diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 22e025ecfe..bad7d30157 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -926,7 +926,7 @@ jobs: extractor.log integration-postgres: - name: pytest integration (Postgres ${{ matrix.postgres }} memory authority) + name: pytest integration (Postgres ${{ matrix.postgres }}, ${{ matrix.profile }}) needs: [validate-ref, clickhouse] runs-on: ubuntu-latest timeout-minutes: 60 @@ -935,6 +935,7 @@ jobs: fail-fast: false matrix: postgres: ["16", "18"] + profile: [storage, isolation] env: LOOM_CLICKHOUSE_IMAGE: ${{ needs.clickhouse.outputs.image }} LOOM_CH_HOST: localhost @@ -1016,6 +1017,7 @@ jobs: tail -100 server.log exit 1 - name: pytest (Postgres authority, memory, knowledge, isolation and the SQL surface) + if: ${{ matrix.profile == 'storage' }} run: >- pytest -q tests/integration/test_pg_authority.py @@ -1096,6 +1098,7 @@ jobs: tests/integration/test_marked_durable_folds_native.py tests/integration/test_marked_sql_surface_native.py - name: pytest online-index durability on a private physical standby + if: ${{ matrix.profile == 'storage' }} timeout-minutes: 8 env: LOOM_PG_IMAGE: postgres:${{ matrix.postgres }} @@ -1124,6 +1127,7 @@ jobs: docker update --memory 1g --memory-swap 1g loom-scale-index-drill-pg loom-scale-index-drill-pg-standby pytest -q tests/integration/test_pg_online_index_durability.py - name: pytest placement durability on a private physical standby + if: ${{ matrix.profile == 'storage' }} timeout-minutes: 8 env: LOOM_PG_IMAGE: postgres:${{ matrix.postgres }} @@ -1162,12 +1166,12 @@ jobs: export LOOM_CELL_PLACE_DRILL_DSN="postgres://loom:loomdev@127.0.0.1:$port/loom" docker update --memory 1g --memory-swap 1g loom-scale-place-drill-pg loom-scale-place-drill-pg-standby pytest -q tests/integration/test_cell_place_durability.py - # The tenant-isolation gates again, against a second server that enforces auth: + # The tenant-isolation profile uses a second server that enforces auth: # the Postgres read paths (heads by key and by id, narrowed scopes) see real tokens. - # They run whether or not the memory suites passed (`!cancelled()`). + # Its matrix job runs independently of the storage profile's outcome. - name: Boot control plane with auth on id: auth_on - if: ${{ !cancelled() && steps.issuer.outcome == 'success' }} + if: ${{ !cancelled() && matrix.profile == 'isolation' && steps.issuer.outcome == 'success' }} env: LOOM_AUTH_REQUIRED: "true" LOOM_INSTANCE_LABEL: "ci:integration-postgres-auth" @@ -1187,7 +1191,7 @@ jobs: # extraction queue, projection failures) hold no Postgres row: they run on the # ClickHouse job. - name: pytest isolation gates (Postgres authority, auth on) - if: ${{ !cancelled() && steps.auth_on.outcome == 'success' }} + if: ${{ !cancelled() && matrix.profile == 'isolation' && steps.auth_on.outcome == 'success' }} env: LOOM_AUTH_REQUIRED: "true" LOOM_BASE_URL: http://localhost:7778 @@ -1209,16 +1213,16 @@ jobs: tests/integration/test_pg_authority.py -k "not test_a_duplicate_never_resolves_to_a_canonical_outside_the_scope and not test_a_superseded_hit_never_names_a_successor_outside_the_scope" - # Last, on its own: `loom reconcile` acts on the whole database, every other test's - # rows included, so it runs after them all and in its file's order. + # Last in the storage profile: `loom reconcile` acts on the whole database, + # the storage tests' rows included, so it runs after them in its file's order. - name: pytest (loom reconcile drills) - if: ${{ !cancelled() }} + if: ${{ !cancelled() && matrix.profile == 'storage' }} run: pytest -q -p no:randomly tests/integration/test_restore_reconcile.py - name: Upload server log on failure if: failure() uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: - name: server-log-postgres-${{ matrix.postgres }} + name: server-log-postgres-${{ matrix.postgres }}-${{ matrix.profile }} path: | server.log server-auth.log diff --git a/bench/conflict_resolver_long_session.py b/bench/conflict_resolver_long_session.py index 0795a84991..9f1b93b42e 100644 --- a/bench/conflict_resolver_long_session.py +++ b/bench/conflict_resolver_long_session.py @@ -5,7 +5,7 @@ namespace: six attributes (home city, employer, manager, standup time, CI provider, diet) change once or twice, each change under a new key and in new words, among restated, elaborated and neighbouring facts, a pinned fact that is -contradicted later, a fact deleted and brought back with Undo, and eighteen +contradicted later, a permanently deleted fact, and eighteen unrelated ones (`bench/conflict_resolver_scenario.py`, the scenario and its ground truth; `tests/integration/test_conflict_resolver_long_session_ch.py` runs it with a scripted judge). The harness then waits for the server's @@ -91,12 +91,13 @@ from bench.conflict_resolver_scenario import ( ATTRIBUTES, BY_KEY, + CHECK_DELETED_IN, CURRENT, DELETE_IN, + DELETED, FACTS, KEPT, OLDER, - RESTORE_IN, SESSIONS, dialogue, relation_of, @@ -150,7 +151,7 @@ def _entry(start: datetime, session: int, i: int, key: str) -> dict[str, Any]: def write_set_session(post: Post, namespace: str, session: int, start: datetime) -> None: - """The session's facts under the scenario's keys, then its delete or Undo.""" + """The session's facts under the scenario's keys, then its deletion or absence check.""" keys = [f.key for f in FACTS if f.session == session] entries = [_entry(start, session, i, k) for i, k in enumerate(keys)] ident = _identity(namespace, session) @@ -162,9 +163,12 @@ def write_set_session(post: Post, namespace: str, session: int, start: datetime) if session in DELETE_IN: key = DELETE_IN[session] post("memory.delete", {**_identity(namespace, BY_KEY[key].session), "memory_key": key}) - if session in RESTORE_IN: - key = RESTORE_IN[session] - post("memory.restore", {**_identity(namespace, BY_KEY[key].session), "memory_key": key}) + if session in CHECK_DELETED_IN: + key = CHECK_DELETED_IN[session] + absent = post( + "memory.get", {**_identity(namespace, BY_KEY[key].session), "memory_key": key} + ) + assert absent["found"] is False, absent def write_messages_session(post: Post, namespace: str, session: int, start: datetime) -> None: @@ -320,7 +324,8 @@ def score_by_key(rows: dict[str, dict[str, Any]], top: dict[str, str]) -> dict[s "flagged_current": sum( 1 for k in CURRENT if k in rows and rows[k]["status"] == "conflicted" ), - "missing_rows": sorted({f.key for f in FACTS} - set(rows)), + "missing_rows": sorted({f.key for f in FACTS} - DELETED - set(rows)), + "deleted_returned": sorted(DELETED & set(rows)), "older_live": sorted(k for k in OLDER if k in rows and _live(rows[k])), } @@ -480,7 +485,7 @@ def print_plan(servers: list[tuple[str, str]], variants: list[str]) -> None: for name, a in ATTRIBUTES.items(): values = " -> ".join(f"{BY_KEY[k].value} (s{BY_KEY[k].session})" for k in a.keys) print(f" {name}: {values}") - print(f"kept live: {len(KEPT)} facts; deleted+restored: {sorted(set(DELETE_IN.values()))}") + print(f"kept live: {len(KEPT)} facts; permanently deleted: {sorted(set(DELETE_IN.values()))}") for label, url in servers or [("on", ""), ("off", "")]: for v in variants: print(f"plan: {label} {url} variant={v}") diff --git a/bench/conflict_resolver_scenario.py b/bench/conflict_resolver_scenario.py index bd5c4130d9..d503fb11c8 100644 --- a/bench/conflict_resolver_scenario.py +++ b/bench/conflict_resolver_scenario.py @@ -10,8 +10,8 @@ key and in new words. The last value is current (`CURRENT`); every earlier one should end superseded behind the next (`OLDER`). - Four facts are restated, four elaborated, four sit near a different fact, one - pinned fact is contradicted later, one fact is deleted and brought back with - Undo, and eighteen stand alone. All of them should stay live and unflagged + pinned fact is contradicted later, one fact is permanently deleted, and eighteen + stand alone. The other facts stay live and unflagged (`KEPT`). - `JUDGE` holds the right verdict for every pair a cluster should hold, except for the scripted judge's deliberate mistakes: two wrong answers at low @@ -170,7 +170,7 @@ def near(topic: str, cos: float, axis: int = 1) -> tuple[Part, ...]: # Pinned, and contradicted later: a pinned row is never retired. F(3, "code_freeze", "Code freeze starts every Friday at noon", alone("freeze"), pinned=True), F(9, "freeze_change", "Code freeze now starts on Thursday evening", near("freeze", 0.75)), - # Deleted in session 4, brought back with Undo in session 6, elaborated in 9. + # Deleted in session 4; session 6 checks its key stays absent. F(3, "badge", "The user's badge number is 4471", alone("badge")), F(9, "badge_lab", "Badge 4471 also opens the third-floor lab", near("badge", 0.75)), # Unrelated, each in its own direction: no peer, never judged. @@ -196,9 +196,9 @@ def near(topic: str, cos: float, axis: int = 1) -> tuple[Part, ...]: SESSIONS = 12 SOLO = {f.key for f in FACTS if f.at[0][0].startswith("solo")} -# The key deleted in a session, and the session whose Undo brings it back (after -# each session's writes). -DELETE_IN, RESTORE_IN = {4: "badge"}, {6: "badge"} +# The key deleted in a session, and the later session that checks its continued absence. +DELETE_IN, CHECK_DELETED_IN = {4: "badge"}, {6: "badge"} +DELETED = set(DELETE_IN.values()) # The relations are not single-valued, so no write closes a fact on its own. ATTRIBUTES: dict[str, Attribute] = { @@ -240,7 +240,6 @@ def near(topic: str, cos: float, axis: int = 1) -> tuple[Part, ...]: frozenset({"marathon", "marathon_plan"}): ("elaboration", 0.96, True, True), frozenset({"rust", "rust_plan"}): ("elaboration", 0.95, True, False), frozenset({"guitar", "guitar_band"}): ("elaboration", 0.96, True, True), - frozenset({"badge", "badge_lab"}): ("elaboration", 0.96, True, False), frozenset({"sister_city", "home_city"}): ("unrelated", 0.97, False, False), frozenset({"skip_level", "manager"}): ("related", 0.95, False, False), frozenset({"car", "partner_car"}): ("unrelated", 0.97, False, True), @@ -262,7 +261,7 @@ def near(topic: str, cos: float, axis: int = 1) -> tuple[Part, ...]: CURRENT = {a.keys[-1] for a in ATTRIBUTES.values()} OLDER = {k for a in ATTRIBUTES.values() for k in a.keys[:-1]} -KEPT = {f.key for f in FACTS} - CURRENT - OLDER +KEPT = {f.key for f in FACTS} - CURRENT - OLDER - DELETED BY_KEY = {f.key: f for f in FACTS} diff --git a/docs/architecture/clickhouse.md b/docs/architecture/clickhouse.md index 148fb11ca8..bdc988ab57 100644 --- a/docs/architecture/clickhouse.md +++ b/docs/architecture/clickhouse.md @@ -652,8 +652,9 @@ on `valid_to`, `invalid_at`, `version`; `set` on `status` and `memory_type`. Loom deletes memory by writing a tombstone version (`tombstoned_at` set, every other column carried). It is visible to the next `FINAL` read, keeps prior versions in -`memory_versions` and leaves `memory.restore` a window. Physical removal is for erasure -with `purge=true`, runner state, and the supersession rows `knowledge.restore` removes. +`memory_versions`. Only automatic retirement without permanent refusal has a restore window. Physical removal is for erasure +with `purge=true`, stale vector caches, runner state, and the supersession rows +`knowledge.restore` removes. - **`knowledge.restore`** runs a lightweight `DELETE FROM loom.chunk_supersessions` for the rows one delete wrote: the document's `source_uri`, the ingestions of its revision and @@ -705,6 +706,18 @@ with `purge=true`, runner state, and the supersession rows `knowledge.restore` r candidate review lease. An invalidated model attempt instead removes only its own candidate versions through a lightweight `DELETE FROM`, bounded by the claim version, so it cannot remove a later review decision. +- **Orphaned vector retirement** deletes cache versions below the caller's cutoff by + the complete vector sort key, across all creation partitions. A missing authoritative + source or matching erasure policy cannot be represented by a newly inserted marker: + even an empty marker would recreate residue after a purge. Admission fences stay held + until the synchronous lightweight delete is acknowledged; equal and newer versions stay. +- **Shared canonical retirement** uses the same bounded lightweight deletion when its + shared head is absent, source identities are unproven or erasure policy matches. The + cutoff preserves a later promotion and avoids writing a tombstone after a purge. +- **Refused extraction jobs** end through a control-only `ALTER TABLE … UPDATE` on the + exact identity, queue id, source version and payload hash. It writes a fixed reason and + terminal status without copying transcript bytes or creating a row. Unrelated jobs in + the same claim batch can still run. - **Extraction retries** (`memory.extraction_retry_mine`) update failed queue rows in place, asynchronously. The update targets the exact identity and version observed, with the caller's namespace, principal and read scope. Copying the transcript into a new diff --git a/docs/architecture/memory-architecture.md b/docs/architecture/memory-architecture.md index bac74089fe..2b7bae5b05 100644 --- a/docs/architecture/memory-architecture.md +++ b/docs/architecture/memory-architecture.md @@ -724,10 +724,10 @@ is left. Its triples close as for a row that holds none. A revision (`memory.set`, `memory.bulk_set`, `memory.update_by_id`) of a row that existed before the write closes the facts its row held that no row holds any longer, and -an Undo of a retirement or of a delete opens again those the restored row's current value +an Undo of reversible retirement opens again those the restored row's current value holds by the same rule (another row's claim keeps a fact from closing, but never reopens one); either, when it fails, is recorded for `projection_repair` as `triples_reconcile`. -A delete closes none of its row's facts, so the only facts a delete's Undo opens again +A tombstone closes none of its row's facts, so the only facts a reversible restore opens again are ones another row's revision or retirement closed while the row was gone. A dropped fact still valid later ("until Dec 1") ends now, and so does one a retired row @@ -1004,14 +1004,15 @@ session, agent or run identity. A read bounded on one of those axes omits the ag graph signals, including hub pruning and graph paths. Its fact legs retain their scoped cosine and query-entity ranking. Namespace-wide reads use the graph aggregates. -Session digests carry the keys they compressed in `derived_from`. Before deduplication, -the summary leg checks those known sources under the digest's full identity. A source -that is missing, unserved, deleted, expired, outside its validity, high-sensitivity or at a newer -version withholds the digest. An unreadable source check withholds all unchecked digests. -Without source revision hashes, a later source version conservatively withholds a digest -during replay too. Lineage covers extracted facts, not every sentence of the transcript: -a digest without a known source remains session evidence, and does not prove that its -statements are current facts. +Session digests contain whole committed source values joined by newlines. Their ordered +`derived_from` and `key_facts` reconstruct every byte of `summary_text`; raw transcript +text and model-generated additions cannot contribute. Every digest reader verifies that +proof against the authoritative heads under the digest's exact agent, user and session. +Missing, changed, unserved, expired, truncated, sensitive or unreviewed sources withhold +the digest. A pin or touch that leaves the same eligible content does not invalidate it. +Legacy digests without complete proof are withheld. Source-read failures fail closed. +A digest hit cannot open unrelated raw turns from its session. Model-generated period +lessons and contrast reflections enter proposed state for independent review. `memory.list` excludes rejected rows from both sort orders and counts. A key lookup can inspect a rejected row. `memory.for_file` also checks `valid_to` before adding a fact to diff --git a/docs/architecture/memory-consent.md b/docs/architecture/memory-consent.md new file mode 100644 index 0000000000..aa6af666ab --- /dev/null +++ b/docs/architecture/memory-consent.md @@ -0,0 +1,123 @@ +# Memory consent and derived text + +A deletion contracts memory and records content refusal before its tombstone commits. +`memory.delete`, a successful `memory.delete_version`, natural-language forgetting and +`memory.clear` keep canonical JSON hashes and normalized literal fingerprints in +`memory_refusals`. Refusal records contain no deleted plaintext and have no TTL. Ordinary +revision, supersession, expiry and an internal move do not express permanent refusal. + +A point deletion applies to the same user in its namespace, across agents and sessions. +A user erasure applies across the organization, including namespaces created later. +A namespace erasure applies to later intake by every user in that namespace. The policy +scope is separate from the original row's identity tuple, so a derived row keeps its +writer identity without narrowing the erased subject's refusal. Captured derived content +also records an indexed erasure refusal under that writer's namespace and user identity. + +Canonical values are compared after JSON decoding. Literal comparisons normalize Unicode, +letter case and whitespace, and check passages within later strings. Wrapping a deleted +value in another JSON object does not bypass the check. These fingerprints compare +content; they do not recognize arbitrary semantic paraphrases. Verification work is +bounded and an incomplete check refuses the write. + +ClickHouse writers share admission leases with deletion and erasure. A successful delete +records refusal under the exact head's write fence. Erasure blocks new intake while it +captures fingerprints and drains admitted sends before deleting content. Postgres stores +policy beside canonical memory. Every publisher takes an organization admission stripe; +erasure drains every stripe before capture. Native writers take the organization policy +lock before namespace, user, generation and head locks. Erasure takes that organization +lock exclusively, including when it spans several namespaces. +A failed policy read refuses intake. Raw evidence, deferred extraction, canonical writes, +embeddings and shared promotion use the same policy. A delayed projection rechecks its +source under admission before publishing. + +Erasure captures ClickHouse revision history under either storage authority. A native +revision whose payload was cleared still supplies its typed identity for raw dependent +capture; ClickHouse history supplies its content before either copy is removed. + +Canonical tombstones check erasure policy even though their content is no longer served. +Point deletion can retain its canonical history. Native authority commits check point +refusal before entering the journal. Delivery can publish those committed revisions after +point deletion, while rechecking erasure policy under the policy locks. Raw evidence, +chunks, extraction payloads and new vector publication still check point refusal. + +A vector retirement keeps cache bytes +only while its authoritative source and policy permit them; point refusal produces an +empty marker. With no source or an erasure match, cleanup deletes older vector versions +across every partition, bounded by the caller's retirement version. It inserts no new row +and preserves equal or newer versions. Native publication proves source absence with a +durable read before borrowing a connection, then rechecks on that connection under policy +locks. Present heads remain locked until publication finishes. + +Native delivery clears search-document debt only for acknowledged publications or a +source that no longer needs a document. A vector withheld after source selection stays +owed for another pass. Receipts match the complete identity and published version; +a newer metadata copy settles its original revision's delivery. Copy follow-ups confirm +the exact document in ClickHouse before settling debt. + +Each native publication batch finishes its canonical send before sending its vectors. +Both sends acquire the same consent fences; ordering them prevents competing partial +claims from blocking the batch. Audit publication runs alongside them. A failed canonical +send still permits the independently checked vector send and preserves each target's debt. + +Shared canonical retirement holds admission through its final send. It checks the +promoted source identities against erasure policy and requires the shared head to remain +present. Missing heads, erased content and unproven legacy sources are removed below the +retirement version without inserting plaintext or a marker. Definition promotion and +demotion acquire the namespace claim before borrowing a source connection. + +Deferred extraction checks caller data in current and prior messages, names, instructions, +schema enums and pipeline arguments. Standard protocol fields do not become refusal +patterns. Enqueue, claim and every payload-carrying transition share this admission. +A transition also requires its exact source revision and payload hash to remain present. +A refused job ends through a control-only update bound to that same source; its transcript +is never copied into the terminal update. A missing source cannot be recreated. + +An owed `memory.delete_version` check carries a hash of the exact checked state. +Recovery compares that hash before writing a missing tombstone. A numeric version +collision cannot turn a different fact into the target of permanent refusal. + +Vector maintenance reports only acknowledged replacements. A legacy vector without a +source hash stays held when its complete source cannot be proved; `--delete-old` keeps +that organization's old vectors. Run `memory.reindex` with `force=true` to rebuild such +vectors from the complete authoritative values. + +Permanent refusal also applies to `memory.restore` and snapshot restoration. Automatic +retirement can still be undone when the content has not been refused. Keeping the +historical row for inspection does not grant permission to publish it again. + +## Session digests + +A session digest contains complete committed source facts joined by newlines. Each fact +has a corresponding key in `derived_from`, under the same complete identity. A model does +not draft session digest text. Search and other digest consumers reconstruct the text +from the authoritative current heads and withhold any digest whose source is missing, +changed, retired, expired, sensitive, incomplete or awaiting review. A metadata touch +keeps an unchanged fact valid. A legacy digest without a complete proof is withheld. + +Digest listing, outline and evidence export scan past withheld candidates to fill their +requested rows. `memory_digest_listing_page_size` bounds candidates per query and +`memory_digest_listing_max_pages` bounds queries per call; both default to 256. If that +budget cannot find enough verified rows or prove exhaustion, the operation returns +`BACKEND_UNAVAILABLE` rather than claiming the listing is complete. + +Period planning selects the newest live digest per session before pagination. Source +verification applies to each digest row; a valid row cannot admit another model's stale +digest from the same session. + +Period and contrast synthesis remain proposals. A model-generated restatement therefore +waits for independent review before ordinary fact retrieval serves it. + +## Rollout + +Stop and drain older APIs, workers and command-line writers before applying the additive +ClickHouse and Postgres migrations. Finish pending versioned deletion checks before +the drain. Without a checked-state hash, recovery closes a check that would need +a missing tombstone without deleting the fact; retry the deletion to check its current state. +Keep writes paused until every publisher enforces the +consent rules. If an older writer ran alongside a migrated store, keep traffic paused +until the affected deletion and erasure scopes have been verified again; a migration does +not certify writes admitted by the older build. Existing rows removed before policy recording was +introduced have no reconstructed consent record. Operators needing a historical guarantee +must establish the intended refusals from their deletion records before resuming intake. +Rolling back restores the earlier admission behavior. Audit records use their separate +retention policy and do not inherit namespace deletion windows. diff --git a/docs/architecture/storage-layout.md b/docs/architecture/storage-layout.md index 278d9712cc..bf249d2301 100644 --- a/docs/architecture/storage-layout.md +++ b/docs/architecture/storage-layout.md @@ -950,6 +950,7 @@ copies of tables listed here, owned and removed by that run, and erasure reaches | `memory` | RMT(version) `(org, namespace, agent, user_id, session_id, memory_key)`, by `toYYYYMM(created_at)` | stays | Derived | census, `memory.tombstone`, user `memory.retire`; step `consolidated.tombstone` | — | the current rows in ClickHouse: the source of `memory_versions_mv`, granted to `insights.query`, read by ClickHouse-side joins; published from `memory_current` and rebuilt by republishing it; named in `server/memory/store/history.py`, `server/memory/store/inserts.py` | | `memory_embeddings` | RMT(version) `(org, namespace, agent, user_id, session_id, scope, embedding_model, memory_key)`, by `toYYYYMM(created_at)` | stays | Derived | census, `memory_embeddings.tombstone`, user scope too; step `shared_embeddings.tombstone`; step `consolidated.tombstone` | — | the vectors of the memory heads (`memory_current`) and the shared heads (`shared`, and from wave 3 `shared_current`); Postgres stores none (re-embedding costs provider calls); wave 4's vector levers change its columns and codec; named in `server/memory/store/inserts.py`, `server/memory/store/shared.py` | | `memory_versions` | RMT `(org, namespace, agent, user_id, session_id, memory_key, version)`, by `toYYYYMM(created_at)` | stays | SoR | census, `memory_versions.delete`, user scope too | — | version history, fed by `memory_versions_mv`: the system of record for history written before a deployment's journal existed, derived from `memory_revisions` for later rows; for a version superseded with its journal payload cleared, the only copy of its content; named in `server/memory/store/conflict_resolver.py`, `server/memory/store/history.py` | +| `memory_refusals` | MT `(org, scope_kind, scope_value, user_id, kind, units, digest)` | stays | SoR | no content | — | Durable consent fingerprints recorded before deletion; policy scopes preserve the original identity and prevent later canonical or literal intake. The native authority stores the same policy in Postgres; named in `server/memory/store/refusals.py`. | | `shared` | RMT(version) `(org, namespace, memory_key)`, by `(org, namespace)` | Postgres `shared_current` + ClickHouse `shared` | SoR (head) | census, `shared.tombstone` | 3 | promoted namespace-wide memory: a head per `(org, namespace, memory_key)` on the memory journal (its revisions in `memory_revisions`, its deliveries in `memory_deliveries`); the ClickHouse table stays its search projection; a namespace erasure tombstones it and a user erasure matches it by provenance; named in `server/memory/store/shared.py`, `server/erasure/store/steps_memory.py` | | `working_memory` | RMT(version) `(org, namespace, agent, user_id, session_id, run_id, block_key)`, by `(org, namespace)` | Postgres `working_memory` | SoR | census, `working_memory.tombstone`, user scope too | 3 | keyed blocks read and replaced by key, never ranked; named in `server/memory/app/working_memory.py`, `server/erasure/store/steps_memory.py` | | `memory_reinforcement` | RMT(version) `(org, namespace, agent, user_id, memory_key)` | stays | Derived | census, `memory_reinforcement.delete`, user scope too | — | folded usage counters, rebuildable from `memory_usage_events`; named in `server/memory/app/reinforcement.py`, `server/erasure/store/steps_memory.py` | @@ -963,7 +964,7 @@ copies of tables listed here, owned and removed by that run, and erasure reaches | `memory_chunks` | RMT(created_at) `(org, namespace, session_id, chunk_index, content_sha256)`, by `toYYYYMM(created_at)` | stays | Derived (SoR for chunks `memory.import_evidence` writes) | census, `memory_chunks.tombstone`, user scope too | — | chunked evidence and chunks of long `memory.set` values, with vectors (`server/memory/app/set_chunks.py`); rebuilt from `memory_evidence` and the heads at embedding cost, except the chunks `memory.import_evidence` writes with no `memory_evidence` row (`server/memory/app/evidence_transfer.py`), held only here; named in `server/memory/app/set_chunks.py`, `server/memory/app/chunk_projection.py` | | `memory_chunk_sources` | RMT(created_at) `(org, namespace, session_id, chunk_index, memory_key, evidence_id)`, by `toYYYYMM(created_at)` | stays | SoR | census, `source_links.tombstone`, user scope too | — | chunk-to-memory links written by the extraction handler after a run and by `memory.import_evidence` (`server/memory/app/evidence_transfer.py`), which takes links a caller states; held only here, so lost without a backup; named in `server/memory/app/chunk_links.py`, `server/erasure/store/steps_graph.py` | | `memory_belief_sources` | RMT(created_at) `(org, namespace, memory_key, evidence_id)` | stays | SoR | census, `source_links.tombstone`, user scope too | — | evidence links of a belief, written by the extraction handler after a run (`server/memory/app/extract/handler.py`), by consolidation linking merged rows to their summary (`server/memory/app/consolidate.py`) and by period digests (`server/memory/app/period_digest.py`); held only here, so lost without a backup; named in `server/memory/app/evidence.py`, `server/erasure/store/steps_graph.py` | -| `session_summaries` | RMT(version) `(org, namespace, agent, user_id, session_id, embedding_model)`, by `toYYYYMM(created_at)` | stays | Derived (SoR for digests `memory.import_evidence` writes) | census, `summaries.tombstone`, user scope too | — | session digests with vectors, written by a model from the session's turns (`server/memory/app/session_summary.py`) and rebuildable at model cost from those turns in `memory_evidence`; the digests `memory.import_evidence` writes are held only here; wave 3 retires a digest whose `derived_from` names a redacted key ([deliveries](#deliveries)); named in `server/memory/app/session_summary.py`, `server/memory/app/evidence_transfer.py` | +| `session_summaries` | RMT(version) `(org, namespace, agent, user_id, session_id, embedding_model)`, by `toYYYYMM(created_at)` | stays | Derived (SoR for digests `memory.import_evidence` writes) | census, `summaries.tombstone`, user scope too | — | Complete committed source facts joined into an extractive digest, with one key per fact. Consumers reconstruct the text from current authoritative heads and withhold unproven or stale digests; named in `server/memory/store/summary_sources.py`. | | `derived_facts` | MT `(org, namespace, subject, predicate, category)`, by `toYYYYMM(created_at)` | stays | Derived (SoR for operands a `memory.bulk_set` entry states) | census, `derived_facts.delete`, user scope too | — | per-occurrence operands for multi-hop answers, append-only, written by `memory.bulk_set` from each entry's `derived` list, which a caller states or extraction from messages fills; a stated operand is held only here, so a rebuild, by re-running extraction over the messages, restores only the extracted ones; wave 3 adds the fact's memory `version` and retires the facts of a redacted version ([deliveries](#deliveries)); named in `server/memory/app/triple_projections.py`, `server/erasure/store/steps_memory.py` | | `memory_records` | RMT(version) `(org, namespace, schema_name, memory_key, record_id)` | Postgres `records` + ClickHouse projection | SoR (head) | census, `records.tombstone`, user scope too | 3 | a record's current fields, read by key and compared on write; Postgres `records` keeps each field's state and evidence id with its value, which only `memory_record_fields` holds today; the projection stays for the record side leg and record query, which join it with `memory_record_fields`, and feeds `memory_record_versions_mv`; named in `server/memory/store/records.py`, `server/memory/app/records.py` | | `memory_record_versions` | RMT `(org, namespace, schema_name, memory_key, record_id, version)`, by `toYYYYMM(created_at)` | stays | SoR | census, `records.tombstone`, user scope too | — | record history, fed by `memory_record_versions_mv` from `memory_records`; from wave 3, one row per delivered revision, reconciled after a restore by the journal replay; named in `server/erasure/store/steps_graph.py`, `server/memory/app/records.py` | @@ -1181,6 +1182,7 @@ Postgres step the flip splits off it, which joins `PG_ONLY_STEPS` | `cache_generations` | `(org, kind)` | Coord | no content | | `memory_current` | [storage authority](storage-authority.md#the-tables) | SoR | census `pg:loom.memory_current` | | `memory_revisions` | [storage authority](storage-authority.md#the-tables) | SoR | census `pg:loom.memory_revisions` | +| `memory_refusals` | `(org, scope_kind, scope_value, user_id, kind, units, digest)` | SoR, hash-only consent policy committed beside memory; no TTL | no content; retained across erasure | | `memory_deliveries` | [storage authority](storage-authority.md#the-tables) | SoR | cascades with its revision | | `idempotency`, `idempotency_fences` | [storage authority](storage-authority.md#the-tables) | SoR | census `pg:loom.idempotency`; `idempotency_fences` holds no content (the census's bound) | | `erased_memories`, `erased_documents` | [storage authority](storage-authority.md#the-tables) | SoR | evidence, no content | diff --git a/docs/concepts/memory.md b/docs/concepts/memory.md index 46b7e25492..29a089801c 100644 --- a/docs/concepts/memory.md +++ b/docs/concepts/memory.md @@ -164,13 +164,13 @@ answer confidently with a fact it has reason to doubt. Five operations cover the memory lifecycle: -| Lens | Operation | Effect | -| --------- | --------------------- | -------------------------------------------------------------------------- | -| Expand | `memory.set` | Learn a fact under a key. | -| Revise | `memory.update_by_id` | Correct one row by id (`memory.update` does the same by key). | -| Contract | `memory.delete` | Tombstone a key. `memory.restore` revives it within 14 days of the delete. | -| Reinforce | `memory.pin` | Store and pin: no later write that does not itself pin replaces it. | -| Refuse | `blacklist.add` | Refuse every future write matching a substring or regex. | +| Lens | Operation | Effect | +| --------- | --------------------- | ------------------------------------------------------------------- | +| Expand | `memory.set` | Learn a fact under a key. | +| Revise | `memory.update_by_id` | Correct one row by id (`memory.update` does the same by key). | +| Contract | `memory.delete` | Tombstone a key and refuse later intake of its removed value. | +| Reinforce | `memory.pin` | Store and pin: no later write that does not itself pin replaces it. | +| Refuse | `blacklist.add` | Refuse every future write matching a substring or regex. | A searchable `memory.set` runs these checks in order: @@ -264,7 +264,7 @@ stateDiagram-v2 active --> superseded: newer write (newest_wins, refines, conflict resolver) superseded --> active: memory.restore active --> tombstoned: memory.delete, blacklist sweep, expiry - tombstoned --> active: memory.restore (within 14 days) + tombstoned --> active: memory.restore (automatic retirement, no refusal) ``` Every arrow is a new version; the old one stays in `loom.memory_versions`. @@ -570,8 +570,8 @@ ahead of the durable sections, so an agent sees its own state first. Loom deletes only what its writer asked for. No memory table has a default TTL and no sweep retires rows by age. A row leaves the served set when its writer deletes or supersedes it, when its writer gave it an expiry (`ttl_seconds`, `valid_to`), or when the -namespace owner or operator configured a retention window. `memory.restore` revives a -tombstone within 14 days. +namespace owner or operator configured a retention window. `memory.restore` reverses an automatic +tombstone within 14 days when no permanent refusal exists. [Retention and erasure](../features/memory/retention-and-erasure.md) covers the settings and the erasure saga. diff --git a/docs/development/testing.md b/docs/development/testing.md index 913881ea58..b23f8e791d 100644 --- a/docs/development/testing.md +++ b/docs/development/testing.md @@ -325,27 +325,27 @@ failure on one merge commit is never hidden by the next. Every action is pinned commit SHA (its version in a comment), and the workflow grants its jobs read-only repository access (`permissions: contents: read`). -| Job | Runs | -| ---------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `clickhouse` | resolves the ClickHouse release `dev/clickhouse_release.sh` names, so `unit`, `schema-check` and `integration` test the same one | -| `lint` | `ruff check`, `ruff format --check`, the `--check` of `sdk_py_gen.py`, `sdk_py_client_gen.py`, `sdk_py_admin_gen.py`, `agent_plugin_sync.py`, `settings_reference.py`, `settings_schema.py`, `docs_reference.py` and `op_reach_census.py`, `mypy --strict` on the SDK, the mypy ratchet, the function-length and secret gates | -| `unit` | `pytest tests/unit` in six parts (`LOOM_TEST_SHARD`), each on its own runner, in random order on every core with coverage | -| `unit-coverage` | the parts' coverage combined (floor 40%) and the per-module floors, then `pytest bench` and the generic integration sample (`integrations/generic/test.sh`); its `shard-weights` artifact is the run's seconds per unit file, to commit as `tests/shard_weights.json` when the parts drift apart | -| `sandbox` | the hostile repository-job sandbox suite under real bubblewrap, where a skip fails (`LOOM_TEST_REQUIRE_SANDBOX=1`) | -| `schema-check` | a fresh ClickHouse (`dev/ci_start_clickhouse.sh`), every migration, the strict dry run, `make schema-check` | -| `ts-typecheck` | the dashboard build (`tsc -b` + Vite) and its bundle budget (`npm run budget`) | -| `format` | Biome, Prettier, Taplo and shfmt checks at pinned versions, and Biome's accessibility lint on `frontend/src` | -| `helm` | `helm lint --strict` and kubeconform on every overlay, kubeconform on `deploy/okf-sync/cronjob.yaml`, the values the chart must refuse, and `scripts/gates/helm_extractor_isolation.py`, `helm_metadata_egress.py` and `helm_assistant_isolation.py` | -| `agent-plugins` | `node --test` for the plugin hooks and the grep shim, the Cursor hook sample (`integrations/cursor/test.sh`), a parse check of every hook, library and `bin/` helper script of both plugins and the shim, and a check that the grep shim's entry points are executable | -| `frontend-vitest` | vitest (unit, component, axe), in parts on separate runners | -| `frontend-e2e` | Playwright, in parts on separate runners | -| `integration` | parts, each with its own ClickHouse, migrations, reference extractor and booted server (fake providers, workers off): the numbered parts split `pytest tests/integration` (`LOOM_TEST_SHARD`), the isolation gates part the provider staging tests, the auth-on gates and `pytest tests/analyzer` | -| `integration-postgres` | the memory, knowledge, isolation, idempotency, erasure and restored-id suites and the auth-on gates against a server with `LOOM_STORAGE_AUTHORITY=postgres`, on Postgres 16 and 18 with a synchronous standby ([Postgres](../operations/postgres.md)) | -| `mixed-version` | `scripts/gates/mixed_version.py`: builds N-1 and N side by side on one ClickHouse, N-1 being `HEAD~1`: on a pull request the first parent of its merge commit, the branch it targets as the merge was made, and on a push the previous commit | -| `sdk-typescript` | `sdk_ts_gen.py --check`, `npm run check` (lint, types, version, README examples, tests), `npm run check:dist` | -| `sdk-rust` | dependency resolution with Rust-version fallback, `cargo fmt --check`, `cargo test --locked` (response, HTTP contract and documentation tests), `cargo clippy --locked --all-targets -- -D warnings`, `cargo doc --locked --no-deps` with warnings refused, `cargo package --locked` and packaged license verification; Rust 1.84.1 and stable | -| `sdk-python-adapters` | `make test-sdk-adapters`: for each adapter whose framework is an SDK extra, in a venv of its own with that framework, outside `uv.lock`, its tests and `mypy --strict` on its module and on the adapters that need no extra (`__init__.py`, `langgraph.py`, `strands.py`); on Python 3.12, and on 3.11 with every direct requirement (the SDK's own, the extra's, pytest) at its lower bound | -| `sdk-python-package` | builds the SDK wheel with uv, installs it into a clean venv and runs `scripts/gates/sdk_wheel_smoke.py`, on Python 3.11, 3.12 and 3.14, plus 3.11 and 3.14 with every declared dependency at its lower bound | +| Job | Runs | +| ---------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `clickhouse` | resolves the ClickHouse release `dev/clickhouse_release.sh` names, so `unit`, `schema-check` and `integration` test the same one | +| `lint` | `ruff check`, `ruff format --check`, the `--check` of `sdk_py_gen.py`, `sdk_py_client_gen.py`, `sdk_py_admin_gen.py`, `agent_plugin_sync.py`, `settings_reference.py`, `settings_schema.py`, `docs_reference.py` and `op_reach_census.py`, `mypy --strict` on the SDK, the mypy ratchet, the function-length and secret gates | +| `unit` | `pytest tests/unit` in six parts (`LOOM_TEST_SHARD`), each on its own runner, in random order on every core with coverage | +| `unit-coverage` | the parts' coverage combined (floor 40%) and the per-module floors, then `pytest bench` and the generic integration sample (`integrations/generic/test.sh`); its `shard-weights` artifact is the run's seconds per unit file, to commit as `tests/shard_weights.json` when the parts drift apart | +| `sandbox` | the hostile repository-job sandbox suite under real bubblewrap, where a skip fails (`LOOM_TEST_REQUIRE_SANDBOX=1`) | +| `schema-check` | a fresh ClickHouse (`dev/ci_start_clickhouse.sh`), every migration, the strict dry run, `make schema-check` | +| `ts-typecheck` | the dashboard build (`tsc -b` + Vite) and its bundle budget (`npm run budget`) | +| `format` | Biome, Prettier, Taplo and shfmt checks at pinned versions, and Biome's accessibility lint on `frontend/src` | +| `helm` | `helm lint --strict` and kubeconform on every overlay, kubeconform on `deploy/okf-sync/cronjob.yaml`, the values the chart must refuse, and `scripts/gates/helm_extractor_isolation.py`, `helm_metadata_egress.py` and `helm_assistant_isolation.py` | +| `agent-plugins` | `node --test` for the plugin hooks and the grep shim, the Cursor hook sample (`integrations/cursor/test.sh`), a parse check of every hook, library and `bin/` helper script of both plugins and the shim, and a check that the grep shim's entry points are executable | +| `frontend-vitest` | vitest (unit, component, axe), in parts on separate runners | +| `frontend-e2e` | Playwright, in parts on separate runners | +| `integration` | parts, each with its own ClickHouse, migrations, reference extractor and booted server (fake providers, workers off): the numbered parts split `pytest tests/integration` (`LOOM_TEST_SHARD`), the isolation gates part the provider staging tests, the auth-on gates and `pytest tests/analyzer` | +| `integration-postgres` | separate storage and authenticated-isolation profiles on Postgres 16 and 18 with a synchronous standby and `LOOM_STORAGE_AUTHORITY=postgres`. Storage covers memory, knowledge, idempotency, erasure, restored ids and durability, followed by whole-database restore drills. Isolation covers the auth-on gates. Each profile uses its own stores and keeps the one-hour job limit ([Postgres](../operations/postgres.md)) | +| `mixed-version` | `scripts/gates/mixed_version.py`: builds N-1 and N side by side on one ClickHouse, N-1 being `HEAD~1`: on a pull request the first parent of its merge commit, the branch it targets as the merge was made, and on a push the previous commit | +| `sdk-typescript` | `sdk_ts_gen.py --check`, `npm run check` (lint, types, version, README examples, tests), `npm run check:dist` | +| `sdk-rust` | dependency resolution with Rust-version fallback, `cargo fmt --check`, `cargo test --locked` (response, HTTP contract and documentation tests), `cargo clippy --locked --all-targets -- -D warnings`, `cargo doc --locked --no-deps` with warnings refused, `cargo package --locked` and packaged license verification; Rust 1.84.1 and stable | +| `sdk-python-adapters` | `make test-sdk-adapters`: for each adapter whose framework is an SDK extra, in a venv of its own with that framework, outside `uv.lock`, its tests and `mypy --strict` on its module and on the adapters that need no extra (`__init__.py`, `langgraph.py`, `strands.py`); on Python 3.12, and on 3.11 with every direct requirement (the SDK's own, the extra's, pytest) at its lower bound | +| `sdk-python-package` | builds the SDK wheel with uv, installs it into a clean venv and runs `scripts/gates/sdk_wheel_smoke.py`, on Python 3.11, 3.12 and 3.14, plus 3.11 and 3.14 with every declared dependency at its lower bound | `.github/workflows/plugin-platforms.yml` runs when a change touches `integrations/`, a plugin marketplace, `.gitattributes` or the workflow itself: the plugin and grep-shim diff --git a/docs/features/memory/history-and-provenance.md b/docs/features/memory/history-and-provenance.md index 331db28f47..2e078d6a94 100644 --- a/docs/features/memory/history-and-provenance.md +++ b/docs/features/memory/history-and-provenance.md @@ -48,8 +48,8 @@ async with Agent(org="acme", namespace="support", agent="auditor") as agent: print(v["written_at"], v["actor"], v["source_class"], v["value"]) ``` -A tombstone is a version too, so a delete shows in history and `memory.restore` can -revive the key within 14 days. For point-in-time reads of current state, use `as_of` on +A tombstone is a version too, so a delete shows in history. `memory.restore` can +reverse an automatic tombstone within 14 days when no permanent refusal exists. For point-in-time reads of current state, use `as_of` on `memory.get`, `memory.search` and `memory.ask` ([validity time](./writing.md#validity-time-valid_at-and-valid_to)). diff --git a/docs/features/memory/retention-and-erasure.md b/docs/features/memory/retention-and-erasure.md index 830898de8f..3472bbe228 100644 --- a/docs/features/memory/retention-and-erasure.md +++ b/docs/features/memory/retention-and-erasure.md @@ -37,14 +37,16 @@ other kinds. See [namespaces and sharing](../namespaces-and-sharing.md). A tombstone is a new version with `tombstoned_at` set. Reads stop serving it at once, but the row and its prior versions in `loom.memory_versions` stay on disk: -- `memory.restore` revives a tombstoned key within 14 days (after that it answers - `expired=true`). It also reverses an automatic supersession. +- Explicit deletion records permanent content refusal. `memory.restore` and snapshot + restoration cannot publish a refused value again. `memory.restore` still reverses an + automatic supersession or expiry when no refusal applies. See + [memory consent](../../architecture/memory-consent.md). - `memory.clear` (admin) tombstones every memory for an agent or user in a namespace; `confirm_subject` must repeat the concrete `user_id` or `agent` (neither `'*'` nor `'-'` confirms). `agent="*"` with a `user_id` clears that user's rows under every agent, and `user_id="*"` with an `agent` that agent's rows for every user. A scope that names nobody but `'-'` is refused, as a namespace-wide clear is. Locked rows stay and are counted in - `locked_skipped`. Cleared keys revive the same way. + `locked_skipped`. Cleared values receive the same permanent refusal as a point deletion. To remove the bytes, use an [erasure](#erasure). To strip one value but keep the row and audit trail, use [`memory.redact`](./history-and-provenance.md#scrubbing-history-memoryredact). @@ -172,17 +174,16 @@ flowchart TD hosts with Keeper, within one process without it. It waits up to 15 minutes for a pass to finish; past that the step fails with `LeaseBusy` and `erasure_recovery` retries it. - **Tombstone or purge.** By default the soft-delete tables (memory, embeddings, chunks, - summaries, links) are tombstoned, and `memory.restore` can still revive them inside - the window; `erasure.list` reports such revivals as `reversals_count`. `purge=true` - deletes the rows instead (a lightweight `DELETE FROM` on the wide tables). A scope + summaries, links) are tombstoned. Permanent refusal prevents publishing the captured + values again, including through `memory.restore`. `purge=true` deletes the rows instead + (a lightweight `DELETE FROM` on the wide tables). A scope that holds a namespace that has ever had a mirror is purged whole, every namespace in it, so none of them has such a window ([document sources](../../architecture/document-sources.md#the-mirror)). A user erasure finds what the user's memories fed through their keys, so in a purge its `loom.memory` rows, and the consolidated summaries built from them, are tombstoned first and deleted in - `memory.retire`, after every step that reads those keys; until then `memory.restore` - can revive them (the user's own rows count as a reversal, the summaries do not), and - `memory.retire` deletes what `memory.restore` revived. + `memory.retire`, after every step that reads those keys. Refusal fingerprints are + recorded before this content is removed and survive the erasure. - **Uploaded originals** of knowledge documents: the namespace's rows of `loom.document_blobs` are deleted first, and `blob_erased_at` is stamped. A user erasure deletes the originals that user uploaded and no other writer's live document shares, @@ -336,13 +337,14 @@ What the receipt does not cover: - backups, point-in-time recovery, WAL and standbys of either service; - copies made outside Loom, such as a table built in an org's own Postgres from the [SQL surface](../../reference/sql-surface.md); -- rows a worker writes after the census from a read it made before it - ([storage authority](../../architecture/storage-authority.md#erasure)); -- a `namespace.restore` from a snapshot taken before the erasure, which brings rows back: - run the erasure again, and its census is the new receipt; +- an unrelated row written after the census, outside its erasure scope; - for a user scope, document originals in the bucket, which the census counts for a namespace scope only. +Consent checks fence delayed memory publishers and refuse removed values in later +snapshot restoration. See [memory consent](../../architecture/memory-consent.md) for +the matching rules and the rollout required to enforce them. + The erased id stays where the proof of erasure needs it: in the ledger row (until the table's TTL drops it, 90 days after the request), in audit, in `loom.erasure_reversals`, and in the deployment's own logs of the statements that named it (the ClickHouse mutation log, diff --git a/docs/features/memory/writing.md b/docs/features/memory/writing.md index 0c99c4db9d..6e7d47eb52 100644 --- a/docs/features/memory/writing.md +++ b/docs/features/memory/writing.md @@ -18,7 +18,7 @@ merge, duplicate, conflict or rejected. The verdicts and conflict policies are i | Correct a fact you found through search or ask | `memory.update` | Resolves the key namespace-wide; keeps a pin. | | Correct one exact row by id | `memory.update_by_id` | Addressed by `(org, memory_id)`. | | Change only the expiry | `memory.touch` | One key or up to 1,000 `memory_keys`; value untouched. | -| Retire a fact | `memory.delete` | Tombstone; `memory.restore` revives it within 14 days. | +| Retire a fact | `memory.delete` | Tombstone and permanent refusal of the removed value. | | Rename a key and keep the old one readable | `memory.move` | `keep_redirect` (default true) leaves a redirect. | | Guard a row against edits or deletion | `memory.lock` / `memory.unlock` | Unlock is admin. | | Apply one change to many existing rows | `memory.apply` | Dry run by default. | @@ -178,14 +178,14 @@ key then answers `conflict_verdict: "conflict"`, `peers[0].memory_key: ## Changing and retiring facts -| Op | Behaviour | -| --------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `memory.update` | Resolves `memory_key` across the namespace when no identity is given (`"-"` counts as none, and `"*"` for `agent`). One live row: updated. None: `updated=false, reason="not_found"`. Several: 409 `AMBIGUOUS_KEY` with `candidates`. Runs the conflict check and keeps a pin. An expired row answers 409 `EXPIRED` unless `revive=true`. | -| `memory.update_by_id` | One row by `(org, memory_id)`. The namespace is resolved and checked before the read; a refusal answers `updated=false`, not 403. | -| `memory.delete` | Writes a tombstone, looking past the caller's identity for the key (`found=false` when no live row exists). `memory.delete_version` instead reads only the caller's exact identity (a concrete agent) and, under the key's write lock, deletes only the version it names, else `tombstoned=false, reason="version_changed"`. Consolidated summaries lose the row: retired when no source survives, else `conflicted`. `memory.restore` revives the key within 14 days. | -| `memory.move` | Renames a key, carrying value, type, tags, entities, validity and provenance. The source becomes a redirect (`canonical_key`, `follow=true`) or, with `keep_redirect=false`, a tombstone. See [optimize a namespace](../../guides/optimize-namespace.md). | -| `memory.lock` | `read_only` refuses value, expiry and deletion changes; `no_delete` refuses tombstone, redact, clear and consolidation. Both refuse `memory.reject` (423 `LOCKED`); a negative-outcome quarantine leaves a locked row as it is, while its negatives keep counting. A writer may lock or tighten (`until`, `ttl_seconds`); `memory.unlock` is admin. `locked=false, reason="not_found"` is not a lock. | -| `memory.apply` | A `target` (keys, a filter or a search; filter and search need `limit` ≤ 1,000), an `action` (`delete`, `expire`, `lock`, `label`, `demote`), a `reason`, `dry_run` on by default. Each row goes through the single-row path; refusals come back as `refused` rows. Refuses `agent="*"`. | +| Op | Behaviour | +| --------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `memory.update` | Resolves `memory_key` across the namespace when no identity is given (`"-"` counts as none, and `"*"` for `agent`). One live row: updated. None: `updated=false, reason="not_found"`. Several: 409 `AMBIGUOUS_KEY` with `candidates`. Runs the conflict check and keeps a pin. An expired row answers 409 `EXPIRED` unless `revive=true`. | +| `memory.update_by_id` | One row by `(org, memory_id)`. The namespace is resolved and checked before the read; a refusal answers `updated=false`, not 403. | +| `memory.delete` | Writes a tombstone, looking past the caller's identity for the key (`found=false` when no live row exists). `memory.delete_version` instead reads only the caller's exact identity (a concrete agent) and, under the key's write lock, deletes only the version it names, else `tombstoned=false, reason="version_changed"`. Consolidated summaries lose the row: retired when no source survives, else `conflicted`. Permanent refusal blocks restoration and later intake of the removed content. | +| `memory.move` | Renames a key, carrying value, type, tags, entities, validity and provenance. The source becomes a redirect (`canonical_key`, `follow=true`) or, with `keep_redirect=false`, a tombstone. See [optimize a namespace](../../guides/optimize-namespace.md). | +| `memory.lock` | `read_only` refuses value, expiry and deletion changes; `no_delete` refuses tombstone, redact, clear and consolidation. Both refuse `memory.reject` (423 `LOCKED`); a negative-outcome quarantine leaves a locked row as it is, while its negatives keep counting. A writer may lock or tighten (`until`, `ttl_seconds`); `memory.unlock` is admin. `locked=false, reason="not_found"` is not a lock. | +| `memory.apply` | A `target` (keys, a filter or a search; filter and search need `limit` ≤ 1,000), an `action` (`delete`, `expire`, `lock`, `label`, `demote`), a `reason`, `dry_run` on by default. Each row goes through the single-row path; refusals come back as `refused` rows. Refuses `agent="*"`. | ## Provenance fields @@ -278,14 +278,12 @@ extraction path (`LOOM_MEMORY_EXTRACT_VALUE_UPDATES`) never runs for an import. (`on_expire=anonymize`) and comes back `skipped` (`revised`), before the bulk write's own pin and lock guards. A status the import does not recognise leaves the key as it is too, as `skipped` (`unknown`). The read and the write are not atomic: a delete that lands - between them is written over. Text deleted, retired or rejected under another key is - imported again under its `import:` key: the bulk write's dedup sees live rows only. -- **A tombstone has no way back through the import:** `memory.delete`, `memory.clear` and - an expiry with `on_expire=tombstone` all write one, so clearing an agent and importing - from the source again answers `skipped` (`deleted`) for every such item. Bring the keys - back with `memory.restore` within 14 days of the delete (a re-import then answers - `unchanged`), or remove the scope's rows with `erasure.request` and `purge=true`: a key - with no rows left is imported as new. + between them must still pass the committed write's policy checks. Refused content is + blocked under another key too; ordinary retirement does not record permanent refusal. +- **Import does not reverse a tombstone:** explicit deletion and clearing also refuse + the removed content. `memory.restore` can reverse an automatic tombstone within + 14 days when no refusal exists. Erasure and purge preserve consent fingerprints, + so removing stored rows does not grant permission to import their content again. - **Provenance:** every row is `source_class=imported`, `valid_at` is the source's time and `created_at` the import's. An entry with no `source_trace` is traced by its key. - **Categories:** a `category` the namespace registered (`namespace.register_category`) is diff --git a/docs/features/tasks-and-decisions.md b/docs/features/tasks-and-decisions.md index 6e870a7584..4861d83156 100644 --- a/docs/features/tasks-and-decisions.md +++ b/docs/features/tasks-and-decisions.md @@ -341,7 +341,7 @@ superseded in the record plane and does not keep the record. with `sweep=true` and `blacklist.sweep`. `decision.chain`, `decision.precedents` and `decision.outcome` stop serving it, and `memory.record.history` shows the last version with `status="tombstoned"`. Two retirements that race can each write a tombstoned version, so history may list the tombstone twice. -- **After a retirement**: `memory.restore`, or `namespace.rollback` past the delete, revives the row but not the +- **After a retirement**: a reversible automatic retirement can be restored, but this revives the row without the record; call `decision.record` again. A row moved with `keep_redirect=false` is no longer a decision: its new key has no record, and `decision.record` with the same category and scenario records it again under its derived key. - **Leaves the record active**: a delete of another identity's row; `memory.clear` and `namespace.rollback`, which diff --git a/docs/glossary.md b/docs/glossary.md index 8dd4bdacf4..954ac759ff 100644 --- a/docs/glossary.md +++ b/docs/glossary.md @@ -199,8 +199,9 @@ the rows agents write (`memory.promote` / `memory.demote`, table `loom.shared`). **Tier** — which extractor produced a code-index fact (parser or compiler); answers say which tier backs them. -**Tombstone** — a higher-version row marking a memory deleted. `memory.restore` can undo -it within the grace window; nothing is physically deleted by `memory.delete`. +**Tombstone** — a higher-version row marking a memory deleted. Automatic tombstones can +be reversed within the grace window when no refusal exists. Explicit deletion also records +permanent refusal; its historical rows remain until physical erasure. **Toolset** — the MCP listing a client asks for (`core`, `coding`, `atomic`, `tasks`, `tracker`, `readonly`, `all`); it changes what is listed, not what the token may call. diff --git a/docs/operations/backup-and-recovery.md b/docs/operations/backup-and-recovery.md index 0a47670b4f..79128e4004 100644 --- a/docs/operations/backup-and-recovery.md +++ b/docs/operations/backup-and-recovery.md @@ -196,7 +196,7 @@ Repairs tab, where a namespace admin downloads a snapshot, restores one and roll | Operation | Role | Does | | -------------------- | ------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `memory.restore` | `WRITER` | brings one tombstoned key back | +| `memory.restore` | `WRITER` | reverses one automatic tombstone when no refusal exists | | `namespace.snapshot` | `ADMIN` | freezes a namespace's memories, types, grants and an audit tail (`include_audit_tail_n`, default 1000) as a JSON payload; `include_embeddings=false` (the default) lets the target re-embed under its own model | | `namespace.restore` | `VIEWER` on a fresh target, `ADMIN` on an existing one | writes a snapshot payload into `target_namespace` (optionally `rebind_agent`); idempotent by `snapshot_id`. Rows keep `version`, `content_sha256` and the attestation columns, so `verified_by` stays; a payload value its column cannot hold (a memory id that is not a UUID) is refused with 422. A collection is refused: its snapshot restores into a plain namespace | | `namespace.rollback` | `ADMIN` | returns every memory head (or `memory_keys`) to the version it had at `as_of`: reinstates the older version as the new head, tombstones keys created after it. Dry run by default (`dry_run=false` applies); at most `max_keys` per call (`truncated` says when to call again) | @@ -382,17 +382,17 @@ again whole before the deployment starts; deliveries it held are listed as parke ## What is and is not recoverable -| Situation | Recoverable? | How | -| ---------------------------------------------------------- | -------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| A memory tombstoned by `memory.delete` or `memory.clear` | yes | `memory.restore`, or `namespace.rollback` to before the delete | -| A memory overwritten by a later version | yes | `memory.history` shows the versions; `namespace.rollback` reinstates one | -| An erasure with `purge: true`, or a purged bench namespace | no | purge hard-deletes; there is no restore window. Only a backup taken before it | -| An erasure that tombstoned (no `purge`) | treat as final | the saga's census verified the scope is gone from every read path; bringing it back would undo a deletion request, so restore from a backup only when the erasure itself was the mistake | -| A newer build's schema, after rolling back the image | not needed | migrations are additive; the previous build reads the newer schema ([upgrades](./upgrades.md#rollback)) | -| An earlier schema | from a backup | there are no down-migrations; restore a backup taken before the upgrade | -| A single-node ClickHouse OSS server's disk | from a backup | no replication and no point-in-time recovery: the node is durable only to its own disk | -| Code index tables | rebuildable | `loom code backfill` re-derives them from the git mirror ([code index operations](./code-index-operations.md#first-index)) | -| Rows removed by an opt-in retention TTL | no | TTL deletes are final; only a backup taken before them | +| Situation | Recoverable? | How | +| ---------------------------------------------------------- | -------------- | -------------------------------------------------------------------------------------------------------------------------- | +| A memory tombstoned by `memory.delete` or `memory.clear` | no | refusal blocks restoration and reimport | +| A memory overwritten by a later version | yes | `memory.history` shows the versions; `namespace.rollback` reinstates one | +| An erasure with `purge: true`, or a purged bench namespace | no | purge removes stored content; permanent refusal also blocks logical restoration | +| An erasure that tombstoned (no `purge`) | treat as final | the saga's census verified the scope is gone from every read path; permanent refusal remains in force during restoration | +| A newer build's schema, after rolling back the image | not needed | migrations are additive; the previous build reads the newer schema ([upgrades](./upgrades.md#rollback)) | +| An earlier schema | from a backup | there are no down-migrations; restore a backup taken before the upgrade | +| A single-node ClickHouse OSS server's disk | from a backup | no replication and no point-in-time recovery: the node is durable only to its own disk | +| Code index tables | rebuildable | `loom code backfill` re-derives them from the git mirror ([code index operations](./code-index-operations.md#first-index)) | +| Rows removed by an opt-in retention TTL | no | TTL deletes are final; only a backup taken before them | On a single self-managed OSS node, schedule `BACKUP DATABASE` (or `clickhouse-backup`) to storage off the host. ClickHouse Cloud replicates and backs up the service itself, so diff --git a/docs/operations/maintenance.md b/docs/operations/maintenance.md index 1decb06de6..aa3536b822 100644 --- a/docs/operations/maintenance.md +++ b/docs/operations/maintenance.md @@ -126,7 +126,7 @@ A namespace whose graph outlived its memories is listed by | Call | Effect | Use for | | ------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------ | -------------------------------------------------------------------- | -| `memory.clear` (`ADMIN`) | tombstones the memories of one `user_id` or `agent` in a namespace; `confirm_subject` must repeat the scope value | cleanup, test resets; reversible with `memory.restore` | +| `memory.clear` (`ADMIN`) | tombstones the memories of one `user_id` or `agent` in a namespace; `confirm_subject` must repeat the scope value | explicit removal; refused content cannot be restored | | `erasure.request` with `scope_type`: `user_id`, `namespace` or `namespace_list` | a resumable saga across every Loom table that can hold the subject's content, then the configured external backends (Langfuse, ClickStack) | erasing a user's data (the audit trail is kept); retiring namespaces | The saga checkpoints each step in `loom.erasure_requests` (`ReplacingMergeTree`, rows read diff --git a/docs/operations/okf-sync.md b/docs/operations/okf-sync.md index d6e5ec3b4a..500c330d8c 100644 --- a/docs/operations/okf-sync.md +++ b/docs/operations/okf-sync.md @@ -37,8 +37,7 @@ LOOM_API_KEY="$PUBLIC_WRITER" loom import-okf /tmp/bundle --server-url "$PUBLIC_ import's `--agent`, written by its token, whose `source_trace` starts with `okf:` and whose key is no longer in the bundle. Rows anyone else wrote in the namespace stay. Give each bundle imported into the namespace its own `--agent` (`--prune` refuses without - one). A delete is a tombstone: `memory.restore` brings the row back within its 14-day - window. + one). Pruning tombstones the row and permanently refuses its removed content. The last line reports the counts: `500 rows from the bundle into acme/shared-lessons: 0 written, 500 unchanged, 0 failed …, @@ -92,8 +91,8 @@ Suspend the CronJob (`kubectl patch cronjob okf-sync -p '{"spec":{"suspend":true delete the copies on the public Loom with `memory.delete`, or through erasure when the content must go. -A key pruned by mistake is written again by the next run whose bundle holds it, as a new -row: its `memory_id` changes. `memory.restore` brings back the original row instead. +Pruning is permanent content refusal. A later bundle or `memory.restore` cannot republish +the removed value. Review the prune preview before applying it. ## What a run costs @@ -162,8 +161,7 @@ knows a `high` row by its `sensitivity`, which it reads through `memory.bulk_get the `sensitivity:high` tag `memory.set` and `memory.update` add. Against a private Loom whose `memory.bulk_get` does not return `sensitivity`, every row stays home (the export counts them under "no known sensitivity"), so the bundle has no concepts and `--prune` -refuses it. A prune is a tombstone: the value stays restorable for 14 days and in the -public Loom's version history. Content that must leave the public Loom goes through +refuses it. Pruning retains version history and refuses later publication of the removed value. Content that must leave the public Loom goes through [erasure](../features/memory/retention-and-erasure.md). ## Network diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index a80be64230..78adae3700 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -1104,7 +1104,9 @@ The memory plane outside search/ask/extraction: writes, conflicts, lifecycle, wo | `LOOM_MEMORY_REINFORCEMENT_CLOSER_WINDOW_DAYS` | `int` | `14` | | Days a consumer's closer keeps its run-less reads credited. | | `LOOM_MEMORY_REINFORCEMENT_CLOSER_CACHE_SECONDS` | `int` | `300` | | Seconds one (org, consumer) closer answer is cached per worker. | | `LOOM_MEMORY_LIST_VIEWER_LIMIT` | `int` | `100` | | memory.list rows per call for a token below writer on the namespace; it pages with `offset` (wire cap stays 1000). 0 = no viewer cap. | -| `LOOM_MEMORY_SESSION_DIGEST_MAX_INPUT_CHARS` | `int` | `3000` | | Transcript characters the session-digest call reads (~4 chars a token); 0 = the built-in 3000. | +| `LOOM_MEMORY_SESSION_DIGEST_MAX_INPUT_CHARS` | `int` | `3000` | | Characters of complete source facts included in an extractive session digest; 0 = the built-in 3000. Facts are never clipped to fit. | +| `LOOM_MEMORY_DIGEST_LISTING_PAGE_SIZE` | `int` | `256` | | Raw digest candidates per query while filling listing, outline and export rows. | +| `LOOM_MEMORY_DIGEST_LISTING_MAX_PAGES` | `int` | `256` | | Queries allowed to find enough proven digests; exhaustion answers BACKEND_UNAVAILABLE. | | `LOOM_MEMORY_STRUCTURED_ANSWER_EXCERPT_LIMIT` | `int` | `4096` | | content_excerpt ceiling for answer-bearing facts (question_topic set) when memory_capture_structured_answers is on, so an enumerated answer is not clipped mid-list; every other fact keeps EXCERPT_LIMIT (1,200 chars). | | `LOOM_MEMORY_VALUE_ADVISORY_CHARS` | `int` | `30000` | | A searchable memory.set value serializing longer than this many chars gets one advisory line (the write still lands): only the first EXCERPT_LIMIT chars are embedded, and past the embedder's 8,191-token cap nothing is. 30,000 chars ≈ 8k tokens. 0 disables. | @@ -1698,7 +1700,7 @@ Telemetry, audit, tracing and the ops surface. | `LOOM_LANGFUSE_CIRCUIT_FAILURE_THRESHOLD` | `int` | `5` | | Score-POST circuit breaker: consecutive failures that open it. | | `LOOM_LANGFUSE_CIRCUIT_COOLDOWN_S` | `float` | `30.0` | | Seconds the open breaker waits before a trial call (floor 1). | | `LOOM_LANGFUSE_TIMEOUT_S` | `float` | `15.0` | live | Seconds one Langfuse API call may take: posting a score, and the traces and scores the prompt evaluation reads. | -| `LOOM_LANGFUSE_CAPTURE_IO_LEVEL` | `'off' \| 'query' \| 'all'` | `'off'` | | How much LLM prompt and completion text is recorded, by what the prompt holds: "off" records none (the default; consent is given, not withdrawn); "query" only the legs whose prompt is the user's own search text (hyde, query_rewrite, enrichment, …); "all" also the legs carrying retrieved memories or conversation turns (inline_rerank, iterative_judge, extract, session_summary, …). A namespace's `capture_generation_io` overrides it (false ⇒ off, true ⇒ all, NULL ⇒ this). It also gates loom.search_trace_log.query. Token counts, cost, latency and quality scores are never gated. | +| `LOOM_LANGFUSE_CAPTURE_IO_LEVEL` | `'off' \| 'query' \| 'all'` | `'off'` | | How much LLM prompt and completion text is recorded, by what the prompt holds: "off" records none (the default; consent is given, not withdrawn); "query" only the legs whose prompt is the user's own search text (hyde, query_rewrite, enrichment, …); "all" also the legs carrying retrieved memories or conversation turns (inline_rerank, iterative_judge, extract, …). A namespace's `capture_generation_io` overrides it (false ⇒ off, true ⇒ all, NULL ⇒ this). It also gates loom.search_trace_log.query. Token counts, cost, latency and quality scores are never gated. | | `LOOM_LANGFUSE_GENERATION_IO_MAX_CHARS` | `int` | `2000` | | Per-field cap on recorded prompt/completion text; truncation adds an explicit marker. | | `LOOM_RETRIEVAL_JUDGE_SAMPLE_RATE` | `float` | `0.0` | | Fraction of searches sent to an independent LLM judge, written back as a `retrieval_relevance` score (unlike loom's own confidence scores, it shares no state with retrieval). 0.0 = off: it costs money per search; a few percent (e.g. 0.05) suits production. | | `LOOM_ANALYTICS_EXPORT_NAME_KEY` | `str` | *(unset)* | | Secret that `loom export-analytics` hashes org, namespace and agent names under: a random string of 32 or more characters; a new key changes every hash. | diff --git a/docs/reference/operations.md b/docs/reference/operations.md index ba1127c137..9211aa7f43 100644 --- a/docs/reference/operations.md +++ b/docs/reference/operations.md @@ -538,7 +538,7 @@ Fields for every `memory` op: [operations/memory.md](operations/memory.md). | [`memory.reject`](operations/memory.md#memoryreject) | Mark a memory as explicitly rejected (extractor / user review decided the candidate fact is wrong), so it isn't re-extracted. | writer | idempotent | no | yes | | [`memory.related`](operations/memory.md#memoryrelated) | Find memories sharing entities with the source memory. | viewer | read | no | | | [`memory.resolve_orphan_duplicates`](operations/memory.md#memoryresolve_orphan_duplicates) | Scan (org, namespace) for orphaned duplicate rows and re-activate them. | writer | idempotent | yes | yes | -| [`memory.restore`](operations/memory.md#memoryrestore) | Bring back a deleted memory (within 14 days) or one Loom retired on its own. | writer | idempotent | yes | yes | +| [`memory.restore`](operations/memory.md#memoryrestore) | Reverse automatic retirement when no deletion or erasure refusal blocks the value. | writer | idempotent | yes | yes | | [`memory.search`](operations/memory.md#memorysearch) | Semantic search over the agent's own memory. | viewer | read | no | | | [`memory.search_trace`](operations/memory.md#memorysearch_trace) | Same shape as memory.search; returns the vector leg's per-candidate scoring breakdown (no fusion, assembly or reranker). | viewer | read | no | | | [`memory.search_trace_log`](operations/memory.md#memorysearch_trace_log) | Read loom.search_trace_log for a namespace (requires the telemetry flag). | viewer | read | yes | | diff --git a/docs/reference/operations/memory.md b/docs/reference/operations/memory.md index 5b0e8c789d..97e81689b2 100644 --- a/docs/reference/operations/memory.md +++ b/docs/reference/operations/memory.md @@ -757,7 +757,7 @@ The raw chunks and session digests behind a namespace's memories, as portable ro | `session_id` | `str` | `''` | optional: one session only; "" or "-" = every session | | `limit` | `int` | `500` | Rows per page (1–2000); `has_more` says whether another page follows. | | `cursor` | `str` | `''` | opaque keyset cursor from a previous page | -| `include_high` | `bool` | `False` | Also export the chunks and digests behind a 'high' row or whose text holds a high shape (a credential, card number, SSN or stated PIN), and the chunks stored without a vector (one may hold part of a key). | +| `include_high` | `bool` | `False` | Include high chunks and history-held digests. | | Output field | Type | Meaning | | --- | --- | --- | @@ -1105,7 +1105,7 @@ Write exported evidence chunks and digests into a namespace, re-embedded here an | `agent` | `str` | required | The writing or reading agent; `'-'` or `'*'` means any agent where the op allows it. | | `user_id` | `str` | `'-'` | The end user the row belongs to; `'-'` for none. | | `chunks` | `list[EvidenceChunk]` | *(computed)* | Chunks to re-embed and insert (≤256); empty-content or session-less ones are skipped, one that reads as high is inserted without a vector, and a key cut between two calls' chunks is not found. | -| `digests` | `list[EvidenceDigest]` | *(computed)* | Session digests to re-embed and insert (≤256); empty summaries are skipped, and one whose text reads as high is inserted without a vector. | +| `digests` | `list[EvidenceDigest]` | *(computed)* | Session digests to re-embed and insert (≤256); only exact extractive summaries with complete, currently eligible source proof are written. | | `source_trace` | `str` | `'okf-import'` | Stamped into memory_chunks.source_url when the chunk carries none. | | Output field | Type | Meaning | @@ -1116,7 +1116,7 @@ Write exported evidence chunks and digests into a namespace, re-embedded here an | `embedding_model` | `str` | The server embedder the imported rows were re-embedded with. | | `embedding_dim` | `int` | Vector length of the written embeddings; 0 when nothing was embedded (every item read as high, or none was written). | | `edited` | `list[str]` | Chunks whose body did not hash to the content_sha256 they arrived with. | -| `skipped` | `int` | Items not written: empty, unembeddable, or refused by a blacklist pattern (Refuse). | +| `skipped` | `int` | Items not written: empty, unembeddable, refused, or missing eligible source proof. | | `unchecked` | `int` | Of the skipped, those the blacklist could not check: the request's values are too long for its regexes; sent in a smaller request they are checked, unless one is too long alone, which must be shortened. | ## `memory.inspect` @@ -1842,7 +1842,7 @@ Scan (org, namespace) for orphaned duplicate rows and re-activate them. ## `memory.restore` -Bring back a deleted memory (within 14 days) or one Loom retired on its own. +Reverse automatic retirement when no deletion or erasure refusal blocks the value. `POST /v1/memory.restore` · MCP `loom_memory_restore` · role `writer` · idempotent · audited diff --git a/docs/reference/schema.md b/docs/reference/schema.md index 7d94cdf67d..534ec76a46 100644 --- a/docs/reference/schema.md +++ b/docs/reference/schema.md @@ -142,7 +142,7 @@ the `loom` tables each view reads, and, for each materialized view, its target, -196 tables, 5 views and 19 materialized views in the `loom` and `loom_telemetry` databases; a `loom_telemetry` object keeps its database prefix. +197 tables, 5 views and 19 materialized views in the `loom` and `loom_telemetry` databases; a `loom_telemetry` object keeps its database prefix. #### Tables @@ -276,6 +276,7 @@ the `loom` tables each view reads, and, for each materialized view, its target, | `memory_record_fields` | `ReplacingMergeTree(version)` | `(org, namespace, schema_name, field_name, record_id)` | | | 1 skip index (bloom_filter) | | `memory_record_versions` | `ReplacingMergeTree` | `(org, namespace, schema_name, memory_key, record_id, version)` | `toYYYYMM(created_at)` | | 1 skip index (bloom_filter) | | `memory_records` | `ReplacingMergeTree(version)` | `(org, namespace, schema_name, memory_key, record_id)` | | | 1 skip index (bloom_filter) | +| `memory_refusals` | `MergeTree` | `(org, scope_kind, scope_value, user_id, kind, units, digest)` | | | | | `memory_reinforcement` | `ReplacingMergeTree(version)` | `(org, namespace, agent, user_id, memory_key)` | | | | | `memory_schema_fields` | `ReplacingMergeTree(version)` | `(org, namespace, schema_id, field_name)` | | | | | `memory_schemas` | `ReplacingMergeTree(version)` | `(org, namespace, name)` | | | | diff --git a/llms.txt b/llms.txt index 3cc5487452..4b32f3b00c 100644 --- a/llms.txt +++ b/llms.txt @@ -166,6 +166,7 @@ Start with [the docs map](docs/index.md). To install Loom, a coding agent reads - [Live connected-source search](docs/architecture/live-source-search.md): `search.query(live=true)` adds an ephemeral section from the asker's own connected Drive and Notion registrations. - [LLM and embedding architecture](docs/architecture/llm-architecture.md): Loom has two pluggable providers: an embedder (`server/embedding/`) that turns text into vectors, and an LLM client (`server/llm/`) that completes prompts. - [Memory architecture](docs/architecture/memory-architecture.md): The memory plane stores facts in `loom.memory`, makes them searchable through `loom.memory_embeddings`, and derives evidence, chunks, graph rows and statistics from them. +- [Memory consent and derived text](docs/architecture/memory-consent.md): A deletion contracts memory and records content refusal before its tombstone commits. - [n8n adapter](docs/architecture/n8n-adapter.md): The n8n community package stores conversation history and durable facts in a Loom deployment through the TypeScript SDK. - [OpenAI Agents conversation sessions](docs/architecture/openai-agents-sessions.md): `LoomSession` implements the native Python and TypeScript session interfaces over Loom episodic memories. - [Pipecat demonstration recording](docs/architecture/pipecat-demo-recording.md): The optional `pipecat-demo.yml` workflow renders a timestamped execution visualization of an authenticated native Pipecat text-memory check. diff --git a/loom.example.yaml b/loom.example.yaml index 523368cf27..26ef156602 100644 --- a/loom.example.yaml +++ b/loom.example.yaml @@ -3237,10 +3237,18 @@ settings: # `offset` (wire cap stays 1000). 0 = no viewer cap. [restart] # memory_list_viewer_limit: 100 - # Transcript characters the session-digest call reads (~4 chars a token); 0 = the - # built-in 3000. [restart] + # Characters of complete source facts included in an extractive session digest; 0 = + # the built-in 3000. Facts are never clipped to fit. [restart] # memory_session_digest_max_input_chars: 3000 + # Raw digest candidates per query while filling listing, outline and export rows. + # [restart] + # memory_digest_listing_page_size: 256 + + # Queries allowed to find enough proven digests; exhaustion answers + # BACKEND_UNAVAILABLE. [restart] + # memory_digest_listing_max_pages: 256 + # content_excerpt ceiling for answer-bearing facts (question_topic set) when # memory_capture_structured_answers is on, so an enumerated answer is not clipped # mid-list; every other fact keeps EXCERPT_LIMIT (1,200 chars). [restart] @@ -4987,10 +4995,10 @@ settings: # records none (the default; consent is given, not withdrawn); "query" only the legs # whose prompt is the user's own search text (hyde, query_rewrite, enrichment, …); # "all" also the legs carrying retrieved memories or conversation turns - # (inline_rerank, iterative_judge, extract, session_summary, …). A namespace's - # `capture_generation_io` overrides it (false ⇒ off, true ⇒ all, NULL ⇒ this). It also - # gates loom.search_trace_log.query. Token counts, cost, latency and quality scores - # are never gated. [restart] + # (inline_rerank, iterative_judge, extract, …). A namespace's `capture_generation_io` + # overrides it (false ⇒ off, true ⇒ all, NULL ⇒ this). It also gates + # loom.search_trace_log.query. Token counts, cost, latency and quality scores are + # never gated. [restart] # langfuse_capture_io_level: "off" # Per-field cap on recorded prompt/completion text; truncation adds an explicit diff --git a/schemas/loom-settings.schema.json b/schemas/loom-settings.schema.json index dea7d18cc5..c40a02f2c9 100644 --- a/schemas/loom-settings.schema.json +++ b/schemas/loom-settings.schema.json @@ -3606,7 +3606,7 @@ "all" ], "type": "string", - "description": "How much LLM prompt and completion text is recorded, by what the prompt holds: \"off\" records none (the default; consent is given, not withdrawn); \"query\" only the legs whose prompt is the user's own search text (hyde, query_rewrite, enrichment, …); \"all\" also the legs carrying retrieved memories or conversation turns (inline_rerank, iterative_judge, extract, session_summary, …). A namespace's `capture_generation_io` overrides it (false ⇒ off, true ⇒ all, NULL ⇒ this). It also gates loom.search_trace_log.query. Token counts, cost, latency and quality scores are never gated." + "description": "How much LLM prompt and completion text is recorded, by what the prompt holds: \"off\" records none (the default; consent is given, not withdrawn); \"query\" only the legs whose prompt is the user's own search text (hyde, query_rewrite, enrichment, …); \"all\" also the legs carrying retrieved memories or conversation turns (inline_rerank, iterative_judge, extract, …). A namespace's `capture_generation_io` overrides it (false ⇒ off, true ⇒ all, NULL ⇒ this). It also gates loom.search_trace_log.query. Token counts, cost, latency and quality scores are never gated." }, "langfuse_circuit_cooldown_s": { "default": 30.0, @@ -4593,6 +4593,19 @@ "type": "integer", "description": "Characters one extraction read takes: a shorter document is read whole, a longer one in windows of whole sections. Sized so a dense window's reply fits the decision_extract cap." }, + "memory_digest_listing_max_pages": { + "default": 256, + "minimum": 1, + "type": "integer", + "description": "Queries allowed to find enough proven digests; exhaustion answers BACKEND_UNAVAILABLE." + }, + "memory_digest_listing_page_size": { + "default": 256, + "maximum": 2000, + "minimum": 1, + "type": "integer", + "description": "Raw digest candidates per query while filling listing, outline and export rows." + }, "memory_durability_verifier_max_facts_per_call": { "default": 60, "minimum": 1, @@ -5347,7 +5360,7 @@ "memory_session_digest_max_input_chars": { "default": 3000, "type": "integer", - "description": "Transcript characters the session-digest call reads (~4 chars a token); 0 = the built-in 3000." + "description": "Characters of complete source facts included in an extractive session digest; 0 = the built-in 3000. Facts are never clipped to fit." }, "memory_set_chunk_max_chunks": { "default": 48, diff --git a/schemas/migrations/115_memory_refusals.sql b/schemas/migrations/115_memory_refusals.sql new file mode 100644 index 0000000000..b51fd9b6b1 --- /dev/null +++ b/schemas/migrations/115_memory_refusals.sql @@ -0,0 +1,24 @@ +-- Durable consent fingerprints contain no removed plaintext and have no TTL. +CREATE TABLE IF NOT EXISTS loom.memory_refusals +( + org String, + namespace String, + agent String DEFAULT '-', + user_id String DEFAULT '-', + session_id String DEFAULT '-', + scope_kind LowCardinality(String) DEFAULT 'identity', + scope_value String, + actor String, + source_class LowCardinality(String) DEFAULT 'derived', + kind LowCardinality(String), + units UInt32, + digest FixedString(64), + created_at DateTime DEFAULT now() +) +ENGINE = MergeTree +ORDER BY (org, scope_kind, scope_value, user_id, kind, units, digest) +SETTINGS non_replicated_deduplication_window = 1000; + +-- A numeric version alone cannot distinguish independently written full states. +ALTER TABLE loom.delete_version_checks + ADD COLUMN IF NOT EXISTS checked_state_sha256 String DEFAULT ''; diff --git a/schemas/pg_migrations/014_memory_refusals.sql b/schemas/pg_migrations/014_memory_refusals.sql new file mode 100644 index 0000000000..2ab028c0b1 --- /dev/null +++ b/schemas/pg_migrations/014_memory_refusals.sql @@ -0,0 +1,16 @@ +CREATE TABLE IF NOT EXISTS loom.memory_refusals ( + org text COLLATE "C" NOT NULL, + namespace text COLLATE "C" NOT NULL, + agent text COLLATE "C" NOT NULL DEFAULT '-', + user_id text COLLATE "C" NOT NULL DEFAULT '-', + session_id text COLLATE "C" NOT NULL DEFAULT '-', + scope_kind text NOT NULL DEFAULT 'identity', + scope_value text NOT NULL, + actor text NOT NULL, + source_class text NOT NULL DEFAULT 'derived', + kind text NOT NULL, + units integer NOT NULL, + digest text NOT NULL, + created_at timestamptz NOT NULL DEFAULT now(), + PRIMARY KEY (org, scope_kind, scope_value, user_id, kind, units, digest) +); diff --git a/schemas/schema.sql b/schemas/schema.sql index e833512461..2b8590f2d8 100644 --- a/schemas/schema.sql +++ b/schemas/schema.sql @@ -16,13 +16,13 @@ -- Do NOT hand-edit. Edit schemas/migrations/*.sql and re-dump. -- CI verifies this file matches what migrations would produce. -- --- Counts: 196 tables + 5 views + 19 materialized views. +-- Counts: 197 tables + 5 views + 19 materialized views. CREATE DATABASE IF NOT EXISTS `loom`; CREATE DATABASE IF NOT EXISTS `loom_telemetry`; -- ====================================================================== --- Tables (196) +-- Tables (197) -- ====================================================================== -- loom.acl_versions @@ -1609,7 +1609,8 @@ CREATE TABLE IF NOT EXISTS loom.delete_version_checks `tombstone` UInt64, `done` UInt8 DEFAULT 0, `created_at` DateTime DEFAULT now(), - `updated_at` DateTime64(3) DEFAULT now64(3) + `updated_at` DateTime64(3) DEFAULT now64(3), + `checked_state_sha256` String DEFAULT '' ) ENGINE = ReplacingMergeTree(updated_at) ORDER BY (org, namespace, agent, user_id, session_id, memory_key, tombstone) @@ -3032,6 +3033,27 @@ ENGINE = ReplacingMergeTree(version) ORDER BY (org, namespace, schema_name, memory_key, record_id) SETTINGS index_granularity = 8192, non_replicated_deduplication_window = 1000; +-- loom.memory_refusals +CREATE TABLE IF NOT EXISTS loom.memory_refusals +( + `org` String, + `namespace` String, + `agent` String DEFAULT '-', + `user_id` String DEFAULT '-', + `session_id` String DEFAULT '-', + `scope_kind` LowCardinality(String) DEFAULT 'identity', + `scope_value` String, + `actor` String, + `source_class` LowCardinality(String) DEFAULT 'derived', + `kind` LowCardinality(String), + `units` UInt32, + `digest` FixedString(64), + `created_at` DateTime DEFAULT now() +) +ENGINE = MergeTree +ORDER BY (org, scope_kind, scope_value, user_id, kind, units, digest) +SETTINGS non_replicated_deduplication_window = 1000, index_granularity = 8192; + -- loom.memory_reinforcement CREATE TABLE IF NOT EXISTS loom.memory_reinforcement ( diff --git a/scripts/codegen/push_langfuse_prompts.py b/scripts/codegen/push_langfuse_prompts.py index a414324959..c6af194791 100644 --- a/scripts/codegen/push_langfuse_prompts.py +++ b/scripts/codegen/push_langfuse_prompts.py @@ -61,7 +61,6 @@ def registry() -> dict[str, str]: from server.memory.app import extract as extract_svc from server.memory.app import period_digest as _period_digest from server.memory.app import rerank - from server.memory.app import session_summary as sess from server.memory.domain import extraction_prompts as extract from server.memory.domain import search_prompts from server.memory.workers import entity_extractor, profile_consolidation @@ -88,7 +87,6 @@ def registry() -> dict[str, str]: # relation backfill, sent bare "loom-extract-relations": extract_svc._RELATION_BACKFILL_SYSTEM_PROMPT, # summarisation / workers - "loom-session-summary": sess._SESSION_SUMMARY_SYSTEM, "loom-period-digest": _period_digest._PERIOD_DIGEST_SYSTEM, "loom-conversation-title": conversation_titler._TITLE_SYSTEM, "loom-profile-fold": profile_consolidation._FOLD_SYSTEM, diff --git a/scripts/gates/function_length_baseline.json b/scripts/gates/function_length_baseline.json index d778beb05e..9c81dcec52 100644 --- a/scripts/gates/function_length_baseline.json +++ b/scripts/gates/function_length_baseline.json @@ -27,7 +27,6 @@ "server/memory/app/consolidate.py": 1, "server/memory/app/entity_resolution.py": 1, "server/memory/app/expiry.py": 1, - "server/memory/app/extract/handler.py": 1, "server/memory/app/extract/parse.py": 1, "server/memory/app/extract/sweep.py": 1, "server/memory/app/extract/typed.py": 1, diff --git a/sdk/python/loom/memory.py b/sdk/python/loom/memory.py index 1a67b33434..816de7822a 100644 --- a/sdk/python/loom/memory.py +++ b/sdk/python/loom/memory.py @@ -1386,8 +1386,10 @@ async def restore( if_superseded_by: str | None = None, only_if_superseded: bool = False, ) -> dict[str, Any]: - """Bring back a deleted memory (within the 14-day grace window) or one - Loom retired on its own (status `superseded`). + """Reverse an automatic tombstone within its 14-day grace window, or + an automatic supersession. Explicit deletion and clearing record a + permanent refusal: restoring that content raises VALIDATION_FAILED + with reason `forgotten`. `if_superseded_by` restores the row only while it is still retired behind that memory id (as a `memory.auto_supersede` record names it); @@ -1399,13 +1401,12 @@ async def restore( - `memory_id: str | None` — UUID of the restored row. - `version: int | None` — new version assigned. - `expired: bool` — True when the tombstone aged past the 14d - grace window. The data is being GC'd; restoration is no longer - possible. - - `from_status: str` — `"tombstoned"` (a delete undone) or + grace window; restoration is no longer available. + - `from_status: str` — `"tombstoned"` (automatic expiry reversed) or `"superseded"` (a retirement reversed); empty on a no-op. Preserves the original char_limit, pinned, tags, and entities - across the restore. An undone delete clears `expires_at`, so the row + across the restore. Reversing an automatic tombstone clears `expires_at`, so the row gets a fresh lease (its old expiry may have passed, which would tombstone it again at once); an undone retirement keeps it. """ @@ -1434,9 +1435,8 @@ async def clear( Returns `{cleared_at, rows_affected: {table: count}, total, locked_skipped}`; locked rows stay and are counted in `locked_skipped`. - Tombstone-not-hard-delete: cleared rows are recoverable for the - 14-day grace window by re-writing the same key. For - cross-system erasure use `erasure.request_user_id` or + History remains, but restoration and re-writing refused content are + blocked. For physical erasure across stores use `erasure.request_user_id` or `erasure.request_namespace`. """ payload: dict[str, Any] = { diff --git a/sdk/python/loom/ops.py b/sdk/python/loom/ops.py index a95526c354..9606382061 100644 --- a/sdk/python/loom/ops.py +++ b/sdk/python/loom/ops.py @@ -15586,7 +15586,7 @@ class MemoryResolveOrphanDuplicatesOutput(TypedDict, total=False): class MemoryRestoreInput(TypedDict, total=False): - "Bring back a deleted memory (within 14 days) or one Loom retired on its own." + "Reverse automatic retirement when no deletion or erasure refusal blocks the value." org: Required[str] namespace: Required[str] @@ -15600,7 +15600,7 @@ class MemoryRestoreInput(TypedDict, total=False): class MemoryRestoreBody(TypedDict, total=False): - "Bring back a deleted memory (within 14 days) or one Loom retired on its own." + "Reverse automatic retirement when no deletion or erasure refusal blocks the value." org: NotRequired[str] namespace: NotRequired[str] diff --git a/sdk/python/loom/profile.py b/sdk/python/loom/profile.py index 1798c27a1d..e2cbae7811 100644 --- a/sdk/python/loom/profile.py +++ b/sdk/python/loom/profile.py @@ -156,8 +156,10 @@ async def delete( user_id: str | None = None, session_id: str | None = None, ) -> bool: - """Tombstone a profile section. Soft-delete via memory.delete; - restoreable via `memory.restore` within the grace window.""" + """Tombstone a profile section and refuse its removed content. + + `memory.restore` cannot republish explicitly deleted content. + """ return await self._agent.memory.delete( memory_key=label, user_id=user_id, diff --git a/server/cli/import_clickgap.py b/server/cli/import_clickgap.py index cc44181142..aae290778f 100644 --- a/server/cli/import_clickgap.py +++ b/server/cli/import_clickgap.py @@ -781,7 +781,7 @@ def _cmd_import_clickgap(args: argparse.Namespace) -> int: from server.core.ch import get_client from server.embedding import get_embedder from server.memory.app.provenance import writer_actor - from server.memory.store import insert_memory_versions + from server.memory.store import insert_embedding_rows, insert_memory_versions async def _run() -> int: sqlite_path = args.sqlite @@ -860,8 +860,8 @@ async def _run() -> int: await insert_memory_versions(mem_batch, ch=ch) total_items += len(mem_batch) if emb_batch: - await ch.insert("loom.memory_embeddings", emb_batch) - total_emb += len(emb_batch) + admitted = await insert_embedding_rows(emb_batch, ch=ch) + total_emb += len(admitted) print( f" ✓ {total_items} lessons → memory ({total_emb} embeddings, " f"{w.polarity_neg} suppression/polarity:negative, " diff --git a/server/core/ch.py b/server/core/ch.py index 151e46a7b3..563ec14134 100644 --- a/server/core/ch.py +++ b/server/core/ch.py @@ -1365,7 +1365,13 @@ async def query_one( return None return dict(zip(result.column_names, rows[0], strict=False)) - async def _query_fail_open(self, op_name: str, sql: str, params: dict[str, Any] | None) -> Any: + async def _query_fail_open( + self, + op_name: str, + sql: str, + params: dict[str, Any] | None, + settings: dict[str, Any] | None = None, + ) -> Any: """Run a SELECT with the standard retry; if the server rejects the quantized-codes setting (a replica that does not know it), turn the setting off for this process and run the query once @@ -1376,7 +1382,7 @@ async def _q(text: str) -> Any: res = await self._statement_client().query( text, parameters=params or {}, - settings=statement_settings(text, cls="read"), + settings=statement_settings(text, settings, cls="read"), **_reader_auth(), ) note_result_summary(res) @@ -1408,8 +1414,10 @@ async def query_all( self, sql: str, params: dict[str, Any] | None = None, + *, + settings: dict[str, Any] | None = None, ) -> list[dict[str, Any]]: - result = await self._query_fail_open("query_all", sql, params) + result = await self._query_fail_open("query_all", sql, params, settings) return [dict(zip(result.column_names, row, strict=False)) for row in result.result_rows] async def query_columns( diff --git a/server/core/repo_base.py b/server/core/repo_base.py index f68ea2cb2c..d9646b4563 100644 --- a/server/core/repo_base.py +++ b/server/core/repo_base.py @@ -29,6 +29,8 @@ async def query_all( self, sql: str, params: dict[str, Any] | None = ..., + *, + settings: dict[str, Any] | None = ..., ) -> list[dict[str, Any]]: ... async def execute( self, diff --git a/server/erasure/app/saga_steps.py b/server/erasure/app/saga_steps.py index 3e77271375..78f89f3cd1 100644 --- a/server/erasure/app/saga_steps.py +++ b/server/erasure/app/saga_steps.py @@ -21,6 +21,7 @@ from server.core.principals import personal_name from server.core.repo_base import _CHClient from server.erasure.store import census as erasure_census +from server.erasure.store import refusals as refusal_steps from server.erasure.store import steps_graph as graph_steps from server.erasure.store import steps_ledgers as ledger_steps from server.erasure.store import steps_memory as memory_steps @@ -327,6 +328,7 @@ def authority_steps[StepT: tuple[str, Any]](steps: tuple[StepT, ...]) -> list[St # The step names are stored in the ledger's `steps_completed`: renaming one # makes a resumed saga run it again. USER_STEPS: tuple[UserStep, ...] = ( + ("memory.refusals", refusal_steps.capture_user), # First: the authority's copy, then the wait for its publishers, so every # ClickHouse copy the steps below look for has landed. ("pg_memory.delete", pg_steps.delete_pg_memory_by_user), @@ -386,6 +388,7 @@ def authority_steps[StepT: tuple[str, Any]](steps: tuple[StepT, ...]) -> list[St # The loom-side steps of a namespace erasure over (org, namespace or list), in # saga order. Names as for USER_STEPS. NAMESPACE_STEPS: tuple[NamespaceStep, ...] = ( + ("memory.refusals", refusal_steps.capture_namespace), ("knowledge_sources.retire", _step_retire_knowledge_sources_by_ns), ("knowledge_source_push.delete", ledger_steps.delete_source_push_by_namespace), ("pg_memory.delete", pg_steps.delete_pg_memory_by_namespace), diff --git a/server/erasure/store/census.py b/server/erasure/store/census.py index ce1551e109..236d052526 100644 --- a/server/erasure/store/census.py +++ b/server/erasure/store/census.py @@ -530,6 +530,8 @@ def owner(self, scope_type: str) -> str | None: ) STEP_ONLY_TABLES: dict[str, tuple[str, ...]] = { + # Captures retained hash-only consent policy; this auxiliary step erases no table. + "memory.refusals": (), "entity_canonical_map.tombstone": ("loom.entity_canonical_map",), "agent_events.scrub": ("loom.agent_events", "loom.agent_events_private"), "subject_last_seen.delete": ("loom.subject_last_seen",), @@ -767,6 +769,10 @@ def owner(self, scope_type: str) -> str | None: "profile built from one is a loom.memory row." ), "loom.blacklist": "consent patterns authored by admins; policy, not subject data.", + "loom.memory_refusals": ( + "durable deletion consent fingerprints; no removed plaintext or TTL. " + "Retained so erasure cannot permit the same value to be learned again." + ), # -- tracker plane: org-shared reference data mirrored from CI, not anyone's # memory; rebuilt from the source, never read into a memory answer. "loom.qa_test_daily": ( diff --git a/server/erasure/store/consent.py b/server/erasure/store/consent.py new file mode 100644 index 0000000000..b0d222fb78 --- /dev/null +++ b/server/erasure/store/consent.py @@ -0,0 +1,32 @@ +"""The consent policy the composition root supplies to erasure stores.""" + +from __future__ import annotations + +from collections.abc import Awaitable, Callable +from dataclasses import dataclass + +from server.core.errors import BackendUnavailableError +from server.core.repo_base import _CHClient + + +@dataclass(frozen=True) +class ConsentPolicy: + fence_names: Callable[[str, str, str], list[str]] + lease_client: Callable[[_CHClient], _CHClient] + lock_postgres: Callable[..., Awaitable[None]] + record_clickhouse: Callable[..., Awaitable[None]] + record_postgres: Callable[..., Awaitable[None]] + + +_policy: ConsentPolicy | None = None + + +def use_policy(policy: ConsentPolicy) -> None: + global _policy + _policy = policy + + +def current_policy() -> ConsentPolicy: + if _policy is None: + raise BackendUnavailableError("erasure consent policy is not wired") + return _policy diff --git a/server/erasure/store/refusals.py b/server/erasure/store/refusals.py new file mode 100644 index 0000000000..e1dd9f9b57 --- /dev/null +++ b/server/erasure/store/refusals.py @@ -0,0 +1,251 @@ +"""Record refusal fingerprints before an erasure removes its source history.""" + +from __future__ import annotations + +from collections.abc import Sequence +from typing import Any + +from server.core import lease_writes, pg +from server.core.ch import sequential_reads, set_query_tag, write_limit_s +from server.core.ch_params import chunks +from server.core.repo_base import _CHClient, _default_client +from server.erasure.store.consent import current_policy +from server.erasure.store.predicates import ( + USER_CHUNKS, + USER_SUMMARIES, + USER_SUMMARY_CHUNKS, + USER_SUMMARY_VECTORS, +) + +_KEYS = ( + "namespace", + "agent", + "user_id", + "session_id", + "memory_key", + "version", + "body_sha", +) +_CURSOR = "Tuple(String,String,String,String,String,UInt64,FixedString(64))" +_BODY_SHA = "hex(SHA256(toJSONString((value_json, value)))) AS body_sha" + + +def _sources() -> list[tuple[str, str]]: + identity = "org, namespace, agent, user_id, session_id, 'process:erasure' AS actor, " + version = "toUInt64(toUnixTimestamp64Nano(toDateTime64(created_at, 9))) AS version, " + sources = [ + ( + "loom.session_summaries", + identity + "concat('summary:', session_id) AS memory_key, version, " + "'{}' AS value_json, " + "toJSONString(arrayJoin(arrayConcat([summary_text], key_facts))) AS value", + ), + ( + "loom.memory_evidence", + identity + + "concat('evidence:', toString(evidence_id)) AS memory_key, " + + version + + "'{}' AS value_json, toJSONString(content) AS value", + ), + ( + "loom.memory_evidence", + identity + + "concat('evidence-metadata:', toString(evidence_id)) AS memory_key, " + + version + + "'{}' AS value_json, metadata AS value, 'metadata' AS capture_kind", + ), + ( + "loom.memory_chunks", + identity + "concat('chunk:', toString(chunk_index), ':', " + "content_sha256) AS memory_key, " + version + "'{}' AS value_json, " + "toJSONString(arrayJoin([content, caption, source_url, source_local_path])) AS value", + ), + ] + sources.append( + ( + "loom.memory_embeddings", + identity + "memory_key, version, '{}' AS value_json, " + "toJSONString(content_excerpt) AS value", + ) + ) + sources.extend( + [ + ( + "loom.memory_extraction_queue", + identity + "concat('queue:', queue_id) AS memory_key, version, " + "'{}' AS value_json, payload AS value, 'payload' AS capture_kind", + ), + ( + "loom.memory_extraction_queue", + identity + "concat('queue-error:', queue_id) AS memory_key, version, " + "'{}' AS value_json, toJSONString(last_error) AS value", + ), + ( + "loom.entity_summaries", + "org, namespace, agent, user_id, '-' AS session_id, 'process:erasure' AS actor, " + "concat('entity:', entity) AS memory_key, version, '{}' AS value_json, " + "toJSONString(arrayJoin(arrayConcat([summary_text], key_facts))) AS value", + ), + ] + ) + # Cleared native revision payloads survive in ClickHouse history. + sources.insert( + 0, + ( + "loom.memory_versions", + identity + "memory_key, version, toString(value_json) AS value_json, value", + ), + ) + return sources + + +def _predicate(table: str, kind: str) -> str: + if kind == "user_id": + if table == "loom.memory_embeddings": + return f"(user_id={{uid:String}} OR {USER_SUMMARY_VECTORS})" + if table == "loom.memory_chunks": + return f"({USER_CHUNKS} OR {USER_SUMMARY_CHUNKS})" + if table == "loom.memory_versions": + return f"(user_id={{uid:String}} OR {USER_SUMMARIES})" + return f"{kind}={{scope:String}}" + + +async def _capture(org: str, kind: str, value: str) -> int: + ch = _default_client() + refusals = current_policy() + names = refusals.fence_names(org, kind, value) + count = 0 + # The ledger blocks new intake. Drain admitted sends before paging history; + # capture can outlast a send lease without admitting new writers. + async with lease_writes.fenced_many( + names, ch=refusals.lease_client(ch), wait_s=write_limit_s() + ): + pass + with sequential_reads(): + for table, projection in _sources(): + count += await _capture_source(ch, table, projection, org, kind, value) + return count + + +async def _capture_source( + ch: _CHClient, table: str, projection: str, org: str, kind: str, value: str +) -> int: + refusals = current_policy() + count = 0 + after: tuple[Any, ...] | None = None + while True: + set_query_tag("erasure.memory_refusals") + predicates = [] + if after: + predicates.append(f"({', '.join(_KEYS)}) > {{after:{_CURSOR}}}") + if table != "loom.memory_versions": + predicates.append("value != {empty_text:String}") + cursor = "WHERE " + " AND ".join(predicates) if predicates else "" + rows = await ch.query_all( + f"SELECT *, {_BODY_SHA} FROM (SELECT {projection} FROM {table} " + f"WHERE org={{org:String}} AND {_predicate(table, kind)}) {cursor} " + f"ORDER BY {', '.join(_KEYS)} LIMIT 1000", + params={ + "org": org, + "scope": value, + "uid": value, + "after": after, + "empty_text": '""', + }, + ) + if not rows: + break + if pg.enabled(): + async with pg.transaction() as conn: + await refusals.record_postgres(conn, rows, scope_kind=kind, scope_value=value) + else: + await refusals.record_clickhouse(ch, rows, scope_kind=kind, scope_value=value) + count += len(rows) + after = tuple(rows[-1][c] for c in _KEYS) + return count + + +async def capture_user(org: str, user_id: str) -> int: + return await _capture(org, "user_id", user_id) + + +async def capture_namespace(org: str, namespace: str | Sequence[str]) -> int: + count = 0 + for ns in [namespace] if isinstance(namespace, str) else sorted(set(namespace)): + count += await _capture(org, "namespace", ns) + return count + + +async def capture_native_chunks(conn: Any, sources: list[dict[str, Any]], user_id: str) -> None: + keys = sorted( + { + tuple( + str(r.get(c) or "-") for c in ("org", "namespace", "agent", "user_id", "memory_key") + ) + for r in sources + } + ) + for part in chunks(keys, limit=16 * 1024): + await _capture_native_chunk_keys(conn, part, user_id) + + +async def _capture_native_chunk_keys( + conn: Any, sources: list[tuple[str, ...]], user_id: str +) -> None: + """Capture raw dependents by authoritative keys before native history disappears.""" + if not sources: + return + org = sources[0][0] + identities = sorted( + { + (ns, agent, uid, key) + for _, ns, agent, source_uid, key in sources + for uid in (source_uid, "-") + } + ) + legacy = sorted({(ns, key) for _, ns, _, source_uid, key in sources if source_uid == user_id}) + ch = _default_client() + for table, parent in ( + ("loom.memory_chunks", "parent_memory_key"), + ("loom.memory_embeddings", "memory_key"), + ): + await _capture_native_dependents(conn, org, user_id, identities, legacy, table, parent, ch) + + +async def _capture_native_dependents( + conn: Any, + org: str, + user_id: str, + identities: Sequence[tuple[str, ...]], + legacy: Sequence[tuple[str, ...]], + table: str, + parent: str, + ch: _CHClient, +) -> None: + projection = dict(_sources())[table] + refusals = current_policy() + after: tuple[Any, ...] | None = None + with sequential_reads(): + while True: + set_query_tag("erasure.memory_refusals") + cursor = f"AND ({', '.join(_KEYS)}) > {{after:{_CURSOR}}}" if after else "" + rows = await ch.query_all( + f"SELECT *, {_BODY_SHA} FROM (SELECT {projection} FROM {table} " + f"WHERE org={{org:String}} AND ((namespace, agent, user_id, {parent}) " + "IN {identities:Array(Tuple(String,String,String,String))} OR " + f"(user_id='-' AND (namespace, {parent}) " + "IN {legacy:Array(Tuple(String,String))}))) " + f"WHERE value != {{empty_text:String}} {cursor} " + f"ORDER BY {', '.join(_KEYS)} LIMIT 1000", + params={ + "org": org, + "identities": identities, + "legacy": legacy, + "after": after, + "empty_text": '""', + }, + ) + if not rows: + return + await refusals.record_postgres(conn, rows, scope_kind="user_id", scope_value=user_id) + after = tuple(rows[-1][c] for c in _KEYS) diff --git a/server/erasure/store/steps_pg.py b/server/erasure/store/steps_pg.py index 7fd6f091ef..410065db0e 100644 --- a/server/erasure/store/steps_pg.py +++ b/server/erasure/store/steps_pg.py @@ -135,6 +135,14 @@ async def _erase_once( # A large scope takes longer than a request's statement budget, on the client's # timeout as on the server's: its own connection, with neither. The scope's writes wait. async with pg.dedicated() as conn, conn.transaction(): + from server.erasure.store.consent import current_policy + + refusals = current_policy() + + for value in sorted(set(namespaces or [])) if user_id is None else [user_id]: + await refusals.lock_postgres( + conn, [], erasure=("namespace" if user_id is None else "user_id", org, value) + ) # Advancing each generation locks it: the scope's commits wait for this # transaction, and an admission that captured the old one checks again. scoped: list[str] | None = None # a user's scope reads the org's evidence @@ -170,6 +178,7 @@ async def _erase_once( "ORDER BY d.org, d.namespace, d.seq FOR UPDATE OF d", *args, ) + await _capture_history(conn, where_r, where_c, args, user_id) # The deliveries go with their revisions (ON DELETE CASCADE). This statement locks # them and reads their leases, any committed since the locks above included, and # moves the audit rows ClickHouse lacks to the outbox: the audit trail outlives the @@ -206,6 +215,50 @@ async def _erase_once( return int(revisions or 0) + int(heads or 0), [float(x) for x in leases] +async def _capture_history( + conn: Any, where_r: str, where_c: str, args: list[Any], user_id: str | None +) -> None: + """Keep payload fingerprints and dependent keys before native history is removed. + + Delivered revisions may retain only typed metadata. Their plaintext is captured + from ClickHouse history by the refusal step; their keys still find raw dependents. + """ + from server.erasure.store.consent import current_policy + from server.erasure.store.refusals import capture_native_chunks + + refusals = current_policy() + + async def capture(batch: list[dict[str, Any]]) -> None: + payloads = [source["row"] for source in batch if source["row"] is not None] + await refusals.record_postgres( + conn, + payloads, + scope_kind="namespace" if user_id is None else "user_id", + scope_value=user_id, + ) + if user_id is not None: + await capture_native_chunks( + conn, + [{key: value for key, value in row.items() if key != "row"} for row in batch], + user_id, + ) + + batch: list[dict[str, Any]] = [] + async for source in conn.cursor( + "SELECT r.row, r.org, r.namespace, coalesce(r.agent, r.row->>'agent', '') AS agent, " + "r.user_id, coalesce(r.memory_key, r.row->>'memory_key', '') AS memory_key " + f"FROM loom.memory_revisions r WHERE {where_r} " + "UNION ALL SELECT row, org, namespace, agent, user_id, memory_key " + f"FROM loom.memory_current WHERE {where_c}", + *args, + ): + batch.append(dict(source)) + if len(batch) >= 1000: + await capture(batch) + batch = [] + await capture(batch) + + async def _mark_step(conn: Any, org: str) -> None: """The running saga's step recorded in its ledger row, in this step's transaction (`pg_ledger.mark_step`); outside a saga, nothing.""" @@ -517,6 +570,7 @@ async def delete_pg_idempotency(org: str) -> int: } # The Postgres tables that hold none, with the reason. PG_NO_MEMORY_CONTENT: dict[str, str] = { + "loom.memory_refusals": "durable consent fingerprints, no removed plaintext; survives erasure", "loom.authority": "the deployment's recorded authority and epoch; no tenant data.", "loom.cell_first": "whether this cell is the deployment's first, and its name; no tenant data.", "loom.tenancy_mode": "the cell's tenancy mode (recording or enforced); no tenant data.", diff --git a/server/memory/api/models_crud.py b/server/memory/api/models_crud.py index 1e7b58452a..9a8c99b292 100644 --- a/server/memory/api/models_crud.py +++ b/server/memory/api/models_crud.py @@ -569,7 +569,7 @@ class MemoryDeleteVersionOutput(BaseModel): class MemoryRestoreInput(_Identity): - """Bring back a deleted memory (within 14 days) or one Loom retired on its own.""" + """Reverse automatic retirement when no deletion or erasure refusal blocks the value.""" # The key to restore, under this exact identity (no namespace-wide fallback). memory_key: str diff --git a/server/memory/api/models_evidence_transfer.py b/server/memory/api/models_evidence_transfer.py index 66646f77df..ad3cd381df 100644 --- a/server/memory/api/models_evidence_transfer.py +++ b/server/memory/api/models_evidence_transfer.py @@ -58,10 +58,8 @@ class MemoryExportEvidenceInput(BaseModel): # Rows per page (1–2000); `has_more` says whether another page follows. limit: int = Field(default=500, ge=1, le=2000) cursor: str = "" # opaque keyset cursor from a previous page - # Also export the chunks and digests behind a 'high' row or whose text holds a high - # shape (a credential, card number, SSN or stated PIN), and the chunks stored without - # a vector (one may hold part of a key). Off: a bundle reaches a wider audience than - # the namespace's grants. + # Include high chunks and history-held digests. A digest still needs complete, + # current, eligible source proof; this flag never overrides that requirement. include_high: bool = False @@ -89,8 +87,8 @@ class MemoryImportEvidenceInput(BaseModel): # one that reads as high is inserted without a vector, and a key cut between # two calls' chunks is not found. chunks: list[EvidenceChunk] = Field(default_factory=list, max_length=IMPORT_MAX_ITEMS) - # Session digests to re-embed and insert (≤256); empty summaries are skipped, and one - # whose text reads as high is inserted without a vector. + # Session digests to re-embed and insert (≤256); only exact extractive summaries + # with complete, currently eligible source proof are written. digests: list[EvidenceDigest] = Field(default_factory=list, max_length=IMPORT_MAX_ITEMS) # Stamped into memory_chunks.source_url when the chunk carries none. source_trace: str = Field(default="okf-import", max_length=512) @@ -110,7 +108,7 @@ class MemoryImportEvidenceOutput(BaseModel): embedding_dim: int = 0 # Chunks whose body did not hash to the content_sha256 they arrived with. edited: list[str] = Field(default_factory=list) - # Items not written: empty, unembeddable, or refused by a blacklist pattern (Refuse). + # Items not written: empty, unembeddable, refused, or missing eligible source proof. skipped: int = 0 # Of the skipped, those the blacklist could not check: the request's values are too # long for its regexes; sent in a smaller request they are checked, unless one is diff --git a/server/memory/app/chunk_projection.py b/server/memory/app/chunk_projection.py index 9106716ea3..faaadfc009 100644 --- a/server/memory/app/chunk_projection.py +++ b/server/memory/app/chunk_projection.py @@ -11,7 +11,6 @@ from contextvars import ContextVar from typing import Any -from server.core.ch_writes import committed_insert_settings from server.core.datetimes import naive_utc_now from server.embedding.documents import document_vector, document_vectors from server.llm import customer @@ -392,11 +391,9 @@ async def write_memory_chunks( # write must find them; a fire-and-forget insert would break that. Committed, # not batched: a mis-sized vector fails the shared part of a batched flush, # and with it every other insert in that flush. - await ch.insert( - "loom.memory_chunks", - rows, - settings=committed_insert_settings(), - ) + from server.memory.store.refusals import insert_content_rows + + await insert_content_rows(ch, "loom.memory_chunks", rows) if provenance_rows: try: await ch.insert( diff --git a/server/memory/app/curation.py b/server/memory/app/curation.py index 4f0473a329..ce454bc367 100644 --- a/server/memory/app/curation.py +++ b/server/memory/app/curation.py @@ -143,12 +143,10 @@ async def handle_memory_digests(input: MemoryDigestsInput) -> MemoryDigestsOutpu where += " AND session_id = {s:String}" params["s"] = input.session_id where += scope_and(request_scope(), params) - rows = await ch.query_all( - "SELECT session_id, summary_text, key_facts, derived_from, valid_at, created_at, " - "version FROM loom.session_summaries FINAL WHERE " - + where - + " ORDER BY created_at, session_id LIMIT {lim:UInt32}", - params, + from server.memory.store.summary_pages import verified_summary_page + + rows = await verified_summary_page( + ch, org=input.org, namespace=input.namespace, where=where, params=params, limit=input.limit ) page = rows[: int(input.limit)] # Checking every digest's text is CPU work; it runs off the event loop. @@ -205,12 +203,16 @@ async def handle_memory_outline(input: MemoryOutlineInput) -> MemoryOutlineOutpu sessions: list[_OutlineSession] = [] sessions_total = 0 if int(input.sessions_limit) > 0: - digests = await ch.query_all( - "SELECT session_id, summary_text, length(key_facts) AS kf, length(derived_from) AS df, " - "coalesce(valid_at, created_at) AS d FROM loom.session_summaries FINAL " - "WHERE org = {o:String} AND namespace = {n:String} AND tombstoned_at IS NULL " - "ORDER BY d DESC, session_id LIMIT {lim:UInt32}", - {**base, "lim": int(input.sessions_limit) + 1}, + from server.memory.store.summary_pages import verified_summary_page + + digests = await verified_summary_page( + ch, + org=input.org, + namespace=input.namespace, + where="org = {o:String} AND namespace = {n:String} AND tombstoned_at IS NULL", + params=base, + limit=input.sessions_limit, + order="newest", ) total = await ch.query_all( "SELECT count() AS c FROM loom.session_summaries FINAL " diff --git a/server/memory/app/deep_evidence.py b/server/memory/app/deep_evidence.py index 71b5d01b64..04a1611069 100644 --- a/server/memory/app/deep_evidence.py +++ b/server/memory/app/deep_evidence.py @@ -3,8 +3,8 @@ `memory.ask` synthesizes from fact excerpts (loom.memory, ≤2,000 chars each). A fact is a compression of the turns it came from, and some questions need the turns: the exact wording, the number the extractor rounded, the second -clause it dropped. The deep path recalls summaries, lets one LLM call pick the -relevant ones, follows the summary→event link to the raw text and answers again. +clause it dropped. The deep path selects relevant facts, follows their evidence +links to the supporting raw text and answers again. Loom stores the two tiers as `loom.memory` and `loom.memory_evidence`, linked by `loom.memory_belief_sources`; this module is the read that follows the link. @@ -358,145 +358,6 @@ async def fetch_raw_evidence( SUMMARY_KEY_PREFIX = "summary:" -_TOKEN_RE = re.compile(r"[a-z0-9]{4,}") - - -def _tokens(text: str) -> set[str]: - return set(_TOKEN_RE.findall((text or "").lower())) - - -def session_ids_of(memory_keys: list[str]) -> list[str]: - """Session ids behind `summary:` hits (the session-summary - leg, fetch_session_summaries), in order, de-duplicated.""" - - out: list[str] = [] - for key in memory_keys: - if key.startswith(SUMMARY_KEY_PREFIX): - sid = key[len(SUMMARY_KEY_PREFIX) :] - if sid and sid not in out: - out.append(sid) - return out - - -def window_turns_by_question( - rows: list[dict[str, Any]], - question: str, - *, - max_chars: int, - context_turns: int = 1, -) -> list[dict[str, Any]]: - """The turns of a session worth opening for `question`, bounded. - - A session summary is one hit for a whole session (tens of thousands of - characters), so it cannot be opened whole. Score each turn by the - distinct question tokens (4+ chars) it contains, keep the best-scoring - turns plus `context_turns` neighbours on each side for coherence, in - chronological order, until `max_chars` is spent. Zero-score turns are - only kept as neighbours. - """ - - if not rows or max_chars <= 0: - return [] - q = _tokens(question) - scored = [(len(q & _tokens(str(r.get("content") or ""))), i) for i, r in enumerate(rows)] - keep: set[int] = set() - spent = 0 - for score, i in sorted(scored, key=lambda p: (-p[0], p[1])): - if score <= 0: - break - block = range(max(0, i - context_turns), min(len(rows), i + context_turns + 1)) - cost = sum(len(str(rows[j].get("content") or "")) for j in block if j not in keep) - if spent + cost > max_chars: - # Take the scoring turn alone if it fits; otherwise stop. - solo = len(str(rows[i].get("content") or "")) - if i not in keep and spent + solo <= max_chars: - keep.add(i) - spent += solo - continue - for j in block: - keep.add(j) - spent += cost - return [rows[i] for i in sorted(keep)] - - -async def fetch_session_turns( - *, - org: str, - namespace: str, - session_ids: list[str], - question: str, - max_chars_per_session: int, - max_chars_total: int, - ch: Any | None = None, -) -> list[EvidenceTurn]: - """Open the sessions behind session-summary hits, windowed by the question. - - The summary leg ranks a whole session; this reads that session's turns - from `loom.memory_evidence` (a key-prefix read on org, namespace, then - session_id) and keeps the question-relevant window per session, capped - per session and in total. Labelled with the `summary:` key so the - synthesizer's citation points back at the hit. - """ - - if not session_ids: - return [] - from server.core.ch import get_client - - client = ch or get_client() - # Session ids repeat across users: a narrowed read opens only its own. - params: dict[str, Any] = {"org": org, "ns": namespace, "sids": list(session_ids)} - in_scope = scope_and(request_scope(), params) - # Versioned, not FINAL: an evidence row's id is minted per - # insert, so the only re-insert is a tombstone; the latest version by - # `created_at` decides, as FINAL would, without the query-time merge - # (same rows, a fraction of the read). - sql = f""" - SELECT session_id, role, toString(observed_at) AS observed_at, content - FROM ( - SELECT session_id, role, observed_at, content, evidence_id, tombstoned_at - FROM loom.memory_evidence - WHERE org = {{org:String}} - AND namespace = {{ns:String}} - AND session_id IN {{sids:Array(String)}} - AND content != ''{in_scope} - ORDER BY evidence_id, created_at DESC - LIMIT 1 BY evidence_id - ) - WHERE tombstoned_at IS NULL - ORDER BY session_id, observed_at ASC, evidence_id - LIMIT 2000 - """ - try: - rows = await client.query_all(sql, params=params) - except Exception as e: # noqa: BLE001 — fail-open - log.warning("deep-evidence session fetch failed (fail-open): %s", e) - return [] - # A turn that reads as high never reaches the synthesis prompt. - rows = await asyncio.to_thread(without_high, rows, lambda r: str(r.get("content") or "")) - by_sid: dict[str, list[dict[str, Any]]] = {} - for r in rows: - by_sid.setdefault(str(r.get("session_id") or ""), []).append(r) - out: list[EvidenceTurn] = [] - total = 0 - for sid in session_ids: - budget = min(max_chars_per_session, max_chars_total - total) - if budget <= 0: - break - for r in window_turns_by_question(by_sid.get(sid, []), question, max_chars=budget): - content = str(r.get("content") or "").strip() - if not content: - continue - total += len(content) - out.append( - EvidenceTurn( - memory_key=f"{SUMMARY_KEY_PREFIX}{sid}", - role=str(r.get("role") or ""), - observed_at=str(r.get("observed_at") or "")[:19], - content=content, - support_type="session", - ) - ) - return out def render_raw_evidence(turns: list[EvidenceTurn]) -> str: @@ -541,22 +402,7 @@ async def deep_evidence_pass( max_chars_total=max_chars_total, ch=ch, ) - # A picked session-summary hit opens its session, windowed by the - # question, with whatever budget the fact evidence left. - sids = session_ids_of(keys) - if sids: - left = max_chars_total - sum(len(t.content) for t in turns) - if left > 0: - turns = turns + await fetch_session_turns( - org=org, - namespace=namespace, - session_ids=sids, - question=question, - max_chars_per_session=max_chars_per_memory, - max_chars_total=left, - ch=ch, - ) - turns.sort(key=lambda t: (t.observed_at, t.memory_key)) + # A digest proves its source facts, not every raw turn from their session. block = render_raw_evidence(turns) return DeepEvidence( selected_keys=keys, @@ -575,10 +421,7 @@ async def deep_evidence_pass( "candidate_lines", "deep_evidence_pass", "fetch_raw_evidence", - "fetch_session_turns", "parse_picks", "render_raw_evidence", "select_candidates", - "session_ids_of", - "window_turns_by_question", ] diff --git a/server/memory/app/evidence.py b/server/memory/app/evidence.py index bdc4b49bd0..2b3709b209 100644 --- a/server/memory/app/evidence.py +++ b/server/memory/app/evidence.py @@ -83,11 +83,9 @@ async def write_evidence_rows( } ) out.append(EvidenceRow(evidence_id=evidence_id, content_sha256=content_sha)) - await ch.insert( - "loom.memory_evidence", - rows, - settings=batched_insert_settings(), - ) + from server.memory.store.refusals import insert_content_rows + + await insert_content_rows(ch, "loom.memory_evidence", rows) return out diff --git a/server/memory/app/evidence_transfer.py b/server/memory/app/evidence_transfer.py index f21e7e3282..b4a1622ea7 100644 --- a/server/memory/app/evidence_transfer.py +++ b/server/memory/app/evidence_transfer.py @@ -134,17 +134,16 @@ async def handle_memory_export_evidence( params["sid"] = input.session_id if input.kind == "digests": - rows = await ch.query_all( - f""" - SELECT session_id, agent, user_id, summary_text, key_facts, derived_from, - valid_at, created_at - FROM loom.session_summaries FINAL - WHERE {" AND ".join(where)} - ORDER BY session_id ASC, version DESC - LIMIT 1 BY session_id - LIMIT {{lim:UInt32}} - """, + from server.memory.store.summary_pages import verified_summary_page + + rows = await verified_summary_page( + ch, + org=input.org, + namespace=input.namespace, + where=" AND ".join(where), params=params, + limit=input.limit, + order="session", ) digests = [ EvidenceDigest( @@ -379,7 +378,9 @@ async def handle_memory_import_evidence( ) ) if rows: - await ch.insert("loom.memory_chunks", rows, settings=committed_insert_settings()) + from server.memory.store.refusals import insert_content_rows + + await insert_content_rows(ch, "loom.memory_chunks", rows) out.chunks_written = len(rows) from server.memory.app.chunk_links import SOURCES_TABLE, import_link_rows @@ -432,10 +433,8 @@ async def handle_memory_import_evidence( ) ) if drows: - await ch.insert( - "loom.session_summaries", - drows, - settings=committed_insert_settings(), - ) - out.digests_written = len(drows) + from server.memory.store.summary_sources import insert_verified_summaries + + out.digests_written = await insert_verified_summaries(ch, drows) + out.skipped += len(drows) - out.digests_written return out diff --git a/server/memory/app/extract/handler.py b/server/memory/app/extract/handler.py index 4c062e55ef..da9088fcc7 100644 --- a/server/memory/app/extract/handler.py +++ b/server/memory/app/extract/handler.py @@ -456,7 +456,6 @@ async def _project_extraction( msgs: list[dict[str, Any]], state: Any, evidence_ids: list[str], - summary_draft_task: asyncio.Task[Any] | None, ) -> int: """The projections of a pipeline write — evidence links, verbatim chunks, the session summary — run together; returns the belief-source links @@ -535,32 +534,11 @@ async def _project_chunks() -> None: payload={"message_count": len(msgs)}, ) - # Session-summary plane: one cheap LLM call per session emits a dense - # dated digest, embedded with the SAME model/dim as the fact plane so - # cosine is comparable. Awaited before the response so an immediate - # post-ingest read sees it. Fail-open — the facts already landed. + # The committed facts, rather than the raw transcript, determine the digest. async def _project_session_summary() -> None: - draft: Any = None - if summary_draft_task is not None: - try: - draft = await summary_draft_task - except Exception as e: # noqa: BLE001 — a projection; the write stands - log.warning("session digest draft failed (non-fatal): %s", e) - await record_input_projection_failure( - input, - projection="session_summaries", - error=e, - memory_keys=written_memory_keys, - payload={"message_count": len(msgs), "stage": "draft"}, - ) - return - if draft is None: - # The model returned nothing (or the embedder did): the same - # skip the inline path takes, without a second model call. - return try: with traced("extract.session_summary", messages=len(msgs)): - await _write_session_summary(input, msgs, state, draft=draft) + await _write_session_summary(input, state) except Exception as e: # noqa: BLE001 — a projection; the write stands log.warning("session_summaries write failed (non-fatal): %s", e) await record_input_projection_failure( @@ -625,7 +603,14 @@ async def _set_from_messages( detect_updates_override.set(input.detect_updates) last_value_updates_linked.set(0) from server.memory.app.extract import ExtractionError + from server.memory.domain.refusals import transcript_content + from server.memory.store.refusals import check_intake + await check_intake( + get_client(), + input.model_dump(), + transcript_content(input.model_dump()), + ) if input.defer: return await _enqueue_deferred(input, rule_agent) if _deferred_by_allowlist(input): @@ -705,28 +690,10 @@ async def _set_from_messages( agent_learnings=learnings_on, **_extract_request_pins(input), ) - # The session digest is a function of the transcript, not of the extraction result, so - # its model call and embedding run WHILE the extractor runs; the row is written once the - # memory keys exist. A session that extracts nothing discards the draft (one cheap call, - # ~0.4% of sessions). - summary_draft_task: asyncio.Task[Any] | None = None - if msgs: - - async def _draft_digest() -> Any: - with traced("extract.session_summary_draft", messages=len(msgs)): - return await _draft_session_summary( - input, msgs, observation_date, operator_policy=operator_policy - ) - - summary_draft_task = create_deferred_task(_draft_digest()) - summary_draft_task.add_done_callback(_observe_task_result) try: with traced("extract.pipeline", steps=len(steps)): state = await run_pipeline(steps, state) except BaseException as e: - # Any failure or cancel of the extraction also stops the digest draft's LLM call. - if summary_draft_task is not None: - summary_draft_task.cancel() if not isinstance(e, ExtractionError): raise raise HTTPException( @@ -742,9 +709,7 @@ async def _draft_digest() -> Any: ) evidence_ids = await evidence_task _ing_clock.mark("evidence_write") # only what the pipeline did not cover - belief_source_links = await _project_extraction( - input, msgs, state, evidence_ids, summary_draft_task - ) + belief_source_links = await _project_extraction(input, msgs, state, evidence_ids) _ing_clock.mark("projections") ontology_warnings = await _ontology_warnings(input, state) @@ -817,16 +782,7 @@ def _observe_task_result(task: asyncio.Task[Any]) -> None: from server.memory.app.inline_records import ( # noqa: E402 (re-export) _write_inline_record_projection, ) -from server.memory.app.session_summary import ( # noqa: E402, F401 (re-export) - _SESSION_SUMMARY_MAX_INPUT_CHARS, - _SESSION_SUMMARY_MAX_KEY_FACTS, - _SESSION_SUMMARY_PROMPT_VERSION, - _SESSION_SUMMARY_SYSTEM, - _build_session_summary_user_prompt, - _draft_session_summary, - _top_key_facts, - _write_session_summary, -) +from server.memory.app.session_summary import _write_session_summary # noqa: E402 (re-export) __all__ = [ "handle_memory_bulk_set", @@ -839,12 +795,7 @@ def _observe_task_result(task: asyncio.Task[Any]) -> None: "_link_written_memories_to_evidence", "_write_source_evidence", "_write_inline_record_projection", - "_build_session_summary_user_prompt", - "_top_key_facts", - "_draft_session_summary", "_write_session_summary", - "_SESSION_SUMMARY_MAX_INPUT_CHARS", - "_SESSION_SUMMARY_MAX_KEY_FACTS", # Patch points: the session-summary helpers read these through this module. "get_client", "get_embedder", diff --git a/server/memory/app/extraction_queue.py b/server/memory/app/extraction_queue.py index 4adbda9fb7..b2130168fb 100644 --- a/server/memory/app/extraction_queue.py +++ b/server/memory/app/extraction_queue.py @@ -33,10 +33,12 @@ from datetime import timedelta from typing import Any, NamedTuple +from server.core.ch import sequential_reads from server.core.ch_names import split_table from server.core.datetimes import UNIX_EPOCH from server.core.datetimes import utc_now as _now -from server.core.errors import stored_caller_text +from server.core.errors import ValidationError, stored_caller_text +from server.core.hashing import sha256_text from server.identity.domain.read_scope import ( READ_SCOPE_KEY, ReadFilters, @@ -46,6 +48,7 @@ ) from server.llm import customer from server.memory.api.models_bulk import MemorySetFromMessagesInput +from server.memory.domain.refusals import transcript_content from server.memory.store.rows import next_version from server.settings import settings @@ -177,10 +180,92 @@ async def enqueue( "version": next_version(), "enqueued_by": str(json.loads(payload).get(ENQUEUED_BY_KEY) or ""), } - await ch.insert(TABLE, [row], settings=_SYNC) + await _insert_rows(ch, [row], skip_refused=False, create=True) return qid +async def _insert_rows( + ch: Any, rows: list[dict[str, Any]], *, skip_refused: bool = True, create: bool = False +) -> list[dict[str, Any]]: + """State transitions carry raw payloads, so they need intake admission too.""" + from server.memory.store.refusals import content_policy_row, publication + + written = [] + for row in rows: + content = [*transcript_content(json.loads(row.get("payload") or "{}"))] + if row.get("last_error"): + content.append(row["last_error"]) + try: + async with publication(ch, [content_policy_row(row, content)]): + if not create and not await _source_present(ch, row): + continue + await ch.insert(TABLE, [row], settings=_SYNC) + written.append(row) + except ValidationError as error: + if not skip_refused or error.extra.get("reason") != "forgotten": + raise + await _drop_refused(ch, row) + return written + + +class _QueueRevision(dict[str, Any]): + def __init__(self, values: dict[str, Any], source_version: int) -> None: + super().__init__(values) + self.source_version = source_version + + +async def _source_present(ch: Any, row: dict[str, Any]) -> bool: + if not isinstance(row, _QueueRevision): + raise ValueError("A queue transition requires its exact source revision") + with sequential_reads(): + found = await ch.query_all( + "SELECT 1 FROM loom.memory_extraction_queue FINAL " + "WHERE org={org:String} AND namespace={namespace:String} AND agent={agent:String} " + "AND user_id={user_id:String} AND session_id={session_id:String} " + "AND queue_id={queue_id:String} AND version={source_version:UInt64} " + "AND hex(SHA256(payload))={payload_sha:String} LIMIT 1", + params={ + **{ + key: row[key] + for key in ( + "org", + "namespace", + "agent", + "user_id", + "session_id", + "queue_id", + ) + }, + "source_version": row.source_version, + "payload_sha": sha256_text(str(row.get("payload") or "")).upper(), + }, + ) + return bool(found) + + +async def _drop_refused(ch: Any, row: dict[str, Any]) -> None: + """End an existing refused job without copying any of its erased payload.""" + if not isinstance(row, _QueueRevision): + raise ValueError("A refused transition requires its exact source revision") + await ch.execute( + "ALTER TABLE loom.memory_extraction_queue UPDATE status='failed', " + "last_error='dropped: refusal policy', lease_until=toDateTime64(0,3) " + "WHERE org={org:String} AND namespace={namespace:String} AND agent={agent:String} " + "AND user_id={user_id:String} AND session_id={session_id:String} " + "AND queue_id={queue_id:String} AND version={source_version:UInt64} " + "AND hex(SHA256(payload))={payload_sha:String} " + "AND status IN ('pending','running')", + params={ + **{ + key: row[key] + for key in ("org", "namespace", "agent", "user_id", "session_id", "queue_id") + }, + "source_version": row.source_version, + "payload_sha": sha256_text(str(row.get("payload") or "")).upper(), + }, + ) + + def _mirror(row: dict[str, Any], **changes: Any) -> dict[str, Any]: """A new version of `row` with `changes` applied and every column carried forward — omitting one zeros it for readers under FINAL.""" @@ -197,7 +282,7 @@ def _mirror(row: dict[str, Any], **changes: Any) -> dict[str, Any]: for c in ("last_error", "payload", "enqueued_by"): if out.get(c) is None: out[c] = "" - return out + return _QueueRevision(out, int(row["version"])) def backoff_s(attempts: int) -> int: @@ -269,18 +354,15 @@ async def claim_batch( claimed = [ _mirror(dict(r), status=STATUS_RUNNING, lease_until=lease, last_error="") for r in rows ] - await ch.insert(TABLE, claimed, settings=_SYNC) - return claimed + return await _insert_rows(ch, claimed) async def mark_done(row: dict[str, Any], *, written: int, ch: Any | None = None) -> None: from server.core.ch import get_client ch = ch or get_client() - await ch.insert( - TABLE, - [_mirror(row, status=STATUS_DONE, written=int(written), lease_until=UNIX_EPOCH)], - settings=_SYNC, + await _insert_rows( + ch, [_mirror(row, status=STATUS_DONE, written=int(written), lease_until=UNIX_EPOCH)] ) @@ -299,8 +381,8 @@ async def mark_error( ch = ch or get_client() attempts = int(row.get("attempts") or 0) + 1 failed = attempts >= max(1, int(max_attempts)) - await ch.insert( - TABLE, + await _insert_rows( + ch, [ _mirror( row, @@ -312,7 +394,6 @@ async def mark_error( else _now() + timedelta(seconds=backoff_s(attempts)), ) ], - settings=_SYNC, ) return STATUS_FAILED if failed else STATUS_PENDING @@ -324,8 +405,8 @@ async def mark_unreachable(row: dict[str, Any], *, error: str, ch: Any | None = ch = ch or get_client() wait = backoff_s(int(row.get("attempts") or 0) + 1) - await ch.insert( - TABLE, + await _insert_rows( + ch, [ _mirror( row, @@ -334,7 +415,6 @@ async def mark_unreachable(row: dict[str, Any], *, error: str, ch: Any | None = lease_until=_now() + timedelta(seconds=wait), ) ], - settings=_SYNC, ) @@ -348,8 +428,8 @@ async def mark_held( from server.core.ch import get_client ch = ch or get_client() - await ch.insert( - TABLE, + await _insert_rows( + ch, [ _mirror( row, @@ -360,7 +440,6 @@ async def mark_held( enqueued_by=str(row.get("enqueued_by") or "") or enqueued_actor(row), ) ], - settings=_SYNC, ) @@ -423,13 +502,12 @@ async def drop_waiting( ch=ch, ) why = f"{DROPPED_LABEL}model calls switched off, waited over {older_than_days} days" - await ch.insert( - TABLE, + await _insert_rows( + ch, [ _mirror(dict(r), status=STATUS_FAILED, last_error=why, lease_until=UNIX_EPOCH) for r in rows ], - settings=_SYNC, ) return len(rows) @@ -482,8 +560,8 @@ async def drop_expiring( why = ( f"{DROPPED_LABEL}still waiting after {_LAST_DAY} days, a day before the {TTL_DAYS}-day TTL" ) - await ch.insert( - TABLE, + await _insert_rows( + ch, [ _mirror( dict(r), @@ -493,7 +571,6 @@ async def drop_expiring( ) for r in rows ], - settings=_SYNC, ) return len(rows) diff --git a/server/memory/app/handlers.py b/server/memory/app/handlers.py index a4efe78689..7124edf928 100644 --- a/server/memory/app/handlers.py +++ b/server/memory/app/handlers.py @@ -24,7 +24,7 @@ from server.core.hashing import served_content_sha256 from server.embedding import get_embedder from server.identity.app.request_scope import request_scope -from server.identity.domain.read_scope import WHOLE_NAMESPACE +from server.identity.domain.read_scope import WHOLE_NAMESPACE, ReadFilters, narrower from server.memory.api.models_crud import ( CodeLink, MemoryClearInput, @@ -55,7 +55,9 @@ from server.memory.domain.served import served_totals from server.memory.domain.webhook_events import memory_change from server.memory.store import key_write_lock +from server.memory.store.delete_checks import checked_state_sha256 from server.memory.store.inserts import insert_memory_versions +from server.memory.store.pg_memory import AdmissionContendedError, unchanged from server.memory.store.rows import Identity, next_version, version_over from server.observability import degradations from server.observability.budget_shape import note_budget_shape @@ -603,10 +605,11 @@ async def handle_memory_unpin(input: MemoryUnpinInput) -> MemoryUnpinOutput: async def handle_memory_delete(input: MemoryDeleteInput) -> MemoryDeleteOutput: - """Soft-delete: write a tombstone row with a higher version, so RMT - collapse picks it and `memory.get`'s `tombstoned_at IS NULL` filter - excludes it. Copies the live row's columns forward. Preserves the row - for the grace window; avoids expensive CH mutations.""" + """Refuse the exact live value, then carry it into a higher-version tombstone. + + The complete row remains available for history; ordinary reads exclude its + tombstone, and later intake cannot restore the removed content. + """ import server.memory.store as memory_repo from server.observability.otel import traced @@ -617,10 +620,18 @@ async def handle_memory_delete(input: MemoryDeleteInput) -> MemoryDeleteOutput: user_id=input.user_id, session_id=input.session_id, ) + requested_scope = request_scope() + if requested_scope.narrowed and requested_scope.user_id != input.user_id: + return MemoryDeleteOutput(tombstoned=False, found=False, version=next_version()) + delete_scope = ReadFilters( + scope=narrower(requested_scope.scope, "own_user"), + user_id=input.user_id, + session_id=requested_scope.session_id, + ) with traced("ch.query.memory.precheck", op="tombstone_lookup"): # fresh: a delete right after a set must not no-op on a replica lag miss. existing = await memory_repo.get_by_key( - identity, input.memory_key, fresh=True, strong=True, scope=request_scope() + identity, input.memory_key, fresh=True, strong=True, scope=delete_scope ) if existing is None and identity.agent != "*": # A delete names a key, not the identity that wrote it (hooks write under their own @@ -634,7 +645,7 @@ async def handle_memory_delete(input: MemoryDeleteInput) -> MemoryDeleteOutput: session_id=input.session_id, ) existing = await memory_repo.get_by_key( - wide, input.memory_key, fresh=True, strong=True, scope=request_scope() + wide, input.memory_key, fresh=True, strong=True, scope=delete_scope ) if existing is not None and not same_user(existing, identity): existing = None # the wide form ignores user_id; another user's row is not ours @@ -646,7 +657,7 @@ async def handle_memory_delete(input: MemoryDeleteInput) -> MemoryDeleteOutput: # The version is drawn here, right before the insert: the vector retire skips rows at # or above it, so the later it is drawn the fewer late vector writes it leaves live. version = version_over(existing) - await _write_tombstone_row(existing, version=version) + await _write_tombstone_row(existing, version=version, refuse_value=True) try: tombstoned, downgraded = await _retire_derived( existing, version=version, op="memory.delete" @@ -748,13 +759,22 @@ async def _delete_version_locked( "memory_key": existing.memory_key, "checked": int(input.version), "tombstone": version, + "checked_state_sha256": checked_state_sha256(existing), } try: await memory_repo.record_delete_check(check, done=False) await _write_tombstone_row( - existing, version=version, trace=delete_version_trace(input.version) + existing, version=version, trace=delete_version_trace(input.version), refuse_value=True ) return await finish_delete_check(existing, check) + except AdmissionContendedError: + # Admission refused before sending or recording a refusal. Reconcile the + # newer head rather than treating it as an uncertain tombstone insert. + try: + answer = await rerun_delete_check(check) + return answer or MemoryDeleteVersionOutput(tombstoned=False, found=False) + except Exception as e: # a recorded check must remain owed on an uncertain result + raise _check_deferred(e) from e except Exception as e: # a retry finishes it, or the worker once the check is recorded raise _check_deferred(e) from e @@ -852,21 +872,85 @@ async def rerun_delete_check(check: dict[str, Any]) -> MemoryDeleteVersionOutput ) mine = {r.version: r for r in rows if (r.agent, r.user_id, r.session_id) == ident} removed = mine.get(checked) + committed = mine.get(tombstone) + if committed is not None and ( + committed.tombstoned_at is None or committed.source_trace != delete_version_trace(checked) + ): + committed = None + if committed is not None: + # History can lose the checked state to a same-version collision. The + # committed tombstone carries the content this deletion actually removed. + removed = committed.model_copy(update={"version": checked, "tombstoned_at": None}) if removed is None: await memory_repo.record_delete_check(check, done=True) return None - if tombstone not in mine: - same_delete, others = await _after_checked(removed, checked=checked, tombstone=tombstone) + if committed is None: + recovered = await _recover_missing_tombstone(removed, check, tombstone) + if isinstance(recovered, MemoryDeleteVersionOutput): + return recovered + tombstone = recovered + if tombstone != int(check["tombstone"]): + committed = await _version_row(removed, tombstone) + if committed.tombstoned_at is None or committed.source_trace != delete_version_trace( + checked + ): + raise BackendUnavailableError("the recovered tombstone no longer matches its check") + removed = committed.model_copy(update={"version": checked, "tombstoned_at": None}) + return await finish_delete_check(removed, check, tombstone=tombstone) + + +async def _recover_missing_tombstone( + removed: Any, check: dict[str, Any], tombstone: int +) -> int | MemoryDeleteVersionOutput: + """Retry known pre-send races; an uncertain send leaves the check owed.""" + import server.memory.store as memory_repo + + for _ in range(4): + same_delete, others = await _after_checked( + removed, checked=int(check["checked"]), tombstone=tombstone + ) if same_delete: - tombstone = max(same_delete) # a retry's, or a late one: the delete's that stands - elif not others: - # Nothing came after the removed version: the delete is still owed, above any - # write a fence lifted over a late commit of the original. - tombstone = _above_lift_band(tombstone) + return max(same_delete) + if others: + return tombstone + expected = check.get("checked_state_sha256", "") + if not expected or checked_state_sha256(removed) != expected: + # No write can recover the requested full state from a numeric version. + # Close this terminal check so it cannot starve newer recovery jobs. + await memory_repo.record_delete_check(check, done=True) + return MemoryDeleteVersionOutput(tombstoned=False, reason="version_changed") + current = await memory_repo.get_by_key( + Identity( + org=removed.org, + namespace=removed.namespace, + agent=removed.agent, + user_id=removed.user_id, + session_id=removed.session_id, + ), + removed.memory_key, + fresh=True, + strong=True, + include_tombstoned=True, + scope=WHOLE_NAMESPACE, + ) + # A same-version collision is invisible to the history's version range. + if not unchanged(removed, current): + await memory_repo.record_delete_check(check, done=True) + if current is None or current.tombstoned_at is not None: + return MemoryDeleteVersionOutput(tombstoned=False, found=False) + return MemoryDeleteVersionOutput(tombstoned=False, reason="version_changed") + try: + replacement = _above_lift_band(tombstone) await _write_tombstone_row( - removed, version=tombstone, trace=delete_version_trace(checked) + removed, + version=replacement, + trace=delete_version_trace(check["checked"]), + refuse_value=True, ) - return await finish_delete_check(removed, check, tombstone=tombstone) + return replacement + except AdmissionContendedError: + continue + raise AdmissionContendedError("the memory kept changing during deletion recovery") async def _key_moved_on( @@ -887,7 +971,9 @@ async def _key_moved_on( row = await _version_row(existing, max(others)) gone = row.tombstoned_at is not None # another remover took the key top = max([tombstone, *same_delete]) - if row.version < top and not (gone and await _head_carries(existing, top, row)): + if row.version == top or ( + row.version < top and not (gone and await _head_carries(existing, top, row)) + ): # A sweep's marker stays on the sweep's tombstone, which keeps the writes it caught. trace = delete_version_trace(checked) if gone else None await _write_above(row, tombstone=top, source_trace=trace) @@ -1035,7 +1121,7 @@ async def _after_checked( same_delete: list[int] = [] others: list[int] = [] for v, deleted, target in above: - if v != tombstone: + if v != tombstone or not deleted or target != checked: (same_delete if deleted and target == checked else others).append(v) return same_delete, others @@ -1084,7 +1170,14 @@ async def _write_above(row: Any, *, tombstone: int, source_trace: str | None = N await insert_memory_versions([back], ch=get_client()) -async def _write_tombstone_row(existing: Any, *, version: int, trace: str = "") -> None: +async def _write_tombstone_row( + existing: Any, + *, + version: int, + trace: str = "", + refuse_value: bool = False, + exact: bool = False, +) -> None: """The loom.memory tombstone of `existing` at `version`, `trace` as its source_trace.""" import server.memory.store as memory_repo from server.observability.otel import traced @@ -1102,7 +1195,13 @@ async def _write_tombstone_row(existing: Any, *, version: int, trace: str = "") ) with traced("ch.insert.memory.tombstone", table="loom.memory"): # Over the head it read: a lock taken since is never dropped by the tombstone. - await memory_repo.insert_versions_over_heads([tombstone_row], [existing], ch=get_client()) + await memory_repo.insert_versions_over_heads( + [tombstone_row], + [existing], + ch=get_client(), + exact=exact or refuse_value, + refuse_values=refuse_value, + ) async def _retire_derived( diff --git a/server/memory/app/move.py b/server/memory/app/move.py index f4a969958e..6c069c942d 100644 --- a/server/memory/app/move.py +++ b/server/memory/app/move.py @@ -22,7 +22,7 @@ from server.core.datetimes import naive_utc_now from server.core.hashing import served_content_sha256 from server.identity.domain.read_scope import WHOLE_NAMESPACE -from server.memory.api.models_crud import MemoryDeleteInput, MemorySetInput +from server.memory.api.models_crud import MemorySetInput from server.memory.api.models_move import MemoryMoveInput, MemoryMoveOutput from server.memory.app import handlers as _mem from server.memory.app import set_chunks as _set_chunks @@ -32,7 +32,9 @@ from server.memory.domain import provenance as _prov from server.memory.domain.delete_marks import carried_trace from server.memory.domain.memory_types import is_searchable_type +from server.memory.domain.webhook_events import memory_change from server.memory.store.rows import version_over +from server.observability.budget_shape import note_budget_shape if TYPE_CHECKING: from server.memory.store import MemoryRow @@ -144,7 +146,7 @@ async def _redirect_source(source: MemoryRow, dest_id: str, ch: Any) -> None: duplicate_of=dest_id, updated_at=now, ) - await memory_repo.insert_versions_over_heads([row], [source], ch=ch) + await memory_repo.insert_versions_over_heads([row], [source], ch=ch, exact=True) # The source key now reads as a redirect to the destination. await webhooks.announce([("memory.revised", row)]) try: @@ -171,16 +173,13 @@ async def _redirect_source(source: MemoryRow, dest_id: str, ch: Any) -> None: async def _tombstone_source(source: MemoryRow) -> None: """`keep_redirect: false` — the delete path, which carries the mirror and the chunk tombstones itself.""" - await _mem.handle_memory_delete( - MemoryDeleteInput( - org=source.org, - namespace=source.namespace, - agent=source.agent, - user_id=source.user_id, - session_id=source.session_id, - memory_key=source.memory_key, - ) - ) + version = version_over(source) + await _mem._write_tombstone_row(source, version=version, exact=True) + try: + await _mem._retire_derived(source, version=version, op="memory.move") + finally: + if await webhooks.announce([("memory.tombstoned", memory_change(source, version))]): + note_budget_shape("webhook") async def _destination_id(set_out: Any, set_input: Any) -> str: diff --git a/server/memory/app/ops.py b/server/memory/app/ops.py index 8d117fa353..cc838707d7 100644 --- a/server/memory/app/ops.py +++ b/server/memory/app/ops.py @@ -363,10 +363,21 @@ async def handle_memory_reindex(input: MemoryReindexInput) -> MemoryReindexOutpu # would; a key without axis text gets memory.set's zero vector at dim 0. embedding_rows = [] for i, r in enumerate(rows): + from server.core.hashing import hash_text, served_row_value, sha256_value + key_vec = key_vecs.get(i, []) + canonical_hash = sha256_value(served_row_value({"value": r["content_excerpt"]})) embedding_rows.append( memory_repo.build_embedding_row( - **{k: v for k, v in r.items() if k not in ("memory_version", "content_excerpt")}, + **{ + k: v + for k, v in r.items() + if k not in ("memory_version", "content_excerpt", "content_sha256", "verified") + }, + content_sha256=canonical_hash, + verified=int( + bool(r.get("verified")) and hash_text(r.get("content_sha256")) == canonical_hash + ), content_excerpt=texts[i][0], embedding_model=target_model, embedding_dim=embedder.dim, @@ -376,13 +387,13 @@ async def handle_memory_reindex(input: MemoryReindexInput) -> MemoryReindexOutpu embed_attempts=1, ) ) - await memory_repo.insert_embedding_rows(embedding_rows, ch=ch) + admitted = await memory_repo.insert_embedding_rows(embedding_rows, ch=ch) await _recheck_reindexed_heads(rows, embedding_rows, ch=ch, model=target_model) # One above versions read before the embed call: under the BM25 watermark. - await memory_repo.index_late_rows(embedding_rows, ch=ch) + await memory_repo.index_late_rows(admitted, ch=ch) return MemoryReindexOutput( - processed=len(embedding_rows), + processed=len(admitted), skipped_already_current=0, target_model=target_model, target_dim=embedder.dim, diff --git a/server/memory/app/period_digest.py b/server/memory/app/period_digest.py index f42d25a535..e4cd012257 100644 --- a/server/memory/app/period_digest.py +++ b/server/memory/app/period_digest.py @@ -22,10 +22,9 @@ Under Postgres the authority's heads decide which periods are written, so one committed but not yet in ClickHouse is not written again. Nothing is tombstoned. -The rows are plain memories: they compete in the vector legs like any fact, -keep out of the session-digest leg (whose keys start with `summary:`) and -out of the digest chain, and read back through `memory.list` -and `memory.evidence` like every other derived row. +The rows are proposals. Independent approval admits them to ordinary fact retrieval; +explicit proposal reads can inspect them before approval. They keep out of the +session-digest leg (whose keys start with `summary:`) and out of the digest chain. """ from __future__ import annotations @@ -34,6 +33,7 @@ import re import uuid from collections import defaultdict +from collections.abc import AsyncIterator from dataclasses import dataclass, field from datetime import datetime from typing import Any @@ -262,6 +262,7 @@ def build_period_memory_rows( source_trace=f"{SOURCE_TRACE}|prompt={PERIOD_DIGEST_PROMPT_VERSION}", actor=writer_actor(fallback="process:period_digest"), source_class="derived", + status="proposed", tags=tags, entities=ents, content_sha256=sha, @@ -291,7 +292,8 @@ def build_period_memory_rows( tags=tags, entities=ents, valid_at=valid_at, - status="active", + status="proposed", + content_sha256=sha, ) return memory_row, embedding_row @@ -342,28 +344,79 @@ async def _fetch_namespaces(ch: Any, cfg: PeriodDigestConfig) -> list[dict[str, return list(rows or []) -async def _fetch_digests(ch: Any, ident: dict[str, Any]) -> list[dict[str, Any]]: - rows = await ch.query_all( - """ - SELECT org, namespace, agent, user_id, session_id, summary_text, derived_from, - coalesce(valid_at, created_at) AS at - FROM loom.session_summaries FINAL - WHERE org = {org:String} AND namespace = {ns:String} - AND agent = {agent:String} AND user_id = {uid:String} - AND tombstoned_at IS NULL - ORDER BY at, session_id - """, - params={ - "org": ident["org"], - "ns": ident["namespace"], - "agent": ident["agent"], - "uid": ident["user_id"], - }, - ) - return list(rows or []) +async def _fetch_digests( + ch: Any, + ident: dict[str, Any], + *, + page_size: int = 256, + after: tuple[datetime, str] | None = None, +) -> AsyncIterator[list[dict[str, Any]]]: + """Page one latest live digest per session before cutting period windows.""" + from server.core.ch import set_query_tag + + while True: + set_query_tag("cli.period_digest.sources") + cursor = " WHERE (at, session_id) > {after:Tuple(DateTime,String)}" if after else "" + rows = await ch.query_all( + "SELECT org, namespace, agent, user_id, session_id, embedding_model, version, " + "summary_text, key_facts, derived_from, at FROM (" + "SELECT org, namespace, agent, user_id, session_id, embedding_model, version, " + "summary_text, key_facts, derived_from, coalesce(valid_at, created_at) AS at " + "FROM loom.session_summaries FINAL " + "WHERE org={org:String} AND namespace={ns:String} " + "AND agent={agent:String} AND user_id={uid:String} AND tombstoned_at IS NULL " + "ORDER BY session_id, version DESC, embedding_model LIMIT 1 BY session_id)" + + cursor + + " ORDER BY at, session_id LIMIT {limit:UInt32}", + params={ + "org": ident["org"], + "ns": ident["namespace"], + "agent": ident["agent"], + "uid": ident["user_id"], + "after": after, + "limit": page_size, + }, + ) + if not rows: + return + yield list(rows) + if len(rows) < page_size: + return + after = (rows[-1]["at"], str(rows[-1]["session_id"])) -async def _skipped_period_keys(ch: Any, ident: dict[str, Any], *, rewrite: bool) -> set[str]: +async def _plan_digest_pages( + ch: Any, ident: dict[str, Any], cfg: PeriodDigestConfig +) -> tuple[list[Period], int]: + """Keep one page, an incomplete window, and at most the requested output cap.""" + from dataclasses import replace + + k, cap = max(2, cfg.sessions_per_period), max(0, cfg.max_periods_per_identity) + planned: list[Period] = [] + tail: list[dict[str, Any]] = [] + skipped = 0 + if not cap: + return planned, skipped + async for page in _fetch_digests(ch, ident, page_size=max(256, k)): + combined = tail + page + complete = len(combined) // k * k + tail = combined[complete:] + periods, count = await _plan_identity( + ch, + combined[:complete], + ident, + replace(cfg, max_periods_per_identity=cap - len(planned)), + ) + planned.extend(periods) + skipped += count + if len(planned) == cap: + break + return planned, skipped + + +async def _skipped_period_keys( + ch: Any, ident: dict[str, Any], keys: list[str], *, rewrite: bool +) -> set[str]: """On ClickHouse, the identity's period keys a pass skips: every key with a version, or under a rewrite only those taken out (deleted, redacted or no longer active: an Undo of a retirement brings a redacted version back as the live head).""" @@ -375,7 +428,7 @@ async def _skipped_period_keys(ch: Any, ident: dict[str, Any], *, rewrite: bool) FROM loom.memory FINAL WHERE org = {org:String} AND namespace = {ns:String} AND agent = {agent:String} AND user_id = {uid:String} AND session_id = '-' - AND startsWith(memory_key, {pfx:String}) + AND startsWith(memory_key, {pfx:String}) AND memory_key IN {keys:Array(String)} """, params={ "org": ident["org"], @@ -383,6 +436,7 @@ async def _skipped_period_keys(ch: Any, ident: dict[str, Any], *, rewrite: bool) "agent": ident["agent"], "uid": ident["user_id"], "pfx": PERIOD_KEY_PREFIX, + "keys": keys, }, ) return {str(r["memory_key"]) for r in rows if not rewrite or r["taken_out"]} @@ -575,15 +629,31 @@ async def _plan_identity( """(the identity's periods to write, how many were skipped as written or taken out). Under Postgres the authority's heads decide which are written, ClickHouse's otherwise; the cap applies to what is left, so a period skipped takes no place in it.""" + from server.memory.store.summary_sources import current_summary_sources + k, rewrite = cfg.sessions_per_period, cfg.rewrite_existing + windows, _ = plan_periods(digests, sessions_per_period=k) + if not windows: + return [], 0 if pg.enabled(): - windows, _ = plan_periods(digests, sessions_per_period=k) writable = await _writable_in_authority(windows, rewrite=rewrite) skipped = len(windows) - len(writable) else: - skip = await _skipped_period_keys(ch, ident, rewrite=rewrite) + skip = await _skipped_period_keys( + ch, ident, [p.memory_key for p in windows], rewrite=rewrite + ) writable, skipped = plan_periods(digests, sessions_per_period=k, skip_keys=skip) - return writable[: max(0, cfg.max_periods_per_identity)], skipped + if not writable: + return [], skipped + qualified = await current_summary_sources( + ch, + [d for p in writable for d in p.digests], + org=ident["org"], + namespace=ident["namespace"], + ) + allowed = {id(d) for d in qualified} + verified = [p for p in writable if all(id(d) in allowed for d in p.digests)] + return verified[: max(0, cfg.max_periods_per_identity)], skipped async def run_period_digest( @@ -615,8 +685,7 @@ async def run_period_digest( result.identities_scanned = len(identities) for ident in identities: try: - digests = await _fetch_digests(ch, ident) - periods, skipped = await _plan_identity(ch, digests, ident, cfg) + periods, skipped = await _plan_digest_pages(ch, ident, cfg) result.periods_planned += len(periods) result.periods_skipped += skipped if dry_run: diff --git a/server/memory/app/pg_delivery.py b/server/memory/app/pg_delivery.py index 045afeee08..e5883049fa 100644 --- a/server/memory/app/pg_delivery.py +++ b/server/memory/app/pg_delivery.py @@ -133,7 +133,7 @@ async def _publish_audits(rows: list[dict[str, Any]], start_by: float) -> Except return None -async def _publish(claims: list[Claimed], *, stub: bool = False) -> None: +async def _publish(claims: list[Claimed], *, stub: bool = False) -> int: """One namespace's claims: memory rows and audit rows (`pg_publish.publish_rows`), then the search documents still owed, the old vectors' retirements, then the deleted decisions' records.""" @@ -169,14 +169,22 @@ async def _publish(claims: list[Claimed], *, stub: bool = False) -> None: [c for c in owed if pg_publish.document_id(c.row) not in held], deadline ) docs = await _search_documents(build, stub=stub) + admitted: list[dict[str, Any]] = [] if docs: with pg_lease.leased(deadline): - await memory_repo.insert_embedding_rows(docs, ch=ch) - after_write.index_late_rows(docs, ch=ch) + admitted = await memory_repo.insert_embedding_rows(docs, ch=ch) + after_write.index_late_rows(admitted, ch=ch) + received = {(row["org"], row["namespace"], *pg_publish.document_id(row)) for row in admitted} + withheld = { + _key(c) + for c, row in zip(build, docs, strict=True) + if (row["org"], row["namespace"], *pg_publish.document_id(row)) not in received + } + settled = [c for c in owed if _key(c) not in withheld] await pg_memory.complete( - [(c.org, c.namespace, c.seq) for c in owed], + [_key(c) for c in settled], pg_memory.SEARCH_DOC, - tokens=[c.lease_token for c in owed], + tokens=[c.lease_token for c in settled], ) old_vectors = await _retire_old_vectors( [c for c in claims if c.targets & pg_memory.OLD_VECTORS], deadline, ch=ch @@ -193,6 +201,9 @@ async def _publish(claims: list[Claimed], *, stub: bool = False) -> None: failed = refused or old_vectors or unretired if failed is not None: raise failed # an attempt that failed, once the rest is in + if withheld: + await pg_memory.release([c for c in owed if _key(c) in withheld]) + return len(claims) - len(withheld) async def _still_heads(claims: list[Claimed], deadline: float) -> list[Claimed]: @@ -333,8 +344,7 @@ async def _publish_namespace(group: list[Claimed], run: _Pass) -> None: batch = await pg_memory.renew([c for c in group if not c.taken_over and c.attempts < limit]) if batch: try: - await _publish_marked(batch, run) - run.landed += len(batch) + run.landed += await _publish_marked(batch, run) batch = [] except pg.PostgresUnavailableError: raise @@ -346,8 +356,7 @@ async def _publish_namespace(group: list[Claimed], run: _Pass) -> None: log.warning("pg_delivery: a batch of %s failed (%s); one at a time", _ns(batch[0]), why) async for c in _one_by_one([*alone, *batch], run): try: - await _publish_marked([c], run) - run.landed += 1 + run.landed += await _publish_marked([c], run) except pg.PostgresUnavailableError: raise except Exception as e: # one revision must not hold back the rest @@ -361,19 +370,20 @@ async def _publish_namespace(group: list[Claimed], run: _Pass) -> None: run.failed += await pg_memory.record_failure([c], e) -async def _publish_marked(claims: list[Claimed], run: _Pass, *, stub: bool = False) -> None: +async def _publish_marked(claims: list[Claimed], run: _Pass, *, stub: bool = False) -> int: """`_publish`, with the claims marked as running while it does: a worker's pass cancelled in it keeps them held. A publish that ends, either way, clears the mark.""" keys = {_key(c) for c in claims} if run.worker else set() run.running |= keys try: - await _publish(claims, stub=stub) + landed = await _publish(claims, stub=stub) except asyncio.CancelledError: raise except BaseException: run.running -= keys raise run.running -= keys + return landed async def _last_attempt(c: Claimed, error: Exception | None, run: _Pass) -> None: @@ -419,8 +429,7 @@ async def _publish_stub(c: Claimed, run: _Pass) -> bool: if held.targets & pg_memory.SEARCHED != pg_memory.SEARCH_DOC: await pg_memory.release(renewed) return False - await _publish_marked([held], run, stub=True) - return True + return bool(await _publish_marked([held], run, stub=True)) def _ns(c: Claimed) -> tuple[str, str]: diff --git a/server/memory/app/pipeline.py b/server/memory/app/pipeline.py index 0c768308bc..2cdcc84a33 100644 --- a/server/memory/app/pipeline.py +++ b/server/memory/app/pipeline.py @@ -401,26 +401,22 @@ async def _fetch_for_forget_requests( async def _retire_forgotten(state: WorkflowState, keys: set[str]) -> int: """Tombstone the live row of every key the user asked to forget, under this identity's org/namespace and any agent/session (the row may have - been written in an earlier session). Fails open per key; returns how many - were retired.""" + been written in an earlier session). Refusal policy failures stop extraction.""" from server.memory.app.handlers import handle_memory_delete n = 0 for key in sorted(keys): - try: - out = await handle_memory_delete( - MemoryDeleteInput( - org=state.org, - namespace=state.namespace, - agent="*", - user_id=state.user_id or "-", - session_id="-", - memory_key=key, - ) + out = await handle_memory_delete( + MemoryDeleteInput( + org=state.org, + namespace=state.namespace, + agent="*", + user_id=state.user_id or "-", + session_id="-", + memory_key=key, ) - n += 1 if getattr(out, "tombstoned", False) else 0 - except Exception as e: # noqa: BLE001 — a forget that fails is logged, never fatal - log.warning("forget request: retiring %r failed (open): %s", key, e) + ) + n += 1 if getattr(out, "tombstoned", False) else 0 return n diff --git a/server/memory/app/profile_search.py b/server/memory/app/profile_search.py index 1541552fab..89f3664da6 100644 --- a/server/memory/app/profile_search.py +++ b/server/memory/app/profile_search.py @@ -354,8 +354,8 @@ async def queue_profile_reembed( ) for r in rows ] - await memory_repo.insert_embedding_rows(stubs, ch=ch) - return len(stubs) + admitted = await memory_repo.insert_embedding_rows(stubs, ch=ch) + return len(admitted) except Exception as e: # noqa: BLE001 — the read-side check heals it later log.warning("profile re-embed stub write failed: %s", e) return 0 diff --git a/server/memory/app/promote.py b/server/memory/app/promote.py index 70f59e20a2..798a4daa4b 100644 --- a/server/memory/app/promote.py +++ b/server/memory/app/promote.py @@ -6,15 +6,17 @@ import logging import uuid from collections.abc import AsyncIterator, Callable -from contextlib import asynccontextmanager +from contextlib import AsyncExitStack, asynccontextmanager from datetime import datetime, timedelta from typing import Any from fastapi import HTTPException from server.core import leases +from server.core.ch import get_client from server.core.datetimes import naive_utc_now, to_naive_utc from server.core.errors import ( + AuthInsufficientError, DefinitionCeilingError, DefinitionNameTakenError, DefinitionPromotionRefusedError, @@ -23,7 +25,7 @@ SensitiveDefinitionRefusedError, ValidationError, ) -from server.core.hashing import served_content_sha256 +from server.core.hashing import served_content_sha256, sha256_value from server.embedding import get_embedder from server.embedding.documents import document_vector from server.embedding.embedder import Embedder @@ -49,8 +51,8 @@ from server.memory.domain.promotion_source import promotion_source from server.memory.domain.search_types import SHARED_AGENT from server.memory.domain.trust import verification_state +from server.memory.store import refusals from server.memory.store import shared as shared_repo -from server.memory.store import source_review from server.memory.store.rows import MemoryRow, next_version, version_over from server.observability.budget_shape import note_budget_shape from server.settings import settings @@ -187,7 +189,7 @@ async def handle_memory_promote( if embedder is None else _shared_vector_row(src, target_key, excerpt, embedder, embedding, now_naive) ) - async with source_review.publication(src) as mark: + async with _promotion_admission(src, target_key, definition=is_definition_promotion) as mark: if is_definition_promotion: await _write_definition(input.org, input.namespace, names, row, vector_row, mark) else: @@ -201,6 +203,20 @@ async def handle_memory_promote( ) +@asynccontextmanager +async def _promotion_admission( + source: MemoryRow, target_key: str, *, definition: bool +) -> AsyncIterator[Callable[[bool], None]]: + # Demotion holds this claim while borrowing its source connection too. + async with AsyncExitStack() as stack: + if definition: + await stack.enter_async_context(_definition_claim(source.org, source.namespace)) + mark = await stack.enter_async_context( + refusals.source_publication(get_client(), source, target_key) + ) + yield mark + + async def _write_shared( row: dict[str, Any], vector: dict[str, Any] | None, mark: Callable[[bool], None] ) -> None: @@ -265,7 +281,10 @@ async def _approved_definition_names( def _refuse_unshareable(src: MemoryRow) -> None: """Refuse a row the shared layer cannot carry: a block row or a 'high' one.""" - source_review.require_reviewed(src) + from server.memory.domain.proposal_review import pending_review + + if pending_review(src): + raise AuthInsufficientError("An unreviewed proposal cannot be promoted") # Block rows are per-agent working memory; loom.shared has no char_limit, # so promoting one would silently drop its overflow cap. Refuse. if src.memory_type == "profile": @@ -320,6 +339,7 @@ def _shared_vector_row( "memory_type": src.memory_type, "scope": "shared", "content_excerpt": excerpt, + "content_sha256": sha256_value(src.value), "embedding_model": embedder.model_name, "embedding_dim": embedder.dim if embedding else 0, "embedding": embedding, # [] when embed failed @@ -332,7 +352,7 @@ def _shared_vector_row( # counts only while it binds the promoted content's hash. "source_class": src.source_class, "verified": int( - verification_state(src.verified_by, src.verified_sha256, src.content_sha256) + verification_state(src.verified_by, src.verified_sha256, sha256_value(src.value)) in ("human", "process") ), } @@ -349,18 +369,17 @@ async def _write_definition( """Write the definition's shared row and its vector unless the namespace is at its ceiling or another shared definition holds one of its names, under the namespace's claim: of a definition's promotions and demotes, the last to run decides.""" - async with _definition_claim(org, namespace): - key = row["memory_key"] - await _check_shared_definitions(org, namespace, names, key) - # Above the row and the vector: a demote cut off between the two left a retired - # vector above the live row. - head = await shared_repo.fetch_shared(org, namespace, key, strong=True) - version = await _definition_version(org, namespace, key, head) - mark(True) - await shared_repo.insert_shared_row({**row, "version": version}) - if vector_row is not None: - await shared_repo.insert_shared_embedding({**vector_row, "version": version}) - mark(False) + key = row["memory_key"] + await _check_shared_definitions(org, namespace, names, key) + # Above the row and the vector: a demote cut off between the two left a retired + # vector above the live row. + head = await shared_repo.fetch_shared(org, namespace, key, strong=True) + version = await _definition_version(org, namespace, key, head) + mark(True) + await shared_repo.insert_shared_row({**row, "version": version}) + if vector_row is not None: + await shared_repo.insert_shared_embedding({**vector_row, "version": version}) + mark(False) async def _definition_version( diff --git a/server/memory/app/reembed.py b/server/memory/app/reembed.py index 31484469fa..15d0451731 100644 --- a/server/memory/app/reembed.py +++ b/server/memory/app/reembed.py @@ -22,7 +22,7 @@ from collections.abc import Awaitable, Callable from typing import Any -from server.core.ch import sequential_reads +from server.core.ch import sequential_reads, set_query_tag from server.core.ch_params import chunks from server.llm import customer from server.memory.app.model_attribution import attribution_for, memory_writers @@ -148,6 +148,16 @@ def in_database_sql( the column type, model/dim columns are rewritten, the version column is bumped the way `rewrite_row` bumps it, every other column is copied. Pure so it can be tested without a server.""" + insert_cols, select = _database_projection(table, cols, model, dim, credentials) + return ( + f"INSERT INTO loom.{table} ({', '.join(insert_cols)})\n" + f"SELECT {', '.join(select)}\nFROM loom.{table} FINAL\nWHERE {where_sql}" + ) + + +def _database_projection( + table: str, cols: list[str], model: str, dim: int, credentials: str +) -> tuple[list[str], list[str]]: cfg = _TABLES[table] m = model.replace("'", "\\'") cred = credentials.replace("'", "\\'") @@ -185,12 +195,72 @@ def emb(text_col: str) -> str: select.append("version + 1") # as rewrite_row: never above a later write else: select.append(c) + return insert_cols, select + + +def _database_memory_page_sql( + cols: list[str], model: str, dim: int, credentials: str, where: str +) -> str: + names, expressions = _database_projection("memory_embeddings", cols, model, dim, credentials) + projected = ", ".join( + f"{expression} AS {name}" for name, expression in zip(names, expressions, strict=True) + ) + order = ", ".join(_KEYS["memory_embeddings"]) + # Filtering uses the original model; the outer aliases cannot shadow its predicate. return ( - f"INSERT INTO loom.{table} ({', '.join(insert_cols)})\n" - f"SELECT {', '.join(select)}\nFROM loom.{table} FINAL\nWHERE {where_sql}" + f"SELECT {projected}, _k FROM (SELECT *, tuple({order}) AS _k " + f"FROM loom.memory_embeddings FINAL WHERE {where} " + f"ORDER BY {order} LIMIT {{page:UInt32}}) ORDER BY _k" ) +def _memory_vector_row(row: dict[str, Any]) -> dict[str, Any]: + from server.memory.store.rows import EMBEDDING_PERSISTED_COLS, build_embedding_row + + known = {key: value for key, value in row.items() if key in EMBEDDING_PERSISTED_COLS} + # Carry columns added by a newer build alongside the complete known row. + return {**row, **build_embedding_row(**known)} + + +async def _publish_database_memory( + ch: Any, + cols: list[str], + model: str, + dim: int, + credentials: str, + where: str, + params: dict[str, Any], + left_orgs: set[str] | None, +) -> int: + from server.memory.store import insert_embedding_rows + + after = None + written = 0 + order = ", ".join(_KEYS["memory_embeddings"]) + while True: + set_query_tag("memory.reembed") + page_where = where + page_params = {**params, "page": 200} + if after is not None: + marks = ", ".join(f"{{k{i}:String}}" for i in range(len(after))) + page_where += f" AND ({order}) > ({marks})" + page_params.update({f"k{i}": str(value) for i, value in enumerate(after)}) + rows = await ch.query_all( + _database_memory_page_sql(cols, model, dim, credentials, page_where), + params=page_params, + settings=in_database_settings(200), + ) + if not rows: + return written + out = [_memory_vector_row(_without_key(row)) for row in rows] + admitted = await insert_embedding_rows(out, ch=ch) + _held_publications(out, admitted, left_orgs) + written += len(admitted) + after = rows[-1]["_k"] + if len(rows) < 200: + return written + + def in_database_settings(rows: int) -> dict[str, Any]: """Session settings for one in-database re-embed statement: the experimental gate, batches of 100 texts per request, and per-query @@ -220,11 +290,12 @@ async def reembed_table_in_database( namespace: str | None = None, dry_run: bool = False, exclude_orgs: frozenset[str] = frozenset(), + left_orgs: set[str] | None = None, log: Callable[..., None] = print, ) -> int: - """Re-embed one table's off-model rows inside ClickHouse. - One INSERT … SELECT per table; ClickHouse batches the embedding calls. - Returns the off-model row count the statement covered.""" + """Re-embed off-model rows with aiEmbed. Memory vectors publish through admission + in bounded pages; other tables use one INSERT … SELECT. Return acknowledged memory + vectors, or the off-model row count covered for the other tables.""" cfg = _TABLES[table] cols = await _cols(ch, table) where = [f"embedding_model != '{model}'", "embedding_dim > 0"] @@ -252,6 +323,10 @@ async def reembed_table_in_database( ) if dry_run or total == 0: return 0 + if table == "memory_embeddings": + return await _publish_database_memory( + ch, cols, model, dim, credentials, where_sql, params, left_orgs + ) _ = cfg sql = in_database_sql( table, cols, model=model, dim=dim, credentials=credentials, where_sql=where_sql @@ -365,10 +440,7 @@ async def reread_page(read: dict[int, dict[str, Any]]) -> dict[int, dict[str, An if left: if left_orgs is not None: left_orgs.update(left) - log( - f" {table}: {sum(left.values())} rows of an org switched off during the run " - "left for later" - ) + log(f" {table}: {sum(left.values())} unpublished rows left for later") after = [str(x) for x in rows[-1]["_k"]] progress = f"{written}/{total}" + (f" ({held} held as high)" if held else "") progress += f" ({changed} removed or rewritten since read, skipped)" if changed else "" @@ -491,12 +563,36 @@ async def _rewrite_page( for i, r in kept.items() ] if out: - await ch.insert( - f"loom.{table}", out, settings={"async_insert": 0, "wait_for_async_insert": 1} - ) + if table == "memory_embeddings": + from server.memory.store import insert_embedding_rows + + out = [_memory_vector_row(row) for row in out] + admitted = await insert_embedding_rows(out, ch=ch) + for org, count in _held_publications(out, admitted).items(): + left_by_org[org] = left_by_org.get(org, 0) + count + else: + await ch.insert( + f"loom.{table}", out, settings={"async_insert": 0, "wait_for_async_insert": 1} + ) return left_by_org +def _held_publications( + rows: list[dict[str, Any]], + admitted: list[dict[str, Any]], + left_orgs: set[str] | None = None, +) -> dict[str, int]: + keys = _KEY_COLUMNS["memory_embeddings"] + accepted = {tuple(row.get(key) for key in keys) for row in admitted} + held: dict[str, int] = {} + for row in rows: + if tuple(row.get(key) for key in keys) not in accepted: + held[row["org"]] = held.get(row["org"], 0) + 1 + if left_orgs is not None: + left_orgs.update(held) + return held + + # Each table's sort key, the keyset a run pages through and resumes after, and the # same as strings (the Enum `scope` by name). _KEY_COLUMNS: dict[str, tuple[str, ...]] = { @@ -649,6 +745,7 @@ async def reembed_all( namespace=namespace, dry_run=dry_run, exclude_orgs=off, + left_orgs=left_orgs, log=log, ) else: diff --git a/server/memory/app/restore.py b/server/memory/app/restore.py index 30d452a7ab..cf582f3aeb 100644 --- a/server/memory/app/restore.py +++ b/server/memory/app/restore.py @@ -17,6 +17,7 @@ from server.core.datetimes import naive_utc_now from server.core.error_summary import error_summary from server.core.errors import BackendUnavailableError, LoomServerError +from server.core.hashing import served_content_sha256 from server.identity.domain.read_scope import WHOLE_NAMESPACE from server.memory.api.models_crud import ( MemoryRestoreInput, @@ -619,7 +620,7 @@ async def _restore_embedding_mirror( entities=list(existing.entities or []), # Carried forward so restore doesn't zero them under FINAL. files=list(getattr(existing, "files", None) or []), - content_sha256=str(getattr(existing, "content_sha256", "") or ""), + content_sha256=served_content_sha256(existing.value), concepts=list(getattr(existing, "concepts", None) or []), status=str(existing.status or "active"), temporal_anchor=existing.temporal_anchor, @@ -637,20 +638,19 @@ async def _restore_embedding_mirror( mentioned_at=existing.mentioned_at, source_resource_id=existing.source_resource_id or "", ) - await ch.insert( - "loom.memory_embeddings", - [restore_emb_row], - settings={"async_insert": 0, "wait_for_async_insert": 1}, - ) + from server.memory.store import insert_embedding_rows + + await insert_embedding_rows([restore_emb_row], ch=ch) async def handle_memory_restore(input: MemoryRestoreInput) -> MemoryRestoreOutput: - """Undo a delete, or an automatic retirement. + """Reverse an automatic tombstone or supersession when consent permits it. + + Within the 14-day grace window, a restorable tombstone is copied forward with + a higher version and tombstoned_at=NULL on memory and its search mirror. + An older tombstone answers expired=True. Explicit deletion or clearing leaves + permanent refusal, so its content cannot be republished. - A delete within the 14-day grace window: copy the tombstoned row forward with a - higher version and tombstoned_at=NULL so RMT collapses the pair live, on both - loom.memory and loom.memory_embeddings. A tombstone past the 14-day TTL returns - expired=True. A retired (`superseded`) row is served again by `_restore_superseded`, whose two preconditions hold it to the retirement on screen: `if_superseded_by` (still diff --git a/server/memory/app/retrieval/summaries.py b/server/memory/app/retrieval/summaries.py index 27d901621c..5abdf216db 100644 --- a/server/memory/app/retrieval/summaries.py +++ b/server/memory/app/retrieval/summaries.py @@ -167,7 +167,7 @@ async def fetch_session_summaries( -- supertype for String, DateTime" — silently swallowed by the -- except below, killing the whole leg under any as_of. toString(coalesce(valid_at, toDateTime(0))) AS valid_at_iso, - derived_from + derived_from, key_facts FROM loom.session_summaries FINAL WHERE {where_sql} -- No `LIMIT 1 BY` and no `session_id` tiebreak here: diff --git a/server/memory/app/runs_manifest.py b/server/memory/app/runs_manifest.py index 2c481ebe38..3e6afb1fc7 100644 --- a/server/memory/app/runs_manifest.py +++ b/server/memory/app/runs_manifest.py @@ -204,8 +204,8 @@ async def fetch_manifest_rows( if sids: digests = await ch.query_all( f""" - SELECT session_id, substring(summary_text, 1, {_DIGEST_CHARS * 2}) AS summary_text, - length(derived_from) AS derived_n + SELECT org, namespace, agent, user_id, session_id, summary_text, key_facts, + derived_from, version, length(derived_from) AS derived_n FROM loom.session_summaries FINAL WHERE {scope} AND session_id IN {{sids:Array(String)}} @@ -214,6 +214,9 @@ async def fetch_manifest_rows( """, params={**params, "sids": sids}, ) + from server.memory.store.summary_sources import current_summary_sources + + digests = await current_summary_sources(ch, digests, org=org, namespace=namespace) return merge_rows(list(observations or []), list(runs or []), list(digests or []), limit=limit) diff --git a/server/memory/app/sensitivity_sweep.py b/server/memory/app/sensitivity_sweep.py index 3eda4a04da..f4b864e627 100644 --- a/server/memory/app/sensitivity_sweep.py +++ b/server/memory/app/sensitivity_sweep.py @@ -550,9 +550,10 @@ async def _tag_embeddings(c: SweepCandidate, *, ch: _CHClient) -> int: ) ) if out: - await insert_embedding_rows(out, ch=ch) + admitted = await insert_embedding_rows(out, ch=ch) await retire_copies_under_unembedded_heads(out, ch=ch) - return len(out) + return len(admitted) + return 0 async def _close_mined_facts( diff --git a/server/memory/app/session_outcomes.py b/server/memory/app/session_outcomes.py index 5068ebc361..805ad6079c 100644 --- a/server/memory/app/session_outcomes.py +++ b/server/memory/app/session_outcomes.py @@ -239,7 +239,7 @@ async def record_outcome(ch: Any, **fields: Any) -> None: _UNLABELLED_SQL = """ SELECT s.org AS org, s.namespace AS namespace, s.agent AS agent, s.user_id AS user_id, s.session_id AS session_id, - s.summary_text AS summary_text, s.key_facts AS key_facts + s.summary_text AS summary_text, s.key_facts AS key_facts, s.derived_from AS derived_from FROM loom.session_summaries AS s FINAL LEFT ANTI JOIN ( SELECT org, namespace, agent, user_id, session_id FROM loom.session_outcomes_by_user FINAL @@ -276,6 +276,9 @@ async def label_sessions(ch: Any, llm: Any, cfg: DistillConfig, result: DistillR "off": sorted(await customer.switched_off_orgs()), }, ) + from server.memory.store.summary_sources import verified_summary_batch + + rows = await verified_summary_batch(ch, rows) result.sessions_scanned = len(rows) # The judge's call is for the session's writer, else its namespace's agent. writers = await _writers(ch, rows) @@ -331,11 +334,12 @@ async def label_sessions(ch: Any, llm: Any, cfg: DistillConfig, result: DistillR SELECT o.org AS org, o.namespace AS namespace, o.agent AS agent, o.session_id AS session_id, o.reason AS reason, o.confidence AS confidence, s.summary_text AS summary_text, s.summary_embedding AS summary_embedding, - s.embedding_model AS embedding_model, s.user_id AS user_id + s.embedding_model AS embedding_model, s.user_id AS user_id, + s.key_facts AS key_facts, s.derived_from AS derived_from FROM loom.session_outcomes_by_user AS o FINAL INNER JOIN ( SELECT org, namespace, agent, session_id, summary_text, summary_embedding, - embedding_model, user_id + embedding_model, user_id, key_facts, derived_from FROM loom.session_summaries FINAL WHERE tombstoned_at IS NULL AND embedding_dim > 0 ) AS s ON s.org = o.org AND s.namespace = o.namespace AND s.agent = o.agent @@ -356,7 +360,9 @@ async def label_sessions(ch: Any, llm: Any, cfg: DistillConfig, result: DistillR """ _NEAREST_POSITIVE_SQL = """ - SELECT s.session_id AS session_id, s.summary_text AS summary_text, + SELECT s.org AS org, s.namespace AS namespace, s.agent AS agent, s.user_id AS user_id, + s.key_facts AS key_facts, s.derived_from AS derived_from, + s.session_id AS session_id, s.summary_text AS summary_text, 1 - cosineDistance(s.summary_embedding, {q:Array(Float32)}) AS similarity FROM loom.session_summaries AS s FINAL INNER JOIN ( @@ -400,27 +406,32 @@ async def write_contrast_reflection( "contradicts": [neg_session], "pair_similarity": round(float(pair_similarity), 4), } - await handle_memory_set( - MemorySetInput( - org=org, - namespace=namespace, - agent=agent or "-", - user_id=user_id or "-", - session_id=neg_session, - memory_key=key, - memory_type="reflection", - value=value, - tags=[ - "reflection", - "contrast", - f"session:{neg_session}", - f"session:{pos_session}", - "status:failed", - ], - confidence=float(max(0.0, min(1.0, confidence))), - source_trace=f"contrast:{neg_session}<{pos_session}", + from server.memory.domain.provenance import pipeline_actor + + with pipeline_actor("process:session_contrast"): + await handle_memory_set( + MemorySetInput( + org=org, + namespace=namespace, + agent=agent or "-", + user_id=user_id or "-", + session_id=neg_session, + memory_key=key, + memory_type="reflection", + proposed=True, + source_class="derived", + value=value, + tags=[ + "reflection", + "contrast", + f"session:{neg_session}", + f"session:{pos_session}", + "status:failed", + ], + confidence=float(max(0.0, min(1.0, confidence))), + source_trace=f"contrast:{neg_session}<{pos_session}", + ) ) - ) return key @@ -439,6 +450,9 @@ async def distil_contrasts( "off": sorted(await customer.switched_off_orgs()), }, ) + from server.memory.store.summary_sources import verified_summary_batch + + negatives = await verified_summary_batch(ch, negatives) result.contrast_candidates = len(negatives) writers = await _writers(ch, negatives) for neg in negatives: @@ -483,12 +497,16 @@ async def _contrast_pair(ch: Any, llm: Any, cfg: DistillConfig, neg: Any, writer "uid": str(neg.get("user_id") or "-"), }, ) + from server.memory.store.summary_sources import verified_summary_batch + + qualified = await verified_summary_batch(ch, [pos] if pos else []) + pos = qualified[0] if qualified else None sim = float((pos or {}).get("similarity") or 0.0) if not pos or sim < float(cfg.min_pair_similarity): return False system, user = contrast_prompt( str(neg.get("summary_text") or ""), - str(neg.get("reason") or ""), + "failed", str(pos.get("summary_text") or ""), ) with llm_stage("session_contrast"): @@ -509,7 +527,7 @@ async def _contrast_pair(ch: Any, llm: Any, cfg: DistillConfig, neg: Any, writer neg_session=sid, pos_session=str(pos.get("session_id") or ""), lessons=lessons, - what_failed=what_failed or str(neg.get("reason") or ""), + what_failed=what_failed, confidence=min(float(neg.get("confidence") or 0.0), sim), pair_similarity=sim, ) diff --git a/server/memory/app/session_summary.py b/server/memory/app/session_summary.py index 7548245c94..7c7ed72b02 100644 --- a/server/memory/app/session_summary.py +++ b/server/memory/app/session_summary.py @@ -1,10 +1,7 @@ -"""Session-summary plane (write side) for memory.set_from_messages.""" +"""Session digests built only from authoritative, committed facts.""" from __future__ import annotations -import json -import logging -from dataclasses import dataclass from typing import Any from server.core.ch import get_client @@ -12,266 +9,94 @@ from server.embedding import get_embedder from server.embedding._provider import MalformedEmbedding from server.embedding.documents import document_vector -from server.memory.domain.extraction_policy import with_policy -from server.memory.domain.sensitivity import reads_high, value_reads_high +from server.identity.domain.read_scope import WHOLE_NAMESPACE +from server.memory.domain.sensitivity import value_reads_high +from server.memory.domain.summary_provenance import render_summary, source_text from server.observability.genai import staged -from server.settings.budgets import llm_max_tokens +from server.settings import settings -log = logging.getLogger(__name__) - -# After extraction, compress a session into one dense dated digest, embed it -# with the fact-plane embedder, and store it in loom.session_summaries. The -# read leg retrieves these by vector as source="summary" hits — a compact, -# high-recall stand-in for a whole session. - -# Bumped when the prompt text changes, so an operator can tell which prompt -# produced a given digest. -_SESSION_SUMMARY_PROMPT_VERSION = "ss-3" - -# Bounds the digest call's input cost. ~3000 chars ≈ 750 tokens. -_SESSION_SUMMARY_MAX_INPUT_CHARS = 3000 - - -# How many fact values ride along as key_facts — a dedup signal for the -# assembler (which facts a summary subsumes); never independently embedded. _SESSION_SUMMARY_MAX_KEY_FACTS = 6 -# Calibrated to emit dated, fact-only sentences. -_SESSION_SUMMARY_SYSTEM = ( - "You are compressing one conversation session into a dense, factual digest " - "for later retrieval. Write 1-3 sentences stating only the concrete facts, " - "decisions, events, and stated values from THIS session. Prefer dated " - "phrasing ('on 2024-03-11, the user ...'). No greetings, no meta-commentary, " - "no 'the user discussed'. If a date is given, attribute facts to it. Date " - "each EVENT at the time the speaker placed it, resolved against the session " - "date: 'dyed my hair last week' in a session dated 2022-04-15 is 'around " - "2022-04-08', never 'on 2022-04-15'; 'yesterday' is the day before the " - "session date. Only what happened during the session itself is dated at " - "the session date. State every fact as the speakers gave it: the digest is " - "their record, so never judge a statement true or false, never call one " - "incorrect or a misstatement, and never replace a stated value with the " - "real-world one. Output the digest only." -) - -def _digest_max_input_chars() -> int: - from server.settings import settings +@staged("session_summary") +async def _write_session_summary(input: Any, state: Any) -> None: + """Persist an extractive digest with enough lineage to reconstruct every byte.""" + import server.memory.store as memory_repo + from server.memory.app.digest_chain import link_digest - try: - return max( - 500, - int(settings.memory_session_digest_max_input_chars or _SESSION_SUMMARY_MAX_INPUT_CHARS), + ch = get_client() + identity = tuple( + str(getattr(input, axis, "-") or "-") for axis in ("agent", "user_id", "session_id") + ) + keys: list[str] = [] + facts: list[str] = [] + cap = int(settings.memory_session_digest_max_input_chars) + if cap == 0: + cap = 3000 # Zero selects the documented built-in character budget. + cap = max(500, cap) + for written in state.written or []: + memory_id = str(written.get("memory_id") or "") + version = int(written.get("version") or 0) + if not memory_id or not version: + continue + source = await memory_repo.get_by_id( + memory_id, + org=input.org, + namespace=input.namespace, + scope=WHOLE_NAMESPACE, + ch=ch, + strong=True, + fresh=True, + include_usage_totals=False, ) - except (TypeError, ValueError): - return _SESSION_SUMMARY_MAX_INPUT_CHARS - - -def _top_key_facts( - extracted_facts: list[dict[str, Any]], - *, - limit: int = _SESSION_SUMMARY_MAX_KEY_FACTS, -) -> list[str]: - """Top-N non-empty fact values as strings, order-preserving + deduped. A fact whose text - reads as `high` is left out: the digest is listed by memory.digests and sent to - the outcome judge, so its key facts never carry such a value.""" - out: list[str] = [] - seen: set[str] = set() - for fact in extracted_facts or []: - if not isinstance(fact, dict): + if source is None or int(source.version) != version: continue - raw = fact.get("value") - if isinstance(raw, str): - text = raw.strip() - elif raw is None: - text = "" - else: - text = json.dumps(raw, default=str) - text = text.strip() - if not text or text in seen: + source_identity = (source.agent or "-", source.user_id or "-", source.session_id or "-") + if source_identity != identity or source.memory_key in keys: continue - if value_reads_high(raw): + text = source_text(source, now=naive_utc_now()) + if not text or len(render_summary([*facts, text])) > cap: continue - seen.add(text) - out.append(text) - if len(out) >= limit: + keys.append(source.memory_key) + facts.append(text) + if len(facts) >= _SESSION_SUMMARY_MAX_KEY_FACTS: break - return out - - -def _build_session_summary_user_prompt( - msgs: list[dict[str, str]], - observation_date: str | None, -) -> str: - """Role-prefixed, length-bounded transcript with a session-date header.""" - transcript = "\n".join( - f"[{m.get('role', 'user')}] {m.get('content', '')}" for m in (msgs or []) - ) - cap = _digest_max_input_chars() - if len(transcript) > cap: - transcript = transcript[:cap] - return ( - f"Session date: {observation_date or 'unknown'}\n\nConversation:\n{transcript}\n\nDigest:" - ) - - -@dataclass(frozen=True) -class SessionSummaryDraft: - """The model's digest and its vector, drafted before the row exists. - - The digest depends on the transcript only, so it is drafted while the extraction - pipeline runs and joined to the written keys after. - """ - - summary_text: str - embedding: list[float] - embedding_model: str - tokens_in: int = 0 - tokens_out: int = 0 - - -@staged("session_summary") -async def _draft_session_summary( - input: Any, - msgs: list[dict[str, str]], - observation_date: str | None, - *, - operator_policy: str, -) -> SessionSummaryDraft | None: - """One LLM call + one embedding: everything about the digest that does not - need the extraction result. None when the model returns nothing or the - embedder returns an invalid vector (the read leg needs a valid embedding); - a digest that reads as high is drafted with no vector. The digest is - written from the raw turns, so it follows the operator's extraction policy as - the extractor does.""" - from server.llm.client import get_llm_client - - if not msgs: - return None - user_prompt = _build_session_summary_user_prompt(msgs, observation_date or None) - llm = get_llm_client() - result = await llm.complete( - with_policy(_SESSION_SUMMARY_SYSTEM, operator_policy), - user_prompt, - max_tokens=llm_max_tokens("session_summary"), - ) - summary_text = (result.text or "").strip() - if not summary_text: - log.info("session_summaries: empty digest for session=%r; skipping", input.session_id) - return None + if not facts: + return + summary = render_summary(facts) + if value_reads_high(summary): + return embedder = get_embedder() - # A digest that reads as high is written without a vector: the summary leg - # reads only digests that have one, and the text never reaches the embedder. - embedding: list[float] = [] - if not reads_high(summary_text): - try: - embedding = await document_vector(embedder, summary_text) - except MalformedEmbedding: - log.info( - "session_summaries: invalid embedding for session=%r; skipping", input.session_id - ) - return None - return SessionSummaryDraft( - summary_text=summary_text, - embedding=embedding, - embedding_model=embedder.model_name, - tokens_in=int(getattr(result, "tokens_in", 0) or 0), - tokens_out=int(getattr(result, "tokens_out", 0) or 0), - ) - - -async def _insert_session_summary( - input: Any, - state: Any, # WorkflowState - draft: SessionSummaryDraft, -) -> None: - """Join a draft to the extraction result and insert the row.""" - from server.memory.store import build_session_summary_row - - extracted_facts = list(getattr(state, "extracted_facts", None) or []) - observation_date = getattr(state, "observation_date", None) or None - ch = get_client() - row = build_session_summary_row( + try: + embedding = await document_vector(embedder, summary) + except MalformedEmbedding: + return + row = memory_repo.build_session_summary_row( org=input.org, namespace=input.namespace, - agent=getattr(input, "agent", "-") or "-", - user_id=getattr(input, "user_id", "-") or "-", - session_id=getattr(input, "session_id", "") or "", - # The digest belongs to the run that wrote it. + agent=identity[0], + user_id=identity[1], + session_id=identity[2], run_id=str(getattr(input, "run_id", "") or "-"), - summary_text=draft.summary_text, - summary_embedding=draft.embedding, - embedding_model=draft.embedding_model, - embedding_dim=len(draft.embedding), - key_facts=_top_key_facts(extracted_facts), - # Lineage: the memory keys this digest compressed. - derived_from=sorted( - { - str(w.get("memory_key") or "") - for w in (getattr(state, "written", None) or []) - if w.get("memory_key") - } - ), - # valid_at = the observation date, surfaced as a per-line LABEL (never a - # reorder key); parse_iso_dt fail-opens to NULL, matching the column DEFAULT. - valid_at=parse_iso_dt(observation_date), + summary_text=summary, + summary_embedding=embedding, + embedding_model=embedder.model_name, + embedding_dim=len(embedding), + key_facts=facts, + derived_from=keys, + valid_at=parse_iso_dt(getattr(state, "observation_date", None)), created_at=naive_utc_now(), ) - await ch.insert( - "loom.session_summaries", - [row], - settings={"async_insert": 0, "wait_for_async_insert": 1}, - ) - log.info( - "session_summaries: wrote digest for session=%r (%d chars, prompt=%s)", - input.session_id, - len(draft.summary_text), - _SESSION_SUMMARY_PROMPT_VERSION, - ) - # The digest chain: this digest is linked to the previous and next digest of the same - # identity by event time, as `precedes` edges in loom.operator_edges. Model-free, two - # indexed reads and one insert; fail-open, after the digest is in place. - from server.memory.app.digest_chain import link_digest + from server.memory.store.summary_sources import insert_verified_summaries + if not await insert_verified_summaries(ch, [row]): + return await link_digest( ch, org=input.org, namespace=input.namespace, - agent=str(row.get("agent") or "-"), - user_id=str(row.get("user_id") or "-"), - session_id=str(row.get("session_id") or ""), + agent=identity[0], + user_id=identity[1], + session_id=identity[2], at=row.get("valid_at") or row["created_at"], ) - - -@staged("session_summary") -async def _write_session_summary( - input: Any, - msgs: list[dict[str, str]], - state: Any, # WorkflowState - *, - draft: SessionSummaryDraft | None = None, -) -> None: - """Generate + embed + insert one session-summary row. Fail-open. - - Skips a session with no extracted facts. One LLM call produces the digest; - the fact-plane embedder embeds it so the read leg's cosine is comparable; - inserted synchronously so the immediate post-ingest read sees it. A - `draft` made ahead of time skips the model and embedder calls; a session that extracted - nothing discards it. - """ - extracted_facts = list(getattr(state, "extracted_facts", None) or []) - if not extracted_facts: - return - if not msgs: - return - if draft is None: - observation_date = getattr(state, "observation_date", None) or None - draft = await _draft_session_summary( - input, - msgs, - observation_date, - operator_policy=state.operator_policy, - ) - if draft is None: - return - await _insert_session_summary(input, state, draft) diff --git a/server/memory/app/set_chunks.py b/server/memory/app/set_chunks.py index ec299a2f4d..089a4a179f 100644 --- a/server/memory/app/set_chunks.py +++ b/server/memory/app/set_chunks.py @@ -30,7 +30,6 @@ from datetime import UTC, datetime from typing import Any -from server.core.ch_writes import committed_insert_settings from server.core.datetimes import naive_utc_now from server.embedding.documents import document_vector, document_vectors from server.memory.app.chunk_projection import chunk_text_with_ranges, chunk_text_with_truncation @@ -218,11 +217,9 @@ def _chunk_state_row( async def _insert_chunk_states(ch: Any, rows: list[dict[str, Any]]) -> None: # Synchronous: the caller's next search must not lift the parent # through a chunk of the value it just replaced or deleted. - await ch.insert( - "loom.memory_chunks", - rows, - settings=committed_insert_settings(), - ) + from server.memory.store.refusals import insert_content_rows + + await insert_content_rows(ch, "loom.memory_chunks", rows) async def tombstone_set_chunks_fail_open( @@ -497,11 +494,9 @@ async def write_set_chunks( # Synchronous for the reason the fact insert is: the chunk plane is read # on the default search path, and a query landing before the chunks embed # must not find the head of the value and miss its body. - await ch.insert( - "loom.memory_chunks", - rows, - settings=committed_insert_settings(), - ) + from server.memory.store.refusals import insert_content_rows + + await insert_content_rows(ch, "loom.memory_chunks", rows) # A newer write may have made the key 'high' (or unsearchable) or deleted it while # these chunks were embedding. It inserts its row before tombstoning the key's # chunks, so either its tombstone saw these or this strong read sees its row. A diff --git a/server/memory/domain/promotion_source.py b/server/memory/domain/promotion_source.py index 07a8576533..3c7f93a921 100644 --- a/server/memory/domain/promotion_source.py +++ b/server/memory/domain/promotion_source.py @@ -8,6 +8,8 @@ from __future__ import annotations +PromotionKey = tuple[str, str, str, str, str, str] + _FIELDS = ("org", "namespace", "agent", "user_id", "session_id", "memory_key") @@ -25,3 +27,75 @@ def parse_promotion_source(entry: str) -> dict[str, str] | None: if len(parts) != len(_FIELDS) or not parts[-1]: return None return dict(zip(_FIELDS, parts, strict=True)) + + +# not a setting: fail-closed work ceiling for ambiguous legacy source identities. +_CONSENT_WORK_LIMIT = 250_000 + + +def promotion_users(entries: list[str], org: str, namespace: str) -> set[str] | None: + """Every possible source user, or no proof when complete denial checking is too large. + + The known organization and namespace fix the prefix. Slashes in any remaining + field leave several possible users; consent checks must cover all of them. + """ + prefix = f"{org}/{namespace}/" + users: set[str] = set() + work = 0 + for entry in entries: + if not isinstance(entry, str) or not entry.startswith(prefix): + return None + work += len(entry) + if work > _CONSENT_WORK_LIMIT: + return None + parts = entry[len(prefix) :].split("/") + if len(parts) < 4: + return None + found = False + for start in range(1, len(parts) - 2): + for stop in range(start + 1, len(parts) - 1): + user = "/".join(parts[start:stop]) + key = "/".join(parts[stop + 1 :]) + work += len(user) + len(key) + stop - start + if work > _CONSENT_WORK_LIMIT: + return None + if key: + users.add(user) + found = True + if not found: + return None + return users or None + + +def promotion_keys(entries: list[str], org: str, namespace: str) -> list[PromotionKey] | None: + """All exact identities a legacy entry may name, with complete bounded verification.""" + from itertools import combinations + + prefix = f"{org}/{namespace}/" + keys: set[PromotionKey] = set() + work = 0 + for entry in entries: + if not isinstance(entry, str) or not entry.startswith(prefix): + return None + work += len(entry) + if work > _CONSENT_WORK_LIMIT: + return None + parts = entry[len(prefix) :].split("/") + if len(parts) < 4: + return None + found = False + for first, second, third in combinations(range(1, len(parts)), 3): + fields = tuple( + "/".join(parts[a:b]) + for a, b in ((0, first), (first, second), (second, third), (third, len(parts))) + ) + work += sum(map(len, fields)) + len(parts) + if work > _CONSENT_WORK_LIMIT: + return None + agent, user, session, key = fields + if key: + keys.add((org, namespace, agent, user, session, key)) + found = True + if not found: + return None + return sorted(keys) or None diff --git a/server/memory/domain/provenance.py b/server/memory/domain/provenance.py index 28197c585d..76f2fce6c4 100644 --- a/server/memory/domain/provenance.py +++ b/server/memory/domain/provenance.py @@ -110,6 +110,8 @@ def cli_actor(command: str) -> str: # server/ is one of these, or a `loom` command's (tests/unit/test_model_call_writers.py). PIPELINE_ACTORS: Final = frozenset( { + "process:memory_refusal", + "process:session_contrast", EXTRACTION_WORKER_ACTOR, "process:budget_warning", "process:code_coverage_notify", diff --git a/server/memory/domain/refusals.py b/server/memory/domain/refusals.py new file mode 100644 index 0000000000..df6ffe59c1 --- /dev/null +++ b/server/memory/domain/refusals.py @@ -0,0 +1,178 @@ +"""Fingerprints for refusing deleted values without storing their plaintext.""" + +from __future__ import annotations + +import re +import unicodedata +from collections.abc import Iterable, Mapping +from typing import Any + +from server.core.errors import ValidationError +from server.core.hashing import sha256_value + +Fingerprint = tuple[str, int, str] +_WORDS = re.compile(r"\w+|[^\w\s]") +# not a setting: fail-closed CPU safety ceiling for complete consent verification. +_WORK_LIMIT = 250_000 + + +def tokens(text: str) -> list[str]: + return _WORDS.findall(unicodedata.normalize("NFKC", text).casefold()) + + +def fingerprint(value: Any) -> list[Fingerprint]: + """One canonical hash and, for a string, a normalized literal fingerprint.""" + out = [("value", 0, sha256_value(value))] + if isinstance(value, str) and (words := tokens(value)): + out.append(("literal", len(words), sha256_value(words))) + return out + + +def inspect(value: Any) -> tuple[set[Fingerprint], list[list[str]]]: + """Canonical subtrees and normalized string leaves, bounded by input work.""" + exact: set[Fingerprint] = set() + text: list[list[str]] = [] + pending = [value] + work = 0 + while pending: + part = pending.pop() + work += 1 + exact.add(("value", 0, sha256_value(part))) + if isinstance(part, dict): + pending.extend(part.values()) + pending.extend(part.keys()) + elif isinstance(part, list): + pending.extend(part) + elif isinstance(part, str): + words = tokens(part) + text.append(words) + work += len(words) + if work > _WORK_LIMIT: + raise ValidationError("The value is too large to verify against refusal policy") + return exact, text + + +def candidates( + inspected: tuple[set[Fingerprint], list[list[str]]], sizes: Iterable[int] +) -> set[Fingerprint]: + exact, texts = inspected + result = set(exact) + work = 0 + sizes = set(sizes) + for words in texts: + for size in sizes: + work += max(0, len(words) - size + 1) * size + if size <= 0 or work > _WORK_LIMIT: + raise ValidationError("The value is too large to verify against refusal policy") + result.update( + ("literal", size, sha256_value(words[i : i + size])) + for i in range(len(words) - size + 1) + ) + return result + + +def matches(value: Any, patterns: Iterable[Fingerprint]) -> bool: + """Check complete JSON subtrees and literal passages; refuse an incomplete scan.""" + patterns = set(patterns) + return bool( + patterns + and patterns & candidates(inspect(value), [p[1] for p in patterns if p[0] == "literal"]) + ) + + +EXTRACTION_CONTENT_FIELDS = frozenset( + { + "messages", + "last_messages", + "session_values", + "extra_tags", + "recalled_keys", + "extraction_instructions", + "record_schema", + "pipeline_steps", + "observation_date", + "memory_key", + } +) +EXTRACTION_PROTOCOL_FIELDS = frozenset( + { + "org", + "namespace", + "agent", + "user_id", + "session_id", + "run_id", + "max_tokens", + "judge_extraction", + "llm_cache", + "detect_updates", + "coverage_sweep", + "agent_learnings", + "defer", + "valid_at", + } +) + + +EXTRACTION_PROTOCOL_STEPS = frozenset( + { + "loom.fetch_namespace_categories", + "loom.fetch_existing_memories", + "loom.llm_extract", + "loom.drop_recall_echoes", + "loom.drop_session_echoes", + "loom.drop_activity_narration", + "loom.drop_non_durable", + "loom.verify_durable", + "loom.hash_dedup", + "loom.bulk_write", + } +) + + +def transcript_content(payload: Mapping[str, Any]) -> list[Any]: + """Caller data, excluding the request's and nested models' protocol defaults.""" + result: list[Any] = [] + for field in ("messages", "last_messages"): + for message in payload.get(field) or []: + result.extend(message[k] for k in ("content", "name") if message.get(k)) + role = message.get("role") + if role and role not in { + "user", + "assistant", + "system", + "developer", + "tool", + "function", + }: + result.append(role) + for field in ("session_values", "extra_tags", "recalled_keys"): + result.extend(payload.get(field) or []) + for field in ("extraction_instructions", "observation_date", "memory_key"): + if payload.get(field): + result.append(payload[field]) + for name, spec in (payload.get("record_schema") or {}).items(): + result.append(name) + if not isinstance(spec, str) or spec not in {"string", "number", "integer", "boolean"}: + result.append(spec) + for step in payload.get("pipeline_steps") or []: + step_id = step.get("step_id") + if "step_id" in step and not ( + isinstance(step_id, str) and step_id in EXTRACTION_PROTOCOL_STEPS + ): + result.append(step_id) + if "config" in step: + result.append(step["config"]) + arguments = {k: v for k, v in step.items() if k not in {"step_id", "config"}} + if arguments: + result.append(arguments) + return result + + +def payload_fingerprints(payload: Mapping[str, Any]) -> set[Fingerprint]: + result: set[Fingerprint] = set() + for value in transcript_content(payload): + exact, texts = inspect(value) + result.update(exact) + result.update(("literal", len(words), sha256_value(words)) for words in texts if words) + return result diff --git a/server/memory/domain/summary_provenance.py b/server/memory/domain/summary_provenance.py new file mode 100644 index 0000000000..3ce89d7d20 --- /dev/null +++ b/server/memory/domain/summary_provenance.py @@ -0,0 +1,57 @@ +"""Extractive digests reconstructed from their authoritative sources.""" + +from __future__ import annotations + +import json +from datetime import UTC, datetime +from typing import Any + +from server.memory.domain.sensitivity import value_reads_high +from server.memory.domain.temporal_sql import SERVED_STATUSES + + +def source_text( + row: Any, *, now: datetime, as_of: datetime | None = None, as_of_basis: str = "valid_time" +) -> str | None: + """Only a complete, currently served source can contribute digest text.""" + if ( + row.tombstoned_at is not None + or row.status not in SERVED_STATUSES + or getattr(row, "requires_review", False) + or row.sensitivity == "high" + or row.value_truncated + or value_reads_high(row.value) + ): + return None + for expiry in (row.expires_at, row.valid_to): + if expiry is not None and _utc(expiry) <= _utc(now): + return None + if as_of is not None: + at = _utc(as_of) + if as_of_basis == "knowledge_time": + elapsed = at - datetime(1970, 1, 1) + cutoff = ( + elapsed.days * 86400 + elapsed.seconds + ) * 1_000_000_000 + elapsed.microseconds * 1000 + if int(row.version) > cutoff: + return None + else: + lower = row.valid_at or row.temporal_anchor or row.created_at + if lower is None or _utc(lower) > at: + return None + if row.valid_to is not None and _utc(row.valid_to) <= at: + return None + return ( + row.value + if isinstance(row.value, str) + else json.dumps(row.value, ensure_ascii=False, sort_keys=True, default=str) + ) + + +def render_summary(facts: list[str]) -> str: + """Every factual byte comes from the source texts, in their recorded order.""" + return "\n".join(facts) + + +def _utc(value: datetime) -> datetime: + return value.astimezone(UTC).replace(tzinfo=None) if value.tzinfo else value diff --git a/server/memory/store/delete_checks.py b/server/memory/store/delete_checks.py index 283bc6f9ae..b43a5424ac 100644 --- a/server/memory/store/delete_checks.py +++ b/server/memory/store/delete_checks.py @@ -7,9 +7,12 @@ from __future__ import annotations +import json from typing import Any +from server.core.hashing import sha256_text from server.core.repo_base import _CHClient, _default_client +from server.memory.store.rows import MemoryRow, memory_head_state from server.memory.store.sql import _SEQUENTIAL, read_your_writes_enabled TABLE = "loom.delete_version_checks" @@ -28,6 +31,11 @@ ) +def checked_state_sha256(row: MemoryRow) -> str: + """Bind recovery to the full checked state, including same-version collisions.""" + return sha256_text(json.dumps(memory_head_state(row), sort_keys=True, default=str)) + + async def record_delete_check( check: dict[str, Any], *, done: bool, ch: _CHClient | None = None ) -> None: @@ -37,6 +45,7 @@ async def record_delete_check( row = {k: check[k] for k in _CHECK_COLUMNS} row["checked"], row["tombstone"] = int(row["checked"]), int(row["tombstone"]) row["done"] = 1 if done else 0 + row["checked_state_sha256"] = check.get("checked_state_sha256", "") await ch.insert(TABLE, [row], settings={"async_insert": 0, "wait_for_async_insert": 1}) @@ -56,7 +65,8 @@ async def find_delete_check( ch = ch or _default_client() rows = await ch.query_all( f""" - SELECT org, namespace, agent, user_id, session_id, memory_key, checked, tombstone, done + SELECT org, namespace, agent, user_id, session_id, memory_key, checked, tombstone, + checked_state_sha256, done FROM {TABLE} FINAL WHERE org = {{org:String}} AND namespace = {{ns:String}} AND agent = {{agent:String}} AND user_id = {{uid:String}} AND session_id = {{sid:String}} @@ -86,7 +96,8 @@ async def pending_delete_checks( ch = ch or _default_client() return await ch.query_all( f""" - SELECT org, namespace, agent, user_id, session_id, memory_key, checked, tombstone + SELECT org, namespace, agent, user_id, session_id, memory_key, checked, tombstone, + checked_state_sha256 FROM {TABLE} FINAL WHERE done = 0 AND created_at <= now() - toIntervalSecond({{age:UInt32}}) ORDER BY created_at diff --git a/server/memory/store/embeddings.py b/server/memory/store/embeddings.py index 27fdbf46ca..78c57ef653 100644 --- a/server/memory/store/embeddings.py +++ b/server/memory/store/embeddings.py @@ -13,7 +13,9 @@ from server.core import pg from server.core.ch_writes import committed_insert_settings, partition_blocks from server.core.datetimes import naive_utc_now +from server.core.hashing import hash_text from server.core.repo_base import _CHClient, _default_client, query_all_sliced +from server.memory.domain.redaction_lineage import is_content_hash from server.memory.domain.temporal_sql import SERVED_STATUSES from server.memory.store.inserts import insert_embedding_rows from server.memory.store.rows import ( @@ -125,7 +127,7 @@ async def mirror_status_to_embeddings( overrides["tombstoned_at"] = m.get("tombstoned_at") if judged: overrides["version"] = drawn - shas[key] = str(m.get("content_sha256") or "") + shas[key] = hash_text(m.get("content_sha256")) elif m.get("version"): overrides["version"] = int(m["version"]) wanted[key] = overrides @@ -143,7 +145,11 @@ async def mirror_status_to_embeddings( continue for r in rows: want = wanted.get(identity_key(r)) - if judged and str(r.get("content_sha256") or "") != shas.get(identity_key(r)): + if ( + judged + and hash_text(r.get("content_sha256")) != shas.get(identity_key(r)) + and not _unproven_retirement(r, want) + ): continue # another revision's vector if judged and str(r.get("status") or "") not in SERVED_STATUSES: continue # a retire (a refines supersede) won meanwhile @@ -160,7 +166,10 @@ async def mirror_status_to_embeddings( return len(out) if out: try: - await insert_embedding_rows(out, ch=ch) + candidates = out + out = await insert_embedding_rows(candidates, ch=ch) + if not judged: + out += await _retire_held_status_copies(candidates, out, wanted, ch=ch) await retire_copies_under_unembedded_heads(out, ch=ch) except Exception as e: # a mirror, never the write; strict re-raises if strict: @@ -170,6 +179,34 @@ async def mirror_status_to_embeddings( return len(out) +async def _retire_held_status_copies( + candidates: list[dict[str, Any]], + written: list[dict[str, Any]], + wanted: Mapping[IdentityKey, dict[str, Any]], + *, + ch: _CHClient, +) -> list[dict[str, Any]]: + """Retire an older, unpublishable vector at its canonical retirement's pin.""" + + def receipt(row: Mapping[str, Any]) -> tuple[Any, ...]: + return tuple(row[c] for c in (*_BARE_TOMBSTONE_KEPT, "version")) + + acknowledged = {receipt(r) for r in written} + now = naive_utc_now() + bare = [] + for row in candidates: + overrides = wanted[identity_key(row)] + pin = overrides.get("version") + if ( + pin is not None + and int(row["version"]) == int(pin) + and overrides.get("status", "active") not in SERVED_STATUSES + and receipt(row) not in acknowledged + ): + bare.append(_bare_tombstone(row, int(pin), now)) + return await insert_embedding_rows(bare, ch=ch) if bare else [] + + def status_divergence_sql() -> str: """Identity rows (org, namespace, agent, user_id, session_id, memory_key) whose latest live memory status differs from the latest live embeddings @@ -380,7 +417,7 @@ async def reemit_embedding_rows_for_keys( if row is not None: out.append(row) if out: - await insert_embedding_rows(out, ch=ch) + out = await insert_embedding_rows(out, ch=ch) await retire_copies_under_unembedded_heads(out, ch=ch) written += len(out) return written @@ -446,6 +483,14 @@ async def _live_rows_for_keys( ) +def _unproven_retirement(row: Mapping[str, Any], overrides: Mapping[str, Any] | None) -> bool: + return bool( + overrides is not None + and overrides.get("status", "active") not in SERVED_STATUSES + and not is_content_hash(hash_text(row.get("content_sha256"))) + ) + + def _copy_row(r: dict[str, Any], overrides: dict[str, Any]) -> dict[str, Any] | None: """A new version of read row `r` with `overrides`: at read + 1, or at the `version` the overrides pin (a retire's, or the expiry sweep's demote). None when the row is @@ -454,7 +499,10 @@ def _copy_row(r: dict[str, Any], overrides: dict[str, Any]) -> dict[str, Any] | pinned = overrides.get("version") if pinned is not None and int(r.get("version") or 0) >= int(pinned): return None - return _reemitted(r, overrides) + copy = _reemitted(r, overrides) + if _unproven_retirement(r, overrides): + return _bare_tombstone(r, int(copy["version"]), naive_utc_now()) + return copy def _reemitted(r: dict[str, Any], overrides: dict[str, Any]) -> dict[str, Any]: @@ -475,7 +523,7 @@ async def reemit_rows( the rows written.""" out = [row for r in rows if (row := _copy_row(r, overrides)) is not None] if out: - await insert_embedding_rows(out, ch=ch) + out = await insert_embedding_rows(out, ch=ch) await retire_copies_under_unembedded_heads(out, ch=ch) return len(out) @@ -528,8 +576,7 @@ async def retire_copies_under_unembedded_heads( for r in copies if identity_key(r) in stale ] - await insert_embedding_rows(tombstones, ch=ch) - return len(tombstones) + return len(await insert_embedding_rows(tombstones, ch=ch)) except Exception as e: # noqa: BLE001 — the copy landed; a missed check is logged, not raised _log.warning("vector copy: head re-check failed for %d rows: %s", len(copies), e) return 0 @@ -719,8 +766,7 @@ async def _bare_tombstones(items: list[tuple[IdentityKey, int]], *, ch: _CHClien ] # A retirement across many keys can span more partitions than an insert takes. for block in partition_blocks(out, created_month): - await insert_embedding_rows(block, ch=ch) - n += len(out) + n += len(await insert_embedding_rows(block, ch=ch)) return n @@ -752,8 +798,7 @@ async def reject_embedding_rows( out.append(_reemitted(r, {"status": "rejected", "version": version})) # A retirement across many keys can span more partitions than an insert takes. for block in partition_blocks(out, created_month): - await insert_embedding_rows(block, ch=ch) - n += len(out) + n += len(await insert_embedding_rows(block, ch=ch)) return n @@ -789,8 +834,8 @@ async def fence_vectors( ) for key, version in items ] - await insert_embedding_rows(rows, ch=ch) - return len(rows) + admitted = await insert_embedding_rows(rows, ch=ch) + return len(admitted) def version_second(version: int) -> datetime: @@ -892,7 +937,7 @@ async def unsupersede_embeddings( if r.get("status") != status or r.get("invalid_at") is not None ] if out: - await insert_embedding_rows(out, ch=ch) + out = await insert_embedding_rows(out, ch=ch) await retire_copies_under_unembedded_heads(out, ch=ch) return len(out) @@ -939,7 +984,7 @@ async def restamp_embedding_rows( overrides = {"status": status, "version": int(version)} out = [row for r in rows if (row := _copy_row(r, overrides)) is not None] if out: - await insert_embedding_rows(out, ch=ch) + out = await insert_embedding_rows(out, ch=ch) await retire_copies_under_unembedded_heads(out, ch=ch) return len(out) diff --git a/server/memory/store/entities.py b/server/memory/store/entities.py index 8e037d5f16..68a9966c32 100644 --- a/server/memory/store/entities.py +++ b/server/memory/store/entities.py @@ -304,9 +304,9 @@ async def _copy_mirrors( ) for r in emb_rows ] - await insert_embedding_rows(copies, ch=ch) - await retire_copies_under_unembedded_heads(copies, ch=ch) - return len(emb_rows) + acknowledged = await insert_embedding_rows(copies, ch=ch) + await retire_copies_under_unembedded_heads(acknowledged, ch=ch) + return len(acknowledged) def _shows(mirror: Mapping[str, Any], head_sha256: str) -> bool: @@ -375,6 +375,6 @@ async def heal_divergent_entities(*, limit: int = 200, ch: _CHClient | None = No ) for r in emb_rows ] - await insert_embedding_rows(copies, ch=ch) - await retire_copies_under_unembedded_heads(copies, ch=ch) - return len(emb_rows) + acknowledged = await insert_embedding_rows(copies, ch=ch) + await retire_copies_under_unembedded_heads(acknowledged, ch=ch) + return len(acknowledged) diff --git a/server/memory/store/inserts.py b/server/memory/store/inserts.py index 0fb1453301..bd4ec0d679 100644 --- a/server/memory/store/inserts.py +++ b/server/memory/store/inserts.py @@ -4,7 +4,6 @@ from __future__ import annotations from collections.abc import Callable, Mapping, Sequence -from contextlib import nullcontext from typing import Any from server.core.repo_base import _CHClient, _default_client @@ -42,6 +41,7 @@ async def insert_memory_versions( expected_heads: Mapping[IdentityKey, MemoryRow | None] | None = None, on_send: Callable[[], None] | None = None, on_inserted: Callable[[], None] | None = None, + refuse_values: bool = False, ) -> int: """The one loom.memory write: every row is checked against MEMORY_PERSISTED_COLS before the insert (see `check_memory_rows`), and @@ -64,19 +64,26 @@ async def insert_memory_versions( admission = pg_memory.Admission(generations=dict(policy)) if policy else None try: if publish_now: - await pg_publish.commit_and_publish(rows, ch=ch, admission=admission) + await pg_publish.commit_and_publish( + rows, ch=ch, admission=admission, refuse_values=refuse_values + ) else: - await pg_publish.commit(rows, publisher_lease=False, admission=admission) + await pg_publish.commit( + rows, publisher_lease=False, admission=admission, refuse_values=refuse_values + ) except pg_memory.AdmissionMoved: raise pg_memory.AdmissionContendedError( "the write policy changed while it was checked" ) from None return len(rows) ch = ch or _default_client() + from server.memory.store import refusals from server.memory.store.write_fence import checked_write guard = ( - nullcontext() if pg.enabled() else checked_write(rows, expected_heads=expected_heads, ch=ch) + refusals.publication(ch, rows, committed=True) + if pg.enabled() + else checked_write(rows, expected_heads=expected_heads, ch=ch, refuse_values=refuse_values) ) async with guard: if on_send is not None: @@ -96,7 +103,7 @@ async def insert_embedding_rows( rows: list[dict[str, Any]], *, ch: _CHClient | None = None, -) -> None: +) -> list[dict[str, Any]]: """Insert N memory_embedding rows, committed. Synchronous because this is the row that makes a memory findable @@ -104,14 +111,29 @@ async def insert_embedding_rows( fire-and-forget insert would let `memory.set` return before its memory can be found. The cost is tens of milliseconds per write. - Empty `rows` is a no-op. + Return the input rows whose publication or bounded orphan cleanup was acknowledged. """ if not rows: - return + return [] ch = ch or _default_client() - await ch.insert( - "loom.memory_embeddings", - rows, - settings={"async_insert": 0, "wait_for_async_insert": 1}, - ) - pg_pipeline.record_insert("memory_embeddings", len(rows)) + from server.memory.store.shared import repair_shared_embeddings + + shared = [r for r in rows if r.get("scope") == "shared" and r.get("tombstoned_at") is None] + acknowledged = [] + if shared: + acknowledged = await repair_shared_embeddings(shared, ch=ch) + rows = [r for r in rows if r not in shared] + if not rows: + return acknowledged + from server.memory.store import refusals + + retired: list[dict[str, Any]] = [] + async with refusals.publication(ch, rows, embeddings=True, retired=retired) as admitted: + if admitted: + await ch.insert( + "loom.memory_embeddings", + [dict(r) for r in admitted], + settings={"async_insert": 0, "wait_for_async_insert": 1}, + ) + pg_pipeline.record_insert("memory_embeddings", len(admitted)) + return acknowledged + [dict(row) for row in admitted] + retired diff --git a/server/memory/store/integrity.py b/server/memory/store/integrity.py index e7c77282e0..13bc4dcf60 100644 --- a/server/memory/store/integrity.py +++ b/server/memory/store/integrity.py @@ -160,15 +160,15 @@ async def repair_dim0_rows( if dry_run or not out: return len(out) ch = ch or _default_client() - await insert_embedding_rows(out, ch=ch) + admitted = await insert_embedding_rows(out, ch=ch) from server.memory.store.embeddings import retire_copies_under_unembedded_heads await retire_copies_under_unembedded_heads(out, ch=ch) # A row that was dim 0 was never counted, and its + 1 sits under the BM25 watermark. from server.memory.store.bm25_stats import index_late_rows - await index_late_rows(out, ch=ch) - return len(out) + await index_late_rows(admitted, ch=ch) + return len(admitted) async def drift_sample( diff --git a/server/memory/store/lifecycle.py b/server/memory/store/lifecycle.py index 8052eeeeb2..98e99233c6 100644 --- a/server/memory/store/lifecycle.py +++ b/server/memory/store/lifecycle.py @@ -316,6 +316,7 @@ async def _insert_clickhouse_copies( landed: Callable[[], Awaitable[None]] | None, on_commit: Callable[[], None] | None, exact: bool, + refuse_values: bool, ) -> int: inserted = not rows count = 0 @@ -337,7 +338,11 @@ async def follow() -> int: {identity_key(r): h for r, h in zip(rows, read, strict=True)} if exact else None ) await insert_memory_versions( - rows, ch=ch, expected_heads=expected, on_inserted=mark_inserted + rows, + ch=ch, + expected_heads=expected, + on_inserted=mark_inserted, + refuse_values=refuse_values, ) finally: # A successful memory insert still owes its follow-ups when holder @@ -355,12 +360,12 @@ async def insert_versions_over_heads( then: Callable[[], Awaitable[int | None]] | None = None, landed: Callable[[], Awaitable[None]] | None = None, documented: Sequence[bool] | None = None, - settle_held: bool = False, sources: Sequence[tuple[str, int]] = (), live_sources: bool = False, policy: Mapping[tuple[str, str], int] | None = None, on_commit: Callable[[], None] | None = None, exact: bool = False, + refuse_values: bool = False, ) -> int: """Write `rows`, each a copy of the authority's head beside it in `read`, drawn above it (`rows.version_over`) or, to yield to a write since, before the read: a copy below @@ -376,16 +381,22 @@ async def insert_versions_over_heads( of it is raised once the commit is published. `landed` runs once the rows are in, outside the lease: on ClickHouse right after the insert, before `then`; under Postgres after the publish, whatever `then` did. `documented`: the rows that owe a live search - document, which `then` writes; under Postgres they are settled once `then` returns - (with `settle_held`, only those ClickHouse holds at their row's version, for a `then` - that copies what it finds), and the delivery worker builds one it did not. + document, which `then` writes; under Postgres only documents ClickHouse holds at + their row's version are settled, and the delivery worker builds one it did not. `sources`: the revisions the rows copy (`Admission.sources`), with `live_sources` each still its key's live head. `on_commit` is called once the rows are in, before holder cleanup or publication can fail. A commit whose publication is pending is done.""" if not pg.enabled() or not rows: return await _insert_clickhouse_copies( - rows, read, ch=ch, then=then, landed=landed, on_commit=on_commit, exact=exact + rows, + read, + ch=ch, + then=then, + landed=landed, + on_commit=on_commit, + exact=exact, + refuse_values=refuse_values, ) check_memory_rows(rows) heads = {identity_key(r): e for r, e in zip(rows, read, strict=True)} @@ -398,7 +409,9 @@ async def insert_versions_over_heads( plain = pg_memory.MEMORY_ROW | pg_memory.AUDIT_ROW owed = [plain | (pg_memory.SEARCH_DOC if d else 0) for d in documented or [False] * len(rows)] try: - committed = await pg_publish.commit(rows, targets=owed, admission=admission) + committed = await pg_publish.commit( + rows, targets=owed, admission=admission, refuse_values=refuse_values + ) except pg_memory.HeadMoved: raise pg_memory.AdmissionContendedError("the memory changed while it was read") from None except pg_memory.AdmissionMoved: @@ -413,7 +426,7 @@ async def insert_versions_over_heads( with pg_lease.leased(committed.deadline), sequential_reads(): answer = await then() or 0 if documented: - await _settle_documents(committed, documented, held=settle_held, ch=ch) + await _settle_documents(committed, documented, ch=ch) except pg_lease.PublicationPending: _log.warning("a copy's follow-up was left: its commit's lease ended first") except Exception as e: # noqa: BLE001 — raised once the commit is published @@ -460,30 +473,27 @@ async def insert_documented_versions( async def _settle_documents( - committed: Any, documented: Sequence[bool], *, held: bool = False, ch: _CHClient | None = None + committed: Any, documented: Sequence[bool], *, ch: _CHClient | None = None ) -> None: - """Clear the search documents written under the commit's lease; with `held`, only - those that ClickHouse holds at their row's version (`held_documents`). Failing that, + """Clear only documents ClickHouse holds at their row's version (`held_documents`). + Failing that, the delivery worker settles one it finds there and builds again one written below it (an Undo's rows, re-emitted at their own).""" owed = [(r, q) for r, q, d in zip(committed.rows, committed.seqs, documented, strict=True) if d] try: - if held: - client = ch or _default_client() - by_scope: dict[tuple[str, str], list[tuple[Any, ...]]] = {} - for r, _ in owed: - by_scope.setdefault((r["org"], r["namespace"]), []).append( - pg_publish.document_id(r) - ) - held_ids: set[tuple[Any, ...]] = set() - for (org, ns), ids in by_scope.items(): - docs = await pg_publish.held_documents(org, ns, ids, ch=client) - held_ids |= {(org, ns, *d) for d in docs} - owed = [ - (r, q) - for r, q in owed - if (r["org"], r["namespace"], *pg_publish.document_id(r)) in held_ids - ] + client = ch or _default_client() + by_scope: dict[tuple[str, str], list[tuple[Any, ...]]] = {} + for r, _ in owed: + by_scope.setdefault((r["org"], r["namespace"]), []).append(pg_publish.document_id(r)) + held_ids: set[tuple[Any, ...]] = set() + for (org, ns), ids in by_scope.items(): + docs = await pg_publish.held_documents(org, ns, ids, ch=client) + held_ids |= {(org, ns, *d) for d in docs} + owed = [ + (r, q) + for r, q in owed + if (r["org"], r["namespace"], *pg_publish.document_id(r)) in held_ids + ] await pg_memory.complete([q for _, q in owed], pg_memory.SEARCH_DOC) except Exception as e: # noqa: BLE001 — the commit stands; the worker settles or builds them _log.warning("documents left owed for the delivery worker: %s", error_summary(e)) @@ -884,7 +894,6 @@ def on_commit() -> None: ch=ch, then=copy_vectors, documented=documented, - settle_held=True, on_commit=on_commit, ) except pg_memory.AdmissionContendedError: @@ -1037,8 +1046,24 @@ async def _tombstone_ch_heads( ) for head in page ] - for block in partition_blocks(rows, created_month): - await insert_memory_versions(block, ch=ch) + blocks = [ + block[start : start + 16] + for block in partition_blocks(rows, created_month) + for start in range(0, len(block), 16) + ] + for block in blocks: + expected = { + identity_key(h): _hydrate(h) + for h in page + if identity_key(h) in {identity_key(r) for r in block} + } + try: + await insert_memory_versions( + block, ch=ch, expected_heads=expected, refuse_values=True + ) + except pg_memory.AdmissionContendedError: + # Re-page at the original cutoff: a new value or lock is not ours to erase. + break heads = await memory_rows_by_identity_keys( [identity_key(r) for r in block], ch=ch, include_tombstoned=True, sequential=True ) @@ -1066,7 +1091,8 @@ async def _tombstone_ch_heads( ], ch=ch, ) - after = key_in_namespace(page[-1]) + else: + after = key_in_namespace(page[-1]) async def _tombstone_where( @@ -1160,6 +1186,7 @@ async def _tombstone_heads( try: committed = await pg_publish.commit( rows, + refuse_values=True, admission=pg_memory.Admission( recheck=pg_memory.heads_unchanged(heads, exact=True) ), diff --git a/server/memory/store/pg_batch.py b/server/memory/store/pg_batch.py index 538f150b4a..b384a0f5f3 100644 --- a/server/memory/store/pg_batch.py +++ b/server/memory/store/pg_batch.py @@ -437,14 +437,22 @@ async def _publish(group: list[_Entry]) -> None: deadline = min(e.committed.deadline for e in group) rows = [r for e in group if not e.rows_sent for r in e.committed.rows] docs = [d for e in group for d in served.get(id(e), {}).values()] + acknowledged: set[tuple[Any, ...]] = set() + rows_finished = asyncio.Event() async def send_rows() -> None: - for block in partition_blocks(rows, created_month): - await inserts.insert_memory_versions(block, ch=ch, published=True) + try: + for block in partition_blocks(rows, created_month): + await inserts.insert_memory_versions(block, ch=ch, published=True) + finally: + rows_finished.set() async def send_documents() -> None: + # Both sends acquire the same consent fences; competing can split their claims. + await rows_finished.wait() for block in partition_blocks(docs, created_month): - await inserts.insert_embedding_rows(block, ch=ch) + admitted = await inserts.insert_embedding_rows(block, ch=ch) + acknowledged.update(_doc_key(row) for row in admitted) with pg_lease.leased(deadline): rows_error, audit_error, docs_error = await asyncio.gather( @@ -464,6 +472,18 @@ async def send_documents() -> None: ) rows_in = not isinstance(rows_error, BaseException) documents_in = not isinstance(docs_error, BaseException) + if documents_in: + for e in group: + attempted = served.get(id(e), {}) + served[id(e)] = { + key: row for key, row in attempted.items() if _doc_key(row) in acknowledged + } + withheld = attempted.keys() - served[id(e)].keys() + e.stale.extend( + seq + for seq, row in zip(e.committed.seqs, e.committed.rows, strict=True) + if _doc_key(row) in withheld + ) await pg_memory.complete_each( _landed( group, diff --git a/server/memory/store/pg_memory.py b/server/memory/store/pg_memory.py index 7ef14e58de..28057dd362 100644 --- a/server/memory/store/pg_memory.py +++ b/server/memory/store/pg_memory.py @@ -17,8 +17,7 @@ import logging import math import time -from collections.abc import AsyncIterator, Callable, Mapping, Sequence -from contextlib import asynccontextmanager +from collections.abc import Callable, Mapping, Sequence from dataclasses import dataclass, field, replace from datetime import UTC, datetime from typing import Any @@ -700,6 +699,7 @@ async def commit_versions( request_id: str = "", idempotency: tuple[str, str, int] | None = None, publisher_lease: bool = True, + refuse_values: bool = False, ) -> Committed: """Commit `rows` (full loom.memory version rows): heads, revisions with `audits[i]` (row i's hash-only loom.audit_log row), deliveries (`targets`: one @@ -725,6 +725,9 @@ async def commit_versions( per_scope, scopes, heads, keys = _batch_indexes(stored) async with pg.transaction() as conn: await _bound_lock_waits(conn) + from server.memory.store import refusals + + await refusals.lock_postgres(conn, stored, forgetting=refuse_values) if admission is not None: await _check_policy(conn, admission, per_scope) bumped = await conn.fetch( @@ -735,6 +738,7 @@ async def commit_versions( # runs, and before the head recheck: an erased memory is a 404, not a moved head. await _refuse_stale_rows(conn, stored, admission) await _refuse_moved_sources(conn, scopes, admission) + await refusals.check_postgres(conn, stored) if admission is not None: moved = { s: last[s] - per_scope[s] @@ -755,6 +759,8 @@ async def commit_versions( recheck(k, head) if _replaces_vectors(stored[heads[k]], head): per_row[heads[k]] |= OLD_VECTORS + if refuse_values: + await refusals.record_postgres(conn, stored) seqs: list[tuple[str, str, int]] = [] next_seq = {s: last[s] - per_scope[s] for s in scopes} for r in stored: @@ -1618,7 +1624,11 @@ async def settled_head_versions(keys: Sequence[IdentityKey]) -> dict[IdentityKey async def heads_by_keys( - keys: Sequence[IdentityKey], *, include_tombstoned: bool = False, durable: bool = False + keys: Sequence[IdentityKey], + *, + include_tombstoned: bool = False, + durable: bool = False, + max_value_chars: int | None = None, ) -> dict[IdentityKey, dict[str, Any]]: """The heads, expired ones included, of exact keys (a deleted one too with `include_tombstoned`); a key with none is absent. `durable`: read for ClickHouse @@ -1629,24 +1639,41 @@ async def heads_by_keys( sql = _HEADS_BY_KEYS if not include_tombstoned: sql += " WHERE c.row->>'tombstoned_at' IS NULL" - rows = await _read(sql, *_key_arrays(keys), durable=durable) + args = _key_arrays(keys) + if max_value_chars is not None: + sql += (" AND " if " WHERE " in sql else " WHERE ") + ( + "length(coalesce(c.row->>'value','')) <= $7 AND " + "length(coalesce(c.row->>'value_json','{}')) <= $7" + ) + bound: list[Any] = [*args, max_value_chars] + else: + bound = list(args) + rows = await _read(sql, *bound, durable=durable) return {identity_key(r["row"]): decode_row(r["row"]) for r in rows} -@asynccontextmanager -async def locked_source(source: MemoryRow) -> AsyncIterator[None]: - """Hold an unchanged durable source while its shared projection is published.""" - key = identity_key(source) - durable = await heads_by_keys([key], include_tombstoned=True, durable=True) - if not unchanged(source, durable.get(key)): - raise AdmissionContendedError("The promotion's source changed") - async with pg.rolled_back() as conn: - await _bound_lock_waits(conn) - rows = await conn.fetch(_LOCK_HEADS, *_key_arrays([key])) - current = decode_row(rows[0]["row"]) if rows else None - if not unchanged(source, current): - raise AdmissionContendedError("The promotion's source changed") - yield +async def heads_on_connection( + conn: Any, + keys: Sequence[IdentityKey], + *, + lock: bool = False, + max_value_chars: int | None = None, +) -> dict[IdentityKey, dict[str, Any]]: + """Recheck heads while the caller holds scope locks on this connection.""" + if not keys: + return {} + sql = _HEADS_BY_KEYS + args: list[Any] = list(_key_arrays(keys)) + if max_value_chars is not None: + sql += ( + " WHERE length(coalesce(c.row->>'value','')) <= $7 AND " + "length(coalesce(c.row->>'value_json','{}')) <= $7" + ) + args.append(max_value_chars) + if lock: + sql += " " + _KEY_ORDER + " FOR UPDATE OF c" + rows = await conn.fetch(sql, *args) + return {identity_key(r["row"]): decode_row(r["row"]) for r in rows} async def heads_owing( diff --git a/server/memory/store/pg_publish.py b/server/memory/store/pg_publish.py index 806185b8c1..69cf5c031f 100644 --- a/server/memory/store/pg_publish.py +++ b/server/memory/store/pg_publish.py @@ -95,6 +95,7 @@ async def commit( admission: Admission | None = None, idempotent: bool = False, publisher_lease: bool = True, + refuse_values: bool = False, ) -> Committed: """The commit alone; `publish` sends it. `idempotent`: these are the op's own rows, which the request's idempotency key records as committed (another write the request @@ -114,6 +115,7 @@ async def commit( request_id=rid, idempotency=idempotency.bound_key() if idempotent else None, publisher_lease=publisher_lease, + refuse_values=refuse_values, ) @@ -338,8 +340,9 @@ async def commit_and_publish( *, ch: _CHClient | None = None, admission: Admission | None = None, + refuse_values: bool = False, ) -> Committed: """`insert_memory_versions` under the Postgres authority, held to its admission.""" - committed = await commit(rows, admission=admission) + committed = await commit(rows, admission=admission, refuse_values=refuse_values) await publish(committed, ch=ch) return committed diff --git a/server/memory/store/reads.py b/server/memory/store/reads.py index c535913d2b..5987266061 100644 --- a/server/memory/store/reads.py +++ b/server/memory/store/reads.py @@ -383,6 +383,7 @@ async def memory_rows_by_identity_keys( ch: _CHClient | None = None, sequential: bool = False, include_tombstoned: bool = False, + max_value_chars: int | None = None, ) -> dict[IdentityKey, MemoryRow]: """`get_by_key` for N exact identity keys, in statements whose keys each fit one form field (`query_all_sliced`): the same projection under FINAL, the full sort key in @@ -393,19 +394,29 @@ async def memory_rows_by_identity_keys( if not keys: return {} if pg.enabled(): - heads = await pg_memory.heads_by_keys(keys, include_tombstoned=include_tombstoned) + heads = await pg_memory.heads_by_keys( + keys, include_tombstoned=include_tombstoned, max_value_chars=max_value_chars + ) return {k: _hydrate(h) for k, h in heads.items()} ch = ch or _default_client() # Bound a slice at a time: one parameter holds as many keys as a form field does. + where = [] + if not include_tombstoned: + where.append("tombstoned_at IS NULL") + if max_value_chars is not None: + where.append( + "length(value) <= {value_cap:UInt64} AND " + "length(toString(value_json)) <= {value_cap:UInt64}" + ) rows = await query_all_sliced( ch, f"""{_MEMORY_FINAL_PROJECTION} PREWHERE (org, namespace, agent, user_id, session_id, memory_key) IN {{keys:{_IDENTITY_KEY_TUPLE}}} - {"" if include_tombstoned else "WHERE tombstoned_at IS NULL"} + {("WHERE " + " AND ".join(where)) if where else ""} {_SEQUENTIAL if sequential else ""} """, - {"keys": list(keys)}, + {"keys": list(keys), "value_cap": max_value_chars}, "keys", ) return {identity_key(r): _hydrate(r) for r in rows} diff --git a/server/memory/store/refusals.py b/server/memory/store/refusals.py new file mode 100644 index 0000000000..488243a7d1 --- /dev/null +++ b/server/memory/store/refusals.py @@ -0,0 +1,697 @@ +"""Durable refusal policy and admission fences shared by writers and erasers.""" + +from __future__ import annotations + +import hashlib +import json +from collections.abc import AsyncIterator, Callable, Iterator, Mapping, Sequence +from contextlib import AsyncExitStack, asynccontextmanager +from contextvars import ContextVar +from datetime import UTC +from typing import Any + +from server.core import lease_writes, pg +from server.core.ch import sequential_reads, set_query_tag, write_limit_s +from server.core.ch_params import chunks +from server.core.ch_writes import batched_insert_settings, committed_insert_settings +from server.core.datetimes import naive_utc_now +from server.core.errors import BackendUnavailableError, ValidationError +from server.core.hashing import hash_text, served_row_value, sha256_value +from server.core.repo_base import _CHClient +from server.memory.domain import refusals + +# Protocol constant: erasers acquire every stripe; every publisher acquires one. +_STRIPES = 32 +_writer: Callable[[], str] | None = None +_shared_permit: ContextVar[tuple[str, str, str, str] | None] = ContextVar( + "shared_refusal_permit", default=None +) + + +_CH_SCOPE = ( + "((scope_kind IN ('identity','erased_identity') AND scope_value={ns:String} " + "AND user_id={uid:String}) " + "OR (scope_kind='namespace' AND scope_value={ns:String}) " + "OR (scope_kind='user_id' AND scope_value={uid:String}))" +) +_PG_SCOPE = ( + "((scope_kind IN ('identity','erased_identity') AND scope_value=$2 AND user_id=$3) " + "OR (scope_kind='namespace' AND scope_value=$2) " + "OR (scope_kind='user_id' AND scope_value=$3))" +) + +_CH_ERASURE_SCOPE = _CH_SCOPE.replace("('identity','erased_identity')", "('erased_identity')") +_PG_ERASURE_SCOPE = _PG_SCOPE.replace("('identity','erased_identity')", "('erased_identity')") + + +def _lease_client(ch: _CHClient) -> _CHClient: + return ch + + +def _admission_wait_s() -> float: + return write_limit_s() + + +def use_writer(writer: Callable[[], str]) -> None: + global _writer + _writer = writer + + +def _scope(kind: str, org: str, value: str) -> str: + return json.dumps([kind, org, value], separators=(",", ":")) + + +def _name(scope: str, stripe: int) -> str: + return "memory_refusal/" + hashlib.sha256(scope.encode()).hexdigest() + f"/{stripe}" + + +def fence_names(rows: Sequence[Mapping[str, Any]], *, forgetting: bool = False) -> list[str]: + names: set[str] = set() + for row in rows: + org, ns, uid = str(row["org"]), str(row["namespace"]), str(row.get("user_id") or "-") + encoded = json.dumps( + [org, ns, row.get("agent"), uid, row.get("session_id"), row.get("memory_key")], + separators=(",", ":"), + default=str, + ) + stripe = int.from_bytes(hashlib.sha256(encoded.encode()).digest()[:4]) % _STRIPES + names.add(_name(_scope("org", org, ""), stripe)) + names.add(_name(_scope("namespace", org, ns), stripe)) + user_scope = _scope("user_id", org, uid) + names.update(_name(user_scope, n) for n in (range(_STRIPES) if forgetting else [stripe])) + return sorted(names) + + +def erasure_fence_names(org: str, kind: str, value: str) -> list[str]: + return sorted( + _name(scope, n) + for scope in (_scope("org", org, ""), _scope(kind, org, value)) + for n in range(_STRIPES) + ) + + +def assert_shared_publication(row: Mapping[str, Any], *, vector: bool = False) -> None: + if row.get("tombstoned_at") is not None: + return + digest = row.get("content_sha256") if vector else sha256_value(served_row_value(row)) + if _shared_permit.get() != (row["org"], row["namespace"], row["memory_key"], digest): + raise ValidationError("A promotion requires its live source's admission") + + +@asynccontextmanager +async def source_publication( + ch: _CHClient, source: Any, target_key: str, *, refusal_users: set[str] | None = None +) -> AsyncIterator[Callable[[bool], None]]: + from server.memory.domain.temporal_sql import SERVED_STATUSES + from server.memory.store import pg_memory + from server.memory.store.reads import memory_rows_by_identity_keys + from server.memory.store.rows import build_memory_version_row, identity_key + + if ( + source.status not in SERVED_STATUSES + or source.requires_review + or source.tombstoned_at + or source.sensitivity == "high" + ): + raise ValidationError("Only an approved live source can be promoted") + row = build_memory_version_row(source) + key = identity_key(row) + async with lease_writes.fenced_many( + fence_names([row]), ch=_lease_client(ch), wait_s=_admission_wait_s() + ) as mark: + async with AsyncExitStack() as stack: + conn = None + if pg.enabled(): + durable = await pg_memory.heads_by_keys( + [key], include_tombstoned=True, durable=True + ) + if not pg_memory.unchanged(source, durable.get(key)): + raise pg_memory.AdmissionContendedError("The promotion's source changed") + conn = await stack.enter_async_context(pg.transaction()) + await lock_postgres(conn, [row]) + heads = await pg_memory.heads_on_connection(conn, [key], lock=True) + await check_postgres(conn, [row]) + else: + heads = { + k: build_memory_version_row(h) + for k, h in ( + await memory_rows_by_identity_keys( + [key], ch=ch, sequential=True, include_tombstoned=True + ) + ).items() + } + await check_clickhouse(ch, [row]) + if refusal_users is not None and ( + not refusal_users + or await _shared_values_refused( + ch, + conn, + [{**row, "user_id": user or "-"} for user in refusal_users], + erasure_only=False, + ) + ): + raise _refused() + if not pg_memory.unchanged(source, heads.get(key)): + raise pg_memory.AdmissionContendedError("The promotion's source changed") + token = _shared_permit.set( + (source.org, source.namespace, target_key, sha256_value(source.value)) + ) + try: + yield mark + finally: + _shared_permit.reset(token) + + +@asynccontextmanager +async def shared_retirement(ch: _CHClient, row: Mapping[str, Any]) -> AsyncIterator[bool]: + """A delayed shared tombstone cannot restore bytes removed by erasure.""" + from server.memory.domain.promotion_source import promotion_users + + users = promotion_users(list(row.get("promoted_from") or []), row["org"], row["namespace"]) + base = content_policy_row(row, served_row_value(row)) + sources = [{**base, "user_id": user or "-"} for user in sorted(users)] if users else [base] + async with ( + lease_writes.fenced_many( + fence_names([base]), ch=_lease_client(ch), wait_s=_admission_wait_s() + ) as mark, + AsyncExitStack() as stack, + ): + conn = await stack.enter_async_context(pg.transaction()) if pg.enabled() else None + if conn is not None: + await lock_postgres(conn, [base]) + erased = await _shared_values_refused(ch, conn, sources) + from server.memory.store.shared import fetch_shared + + head = await fetch_shared( + row["org"], row["namespace"], row["memory_key"], strong=True, ch=ch + ) + publish = bool(users) and head is not None and not erased + mark(True) + if not publish: + await _delete_old_shared(ch, row) + mark(False) + yield publish + mark(False) + + +async def _delete_old_shared(ch: _CHClient, row: Mapping[str, Any]) -> None: + set_query_tag("memory.retire_orphan_shared") + await ch.execute( + "DELETE FROM loom.shared WHERE org={org:String} " + "AND namespace={namespace:String} AND memory_key={memory_key:String} " + "AND version < {version:UInt64}", + params={c: row[c] for c in ("org", "namespace", "memory_key", "version")}, + settings={"lightweight_deletes_sync": 2}, + ) + + +async def _shared_values_refused( + ch: _CHClient, conn: Any, sources: Sequence[Mapping[str, Any]], *, erasure_only: bool = True +) -> bool: + refused = False + inspected = [refusals.inspect(served_row_value(sources[0]))] + keys = {(str(row["org"]), str(row["namespace"]), str(row["user_id"])) for row in sources} + with sequential_reads(): + for key in sorted(keys): + set_query_tag("memory.shared_consent") + refused |= ( + await _matches_postgres(conn, key, inspected, erasure_only=erasure_only) + if conn is not None + else await _matches_clickhouse(ch, key, inspected, erasure_only=erasure_only) + ) + return refused + + +def _policy( + rows: Sequence[Mapping[str, Any]], + *, + scope_kind: str = "identity", + scope_value: str | None = None, +) -> list[dict[str, Any]]: + if _writer is None: + raise RuntimeError("Refusal policy writer is not wired") + result = [] + for row in rows: + if row.get("value_truncated"): + raise ValidationError("A complete value is required to record refusal policy") + value = served_row_value(row) + if row.get("capture_kind") == "metadata" and value in (None, "", {}, []): + continue + fingerprints = ( + refusals.payload_fingerprints(value) + if row.get("capture_kind") == "payload" + else refusals.fingerprint(value) + ) + for kind, units, digest in sorted(fingerprints): + result.append( + { + **{ + c: str(row.get(c) or "-") + for c in ("org", "namespace", "agent", "user_id", "session_id") + }, + "scope_kind": scope_kind, + "scope_value": scope_value + if scope_value is not None + else str(row["namespace"]), + "actor": _writer() or "process:memory_refusal", + "source_class": "derived", + "kind": kind, + "units": units, + "digest": digest, + "created_at": naive_utc_now(), + } + ) + return result + + +def _recording_policy(rows: Sequence[Mapping[str, Any]], **scope: Any) -> list[dict[str, Any]]: + policy = _policy(rows, **scope) + if scope.get("scope_kind") == "user_id": + others = [r for r in rows if str(r.get("user_id") or "-") != scope.get("scope_value")] + policy += _policy(others, scope_kind="erased_identity") + return policy + + +def _live_groups(rows: Sequence[Mapping[str, Any]]) -> dict[tuple[str, str, str], list[Any]]: + groups: dict[tuple[str, str, str], list[Any]] = {} + for row in rows: + if row.get("tombstoned_at") is None: + key = (str(row["org"]), str(row["namespace"]), str(row.get("user_id") or "-")) + groups.setdefault(key, []).append(refusals.inspect(served_row_value(row))) + return groups + + +def _candidate_batch(inspected: list[Any], sizes: list[int]) -> list[refusals.Fingerprint]: + if -1 in sizes: + raise BackendUnavailableError("Memory intake waits for the organization's erasure") + found: set[refusals.Fingerprint] = set() + for value in inspected: + found.update(refusals.candidates(value, sizes)) + return sorted(found) + + +def _max_units(inspected: list[Any]) -> int: + return max((len(words) for _, texts in inspected for words in texts), default=0) + + +def _refused() -> ValidationError: + return ValidationError("The value is refused by a previous deletion", reason="forgotten") + + +def _check_groups( + rows: Sequence[Mapping[str, Any]], +) -> Iterator[tuple[tuple[str, str, str], list[Any], bool]]: + yield from ((key, values, False) for key, values in _live_groups(rows).items()) + dead = [{**r, "tombstoned_at": None} for r in rows if r.get("tombstoned_at") is not None] + yield from ((key, values, True) for key, values in _live_groups(dead).items()) + + +async def _matches_clickhouse( + ch: _CHClient, key: tuple[str, str, str], inspected: list[Any], *, erasure_only: bool = False +) -> bool: + scope = _CH_ERASURE_SCOPE if erasure_only else _CH_SCOPE + params = {"org": key[0], "ns": key[1], "uid": key[2], "max": _max_units(inspected)} + lengths = await ch.query_all( + "SELECT DISTINCT toInt64(units) AS units FROM loom.memory_refusals " + "WHERE org={org:String} AND " + scope + " AND kind='literal' AND units <= {max:UInt32} " + "UNION ALL SELECT toInt64(-1) AS units FROM loom.erasure_requests FINAL " + "WHERE org={org:String} AND status IN ('queued', 'in_progress', 'failed') " + "AND NOT startsWith(failure_reason, 'max_attempts (') LIMIT 1", + params=params, + ) + candidates = _candidate_batch(inspected, [int(r["units"]) for r in lengths]) + for batch in chunks(candidates): + match = await ch.query_all( + "SELECT 1 FROM loom.memory_refusals WHERE org={org:String} AND " + + scope + + " AND (kind, units, digest) IN " + "{candidates:Array(Tuple(String,UInt32,FixedString(64)))} LIMIT 1", + params={**params, "candidates": batch}, + ) + if match: + return True + return False + + +async def check_clickhouse(ch: _CHClient, rows: Sequence[Mapping[str, Any]]) -> None: + with sequential_reads(): + for key, inspected, erasure_only in _check_groups(rows): + if await _matches_clickhouse(ch, key, inspected, erasure_only=erasure_only): + raise _refused() + + +async def record_clickhouse( + ch: _CHClient, + rows: Sequence[Mapping[str, Any]], + *, + scope_kind: str = "identity", + scope_value: str | None = None, +) -> None: + policy = _recording_policy(rows, scope_kind=scope_kind, scope_value=scope_value) + if policy: + await ch.insert( + "loom.memory_refusals", policy, settings={"async_insert": 0, "wait_for_async_insert": 1} + ) + + +async def lock_postgres( + conn: Any, + rows: Sequence[Mapping[str, Any]], + *, + forgetting: bool = False, + erasure: tuple[str, str, str] | None = None, +) -> None: + """Lock scope admission before generation rows, including empty/new namespaces.""" + scopes: dict[str, bool] = {} + orgs = {str(row["org"]) for row in rows} + if erasure is not None: + orgs.add(erasure[1]) + # Every transaction takes organization locks first, including multi-namespace erasure. + for org in sorted(orgs): + fn = "pg_advisory_xact_lock" if erasure is not None else "pg_advisory_xact_lock_shared" + await conn.execute(f"SELECT {fn}(hashtextextended($1, 0))", _scope("org", org, "")) + for row in rows: + org = str(row["org"]) + scopes[_scope("namespace", org, str(row["namespace"]))] = False + scopes[_scope("user_id", org, str(row.get("user_id") or "-"))] = forgetting + if erasure is not None: + scopes[_scope(*erasure)] = True + for name, exclusive in sorted(scopes.items()): + fn = "pg_advisory_xact_lock" if exclusive else "pg_advisory_xact_lock_shared" + await conn.execute(f"SELECT {fn}(hashtextextended($1, 0))", name) + + +async def _matches_postgres( + conn: Any, key: tuple[str, str, str], inspected: list[Any], *, erasure_only: bool = False +) -> bool: + scope = _PG_ERASURE_SCOPE if erasure_only else _PG_SCOPE + lengths = await conn.fetch( + "SELECT DISTINCT units::bigint AS units FROM loom.memory_refusals " + "WHERE org=$1 AND " + scope + " AND kind='literal' AND units<=$4 " + "UNION ALL SELECT -1::bigint FROM loom.erasure_ledger " + "WHERE org=$1 AND completed_at IS NULL AND abandoned_at IS NULL", + *key, + _max_units(inspected), + ) + candidates = _candidate_batch(inspected, [int(r["units"]) for r in lengths]) + for start in range(0, len(candidates), 2000): + batch = candidates[start : start + 2000] + match = await conn.fetchval( + "SELECT 1 FROM loom.memory_refusals r JOIN unnest($4::text[], $5::integer[], " + "$6::text[]) AS k(kind, units, digest) USING(kind, units, digest) " + "WHERE r.org=$1 AND " + scope + " LIMIT 1", + *key, + [p[0] for p in batch], + [p[1] for p in batch], + [p[2] for p in batch], + ) + if match: + return True + return False + + +async def check_postgres( + conn: Any, rows: Sequence[Mapping[str, Any]], *, erasure_only: bool = False +) -> None: + for key, inspected, retired in _check_groups(rows): + if await _matches_postgres(conn, key, inspected, erasure_only=erasure_only or retired): + raise _refused() + + +async def record_postgres( + conn: Any, + rows: Sequence[Mapping[str, Any]], + *, + scope_kind: str = "identity", + scope_value: str | None = None, +) -> None: + policy = _recording_policy(rows, scope_kind=scope_kind, scope_value=scope_value) + if policy: + for row in policy: + row["created_at"] = row["created_at"].replace(tzinfo=UTC) + columns = tuple(policy[0]) + placeholders = ",".join(f"${n}" for n in range(1, len(columns) + 1)) + await conn.executemany( + f"INSERT INTO loom.memory_refusals ({','.join(columns)}) VALUES ({placeholders}) " + "ON CONFLICT DO NOTHING", + [tuple(row[c] for c in columns) for row in policy], + ) + + +def content_policy_row(row: Mapping[str, Any], value: Any) -> dict[str, Any]: + return { + **{ + c: str(row.get(c) or "-") + for c in ("org", "namespace", "agent", "user_id", "session_id") + }, + "memory_key": str(row.get("memory_key") or row.get("parent_memory_key") or "-"), + "value_json": json.dumps(value, default=str), + "value": json.dumps(value, default=str), + "tombstoned_at": row.get("tombstoned_at"), + } + + +async def check_intake(ch: _CHClient, row: Mapping[str, Any], value: Any) -> None: + policy = [content_policy_row(row, value)] + if pg.enabled(): + async with pg.transaction() as conn: + await lock_postgres(conn, policy) + await check_postgres(conn, policy) + else: + await check_clickhouse(ch, policy) + + +def _raw_content_values(row: Mapping[str, Any]) -> list[Any]: + values = [ + row[c] + for c in ("content", "caption", "source_url", "source_local_path") + if row.get(c) not in (None, "") + ] + metadata = row.get("metadata") + if isinstance(metadata, str): + try: + metadata = json.loads(metadata) + except (ValueError, TypeError): + pass + if metadata not in (None, "", {}, []): + values.append(metadata) + return values + + +async def insert_content_rows(ch: _CHClient, table: str, rows: Sequence[Mapping[str, Any]]) -> None: + """Refuse raw content under the same policy fences as canonical memory.""" + if table not in {"loom.memory_evidence", "loom.memory_chunks"}: + raise ValueError("unsupported refusal-protected content table") + policy = [content_policy_row(row, _raw_content_values(row)) for row in rows] + if policy: + async with publication(ch, policy): + if table == "loom.memory_evidence": + await ch.insert( + "loom.memory_evidence", + [dict(r) for r in rows], + settings=batched_insert_settings(), + ) + else: + await ch.insert( + "loom.memory_chunks", + [dict(r) for r in rows], + settings=committed_insert_settings(), + ) + + +async def _embedding_sources( + ch: _CHClient, rows: Sequence[Mapping[str, Any]] +) -> tuple[list[Mapping[str, Any]], list[Mapping[str, Any]]]: + from server.memory.store import pg_memory + from server.memory.store.reads import memory_rows_by_identity_keys + from server.memory.store.rows import build_memory_version_row, identity_key + + keys = [identity_key(r) for r in rows] + if pg.enabled(): + heads = await pg_memory.heads_by_keys(keys, include_tombstoned=True, durable=True) + else: + found = ( + await memory_rows_by_identity_keys( + keys, ch=ch, sequential=True, include_tombstoned=True + ) + if keys + else {} + ) + heads = {k: build_memory_version_row(h) for k, h in found.items()} + accepted: list[Mapping[str, Any]] = [] + sources: list[Mapping[str, Any]] = [] + for row in rows: + head = heads.get(identity_key(row)) + if row.get("tombstoned_at") is not None: + accepted.append(row) + if head is not None: + sources.append(head) + continue + if ( + head is not None + and head.get("tombstoned_at") is None + and hash_text(row.get("content_sha256")) == sha256_value(served_row_value(head)) + and row.get("status", "active") == head.get("status", "active") + ): + accepted.append(row) + sources.append(head) + return accepted, sources + + +def _publication_values( + sources: Sequence[Mapping[str, Any]], admitted: Sequence[Mapping[str, Any]], *, embeddings: bool +) -> list[Mapping[str, Any]]: + from server.memory.store.rows import identity_key + + admitted_keys = {identity_key(r) for r in admitted} + values = [s for s in sources if not embeddings or identity_key(s) in admitted_keys] + if embeddings: + values.extend( + content_policy_row(row, row["content_excerpt"]) + for row in admitted + if row.get("tombstoned_at") is None and row.get("content_excerpt") + ) + return values + + +@asynccontextmanager +async def publication( + ch: _CHClient, + rows: Sequence[Mapping[str, Any]], + *, + embeddings: bool = False, + committed: bool = False, + retired: list[dict[str, Any]] | None = None, +) -> AsyncIterator[Sequence[Mapping[str, Any]]]: + """Hold the policy admission through the complete memory/vector send.""" + async with lease_writes.fenced_many( + fence_names(rows), ch=_lease_client(ch), wait_s=_admission_wait_s() + ) as mark: + if pg.enabled(): + from server.memory.store import pg_memory + from server.memory.store.rows import identity_key + + # A durable read must finish before any bytes it returned can be published. + admitted, sources = await _embedding_sources(ch, rows) if embeddings else (rows, rows) + async with pg.transaction() as conn: + await lock_postgres(conn, rows) + if embeddings: + current = await pg_memory.heads_on_connection( + conn, [identity_key(r) for r in rows], lock=True + ) + unchanged = { + identity_key(r) for r in sources if current.get(identity_key(r)) == r + } + admitted = [ + r + for r in admitted + if identity_key(r) in unchanged + or ( + r.get("tombstoned_at") is not None + and identity_key(r) not in current + and identity_key(r) not in {identity_key(s) for s in sources} + ) + ] + sources = [r for r in sources if identity_key(r) in unchanged] + if embeddings: + admitted = await _embedding_retirements( + ch, admitted, sources, mark, retired, conn=conn + ) + await check_postgres( + conn, + _publication_values(sources, admitted, embeddings=embeddings), + # Authority commits enforce point refusal before becoming history. + erasure_only=committed and not embeddings, + ) + mark(True) + yield admitted + mark(False) + else: + admitted, sources = await _embedding_sources(ch, rows) if embeddings else (rows, rows) + if embeddings: + admitted = await _embedding_retirements(ch, admitted, sources, mark, retired) + await check_clickhouse( + ch, _publication_values(sources, admitted, embeddings=embeddings) + ) + mark(True) + yield admitted + mark(False) + + +async def _embedding_retirements( + ch: _CHClient, + rows: Sequence[Mapping[str, Any]], + sources: Sequence[Mapping[str, Any]], + mark: Callable[[bool], None], + retired: list[dict[str, Any]] | None, + *, + conn: Any = None, +) -> list[Mapping[str, Any]]: + from server.memory.domain.redaction_lineage import is_content_hash + from server.memory.store.embeddings import _bare_tombstone + from server.memory.store.rows import identity_key + + heads = {identity_key(r): r for r in sources} + accepted: list[Mapping[str, Any]] = [] + for row in rows: + if row.get("tombstoned_at") is None: + accepted.append(row) + continue + head = heads.get(identity_key(row)) + values = [served_row_value(head)] if head is not None else [] + if row.get("content_excerpt"): + values.append(row["content_excerpt"]) + inspected = refusals.inspect(values) + digest = hash_text(row.get("content_sha256")) + if is_content_hash(digest): + inspected[0].add(("value", 0, digest)) + key = (str(row["org"]), str(row["namespace"]), str(row.get("user_id") or "-")) + with sequential_reads(): + erased = ( + await _matches_postgres(conn, key, [inspected], erasure_only=True) + if conn is not None + else await _matches_clickhouse(ch, key, [inspected], erasure_only=True) + ) + if head is None or erased: + mark(True) + await _delete_old_vector(ch, row) + mark(False) + if retired is not None: + retired.append(dict(row)) + continue + refused = ( + await _matches_postgres(conn, key, [inspected]) + if conn is not None + else await _matches_clickhouse(ch, key, [inspected]) + ) + accepted.append( + _bare_tombstone(dict(row), int(row["version"]), naive_utc_now()) if refused else row + ) + return accepted + + +async def _delete_old_vector(ch: _CHClient, row: Mapping[str, Any]) -> None: + """Clean an orphan without introducing a row after a physical erasure.""" + set_query_tag("memory.retire_orphan_vector") + await ch.execute( + "DELETE FROM loom.memory_embeddings WHERE org={org:String} " + "AND namespace={namespace:String} AND agent={agent:String} AND user_id={user_id:String} " + "AND session_id={session_id:String} AND scope={scope:String} " + "AND embedding_model={embedding_model:String} AND memory_key={memory_key:String} " + "AND version < {version:UInt64}", + params={ + c: row[c] + for c in ( + "org", + "namespace", + "agent", + "user_id", + "session_id", + "scope", + "embedding_model", + "memory_key", + "version", + ) + }, + settings={"lightweight_deletes_sync": 2}, + ) diff --git a/server/memory/store/shared.py b/server/memory/store/shared.py index 475bb95872..d70c4bcdf2 100644 --- a/server/memory/store/shared.py +++ b/server/memory/store/shared.py @@ -12,6 +12,7 @@ from __future__ import annotations +import json from datetime import datetime from typing import Any @@ -63,6 +64,16 @@ async def insert_shared_row( f"missing={missing} unknown={unknown}" ) ch = ch or _default_client() + from server.memory.store.refusals import assert_shared_publication, shared_retirement + + if row.get("tombstoned_at") is not None: + async with shared_retirement(ch, row) as publish: + if publish: + await ch.insert( + "loom.shared", [row], settings={"async_insert": 0, "wait_for_async_insert": 1} + ) + return + assert_shared_publication(row) await ch.insert( "loom.shared", [row], @@ -86,6 +97,9 @@ async def insert_shared_embedding( from server.memory.store.rows import build_embedding_row ch = ch or _default_client() + from server.memory.store.refusals import assert_shared_publication + + assert_shared_publication(row, vector=True) await ch.insert( "loom.memory_embeddings", [build_embedding_row(**row)], @@ -93,6 +107,94 @@ async def insert_shared_embedding( ) +async def shared_embedding_source( + row: dict[str, Any], *, ch: _CHClient +) -> tuple[dict[str, Any], Any, set[str]] | None: + """The complete approved source behind this shared cache row, if it still agrees.""" + from server.core import pg + from server.core.hashing import served_row_value, sha256_value + from server.memory.domain.excerpt import build_excerpt + from server.memory.domain.promotion_source import promotion_keys, promotion_users + from server.memory.domain.temporal_sql import SERVED_STATUSES + from server.memory.store import pg_memory + from server.memory.store.reads import memory_rows_by_identity_keys + from server.memory.store.rows import _hydrate + + shared = await fetch_shared(row["org"], row["namespace"], row["memory_key"], strong=True, ch=ch) + if not shared or shared.get("tombstoned_at") is not None: + return None + entries = shared.get("promoted_from") or [] + users = promotion_users(entries, row["org"], row["namespace"]) + if users is None: + return None + keys = promotion_keys(entries, row["org"], row["namespace"]) + if keys is None: + return None + if pg.enabled(): + raw_heads = await pg_memory.heads_by_keys(keys, include_tombstoned=True, durable=True) + heads = {key: _hydrate(head) for key, head in raw_heads.items()} + else: + heads = await memory_rows_by_identity_keys( + keys, include_tombstoned=True, sequential=True, ch=ch + ) + digest = sha256_value(served_row_value(shared)) + for key in keys: + source = heads.get(key) + if source is None or sha256_value(source.value) != digest: + continue + if ( + source.status not in SERVED_STATUSES + or source.requires_review + or source.tombstoned_at + or source.sensitivity == "high" + ): + continue + if row.get("content_excerpt") == build_excerpt( + json.dumps(source.value, default=str), memory_key=source.memory_key + ): + return shared, source, users + return None + + +async def repair_shared_embeddings( + rows: list[dict[str, Any]], *, ch: _CHClient +) -> list[dict[str, Any]]: + """Repair a shared mirror only while its full promoted source still qualifies.""" + from server.core.hashing import sha256_value + from server.memory.domain.trust import verification_state + from server.memory.store.refusals import source_publication + + written = [] + for row in rows: + proof = await shared_embedding_source(row, ch=ch) + if proof is None: + continue + shared, source, users = proof + digest = sha256_value(source.value) + async with source_publication(ch, source, row["memory_key"], refusal_users=users) as mark: + current = await fetch_shared( + row["org"], row["namespace"], row["memory_key"], strong=True, ch=ch + ) + if current != shared: + continue + mark(True) + await insert_shared_embedding( + { + **row, + "content_sha256": digest, + "source_class": source.source_class, + "verified": int( + verification_state(source.verified_by, source.verified_sha256, digest) + in ("human", "process") + ), + }, + ch=ch, + ) + mark(False) + written.append(row) + return written + + async def fetch_shared( org: str, namespace: str, diff --git a/server/memory/store/source_review.py b/server/memory/store/source_review.py deleted file mode 100644 index 7fb4134110..0000000000 --- a/server/memory/store/source_review.py +++ /dev/null @@ -1,52 +0,0 @@ -"""Keep promotion tied to the reviewed source through its shared writes.""" - -from __future__ import annotations - -from collections.abc import AsyncIterator, Callable -from contextlib import asynccontextmanager - -from server.core import lease_writes, pg -from server.core.errors import AuthInsufficientError -from server.core.repo_base import _default_client -from server.identity.domain.read_scope import WHOLE_NAMESPACE -from server.memory.domain.proposal_review import pending_review -from server.memory.store import pg_memory, write_fence -from server.memory.store.rows import Identity, MemoryRow, identity_key - - -def require_reviewed(source: MemoryRow) -> None: - if pending_review(source): - raise AuthInsufficientError("An unreviewed proposal cannot be promoted") - - -@asynccontextmanager -async def publication(source: MemoryRow) -> AsyncIterator[Callable[[bool], None]]: - import server.memory.store as memory_repo - - require_reviewed(source) - ch = _default_client() - async with lease_writes.fenced_many( - [write_fence.lease_name(identity_key(source))], - ch=write_fence._lease_client(ch), - wait_s=write_fence._admission_wait_s(), - ) as mark: - if pg.enabled(): - async with pg_memory.locked_source(source): - yield mark - else: - current = await memory_repo.get_by_key( - Identity( - org=source.org, - namespace=source.namespace, - agent=source.agent, - user_id=source.user_id, - session_id=source.session_id, - ), - source.memory_key, - scope=WHOLE_NAMESPACE, - strong=True, - fresh=True, - ) - if not pg_memory.unchanged(source, current): - raise pg_memory.AdmissionContendedError("The promotion's source changed") - yield mark diff --git a/server/memory/store/summary_pages.py b/server/memory/store/summary_pages.py new file mode 100644 index 0000000000..71d1cdb74d --- /dev/null +++ b/server/memory/store/summary_pages.py @@ -0,0 +1,75 @@ +"""Fill digest listings from eligible sources rather than raw candidate limits.""" + +from __future__ import annotations + +from typing import Any, Literal + +from server.core.ch import set_query_tag +from server.core.errors import BackendUnavailableError +from server.core.repo_base import _CHClient +from server.memory.store import summary_sources +from server.settings import settings + +Order = Literal["oldest", "newest", "session"] + + +def _ordering(order: Order) -> tuple[str, str, str]: + if order == "session": + return "session_id", "session_id, version DESC, agent, user_id, embedding_model", ">" + date = "d" if order == "newest" else "created_at" + key = f"({date}, session_id, agent, user_id, embedding_model)" + return key, key + (" DESC" if order == "newest" else " ASC"), "<" if order == "newest" else ">" + + +async def verified_summary_page( + ch: _CHClient, + *, + org: str, + namespace: str, + where: str, + params: dict[str, Any], + limit: int, + order: Order = "oldest", +) -> list[dict[str, Any]]: + """Return up to limit+1 proven rows; refuse an incomplete bounded scan.""" + key, ordering, comparison = _ordering(order) + after: Any = None + found: list[dict[str, Any]] = [] + for page in range(settings.memory_digest_listing_max_pages): + set_query_tag("memory.verified_digest_page") + size = settings.memory_digest_listing_page_size + if page == 0: + size = min(size, limit + 1) + cursor_type = ( + "String" if order == "session" else "Tuple(DateTime,String,String,String,String)" + ) + cursor = f" AND {key} {comparison} {{after:{cursor_type}}}" if after is not None else "" + rows = await ch.query_all( + "SELECT agent, user_id, session_id, embedding_model, summary_text, key_facts, " + "derived_from, valid_at, created_at, version, length(key_facts) AS kf, " + "length(derived_from) AS df, coalesce(valid_at, created_at) AS d " + "FROM loom.session_summaries FINAL WHERE " + + where + + cursor + + " ORDER BY " + + ordering + + (" LIMIT 1 BY session_id" if order == "session" else "") + + " LIMIT {lim:UInt32}", + params={**params, "lim": size, "after": after}, + ) + found.extend( + await summary_sources.current_summary_sources(ch, rows, org=org, namespace=namespace) + ) + if len(found) > limit or len(rows) < size: + return found[: limit + 1] + last = rows[-1] + after = ( + str(last["session_id"]) + if order == "session" + else ( + last["d" if order == "newest" else "created_at"], + *(str(last.get(c, "-")) for c in ("session_id", "agent", "user_id")), + str(last.get("embedding_model") or ""), + ) + ) + raise BackendUnavailableError("Digest source verification exceeded its scan budget") diff --git a/server/memory/store/summary_sources.py b/server/memory/store/summary_sources.py index 2be31f0c33..7748cbb238 100644 --- a/server/memory/store/summary_sources.py +++ b/server/memory/store/summary_sources.py @@ -1,24 +1,97 @@ -"""Source-state checks for the session-summary shortlist.""" +"""Verify every digest byte against its authoritative, currently served sources.""" from __future__ import annotations +from collections import defaultdict +from collections.abc import Mapping +from contextlib import AsyncExitStack from typing import Any -from server.core.repo_base import _CHClient, query_all_sliced -from server.memory.domain.temporal_sql import ( - as_of_predicate, - status_predicate, - tombstone_predicate, -) +from server.core import lease_writes, pg +from server.core.ch import sequential_reads +from server.core.datetimes import naive_utc_now +from server.core.repo_base import _CHClient +from server.memory.domain.sensitivity import value_reads_high +from server.memory.domain.summary_provenance import render_summary, source_text +from server.memory.store import pg_memory +from server.memory.store.reads import memory_rows_by_identity_keys +from server.memory.store.rows import IdentityKey, MemoryRow, _hydrate -SourceKey = tuple[str, str, str, str] +def _identity(row: Mapping[str, Any]) -> tuple[str, str, str]: + return tuple(str(row.get(c) or "-") for c in ("agent", "user_id", "session_id")) # type: ignore[return-value] -def _sources(row: dict[str, Any]) -> list[SourceKey]: - agent, user, session = ( - str(row.get(axis) or "-") for axis in ("agent", "user_id", "session_id") + +def _complete(row: dict[str, Any]) -> bool: + keys, facts = row.get("derived_from"), row.get("key_facts") + return bool( + isinstance(keys, list) + and isinstance(facts, list) + and keys + and len(keys) == len(facts) + and all(isinstance(k, str) and k for k in keys) + and len(set(keys)) == len(keys) + and all(isinstance(f, str) and f for f in facts) + and row.get("summary_text") == render_summary(facts) + and not value_reads_high(row["summary_text"]) + ) + + +async def _source_heads( + ch: _CHClient, + rows: list[dict[str, Any]], + *, + org: str, + namespace: str, + durable: bool = False, + conn: Any = None, +) -> dict[IdentityKey, MemoryRow]: + candidates = [r for r in rows if _complete(r)] + keys = sorted( + {(org, namespace, *_identity(r), key) for r in candidates for key in r["derived_from"]} ) - return [(agent, user, session, str(key)) for key in row.get("derived_from", []) if key] + if not keys: + return {} + # A JSON string may use two UTF-16 escapes per code point, plus its quotes. + cap = 12 * max(len(f) for r in candidates for f in r["key_facts"]) + 2 + if pg.enabled(): + if conn is not None: + found = await pg_memory.heads_on_connection(conn, keys, lock=True, max_value_chars=cap) + else: + found = await pg_memory.heads_by_keys(keys, durable=durable, max_value_chars=cap) + return {key: _hydrate(row) for key, row in found.items()} + with sequential_reads(): + return await memory_rows_by_identity_keys(keys, ch=ch, sequential=True, max_value_chars=cap) + + +def _verified( + rows: list[dict[str, Any]], + heads: Mapping[IdentityKey, MemoryRow], + *, + org: str, + namespace: str, + as_of: Any = None, + as_of_basis: str = "valid_time", +) -> list[dict[str, Any]]: + now = naive_utc_now() + result = [] + for row in rows: + if not _complete(row): + continue + texts = [] + for key in row["derived_from"]: + source = heads.get((org, namespace, *_identity(row), key)) + text = ( + source_text(source, now=now, as_of=as_of, as_of_basis=as_of_basis) + if source + else None + ) + if text is None: + break + texts.append(text) + if texts == row["key_facts"]: + result.append(row) + return result async def current_summary_sources( @@ -30,55 +103,77 @@ async def current_summary_sources( as_of: Any = None, as_of_basis: str = "valid_time", ) -> list[dict[str, Any]]: - """Withhold digests of known unserved or rewritten sources, within the full identity. - - A digest has no source revision hashes, so a later version conservatively invalidates - it. Missing lineage remains session evidence; it is not proof of a current fact. - """ - sources = sorted({key for row in rows for key in _sources(row)}) - if not sources: - return rows - replaying = as_of is not None - state = [ - tombstone_predicate(as_of_basis, replaying=replaying), - status_predicate(as_of_basis, replaying=replaying), - "sensitivity != 'high'", - "(expires_at IS NULL OR expires_at > now())", - "(valid_to IS NULL OR valid_to > now())", - ] - params: dict[str, Any] = {"org": org, "ns": namespace, "sources": sources} - if replaying: - params["as_of"] = as_of - state.append(as_of_predicate(as_of_basis)) - parents = await query_all_sliced( - ch, - f""" - SELECT agent, if(user_id = '', '-', user_id) AS source_user, - if(session_id = '', '-', session_id) AS source_session, memory_key, - version, {" AND ".join(state)} AS source_served - FROM loom.memory FINAL - WHERE org = {{org:String}} AND namespace = {{ns:String}} - AND (agent, if(user_id = '', '-', user_id), if(session_id = '', '-', session_id), - memory_key) IN {{sources:Array(Tuple(String,String,String,String))}} - """, - params, - "sources", + """Withhold missing, unverifiable and rewritten lineage, including legacy digests.""" + heads = await _source_heads(ch, rows, org=org, namespace=namespace) + return _verified( + rows, heads, org=org, namespace=namespace, as_of=as_of, as_of_basis=as_of_basis ) - heads: dict[SourceKey, tuple[bool, int]] = {} - for p in parents: - key = ( - str(p["agent"]), - str(p["source_user"]), - str(p["source_session"]), - str(p["memory_key"]), - ) - served, version = heads.get(key, (True, 0)) - heads[key] = (served and bool(p["source_served"]), max(version, int(p["version"]))) - return [ - row - for row in rows - if all( - key in heads and heads[key][0] and heads[key][1] <= int(row["version"]) - for key in _sources(row) + + +async def verified_summary_batch(ch: _CHClient, rows: list[dict[str, Any]]) -> list[dict[str, Any]]: + """Verify a worker batch without mixing source identities across tenants.""" + groups: dict[tuple[str, str], list[dict[str, Any]]] = defaultdict(list) + for row in rows: + if row.get("org") and row.get("namespace"): + groups[(str(row["org"]), str(row["namespace"]))].append(row) + verified: set[int] = set() + for (org, namespace), batch in groups.items(): + verified.update( + id(r) for r in await current_summary_sources(ch, batch, org=org, namespace=namespace) ) - ] + return [row for row in rows if id(row) in verified] + + +async def insert_verified_summaries(ch: _CHClient, rows: list[dict[str, Any]]) -> int: + """Publish only current proven text, fenced against deletion and erasure.""" + from server.memory.store import refusals + + groups: dict[tuple[str, str], list[dict[str, Any]]] = defaultdict(list) + for row in rows: + groups[(str(row["org"]), str(row["namespace"]))].append(row) + written = 0 + for (org, namespace), batch in groups.items(): + policy = [ + refusals.content_policy_row(r, [r["summary_text"], *r["key_facts"]]) for r in batch + ] + async with lease_writes.fenced_many( + refusals.fence_names(policy), + ch=refusals._lease_client(ch), + wait_s=refusals._admission_wait_s(), + ) as mark: + async with AsyncExitStack() as stack: + if pg.enabled(): + durable = await _source_heads( + ch, batch, org=org, namespace=namespace, durable=True + ) + conn = await stack.enter_async_context(pg.transaction()) + await refusals.lock_postgres(conn, policy) + current = await _source_heads( + ch, batch, org=org, namespace=namespace, conn=conn + ) + heads = { + key: row + for key, row in current.items() + if key in durable and pg_memory.unchanged(durable[key], row) + } + else: + heads = await _source_heads(ch, batch, org=org, namespace=namespace) + admitted = _verified(batch, heads, org=org, namespace=namespace) + admitted_policy = [ + refusals.content_policy_row(r, [r["summary_text"], *r["key_facts"]]) + for r in admitted + ] + if pg.enabled(): + await refusals.check_postgres(conn, admitted_policy) + else: + await refusals.check_clickhouse(ch, admitted_policy) + if admitted: + mark(True) + await ch.insert( + "loom.session_summaries", + admitted, + settings={"async_insert": 0, "wait_for_async_insert": 1}, + ) + mark(False) + written += len(admitted) + return written diff --git a/server/memory/store/write_fence.py b/server/memory/store/write_fence.py index 0ec927f9a2..ced15c7ba2 100644 --- a/server/memory/store/write_fence.py +++ b/server/memory/store/write_fence.py @@ -36,6 +36,7 @@ async def checked_write( *, expected_heads: Mapping[IdentityKey, MemoryRow | None] | None, ch: _CHClient, + refuse_values: bool = False, ) -> AsyncIterator[None]: """Every value writer takes these fences too, closing the check/insert window. @@ -45,7 +46,13 @@ async def checked_write( keys = {identity_key(r) for r in rows} if expected_heads is not None and set(expected_heads) != keys: raise ValueError("expected heads must name exactly the inserted keys") - names = [lease_name(key) for key in sorted(keys)] + from server.memory.store import refusals + + names = sorted( + set( + [lease_name(key) for key in keys] + refusals.fence_names(rows, forgetting=refuse_values) + ) + ) async with lease_writes.fenced_many( names, ch=_lease_client(ch), wait_s=_admission_wait_s() ) as mark: @@ -60,6 +67,9 @@ async def checked_write( for key, head in expected_heads.items() ): raise AdmissionContendedError("the memory changed while it was read") + await refusals.check_clickhouse(ch, rows) mark(True) + if refuse_values: + await refusals.record_clickhouse(ch, rows) yield mark(False) diff --git a/server/memory/workers/embedding.py b/server/memory/workers/embedding.py index 5dfac20a09..373898bd36 100644 --- a/server/memory/workers/embedding.py +++ b/server/memory/workers/embedding.py @@ -289,13 +289,19 @@ async def _stubs_still_served( rows: list[dict[str, Any]], stub_embedding: list[float], now: datetime ) -> tuple[list[dict[str, Any]], list[dict[str, Any]]]: """The stubs whose head may still keep a vector, and tombstones (two above the - stub, its excerpt cleared) for the rest. Shared-scope stubs have no head of - their own and are kept.""" + stub, its excerpt cleared) for the rest. Shared stubs require their promoted + source proof before any provider work; an unbacked cache row is withheld.""" + from server.core.ch import get_client from server.memory.store import read_your_writes_enabled + from server.memory.store.shared import shared_embedding_source + shared = { + id(row) + for row in rows + if row.get("scope") == "shared" + and await shared_embedding_source(row, ch=get_client()) is not None + } own = [r for r in rows if str(r.get("scope") or "") != "shared"] - if not own: - return rows, [] sequential = read_your_writes_enabled() heads = await memory_repo.head_sensitivities( sorted({memory_repo.identity_key(r) for r in own}), sequential=sequential @@ -303,7 +309,7 @@ async def _stubs_still_served( def served(r: dict[str, Any]) -> bool: if str(r.get("scope") or "") == "shared": - return True + return id(r) in shared head = heads.get(memory_repo.identity_key(r)) return head != "high" and (head is not None or not sequential) @@ -317,7 +323,7 @@ def served(r: dict[str, Any]) -> bool: embedding=stub_embedding, tombstoned_at=now, ) - for r in rows + for r in own if not served(r) ] return keep, dropped @@ -407,6 +413,29 @@ def _outcome(result: EmbeddingWorkerResult, *, idle: bool, rows_written: int) -> } +def _record_fill_outcomes( + replacements: list[dict[str, Any]], retries: list[dict[str, Any]], result: EmbeddingWorkerResult +) -> None: + result.embedded = [ + {"namespace": row["namespace"], "memory_key": row["memory_key"]} + for row in replacements + if row.get("tombstoned_at") is None and row.get("embedding_dim", 0) > 0 + ] + result.failed, result.poisoned = [], [] + for row in retries: + if row.get("embed_last_error") == HELD: + continue + target = result.poisoned if row.get("embed_failed_at") is not None else result.failed + target.append( + { + "namespace": row["namespace"], + "memory_key": row["memory_key"], + "attempts": row.get("embed_attempts", 0), + "error": row.get("embed_last_error", ""), + } + ) + + async def run_pass( cfg: EmbeddingWorkerConfig | None = None, ) -> EmbeddingWorkerResult: @@ -448,7 +477,7 @@ async def run_pass( since_seconds=_orphan_scan_window(), ) if orphans: - await memory_repo.insert_embedding_rows( + reconciled = await memory_repo.insert_embedding_rows( [ memory_repo.build_embedding_row( **{k: v for k, v in o.items() if k != "created_at"}, @@ -465,12 +494,12 @@ async def run_pass( ] ) result.reconciled = [ - {"namespace": o["namespace"], "memory_key": o["memory_key"]} for o in orphans + {"namespace": o["namespace"], "memory_key": o["memory_key"]} for o in reconciled ] log.warning( "embedding worker: reconciled %d memories that had NO embedding " "row (unsearchable until now)", - len(orphans), + len(reconciled), ) except Exception as e: # noqa: BLE001 — repair is additive; never break the pass # The pass goes on, the supervisor hears about it. @@ -493,7 +522,7 @@ async def run_pass( # to the embedding provider: its excerpt may hold the high text. It is retired. rows, dropped = await _stubs_still_served(rows, stub_embedding, now) if dropped: - await memory_repo.insert_embedding_rows(dropped) + dropped = await memory_repo.insert_embedding_rows(dropped) writers = await memory_writers(rows) if rows and identity.reads_writer() else {} if rows: embedded = await _embed_rows(embedder, rows, writers) @@ -552,31 +581,18 @@ async def run_pass( ) continue replacements.extend(_replacement_rows(r, vec, embedder, stub_embedding, now)) - result.embedded.append( - { - "namespace": r["namespace"], - "memory_key": r["memory_key"], - } - ) - # Entities can advance between the stub read and this insert - # (the entity extractor's apply_entities_batch re-emits version rows); - # a re-emit built from the stale snapshot would resurrect the - # old list under FINAL. - # Re-read at insert time to shrink that window from - # minutes (embed round-trips) to milliseconds. + # Refresh mutable facets, then report only acknowledged publications. await _refresh_entities(replacements + retries) - await memory_repo.insert_embedding_rows(replacements) - # A stub drawn from a read before a 'high' write landed can sit above - # its fence: the fill re-checks its heads like every vector copy. + replacements = await memory_repo.insert_embedding_rows(replacements) await memory_repo.retire_copies_under_unembedded_heads(replacements) - # A fill sits at its stub's version + 1, usually far under the BM25 watermark. await memory_repo.index_late_rows(replacements) - await memory_repo.insert_embedding_rows(retries) + retries = await memory_repo.insert_embedding_rows(retries) + _record_fill_outcomes(replacements, retries, result) rec.outcome = _outcome( result, idle=not rows and not result.reconciled, - rows_written=len(replacements) + len(retries) + len(result.reconciled), + rows_written=len(replacements) + len(retries) + len(result.reconciled) + len(dropped), ) log.info( diff --git a/server/memory/workers/usage_fold.py b/server/memory/workers/usage_fold.py index e6c78c2034..9d17c28aae 100644 --- a/server/memory/workers/usage_fold.py +++ b/server/memory/workers/usage_fold.py @@ -111,7 +111,11 @@ async def query_one( ) -> dict[str, Any] | None: ... async def query_all( - self, sql: str, params: dict[str, Any] | None = None + self, + sql: str, + params: dict[str, Any] | None = None, + *, + settings: dict[str, Any] | None = None, ) -> list[dict[str, Any]]: ... async def execute( diff --git a/server/observability/langfuse_prompts.lock.json b/server/observability/langfuse_prompts.lock.json index a107fc7b03..0eb1c0d9b8 100644 --- a/server/observability/langfuse_prompts.lock.json +++ b/server/observability/langfuse_prompts.lock.json @@ -38,10 +38,6 @@ "name": "loom-period-digest", "version": 1 }, - "84e51221d19f": { - "name": "loom-session-summary", - "version": 7 - }, "87e36f72aac0": { "name": "loom-code-query-expand", "version": 2 diff --git a/server/settings/catalog.py b/server/settings/catalog.py index aa657c59b5..c775f0a266 100644 --- a/server/settings/catalog.py +++ b/server/settings/catalog.py @@ -2780,9 +2780,13 @@ class MemorySettings(BaseModel): # memory.list rows per call for a token below writer on the namespace; it # pages with `offset` (wire cap stays 1000). 0 = no viewer cap. memory_list_viewer_limit: int = 100 - # Transcript characters the session-digest call reads (~4 chars a token); - # 0 = the built-in 3000. + # Characters of complete source facts included in an extractive session digest; + # 0 = the built-in 3000. Facts are never clipped to fit. memory_session_digest_max_input_chars: int = 3000 + # Raw digest candidates per query while filling listing, outline and export rows. + memory_digest_listing_page_size: int = Field(default=256, ge=1, le=2000) + # Queries allowed to find enough proven digests; exhaustion answers BACKEND_UNAVAILABLE. + memory_digest_listing_max_pages: int = Field(default=256, ge=1) # content_excerpt ceiling for answer-bearing facts (question_topic set) # when memory_capture_structured_answers is on, so an enumerated answer # is not clipped mid-list; every other fact keeps EXCERPT_LIMIT (1,200 chars). @@ -4443,7 +4447,7 @@ class ObservabilitySettings(BaseModel): # "off" records none (the default; consent is given, not withdrawn); "query" only # the legs whose prompt is the user's own search text (hyde, query_rewrite, # enrichment, …); "all" also the legs carrying retrieved memories or conversation - # turns (inline_rerank, iterative_judge, extract, session_summary, …). + # turns (inline_rerank, iterative_judge, extract, …). # A namespace's `capture_generation_io` overrides it (false ⇒ off, true ⇒ # all, NULL ⇒ this). It also gates loom.search_trace_log.query. Token # counts, cost, latency and quality scores are never gated. @@ -6793,8 +6797,6 @@ class TrackerSettings(BaseModel): # session.brief: one window's revision (about 600-1,500 tokens); a cut-off answer is # salvaged key by key. "session_brief_revision": 3000, - # The session summary: one to three dated sentences. - "session_summary": 160, # memory.ask: the judge of an answer against its evidence. "ask_consistency": 200, # memory.ask: the split of a question into sub-questions. diff --git a/server/wiring.py b/server/wiring.py index bf716e334a..9918b6d7f8 100644 --- a/server/wiring.py +++ b/server/wiring.py @@ -21,11 +21,12 @@ def wire() -> None: from server.code.domain.index import repos from server.code.domain.index.compiler_tiers import TIERS from server.core.ch import add_insert_observer + from server.erasure.store.consent import ConsentPolicy, use_policy from server.identity.app.directory import IdentityDirectory from server.knowledge.domain.live_chunks import LIVE_CHUNK_SQL from server.memory.app import provenance, recent_writes from server.memory.domain.memory_types import NON_SEARCHABLE_TYPES - from server.memory.store import pg_publish + from server.memory.store import pg_publish, refusals from server.memory.store.usage import AGENT_LABEL_EVENT_TYPES from server.observability.budget_notifications import use_observer from server.observability.state_inputs import StateInputs, use_state_inputs @@ -35,6 +36,16 @@ def wire() -> None: use_directory(IdentityDirectory()) use_observer(schedule) pg_publish.use_writer(provenance.writer_actor) + refusals.use_writer(provenance.writer_actor) + use_policy( + ConsentPolicy( + fence_names=lambda org, kind, value: refusals.erasure_fence_names(org, kind, value), + lease_client=lambda ch: refusals._lease_client(ch), + lock_postgres=lambda *args, **kwargs: refusals.lock_postgres(*args, **kwargs), + record_clickhouse=lambda *args, **kwargs: refusals.record_clickhouse(*args, **kwargs), + record_postgres=lambda *args, **kwargs: refusals.record_postgres(*args, **kwargs), + ) + ) use_state_inputs( StateInputs( live_chunk_sql=LIVE_CHUNK_SQL, diff --git a/tests/analyzer/cases/erasure.py b/tests/analyzer/cases/erasure.py index e3994a49f5..881361356e 100644 --- a/tests/analyzer/cases/erasure.py +++ b/tests/analyzer/cases/erasure.py @@ -170,6 +170,14 @@ async def _exports(ch: RecordingCH, c: dict[str, Any]) -> None: await erasure_exports.export_audit_by_subject_substring(ORG, "alice", limit=100, ch=ch) +async def _consent_capture(ch: RecordingCH, c: dict[str, Any]) -> None: + from server.erasure.store import refusals + + for kind, value in (("namespace", "n"), ("user_id", "alice")): + for table, projection in refusals._sources(): + await refusals._capture_source(ch, table, projection, ORG, kind, value) + + def cases() -> list[Case]: steps: list[Case] = [] for module in STEP_MODULES: @@ -199,6 +207,7 @@ def cases() -> list[Case]: ] return [ *steps, + Case("erasure.consent_capture", "server.erasure.store.refusals", _consent_capture, {}), Case( "erasure.census_and_ledger", CENSUS_MODULE, diff --git a/tests/analyzer/cases/legs.py b/tests/analyzer/cases/legs.py index e0e78e805b..ab7e017aa8 100644 --- a/tests/analyzer/cases/legs.py +++ b/tests/analyzer/cases/legs.py @@ -593,7 +593,7 @@ async def _entity_write(ch: RecordingCH, c: dict[str, Any]) -> None: async def _deep_evidence(ch: RecordingCH, c: dict[str, Any]) -> None: - from server.memory.app.deep_evidence import fetch_raw_evidence, fetch_session_turns + from server.memory.app.deep_evidence import fetch_raw_evidence await fetch_raw_evidence( org=ORG, @@ -603,15 +603,6 @@ async def _deep_evidence(ch: RecordingCH, c: dict[str, Any]) -> None: max_chars_total=4000, ch=ch, ) - await fetch_session_turns( - org=ORG, - namespace=NS, - session_ids=["s1"], - question="q", - max_chars_per_session=800, - max_chars_total=4000, - ch=ch, - ) async def _stale_links(ch: RecordingCH, c: dict[str, Any]) -> None: diff --git a/tests/analyzer/cases/workers.py b/tests/analyzer/cases/workers.py index 075661a0b2..b71bea0b03 100644 --- a/tests/analyzer/cases/workers.py +++ b/tests/analyzer/cases/workers.py @@ -275,6 +275,38 @@ async def _resolver_cluster(ch: RecordingCH, c: dict[str, Any]) -> None: ) +async def _summary_pages(ch: RecordingCH, c: dict[str, Any]) -> None: + from server.memory.store.summary_pages import verified_summary_page + + await verified_summary_page( + ch, + org=ORG, + namespace=NS, + where="org={org:String} AND namespace={ns:String}", + params={"org": ORG, "ns": NS}, + limit=1, + order=c["order"], + ) + + +def _summary_page_rows(sql: str, params: dict[str, Any]) -> list[dict[str, Any]]: + if params.get("after") is not None: + return [] + return [ + dict( + agent="bot", + user_id="alice", + session_id="s1", + embedding_model="fake", + created_at=datetime(2026, 1, 1), + d=datetime(2026, 1, 1), + summary_text="legacy", + key_facts=[], + derived_from=[], + ) + ] * params["lim"] + + async def _period_sources(ch: RecordingCH, c: dict[str, Any]) -> None: """The facts a period's digests compressed: their ids and entities.""" from types import SimpleNamespace @@ -292,7 +324,10 @@ async def _period_keys(ch: RecordingCH, c: dict[str, Any]) -> None: from server.memory.app import period_digest ident = {"org": ORG, "namespace": NS, "agent": "bot", "user_id": "alice"} - await period_digest._skipped_period_keys(ch, ident, rewrite=False) + await period_digest._skipped_period_keys(ch, ident, ["period_digest_first_last"], rewrite=False) + for after in (None, (datetime(2026, 1, 1), "s1")): + async for _ in period_digest._fetch_digests(ch, ident, after=after): + pass async def _conflict_resolver(ch: RecordingCH, c: dict[str, Any]) -> None: @@ -399,8 +434,60 @@ async def _source_retirements(ch: RecordingCH, c: dict[str, Any]) -> None: await queue.write(item, pending=False, ch=ch) +async def _refusal_publication(ch: RecordingCH, c: dict[str, Any]) -> None: + from server.memory.app import extraction_queue as queue + from server.memory.domain.refusals import inspect + from server.memory.store import refusals + + key = (ORG, NS, "alice") + await refusals._matches_clickhouse(ch, key, [inspect("a removed passage")]) + await refusals._matches_clickhouse(ch, key, [inspect("a removed passage")], erasure_only=True) + await refusals._delete_old_vector( + ch, + { + "org": ORG, + "namespace": NS, + "agent": "a", + "user_id": "alice", + "session_id": "s", + "memory_key": "k", + "scope": "private", + "embedding_model": "m", + "version": 7, + }, + ) + await refusals._delete_old_shared( + ch, {"org": ORG, "namespace": NS, "memory_key": "k", "version": 7} + ) + row = queue._mirror( + { + "org": ORG, + "namespace": NS, + "agent": "a", + "user_id": "alice", + "session_id": "s", + "queue_id": "q", + "payload": '{"messages":[]}', + "version": 6, + }, + status="done", + ) + await queue._source_present(ch, row) + await queue._drop_refused(ch, row) + + def cases() -> list[Case]: return [ + Case( + "memory.summary_pages", + "server.memory.store.summary_pages", + _summary_pages, + {"order": ("oldest", "newest", "session")}, + rows=_summary_page_rows, + ), + Case( + "memory.refusal_publication", "server.memory.store.refusals", _refusal_publication, {} + ), Case( "workers.source_retirements", "server.knowledge.store.source_retirements", diff --git a/tests/analyzer/corpus.py b/tests/analyzer/corpus.py index b9e57739b2..a6311aa0cc 100644 --- a/tests/analyzer/corpus.py +++ b/tests/analyzer/corpus.py @@ -163,7 +163,11 @@ def _answer(self, sql: str, params: dict[str, Any]) -> list[dict[str, Any]]: return [r if isinstance(r, _Row) else _Row(r) for r in self._rows(sql, params)] async def query_all( - self, sql: str, params: dict[str, Any] | None = None + self, + sql: str, + params: dict[str, Any] | None = None, + *, + settings: dict[str, Any] | None = None, ) -> list[dict[str, Any]]: p = dict(params or {}) self.calls.append(("query_all", sql, p)) diff --git a/tests/analyzer/shapes.txt b/tests/analyzer/shapes.txt index dd8f3abfe7..997dd957e5 100644 --- a/tests/analyzer/shapes.txt +++ b/tests/analyzer/shapes.txt @@ -131,6 +131,7 @@ 0ec5f4fd6da5 SELECT max(version) AS version FROM loom.memory_embeddings WHERE org = {org:String} AND namespace = {ns:String} AND agent = {agent:String} AND user_id = ? AND session_id = ? AND scope = ? AND memory_k 0ef6f4537013 SELECT path, scope, symbol_ids, kind, start_line, end_line, content FROM ( SELECT path, scope, symbol_ids, kind, start_line, end_line, content, chunk_id, tombstoned_at FROM loom.code_chunks WHERE org 0f371a9d43dd SELECT memory_key, support_type, char_start, char_end, toString(evidence_id) AS evidence_id FROM ( SELECT memory_key, support_type, char_start, char_end, evidence_id, tombstoned_at FROM loom.memory_be +0f510c19553c SELECT *, hex(SHA256(toJSONString((value_json, value)))) AS body_sha FROM (SELECT org, namespace, agent, user_id, session_id, ? AS actor, memory_key, version, ? AS value_json, toJSONString(content_exc 0f5951ba0297 SELECT DISTINCT kind, start_line, end_line FROM loom.code_chunks FINAL WHERE org = {org:String} AND namespace = {ns:String} AND path = {path:String} AND tombstoned_at IS NULL 0f5fb44818c9 SELECT memory_id, org, namespace, agent, user_id, session_id, memory_key, memory_type, value, ? AS value_truncated, confidence, version, pinned, char_limit, created_at, updated_at, expires_at, valid_a 0fd1cbc1eecb SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, ? - cosineDistance(e.embedding, {qv:Array(Float32)}) AS similarity, toString(coalesce(e.temporal_ @@ -140,6 +141,7 @@ 1030febb1e77 ALTER TABLE loom.chunks UPDATE doc_superseded_at = {now:DateTime} WHERE (org = {org:String} AND user_id = {uid:String} AND doc_superseded_at IS NULL) AND ({erasure_mutation_tag:String} = {erasure_muta 1032019497c6 SELECT value, memory_type, confidence, promoted_from, promotion_event, promoted_from_candidate_id, promoted_by, rationale, promoted_at, reversible_until, tombstoned_at, version FROM loom.shared FINAL 1036d9982eb4 SELECT concat(?, c.session_id, ?, toString(c.chunk_index)) AS memory_key, ? AS memory_type, c.content AS content_excerpt, toString(ss.session_date) AS created_at, toString(ss.session_date) AS valid_at +1069a110039a SELECT agent, user_id, session_id, version, summary_text, cosineDistance(summary_embedding, {q:Array(Float32)}) AS distance, -- NB: alias must NOT be `valid_at` — under the analyzer a -- SELECT alias 107be5b763da SELECT count() AS n FROM loom.memory_categories FINAL WHERE org = {org:String} AND namespace IN {ns:Array(String)} AND tombstoned_at IS NULL 109db307d85f SELECT count() AS n FROM loom.chunks WHERE org = {org:String} AND user_id = {uid:String} AND doc_superseded_at IS NULL 10ff90cbf7b6 SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, e.status AS status, e.version AS version, -- Multi-vector axis: min(value_dist, key_dist). CH?t g @@ -198,6 +200,7 @@ 17612729b3e2 SELECT * FROM loom.learnings_queue FINAL WHERE org = {org:String} AND candidate_id IN {cids:Array(UUID)} 177807b65c1e SELECT f AS path, uniqExact(pr_number) AS prs FROM loom.code_pr_history FINAL ARRAY JOIN files AS f WHERE org = {org:String} AND namespace = {ns:String} AND merged_at > now() - INTERVAL {days:UInt32} 1799e5db040e SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, e.status AS status, e.version AS version, -- Multi-vector axis: min(value_dist, key_dist). CH?t g +17c5bb172f19 SELECT DISTINCT toInt64(units) AS units FROM loom.memory_refusals WHERE org={org:String} AND ((scope_kind IN (?) AND scope_value={ns:String} AND user_id={uid:String}) OR (scope_kind=? AND scope_value= 17d1d3c496e4 SELECT count() AS n FROM loom.entity_communities WHERE org = {org:String} AND (cityHash64(namespace, entity) IN {entities:Array(UInt64)} OR cityHash64(namespace, label) IN {entities:Array(UInt64)}) 17dac6a35956 SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, e.status AS status, e.version AS version, -- Multi-vector axis: min(value_dist, key_dist). CH?t g 17e1defc0d07 SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, e.status AS status, e.version AS version, -- Multi-vector axis: min(value_dist, key_dist). CH?t g @@ -209,6 +212,7 @@ 1891df92bb07 SELECT DISTINCT org FROM ( SELECT org FROM loom.namespaces FINAL WHERE owner_email = {s:String} AND active = ? UNION ALL SELECT org FROM loom.namespace_grants FINAL WHERE lower(subject) = lower({s:Str 18beffc43c3d SELECT memory_key_a, memory_key_b, allowed_at, allowed_by, reason FROM loom.conflict_allowlist FINAL WHERE org = {org:String} AND namespace = {ns:String} AND tombstoned_at IS NULL ORDER BY allowed_at 1904c89aac43 SELECT uniqExact(path) AS n FROM loom.code_chunks FINAL WHERE org = {org:String} AND namespace = {ns:String} AND embedding_model = {model:String} AND tombstoned_at IS NULL AND startsWith(path, {prefix +1906e06d78a5 SELECT *, hex(SHA256(toJSONString((value_json, value)))) AS body_sha FROM (SELECT org, namespace, agent, user_id, session_id, ? AS actor, memory_key, version, ? AS value_json, toJSONString(content_exc 190702c74f85 SELECT ts, actor, subject FROM loom.audit_log WHERE org = ? AND namespace = ? AND event_type = {event:String} AND ({name:String} = ? OR JSONExtractString(subject, ?) = {name:String}) ORDER BY ts DESC 190758ec6ba2 SELECT count() AS n FROM loom.memory_drops WHERE org = {org:String} AND namespace IN {ns:Array(String)} AND ? 19091cfef8cd SELECT memory_key, memory_type, content_excerpt, valid_at, created_at, namespace, count() OVER () AS total_matching FROM loom.memory_embeddings FINAL WHERE org = {org:String} AND namespace = {ns:Strin @@ -237,7 +241,6 @@ 1bfe4bc713e1 SELECT DISTINCT org, namespace, agent, user_id, session_id, memory_key FROM loom.memory WHERE tombstoned_at IS NULL AND sensitivity = ? AND (expires_at IS NULL OR expires_at > now()) AND (match(value, 1c34dca1df62 SELECT org, namespace, kind, number, source_id, component, is_open, created_at, updated_at, embedding, embedding_dim, embedding_model, text_sha256, tombstoned_at, version FROM loom.qa_item_embeddings 1c545b956083 SELECT uniqExact(issue_number) AS issues, count() AS pairs, max(computed_at) AS last_computed FROM loom.qa_link_candidates FINAL WHERE org = {org:String} AND qa_namespace = {ns:String} -1c7cbb1b18f8 SELECT session_id, summary_text, key_facts, derived_from, valid_at, created_at, version FROM loom.session_summaries FINAL WHERE org = {o:String} AND namespace = {n:String} AND tombstoned_at IS NULL AN 1ca42e357807 ALTER TABLE loom.entity_triple_rows DELETE WHERE (org = {org:String} AND namespace = {ns:String}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync = ? 1ca47c4c11f3 SELECT toString(evidence_id) AS eid, session_id, role, actor, toString(observed_at) AS observed, content, matched FROM ( SELECT evidence_id, session_id, role, actor, observed_at, content, tombstoned_a 1cb430ccdf4d SELECT ? FROM (SELECT * FROM loom.conversation_embeddings FINAL WHERE org = {org:String} AND namespace = {ns:String} AND thread_id = {tid:String} AND tombstoned_at IS NULL AND msg_id IN {ids:Array(Str @@ -287,6 +290,7 @@ 225194c2f6a6 DELETE FROM loom.entity_vectors WHERE (org = {org:String} AND namespace IN {ns:Array(String)}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS lightweight_deletes_sync = ? 22612cf3a394 SELECT namespace, source_uri, public_namespace, public_uri FROM (SELECT namespace, source_uri, indexOf({bindings:Array(Tuple(String, String))}, (namespace, right(source_uri, ?))) AS binding, arrayElem 2261d5bd6945 SELECT count() AS n FROM loom.chunks WHERE org = {org:String} AND namespace = {ns:String} AND content_sha256 IN {hashes:Array(String)} AND source_uri != {ex:String} AND doc_superseded_at IS NULL AND ( +22e0a45e948a SELECT *, hex(SHA256(toJSONString((value_json, value)))) AS body_sha FROM (SELECT org, namespace, agent, user_id, session_id, ? AS actor, concat(?, toString(chunk_index), ?, content_sha256) AS memory_ 22e4b395ef49 ALTER TABLE loom.conversation_threads UPDATE tombstoned_at = {now:DateTime} WHERE (org = {org:String} AND tombstoned_at IS NULL AND namespace IN {ns:Array(String)}) AND ({erasure_mutation_tag:String} 22eb12673f82 SELECT count() AS n FROM loom.memory_extraction_queue FINAL WHERE org = {org:String} AND user_id = {uid:String} 2303571b09cb ALTER TABLE loom.memory_versions DELETE WHERE (org = {org:String} AND user_id = {uid:String}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync = ? @@ -304,6 +308,7 @@ 2448faf35e3e SELECT DISTINCT src_path FROM loom.code_test_map FINAL WHERE org = {org:String} AND namespace = {ns:String} AND tombstoned_at IS NULL 244b76b58819 ALTER TABLE loom.memory UPDATE tombstoned_at = {now:DateTime} WHERE (org = {org:String} AND tombstoned_at IS NULL AND namespace = {ns:String}) AND ({erasure_mutation_tag:String} = {erasure_mutation_ta 246241942bf7 SELECT chunk_id, path, scope, kind, start_line, end_line, content, symbol_ids, context_description, arraySum(arrayMap((tok, idf) -> idf * countSubstrings(concat(?, lexemes, ?), concat(?, tok, ?)), {to +2490acfcecb6 DELETE FROM loom.shared WHERE org={org:String} AND namespace={namespace:String} AND memory_key={memory_key:String} AND version < {version:UInt64} 24c3fd0e6b29 SELECT path, chunk_id FROM loom.code_chunks WHERE org = {org:String} AND namespace = {ns:String} AND embedding_model = {model:String} GROUP BY path, chunk_id HAVING argMax(embedding_dim, version) = ? 24d4d8775291 SELECT start_line, end_line, scope, substring(content, greatest(?, position(content, {lit:String}) - ?), ?) AS excerpt FROM loom.code_chunks FINAL WHERE org = {org:String} AND namespace = {ns:String} 24df34974019 SELECT ts, org, namespace, agent, user_id, session_id, operation, request_id, trace_id, latency_ms, input_tokens, output_tokens, embedding_tokens, upstream_cost_usd, bytes_written, success, error_code @@ -405,7 +410,6 @@ 30406949d371 SELECT memory_id, version, agent, user_id, session_id, memory_key, created_at FROM loom.memory FINAL WHERE org = {org:String} AND namespace = {ns:String} AND memory_key IN {keys:Array(String)} AND tom 30e3abcc719b ALTER TABLE loom.query_nodes DELETE WHERE (org = {org:String} AND namespace = {ns:String}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync = ? 30ef01ea55bc SELECT count() AS n FROM loom.derived_facts WHERE org = {org:String} AND user_id = {uid:String} AND ? -3112e7749090 SELECT agent, user_id, session_id, version, summary_text, cosineDistance(summary_embedding, {q:Array(Float32)}) AS distance, -- NB: alias must NOT be `valid_at` — under the analyzer a -- SELECT alias 311579a423b9 SELECT created_at, memory_key FROM loom.memory WHERE memory_id = {mid:UUID} AND org = {org:String} AND namespace = {in_ns:String} LIMIT ? 3117b6749ab2 SELECT thread_id, msg_id, role, content_excerpt AS embedded_text, ts, cosineDistance(embedding, {q:Array(Float32)}) AS distance FROM loom.conversation_embeddings FINAL WHERE org = {org:String} AND nam 311c3ee7bb8b SELECT j.source_uri AS source_uri, if(j.open_jobs > ?, j.open_ingestion_id, j.latest_ingestion_id) AS ingestion_id, if(j.open_jobs > ?, j.open_filename, j.latest_filename) AS filename, if(j.open_jobs @@ -448,7 +452,6 @@ 342f026b3008 SELECT count() AS n FROM loom.memory_code_link_queue WHERE org = {org:String} AND namespace = {ns:String} AND ? 343b7c396165 SELECT count() AS n FROM loom.session_brief_revisions WHERE org = {org:String} AND namespace IN {ns:Array(String)} 345bb4f0bdc6 SELECT count() AS n FROM loom.learnings_supporters WHERE org = {org:String} AND namespace IN {ns:Array(String)} AND ? -346a0c5da453 -- schemas/schema.sql -- -- Reference dump of `loom.*` and `loom_telemetry.*`: what replaying schemas/migrations/*.sql -- produces, auto-generated by scripts/ops/dump_schema.py from a live -- ClickHou 348ab5a553ad SELECT org, namespace, source_uri, chunk_idx, content, tags, doc_metadata, chunk_metadata, ingested_at, doc_superseded_at, embedding_model, embedding_dim, embedding, created_at, embed_attempts, embed_ 34912ecbbcb8 SELECT count() AS n FROM loom.memory_chunk_sources FINAL WHERE org = {org:String} AND tombstoned_at IS NULL AND (namespace, memory_key) IN {keys:Array(Tuple(String, String))} 349d1b15dfc4 SELECT count() AS n FROM loom.dataset_rows WHERE org = {org:String} AND user_id = {uid:String} @@ -620,6 +623,7 @@ 47d6216caf2e SELECT number, state, author, title, labels, opened_at, closed_at, updated_at, files, title_hits, file_hit FROM ( SELECT number, state, author, title, labels, opened_at, closed_at, updated_at, files, 47db1deeabfd SELECT DISTINCT namespace FROM ( SELECT namespace FROM loom.memory WHERE org = {org:String} AND (namespace, agent, user_id, session_id, memory_key) IN ( SELECT namespace, agent, user_id, session_id, m 47e13d18aa4f SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, ? - cosineDistance(e.embedding, {qv:Array(Float32)}) AS similarity, toString(coalesce(e.temporal_ +47ea41c2e2e8 SELECT agent, user_id, session_id, embedding_model, summary_text, key_facts, derived_from, valid_at, created_at, version, length(key_facts) AS kf, length(derived_from) AS df, coalesce(valid_at, create 482347c34a21 SELECT memory_key, groupUniqArray(?)(ent) AS shared FROM ( SELECT other.memory_key AS memory_key, other.entity_a AS ent FROM loom.entity_edge_memories AS other WHERE other.org = {o:String} AND other.n 48419f5696b5 SELECT count() AS n FROM loom.consolidation_runs WHERE org = {org:String} AND user_id = {uid:String} AND ? 4888e1579c2e SELECT count() AS n FROM loom.context_blobs FINAL WHERE org = {org:String} AND tombstoned_at IS NULL AND user_id = {uid:String} @@ -687,6 +691,7 @@ 525d9e3b3774 SELECT count() AS n FROM loom.working_memory FINAL WHERE org = {org:String} AND user_id = {uid:String} AND tombstoned_at IS NULL 5287f5e8fe20 SELECT operation, sumMerge(cost_usd_state) AS cost_usd, sumMerge(tokens_state) AS tokens, sumMerge(embed_tokens_state) AS embed_tokens, countMerge(ops_state) AS ops, sumMerge(errors_state) AS errors, 528a44452aa5 SELECT namespace, agent, user_id, session_id, memory_key, toString(status) AS row_status, duplicate_of AS canonical_id FROM loom.memory FINAL WHERE org = {org:String} AND (namespace, agent, user_id, s +52c2e4a12bb9 SELECT agent, user_id, session_id, version, summary_text, cosineDistance(summary_embedding, {q:Array(Float32)}) AS distance, -- NB: alias must NOT be `valid_at` — under the analyzer a -- SELECT alias 52d32818a393 SELECT namespace, toString(memory_id) AS canonical_id, memory_key, memory_type, value, tags, version, agent, session_id, user_id, source_class, sensitivity, toString(created_at) AS created_at, toStrin 52ef9d28c739 SELECT org, namespace, agent, user_id, session_id, memory_key, memory_type, embedding_model, embedding, version, content_sha256, status FROM loom.memory_embeddings FINAL WHERE (org, namespace, agent, 52f36dc110b9 SELECT memory_id, memory_key, agent, value, created_at FROM loom.memory FINAL WHERE org = {org:String} AND memory_id = {mid:UUID} AND tombstoned_at IS NULL LIMIT ? @@ -702,7 +707,6 @@ 54082f4a0bda SELECT c.session_id AS session_id, c.chunk_index AS chunk_index, c.parent_memory_key AS parent_memory_key, cosineDistance(c.embedding, {q:Array(Float32)}) AS distance FROM loom.memory_chunks AS c WHER 541a4c06fca2 SELECT agent, user_id, session_id, memory_key, memory_type, content_excerpt, created_at, valid_at, version, status FROM ( SELECT agent, user_id, session_id, memory_key, memory_type, content_excerpt, t 542242ea89a0 SELECT chunk_id, path, scope, kind, start_line, end_line, content, symbol_ids, context_description, arraySum(arrayMap((tok, idf) -> idf * countSubstrings(concat(?, lexemes, ?), concat(?, tok, ?)), {to -54325369241a SELECT org, namespace, agent, user_id, session_id, memory_key, checked, tombstone, done FROM loom.delete_version_checks FINAL WHERE org = {org:String} AND namespace = {ns:String} AND agent = {agent:St 544bda275e23 ALTER TABLE loom.memory_code_link_queue DELETE WHERE (org = {org:String} AND user_id = {uid:String}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync = ? 548b3de5bbd4 SELECT memory_key, toString(memory_id) AS mid, content_sha256, agent, memory_type, -- a high value is readable by key only if(sensitivity = ?, ?, substring(value, ?, ?)) AS preview, sensitivity, tags, 549dd1314271 SELECT count() AS n FROM loom.conversation_embeddings FINAL WHERE org = {org:String} AND namespace IN {ns:Array(String)} AND tombstoned_at IS NULL @@ -728,6 +732,7 @@ 566af30aabf5 SELECT count() AS n FROM (SELECT ? FROM loom.memory LIMIT ?) SETTINGS select_sequential_consistency = ? 569c9b218f83 ALTER TABLE loom.conflict_allowlist DELETE WHERE (org = {org:String} AND namespace IN {ns:Array(String)}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync = 56a045b3692e SELECT kind, number, source_id, component, is_open, created_at, cosineDistance(embedding, {q:Array(Float32)}) AS distance FROM loom.qa_item_embeddings FINAL WHERE org = {org:String} AND namespace = {n +56ab4d25e910 DELETE FROM loom.memory_embeddings WHERE org={org:String} AND namespace={namespace:String} AND agent={agent:String} AND user_id={user_id:String} AND session_id={session_id:String} AND scope={scope:Str 56f4c4320d80 SELECT f AS path, uniqExact(pr_number) AS prs FROM loom.code_pr_history FINAL ARRAY JOIN files AS f WHERE org = {org:String} AND namespace = {ns:String} AND merged_at > now() - INTERVAL {days:UInt32} 56f6ba832916 SELECT agent, user_id, session_id, memory_key, memory_type, content_excerpt, toString(created_at) AS created_at, toString(valid_at) AS valid_at, version FROM loom.memory_embeddings FINAL WHERE org = { 56f6eca8bb7f SELECT count() AS n FROM loom.session_briefs WHERE org = {org:String} AND user_id = {uid:String} @@ -747,6 +752,7 @@ 57fc18cfd1a4 SELECT pr_number, any(title) AS title, any(files) AS files, any(author) AS author, any(merge_commit_sha) AS merge_sha, min(merged_at) AS merged_at FROM loom.code_pr_history FINAL WHERE org = {org:Stri 5805dc626bcc SELECT countIf(NOT startsWith(path, ?)) AS source_files, countIf(NOT startsWith(path, ?) AND path NOT IN ( SELECT src_path FROM loom.code_test_map FINAL WHERE org = {org:String} AND namespace = {ns:St 580b700ab41c SELECT name, kind, path FROM ( SELECT tier, qualified_name, symbol_id, name, kind, path, argMax(tombstoned_at IS NULL, version) AS is_live FROM loom.code_symbols WHERE org = {org:String} AND namespace +5832b8ab5ff9 SELECT agent, user_id, session_id, version, summary_text, cosineDistance(summary_embedding, {q:Array(Float32)}) AS distance, -- NB: alias must NOT be `valid_at` — under the analyzer a -- SELECT alias 5834a5a32d06 SELECT id, org, scope_type, scope_value, requester, actor, urgency, purge, mirror_marked, attempts, steps_completed, requested_at, failure_reason, residue_count, residue, verified_at, census FROM loom 5835eff36a44 SELECT count() AS n FROM loom.memory_evidence FINAL WHERE org = {org:String} AND namespace IN {ns:Array(String)} 5837a3e90ebe ALTER TABLE loom.context_blobs DELETE WHERE (org = {org:String} AND namespace = {ns:String}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync = ? @@ -755,6 +761,7 @@ 5852405dd8a6 SELECT id, org, scope_type, scope_value, requester, actor, urgency, purge, mirror_marked, attempts, steps_completed, requested_at, failure_reason, residue_count, residue, verified_at, census FROM loom 5862d4ef7b73 ALTER TABLE loom.session_brief_revisions DELETE WHERE (org = {org:String} AND user_id = {uid:String}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync = ? 586a6ce18c21 SELECT concat(?, toString(c.session_id), ?, toString(c.chunk_index)) AS memory_key, ? AS memory_type, substring(c.content, ?, ?) AS content_excerpt, c.content AS content_full, c.agent AS agent, c.user +58cdc04fc385 SELECT agent, user_id, session_id, embedding_model, summary_text, key_facts, derived_from, valid_at, created_at, version, length(key_facts) AS kf, length(derived_from) AS df, coalesce(valid_at, create 58d8e597b166 SELECT chunk_id, path, scope, kind, start_line, end_line, content, symbol_ids, context_description, arraySum(arrayMap((tok, idf) -> idf * countSubstrings(concat(?, lexemes, ?), concat(?, tok, ?)), {to 58dd61abeb39 WITH m AS ( SELECT org, namespace, agent, user_id, session_id, memory_key, argMax(status, version) AS ms, argMax(memory_id, version) AS mid FROM loom.memory WHERE org = {org:String} AND namespace = {n 58f4b29e4e3a SELECT org, namespace, policy_id, effect, description, actions, condition, enabled, mode, created_by, created_at, updated_at, tombstoned_at FROM loom.namespace_policies FINAL WHERE org = {org:String} @@ -770,6 +777,7 @@ 599725704d75 SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, e.status AS status, e.version AS version, -- Multi-vector axis: min(value_dist, key_dist). CH?t g 59b30916ba34 SELECT chunk_id, path, scope, kind, start_line, end_line, content, symbol_ids, context_description, arraySum(arrayMap((tok, idf) -> idf * countSubstrings(concat(?, lexemes, ?), concat(?, tok, ?)), {to 59b3f6e281d9 SELECT count() AS n FROM loom.entity_edge_rows WHERE org = {org:String} AND namespace IN {ns:Array(String)} AND ? +59b463c0ef4b SELECT *, hex(SHA256(toJSONString((value_json, value)))) AS body_sha FROM (SELECT org, namespace, agent, user_id, session_id, ? AS actor, concat(?, queue_id) AS memory_key, version, ? AS value_json, p 59bc4767d952 SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, ? - cosineDistance(e.embedding, {qv:Array(Float32)}) AS similarity, toString(coalesce(e.temporal_ 59cfa551438a ALTER TABLE loom.learnings_supporters DELETE WHERE (org = {org:String} AND namespace = {ns:String}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync = ? 59de9570d846 ALTER TABLE loom.context_blobs UPDATE tombstoned_at = {now:DateTime} WHERE (org = {org:String} AND tombstoned_at IS NULL AND namespace = {ns:String}) AND ({erasure_mutation_tag:String} = {erasure_muta @@ -792,7 +800,6 @@ 5c174b8803f6 SELECT concat(?, toString(c.session_id), ?, toString(c.chunk_index)) AS memory_key, ? AS memory_type, substring(c.content, ?, ?) AS content_excerpt, c.content AS content_full, c.agent AS agent, c.user 5c2243636c81 SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, ? - cosineDistance(e.embedding, {qv:Array(Float32)}) AS similarity, toString(coalesce(e.temporal_ 5c5bd0cdac42 SELECT count() AS n, if(n = ?, NULL, max(ingested_at)) AS latest FROM loom.documents FINAL WHERE org = {org:String} AND namespace = {ns:String} AND superseded_at IS NULL -5c68e73d0f88 SELECT agent, user_id, session_id, version, summary_text, cosineDistance(summary_embedding, {q:Array(Float32)}) AS distance, -- NB: alias must NOT be `valid_at` — under the analyzer a -- SELECT alias 5c7221c42763 SELECT to_key AS memory_key, relation, toString(created_at) AS created_at FROM loom.operator_edges FINAL WHERE org = {o:String} AND namespace = {n:String} AND from_key = {k:String} AND tombstoned_at I 5c93fe003e33 SELECT path, scope, symbol_ids, kind, start_line, end_line, content FROM ( SELECT path, scope, symbol_ids, kind, start_line, end_line, content, chunk_id, tombstoned_at FROM loom.code_chunks WHERE org 5caa1782814d SELECT ts, org, namespace, agent, operation, latency_ms, success, error_code, request_id, trace_id FROM (SELECT raw.ts AS ts, raw.org AS org, raw.namespace AS namespace, raw.agent AS agent, raw.user_i @@ -921,6 +928,7 @@ 6befffe2772b SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, e.status AS status, e.version AS version, -- Multi-vector axis: min(value_dist, key_dist). CH?t g 6c17c9f6e702 SELECT org, namespace, agent, thread_id, msg_id, role, content, reasoning, user_id, run_id, metadata, tokens, ts FROM loom.conversations WHERE org = {org:String} AND namespace IN {nss:Array(String)} A 6c1cb0579e58 SELECT entity, community_id FROM loom.entity_communities FINAL WHERE org = {org:String} AND namespace IN {nss:Array(String)} AND level = ? AND entity IN {ents:Array(String)} AND (namespace, computed_a +6c2e2e39e107 SELECT *, hex(SHA256(toJSONString((value_json, value)))) AS body_sha FROM (SELECT org, namespace, agent, user_id, session_id, ? AS actor, concat(?, queue_id) AS memory_key, version, ? AS value_json, p 6c4a272aa608 SELECT (SELECT count() FROM loom.entity_triples WHERE org = {org:String} AND namespace = {ns:String} AND adapter_name = {adapter:String} AND confidence < {conf:Float32}) AS t_count, (SELECT count() FR 6c4d6b79f018 SELECT agent, count() AS rows, count() OVER () AS total, sum(count()) OVER () AS total_rows FROM loom.memory FINAL WHERE org = {org:String} AND namespace = {ns:String} AND tombstoned_at IS NULL AND st 6cf657fa1bf5 SELECT count() AS n, lower(hex(SHA256(arrayStringConcat(arraySort(groupArray(toString(row_hash))))))) AS digest FROM loom.audit_log WHERE ts >= toDateTime64({ws:DateTime}, ?, ?) AND ts < toDateTime64( @@ -928,14 +936,13 @@ 6d1913594aa8 ALTER TABLE loom.conversation_threads UPDATE tombstoned_at = {now:DateTime} WHERE (org = {org:String} AND tombstoned_at IS NULL AND user_id = {uid:String}) AND ({erasure_mutation_tag:String} = {erasur 6d2ff1c4506a SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, e.status AS status, e.version AS version, cosineDistance(e.embedding, {q:Array(Float32)}) AS dist 6d37b84f121a ALTER TABLE loom.conversation_threads UPDATE tombstoned_at = {now:DateTime} WHERE (org = {org:String} AND tombstoned_at IS NULL AND namespace = {ns:String}) AND ({erasure_mutation_tag:String} = {erasu -6d42ff967211 SELECT org, namespace, agent, user_id, session_id, memory_key, sensitivity FROM loom.memory FINAL PREWHERE (org, namespace, agent, user_id, session_id, memory_key) IN {keys:Array(Tuple(String, String, 6d57c5550379 SELECT symbol_id FROM loom.code_symbol_stats FINAL WHERE org = {org:String} AND namespace = {ns:String} AND fan_in_calls > ? 6d918add4f31 ALTER TABLE loom.memory_categories DELETE WHERE (org = {org:String} AND namespace IN {ns:Array(String)}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync = ? 6d98dc2e3672 SELECT count() AS n FROM loom.memory_embeddings FINAL WHERE org = {org:String} AND namespace IN {ns:Array(String)} AND tombstoned_at IS NULL 6da9f7f60075 SELECT count() AS n FROM loom.entity_canonical_map FINAL WHERE org = {org:String} AND namespace IN {ns:Array(String)} 6daa2655ee73 SELECT count() AS n FROM loom.entity_degree WHERE org = {org:String} AND namespace IN {ns:Array(String)} AND ? 6e07272917e3 SELECT count() AS n FROM loom.memory_versions WHERE org = {org:String} AND namespace = {ns:String} AND ? -6e2bd7b72f84 SELECT DISTINCT org, namespace, agent, user_id, session_id, memory_key, scope, embedding_model FROM loom.memory_embeddings WHERE (org, namespace, agent, user_id, session_id, memory_key) IN {keys:Array +6e1154b224c2 SELECT *, hex(SHA256(toJSONString((value_json, value)))) AS body_sha FROM (SELECT org, namespace, agent, user_id, session_id, ? AS actor, concat(?, queue_id) AS memory_key, version, ? AS value_json, t 6e5e297bbb30 SELECT count() AS n FROM loom.memory FINAL WHERE org = {org:String} AND namespace = {ns:String} AND status != ? AND tombstoned_at IS NULL AND status != ? AND agent = {agent:String} AND user_id = {uid: 6e894706d089 SELECT count() AS n FROM loom.code_search_outcome WHERE org = {org:String} AND namespace IN {ns:Array(String)} AND ? 6e90fc205193 SELECT org, namespace, agent, user_id, session_id, memory_key FROM ( SELECT DISTINCT org, namespace, agent, user_id, session_id, memory_key FROM loom.memory WHERE tombstoned_at IS NULL AND (arrayExist @@ -996,11 +1003,10 @@ 7584931ee488 SELECT c.category_id AS category_id, count() AS rows FROM (SELECT memory_key, category_id FROM loom.memory_categories FINAL WHERE org = {org:String} AND namespace = {ns:String} AND tombstoned_at IS NU 7585095cfe72 SELECT ? AS x FROM loom.scim_users FINAL WHERE id = {id:String} AND active = ? 7588988984a6 ALTER TABLE loom.conversations DELETE WHERE (org = {org:String} AND namespace IN {ns:Array(String)}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync = ? -75cbcf76345b SELECT * FROM loom.memory_embeddings FINAL PREWHERE (org, namespace, agent, user_id, session_id, scope, embedding_model, memory_key) IN {keys:Array(Tuple(String, String, String, String, String, String 75d1ab9d987d SELECT memory_key, memory_type, content_excerpt, ? - cosineDistance(embedding, {qv:Array(Float32)}) AS similarity, toString(coalesce(temporal_anchor, toDateTime(?))) AS temporal_anchor, temporal_confi 75e9410435ef SELECT path, scope, symbol_ids, kind, start_line, end_line, content, shingles, uniqExact(path) OVER () AS admitted_files FROM ( SELECT path, scope, symbol_ids, kind, start_line, end_line, content, chu 75f3fd72118d ALTER TABLE loom.memory_term_freq DELETE WHERE (org = {org:String} AND namespace IN {ns:Array(String)}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync = ? -762c79ce2e4f SELECT agent, user_id, session_id, version, summary_text, cosineDistance(summary_embedding, {q:Array(Float32)}) AS distance, -- NB: alias must NOT be `valid_at` — under the analyzer a -- SELECT alias +7615ac826d0c SELECT *, hex(SHA256(toJSONString((value_json, value)))) AS body_sha FROM (SELECT org, namespace, agent, user_id, session_id, ? AS actor, concat(?, toString(evidence_id)) AS memory_key, toUInt64(toUni 764e065d440e SELECT count() AS n FROM loom.knowledge_ingest_cancels WHERE org = {org:String} AND user_id = {uid:String} AND ? 764e0e259701 SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, e.status AS status, e.version AS version, -- Multi-vector axis: min(value_dist, key_dist). CH?t g 765404bd5d8d SELECT toDate(coalesce(temporal_anchor, valid_at)) AS event_date, cosineDistance(embedding, {q:Array(Float32)}) AS distance, substring(content_excerpt, ?, ?) AS excerpt, memory_key, temporal_anchor IS @@ -1009,6 +1015,7 @@ 76f47ab9df72 SELECT ? AS present FROM loom.code_caller_guards WHERE org = {org:String} AND namespace = {ns:String} LIMIT ? 77424b91d322 SELECT count() AS n FROM loom.memory FINAL WHERE org = {org:String} AND namespace IN {ns:Array(String)} 778a3bb04c67 SELECT namespace, source_uri, title, description, tags, ingested_at, superseded_at, blob_path, blob_size, toString(blob_content_type) AS blob_content_type, original_filename, content_preview FROM loom +7799bb45ee57 SELECT ? FROM loom.memory_refusals WHERE org={org:String} AND ((scope_kind IN (?,?) AND scope_value={ns:String} AND user_id={uid:String}) OR (scope_kind=? AND scope_value={ns:String}) OR (scope_kind=? 77abb6560e82 ALTER TABLE loom.memory_code_links DELETE WHERE (org = {org:String} AND user_id = {uid:String}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync = ? 77b5d76bd16d SELECT count() AS n FROM loom.entity_triple_rows WHERE org = {org:String} AND namespace IN {ns:Array(String)} AND ? 77be1ad28338 SELECT DISTINCT agent FROM loom.memory_embeddings WHERE org = {org:String} AND namespace = {ns:String} AND tombstoned_at IS NULL ORDER BY agent @@ -1041,7 +1048,7 @@ 7a543640c602 ALTER TABLE loom.working_memory DELETE WHERE (org = {org:String} AND user_id = {uid:String}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync = ? 7a8545148669 SELECT memory_key, memory_type, content_excerpt, valid_at, created_at, namespace, count() OVER () AS total_matching FROM loom.memory_embeddings FINAL WHERE org = {org:String} AND namespace = {ns:Strin 7a87bbd25387 SELECT org, namespace, agent, user_id, thread_id, title, metadata, run_id, created_at, updated_at, tombstoned_at FROM loom.conversation_threads FINAL WHERE org = {org:String} AND user_id = {scope:Stri -7a9d47bbc467 SELECT session_id, summary_text, key_facts, derived_from, valid_at, created_at, version FROM loom.session_summaries FINAL WHERE org = {o:String} AND namespace = {n:String} AND tombstoned_at IS NULL OR +7aab1cf5e970 SELECT *, hex(SHA256(toJSONString((value_json, value)))) AS body_sha FROM (SELECT org, namespace, agent, user_id, session_id, ? AS actor, concat(?, session_id) AS memory_key, version, ? AS value_json, 7aad3247f4a7 SELECT count() AS n FROM loom.session_summaries FINAL WHERE org = {org:String} AND user_id = {uid:String} AND tombstoned_at IS NULL 7ac14ad61019 WITH ( SELECT max(mirror_marked) FROM (SELECT raw.ts AS ts, raw.org AS org, raw.namespace AS namespace, raw.agent AS agent, raw.user_id AS user_id, raw.session_id AS session_id, raw.run_id AS run_id, 7ada090d2235 ALTER TABLE loom.shared UPDATE tombstoned_at = {now:DateTime} WHERE (org = {org:String} AND tombstoned_at IS NULL AND namespace = {ns:String}) AND ({erasure_mutation_tag:String} = {erasure_mutation_ta @@ -1093,6 +1100,7 @@ 7e726e6f73c0 SELECT model, count() AS turns, uniqExact(session_id) AS sessions, sum(toUInt64(input_tokens)) AS input_tokens, sum(toUInt64(cache_read_tokens)) AS cache_read_tokens, sum(toUInt64(cache_creation_5m_to 7eccab6dfda3 SELECT org, namespace, name, source_id, kind, source_kind, config_json, items_json, credential_enc, secret_enc, access, credential_mode, owner_subject, connection_id, created_by, active, pending_delet 7ed9e5f6ceb3 ALTER TABLE loom.memory_chunk_sources UPDATE tombstoned_at = {now:DateTime} WHERE (org = {org:String} AND tombstoned_at IS NULL AND namespace = {ns:String}) AND ({erasure_mutation_tag:String} = {erasu +7eea2ee7caf9 SELECT memory_key, (status != ? OR tombstoned_at IS NOT NULL OR redacted_at IS NOT NULL) AS taken_out FROM loom.memory FINAL WHERE org = {org:String} AND namespace = {ns:String} AND agent = {agent:Str 7ef1d533ac81 SELECT count() AS n FROM loom.memory_evidence WHERE org = {org:String} AND namespace = {ns:String} AND ? 7f22d53d8b16 SELECT uniqExact(k) AS n FROM ( SELECT k FROM (SELECT arrayJoin(memories_used) AS k FROM loom.agent_runs FINAL WHERE org = {org:String} AND namespace = {ns:String} AND agent = {agent:String} AND start 7f3000230a53 SELECT memory_type, toString(status) AS status, source_class, count() AS rows, min(updated_at) AS first_at, max(updated_at) AS last_at FROM loom.memory FINAL WHERE org = {org:String} AND namespace = { @@ -1116,6 +1124,7 @@ 80615d70d925 SELECT count() AS n FROM loom.entity_communities WHERE org = {org:String} AND namespace = {ns:String} AND ? 809ffd3a9a82 SELECT tag, count() AS rows FROM (SELECT arrayJoin(tags) AS tag FROM loom.memory FINAL WHERE org = {org:String} AND namespace = {ns:String} AND tombstoned_at IS NULL) GROUP BY tag HAVING rows >= {min: 80ac0a0e1397 SELECT toYYYYMM(ts) AS partition, ts, event_type, org, namespace, actor, subject, request_id, row_hash FROM loom.audit_log WHERE ts >= toDateTime64(now() - INTERVAL {lb:UInt32} SECOND, ?, ?) AND (ts, +80cfa2f097df SELECT org, namespace, agent, user_id, session_id, embedding_model, version, summary_text, key_facts, derived_from, at FROM (SELECT org, namespace, agent, user_id, session_id, embedding_model, version 8107af48e1df SELECT count() AS n FROM ( SELECT evidence_id, tombstoned_at FROM loom.memory_evidence WHERE org = {org:String} AND namespace = {ns:String} AND user_id = {uid:String} AND session_id = {sid:String} AND 81147eeb107a SELECT count() AS n FROM loom.entity_communities WHERE org = {org:String} AND namespace = {ns:String} 812c33d11f88 ALTER TABLE loom.memory_projection_failures DELETE WHERE (org = {org:String} AND namespace IN {ns:Array(String)}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations @@ -1192,6 +1201,7 @@ 89f40a73fa83 SELECT memory_id, org, namespace, agent, user_id, session_id, memory_key, version, value, toString(value_json) AS object_text FROM loom.memory FINAL WHERE tombstoned_at IS NULL AND sensitivity = ? AND 8a1fca063e7f SELECT org, agent, count() AS calls, countIf(if(consumer = ?, args[?], consumer) NOT IN {untagged:Array(String)}) AS tagged, calls - tagged AS untagged, round(tagged / calls, ?) AS tagged_ratio, anyIf 8a44f1b04017 SELECT count() AS n FROM loom.entity_triple_rows WHERE org = {org:String} AND namespace IN {ns:Array(String)} AND memory_id NOT IN (SELECT memory_id FROM loom.memory FINAL WHERE org = {org:String} AND +8a5fb93b3273 SELECT agent, user_id, session_id, embedding_model, summary_text, key_facts, derived_from, valid_at, created_at, version, length(key_facts) AS kf, length(derived_from) AS df, coalesce(valid_at, create 8a725f267f59 SELECT count() AS n FROM loom.memory_reinforcement WHERE org = {org:String} AND namespace = {ns:String} AND ? 8a8b47f8466b SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, e.status AS status, e.version AS version, -- Multi-vector axis: min(value_dist, key_dist). CH?t g 8a8bd6cc9556 SELECT toString(memory_id) AS memory_id, org, namespace, agent, user_id, session_id, memory_key, value, actor, version, redacted_at, toString(coalesce(valid_at, updated_at)) AS since FROM loom.memory @@ -1233,6 +1243,7 @@ 8ea78684e9ed SELECT entities, sensitivity FROM loom.memory FINAL WHERE org = {org:String} AND namespace = {ns:String} AND agent = {agent:String} AND user_id = {uid:String} AND session_id = {sid:String} AND memory_ 8ee4f33d0c2f SELECT count() AS n FROM loom.entity_summaries FINAL WHERE org = {org:String} AND namespace = {ns:String} 8ee5e12ce3f8 SELECT memory_type, retention_days, dedup_threshold, hot_quota, conflict_policy FROM loom.namespace_types FINAL WHERE org = {org:String} AND namespace = {ns:String} ORDER BY memory_type +8f11758986c7 SELECT agent, user_id, session_id, embedding_model, summary_text, key_facts, derived_from, valid_at, created_at, version, length(key_facts) AS kf, length(derived_from) AS df, coalesce(valid_at, create 8f5abdba34be SELECT agent, user_id, session_id, memory_key, memory_type, content_excerpt, toString(created_at) AS created_at, toString(valid_at) AS valid_at, version FROM loom.memory_embeddings FINAL WHERE org = { 8fa56eef0912 SELECT count() AS n FROM loom.task_events WHERE org = {org:String} AND namespace IN {ns:Array(String)} AND ? 8fabc8412cfc SELECT subject, role, granted_by FROM loom.namespace_grants FINAL WHERE org = {org:String} AND namespace = {ns:String} AND revoked_at IS NULL AND status = ? @@ -1293,6 +1304,7 @@ 9648988c847c SELECT count() AS n FROM loom.documents FINAL WHERE org = {org:String} AND namespace = {ns:String} 96a4e124aaa7 SELECT agent, user_id, session_id, memory_key, memory_type, content_excerpt, toString(created_at) AS created_at, toString(valid_at) AS valid_at, version FROM loom.memory_embeddings FINAL WHERE org = { 96cd9708162d SELECT count() AS n FROM loom.memory_reinforcement WHERE org = {org:String} AND user_id = {uid:String} +96e6221f7f9f SELECT agent, user_id, session_id, embedding_model, summary_text, key_facts, derived_from, valid_at, created_at, version, length(key_facts) AS kf, length(derived_from) AS df, coalesce(valid_at, create 96ecb876ec8c SELECT * FROM loom.scim_users FINAL WHERE id = {id:String} 971a9c534152 SELECT entity, substring(summary_text, ?, ?) AS content_excerpt, cosineDistance(summary_embedding, {q:Array(Float32)}) AS distance, -- alias must NOT be `valid_from`: it would shadow the base column - 9761b739c189 SELECT m.org AS org, m.namespace AS namespace, m.user_id AS user_id, count() AS n FROM loom.memory AS m FINAL INNER JOIN loom.namespaces AS n FINAL ON m.org = n.org AND m.namespace = n.namespace WHERE @@ -1315,8 +1327,10 @@ 998d3948018f ALTER TABLE loom.session_brief_revisions DELETE WHERE (org = {org:String} AND namespace = {ns:String}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync = ? 99a41729ee1c SELECT concat(?, toString(c.session_id), ?, toString(c.chunk_index)) AS memory_key, ? AS memory_type, substring(c.content, ?, ?) AS content_excerpt, c.content AS content_full, c.agent AS agent, c.user 99b2b4b77f2a SELECT memory_key, memory_type, content_excerpt, ? - cosineDistance(embedding, {qv:Array(Float32)}) AS similarity, toString(coalesce(temporal_anchor, toDateTime(?))) AS temporal_anchor, temporal_confi +99b329fe464e SELECT agent, user_id, session_id, version, summary_text, cosineDistance(summary_embedding, {q:Array(Float32)}) AS distance, -- NB: alias must NOT be `valid_at` — under the analyzer a -- SELECT alias 99bb0fb8f03c ALTER TABLE loom.session_summaries UPDATE tombstoned_at = {now:DateTime} WHERE (org = {org:String} AND tombstoned_at IS NULL AND user_id = {uid:String}) AND ({erasure_mutation_tag:String} = {erasure_m 99d86c8fa8ea SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, e.status AS status, e.version AS version, -- Multi-vector axis: min(value_dist, key_dist). CH?t g +99ebc0e5d139 SELECT agent, user_id, session_id, version, summary_text, cosineDistance(summary_embedding, {q:Array(Float32)}) AS distance, -- NB: alias must NOT be `valid_at` — under the analyzer a -- SELECT alias 99ebe72c9d4f SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, e.status AS status, e.version AS version, cosineDistance(e.embedding, {q:Array(Float32)}) AS dist 9a356be92fff SELECT path, commit_sha, run_at, lines, tests FROM loom.code_coverage_files FINAL WHERE org = {org:String} AND namespace = {ns:String} AND path IN {paths:Array(String)} AND tombstoned_at IS NULL 9a3d8fb32435 SELECT count() AS n FROM loom.memory_chunk_sources FINAL WHERE org = {org:String} AND namespace IN {ns:Array(String)} @@ -1324,6 +1338,7 @@ 9a9428a61df0 INSERT INTO loom.chunk_supersessions (org, namespace, embedding_model, source_uri, doc_version, superseded_at) SELECT DISTINCT org, namespace, embedding_model, source_uri, doc_version, {when:DateTime} 9a9a371beffd SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, e.status AS status, e.version AS version, cosineDistance(e.embedding, {q:Array(Float32)}) AS dist 9a9aaad3e6e4 SELECT DISTINCT org, namespace, agent, user_id, session_id, memory_key FROM loom.memory WHERE tombstoned_at IS NULL AND sensitivity = ? AND (expires_at IS NULL OR expires_at > now()) AND (match(value, +9ab0f1177c93 SELECT *, hex(SHA256(toJSONString((value_json, value)))) AS body_sha FROM (SELECT org, namespace, agent, user_id, session_id, ? AS actor, concat(?, toString(chunk_index), ?, content_sha256) AS memory_ 9adfa8216cee ALTER TABLE loom.memory_code_link_queue DELETE WHERE (org = {org:String} AND namespace = {ns:String}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync = ? 9ae3708317b3 SELECT if(consumer = ?, args[?], consumer) AS consumer_key, count() AS calls, countIf(NOT success) AS errors, countIf(NOT success AND error_code = ?) AS server_errors, round(errors / calls, ?) AS erro 9b1e4cc93d63 ALTER TABLE loom.memory_exposures DELETE WHERE (org = {org:String} AND user_id = {uid:String}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync = ? @@ -1373,11 +1388,13 @@ 9fad0b356a81 SELECT namespace, countIf(status=?) AS pending, countIf(status=?) AS promoted, countIf(status=?) AS rejected, if(countIf(status IN (?,?)) > ?, countIf(status=?) / countIf(status IN (?,?)), ?) AS appro 9ff4c37ed354 WITH spend AS ( SELECT org, namespace, sum(upstream_cost_usd) AS spent, sumIf(upstream_cost_usd, operation = {op:String}) AS own FROM (SELECT ts, org, namespace, agent, user_id, session_id, run_id, op 9fff0b609d85 SELECT toUInt32(quantileExact(?)(failing_tests)) AS median_tests, count() AS hours FROM loom.qa_fail_hours FINAL WHERE org = {org:String} AND qa_namespace = {ns:String} AND hour >= {h0:DateTime} AND h +a010bc956c57 SELECT org, namespace, agent, user_id, session_id, memory_key, checked, tombstone, checked_state_sha256, done FROM loom.delete_version_checks FINAL WHERE org = {org:String} AND namespace = {ns:String} a055584d8e91 SELECT toString(f.failure_id) AS failure_id, org, namespace, agent, projection, source_operation, source_resource_id, memory_keys, error_type, error_message, payload, status, created_at, resolved_at F a0585852a421 SELECT memory_id, memory_key, value, pinned, agent, user_id, session_id, run_id, valid_at FROM loom.memory FINAL PREWHERE org = {org:String} AND namespace = {ns:String} AND (agent, user_id, session_id a0706937823b SELECT memory_type, toString(status) AS status, source_class, count() AS rows, min(updated_at) AS first_at, max(updated_at) AS last_at FROM loom.memory FINAL WHERE org = {org:String} AND namespace = { a08014de0c22 SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, e.status AS status, e.version AS version, -- Multi-vector axis: min(value_dist, key_dist). CH?t g a09ec2d88823 SELECT pr_number, title, author, labels, files, merged_at, opened_at, linked_issues, lexemes FROM loom.code_pr_history FINAL WHERE org = {org:String} AND namespace = {ns:String} AND tombstoned_at IS N +a0b4dab9a372 SELECT *, hex(SHA256(toJSONString((value_json, value)))) AS body_sha FROM (SELECT org, namespace, agent, user_id, session_id, ? AS actor, concat(?, session_id) AS memory_key, version, ? AS value_json, a0be1210dc79 SELECT count() AS n FROM loom.memory_ask_precedents WHERE org = {org:String} AND namespace = {ns:String} AND ? a0e30a489601 SELECT * EXCEPT (embedding) FROM loom.code_chunks FINAL WHERE org = {org:String} AND namespace = {ns:String} AND path = {path:String} AND embedding_model = {model:String} AND tombstoned_at IS NULL a0f9368cb508 SELECT count() AS n FROM loom.code_answers WHERE org = {org:String} AND hasAny(memory_keys, arrayConcat((SELECT groupUniqArray(memory_key) FROM loom.memory WHERE org = {org:String} AND user_id = {uid: @@ -1430,6 +1447,7 @@ a543652b324c SELECT org, namespace, agent, user_id, session_id, run_id, goal, us a55966ee4380 SELECT f.lang AS lang, toString(s.tier) AS tier, count() AS symbols, max(s.created_at) AS last_at FROM ( SELECT path, tier, created_at FROM loom.code_symbols FINAL WHERE org = {org:String} AND namespa a5598bacc84d SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, e.status AS status, e.version AS version, -- Multi-vector axis: min(value_dist, key_dist). CH?t g a563ab145bc6 SELECT org, namespace, agent, thread_id, msg_id, role, content, reasoning, user_id, run_id, metadata, tokens, ts FROM loom.conversations WHERE org = {org:String} AND namespace IN {nss:Array(String)} A +a56fae89e202 SELECT *, hex(SHA256(toJSONString((value_json, value)))) AS body_sha FROM (SELECT org, namespace, agent, user_id, session_id, ? AS actor, concat(?, toString(evidence_id)) AS memory_key, toUInt64(toUni a5a4769e8d84 ALTER TABLE loom.memory DELETE WHERE (org = {org:String} AND namespace IN {ns:Array(String)}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync = ? a5c02075396f SELECT c.category_id AS category_id, count() AS rows FROM (SELECT memory_key, category_id FROM loom.memory_categories FINAL WHERE org = {org:String} AND namespace = {ns:String} AND tombstoned_at IS NU a5e98e7d16d8 SELECT m.memory_id, m.memory_key, m.value, m.pinned, m.agent, m.user_id, m.session_id, m.run_id, m.valid_at, e.dist AS dist FROM loom.memory m FINAL JOIN ( SELECT agent, user_id, session_id, memory_ke @@ -1441,7 +1459,6 @@ a627d0337788 ALTER TABLE loom.memory_ask_precedents DELETE WHERE (org = {org:Str a63dc159ea03 SELECT count() AS n, argMax(role, ts) AS role, argMax(content, ts) AS content, argMax(run_id, ts) AS run_id, max(ts) AS last_ts, max(saved_at) AS last_saved_at, groupUniqArray(user_id) AS owners, coun a64937812d3d SELECT org, namespace, toString(memory_id) AS memory_id, sum(recall_count) AS recall_total, sum(evidence_count) AS evidence_total, sum(used_count) AS used_total, sum(harm_count) AS harm_total, max(las a661d7b26199 ALTER TABLE loom.memory_extraction_queue DELETE WHERE (org = {org:String} AND user_id = {uid:String}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync = ? -a662cd95ca59 SELECT memory_key, (status != ? OR tombstoned_at IS NOT NULL OR redacted_at IS NOT NULL) AS taken_out FROM loom.memory FINAL WHERE org = {org:String} AND namespace = {ns:String} AND agent = {agent:Str a67dc6a1b98a SELECT path, start_line, end_line, content_hash FROM loom.code_chunks FINAL WHERE org = {org:String} AND namespace = {ns:String} AND path IN {paths:Array(String)} AND tombstoned_at IS NULL AND start_l a68b12d4c3b3 SELECT memory_type, count() AS c FROM loom.memory FINAL WHERE org = {o:String} AND namespace = {n:String} AND tombstoned_at IS NULL AND status = ? AND memory_key NOT LIKE ? GROUP BY memory_type a6ae128a7aef SELECT callee_name, path, line, from_symbol_id, argMax(from_qualified_name, version) AS from_qualified_name, argMax(call_text, version) AS call_text, argMax(disposition, version) AS disposition, argMa @@ -1509,8 +1526,10 @@ acdf77ff85a4 SELECT count() AS n FROM loom.shared FINAL WHERE org = {org:String} acfa8ad8dceb SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, e.status AS status, e.version AS version, cosineDistance(e.embedding, {q:Array(Float32)}) AS dist ad229cd4a4cf SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, e.status AS status, e.version AS version, -- Multi-vector axis: min(value_dist, key_dist). CH?t g ad2a1b0821de DELETE FROM loom.memory_embeddings WHERE (org = {org:String} AND namespace IN {ns:Array(String)}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS lightweight_deletes_sync +ad2f3798c352 SELECT ? FROM loom.memory_refusals WHERE org={org:String} AND ((scope_kind IN (?) AND scope_value={ns:String} AND user_id={uid:String}) OR (scope_kind=? AND scope_value={ns:String}) OR (scope_kind=? A ad39ba238825 SELECT count() AS n FROM loom.working_memory FINAL WHERE org = {org:String} AND namespace IN {ns:Array(String)} ad5724799b14 SELECT namespace, source_uri, chunk_idx, doc_version, JSONExtractUInt(chunk_metadata, ?) AS revision, content, context, tags, chunk_metadata, ingested_at, agent, session_id, toString(source_class) AS +ad575f71f510 SELECT *, hex(SHA256(toJSONString((value_json, value)))) AS body_sha FROM (SELECT org, namespace, agent, user_id, ? AS session_id, ? AS actor, concat(?, entity) AS memory_key, version, ? AS value_json ad69a81ba9d2 SELECT queue_id, agent, user_id, session_id, status, attempts, message_count, content_chars, written, last_error, enqueued_at, updated_at FROM loom.memory_extraction_queue FINAL WHERE org = {org:Strin ad717da278a8 SELECT count() AS n FROM loom.entity_edge_rows WHERE org = {org:String} AND (user_id = {uid:String} OR (namespace, memory_id) IN (SELECT toString(namespace), memory_id FROM (SELECT namespace, agent, u ad8e16687ea3 SELECT count() AS n FROM loom.query_history_sources WHERE org = {org:String} AND namespace = {ns:String} @@ -1524,7 +1543,6 @@ ae7d2da05030 SELECT count() AS n FROM loom.memory_extraction_queue WHERE org = { ae99c20b3d1e SELECT concat(?, toString(c.session_id), ?, toString(c.chunk_index)) AS memory_key, ? AS memory_type, substring(c.content, ?, ?) AS content_excerpt, c.content AS content_full, c.agent AS agent, c.user af399844e950 ALTER TABLE loom.memory_record_fields DELETE WHERE (org = {org:String} AND record_id IN {ids:Array(UUID)}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync = af8aae57db93 SELECT count() AS n FROM loom.memory_chunk_sources FINAL WHERE org = {org:String} AND tombstoned_at IS NULL AND namespace = {ns:String} -af9585af4ffb SELECT session_id, role, toString(observed_at) AS observed_at, content FROM ( SELECT session_id, role, observed_at, content, evidence_id, tombstoned_at FROM loom.memory_evidence WHERE org = {org:Strin afdcda2bf23a SELECT count() AS n FROM loom.memory_ask_precedents WHERE org = {org:String} AND hasAny(arrayConcat(window_keys, cited_keys), arrayConcat((SELECT groupUniqArray(memory_key) FROM loom.memory WHERE org aff1e632ca9c SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, e.status AS status, e.version AS version, e.agent AS agent, e.user_id AS user_id, e.session_id AS b0198f2a1337 SELECT uniqExact(path) AS n FROM loom.code_chunks FINAL WHERE org = {org:String} AND namespace = {ns:String} AND embedding_model = {model:String} AND tombstoned_at IS NULL AND startsWith(path, {prefix @@ -1580,7 +1598,6 @@ b517057ddbb9 SELECT count() AS n FROM loom.memory_ask_precedents WHERE org = {or b546d3bd8553 SELECT count() AS n FROM loom.entity_edge_memories WHERE org = {org:String} AND namespace = {ns:String} AND ? b556883f0824 ALTER TABLE loom.entity_triple_memories DELETE WHERE (org = {org:String} AND namespace = {ns:String}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync = ? b57226087b36 ALTER TABLE loom.memory DELETE WHERE (org = {org:String} AND namespace = {ns:String}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync = ? -b583fd3cb60b SELECT agent, user_id, session_id, version, summary_text, cosineDistance(summary_embedding, {q:Array(Float32)}) AS distance, -- NB: alias must NOT be `valid_at` — under the analyzer a -- SELECT alias b599a544d8e9 SELECT count() AS n FROM loom.documents FINAL WHERE org = {org:String} AND namespace = {ns:String} AND superseded_at IS NULL b5c1f249d43b ALTER TABLE loom.delete_version_checks DELETE WHERE (org = {org:String} AND user_id = {uid:String}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync = ? b5cfdd2b8732 DELETE FROM loom.memory_embeddings WHERE (org = {org:String} AND scope = ? AND (namespace, memory_key) IN {keys:Array(Tuple(String, String))}) AND ({erasure_mutation_tag:String} = {erasure_mutation_ta @@ -1659,6 +1676,7 @@ bcf5fd2c0231 SELECT count() AS n FROM ( SELECT evidence_id, tombstoned_at FROM l bd095b55bf5d SELECT count() AS n FROM loom.memory_embeddings WHERE org = {org:String} AND namespace = {ns:String} AND agent = {a:String} AND user_id = {u:String} AND session_id = {s:String} AND memory_key = {k:Str bd4266cbfe15 SELECT count() AS n FROM loom.conflict_allowlist FINAL WHERE org = {org:String} AND tombstoned_at IS NULL AND namespace = {ns:String} bda43b1f632f WITH arrayConcat(splitByNonAlpha(context_lc), splitByNonAlpha(content_lc)) AS words SELECT namespace, source_uri, chunk_idx, doc_version, JSONExtractUInt(chunk_metadata, ?) AS revision, cosineDistance +bdde77a5c044 SELECT *, hex(SHA256(toJSONString((value_json, value)))) AS body_sha FROM (SELECT org, namespace, agent, user_id, session_id, ? AS actor, concat(?, toString(evidence_id)) AS memory_key, toUInt64(toUni bde7c432bca1 ALTER TABLE loom.entity_communities DELETE WHERE (org = {org:String} AND (cityHash64(namespace, entity) IN {entities:Array(UInt64)} OR cityHash64(namespace, label) IN {entities:Array(UInt64)})) AND ({ bde98d358a2b SELECT path, uniqExact(facet_kind, name) AS matched, groupArray((facet_kind, name, ordinal, line, confidence)) AS hits, length(arrayIntersect(groupUniqArrayIf(ordinal, facet_kind = {k0:String} AND nam be29d7515a0a SELECT count() AS n FROM loom.memory_doc_stats WHERE org = {org:String} AND namespace = {ns:String} @@ -1673,6 +1691,7 @@ bef8fe4e58b7 SELECT tier, edge_kind, to_symbol_id, from_symbol_id, path, line, a bf09992e73fc SELECT memory_key, toString(memory_id) AS id, version, memory_type, toString(status) AS status, source_class, agent, user_id, session_id, actor, updated_at, sensitivity, org, namespace, redacted_at FR bf0c10f9a95a SELECT namespace, source_uri, chunk_idx, doc_version, JSONExtractUInt(chunk_metadata, ?) AS revision FROM loom.chunks WHERE org = {org:String} AND namespace IN {nss:Array(String)} AND embedding_model bf193ac927b4 SELECT count() AS n FROM loom.memory_record_conflicts FINAL WHERE org = {org:String} AND namespace = {ns:String} +bf1ce8fef88c SELECT agent, user_id, session_id, version, summary_text, cosineDistance(summary_embedding, {q:Array(Float32)}) AS distance, -- NB: alias must NOT be `valid_at` — under the analyzer a -- SELECT alias bf4ffadd242a SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, e.status AS status, e.version AS version, -- Multi-vector axis: min(value_dist, key_dist). CH?t g bfbdb1dc9399 SELECT namespace, count() AS turns, uniqExact(session_id) AS sessions, sum(toUInt64(input_tokens)) AS input_tokens, sum(toUInt64(cache_read_tokens)) AS cache_read_tokens, sum(toUInt64(cache_creation_5 bfc004b68fb9 DELETE FROM loom.memory_embeddings WHERE org = {org:String} AND namespace = {ns:String} AND agent = {a:String} AND user_id = {u:String} AND session_id = {s:String} AND memory_key = {k:String} AND (((c @@ -1716,7 +1735,7 @@ c46246e37ab2 SELECT count() AS n FROM loom.derived_facts WHERE org = {org:String c464d4b1053c SELECT count() AS n FROM loom.knowledge_uploads WHERE org = {org:String} AND user_id = {uid:String} AND ? c466198a7fda WITH (SELECT toFloat64(count()) FROM loom.memory_embeddings FINAL WHERE org = {org:String} AND namespace = {ns:String} AND tombstoned_at IS NULL) AS n_keys, (SELECT mapFromArrays(groupArray(s), groupA c4831b86cc45 SELECT now64(?) AS now, groupArray((token, started_at, finished)) AS keys FROM loom.graph_projections WHERE token IN {tokens:Array(String)} -c4c23cde5373 SELECT agent, user_id, session_id, version, summary_text, cosineDistance(summary_embedding, {q:Array(Float32)}) AS distance, -- NB: alias must NOT be `valid_at` — under the analyzer a -- SELECT alias +c48726998045 SELECT agent, user_id, session_id, embedding_model, summary_text, key_facts, derived_from, valid_at, created_at, version, length(key_facts) AS kf, length(derived_from) AS df, coalesce(valid_at, create c4cee4797736 SELECT count() AS sources, countIf(j.has_document = ? AND NOT j.replaced) AS without_document, countIf(j.replaced) AS replaced, sum(j.pending) AS pending, sum(j.processing) AS processing, sum(j.ready) c4d11f8d928a SELECT count() AS n FROM loom.memory_record_fields FINAL WHERE org = {org:String} AND tombstoned_at IS NULL AND namespace = {ns:String} c4d8e27d9015 SELECT tier, qualified_name, symbol_id, name, path, start_line, end_line FROM loom.code_symbols FINAL WHERE org = {org:String} AND namespace = {ns:String} AND (tier, qualified_name, symbol_id) IN ( SE @@ -1825,6 +1844,7 @@ cec3f5ad060e SELECT org, namespace, agent, user_id, session_id, memory_id, memor cee7e1df6fac SELECT jti, revoked_at, expires_at, reason, revoked_by, issued_before, version FROM loom.revoked_tokens FINAL WHERE jti = {j:String} AND expires_at > now() cf06ff759823 SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, e.status AS status, e.version AS version, -- Multi-vector axis: min(value_dist, key_dist). CH?t g cf0cc9e2222f SELECT count() AS n FROM loom.derived_facts WHERE org = {org:String} AND namespace = {ns:String} +cf104f8e96d5 SELECT ? FROM loom.memory_extraction_queue FINAL WHERE org={org:String} AND namespace={namespace:String} AND agent={agent:String} AND user_id={user_id:String} AND session_id={session_id:String} AND qu cf1e114983a0 SELECT count() AS n FROM loom.memory_embeddings FINAL WHERE org = {org:String} AND tombstoned_at IS NULL AND has(tags, ?) AND hasAny(tags, {src:Array(String)}) AND hasAny(arrayMap(t -> (namespace, t), cf1f27acd065 SELECT agent, user_id, session_id, memory_key, memory_type, content_excerpt, toString(created_at) AS created_at, version FROM loom.memory_embeddings FINAL WHERE org = {org:String} AND namespace = {ns: cf3d0a421f45 SELECT s.org AS org, s.namespace AS namespace, s.schema_json AS schema_json, s.schema_sha256 AS schema_sha256 FROM loom.profile_schemas AS s FINAL WHERE s.tombstoned_at IS NULL AND NOT has({off:Array( @@ -1874,7 +1894,6 @@ d3ec29699853 SELECT count() AS n FROM loom.code_answers WHERE org = {org:String} d41fb49bc0a1 SELECT concat(?, toString(c.session_id), ?, toString(c.chunk_index)) AS memory_key, ? AS memory_type, substring(c.content, ?, ?) AS content_excerpt, c.content AS content_full, c.agent AS agent, c.user d44a207c193a SELECT concat(?, toString(c.session_id), ?, toString(c.chunk_index)) AS memory_key, ? AS memory_type, substring(c.content, ?, ?) AS content_excerpt, c.content AS content_full, c.agent AS agent, c.user d455eae6762a SELECT count() AS n FROM loom.procedure_runs WHERE org = {org:String} AND namespace = {ns:String} -d462d47f669a SELECT agent, user_id, session_id, version, summary_text, cosineDistance(summary_embedding, {q:Array(Float32)}) AS distance, -- NB: alias must NOT be `valid_at` — under the analyzer a -- SELECT alias d469073eeef7 SELECT subject, relation, object, sum(co_occur) AS co_occur, max(last_seen) AS last_seen, -- The strongest tier behind the aggregated triple. -- Not `AS provenance`: with the provenance filter in WHER d4a86d775cfd SELECT path, chunk_id, argMax(tombstoned_at IS NULL, version) AS is_live FROM loom.code_chunks WHERE org = {org:String} AND namespace = {ns:String} AND embedding_model = {model:String} AND path IN {pa d4dce40507f9 SELECT state_key, state_value, version FROM loom.code_index_state FINAL WHERE org = ? AND namespace = {ns:String} AND state_key IN {keys:Array(String)} @@ -1899,8 +1918,10 @@ d749ae65d41a SELECT toString(memory_id) AS id, argMax(status, version) AS status d76186bb002a SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, e.status AS status, e.version AS version, -- Multi-vector axis: min(value_dist, key_dist). CH?t g d77696150f16 SELECT id, org, scope_type, scope_value, requester, actor, urgency, purge, mirror_marked, attempts, steps_completed, requested_at, failure_reason, residue_count, residue, verified_at, census FROM loom d7794951d874 SELECT count() AS n FROM loom.code_search_trace WHERE org = {org:String} AND namespace = {ns:String} +d782528717e8 ALTER TABLE loom.memory_extraction_queue UPDATE status=?, last_error=?, lease_until=toDateTime64(?,?) WHERE org={org:String} AND namespace={namespace:String} AND agent={agent:String} AND user_id={user d791a50120b8 SELECT count() AS n FROM loom.memory FINAL WHERE org = {org:String} AND tombstoned_at IS NULL AND user_id = {uid:String} d795664c37b5 WITH reversals AS ( SELECT erasure_id, count() AS n FROM loom.erasure_reversals FINAL WHERE org = {org:String} GROUP BY erasure_id ) SELECT e.id AS id, e.scope_type AS scope_type, e.scope_value AS sco +d7ac29f6b87f SELECT *, hex(SHA256(toJSONString((value_json, value)))) AS body_sha FROM (SELECT org, namespace, agent, user_id, session_id, ? AS actor, concat(?, toString(evidence_id)) AS memory_key, toUInt64(toUni d7ad5fb147fc SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, ? - cosineDistance(e.embedding, {qv:Array(Float32)}) AS similarity, toString(coalesce(e.temporal_ d7c28db7e874 SELECT org, code_namespace, path, qualified_name, namespace, memory_id FROM loom.memory_code_links WHERE org >= {a0:String} AND (org > {a0:String} OR (code_namespace >= {a1:String} AND (code_namespace d7cfaad88c70 SELECT toString(memory_id) AS memory_id, version, value, tags, valid_at, pinned, lock_mode, lock_until, sensitivity FROM loom.memory FINAL WHERE org = {org:String} AND namespace = {ns:String} AND user @@ -1967,6 +1988,7 @@ dd9185239ef3 ALTER TABLE loom.memory_extraction_queue UPDATE status = ?, attempt dd94ab4c37d1 SELECT count() AS n FROM loom.working_memory FINAL WHERE org = {org:String} AND namespace = {ns:String} AND tombstoned_at IS NULL dda464c89047 SELECT neighbor, role, sum(co_occur) AS co_occur FROM ( SELECT object AS neighbor, ? AS role, sum(co_occur) AS co_occur FROM loom.entity_triples WHERE org = {org:String} AND namespace = {ns:String} AN ddb1a5ea55c9 SELECT count() AS n FROM loom.memory_record_fields FINAL WHERE org = {org:String} AND namespace IN {ns:Array(String)} AND tombstoned_at IS NULL +ddc00b268be8 SELECT *, hex(SHA256(toJSONString((value_json, value)))) AS body_sha FROM (SELECT org, namespace, agent, user_id, session_id, ? AS actor, concat(?, queue_id) AS memory_key, version, ? AS value_json, t dde8208a93a2 SELECT count() AS n FROM loom.entity_triple_memories WHERE org = {org:String} AND namespace IN {ns:Array(String)} AND ? ddfb30beb525 SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, e.status AS status, e.version AS version, -- Multi-vector axis: min(value_dist, key_dist). CH?t g de25c2cb1919 SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, e.status AS status, e.version AS version, -- Multi-vector axis: min(value_dist, key_dist). CH?t g @@ -1995,11 +2017,13 @@ dfab46389b75 SELECT count() AS n FROM loom.memory_record_versions WHERE org = {o dfb389c3eaaa SELECT count() AS n FROM loom.memory_records FINAL WHERE org = {org:String} AND namespace = {ns:String} AND tombstoned_at IS NULL e01b3f138368 ALTER TABLE loom.memory_ask_precedents DELETE WHERE (org = {org:String} AND hasAny(arrayConcat(window_keys, cited_keys), {keys:Array(String)})) AND ({erasure_mutation_tag:String} = {erasure_mutation_t e026c7f5b6eb SELECT memory_type, toString(status) AS status, source_class, count() AS rows, min(updated_at) AS first_at, max(updated_at) AS last_at FROM loom.memory FINAL WHERE org = {org:String} AND namespace = { +e05f46166d69 SELECT org, namespace, agent, user_id, session_id, embedding_model, version, summary_text, key_facts, derived_from, at FROM (SELECT org, namespace, agent, user_id, session_id, embedding_model, version e08e70804055 SELECT version, toStringCutToZero(content_sha256) AS sha FROM loom.memory_versions WHERE org = {org:String} AND namespace = {ns:String} AND agent = {agent:String} AND user_id = {u:String} AND session_ e094b0459706 SELECT count() AS n FROM loom.knowledge_ingest_cancels WHERE org = {org:String} AND user_id = {uid:String} e0a2fdcb10f9 SELECT count() AS n FROM loom.knowledge_ingest_cancels WHERE org = {org:String} AND namespace = {ns:String} AND ? e0a394a9b663 SELECT pattern_id, org, namespace, pattern, pattern_type, scope, reason, created_by, hit_count, last_hit_at, created_at FROM loom.blacklist FINAL WHERE tombstoned_at IS NULL AND ((org = {org:String} A e0ce2526325b SELECT count() AS n FROM loom.operator_edges FINAL WHERE org = {org:String} AND namespace = {ns:String} +e0f548d3f7a8 SELECT agent, user_id, session_id, embedding_model, summary_text, key_facts, derived_from, valid_at, created_at, version, length(key_facts) AS kf, length(derived_from) AS df, coalesce(valid_at, create e0f90e7e0243 ALTER TABLE loom.memory_embeddings UPDATE tombstoned_at = {now:DateTime} WHERE (org = {org:String} AND tombstoned_at IS NULL AND namespace = {ns:String}) AND ({erasure_mutation_tag:String} = {erasure_ e107d944ecff SELECT n.namespace AS namespace, n.description AS description, n.kind AS kind, n.owner_email AS owner_email, n.requires_approval AS requires_approval, toString(n.visibility) AS visibility, toString(n. e10df0e1e1d9 SELECT count() AS n FROM loom.memory_chunk_sources FINAL WHERE org = {org:String} AND tombstoned_at IS NULL AND namespace IN {ns:Array(String)} @@ -2042,6 +2066,7 @@ e4e978084474 SELECT namespace, kind, toString(visibility) AS visibility, title, e5024546eff6 SELECT count() AS n FROM loom.knowledge_group_members WHERE org = {org:String} AND namespace = {ns:String} e51463feccae SELECT namespace, sum(upstream_cost_usd) AS cost_usd, sum(toUInt64(input_tokens) + toUInt64(output_tokens)) AS tokens, sum(toUInt64(embedding_tokens)) AS embed_tokens, count() AS ops, countIf(mirror_m e528e5544515 SELECT count() AS n FROM loom.document_blobs WHERE org = {org:String} AND namespace = {ns:String} AND ? +e52c0bc8a1ba SELECT *, hex(SHA256(toJSONString((value_json, value)))) AS body_sha FROM (SELECT org, namespace, agent, user_id, ? AS session_id, ? AS actor, concat(?, entity) AS memory_key, version, ? AS value_json e583636bd338 ALTER TABLE loom.entity_edge_memories DELETE WHERE (org = {org:String} AND (namespace, memory_key) IN {keys:Array(Tuple(String, String))}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:St e5977d678a52 ALTER TABLE loom.code_search_trace DELETE WHERE (org = {org:String} AND namespace = {ns:String}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync = ? e5a6b0c13d38 SELECT chunk_id, path, scope, kind, start_line, end_line, content, symbol_ids, context_description FROM loom.code_chunks FINAL WHERE (org, namespace, embedding_model, path, chunk_id) IN {keys:Array(Tu @@ -2051,7 +2076,9 @@ e5d3479fff14 SELECT agent, user_id, session_id, memory_key, cosineDistance(embed e5df581175ec SELECT count() AS n FROM loom.knowledge_gaps WHERE org = {org:String} AND namespace IN {ns:Array(String)} e5e482e9d615 SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, e.status AS status, e.version AS version, -- Multi-vector axis: min(value_dist, key_dist). CH?t g e60347067264 SELECT DISTINCT org, namespace, agent, user_id, session_id, memory_key FROM loom.memory WHERE tombstoned_at IS NULL AND sensitivity = ? AND (expires_at IS NULL OR expires_at > now()) AND (match(value, +e62faca61816 SELECT agent, user_id, session_id, embedding_model, summary_text, key_facts, derived_from, valid_at, created_at, version, length(key_facts) AS kf, length(derived_from) AS df, coalesce(valid_at, create e67a89c4f6c6 SELECT group_id, members FROM loom.principal_groups FINAL WHERE org = {org:String} AND NOT deleted +e68412b65969 SELECT *, hex(SHA256(toJSONString((value_json, value)))) AS body_sha FROM (SELECT org, namespace, agent, user_id, session_id, ? AS actor, memory_key, version, toString(value_json) AS value_json, value e6990fcc8c82 SELECT thread_id, msg_id, role, content_excerpt AS embedded_text, ts, cosineDistance(embedding, {q:Array(Float32)}) AS distance FROM loom.conversation_embeddings FINAL WHERE org = {org:String} AND nam e6f6f6bc74e2 SELECT count() AS n FROM loom.memory_chunks FINAL WHERE org = {org:String} AND tombstoned_at IS NULL AND namespace IN {ns:Array(String)} e7331ee31d1b SELECT concat(?, toString(c.session_id), ?, toString(c.chunk_index)) AS memory_key, ? AS memory_type, substring(c.content, ?, ?) AS content_excerpt, c.content AS content_full, c.agent AS agent, c.user @@ -2062,6 +2089,7 @@ e77afbb3fb28 SELECT count() AS n FROM loom.chunks WHERE org = {org:String} AND n e77b590b9f54 SELECT org, namespace, name, source_id, provider, generation, fingerprint, provider_id, status, secret_enc, agent, user_id, session_id, actor FROM loom.knowledge_source_push FINAL WHERE org = {org:Str e77c1e05f3c4 SELECT count() AS n FROM loom.blob_uploads WHERE org = {org:String} AND namespace IN {ns:Array(String)} AND ? e7d4c583cc0e ALTER TABLE loom.procedure_runs DELETE WHERE (org = {org:String} AND namespace = {ns:String}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync = ? +e7ef19d41a90 SELECT agent, user_id, session_id, embedding_model, summary_text, key_facts, derived_from, valid_at, created_at, version, length(key_facts) AS kf, length(derived_from) AS df, coalesce(valid_at, create e7f1d693d5b1 SELECT count() AS n FROM loom.entity_vectors WHERE org = {org:String} AND (user_id = {uid:String} OR (namespace, memory_key) IN {keys:Array(Tuple(String, String))}) e8009bd53bc4 SELECT number, head_sha FROM loom.code_gh_items FINAL WHERE org = {org:String} AND namespace = {ns:String} AND tombstoned_at IS NULL AND is_pr AND state = ? AND head_sha != ? ORDER BY updated_at DESC, e82345a88eca SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, e.status AS status, e.version AS version, -- Multi-vector axis: min(value_dist, key_dist). CH?t g @@ -2175,11 +2203,11 @@ f516a9cc81d3 SELECT * FROM loom.entity_vectors FINAL WHERE org = {org:String} AN f51d6af32512 SELECT e.org AS org, e.namespace AS namespace, e.agent AS agent, e.user_id AS user_id, e.session_id AS session_id, e.memory_key AS memory_key FROM (SELECT org, namespace, agent, user_id, session_id, m f51e0ce42dcd SELECT count() OVER () AS total, memory_key, toString(memory_id) AS memory_id, expires_at FROM loom.memory FINAL WHERE org = {org:String} AND namespace = {ns:String} AND tombstoned_at IS NULL AND (emp f52e30d82733 SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, e.status AS status, e.version AS version, -- Multi-vector axis: min(value_dist, key_dist). CH?t g -f5622add735c SELECT session_id, summary_text, length(key_facts) AS kf, length(derived_from) AS df, coalesce(valid_at, created_at) AS d FROM loom.session_summaries FINAL WHERE org = {o:String} AND namespace = {n:St f5b6a3a81817 SELECT path, scope, symbol_ids, kind, start_line, end_line, content FROM ( SELECT path, scope, symbol_ids, kind, start_line, end_line, content, chunk_id, tombstoned_at FROM loom.code_chunks WHERE org f5ddaf977ab5 SELECT state_key, state_value AS value FROM loom.code_index_state FINAL WHERE org = {org:String} AND namespace = {ns:String} AND state_key IN {keys:Array(String)} f60a04215196 SELECT concat(?, toString(c.session_id), ?, toString(c.chunk_index)) AS memory_key, ? AS memory_type, substring(c.content, ?, ?) AS content_excerpt, c.content AS content_full, c.agent AS agent, c.user f617d9867707 SELECT count() AS n FROM loom.memory_code_links WHERE org = {org:String} AND namespace IN {ns:Array(String)} +f6389b5cd8c4 -- schemas/schema.sql -- -- Reference dump of `loom.*` and `loom_telemetry.*`: what replaying schemas/migrations/*.sql -- produces, auto-generated by scripts/ops/dump_schema.py from a live -- ClickHou f646cb42e8e3 ALTER TABLE loom.derived_facts DELETE WHERE (org = {org:String} AND namespace IN {ns:Array(String)}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync = ? f65fc198c24d ALTER TABLE loom.memory_record_conflicts UPDATE tombstoned_at = {now:DateTime} WHERE (org = {org:String} AND tombstoned_at IS NULL AND namespace IN {ns:Array(String)}) AND ({erasure_mutation_tag:Strin f6669612c23e SELECT p.pr_number AS pr_number, p.merge_commit_sha AS merge_commit_sha, p.pack_tokens AS pack_tokens, p.created_at AS created_at, ifNull(h.title, ?) AS title, ifNull(h.author, ?) AS author, h.merged_ @@ -2244,6 +2272,7 @@ fcb2b681c3cc SELECT pr_number, title, author, files, merged_at, linked_issues, a fcf47e494773 SELECT count() AS n FROM loom.knowledge_group_members WHERE org = {org:String} AND namespace IN {ns:Array(String)} fd4b9de98c6f SELECT count() AS live_docs, countIf(ifNull(c.live, ?) = ?) AS without_chunks, groupArrayIf(?)(d.source_uri, ifNull(c.live, ?) = ?) AS without_chunks_uris, countIf(ifNull(c.live, ?) = ? AND d.ingestio fd68c20c1c4c SELECT content_sha256, count() AS doc_count, groupArray(?)((doc_id, source_uri, title, ingested_at)) AS docs FROM loom.documents FINAL WHERE org = {org:String} AND namespace = {ns:String} AND supersed +fd694d19d0d4 SELECT DISTINCT toInt64(units) AS units FROM loom.memory_refusals WHERE org={org:String} AND ((scope_kind IN (?,?) AND scope_value={ns:String} AND user_id={uid:String}) OR (scope_kind=? AND scope_valu fd8bd0425be6 INSERT INTO loom.knowledge_ingest_cancels (org, namespace, ingestion_id, requested_at) SELECT org, namespace, ingestion_id, requested_at FROM loom.knowledge_ingest_cancels WHERE org = {org:String} AND fd94b00f3b5d SELECT concat(?, toString(c.session_id), ?, toString(c.chunk_index)) AS memory_key, ? AS memory_type, substring(c.content, ?, ?) AS content_excerpt, c.content AS content_full, c.agent AS agent, c.user fd95e7268232 ALTER TABLE loom.events DELETE WHERE (org = {org:String} AND namespace IN {ns:Array(String)}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync = ? @@ -2264,3 +2293,4 @@ ff23e7932268 ALTER TABLE loom.working_memory DELETE WHERE (org = {org:String} AN ff438b330e4f SELECT count() AS n FROM loom.memory FINAL WHERE org = {org:String} AND namespace = {ns:String} AND tombstoned_at IS NULL AND status != ? AND session_id = {sid:String} AND startsWith(memory_key, {p:St ffa5c94364aa SELECT DISTINCT namespace FROM ( SELECT namespace FROM loom.memory WHERE org = {org:String} AND (namespace, agent, user_id, session_id, memory_key) IN ( SELECT namespace, agent, user_id, session_id, m ffa9b9e1eddc SELECT c.category_id AS category_id, count() AS rows FROM (SELECT memory_key, category_id FROM loom.memory_categories FINAL WHERE org = {org:String} AND namespace = {ns:String} AND tombstoned_at IS NU +ffbcd8654cca SELECT *, hex(SHA256(toJSONString((value_json, value)))) AS body_sha FROM (SELECT org, namespace, agent, user_id, session_id, ? AS actor, memory_key, version, toString(value_json) AS value_json, value diff --git a/tests/integration/_memory.py b/tests/integration/_memory.py new file mode 100644 index 0000000000..af1fe26048 --- /dev/null +++ b/tests/integration/_memory.py @@ -0,0 +1,49 @@ +"""Canonical retirement fixtures that do not express permanent content refusal.""" + +from server.core.ch import get_client +from server.identity.domain.read_scope import WHOLE_NAMESPACE +from server.memory.api.models_crud import MemoryDeleteInput, MemoryDeleteOutput +from server.memory.store.rows import Identity, version_over + + +async def retire_reversibly(input: MemoryDeleteInput) -> MemoryDeleteOutput: + """Seed an automatic or legacy tombstone through the real canonical write path. + + Restore tests use this state to keep covering resurrection without cancelling + the permanent refusal expressed by an explicit deletion. + """ + import server.memory.store as memory_repo + from server.memory.app.handlers import _retire_derived, _write_tombstone_row + + ch = get_client() + connected_here = not ch.connected + if connected_here: + await ch.connect() + try: + source = await memory_repo.get_by_key( + Identity( + org=input.org, + namespace=input.namespace, + agent=input.agent, + user_id=input.user_id, + session_id=input.session_id, + ), + input.memory_key, + fresh=True, + strong=True, + scope=WHOLE_NAMESPACE, + ch=ch, + ) + assert source is not None, "the reversible retirement fixture needs a live source" + version = version_over(source) + await _write_tombstone_row(source, version=version, refuse_value=False) + tombstoned, downgraded = await _retire_derived(source, version=version, op="memory.expire") + return MemoryDeleteOutput( + tombstoned=True, + version=version, + dependents_tombstoned=tombstoned, + dependents_downgraded=downgraded, + ) + finally: + if connected_here: + await ch.disconnect() diff --git a/tests/integration/same_key/_rows.py b/tests/integration/same_key/_rows.py index eeb8aee21d..64cb1c47da 100644 --- a/tests/integration/same_key/_rows.py +++ b/tests/integration/same_key/_rows.py @@ -853,18 +853,20 @@ async def _open_edges(ch: Any, org: str, want: set[str]) -> list[str]: async def _retired_embedding(ch: Any, row: dict[str, Any]) -> None: """`row`'s search row as its retirement left it: superseded, with a vector.""" + from server.core.hashing import served_row_value, sha256_value from server.memory import store as M - cols = ("org", "namespace", "agent", "user_id", "session_id", "memory_key", "content_sha256") + cols = ("org", "namespace", "agent", "user_id", "session_id", "memory_key") emb = M.build_embedding_row( **{c: row[c] for c in cols}, + content_sha256=sha256_value(served_row_value(row)), embedding_model="itest", embedding_dim=3072, embedding=[0.01] * 3072, status="superseded", invalid_at=naive_utc_now(), ) - await M.insert_embedding_rows([emb], ch=ch) + assert await M.insert_embedding_rows([emb], ch=ch) == [emb] async def _embedding_status(ch: Any, row: dict[str, Any]) -> str: diff --git a/tests/integration/test_blacklist.py b/tests/integration/test_blacklist.py index 173ab3edb5..c126b51a5d 100644 --- a/tests/integration/test_blacklist.py +++ b/tests/integration/test_blacklist.py @@ -63,6 +63,8 @@ async def test_matching_write_rejected_with_422( from loom import Agent from loom.errors import LoomError + from tests.integration._eventually import eventually + ns = f"bl-match-{uuid.uuid4().hex[:6]}" tok = mint_test_token(org=ORG, namespaces=[ns], roles=["admin"]) async with Agent( @@ -90,9 +92,14 @@ async def test_matching_write_rejected_with_422( ok = await a.memory.set("other", "unrelated content", memory_type="semantic") assert ok["content_sha256"] - # List reports the hit_count incremented - rows = await a.blacklist.list() - ours = [r for r in rows if r["pattern_id"] == added["pattern_id"]] + # The hit counter is an asynchronous mutation. + async def counted(): + rows = await a.blacklist.list() + ours = [r for r in rows if r["pattern_id"] == added["pattern_id"]] + assert len(ours) == 1 + return ours if int(ours[0]["hit_count"]) >= 1 else None + + ours = await eventually(counted, description="blacklist hit count to be published") assert len(ours) == 1 assert int(ours[0]["hit_count"]) >= 1 @@ -173,6 +180,16 @@ async def persisted(table): after = [await persisted(table) for table in ("memory", "memory_embeddings")] assert after == before, "a refusal changes no persisted memory or search field" await agent.memory.set("allowed", "unrelated value", memory_type="semantic") - await agent.memory.delete("allowed") + from server.memory.api.models_crud import MemoryDeleteInput + from tests.integration._memory import retire_reversibly + + await retire_reversibly( + MemoryDeleteInput( + org=ORG, + namespace=ns, + agent="t", + memory_key="allowed", + ) + ) restored = await agent.memory.restore("allowed") assert restored["restored"] is True diff --git a/tests/integration/test_conflict_resolver_ch.py b/tests/integration/test_conflict_resolver_ch.py index cefd4c4708..8be366cf3c 100644 --- a/tests/integration/test_conflict_resolver_ch.py +++ b/tests/integration/test_conflict_resolver_ch.py @@ -3253,14 +3253,16 @@ async def statuses() -> dict[str, tuple[str, str]]: MemoryStatusesInput(org=org, namespace="elsewhere", memory_ids=[loser]) ) assert other.rows == [], "another namespace holds none of them" - # A delete, then its restore: the row reads deleted, then live again. + # Explicit deletion remains deleted when restoration is refused. key = {"org": org, "namespace": NS, "agent": "bot", "user_id": USER, "session_id": "s2"} await handle_memory_delete(MemoryDeleteInput(**key, memory_key="office_location")) assert (await statuses())[winner][0] == "deleted" - assert ( + from server.core.errors import ValidationError + + with pytest.raises(ValidationError) as refused: await handle_memory_restore(MemoryRestoreInput(**key, memory_key="office_location")) - ).restored - assert (await statuses())[winner] == ("active", "") + assert refused.value.extra["reason"] == "forgotten" + assert (await statuses())[winner][0] == "deleted" async def test_a_retried_undo_finishes_a_restore_that_landed_after_its_503( @@ -3710,6 +3712,28 @@ async def down(*a: Any, **kw: Any) -> Any: assert len(llm.calls) > before, "the pause ended: rows are judged again" +async def _undelete_after_uncertain_sends_settle(input: Any) -> Any: + from server.core import lease_writes, rmt_carry + from server.core.ch import write_limit_s + from server.core.lease_writes import FenceUnavailableError + from server.memory.app import restore + from tests.integration._eventually import eventually + + async def retry() -> Any: + try: + return await restore.handle_memory_restore(input) + except FenceUnavailableError: + return None + + # Visibility of the canonical row cannot certify that duplicate sends have ended. + return await eventually( + retry, + max_wait=lease_writes.lease_ttl_s(rmt_carry.send_bound_s()) + 2 * write_limit_s(), + interval=1.0, + description="the uncertain restore's writers to settle", + ) + + async def test_an_undelete_whose_insert_answer_was_lost_finishes(ch, wired, monkeypatch) -> None: """The undelete's insert lands with its answer lost: the re-read finds it, so the search row and the facts come back as on any undelete, and it answers restored.""" @@ -3717,13 +3741,13 @@ async def test_an_undelete_whose_insert_answer_was_lost_finishes(ch, wired, monk from server.memory.api.models_crud import MemoryDeleteInput, MemoryRestoreInput from server.memory.app import restore - from server.memory.app.handlers import handle_memory_delete + from tests.integration._memory import retire_reversibly org, _ = wired key = {"org": org, "namespace": NS, "agent": "bot", "user_id": USER, "session_id": "s1"} await _set(org, "s1", "home2", "Ana lives in Porto", relations=[("ana", "lives_in", "porto")]) await asyncio.sleep(1.0) - await handle_memory_delete(MemoryDeleteInput(**key, memory_key="home2")) + await retire_reversibly(MemoryDeleteInput(**key, memory_key="home2")) await asyncio.sleep(1.0) real = ch.insert @@ -3736,6 +3760,7 @@ async def lands_then_fails(table: str, rows: Any, **kw: Any) -> None: m.setattr(ch, "insert", lands_then_fails) out = await restore.handle_memory_restore(MemoryRestoreInput(**key, memory_key="home2")) assert out.restored and out.from_status == "tombstoned" + await _undelete_after_uncertain_sends_settle(MemoryRestoreInput(**key, memory_key="home2")) search = await ch.query_all( "SELECT tombstoned_at FROM loom.memory_embeddings FINAL WHERE org = {o:String}" " AND memory_key = 'home2' SETTINGS select_sequential_consistency = 1", @@ -3839,13 +3864,13 @@ async def test_an_undelete_whose_answer_and_reread_were_lost_finishes_on_retry( from server.core.errors import BackendUnavailableError from server.memory.api.models_crud import MemoryDeleteInput, MemoryRestoreInput from server.memory.app import restore - from server.memory.app.handlers import handle_memory_delete + from tests.integration._memory import retire_reversibly org, _ = wired key = {"org": org, "namespace": NS, "agent": "bot", "user_id": USER, "session_id": "s1"} await _set(org, "s1", "home3", "Ana lives in Braga") await asyncio.sleep(1.0) - await handle_memory_delete(MemoryDeleteInput(**key, memory_key="home3")) + await retire_reversibly(MemoryDeleteInput(**key, memory_key="home3")) await asyncio.sleep(1.0) real = ch.insert @@ -3863,7 +3888,9 @@ async def unread(*a: Any, **kw: Any) -> None: with pytest.raises(BackendUnavailableError) as lost: await restore.handle_memory_restore(MemoryRestoreInput(**key, memory_key="home3")) assert lost.value.extra["outcome_unknown"] - out = await restore.handle_memory_restore(MemoryRestoreInput(**key, memory_key="home3")) + out = await _undelete_after_uncertain_sends_settle( + MemoryRestoreInput(**key, memory_key="home3") + ) assert out.restored and out.from_status == "tombstoned" search = await ch.query_all( "SELECT tombstoned_at FROM loom.memory_embeddings FINAL WHERE org = {o:String}" @@ -3920,13 +3947,13 @@ async def test_a_high_row_restored_after_a_lost_undelete_answers_not_restored( from server.core.errors import BackendUnavailableError from server.memory.api.models_crud import MemoryDeleteInput, MemoryRestoreInput from server.memory.app import restore - from server.memory.app.handlers import handle_memory_delete + from tests.integration._memory import retire_reversibly org, _ = wired key = {"org": org, "namespace": NS, "agent": "bot", "user_id": USER, "session_id": "s1"} await _set(org, "s1", "vault4", "The vault code is 4417", sensitivity="high") await asyncio.sleep(1.0) - await handle_memory_delete(MemoryDeleteInput(**key, memory_key="vault4")) + await retire_reversibly(MemoryDeleteInput(**key, memory_key="vault4")) await asyncio.sleep(1.0) real = ch.insert diff --git a/tests/integration/test_conflict_resolver_invariants_ch.py b/tests/integration/test_conflict_resolver_invariants_ch.py index b8465affa6..dcbaba3b0c 100644 --- a/tests/integration/test_conflict_resolver_invariants_ch.py +++ b/tests/integration/test_conflict_resolver_invariants_ch.py @@ -4,7 +4,7 @@ - no user write or pin is lost: each key holds what its last user write left; - every retired row has a `memory.auto_supersede` record and can be undone; -- no flag, clear or retirement below 0.90; +- no model action below 0.90; dead flags clear without a model judgement; - no text of another user or another tenant reaches a prompt; - nothing is written in an org whose erasure was requested while a call was in flight; - once the writes stop and a pass drains, the next pass makes no call. @@ -140,10 +140,17 @@ async def write(self, keys: list[str]) -> None: self.wrote(key, session, out.version, ("value", value, False)) elif op == "pin" and head is not None and head["tombstoned_at"] is None: value = _text(head["value"]) - out = await handle_memory_pin( - MemorySetInput(**ident, session_id=session, memory_key=key, value=value) - ) - self.wrote(key, session, out.version, ("value", value, True)) + from server.core.errors import ValidationError + + try: + out = await handle_memory_pin( + MemorySetInput(**ident, session_id=session, memory_key=key, value=value) + ) + except ValidationError as error: + if error.extra.get("reason") != "forgotten": + raise + else: + self.wrote(key, session, out.version, ("value", value, True)) elif op == "delete": gone = await handle_memory_delete( MemoryDeleteInput(**ident, session_id=session, memory_key=key) @@ -206,6 +213,7 @@ async def test_the_resolver_keeps_its_invariants_under_concurrent_writes( from server.memory.api.models_crud import MemoryRestoreInput from server.memory.app import after_write, write from server.memory.app.restore import handle_memory_restore + from server.memory.store import conflict_resolver as queue from server.memory.workers.conflict_resolver import run_pass from server.settings import settings @@ -214,6 +222,22 @@ async def test_the_resolver_keeps_its_invariants_under_concurrent_writes( writer = _Writer(ch, org, seed) erasing = seed % 3 == 0 stored: dict[str, set[int]] = {} + dead_peers: set[str] = set() + read_dead_flags = queue.dead_flags + + async def observed_dead_flags( + read_org: str, + namespace: str, + identity: tuple[str, str, str, str], + *, + ch: Any = None, + ) -> list[str]: + peers = await read_dead_flags(read_org, namespace, identity, ch=ch) + if (read_org, namespace) == (org, NS): + dead_peers.update(peers) + return peers + + monkeypatch.setattr(queue, "dead_flags", observed_dead_flags) async def in_call(entries: list[dict[str, Any]]) -> None: acts = any(e["confidence"] >= 0.9 and e["same_entity"] for e in entries) @@ -276,7 +300,17 @@ async def slow_embed(w: Any, embedder: Any = None) -> Any: assert not any(_SECRET in prompt for prompt in calls), "another user's or tenant's text" for record in await _audit(ch, org, "memory.conflict_resolver"): - assert all(p["confidence"] >= 0.90 for p in record["pairs"]), record + assert record["pairs"], record + if record["model"]: + assert all(p["confidence"] >= 0.90 for p in record["pairs"]), record + else: + for pair in record["pairs"]: + assert pair == { + "peer_id": pair["peer_id"], + "action": "clear", + "reason": "peer_not_live", + }, record + assert pair["peer_id"] in dead_peers, record recorded = { e["memory_id"] for r in await _audit(ch, org, "memory.auto_supersede") for e in r["retired"] diff --git a/tests/integration/test_conflict_resolver_long_session_ch.py b/tests/integration/test_conflict_resolver_long_session_ch.py index 1f47852963..c3afe36c62 100644 --- a/tests/integration/test_conflict_resolver_long_session_ch.py +++ b/tests/integration/test_conflict_resolver_long_session_ch.py @@ -6,8 +6,8 @@ through `memory.set` (odd sessions) and `memory.bulk_set` (even sessions), each a week apart. Six attributes change once or twice, each change under a new key and in new words; other facts are restated, elaborated, or sit near a different fact; -one pinned fact is contradicted later; one fact is deleted and brought back -with Undo; the rest are unrelated (`bench/conflict_resolver_scenario.py`, the +one pinned fact is contradicted later; one fact is deleted and stays absent; +the rest are unrelated (`bench/conflict_resolver_scenario.py`, the scenario and its ground truth). A scenario embedder gives every text a fixed direction, so the write path's own check decides which rows the resolver owes a judgement, and the judge answers each (row, peer) pair from the scenario's @@ -37,8 +37,10 @@ from bench.conflict_resolver_scenario import ( ATTRIBUTES, BY_KEY, + CHECK_DELETED_IN, CURRENT, DELETE_IN, + DELETED, FACTS, FLAGGED_AT_WRITE, JUDGE, @@ -46,7 +48,6 @@ LOW_CONFIDENCE, MALFORMED, OLDER, - RESTORE_IN, SESSIONS, SOLO, relation_of, @@ -146,11 +147,40 @@ async def query_all(self, sql: str, params: Any = None) -> Any: return await self.ch.query_all(sql, params={**(params or {}), "only_org": self.org}) +async def _insert_queue_vector(ch: Any, row: dict[str, Any]) -> None: + from server.core.hashing import sha256_value + from server.memory.store.inserts import insert_embedding_rows, insert_memory_versions + from server.memory.store.rows import build_first_version_row + + value = row["org"] + row["content_sha256"] = sha256_value(value) + row["content_excerpt"] = value + source = build_first_version_row( + { + **{ + k: row[k] + for k in ("org", "namespace", "agent", "user_id", "session_id", "memory_key") + }, + "value": json.dumps(value), + "value_json": "{}", + "memory_type": row["memory_type"], + "confidence": 1.0, + "content_sha256": row["content_sha256"], + "status": row["status"], + "version": row["version"], + "actor": "process:itest", + "source_class": "asserted", + }, + now=naive_utc_now(), + ) + await insert_memory_versions([source], ch=ch) + assert await insert_embedding_rows([row], ch=ch) == [row] + + @pytest.mark.parametrize("leg", ["owed", "unjudged"]) async def test_org_queue_scope_precedes_the_candidate_limit(ch, leg: str) -> None: from server.core.vectors.codes_state import stored_width from server.memory.store import conflict_resolver as queue - from server.memory.store.inserts import insert_embedding_rows from server.memory.store.rows import build_embedding_row orgs = [unique_org("queue-scope") for _ in range(2)] @@ -184,9 +214,9 @@ def vector(org: str) -> dict[str, Any]: ) try: - await insert_embedding_rows([vector(foreign)], ch=ch) + await _insert_queue_vector(ch, vector(foreign)) assert await fetch(limit=1, seed=0, ch=_OnlyOrg(ch, own)) == [] - await insert_embedding_rows([vector(own)], ch=ch) + await _insert_queue_vector(ch, vector(own)) rows = await fetch(limit=1, seed=0, ch=_OnlyOrg(ch, own)) assert [(row["org"], row["memory_key"]) for row in rows] == [(own, "pending")] rows = await fetch(limit=1, seed=0, ch=_OnlyOrg(ch, foreign)) @@ -318,10 +348,11 @@ async def _write_session(org: str, session: int) -> None: await after_write.drain() -async def _delete_or_restore(org: str, session: int) -> None: - from server.memory.api.models_crud import MemoryDeleteInput, MemoryRestoreInput +async def _delete_and_check_absent(org: str, session: int) -> None: + from server.identity.domain.read_scope import WHOLE_NAMESPACE + from server.memory import store as memory_repo + from server.memory.api.models_crud import MemoryDeleteInput from server.memory.app.handlers import handle_memory_delete - from server.memory.app.restore import handle_memory_restore if session in DELETE_IN: key = DELETE_IN[session] @@ -329,12 +360,16 @@ async def _delete_or_restore(org: str, session: int) -> None: MemoryDeleteInput(**_identity(org, BY_KEY[key].session), memory_key=key) ) assert out.tombstoned, key - if session in RESTORE_IN: - key = RESTORE_IN[session] - back = await handle_memory_restore( - MemoryRestoreInput(**_identity(org, BY_KEY[key].session), memory_key=key) + if session in CHECK_DELETED_IN: + key = CHECK_DELETED_IN[session] + current = await memory_repo.get_by_key( + memory_repo.Identity(**_identity(org, BY_KEY[key].session)), + key, + fresh=True, + strong=True, + scope=WHOLE_NAMESPACE, ) - assert (back.restored, back.from_status) == (True, "tombstoned"), key + assert current is None, key async def _drain(run: Run) -> None: @@ -434,7 +469,7 @@ async def test_a_long_run_of_sessions_ends_with_one_live_value_per_attribute(ch, org = run.org for session in range(1, SESSIONS + 1): await _write_session(org, session) - await _delete_or_restore(org, session) + await _delete_and_check_absent(org, session) rows = await _rows(ch, org) for f in FACTS: if f.session == session: @@ -442,7 +477,7 @@ async def test_a_long_run_of_sessions_ends_with_one_live_value_per_attribute(ch, await _drain(run) rows = await _rows(ch, org) - assert set(rows) == {f.key for f in FACTS} + assert set(rows) == {f.key for f in FACTS} - DELETED assert run.judge.unexpected == [], "a cluster held a peer the scenario did not place" # Each attribute: its current value is live and unflagged; every older value is diff --git a/tests/integration/test_digest_listing_sources.py b/tests/integration/test_digest_listing_sources.py new file mode 100644 index 0000000000..e8ff023b3d --- /dev/null +++ b/tests/integration/test_digest_listing_sources.py @@ -0,0 +1,145 @@ +"""Stale digest candidates cannot hide a later page of proven facts.""" + +from __future__ import annotations + +from datetime import timedelta + +import httpx +import pytest + +from server.core.datetimes import naive_utc_now +from tests.integration._resources import unique_org + + +@pytest.mark.parametrize("op", ["memory.digests", "memory.outline", "memory.export_evidence"]) +async def test_digest_listings_scan_past_withheld_sources( + control_plane_url, mint_test_token, ch_client, op +): + org = unique_org("digest-pages") + token = mint_test_token(org=org, namespaces=["facts"], roles=["writer"]) + identity = dict(org=org, namespace="facts", agent="a", user_id="subject") + async with httpx.AsyncClient( + base_url=control_plane_url, headers={"Authorization": f"Bearer {token}"}, timeout=120 + ) as client: + for i in (2, 3): + response = await client.post( + "/v1/memory.set", + json={**identity, "session_id": f"s{i}", "memory_key": "k", "value": f"fact {i}"}, + ) + assert response.status_code == 200, response.text + now = naive_utc_now() + columns = [ + "org", + "namespace", + "agent", + "user_id", + "session_id", + "embedding_model", + "summary_text", + "key_facts", + "derived_from", + "created_at", + "version", + "summary_embedding", + ] + ch_client.insert( + "loom.session_summaries", + [ + [ + org, + "facts", + "a", + "subject", + f"s{i}", + "test", + f"fact {i}", + [f"fact {i}"], + ["k"] if i >= 2 else [], + now + timedelta(seconds=-i if op == "memory.outline" else i), + 1, + [0.0] * 3072, + ] + for i in range(4) + ], + column_names=columns, + settings={"async_insert": 0}, + ) + fields = ( + {"sessions_limit": 1, "hubs_limit": 0, "communities_limit": 0} + if op == "memory.outline" + else {"limit": 1} + ) + if op == "memory.export_evidence": + fields["kind"] = "digests" + response = await client.post(f"/v1/{op}", json={**identity, **fields}) + assert response.status_code == 200, response.text + result = response.json() + field = { + "memory.digests": "rows", + "memory.outline": "sessions", + "memory.export_evidence": "digests", + }[op] + assert [row["session_id"] for row in result[field]] == ["s2"], result + if op == "memory.digests": + assert result["truncated"] is True + if op == "memory.export_evidence": + assert result["has_more"] is True + + +@pytest.mark.parametrize("page_size", [1, 2]) +async def test_period_pages_choose_latest_model_before_applying_the_cursor(ch_client, page_size): + from server.core.ch import get_client + from server.memory.app.period_digest import _fetch_digests + + org, now = unique_org("period-pages"), naive_utc_now() + columns = [ + "org", + "namespace", + "agent", + "user_id", + "session_id", + "embedding_model", + "summary_text", + "key_facts", + "derived_from", + "created_at", + "version", + "summary_embedding", + ] + rows = [ + [ + org, + "facts", + "a", + "subject", + sid, + model, + text, + [text], + ["k"], + now + timedelta(seconds=offset), + version, + [0.0] * 3072, + ] + for sid, model, text, offset, version in ( + ("s1", "old", "retired fact", 10, 1), + ("s1", "middle", "intermediate fact", 5, 2), + ("s1", "current", "current fact", 0, 3), + ("s2", "current", "second fact", 1, 1), + ("s3", "current", "third fact", 2, 1), + ) + ] + ch_client.insert( + "loom.session_summaries", rows, column_names=columns, settings={"async_insert": 0} + ) + ch = get_client() + await ch.connect() + try: + identity = dict(org=org, namespace="facts", agent="a", user_id="subject") + found = [ + row async for page in _fetch_digests(ch, identity, page_size=page_size) for row in page + ] + finally: + await ch.disconnect() + assert [row["session_id"] for row in found] == ["s1", "s2", "s3"] + assert found[0]["summary_text"] == "current fact" and found[0]["version"] == 3 diff --git a/tests/integration/test_embedding_worker.py b/tests/integration/test_embedding_worker.py index 47116e43d6..315639b2d6 100644 --- a/tests/integration/test_embedding_worker.py +++ b/tests/integration/test_embedding_worker.py @@ -13,6 +13,7 @@ sys.path.insert(0, str(ROOT / "sdk" / "python")) sys.path.insert(0, str(ROOT)) +from server.core.hashing import sha256_value # noqa: E402 from server.core.vectors.placeholders import placeholder # noqa: E402 from server.memory.workers.embedding import ( # noqa: E402 EmbeddingWorkerConfig, @@ -88,14 +89,19 @@ async def test_stub_inserted_directly_gets_embedded( """ INSERT INTO loom.memory_embeddings (org, namespace, agent, user_id, session_id, memory_key, - memory_type, scope, content_excerpt, embedding_model, + memory_type, scope, content_excerpt, content_sha256, embedding_model, embedding_dim, embedding, memory_key_embedding, tombstoned_at, created_at) VALUES ({org:String}, {ns:String}, 'w', '-', '-', 'pending_k', - 'semantic', 'private', 'clickhouse columnar speed', + 'semantic', 'private', 'clickhouse columnar speed', {sha:FixedString(64)}, 'fake-ngram-1024', 0, {ph:Array(BFloat16)}, {ph:Array(BFloat16)}, NULL, now() - INTERVAL 1 MINUTE) """, - parameters={"org": ORG, "ns": ns, "ph": PLACEHOLDER}, + parameters={ + "org": ORG, + "ns": ns, + "ph": PLACEHOLDER, + "sha": sha256_value("clickhouse columnar speed"), + }, ) assert _embedding_dim(ch_client, ns, "pending_k") == 0 # stub @@ -127,7 +133,6 @@ async def _seed_memory_row(ns: str, key: str, value: str) -> None: """The memory row an async-mode memory.set writes beside its stub: the worker fills only stubs whose head may keep a vector, and a stub with no head is an orphan.""" - import hashlib import json from datetime import UTC, datetime @@ -147,7 +152,7 @@ async def _seed_memory_row(ns: str, key: str, value: str) -> None: "memory_type": "semantic", "value": json.dumps(value), "value_json": "{}", - "content_sha256": hashlib.sha256(value.encode()).hexdigest(), + "content_sha256": sha256_value(value), "confidence": 1.0, "actor": "process:itest", "source_class": "asserted", @@ -176,13 +181,18 @@ async def test_stub_invisible_to_search_until_embedded( """ INSERT INTO loom.memory_embeddings (org, namespace, agent, user_id, session_id, memory_key, - memory_type, scope, content_excerpt, embedding_model, + memory_type, scope, content_excerpt, content_sha256, embedding_model, embedding_dim, embedding, memory_key_embedding, tombstoned_at, created_at) VALUES ({org:String}, {ns:String}, 'w', '-', '-', 'pending', - 'semantic', 'private', 'distinctive query content here', + 'semantic', 'private', 'distinctive query content here', {sha:FixedString(64)}, 'fake-ngram-1024', 0, {ph:Array(BFloat16)}, {ph:Array(BFloat16)}, NULL, now()) """, - parameters={"org": ORG, "ns": ns, "ph": PLACEHOLDER}, + parameters={ + "org": ORG, + "ns": ns, + "ph": PLACEHOLDER, + "sha": sha256_value("distinctive query content here"), + }, ) async with Agent(org=ORG, namespace=ns, agent="w", base_url=control_plane_url) as a: @@ -257,11 +267,11 @@ def _insert(text: str, version: int) -> None: """ INSERT INTO loom.memory_embeddings (org, namespace, agent, user_id, session_id, memory_key, - memory_type, scope, content_excerpt, embedding_model, + memory_type, scope, content_excerpt, content_sha256, embedding_model, embedding_dim, embedding, memory_key_embedding, tombstoned_at, created_at, version) VALUES ({org:String}, {ns:String}, 'w', '-', '-', 'fenced_k', - 'semantic', 'private', {text:String}, {model:String}, + 'semantic', 'private', {text:String}, {sha:FixedString(64)}, {model:String}, 0, {ph:Array(BFloat16)}, {ph:Array(BFloat16)}, NULL, now() - INTERVAL 1 MINUTE, {v:UInt64}) """, @@ -269,6 +279,7 @@ def _insert(text: str, version: int) -> None: "org": ORG, "ns": ns, "text": text, + "sha": sha256_value(text), "model": model, "v": version, "ph": PLACEHOLDER, diff --git a/tests/integration/test_erasure.py b/tests/integration/test_erasure.py index e19d614955..c4232cfa3d 100644 --- a/tests/integration/test_erasure.py +++ b/tests/integration/test_erasure.py @@ -93,6 +93,13 @@ async def test_user_id_erasure_tombstones_memory(admin_agent, ch_client) -> None # user_id filter). bob's True would merge into alice's row # and never reach loom.memory under bob's user_id. await a.memory.set("prefers_yaml", "yaml-please", memory_type="preference", user_id="bob") + # The ClickHouse tombstone assertions require the committed seeds to be published. + listed = await a.memory.list(user_id=target_user) + assert {"prefers_json", "favorite_table"} <= { + row["memory_key"] for row in listed["entries"] + } + control = await a.memory.list(user_id="bob") + assert "prefers_yaml" in {row["memory_key"] for row in control["entries"]} # Run erasure for alice result = await admin_agent.erasure.request_user_id(target_user, requester="ops@example.com") @@ -340,6 +347,8 @@ async def test_namespace_erasure_tombstones_only_target(admin_agent, ch_client) source_uri=f"src://{ns}", tags=["test"], ) + listed = await a.memory.list() + assert any(row["memory_key"] == "k1" for row in listed["entries"]) # Erase ns_target result = await admin_agent.erasure.request_namespace(ns_target, requester="ops@example.com") diff --git a/tests/integration/test_erasure_replicated_ch.py b/tests/integration/test_erasure_replicated_ch.py index 04da3cc513..341501c8f1 100644 --- a/tests/integration/test_erasure_replicated_ch.py +++ b/tests/integration/test_erasure_replicated_ch.py @@ -36,6 +36,8 @@ "documents", "memory_code_links", "memory_versions", + "memory_refusals", + "erasure_requests", "shared", ) # loom.entity_vectors is left out: this fixture's copy of its quantized codec column is diff --git a/tests/integration/test_inference_identity_workers.py b/tests/integration/test_inference_identity_workers.py index d11cca3218..3172ff74db 100644 --- a/tests/integration/test_inference_identity_workers.py +++ b/tests/integration/test_inference_identity_workers.py @@ -59,7 +59,13 @@ async def test_an_older_worker_reclaims_a_hold_when_its_lease_ends( assert response.status_code == 200, response.text assert response.json()["pending"] == 1 monkeypatch.setattr(queue, "_now", lambda: datetime.now(UTC) - timedelta(seconds=1)) - await queue.mark_held(claimed[0], reason="refused", seconds=0, ch=ch) + current = await ch.query_one( + f"SELECT {', '.join(queue.COLUMNS)} FROM loom.memory_extraction_queue FINAL " + "WHERE org={org:String} AND queue_id={qid:String}", + params={"org": org, "qid": claimed[0]["queue_id"]}, + ) + assert current is not None + await queue.mark_held(current, reason="refused", seconds=0, ch=ch) assert len(await ch.query_all(old_selection, params={"org": org})) == 1 diff --git a/tests/integration/test_mcp_session_run_scope.py b/tests/integration/test_mcp_session_run_scope.py index 9a5ccebedb..5a75638281 100644 --- a/tests/integration/test_mcp_session_run_scope.py +++ b/tests/integration/test_mcp_session_run_scope.py @@ -70,6 +70,19 @@ async def _require_auth(c: httpx.AsyncClient) -> None: pytest.skip("the server does not enforce auth") +async def _published_memories(ch_client: Any, org: str, expected: int) -> None: + async def published() -> bool: + return ( + ch_client.query( + "SELECT count() FROM loom.memory FINAL WHERE org = {org:String}", + parameters={"org": org}, + ).result_rows[0][0] + == expected + ) + + await eventually(published, description="the MCP memories to be published") + + @pytest.mark.asyncio async def test_a_session_scoped_tokens_run_is_filed_under_its_session( control_plane_url: str, ch_client: Any @@ -92,6 +105,7 @@ async def test_a_session_scoped_tokens_run_is_filed_under_its_session( assert len(runs) == 1 run_id, user, row_session, status = runs[0] assert (user, row_session, status) == (USER, APP_SESSION, "completed") + await _published_memories(ch_client, org, 3) stamped = ch_client.query( "SELECT DISTINCT run_id FROM loom.memory FINAL WHERE org = {org:String}", parameters={"org": org}, @@ -120,6 +134,7 @@ async def test_two_tokens_sending_one_session_id_hold_two_runs( ).result_rows ) assert runs == {"ns-a": "active", "ns-b": "completed"} + await _published_memories(ch_client, org, 4) stamped = ch_client.query( "SELECT namespace, uniqExact(run_id) FROM loom.memory FINAL WHERE org = {org:String} " "GROUP BY namespace ORDER BY namespace", diff --git a/tests/integration/test_memory_code_links.py b/tests/integration/test_memory_code_links.py index 53aae5460b..8f99de6a5e 100644 --- a/tests/integration/test_memory_code_links.py +++ b/tests/integration/test_memory_code_links.py @@ -1988,7 +1988,7 @@ async def test_a_link_written_after_an_erasure_is_dropped(env, monkeypatch, ch_c await _pass() assert queue.erased.status == "completed" and _all_links(ch_client) == [] # the recheck: moving a link after the erasure - await _remember(MEM, "e", "`erased_value` returns one") + await _remember(MEM, "e", "`erased_value` yields one") await _pass() assert _all_links(ch_client) with monkeypatch.context() as m: @@ -2580,15 +2580,15 @@ async def test_imports_and_member_assignments_in_a_rename(env, path, before, aft async def test_a_delayed_cleanup_keeps_links_made_after_a_restore(env) -> None: from server.core.ch import get_client from server.memory.api.models_crud import MemoryDeleteInput, MemoryRestoreInput - from server.memory.app.handlers import handle_memory_delete from server.memory.app.restore import handle_memory_restore from server.memory.store import code_links as store + from tests.integration._memory import retire_reversibly await _drop_fetch_rows(env, TRUNK) ident = {"org": ORG, "namespace": MEM, "agent": "claude-code", "memory_key": "back"} mid = await _remember(MEM, "back", "`fetch_rows` returns n") await _pass() - await handle_memory_delete(MemoryDeleteInput(**ident)) + await retire_reversibly(MemoryDeleteInput(**ident)) class _RestoreBeforeDelete: def __init__(self, real: Any) -> None: diff --git a/tests/integration/test_memory_promote.py b/tests/integration/test_memory_promote.py index 54ac639d8a..c9277b2191 100644 --- a/tests/integration/test_memory_promote.py +++ b/tests/integration/test_memory_promote.py @@ -451,7 +451,8 @@ async def approved(body: str) -> str: " AND memory_key = {k:String}", parameters=params, ).result_rows[0] - assert vectors[1] and bool(vectors[0]) == bool(live_row), (live_row, vectors) + # Retired mirrors may be physically removed when their source is absent. + assert bool(vectors[0]) == bool(live_row), (live_row, vectors) @pytest.mark.asyncio diff --git a/tests/integration/test_memory_read_lifecycle.py b/tests/integration/test_memory_read_lifecycle.py index d4cb4519cb..4d3cc130dc 100644 --- a/tests/integration/test_memory_read_lifecycle.py +++ b/tests/integration/test_memory_read_lifecycle.py @@ -12,6 +12,7 @@ from server.core.ch import CHClient from server.core.datetimes import naive_utc_now from server.memory.app.retrieval.summaries import fetch_session_summaries +from server.memory.domain.summary_provenance import render_summary from server.memory.store import lifecycle from server.memory.store.lifecycle import bulk_tombstone_scope from server.memory.store.rows import MEMORY_PERSISTED_COLS, next_version @@ -93,13 +94,14 @@ async def test_session_summaries_inherit_known_source_state_and_version( vector = [1.0] + [0.0] * 3071 states = ["active", "conflicted", "rejected", "proposed", "superseded", "tombstoned"] for status in [*states, "rewritten", "high", "expired", "past", "standalone", "missing"]: - fields: dict[str, Any] = {"version": 100, "session_id": status} + fields: dict[str, Any] = {"version": 100, "session_id": status, "value": json.dumps(status)} if status in states: fields["status"] = status if status == "tombstoned": fields["tombstoned_at"] = naive_utc_now() if status == "rewritten": fields["version"] = 300 + fields["value"] = json.dumps("a replacement fact") if status == "high": fields["sensitivity"] = "high" if status in ("past", "expired"): @@ -115,7 +117,8 @@ async def test_session_summaries_inherit_known_source_state_and_version( "user_id": "reader", "session_id": status, "version": 200, - "summary_text": status, + "summary_text": render_summary([status]), + "key_facts": [status], "summary_embedding": vector, "embedding_model": "fixture-model", "embedding_dim": len(vector), @@ -140,7 +143,7 @@ async def test_session_summaries_inherit_known_source_state_and_version( ) finally: await ch.disconnect() - assert {h.content_excerpt for h in hits} == {"active", "conflicted", "standalone"} + assert {h.content_excerpt for h in hits} == {"active", "conflicted"} async def test_clear_keeps_a_complete_tombstone_in_version_history( @@ -452,7 +455,15 @@ async def test_nearest_stale_summary_does_not_hide_the_next_valid_digest( ("stale", "rejected", query), ("fresh", "active", [0.7, 0.2] + [0.0] * 3070), ): - _memory(ch_client, me, "source", session_id=session, status=status, version=100) + _memory( + ch_client, + me, + "source", + session_id=session, + status=status, + version=100, + value=json.dumps(session), + ) row = { "org": me["org"], "namespace": me["namespace"], @@ -460,7 +471,8 @@ async def test_nearest_stale_summary_does_not_hide_the_next_valid_digest( "user_id": "reader", "session_id": session, "version": 200, - "summary_text": session, + "summary_text": render_summary([session]), + "key_facts": [session], "summary_embedding": vector, "embedding_dim": 3072, "embedding_model": "fixture-model", diff --git a/tests/integration/test_memory_refusal_capture_live_ch.py b/tests/integration/test_memory_refusal_capture_live_ch.py new file mode 100644 index 0000000000..89153943e1 --- /dev/null +++ b/tests/integration/test_memory_refusal_capture_live_ch.py @@ -0,0 +1,124 @@ +"""Erasure captures consent scopes and raw metadata before the content disappears.""" + +from __future__ import annotations + +import json +import uuid + +import pytest + +from server.core.ch import get_client +from server.core.errors import ValidationError +from server.erasure.store import refusals as capture +from server.memory.store import refusals as policy +from tests.integration._resources import unique_org + +pytestmark = pytest.mark.asyncio + + +@pytest.fixture +async def ch(monkeypatch): + client = get_client() + await client.connect() + monkeypatch.setattr(capture, "_default_client", lambda: client) + monkeypatch.setattr(policy, "_writer", lambda: "process:test") + yield client + await client.disconnect() + + +def _row(org, ns, uid, value, *, key="source", **fields): + return { + "org": org, + "namespace": ns, + "agent": "writer", + "user_id": uid, + "session_id": "-", + "memory_key": key, + "memory_type": "fact", + "tags": [], + "value": json.dumps(value), + "version": 1, + **fields, + } + + +async def _refused(ch, row): + with pytest.raises(ValidationError, match="previous deletion"): + await policy.check_clickhouse(ch, [row]) + + +async def test_namespace_capture_refuses_every_user_only_in_its_namespace(ch): + org = unique_org("refusal-namespace") + values = [f"retired namespace fact {uuid.uuid4().hex}" for _ in range(2)] + await ch.insert( + "loom.memory_versions", + [ + _row(org, "closed", uid, value) + for uid, value in zip(("alice", "bob"), values, strict=True) + ], + ) + assert await capture.capture_namespace(org, "closed") == 2 + for value in values: + await _refused(ch, _row(org, "closed", "future-user", value)) + await policy.check_clickhouse(ch, [_row(org, "elsewhere", "alice", value)]) + await policy.check_clickhouse(ch, [_row(org, "closed", "alice", "unrelated fact")]) + + +async def test_user_capture_reaches_future_namespaces_and_derived_writer_identity(ch): + org = unique_org("refusal-user") + parent = _row(org, "first", "alice", "original private fact") + derived = _row( + org, + "first", + "-", + "consolidated private fact", + key="summary", + tags=["consolidated", "src:source"], + ) + other = _row(org, "second", "alice", "second private fact") + await ch.insert("loom.memory", [parent, derived, other]) + assert await capture.capture_user(org, "alice") == 3 + for value in ("original private fact", "consolidated private fact", "second private fact"): + await _refused(ch, _row(org, "future", "alice", value)) + await policy.check_clickhouse(ch, [_row(org, "future", "bob", value)]) + + +async def test_capture_decodes_structured_metadata_and_keeps_literal_null(ch): + org = unique_org("refusal-metadata") + await ch.insert( + "loom.memory_evidence", + [ + { + "org": org, + "namespace": "facts", + "agent": "writer", + "user_id": "alice", + "session_id": "-", + "evidence_id": uuid.uuid4(), + "content": "null", + "metadata": json.dumps({"credential": "copper lark secret"}), + } + ], + ) + assert await capture.capture_user(org, "alice") == 2 + for value in ("null", {"credential": "copper lark secret"}): + await _refused(ch, _row(org, "future", "alice", value)) + await policy.check_clickhouse(ch, [_row(org, "future", "alice", {"other": "allowed"})]) + + +async def test_a_large_source_value_never_becomes_a_cursor_parameter(ch, monkeypatch): + org = unique_org("refusal-cursor") + value = "x" * 150_000 + await ch.insert("loom.memory_versions", [_row(org, "facts", "alice", value)]) + cursors = [] + query = ch.query_all + + async def observe(sql, params=None): + if params and params.get("after"): + cursors.append(params["after"]) + return await query(sql, params=params) + + monkeypatch.setattr(ch, "query_all", observe) + assert await capture.capture_user(org, "alice") == 1 + assert cursors and all(len(str(cursor)) < 1000 for cursor in cursors) + await _refused(ch, _row(org, "future", "alice", value)) diff --git a/tests/integration/test_memory_refusal_policy.py b/tests/integration/test_memory_refusal_policy.py new file mode 100644 index 0000000000..da733d9ffc --- /dev/null +++ b/tests/integration/test_memory_refusal_policy.py @@ -0,0 +1,191 @@ +"""Deletion refuses later intake while preserving unrelated identities and values.""" + +from __future__ import annotations + +import json +import uuid + +import httpx +import pytest + +from tests.integration._resources import unique_org + + +@pytest.mark.asyncio +async def test_wildcard_forgetting_refuses_only_the_requested_users_value( + control_plane_url: str, mint_test_token +) -> None: + org = unique_org("refusal-users") + token = mint_test_token(org=org, namespaces=["facts"], roles=["writer"]) + identity = {"org": org, "namespace": "facts", "agent": "writer"} + values = {"alice": "A copper lark stays near the lake", "bob": "A silver heron stays upstream"} + async with httpx.AsyncClient( + base_url=control_plane_url, headers={"Authorization": f"Bearer {token}"}, timeout=120 + ) as client: + for uid, value in values.items(): + wrote = await client.post( + "/v1/memory.set", + json={ + **identity, + "user_id": uid, + "memory_key": "shared-key", + "value": value, + "memory_type": "episodic", + }, + ) + assert wrote.status_code == 200, wrote.text + deleted = await client.post( + "/v1/memory.delete", + json={ + **identity, + "agent": "*", + "user_id": "alice", + "memory_key": "shared-key", + }, + ) + assert deleted.status_code == 200 and deleted.json()["tombstoned"], deleted.text + for uid, expected in (("alice", 422), ("bob", 200)): + got = await client.post( + "/v1/memory.get", + json={ + **identity, + "user_id": uid, + "memory_key": "shared-key", + }, + ) + assert got.status_code == 200, got.text + assert bool(got.json()["found"]) is (uid == "bob") + replay = await client.post( + "/v1/memory.set", + json={ + **identity, + "user_id": uid, + "memory_key": "replayed", + "value": values[uid], + "memory_type": "episodic", + }, + ) + assert replay.status_code == expected, replay.text + + +@pytest.mark.asyncio +async def test_deleted_content_cannot_return_through_another_intake( + control_plane_url: str, mint_test_token +) -> None: + org = unique_org("refusal") + ns = "facts" + token = mint_test_token(org=org, namespaces=[ns], roles=["writer"]) + identity = {"org": org, "namespace": ns, "agent": "writer", "user_id": "subject"} + value = f"The retired phrase is copper lark {uuid.uuid4().hex}" + async with httpx.AsyncClient( + base_url=control_plane_url, headers={"Authorization": f"Bearer {token}"}, timeout=120 + ) as client: + + async def call(op: str, **fields): + return await client.post(f"/v1/{op}", json={**identity, **fields}) + + wrote = await call("memory.set", memory_key="source", value=value) + assert wrote.status_code == 200, wrote.text + deleted = await call("memory.delete", memory_key="source") + assert deleted.status_code == 200 and deleted.json()["tombstoned"], deleted.text + restored = await call("memory.restore", memory_key="source") + assert restored.status_code == 422, restored.text + assert restored.json()["detail"]["reason"] == "forgotten" + for new_value in (value, f"Reminder: {value.upper()}.", {"wrapped": value}): + refused = await call("memory.set", memory_key="returned", value=new_value) + assert refused.status_code == 422, refused.text + assert refused.json()["detail"]["reason"] == "forgotten" + deferred = await call( + "memory.set_from_messages", + defer=True, + messages=[{"role": "user", "content": value}], + ) + assert deferred.status_code == 422, deferred.text + unrelated = await call( + "memory.set", memory_key="unrelated", value="A separate current fact" + ) + assert unrelated.status_code == 200, unrelated.text + other = await call("memory.set", memory_key="other", user_id="another", value=value) + assert other.status_code == 200, other.text + + +@pytest.mark.asyncio +async def test_structured_deleted_values_are_refused_canonically( + control_plane_url: str, mint_test_token +) -> None: + org = unique_org("refusal-object") + token = mint_test_token(org=org, namespaces=["facts"], roles=["writer"]) + identity = {"org": org, "namespace": "facts", "agent": "writer", "user_id": "subject"} + async with httpx.AsyncClient( + base_url=control_plane_url, headers={"Authorization": f"Bearer {token}"}, timeout=120 + ) as client: + for op, fields, expected in ( + ("memory.set", {"memory_key": "source", "value": {"a": "one", "b": "two"}}, 200), + ("memory.delete", {"memory_key": "source"}, 200), + ("memory.set", {"memory_key": "returned", "value": {"b": "two", "a": "one"}}, 422), + ): + result = await client.post(f"/v1/{op}", json={**identity, **fields}) + assert result.status_code == expected, result.text + + +@pytest.mark.asyncio +@pytest.mark.parametrize("scope_type", ["user_id", "namespace"]) +@pytest.mark.parametrize("residue", [False, True]) +async def test_delayed_shared_retirement_cannot_recreate_erased_content( + control_plane_url, mint_test_token, scope_type, residue +): + from server.core.ch import get_client + from server.core.datetimes import naive_utc_now + from server.memory.store import shared + from server.memory.store.rows import next_version + + org, namespace = unique_org("refusal-shared-retirement"), "facts" + token = mint_test_token(org=org, namespaces=["*"], roles=["org_admin", "admin", "writer"]) + identity = {"org": org, "namespace": namespace, "agent": "writer", "user_id": "alice"} + async with httpx.AsyncClient( + base_url=control_plane_url, headers={"Authorization": f"Bearer {token}"}, timeout=120 + ) as client: + for op, fields in ( + ( + "memory.set", + {"memory_key": "source", "value": f"Private copper lark {uuid.uuid4().hex}"}, + ), + ("memory.promote", {"memory_key": "source"}), + ): + result = await client.post(f"/v1/{op}", json={**identity, **fields}) + assert result.status_code == 200, result.text + ch = get_client() + await ch.connect() + try: + original = await shared.fetch_shared(org, namespace, "source", strong=True, ch=ch) + assert original is not None + original = { + "superseded_by": None, + **original, + "org": org, + "namespace": namespace, + "memory_key": "source", + } + erased = await client.post( + "/v1/erasure.request", + json={ + "org": org, + "scope_type": "user_id" if scope_type == "user_id" else "namespace_list", + "scope_value": "alice" + if scope_type == "user_id" + else json.dumps({"namespaces": [namespace], "purge": True}), + "requester": "operator", + "purge": True, + }, + ) + assert erased.status_code == 200 and erased.json()["status"] == "completed", erased.text + assert await shared.fetch_shared(org, namespace, "source", strong=True, ch=ch) is None + if residue: + # An old writer's admitted send leaves a copy for the retirement to remove. + await ch.insert("loom.shared", [original], settings={"async_insert": 0}) + await shared.insert_shared_row( + {**original, "version": next_version(), "tombstoned_at": naive_utc_now()}, ch=ch + ) + assert await shared.fetch_shared(org, namespace, "source", strong=True, ch=ch) is None + finally: + await ch.disconnect() diff --git a/tests/integration/test_memory_webhooks.py b/tests/integration/test_memory_webhooks.py index 1120d72dc5..662cfca04e 100644 --- a/tests/integration/test_memory_webhooks.py +++ b/tests/integration/test_memory_webhooks.py @@ -318,7 +318,16 @@ async def test_each_write_path_leaves_its_event_and_version( await ops("memory.bulk_set", entries=[{"memory_key": "k/two", "value": "bulk new"}], **a) by_id = await ops("memory.update_by_id", memory_id=first["memory_id"], value="by id") deleted = await ops("memory.delete", memory_key="k/one", **a) - restored = await ops("memory.restore", memory_key="k/one", **a) + refused = await ops("memory.restore", memory_key="k/one", expect=422, **a) + assert refused["detail"]["reason"] == "forgotten" + automatic = await ops("memory.set", memory_key="k/auto", value="automatic retirement", **a) + from server.memory.api.models_crud import MemoryDeleteInput + from tests.integration._memory import retire_reversibly + + await retire_reversibly( + MemoryDeleteInput(org=ORG, namespace=ops.ns, agent="assistant", memory_key="k/auto") + ) + restored = await ops("memory.restore", memory_key="k/auto", **a) head = await ops("memory.get", memory_key="k/two", **a) dv = await ops("memory.delete_version", memory_key="k/two", version=head["version"], **a) redacted = await ops( @@ -338,7 +347,8 @@ async def test_each_write_path_leaves_its_event_and_version( ("memory.revised", "k/one", versions["k/one"]), ("memory.revised", "k/one", by_id["new_version"]), ("memory.tombstoned", "k/one", deleted["version"]), - ("memory.restored", "k/one", restored["version"]), + ("memory.written", "k/auto", automatic["version"]), + ("memory.restored", "k/auto", restored["version"]), ("memory.tombstoned", "k/two", dv["version"]), ("memory.redacted", "k/one", first["version"]), # The restatement copies the redacted text, so the redaction scrubs it too. diff --git a/tests/integration/test_okf_sync.py b/tests/integration/test_okf_sync.py index 7450e66eaf..9006eb5183 100644 --- a/tests/integration/test_okf_sync.py +++ b/tests/integration/test_okf_sync.py @@ -199,12 +199,12 @@ def get(agent: str, key: str) -> dict[str, Any]: assert get("okf-sync", keys[0])["found"] assert get("bot", "bot-note")["found"] assert get("okf-sync", "bot-as-sync")["found"] - # A prune is a tombstone: memory.restore brings the copy back. - restored = _op( - url, - writer, - DST_ORG, - "memory.restore", - {"namespace": ns, "agent": "okf-sync", "memory_key": keys[2]}, + # Pruning explicitly refuses the removed copy, including restoration. + restored = httpx.post( + f"{url}/v1/memory.restore", + headers={"Authorization": f"Bearer {writer}"}, + json={"org": DST_ORG, "namespace": ns, "agent": "okf-sync", "memory_key": keys[2]}, ) - assert restored["restored"] and get("okf-sync", keys[2])["found"] + assert restored.status_code == 422, restored.text + assert restored.json()["detail"]["reason"] == "forgotten" + assert not get("okf-sync", keys[2])["found"] diff --git a/tests/integration/test_pg_authority.py b/tests/integration/test_pg_authority.py index fdae7ce954..dbf13b682d 100644 --- a/tests/integration/test_pg_authority.py +++ b/tests/integration/test_pg_authority.py @@ -30,6 +30,127 @@ _SEQUENTIAL = {"select_sequential_consistency": 1} +@pytest.mark.parametrize("scope_type", ["user_id", "namespace"]) +async def test_erasure_refuses_old_values_after_native_revision_payload_cleanup( + pg_conn, call, org, ch_client, scope_type +): + from server.core import pg + from server.core.ch import get_client + from server.memory.store import history, pg_memory + from server.memory.workers.pg_payload_clear import effective_horizon_minutes + + namespace, user = f"cleared-{uuid.uuid4().hex[:8]}", "alice" + values = [f"retired secret {uuid.uuid4().hex}", f"replacement secret {uuid.uuid4().hex}"] + for value in values: + response = await call( + "memory.set", _set(namespace, "secret", value, user_id=user, classify=False) + ) + assert response.status_code == 200, response.text + await _published(pg_conn, org) + await pg_conn.execute("DELETE FROM loom.memory_deliveries WHERE org=$1 AND settled", org) + await pg_conn.execute( + "UPDATE loom.memory_revisions SET committed_at=now()-interval '30 days', " + "delivered_at=now()-interval '30 days' WHERE org=$1", + org, + ) + ch = get_client() + await ch.connect() + try: + horizon = effective_horizon_minutes() * 60.0 + offered = [ + r for r in await pg_memory.clearable_revisions(horizon, None, 100_000) if r.org == org + ] + assert len(offered) == 2 + assert await pg_memory.clear_payloads(horizon, await history.versions_held(offered)) == 2 + finally: + await ch.disconnect() + await pg.close_pool() + assert ( + await pg_conn.fetchval( + "SELECT count(*) FROM loom.memory_revisions WHERE org=$1 AND row IS NULL", org + ) + == 2 + ) + erased = await call( + "erasure.request", + { + "scope_type": scope_type, + "scope_value": user if scope_type == "user_id" else namespace, + "requester": "operator", + }, + ) + assert erased.status_code == 200 and erased.json()["status"] == "completed", erased.text + assert ( + await pg_conn.fetchval("SELECT count(*) FROM loom.memory_revisions WHERE org=$1", org) == 0 + ) + target = "future" if scope_type == "user_id" else namespace + for value in values: + returned = await call( + "memory.set", _set(target, uuid.uuid4().hex, value, user_id=user, classify=False) + ) + assert returned.status_code == 422, returned.text + assert returned.json()["detail"]["reason"] == "forgotten" + unrelated = await call( + "memory.set", + _set(target, "unrelated", "A separate current fact", user_id=user, classify=False), + ) + assert unrelated.status_code == 200, unrelated.text + + +async def test_erasure_keeps_consent_for_native_dependents_without_a_clickhouse_parent( + pg_conn, call, org, ch_client +): + """Authoritative keys capture another writer's raw quote despite projection lag.""" + private = f"private source {uuid.uuid4().hex}" + quote = f"derived quote {uuid.uuid4().hex}" + for key, user, value, tags in ( + ("source", "alice", private, []), + ("summary", "bob", "A consolidated private source", ["consolidated", "src:source"]), + ): + response = await call( + "memory.set", _set("facts", key, value, user_id=user, tags=tags, classify=False) + ) + assert response.status_code == 200, response.text + await _published(pg_conn, org) + ch_client.insert( + "loom.memory_chunks", + [[org, "facts", "a", "bob", "-", "summary", 0, quote, [0.0] * 3072, 3072]], + column_names=[ + "org", + "namespace", + "agent", + "user_id", + "session_id", + "parent_memory_key", + "chunk_index", + "content", + "embedding", + "embedding_dim", + ], + ) + for table in ("memory", "memory_versions"): + ch_client.command( + f"DELETE FROM loom.{table} WHERE org={{org:String}}", parameters={"org": org} + ) + erased = await call( + "erasure.request", + {"scope_type": "user_id", "scope_value": "alice", "requester": "operator"}, + ) + assert erased.status_code == 200 and erased.json()["status"] == "completed", erased.text + assert await pg_conn.fetchval("SELECT count(*) FROM loom.memory_current WHERE org=$1", org) == 0 + for value in (private, quote, "A consolidated private source"): + returned = await call( + "memory.set", _set("future", uuid.uuid4().hex, value, user_id="alice", classify=False) + ) + assert returned.status_code == 422, returned.text + assert returned.json()["detail"]["reason"] == "forgotten" + unrelated = await call( + "memory.set", + _set("future", "unrelated", "An unrelated current fact", user_id="alice", classify=False), + ) + assert unrelated.status_code == 200, unrelated.text + + @pytest.fixture async def pg_authority(control_plane_url: str) -> str: """The DSN of the Postgres authority the server under test runs on; skips otherwise.""" @@ -69,8 +190,9 @@ def call(control_plane_url: str, mint_test_token: Any, org: str) -> Any: token = mint_test_token(org=org, namespaces=["*"], roles=["writer", "admin", "org_admin"]) async def _call(op: str, body: dict[str, Any], **headers: str) -> httpx.Response: - # An erasure on a remote ClickHouse runs longer than a write. - async with httpx.AsyncClient(base_url=control_plane_url, timeout=120) as c: + # Erasure waits out publishers before waiting for ClickHouse mutations. + timeout = httpx.Timeout(120, read=600 if op == "erasure.request" else 120) + async with httpx.AsyncClient(base_url=control_plane_url, timeout=timeout) as c: return await c.post( f"/v1/{op}", json={"org": org, **body}, @@ -164,8 +286,8 @@ async def test_a_bulk_write_of_bytes_another_key_holds_is_judged_on_the_authorit ) -> None: """Identity dedup holds an entry whose bytes a live key of the identity holds, even one whose delivery ClickHouse is still owed (a write that answered at its commit); - a key deleted or rewritten since holds them no more, whatever ClickHouse holds yet, - and the entry is written.""" + a key rewritten since holds them no more, whatever ClickHouse holds yet. An explicit + deletion refuses those bytes under another key too.""" from server.core.hashing import sha256_value from server.memory.store import pg_memory @@ -206,6 +328,19 @@ async def test_a_bulk_write_of_bytes_another_key_holds_is_judged_on_the_authorit "memory.bulk_set", {"namespace": ns, "agent": "a", "entries": [{"memory_key": "b", "value": value}]}, ) + if keeper == "deleted": + assert out.status_code == 422, out.text + assert out.json()["detail"]["reason"] == "forgotten" + assert ( + await pg_conn.fetchval( + "SELECT count(*) FROM loom.memory_current " + "WHERE org = $1 AND namespace = $2 AND memory_key = 'b'", + org, + ns, + ) + == 0 + ) + return assert out.status_code == 200, out.text written = [r["memory_key"] for r in out.json()["written"] if r.get("version")] assert written == ([] if keeper == "undelivered" else ["b"]), out.text @@ -775,6 +910,7 @@ async def test_a_redact_reaches_a_revision_by_its_metadata( whose payload is cleared through its columns, which it then marks.""" ns = f"pg-{uuid.uuid4().hex[:8]}" first = (await call("memory.set", _set(ns, "k", "the token is hunter2"))).json() + await _published(pg_conn, org) assert (await call("memory.set", _set(ns, "k", "the token was rotated"))).status_code == 200 await _published(pg_conn, org) at = "WHERE org = $1 AND namespace = $2 AND version = $3" @@ -1231,6 +1367,8 @@ async def test_a_relabel_refused_for_a_head_written_since_its_read_fences_nothin ns = f"pg-{uuid.uuid4().hex[:8]}" scanned = (await call("memory.set", _set(ns, "k", "the vault password is hunter2"))).json() + await _published(pg_conn, org) + assert _live_sweep_vectors(ch_client, org, ns, "a") >= 1 key = (org, ns, "a", "-", "-", "k") candidate = sensitivity_sweep.SweepCandidate( memory_id=scanned["memory_id"], @@ -1558,6 +1696,7 @@ async def test_a_user_erasure_leaves_a_consolidated_row_another_namespace_built( _set(theirs, "profile", "uses emacs", **tags), ): assert (await call("memory.set", body)).status_code == 200 + await _published(pg_conn, org) r = await call( "erasure.request", {"scope_type": "user_id", "scope_value": user, "requester": "t"} ) @@ -2034,6 +2173,10 @@ async def test_a_redaction_leaves_another_key_holding_the_same_id_alone( ns = f"pg-{uuid.uuid4().hex[:8]}" first = (await call("memory.set", _set(ns, "k", "the badge number is 99"))).json() copy = {**await _head_row(pg_conn, org, ns), "agent": "b", "version": rows.next_version()} + try: + await pg_memory.commit_versions([copy], [audit_row_for(copy, request_id="", writer="")]) + finally: + await pg.close_pool() gone = await call("memory.delete", {"namespace": ns, "agent": "a", "memory_key": "k"}) assert gone.status_code == 200, gone.text tombstone = await pg_conn.fetchval( @@ -2042,7 +2185,6 @@ async def test_a_redaction_leaves_another_key_holding_the_same_id_alone( ns, ) try: - await pg_memory.commit_versions([copy], [audit_row_for(copy, request_id="", writer="")]) _, copies = await pg_memory.redact( (org, ns, "a", "-", "-", "k"), first["memory_id"], @@ -2420,6 +2562,8 @@ async def test_a_reject_is_the_authoritys_head_and_clickhouses_row( and ClickHouse's memory row both read rejected, with the reason.""" ns = f"pg-{uuid.uuid4().hex[:8]}" assert (await call("memory.set", _set(ns, "k", "a fact to reject"))).status_code == 200 + await _published(pg_conn, org) + assert _live_sweep_vectors(ch_client, org, ns, "a") >= 1 before = int((await _head_row(pg_conn, org, ns, "k"))["version"]) body = {"namespace": ns, "agent": "a", "memory_key": "k", "rejection_reason": "wrong"} rejected = await call("memory.reject", body) @@ -2615,6 +2759,7 @@ async def test_an_erasure_takes_an_in_flight_requests_claim( """A claim a request still holds is fenced by the org's erasure: its commit then finds no claim under its token and is refused, so it stores no answer.""" from server.core import pg + from server.core.hashing import sha256_value from server.memory.store import pg_memory ns = f"pg-{uuid.uuid4().hex[:8]}" @@ -2640,6 +2785,8 @@ async def test_an_erasure_takes_an_in_flight_requests_claim( == 2 ) row["memory_id"], row["version"] = str(uuid.uuid4()), int(row["version"]) + 1 + value = "A different request must still honor the fenced claim" + row["value"], row["content_sha256"] = json.dumps(value), sha256_value(value) try: with pytest.raises(pg_memory.IdempotencyFenced): await pg_memory.commit_versions([row], [{}], idempotency=(org, key, 1)) @@ -2834,6 +2981,7 @@ async def test_a_redaction_fences_the_claim_still_sending_the_old_text( ns = f"pg-{uuid.uuid4().hex[:8]}" first = (await call("memory.set", _set(ns, "k", "the token is hunter2"))).json() + await _published(pg_conn, org) assert (await call("memory.set", _set(ns, "k", "the token was rotated"))).status_code == 200 await _published(pg_conn, org) seq = await pg_conn.fetchval( @@ -3116,6 +3264,7 @@ async def test_a_user_erasure_run_again_waits_out_what_an_earlier_run_removed( """A user's scope reads the org's evidence: a run that finds the user's rows gone still waits out the publishers the earlier run removed.""" from server.core import pg + from server.core.ch import get_client from server.erasure.store import steps_pg from server.memory.store import pg_memory @@ -3133,13 +3282,18 @@ async def test_a_user_erasure_run_again_waits_out_what_an_earlier_run_removed( seq, pg_memory.MEMORY_ROW, ) + ch = get_client() + await ch.connect() try: removed, leases = await steps_pg._erase_once(org, None, user) assert removed == 2 and 15 < max(leases) <= 20 removed, leases = await steps_pg._erase_once(org, None, user) assert removed == 0 and 15 < max(leases) <= 20 finally: - await pg.close_pool() + try: + await ch.disconnect() + finally: + await pg.close_pool() @pytest.mark.usefixtures("pg_authority") @@ -3381,8 +3535,29 @@ def memory_status(key: str) -> str: tombstoned = {"tombstoned_at": "2026-01-01T00:00:00"} await _unpublished(pg_conn, org, ns, "deleted", tombstoned, owed, an_hour) for k in ("deleted", "erased", "published"): - await memory_repo.reemit_embedding_rows_for_keys( - [(ident[k], {"status": "superseded"})], ch=ch + vectors = await memory_repo.live_embedding_rows_for_key( + ch, org=org, namespace=ns, agent="a", user_id="-", session_id="-", memory_key=k + ) + assert vectors + # Corrupt the projection directly; publication admission refuses this mismatch. + await ch.insert( + "loom.memory_embeddings", + [ + memory_repo.build_embedding_row( + **( + { + c: v + for c, v in vector.items() + if c in memory_repo.EMBEDDING_PERSISTED_COLS + } + | { + "status": "superseded", + "version": memory_repo.reemit_version(vector), + } + ) + ) + for vector in vectors + ], ) await pg_conn.execute( "DELETE FROM loom.memory_current WHERE org = $1 AND namespace = $2 " @@ -3497,8 +3672,19 @@ async def test_a_restores_failed_search_document_is_left_to_the_worker( ns = f"pg-{uuid.uuid4().hex[:8]}" assert (await call("memory.set", _set(ns, "k", "a fact to bring back"))).status_code == 200 - gone = await call("memory.delete", {"namespace": ns, "agent": "a", "memory_key": "k"}) - assert gone.status_code == 200, gone.text + # Automatic retirement has no permanent refusal, so a restore remains permitted. + from server.memory.store.pg_publish import audit_row_for + + original = rows._hydrate(await _head_row(pg_conn, org, ns)) + retired = rows.build_memory_version_row( + original, version=rows.version_over(original), tombstoned_at=datetime.now(UTC) + ) + try: + await pg_memory.commit_versions( + [retired], [audit_row_for(retired, request_id="", writer="")], publisher_lease=False + ) + finally: + await pg.close_pool() head = rows._hydrate(await _head_row(pg_conn, org, ns)) row = rows.build_memory_version_row( head, version=rows.version_over(head), tombstoned_at=None, expires_at=None @@ -3901,6 +4087,7 @@ async def test_a_fenced_holder_is_waited_out_after_its_delivery_moves_on( ns = f"pg-{uuid.uuid4().hex[:8]}" first = (await call("memory.set", _set(ns, "k", "the token is hunter2"))).json() + await _published(pg_conn, org) assert (await call("memory.set", _set(ns, "k", "the token was rotated"))).status_code == 200 await _published(pg_conn, org) seq = await pg_conn.fetchval( @@ -4126,8 +4313,11 @@ async def test_a_retry_that_took_over_a_fenced_claim_stores_no_residue( from server.erasure.store.steps_pg import pg_residue ns = f"pg-{uuid.uuid4().hex[:8]}" - body, key = _set(ns, "k", "a fact in flight"), uuid.uuid4().hex - assert (await call("memory.set", body, **{"Idempotency-Key": key})).status_code == 200 + original = _set(ns, "k", "a fact in flight") + assert (await call("memory.set", original)).status_code == 200 + body, key = {"namespace": ns, "agent": "a", "memory_key": "k"}, uuid.uuid4().hex + deleted = await call("memory.delete", body, **{"Idempotency-Key": key}) + assert deleted.status_code == 200 and deleted.json()["tombstoned"], deleted.text # The request as it was before the erasure: claimed long ago, still running. await pg_conn.execute( "UPDATE loom.idempotency SET state = 'running', response_body = NULL, " @@ -4139,11 +4329,16 @@ async def test_a_retry_that_took_over_a_fenced_claim_stores_no_residue( "erasure.request", {"scope_type": "namespace", "scope_value": ns, "requester": "t"} ) assert r.json()["status"] == "completed", r.text - retry = await call("memory.set", body, **{"Idempotency-Key": key}) + retry = await call("memory.delete", body, **{"Idempotency-Key": key}) assert retry.status_code == 200, retry.text + assert not retry.json()["tombstoned"], retry.text assert ( await pg_conn.fetchval("SELECT state FROM loom.idempotency WHERE org = $1", org) == "done" ) + refused = await call("memory.set", original) + assert refused.status_code == 422 and refused.json()["detail"]["reason"] == "forgotten", ( + refused.text + ) try: residue = await pg_residue(org, "namespace", ns) finally: @@ -4429,10 +4624,7 @@ async def test_erasing_a_namespace_leaves_its_restored_copy_writable( async def test_a_restore_leaves_out_the_memories_an_erasure_removed_while_evidence_lasts( call: Any, pg_conn: Any, org: str ) -> None: - """A restore into the namespace writes the snapshot's memories but those an erasure - removed from it, counts the ones left out, and is done for that snapshot. The - evidence expires by its age, not by the sweep: a snapshot first restored after it - writes the memory, and an erasure of it again starts its evidence again.""" + """A restore skips recent erasures and still refuses them after evidence expiry.""" from server.core import pg from server.memory.store import pg_memory from server.settings import settings @@ -4475,10 +4667,10 @@ async def _age_evidence() -> None: ) await _erase() - gone = await call( - "memory.delete", {"namespace": ns, "agent": "a", "memory_key": "k2", "user_id": "other"} + rejected = await call( + "memory.reject", {"namespace": ns, "agent": "a", "memory_key": "k2", "user_id": "other"} ) - assert gone.status_code == 200, gone.text + assert rejected.status_code == 200, rejected.text sid = snapshot.json()["snapshot_id"] restored = await _restore(sid) assert restored["restored"] is True and restored["counts"]["erased_skipped"] == 1 @@ -4496,14 +4688,11 @@ async def _age_evidence() -> None: replay = await _restore(sid) assert replay["restored"] is False and replay["counts"]["erased_skipped"] == 1 assert (await _get("k1", erased_user)).get("found") is False - # Another snapshot id restores it: its commit is no longer refused. - again = await _restore(str(uuid.uuid4())) - assert again["counts"]["erased_skipped"] == 0 - assert (await _get("k1", erased_user))["memory_id"] == mid - # Erased again while the old evidence row is still there: the evidence restarts. - await _erase() - assert await pg_memory.erased_ids(org, ns, [mid]) == {mid} - await _age_evidence() + payload = {**snapshot.json(), "snapshot_id": str(uuid.uuid4())} + again = await call("namespace.restore", {"target_namespace": ns, "payload": payload}) + assert again.status_code == 422, again.text + assert again.json()["detail"]["reason"] == "forgotten" + assert (await _get("k1", erased_user)).get("found") is False await pg_memory.sweep_erasure_evidence(10_000) finally: await pg.close_pool() diff --git a/tests/integration/test_profile.py b/tests/integration/test_profile.py index 038e454b15..0826e498c3 100644 --- a/tests/integration/test_profile.py +++ b/tests/integration/test_profile.py @@ -159,7 +159,17 @@ async def test_profile_append_respects_existing_cap(agent) -> None: @pytest.mark.asyncio async def test_profile_restore_preserves_char_limit(agent) -> None: await agent.profile.set("revivable", "Stable persona text.", char_limit=1024) - await agent.profile.delete("revivable") + from server.memory.api.models_crud import MemoryDeleteInput + from tests.integration._memory import retire_reversibly + + await retire_reversibly( + MemoryDeleteInput( + org=agent.org, + namespace=agent.namespace, + agent=agent.agent, + memory_key="revivable", + ) + ) await agent.memory.restore("revivable") got = await agent.profile.get("revivable") assert got is not None @@ -226,7 +236,17 @@ async def test_memory_restore_preserves_pinned(agent) -> None: got = await agent.memory.get(memory_key="important") assert got is not None and got.get("pinned") is True, "pin failed to take" - await agent.memory.delete(memory_key="important") + from server.memory.api.models_crud import MemoryDeleteInput + from tests.integration._memory import retire_reversibly + + await retire_reversibly( + MemoryDeleteInput( + org=agent.org, + namespace=agent.namespace, + agent=agent.agent, + memory_key="important", + ) + ) await agent.memory.restore("important") got = await agent.memory.get(memory_key="important") diff --git a/tests/integration/test_ratelimit.py b/tests/integration/test_ratelimit.py index a0005c0b3d..9bef4ee7b9 100644 --- a/tests/integration/test_ratelimit.py +++ b/tests/integration/test_ratelimit.py @@ -177,7 +177,12 @@ async def test_memory_class_more_generous_than_knowledge_add( async with httpx.AsyncClient( base_url=control_plane_url, timeout=10.0, headers=_auth_headers() ) as c: - responses = await asyncio.gather(*[c.post("/v1/memory.set", json=body) for _ in range(15)]) + responses = await asyncio.gather( + *[ + c.post("/v1/memory.set", json={**body, "memory_key": f"k{i}", "value": i}) + for i in range(15) + ] + ) statuses = [r.status_code for r in responses] assert all(s == 200 for s in statuses), f"memory.set should not throttle at 15 reqs: {statuses}" diff --git a/tests/integration/test_raw_evidence_sensitivity.py b/tests/integration/test_raw_evidence_sensitivity.py index 4a83fc6ee0..492595bd03 100644 --- a/tests/integration/test_raw_evidence_sensitivity.py +++ b/tests/integration/test_raw_evidence_sensitivity.py @@ -198,9 +198,22 @@ def test_a_chunk_an_earlier_build_embedded_is_held_back_by_its_text( def test_a_session_digest_that_reads_as_a_credential_has_no_vector( control_plane_url: str, ch_client, token: str ) -> None: - """Imported so the digest's text is known: a plain digest is embedded and found, one - that reads as high is stored as written with no vector and never a hit.""" + """A proven plain digest is found; credential-bearing digest text is withheld.""" plain, secret = f"s-{uuid.uuid4().hex[:8]}", f"s-{uuid.uuid4().hex[:8]}" + for session, text in ((plain, _PLAIN_TURN[:300]), (secret, _SECRET_TURN[:300])): + source = _post( + control_plane_url, + token, + NS_DIGEST, + "memory.set", + { + "session_id": session, + "memory_key": "digest-source", + "memory_type": "episodic", + "value": text, + }, + ) + assert source.status_code == 200, source.text r = _post( control_plane_url, token, @@ -208,12 +221,18 @@ def test_a_session_digest_that_reads_as_a_credential_has_no_vector( "memory.import_evidence", { "digests": [ - {"session_id": plain, "summary_text": _PLAIN_TURN[:300]}, - {"session_id": secret, "summary_text": _SECRET_TURN[:300]}, + { + "session_id": session, + "summary_text": text, + "key_facts": [text], + "derived_from": ["digest-source"], + } + for session, text in ((plain, _PLAIN_TURN[:300]), (secret, _SECRET_TURN[:300])) ] }, ) - assert r.status_code == 200 and r.json()["digests_written"] == 2, r.text + assert r.status_code == 200 and r.json()["digests_written"] == 1, r.text + assert r.json()["skipped"] == 1, r.text dims = dict( ch_client.query( "SELECT session_id, embedding_dim FROM loom.session_summaries FINAL " @@ -221,7 +240,7 @@ def test_a_session_digest_that_reads_as_a_credential_has_no_vector( parameters={"o": ORG, "n": NS_DIGEST}, ).result_rows ) - assert int(dims[secret]) == 0 and int(dims[plain]) > 0, dims + assert secret not in dims and int(dims[plain]) > 0, dims r = _search(control_plane_url, token, NS_DIGEST, "nightly export billing job key") assert f"summary:{secret}" not in _keys(r) assert _API_KEY not in r.text and _AWS_KEY_ID not in r.text diff --git a/tests/integration/test_read_scope_isolation.py b/tests/integration/test_read_scope_isolation.py index 18f698c3b4..2b0d4a6cbc 100644 --- a/tests/integration/test_read_scope_isolation.py +++ b/tests/integration/test_read_scope_isolation.py @@ -348,7 +348,11 @@ def _bounded_writes() -> list[Any]: names = {n for n, why in NARROW_SAFE_OPS.items() if why.startswith("a write")} names |= {n for n in SCOPED_OPS if _writes(OPERATIONS_BY_NAME[n])} - return [OPERATIONS_BY_NAME[n] for n in sorted(names)] + # A permanent delete runs after writes that reuse this walk's canary value. + return [ + OPERATIONS_BY_NAME[n] + for n in sorted(names, key=lambda name: (name == "memory.delete", name)) + ] def _refused_ops() -> list[Any]: @@ -992,6 +996,18 @@ def test_writes_aimed_at_vs_runs_threads_records_and_blobs_leave_them_alone( "memory.set", {**v, "memory_key": f"pinned/{tag}", "value": pinned_text, "pinned": True}, ) + wait_until( + lambda: ( + ch_client.query( + "SELECT count() FROM loom.memory_embeddings FINAL " + "WHERE org = {org:String} AND namespace = {ns:String} " + "AND memory_key = {key:String} AND tombstoned_at IS NULL", + parameters={"org": seed.org, "ns": seed.ns, "key": f"pinned/{tag}"}, + ).result_rows[0][0] + == 1 + ), + description="the pinned conflict peer to be published", + ) near = {"memory_key": f"near/{tag}", "value": pinned_text.replace(code, "unknown")} # Control: an unbounded writer is blocked by the pinned row, so the probe can see it. control = _write( diff --git a/tests/integration/test_repositories_live_ch.py b/tests/integration/test_repositories_live_ch.py index 4a663f29b4..664cfe0947 100644 --- a/tests/integration/test_repositories_live_ch.py +++ b/tests/integration/test_repositories_live_ch.py @@ -6,6 +6,7 @@ from __future__ import annotations +import json import os import uuid from datetime import UTC, datetime @@ -13,6 +14,7 @@ import pytest +from server.core.hashing import sha256_value from server.identity.domain.read_scope import WHOLE_NAMESPACE pytestmark = pytest.mark.skipif( @@ -87,6 +89,7 @@ async def _seed_memory( "memory_key": memory_key, "memory_type": memory_type, "value": json.dumps(value, default=str), + "content_sha256": sha256_value(value), "confidence": confidence, "version": version, "pinned": 1 if pinned else 0, @@ -97,6 +100,44 @@ async def _seed_memory( return version +async def _seed_promoted( + ch: Any, org: str, ns: str, key: str, value: str, memory_type: str = "semantic" +) -> Any: + from server.memory import store as memory_repo + from server.memory.domain.promotion_source import promotion_source + from server.memory.store import shared + from server.memory.store.refusals import source_publication + + identity = memory_repo.Identity(org=org, namespace=ns, agent="itest-bot") + await _seed_memory(ch, identity, key, memory_type, value) + source = await memory_repo.get_by_key(identity, key, ch=ch, fresh=True, scope=WHOLE_NAMESPACE) + assert source is not None + now = datetime.now(UTC).replace(tzinfo=None) + row = { + "org": org, + "namespace": ns, + "memory_key": key, + "value": json.dumps(value), + "memory_type": memory_type, + "confidence": 0.9, + "promoted_from": [ + promotion_source(org, ns, source.agent, source.user_id, source.session_id, key) + ], + "promotion_event": str(uuid.uuid4()), + "promoted_from_candidate_id": None, + "promoted_by": "process:test", + "rationale": "fixture", + "promoted_at": now, + "reversible_until": now, + "superseded_by": None, + "tombstoned_at": None, + "version": memory_repo.next_version(), + } + async with source_publication(ch, source, key): + await shared.insert_shared_row(row, ch=ch) + return source + + async def _tombstone_memory(ch: Any, identity: Any, memory_key: str) -> None: await ch.execute( """ @@ -472,33 +513,9 @@ async def test_shared_repo_insert_fetch_promoted( org, ns = "itest-org", isolated_id key = "itest-shared-key" - now = datetime.now(UTC).replace(tzinfo=None) - promo_id = str(uuid.uuid4()) - await shared_repo.insert_shared_row( - { - "org": org, - "namespace": ns, - "memory_key": key, - "value": "v1", - "memory_type": "semantic", - "confidence": 0.9, - "promoted_from": [], - "promotion_event": promo_id, - "promoted_from_candidate_id": None, - "promoted_by": "itest", - "rationale": "itest", - "promoted_at": now, - "reversible_until": now, - "superseded_by": None, - "tombstoned_at": None, - "version": 1, - }, - ch=ch, - ) - + await _seed_promoted(ch, org, ns, key, "v1") row = await shared_repo.fetch_shared(org, ns, key, ch=ch) - assert row is not None - assert row["value"] == "v1" + assert row is not None and json.loads(row["value"]) == "v1" @pytest.mark.asyncio @@ -699,6 +716,10 @@ async def test_embedding_worker_reconciles_shared_stub( org = "itest-org" ns = isolated_id memory_key = f"shared-orphan-{uuid.uuid4().hex[:6]}" + from server.memory.domain.excerpt import build_excerpt + + value = "this is the content the worker will embed" + await _seed_promoted(ch, org, ns, memory_key, value, "preference") # tz-aware UTC matches the worker's INSERT path (collapses on FINAL) now = datetime.now(UTC) # stub must use the worker's model_name or rows never collapse on FINAL @@ -720,7 +741,8 @@ async def test_embedding_worker_reconciles_shared_stub( "memory_key": memory_key, "memory_type": "preference", "scope": "shared", - "content_excerpt": "this is the content the worker will embed", + "content_excerpt": build_excerpt(json.dumps(value), memory_key=memory_key), + "content_sha256": sha256_value(value), "embedding_model": embedder_model, "embedding_dim": 0, "embedding": _wide([]), diff --git a/tests/integration/test_restore_reopens_facts_ch.py b/tests/integration/test_restore_reopens_facts_ch.py index bdf66d74fc..9779ae69e4 100644 --- a/tests/integration/test_restore_reopens_facts_ch.py +++ b/tests/integration/test_restore_reopens_facts_ch.py @@ -52,15 +52,15 @@ async def _paris_open(ch: Any, org: str, want: bool) -> bool: async def test_undoing_a_delete_reopens_the_facts_the_row_states(ch) -> None: from server.memory.api.models_crud import MemoryDeleteInput, MemoryRestoreInput - from server.memory.app.handlers import handle_memory_delete from server.memory.app.restore import handle_memory_restore + from tests.integration._memory import retire_reversibly org = unique_org("restore-facts") fact = ("user", "live_in", "paris") a, b = f"a-{uuid.uuid4().hex[:6]}", f"b-{uuid.uuid4().hex[:6]}" await _set(org, a, "I live in Paris", [fact]) await _set(org, b, "Home for me is the city of Paris, France", [fact]) - await handle_memory_delete(MemoryDeleteInput(org=org, namespace=NS, agent="bot", memory_key=a)) + await retire_reversibly(MemoryDeleteInput(org=org, namespace=NS, agent="bot", memory_key=a)) # B still states it, so it stays open; B's revision drops it, and nothing holds it. assert await _paris_open(ch, org, want=True) await _set(org, b, "Weekends go to sailing lessons on the lake", []) diff --git a/tests/integration/test_scim_provider_flows.py b/tests/integration/test_scim_provider_flows.py index d6121905ec..4f63f2e5ec 100644 --- a/tests/integration/test_scim_provider_flows.py +++ b/tests/integration/test_scim_provider_flows.py @@ -78,7 +78,7 @@ def test_okta_spec_test(scim_server) -> None: # Deactivated by a path-less replace (Okta OIN) or a full PUT (Okta AIW). off = _patch(base, user["id"], {"op": "replace", "value": {"active": False}}) - assert off.status_code == 200 and off.json()["active"] is False + assert off.status_code == 200 and off.json()["active"] is False, off.text on = _scim(base, "PUT", f"/Users/{user['id']}", json={**okta, "active": True}) assert on.status_code == 200 and on.json()["active"] is True assert _scim(base, "GET", "/Groups", params={"startIndex": "1", "count": "100"}).json() == { @@ -131,7 +131,7 @@ def test_entra_provisioning_and_validator(scim_server) -> None: ) assert edits.status_code == 200, edits.text off = _patch(base, uid, {"op": "Replace", "path": "active", "value": "False"}) - assert off.status_code == 200 and off.json()["active"] is False + assert off.status_code == 200 and off.json()["active"] is False, off.text assert _scim(base, "GET", f"/Users/{uid}").json()["active"] is False # still listed # The aadOptscim062020 dialect: a path-less replace with dotted keys. on = _patch( diff --git a/tests/integration/test_sensitivity_high_canary.py b/tests/integration/test_sensitivity_high_canary.py index 3f8d25c74f..6821bcf854 100644 --- a/tests/integration/test_sensitivity_high_canary.py +++ b/tests/integration/test_sensitivity_high_canary.py @@ -285,12 +285,17 @@ def test_payment_and_identity_numbers_are_labelled_on_write( "recall_by_agent": [], "sensitivity": level, }, r.text[:500] - stored = ch_client.query( - "SELECT memory_id, org, namespace, agent, user_id, session_id, sensitivity " - "FROM loom.memory FINAL WHERE org = {o:String} AND namespace = {n:String} " - "AND memory_key = {k:String} AND tombstoned_at IS NULL", - parameters={"o": ORG, "n": NAMESPACE, "k": key}, - ).result_rows + + def stored_rows(): + return ch_client.query( + "SELECT memory_id, org, namespace, agent, user_id, session_id, sensitivity " + "FROM loom.memory FINAL WHERE org = {o:String} AND namespace = {n:String} " + "AND memory_key = {k:String} AND tombstoned_at IS NULL", + parameters={"o": ORG, "n": NAMESPACE, "k": key}, + ).result_rows + + _wait(lambda: bool(stored_rows())) + stored = stored_rows() assert [tuple(map(str, row)) for row in stored] == [ (got["memory_id"], ORG, NAMESPACE, "canary", "-", "-", level) ] diff --git a/tests/integration/test_vector_retire_race.py b/tests/integration/test_vector_retire_race.py index 9571ced922..291594413c 100644 --- a/tests/integration/test_vector_retire_race.py +++ b/tests/integration/test_vector_retire_race.py @@ -9,7 +9,6 @@ from __future__ import annotations import asyncio -import hashlib import json import os import uuid @@ -19,6 +18,7 @@ import pytest +from server.core.hashing import sha256_value from server.settings import settings from tests.integration._eventually import eventually @@ -83,8 +83,8 @@ def _memory_row(ns: str, version: int, value: str, sensitivity: str) -> dict[str "memory_key": KEY, "memory_type": "semantic", "value": json.dumps(value), - "value_json": json.dumps({"v": value}), - "content_sha256": hashlib.sha256(value.encode()).hexdigest(), + "value_json": "{}", + "content_sha256": sha256_value(value), "confidence": 1.0, "actor": "process:itest", "source_class": "asserted", @@ -119,6 +119,7 @@ async def _embedded_write(ch: Any, ns: str, value: str, entity: str) -> None: memory_type="semantic", scope="private", content_excerpt=value, + content_sha256=sha256_value(value), embedding_model="fake", embedding_dim=_DIM, embedding=_wide([0.1, 0.2, 0.3]), @@ -870,10 +871,11 @@ async def live_chunks() -> int: async def test_a_deleted_memory_restores_searchable(ch: Any) -> None: - """set, delete, restore: the restored key has a live vector again (the restore's + """Set, automatic retirement, restore: the restored key has a live vector again (the restore's embeddings read once failed on every call with ILLEGAL_AGGREGATION).""" from server.memory.api.models_crud import MemoryDeleteInput, MemoryRestoreInput, MemorySetInput - from server.memory.app import handlers, restore, write + from server.memory.app import restore, write + from tests.integration._memory import retire_reversibly ns = f"itest-{uuid.uuid4().hex[:12]}" key = f"restore_live_{uuid.uuid4().hex[:6]}" @@ -892,7 +894,7 @@ async def live_vectors() -> int: value = f"the {key} vault is audited each quarter" await write.handle_memory_set(MemorySetInput(**ident, memory_key=key, value=value)) assert await live_vectors() == 1 - out = await handlers.handle_memory_delete(MemoryDeleteInput(**ident, memory_key=key)) + out = await retire_reversibly(MemoryDeleteInput(**ident, memory_key=key)) assert out.tombstoned and await live_vectors() == 0 res = await restore.handle_memory_restore(MemoryRestoreInput(**ident, memory_key=key)) assert res.restored @@ -1279,6 +1281,7 @@ async def query_all(self, sql: str, *a: Any, **kw: Any) -> Any: and "head" not in self.fired and "PREWHERE (org, namespace, agent, user_id, session_id, memory_key)" in sql and "FROM loom.memory FINAL" in sql + and "WHERE tombstoned_at IS NULL" in sql ): self.fired.append("head") await self._before_head() @@ -1355,6 +1358,7 @@ async def query_all(self, sql: str, *a: Any, **kw: Any) -> Any: and "head" not in self.fired and "PREWHERE (org, namespace, agent, user_id, session_id, memory_key)" in sql and "FROM loom.memory FINAL" in sql + and "WHERE tombstoned_at IS NULL" in sql ): self.fired.append("head") elif ( @@ -1607,7 +1611,8 @@ async def _high_key_with_row_above_its_fence( created_at=datetime.now(UTC).replace(tzinfo=None), version=int(head["version"]) + 10, ) - await M.insert_embedding_rows([row], ch=ch) + # Seed a legacy cache write already admitted before its source became high. + await ch.insert("loom.memory_embeddings", [row], settings={"async_insert": 0}) return secret, int(head["version"]) @@ -1636,8 +1641,8 @@ async def test_a_high_candidate_whose_head_moved_retires_nothing(ch: Any) -> Non newer = scanned | { "version": M.next_version(), "value": json.dumps(text), - "value_json": json.dumps({"v": text}), - "content_sha256": hashlib.sha256(text.encode()).hexdigest(), + "value_json": "{}", + "content_sha256": sha256_value(text), } await M.insert_memory_versions([newer], ch=ch) embedder = get_embedder() @@ -1651,6 +1656,7 @@ async def test_a_high_candidate_whose_head_moved_retires_nothing(ch: Any) -> Non memory_type="semantic", scope="private", content_excerpt=text, + content_sha256=newer["content_sha256"], embedding_model=embedder.model_name, embedding_dim=embedder.dim, embedding=[0.1] * embedder.dim, @@ -1702,6 +1708,7 @@ async def test_a_high_candidate_is_labelled_whatever_other_key_holds_its_id(ch: memory_type="semantic", scope="private", content_excerpt="the vault password", + content_sha256=scanned["content_sha256"], embedding_model=embedder.model_name, embedding_dim=embedder.dim, embedding=[0.1] * embedder.dim, @@ -1757,6 +1764,7 @@ async def test_the_sweep_labels_a_row_stored_with_no_user_or_session(ch: Any) -> memory_type="semantic", scope="private", content_excerpt="the badge PIN", + content_sha256=row["content_sha256"], embedding_model=embedder.model_name, embedding_dim=embedder.dim, embedding=[0.1] * embedder.dim, @@ -1879,6 +1887,7 @@ def late_row(key: str, *, ago_s: int = 30) -> dict[str, Any]: memory_type="semantic", scope="private", content_excerpt=f"the {word} runbook", + content_sha256=sha256_value(f"the {word} runbook"), embedding_model=embedder.model_name, embedding_dim=embedder.dim, embedding=[0.1] * embedder.dim, @@ -1886,6 +1895,12 @@ def late_row(key: str, *, ago_s: int = 30) -> dict[str, Any]: version=time.time_ns() - ago_s * 1_000_000_000, # drawn ago_s seconds ago ) + async def publish(rows: list[dict[str, Any]]) -> None: + for row in rows: + canonical = _memory_row(ns, row["version"], row["content_excerpt"], "") + await M.insert_memory_versions([{**canonical, "memory_key": row["memory_key"]}], ch=ch) + assert len(await M.insert_embedding_rows(rows, ch=ch)) == len(rows) + async def marker_indexed() -> bool: await refresh() row = await ch.query_one( @@ -1898,10 +1913,10 @@ async def marker_indexed() -> bool: try: # A row drawn now: once a pass has indexed it, the watermarks are past it and so # past the late rows below, drawn 30 s ago, whatever other namespaces wrote. - await M.insert_embedding_rows([late_row("marker", ago_s=0)], ch=ch) + await publish([late_row("marker", ago_s=0)]) await eventually(marker_indexed, max_wait=60.0, interval=1.0, description="the marker") indexed, control = late_row("slow_write"), late_row("slow_write_unindexed") - await M.insert_embedding_rows([indexed, control], ch=ch) + await publish([indexed, control]) assert await M.index_late_rows([indexed], ch=ch) == 1 await refresh() stats = await ch.query_all( @@ -1917,7 +1932,13 @@ async def marker_indexed() -> bool: ) assert int(df["n"]) == 2 # the marker and the indexed late row, not the control finally: - for table in ("memory_embeddings", "memory_doc_stats", "memory_term_freq"): + for table in ( + "memory", + "memory_versions", + "memory_embeddings", + "memory_doc_stats", + "memory_term_freq", + ): await ch.execute( f"DELETE FROM loom.{table} WHERE org = {{o:String}} AND namespace = {{n:String}}", params={"o": ORG, "n": ns}, diff --git a/tests/unit/bulk_stubs.py b/tests/unit/bulk_stubs.py index 615ca5e014..90195c76da 100644 --- a/tests/unit/bulk_stubs.py +++ b/tests/unit/bulk_stubs.py @@ -36,7 +36,11 @@ def __init__(self, lookup_rows: list[dict[str, Any]] | None = None) -> None: async def query_all( self, sql: str, params: dict[str, Any] | None = None ) -> list[dict[str, Any]]: - if "FROM loom.memory" in sql: + if "FROM loom.memory FINAL" in sql and "PREWHERE (org, namespace" in sql: + from server.memory.store import identity_key + + return [r for r in self.lookup_rows if identity_key(r) in (params or {})["keys"]] + if "FROM loom.memory " in sql: return self.lookup_rows return [] diff --git a/tests/unit/ch_fake.py b/tests/unit/ch_fake.py index ebdbe73779..743756e26d 100644 --- a/tests/unit/ch_fake.py +++ b/tests/unit/ch_fake.py @@ -88,7 +88,14 @@ def _read(self, kind: str, sql: str, params: dict[str, Any] | None) -> Rows: raise AssertionError(f"unrouted {kind}: {head!r}; routes: {known}") return [] - async def query_all(self, sql: str, params: dict[str, Any] | None = None, **_: Any) -> Rows: + async def query_all( + self, + sql: str, + params: dict[str, Any] | None = None, + *, + settings: dict[str, Any] | None = None, + **_: Any, + ) -> Rows: return self._read("query_all", sql, params) async def query_one( diff --git a/tests/unit/clickhouse_local.py b/tests/unit/clickhouse_local.py index f16aaf703a..04c90801ba 100644 --- a/tests/unit/clickhouse_local.py +++ b/tests/unit/clickhouse_local.py @@ -126,6 +126,13 @@ def __init__(self, path: Path, *setup: str) -> None: def create(self, *statements: str) -> None: """Run DDL (with any session `SET` it needs) in one session.""" + created = {m.group(1) for statement in statements for m in _CREATED.finditer(statement)} + if "loom.memory" in created: + # Canonical writers read consent and erasure state even when both are empty. + dependencies = ("memory_refusals", "erasure_requests") + statements += tuple( + ddl(table) for table in dependencies if f"loom.{table}" not in created | self.tables + ) run(list(statements), "SELECT 1", path=self.path) self.tables |= {m.group(1) for s in statements for m in _CREATED.finditer(s)} diff --git a/tests/unit/conftest.py b/tests/unit/conftest.py index 96cd97172f..0577304b9d 100644 --- a/tests/unit/conftest.py +++ b/tests/unit/conftest.py @@ -76,6 +76,11 @@ async def capture(org): monkeypatch.setattr(blacklist_guard, "capture_policy", capture) monkeypatch.setattr(blacklist_guard, "enforce_rows", allow) + async def no_refusal(ch, rows): + return None + + monkeypatch.setattr("server.memory.store.refusals.check_clickhouse", no_refusal) + @pytest.fixture(autouse=True) def auth_account_lifecycle(monkeypatch): @@ -1210,11 +1215,13 @@ def memory_insert_keeper(monkeypatch): Business-domain ClickHouse fakes need no Keeper SQL implementation. Tests of contention and failure can change this store or replace the fence explicitly. """ - from server.memory.store import write_fence + from server.memory.store import refusals, write_fence from tests.unit.claims_fake import FakeLockTable keeper = FakeLockTable() monkeypatch.setattr(write_fence, "_lease_client", lambda ch: keeper) + monkeypatch.setattr(refusals, "_lease_client", lambda ch: keeper) + monkeypatch.setattr(refusals, "_admission_wait_s", lambda: 0.0) # Domain tests fail fast on deliberate contention; waiting has explicit coverage. monkeypatch.setattr(write_fence, "_admission_wait_s", lambda: 0.0) return keeper diff --git a/tests/unit/memory_store.py b/tests/unit/memory_store.py index e3fda9da92..47599a4d8c 100644 --- a/tests/unit/memory_store.py +++ b/tests/unit/memory_store.py @@ -347,3 +347,37 @@ async def before(row: mr.MemoryRow, *, ch: Any = None) -> str: monkeypatch.setattr(mr, "status_before", before) return asked + + +def seed_summary_sources(monkeypatch: pytest.MonkeyPatch, rows: list[dict[str, Any]]) -> None: + """Give model-policy fixtures real, complete extractive digests and source heads.""" + heads = {} + for row in rows: + key = (row.get("derived_from") or [f"source:{row['session_id']}"])[0] + for column in ("agent", "user_id", "session_id"): + row.setdefault(column, "-") + text = row["summary_text"] + row.update(derived_from=[key], key_facts=[text]) + source = mr.MemoryRow( + memory_id=str(uuid.uuid4()), + org=row["org"], + namespace=row["namespace"], + agent=row["agent"], + user_id=row["user_id"], + session_id=row["session_id"], + memory_key=key, + memory_type="semantic", + value=text, + confidence=1, + version=1, + pinned=False, + created_at=datetime.now(UTC), + expires_at=None, + ) + heads[mr.identity_key(source)] = source + + async def read(keys: Any, **kwargs: Any) -> dict[Any, Any]: + assert kwargs["sequential"] + return {key: heads[key] for key in keys if key in heads} + + monkeypatch.setattr("server.memory.store.summary_sources.memory_rows_by_identity_keys", read) diff --git a/tests/unit/session_content_senders.json b/tests/unit/session_content_senders.json index 2ab7d3f24c..a88a417f63 100644 --- a/tests/unit/session_content_senders.json +++ b/tests/unit/session_content_senders.json @@ -406,7 +406,7 @@ "sdk/python/loom/integrations/langgraph.py": "5d68e6ab1645d868477a13b14aa6d734c72f72057464cac9a21aa707add17e7b", "sdk/python/loom/integrations/llamaindex.py": "06eaa071115cad752d27bdc9bf7c4d5b6f5d7d39d9896267fcd459a4e06bf661", "sdk/python/loom/integrations/openai_agents.py": "143fe1df060b96550a33f77fa44416e291c2154552e433f8309275f61c8fca2a", - "sdk/python/loom/memory.py": "e1384e84ddf58b503dc430eca1b2e734eb5e5680ac6904a64785bb50a63316d1", + "sdk/python/loom/memory.py": "ce7a8616b3d079d7c5241d932f04d85b5c6da6f4843d68d3938e52a3ddb6e8ee", "sdk/python/loom/run.py": "1d47d5f4d197f78de347d2cf66b9f59f4292846d214f7116c444035971c6ab42", "sdk/python/loom/session.py": "5f0fd61fcfdbf9d60888b111887fca8f32e24eb51ca9ff04a256901657c3202b", "sdk/typescript/src/integrations/claude-memory.ts": "c846f0011d7cf1ba4b30e99fbfd8adb6b29abff56856d0dbe14fba620007f6bc", diff --git a/tests/unit/snapshots/settings_defaults.json b/tests/unit/snapshots/settings_defaults.json index a44ad873f5..088ea4195a 100644 --- a/tests/unit/snapshots/settings_defaults.json +++ b/tests/unit/snapshots/settings_defaults.json @@ -810,6 +810,8 @@ "memory_decision_extract_known_max": 30, "memory_decision_extract_restatement_top_k": 20, "memory_decision_extract_window_chars": 24000, + "memory_digest_listing_max_pages": 256, + "memory_digest_listing_page_size": 256, "memory_durability_verifier_max_facts_per_call": 60, "memory_embed_answer_topic": true, "memory_embeddings_patch_parts_probe_seconds": 60, diff --git a/tests/unit/test_bm25_late_rows.py b/tests/unit/test_bm25_late_rows.py index c32c235946..7c9943a0eb 100644 --- a/tests/unit/test_bm25_late_rows.py +++ b/tests/unit/test_bm25_late_rows.py @@ -174,7 +174,11 @@ async def _noop(*_a: Any, **_k: Any) -> Any: monkeypatch.setattr(memory_repo, "index_late_rows", _index) monkeypatch.setattr(memory_repo, "insert_memory_versions", _noop) - monkeypatch.setattr(memory_repo, "insert_embedding_rows", _noop) + + async def _insert(rows: Any, **_: Any) -> Any: + return rows + + monkeypatch.setattr(memory_repo, "insert_embedding_rows", _insert) monkeypatch.setattr(memory_repo, "retire_vectors_for_keys", _noop) state = _BulkWriteState() state.memory_rows.append({**_row(), "memory_type": "semantic", "sensitivity": "", "version": 1}) @@ -204,9 +208,10 @@ async def _stubs(**_k: Any) -> list[dict[str, Any]]: async def _none(**_k: Any) -> list[Any]: return [] - async def _insert(rows: Any, **_k: Any) -> None: + async def _insert(rows: Any, **_k: Any) -> Any: if rows: order.append("insert") + return rows async def _index_after(rows: Any, **_k: Any) -> int: order.append("index") @@ -251,11 +256,11 @@ async def _index(rows: Any, **_k: Any) -> int: handed.extend(r["memory_key"] for r in rows) return 0 - async def _noop(*_a: Any, **_k: Any) -> Any: - return 0 + async def _insert(rows: Any, **_k: Any) -> Any: + return rows monkeypatch.setattr(bm25_stats, "index_late_rows", _index) - monkeypatch.setattr(memory_repo, "insert_embedding_rows", _noop) + monkeypatch.setattr(memory_repo, "insert_embedding_rows", _insert) await integrity.repair_dim0_rows( [{**_row(memory_key="dim0"), "embedding": [0.1, 0.2], "version": 1}], ch=object() ) diff --git a/tests/unit/test_cell_scan_tables.py b/tests/unit/test_cell_scan_tables.py index c558ec5e27..0305405d41 100644 --- a/tests/unit/test_cell_scan_tables.py +++ b/tests/unit/test_cell_scan_tables.py @@ -21,6 +21,7 @@ "loom.analysis_canary", "loom.audit_log", "loom.blacklist", + "loom.memory_refusals", "loom.blob_uploads", "loom.categories", "loom.chunk_supersessions", diff --git a/tests/unit/test_ch_client_settings.py b/tests/unit/test_ch_client_settings.py index 9a3bd3eaa3..7287cdbc43 100644 --- a/tests/unit/test_ch_client_settings.py +++ b/tests/unit/test_ch_client_settings.py @@ -14,6 +14,28 @@ from server.settings import settings +async def test_query_all_keeps_its_statement_settings_through_the_driver(monkeypatch): + seen = [] + + class Driver: + async def query(self, sql, parameters, settings, **kwargs): + seen.append((sql, parameters, settings)) + return SimpleNamespace(column_names=["v"], result_rows=[(1,)], summary={}) + + async def guarded(*args, **kwargs): + return None + + client = ch_mod.CHClient() + monkeypatch.setattr(type(client), "client", property(lambda self: Driver())) + monkeypatch.setattr(ch_mod, "guard_pending_schema", guarded) + result = await client.query_all( + "SELECT {v:UInt8} AS v", params={"v": 1}, settings={"allow_experimental_ai_functions": 1} + ) + assert result == [{"v": 1}] + assert seen[0][1] == {"v": 1} + assert seen[0][2]["allow_experimental_ai_functions"] == 1 + + @pytest.mark.asyncio async def test_session_settings_enable_filesystem_cache(monkeypatch: pytest.MonkeyPatch) -> None: captured: dict[str, Any] = {} diff --git a/tests/unit/test_chunk_parent_identity.py b/tests/unit/test_chunk_parent_identity.py index 6acff7aa7a..368651ecad 100644 --- a/tests/unit/test_chunk_parent_identity.py +++ b/tests/unit/test_chunk_parent_identity.py @@ -9,7 +9,7 @@ async def _project(monkeypatch, user, session): - ch = SimpleNamespace(insert=AsyncMock()) + ch = SimpleNamespace(insert=AsyncMock(), query_all=AsyncMock(return_value=[])) async def embedded(text, ranges, chunks, embed): return [SimpleNamespace(written=True, vector=[1.0, 0.0]) for _ in chunks] diff --git a/tests/unit/test_clickhouse_mutation_inventory.py b/tests/unit/test_clickhouse_mutation_inventory.py index 7bf18dcb36..d4c18a6a4a 100644 --- a/tests/unit/test_clickhouse_mutation_inventory.py +++ b/tests/unit/test_clickhouse_mutation_inventory.py @@ -29,8 +29,8 @@ # written meanwhile holds (rare: only in that race), and end_triples_now, # which ends now a dropped fact that ends later (only when one does). ("server/memory/app/projections.py", "entity_triples", "UPDATE"): 3, - # In-place retries cannot recreate a transcript erased before a delayed command lands. - ("server/memory/app/extraction_queue.py", "memory_extraction_queue", "UPDATE"): 1, + # In-place retries and refusal termination never copy an erased transcript. + ("server/memory/app/extraction_queue.py", "memory_extraction_queue", "UPDATE"): 2, ("server/memory/store/blacklist.py", "blacklist", "UPDATE"): 1, ("server/memory/store/conflicts.py", "conflict_allowlist", "UPDATE"): 1, ("server/knowledge/store/conversations.py", "conversation_threads", "UPDATE"): 1, diff --git a/tests/unit/test_code_member_writes.py b/tests/unit/test_code_member_writes.py index 507d4ad159..eb006c7d36 100644 --- a/tests/unit/test_code_member_writes.py +++ b/tests/unit/test_code_member_writes.py @@ -175,16 +175,25 @@ def _row(actor: str, **over: Any) -> memory_repo.MemoryRow: class _CH: - def __init__(self) -> None: + def __init__(self, existing=None) -> None: self.inserts: list[tuple[str, list[dict[str, Any]]]] = [] + self.heads = {} + if existing is not None: + row = memory_repo.build_memory_version_row(existing) + self.heads[memory_repo.identity_key(row)] = row async def insert(self, table: str, rows: list[dict[str, Any]], **_: Any) -> None: self.inserts.append((table, rows)) + if table == "loom.memory": + self.heads.update((memory_repo.identity_key(r), dict(r)) for r in rows) async def query_one(self, *_: Any, **__: Any) -> dict[str, Any] | None: return None - async def query_all(self, *_: Any, **__: Any) -> list[dict[str, Any]]: + async def query_all(self, sql, params=None) -> list[dict[str, Any]]: + if "FROM loom.memory FINAL" in sql: + keys = set((params or {}).get("keys", self.heads)) + return [r for key, r in self.heads.items() if key in keys] return [] async def execute(self, *_: Any, **__: Any) -> None: @@ -206,7 +215,7 @@ def _wire(monkeypatch: pytest.MonkeyPatch, existing: memory_repo.MemoryRow | Non from server.memory.app import restore as _restore_mod from server.memory.app import stack as _stack_mod - ch = _CH() + ch = _CH(existing) monkeypatch.setattr(mem_handlers, "get_client", lambda: ch) monkeypatch.setattr(_stack_mod, "get_client", lambda: ch) monkeypatch.setattr(_restore_mod, "get_client", lambda: ch) diff --git a/tests/unit/test_dedup_ops.py b/tests/unit/test_dedup_ops.py index c4cfeb3d5e..f651a35f09 100644 --- a/tests/unit/test_dedup_ops.py +++ b/tests/unit/test_dedup_ops.py @@ -8,6 +8,7 @@ import pytest from fastapi import HTTPException +from server.core.hashing import sha256_value from server.memory.domain.webhook_events import memory_change from server.memory.store.dedup import CanonicalRef, DuplicateCluster from server.memory.store.rows import MemoryRow @@ -557,17 +558,28 @@ async def query_all(self, sql, params=None): if "FROM loom.memory_refusals" in sql: return [] if "FROM loom.memory FINAL" in sql and "PREWHERE (org, namespace" in sql: - return [mr.build_memory_version_row(existing)] - emb = {c: None for c in mr.EMBEDDING_PERSISTED_COLS} - identity = { - "org": "src-org", - "namespace": "src-ns", - "agent": "a", - "user_id": "u", - "session_id": "s", - "memory_key": "k", - } - return [emb | identity | {"status": "duplicate"}] + return next( + (rows for table, rows in reversed(self.inserts) if table == "loom.memory"), + [mr.build_memory_version_row(existing)], + ) + return [ + mr.build_embedding_row( + **{ + c: getattr(existing, c) + for c in ( + "org", + "namespace", + "agent", + "user_id", + "session_id", + "memory_key", + ) + }, + status="duplicate", + version=existing.version, + content_sha256=sha256_value(existing.value), + ) + ] async def insert(self, table, rows, settings=None): self.inserts.append((table, rows)) diff --git a/tests/unit/test_delete_by_key.py b/tests/unit/test_delete_by_key.py index 10be6744d0..ec6851f344 100644 --- a/tests/unit/test_delete_by_key.py +++ b/tests/unit/test_delete_by_key.py @@ -8,6 +8,7 @@ import asyncio from collections.abc import AsyncIterator from contextlib import asynccontextmanager +from datetime import datetime from types import SimpleNamespace from typing import Any @@ -44,11 +45,16 @@ def test_home_namespace_prefers_the_callers_own() -> None: class _CH: def __init__(self) -> None: self.inserts: list[tuple[str, list[dict[str, Any]]]] = [] + self.read_head = lambda: None async def insert(self, table: str, rows: list[dict[str, Any]], settings: Any = None) -> None: self.inserts.append((table, rows)) async def query_all(self, sql: str, params: Any = None) -> list[dict[str, Any]]: + if params and "keys" in params: + head = self.read_head() + if head is not None and memory_repo.identity_key(head) in params["keys"]: + return [memory_repo.build_memory_version_row(head)] return [] async def query_one(self, sql: str, params: Any = None) -> Any: @@ -56,7 +62,7 @@ async def query_one(self, sql: str, params: Any = None) -> Any: def _row(agent: str, session_id: str) -> Any: - return SimpleNamespace( + return memory_repo.MemoryRow( org="clickhouse", namespace="jdoe", agent=agent, @@ -66,12 +72,15 @@ def _row(agent: str, session_id: str) -> Any: memory_id="33333333-3333-3333-3333-333333333333", memory_type="semantic", value='"x"', - run_id="", + run_id="-", source_class="extracted", tags=[], entities=[], version=1, - locked=False, + confidence=1.0, + pinned=False, + expires_at=None, + created_at=datetime(2026, 1, 1), ) @@ -83,6 +92,8 @@ def world(monkeypatch: pytest.MonkeyPatch) -> dict[str, Any]: "ch": _CH(), } + state["ch"].read_head = lambda: state["row"] + async def _get_by_key(identity: Any, key: str, **kw: Any) -> Any: state["lookups"].append((identity.agent, identity.user_id, identity.session_id, key)) if identity.agent == "*": @@ -115,22 +126,6 @@ async def _no_check(*_a: Any, **_k: Any) -> None: monkeypatch.setattr(_restore_mod, "get_client", lambda: state["ch"]) monkeypatch.setattr("server.memory.app.write.get_client", hm.get_client) monkeypatch.setattr("server.memory.app.update.get_client", hm.get_client) - monkeypatch.setattr( - memory_repo, - "build_memory_version_row", - lambda existing, **kw: { - "org": existing.org, - "namespace": existing.namespace, - "memory_key": existing.memory_key, - "agent": existing.agent, - "user_id": existing.user_id, - "session_id": existing.session_id, - **kw, - }, - ) - # The stub builder returns partial rows; the funnel's full-row check is - # pinned in test_rmt_write_is_typed. - monkeypatch.setattr("server.memory.store.inserts.check_memory_rows", lambda rows: None) monkeypatch.setattr(hm._lock, "assert_unlocked", lambda *a, **k: None) return state @@ -153,7 +148,10 @@ async def _run() -> Any: out = asyncio.run(_run()) assert out.tombstoned is True and out.found is True assert world["lookups"][0][0] == "claude-desktop" and world["lookups"][1][0] == "*" - table, rows = world["ch"].inserts[0] + refused, fingerprints = world["ch"].inserts[0] + assert refused == "loom.memory_refusals" and fingerprints + assert all(len(row["digest"]) == 64 for row in fingerprints) + table, rows = next(item for item in world["ch"].inserts if item[0] == "loom.memory") assert table == "loom.memory" assert rows[0]["agent"] == "claude-code" and rows[0]["session_id"] == "sess-65187490" @@ -378,3 +376,62 @@ async def _hold(*key: str): assert held == [ ("clickhouse", "jdoe", "claude-code", "-", "sess-65187490", "user_home_location") ] + + +@pytest.mark.asyncio +@pytest.mark.parametrize("scope_name", ["namespace", "own_session"]) +async def test_wildcard_deletion_keeps_the_requested_user_and_session_ceiling( + world, monkeypatch, scope_name +): + from server.identity.domain.read_scope import ReadFilters + from server.memory.api.models_crud import MemoryDeleteInput + + world["row"] = world["row"].model_copy(update={"user_id": "alice"}) + scopes = [] + original = memory_repo.get_by_key + + async def read(identity, key, **kwargs): + scopes.append(kwargs["scope"]) + return await original(identity, key, **kwargs) + + scope = ReadFilters( + scope_name, + "alice" if scope_name == "own_session" else None, + "sess-65187490" if scope_name == "own_session" else None, + ) + monkeypatch.setattr(hm, "request_scope", lambda: scope) + monkeypatch.setattr(memory_repo, "get_by_key", read) + out = await hm.handle_memory_delete( + MemoryDeleteInput( + org="clickhouse", + namespace="jdoe", + agent="*", + user_id="alice", + memory_key="user_home_location", + ) + ) + assert out.tombstoned + assert scopes == [ + ReadFilters( + "own_session" if scope_name == "own_session" else "own_user", "alice", scope.session_id + ) + ] + + +@pytest.mark.asyncio +async def test_a_pinned_user_cannot_wildcard_delete_a_different_users_row(world, monkeypatch): + from server.identity.domain.read_scope import ReadFilters + from server.memory.api.models_crud import MemoryDeleteInput + + monkeypatch.setattr(hm, "request_scope", lambda: ReadFilters("own_user", "bob", None)) + out = await hm.handle_memory_delete( + MemoryDeleteInput( + org="clickhouse", + namespace="jdoe", + agent="*", + user_id="alice", + memory_key="user_home_location", + ) + ) + assert not out.found and not out.tombstoned + assert world["lookups"] == [] and world["ch"].inserts == [] diff --git a/tests/unit/test_delete_version_consent_races.py b/tests/unit/test_delete_version_consent_races.py new file mode 100644 index 0000000000..8b56c82d5b --- /dev/null +++ b/tests/unit/test_delete_version_consent_races.py @@ -0,0 +1,249 @@ +"""A deletion refuses only the exact fact its tombstone actually removes.""" + +from datetime import datetime +from unittest.mock import AsyncMock + +import pytest + +from server.memory.api.models_crud import MemoryDeleteVersionInput +from server.memory.app import handlers +from server.memory.store import pg_memory, rows +from server.memory.store.delete_checks import checked_state_sha256 + +_AFTER_CHECKED = handlers._after_checked + + +def _head(value="first", version=10): + return rows.MemoryRow( + org="org", + namespace="ns", + agent="agent", + user_id="-", + session_id="-", + memory_key="key", + memory_type="semantic", + value=value, + confidence=1.0, + version=version, + created_at=datetime(2026, 1, 1), + memory_id="00000000-0000-0000-0000-000000000001", + pinned=False, + expires_at=None, + ) + + +def _check(): + return dict( + org="org", + namespace="ns", + agent="agent", + user_id="-", + session_id="-", + memory_key="key", + checked=10, + tombstone=20, + checked_state_sha256=checked_state_sha256(_head()), + ) + + +@pytest.fixture +def store(monkeypatch): + import server.memory.store as store + + monkeypatch.setattr(store, "record_delete_check", AsyncMock()) + monkeypatch.setattr(store, "fetch_version_rows", AsyncMock(return_value=[_head()])) + monkeypatch.setattr(store, "get_by_key", AsyncMock(return_value=_head())) + monkeypatch.setattr(handlers, "_after_checked", AsyncMock(return_value=([], []))) + return store + + +@pytest.mark.asyncio +@pytest.mark.parametrize("same_version", [False, True]) +async def test_a_known_pre_send_race_returns_changed_without_refusing_the_old_value( + store, monkeypatch, same_version +): + current = _head("second", 10 if same_version else 11) + if same_version: + store.fetch_version_rows.return_value = [current] + store.get_by_key.side_effect = [_head(), current] + monkeypatch.setattr( + handlers, + "_write_tombstone_row", + AsyncMock(side_effect=pg_memory.AdmissionContendedError("head changed")), + ) + out = await handlers._delete_version_locked( + MemoryDeleteVersionInput( + **{ + k: v + for k, v in _check().items() + if k not in {"checked", "tombstone", "checked_state_sha256"} + }, + version=10, + ), + rows.Identity(org="org", namespace="ns", agent="agent"), + ) + assert not out.tombstoned and out.reason == "version_changed" + assert handlers._write_tombstone_row.await_count == 1 + assert store.record_delete_check.call_args.kwargs == {"done": True} + + +@pytest.mark.asyncio +async def test_recovery_rechecks_a_second_known_pre_send_race(store, monkeypatch): + store.get_by_key.side_effect = [_head(), _head("second", 11)] + monkeypatch.setattr( + handlers, + "_write_tombstone_row", + AsyncMock(side_effect=pg_memory.AdmissionContendedError("head changed")), + ) + out = await handlers.rerun_delete_check(_check()) + assert out is not None and not out.tombstoned and out.reason == "version_changed" + assert handlers._write_tombstone_row.await_count == 1 + assert store.record_delete_check.call_args.kwargs == {"done": True} + + +@pytest.mark.asyncio +async def test_durable_recovery_cannot_substitute_a_same_version_historical_row(store, monkeypatch): + store.fetch_version_rows.return_value = [_head("second")] + write = AsyncMock() + monkeypatch.setattr(handlers, "_write_tombstone_row", write) + out = await handlers.rerun_delete_check(_check()) + assert out is not None and out.reason == "version_changed" + write.assert_not_awaited() + assert store.record_delete_check.call_args.kwargs == {"done": True} + + +@pytest.mark.asyncio +async def test_committed_tombstone_supplies_the_content_for_owed_retirement(store, monkeypatch): + tombstone = _head().model_copy( + update={ + "version": 20, + "tombstoned_at": datetime(2026, 1, 2), + "source_trace": "memory.delete_version:10", + } + ) + store.fetch_version_rows.return_value = [_head("second"), tombstone] + finish = AsyncMock(return_value=None) + monkeypatch.setattr(handlers, "finish_delete_check", finish) + await handlers.rerun_delete_check(_check()) + removed = finish.call_args.args[0] + assert removed.value == "first" and removed.version == 10 + + +@pytest.mark.asyncio +async def test_a_retrys_tombstone_supplies_content_when_checked_history_collides( + store, monkeypatch +): + tombstone = _head().model_copy( + update={ + "version": 30, + "tombstoned_at": datetime(2026, 1, 2), + "source_trace": "memory.delete_version:10", + } + ) + store.fetch_version_rows.return_value = [_head("second")] + handlers._after_checked.return_value = ([30], []) + monkeypatch.setattr(handlers, "_version_row", AsyncMock(return_value=tombstone)) + finish = AsyncMock(return_value=None) + monkeypatch.setattr(handlers, "finish_delete_check", finish) + await handlers.rerun_delete_check(_check()) + removed = finish.call_args.args[0] + assert removed.value == "first" and removed.version == 10 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("other_versions", [([30], []), ([], [11])]) +async def test_an_unknown_checked_state_can_reconcile_without_recreating_a_tombstone( + store, monkeypatch, other_versions +): + handlers._after_checked.return_value = other_versions + monkeypatch.setattr( + handlers, + "_version_row", + AsyncMock( + return_value=_head().model_copy( + update={ + "version": 30, + "tombstoned_at": datetime(2026, 1, 2), + "source_trace": "memory.delete_version:10", + } + ) + ), + ) + finish = AsyncMock(return_value=None) + write = AsyncMock() + monkeypatch.setattr(handlers, "finish_delete_check", finish) + monkeypatch.setattr(handlers, "_write_tombstone_row", write) + check = _check() + check.pop("checked_state_sha256") + await handlers.rerun_delete_check(check) + finish.assert_awaited_once() + write.assert_not_awaited() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("uncertain", [False, True]) +async def test_recovery_leaves_a_check_owed_after_bounded_contention_or_an_uncertain_send( + store, monkeypatch, uncertain +): + error = RuntimeError("uncertain send") if uncertain else pg_memory.AdmissionContendedError() + write = AsyncMock(side_effect=error) + monkeypatch.setattr(handlers, "_write_tombstone_row", write) + with pytest.raises(type(error)): + await handlers.rerun_delete_check(_check()) + assert write.await_count == (1 if uncertain else 4) + store.record_delete_check.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_recovery_without_a_checked_state_cannot_refuse_a_historical_value( + store, monkeypatch +): + write = AsyncMock() + monkeypatch.setattr(handlers, "_write_tombstone_row", write) + check = _check() + check.pop("checked_state_sha256") + out = await handlers.rerun_delete_check(check) + assert out is not None and out.reason == "version_changed" + assert store.record_delete_check.call_args.kwargs == {"done": True} + write.assert_not_awaited() + + +@pytest.mark.asyncio +@pytest.mark.parametrize("deleted", [False, True]) +async def test_a_foreign_row_at_the_intended_tombstone_version_is_preserved( + store, monkeypatch, deleted +): + foreign = _head("second", 20) + if deleted: + foreign = foreign.model_copy( + update={ + "tombstoned_at": datetime(2026, 1, 2), + "source_trace": "memory.delete_version:9", + } + ) + history = {10: _head(), 20: foreign} + + async def fetch(org, namespace, pairs, **kwargs): + return [history[version] for key, version in pairs if version in history] + + store.fetch_version_rows.side_effect = fetch + monkeypatch.setattr(handlers, "_after_checked", _AFTER_CHECKED) + monkeypatch.setattr( + store, + "versions_above", + AsyncMock( + return_value={ + ("agent", "-", "-", "key"): [(20, deleted, 9 if deleted else None)], + } + ), + ) + lift, retire = AsyncMock(), AsyncMock() + monkeypatch.setattr(handlers, "_write_above", lift) + monkeypatch.setattr(handlers, "_retire_derived", retire) + out = await handlers.rerun_delete_check(_check()) + assert out is not None and not out.tombstoned + assert out.found is (not deleted) + assert out.reason == ("" if deleted else "version_changed") + lift.assert_awaited_once() + assert lift.call_args.args[0].value == "second" + retire.assert_not_awaited() diff --git a/tests/unit/test_delete_version_sql.py b/tests/unit/test_delete_version_sql.py index c4e741579a..1b69924959 100644 --- a/tests/unit/test_delete_version_sql.py +++ b/tests/unit/test_delete_version_sql.py @@ -18,6 +18,7 @@ import pytest import server.memory.app.handlers as hm +from server.core.hashing import sha256_value from server.identity.domain.read_scope import WHOLE_NAMESPACE from server.memory import store as memory_repo from server.memory.api.models_crud import ( @@ -27,6 +28,7 @@ ) from server.memory.app import set_commit from server.memory.store import key_write_lock +from server.memory.store.refusals import check_clickhouse as check_consent from server.memory.store.rows import build_first_version_row from tests.unit.clickhouse_local import CLICKHOUSE, KeptCH, ddl, view @@ -54,7 +56,7 @@ def _row(value: str, version: int) -> dict[str, Any]: "memory_type": "procedural", # not embedded: memory.set's plain insert "value": json.dumps(value), "value_json": "{}", - "content_sha256": "", + "content_sha256": sha256_value(value), "actor": "process:okf-sync", "source_class": "imported", "confidence": 1.0, @@ -67,7 +69,7 @@ def _row(value: str, version: int) -> dict[str, Any]: async def _recorded_checks(ch: KeptCH) -> list[dict[str, Any]]: """Every check delete_version recorded, done or not, oldest tombstone first.""" return await ch.query_all( - "SELECT checked, tombstone, done FROM loom.delete_version_checks FINAL ORDER BY tombstone" + "SELECT checked, tombstone, checked_state_sha256, done FROM loom.delete_version_checks FINAL ORDER BY tombstone" ) @@ -77,6 +79,7 @@ async def _times_out(*_a: Any, **_k: Any) -> Any: @pytest.fixture def ch(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> KeptCH: + monkeypatch.setattr("server.memory.store.refusals.check_clickhouse", check_consent) ch = KeptCH( tmp_path, ddl("memory"), @@ -111,6 +114,7 @@ def _in_flight_set(version: int) -> set_commit._SetWrite: expires_at=None, ) w.memory_row = _row("Thursdays", version) + w.new_sha = sha256_value("Thursdays") w.memory_id = _MID return w @@ -363,6 +367,7 @@ async def _noop(*_a: Any, **_k: Any) -> int: memory_key=_KEY, memory_type="semantic", content_excerpt=value, + content_sha256=sha256_value(value), version=v, embedding=[0.0] * 3072, embedding_dim=3072, @@ -740,7 +745,13 @@ async def check_then_land(identity: Any, key: str, **kw: Any) -> Any: # Run again (its done mark lost), the check answers the same and copies nothing twice. rows_before = await ch.query_all("SELECT version FROM loom.memory_versions ORDER BY version") (done,) = await _recorded_checks(ch) - check = {**_ID, "memory_key": _KEY, "checked": done["checked"], "tombstone": done["tombstone"]} + check = { + **_ID, + "memory_key": _KEY, + "checked": done["checked"], + "tombstone": done["tombstone"], + "checked_state_sha256": done["checked_state_sha256"], + } again = await hm.rerun_delete_check(check) assert again is not None and (again.tombstoned, again.found) == (False, False) assert ( @@ -754,9 +765,18 @@ async def check_then_land(identity: Any, key: str, **kw: Any) -> Any: monkeypatch.setattr(restore_mod, "get_client", lambda: ch) monkeypatch.setattr(memory_repo, "get_by_key", get_by_key) - restored = await restore_mod.handle_memory_restore(MemoryRestoreInput(**_ID, memory_key=_KEY)) - assert restored.restored is True - got = await _read_back(ch) + from server.core.errors import ValidationError + + with pytest.raises(ValidationError, match="refused by a previous deletion"): + await restore_mod.handle_memory_restore(MemoryRestoreInput(**_ID, memory_key=_KEY)) + got = await memory_repo.get_by_key( + memory_repo.Identity(**_ID), + _KEY, + strong=True, + scope=WHOLE_NAMESPACE, + include_tombstoned=True, + ch=ch, + ) assert got is not None and got.value == "Thursdays" @@ -795,19 +815,27 @@ async def check_then_land(identity: Any, key: str, **kw: Any) -> Any: assert out.tombstoned is False and out.found is False monkeypatch.setattr(restore_mod, "get_client", lambda: ch) monkeypatch.setattr(memory_repo, "get_by_key", get_by_key) - restored = await restore_mod.handle_memory_restore(MemoryRestoreInput(**_ID, memory_key=_KEY)) - assert restored.restored is True - got = await _read_back(ch) + from server.core.errors import ValidationError + + with pytest.raises(ValidationError, match="refused by a previous deletion"): + await restore_mod.handle_memory_restore(MemoryRestoreInput(**_ID, memory_key=_KEY)) + got = await memory_repo.get_by_key( + memory_repo.Identity(**_ID), + _KEY, + strong=True, + scope=WHOLE_NAMESPACE, + include_tombstoned=True, + ch=ch, + ) assert got is not None and got.value == "Tuesdays" assert (list(got.tags) if column == "tags" else got.expire_reason) == changed @pytest.mark.asyncio -async def test_a_restored_key_does_not_carry_the_delete_version_marker( +async def test_a_deleted_version_refuses_restore_and_retains_its_marker( ch: KeptCH, monkeypatch: pytest.MonkeyPatch ) -> None: - # The marker stays on delete_version's tombstone; the row memory.restore brings back, - # and every version after it, drops it. + # Refused restoration preserves the deletion marker. from server.memory.api.models_crud import MemoryRestoreInput from server.memory.app import restore as restore_mod @@ -817,10 +845,19 @@ async def test_a_restored_key_does_not_carry_the_delete_version_marker( await hm.handle_memory_delete_version( MemoryDeleteVersionInput(**_ID, memory_key=_KEY, version=head) ) - out = await restore_mod.handle_memory_restore(MemoryRestoreInput(**_ID, memory_key=_KEY)) - assert out.restored is True - got = await _read_back(ch) - assert got is not None and not got.source_trace.startswith("memory.delete_version:") + from server.core.errors import ValidationError + + with pytest.raises(ValidationError, match="refused by a previous deletion"): + await restore_mod.handle_memory_restore(MemoryRestoreInput(**_ID, memory_key=_KEY)) + got = await memory_repo.get_by_key( + memory_repo.Identity(**_ID), + _KEY, + strong=True, + scope=WHOLE_NAMESPACE, + include_tombstoned=True, + ch=ch, + ) + assert got is not None and got.source_trace == f"memory.delete_version:{head}" @pytest.mark.asyncio @@ -835,11 +872,11 @@ async def test_a_later_write_listed_but_not_read_leaves_the_check_owed( head = time.time_ns() - 10_000_000_000 await ch.insert("loom.memory", [_row("Tuesdays", head)]) w = _in_flight_set(time.time_ns() - 1_000_000) - write_row = hm._write_tombstone_row + key_moved_on = hm._key_moved_on - async def set_lands_first(existing: Any, **kw: Any) -> None: - await set_commit._set_plain_insert(w, ch) - await write_row(existing, **kw) + async def set_lands_first(existing: Any, **kw: Any) -> Any: + await ch.insert("loom.memory", [w.memory_row]) + return await key_moved_on(existing, **kw) async def nothing(*_a: Any, **_k: Any) -> list[Any]: return [] @@ -847,7 +884,7 @@ async def nothing(*_a: Any, **_k: Any) -> list[Any]: async def no_retire(*_a: Any, **_k: Any) -> None: raise AssertionError("nothing retires before the check runs") - monkeypatch.setattr(hm, "_write_tombstone_row", set_lands_first) + monkeypatch.setattr(hm, "_key_moved_on", set_lands_first) monkeypatch.setattr(hm, "_retire_derived", no_retire) monkeypatch.setattr(memory_repo, "fetch_version_rows", nothing) with pytest.raises(BackendUnavailableError): @@ -887,6 +924,7 @@ async def _noop(*_a: Any, **_k: Any) -> int: memory_key=_KEY, memory_type="semantic", content_excerpt=value, + content_sha256=sha256_value(value), version=v, embedding=[0.0] * 3072, embedding_dim=3072, @@ -1165,12 +1203,12 @@ async def test_a_check_that_fails_after_the_tombstone_is_finished_by_the_worker( head = time.time_ns() - 10_000_000_000 await ch.insert("loom.memory", [_row("Tuesdays", head)]) w = _in_flight_set(time.time_ns() - 1_000_000) - write_row = hm._write_tombstone_row + key_moved_on = hm._key_moved_on retired: list[Any] = [] async def set_lands_first(existing: Any, **kw: Any) -> None: - await set_commit._set_plain_insert(w, ch) - await write_row(existing, **kw) + await ch.insert("loom.memory", [w.memory_row]) + raise RuntimeError("timeout") retire = hm._retire_derived @@ -1178,10 +1216,8 @@ async def count_retire(*a: Any, **k: Any) -> Any: retired.append(True) return await retire(*a, **k) - monkeypatch.setattr(hm, "_write_tombstone_row", set_lands_first) + monkeypatch.setattr(hm, "_key_moved_on", set_lands_first) monkeypatch.setattr(hm, "_retire_derived", count_retire) - key_moved_on = hm._key_moved_on - monkeypatch.setattr(hm, "_key_moved_on", _times_out) with pytest.raises(BackendUnavailableError): await hm.handle_memory_delete_version( MemoryDeleteVersionInput(**_ID, memory_key=_KEY, version=head) @@ -1333,12 +1369,11 @@ async def _live_chunk_contents(ch: KeptCH) -> list[str]: @pytest.mark.asyncio -async def test_a_short_versions_passage_shared_with_the_head_stays_live( +async def test_a_deleted_passage_is_not_restored_through_a_newer_heads_chunks( ch: KeptCH, monkeypatch: pytest.MonkeyPatch ) -> None: - # The removed version is too short to chunk, but its one passage is chunk 0 of the head - # that lands after the delete's check: the delete retires that row with the version's - # passages and puts it back for the head. + # A delayed row shares the deleted passage as its first chunk. Refusal keeps + # that passage retired even when the newer row still carries it. from server.memory.app import set_chunks from server.memory.app.chunk_projection import chunk_text @@ -1363,7 +1398,8 @@ async def head_lands_after_the_check(existing: Any, **kw: Any) -> Any: MemoryDeleteVersionInput(**_ID, memory_key=_KEY, version=old) ) assert out.tombstoned is True - assert await _live_chunk_contents(ch) == sorted(r["content"] for r in rows) + assert await _live_chunk_contents(ch) == sorted(r["content"] for r in rows[1:]) + assert rows[0]["content"] not in await _live_chunk_contents(ch) _PAST = datetime(2020, 1, 1) @@ -1581,15 +1617,15 @@ async def _owed_after_a_failed_check( head = time.time_ns() - 10_000_000_000 await ch.insert("loom.memory", [_row("Tuesdays", head)]) w = _in_flight_set(time.time_ns() - 1_000_000) - write_row = hm._write_tombstone_row - async def set_lands_first(existing: Any, **kw: Any) -> None: - await set_commit._set_plain_insert(w, ch) - await write_row(existing, **kw) + async def late_set_then_timeout(existing: Any, **kw: Any) -> None: + # A previously admitted insert lands after the tombstone; its head check + # times out before it can lift the live write above the delete. + await ch.insert("loom.memory", [w.memory_row]) + raise RuntimeError("timeout") with monkeypatch.context() as m: - m.setattr(hm, "_write_tombstone_row", set_lands_first) - m.setattr(hm, "_key_moved_on", _times_out) + m.setattr(hm, "_key_moved_on", late_set_then_timeout) m.setattr(hm, "next_version", lambda: time.time_ns() + clock_ahead_ns) with pytest.raises(BackendUnavailableError): await hm.handle_memory_delete_version( @@ -1664,25 +1700,31 @@ async def _tombstone_held_back(ch: KeptCH, monkeypatch: pytest.MonkeyPatch) -> t queued: the version it named, and the call that commits the insert late.""" head = time.time_ns() - 10_000_000_000 await ch.insert("loom.memory", [_row("Tuesdays", head)]) - write_row = hm._write_tombstone_row - queued: list[tuple[Any, dict[str, Any]]] = [] + insert = ch.insert + queued: list[tuple[str, list[dict[str, Any]], Any]] = [] - async def times_out_queued(existing: Any, **kw: Any) -> None: - queued.append((existing, kw)) - raise RuntimeError("timeout") + async def times_out_queued(table: str, rows: list[dict[str, Any]], settings=None) -> None: + if table == "loom.memory" and rows[0]["source_trace"].startswith("memory.delete_version:"): + queued.append((table, rows, settings)) + raise RuntimeError("timeout") + await insert(table, rows, settings=settings) async def commit_late() -> None: - ((existing, kw),) = queued - await write_row(existing, **kw) + ((table, rows, settings),) = queued + await insert(table, rows, settings=settings) from server.core.errors import BackendUnavailableError with monkeypatch.context() as m: - m.setattr(hm, "_write_tombstone_row", times_out_queued) + m.setattr(ch, "insert", times_out_queued) with pytest.raises(BackendUnavailableError): await hm.handle_memory_delete_version( MemoryDeleteVersionInput(**_ID, memory_key=_KEY, version=head) ) + from server.memory.store import write_fence + + keeper = write_fence._lease_client(ch) + keeper.advance(max((row["ttl"] for row in keeper.keys.values()), default=0) + 1) return head, commit_late @@ -1765,10 +1807,7 @@ async def record(event_type: str, **kw: Any) -> str: finally: request_id_var.reset(rid) setup_receipts = [row for event, row in emitted if event == "memory.set"] - assert len(setup_receipts) == int(failure in {"the check after it", "a retry"}) - if setup_receipts: - assert setup_receipts[0]["subject"]["memory_key"] == _KEY - emitted[:] = [(event, row) for event, row in emitted if event != "memory.set"] + assert setup_receipts == [] calls = 2 if failure == "a retry" else 1 assert len(emitted) == calls expected = MemoryDeleteVersionInput(**_ID, memory_key=_KEY, version=head).model_dump( @@ -1909,9 +1948,18 @@ async def check_then_land(identity: Any, key: str, **kw: Any) -> Any: assert out.tombstoned is False and out.found is False monkeypatch.setattr(memory_repo, "get_by_key", get_by_key) monkeypatch.setattr(restore_mod, "get_client", lambda: ch) - restored = await restore_mod.handle_memory_restore(MemoryRestoreInput(**_ID, memory_key=_KEY)) - assert restored.restored is True - got = await _read_back(ch) + from server.core.errors import ValidationError + + with pytest.raises(ValidationError, match="refused by a previous deletion"): + await restore_mod.handle_memory_restore(MemoryRestoreInput(**_ID, memory_key=_KEY)) + got = await memory_repo.get_by_key( + memory_repo.Identity(**_ID), + _KEY, + strong=True, + scope=WHOLE_NAMESPACE, + include_tombstoned=True, + ch=ch, + ) assert got is not None and got.value == "Thursdays" @@ -1923,7 +1971,13 @@ async def test_a_recovered_check_run_again_stands_on_the_tombstone_it_wrote( # stands on that tombstone, answers the same and writes nothing. head, _commit_late = await _tombstone_held_back(ch, monkeypatch) (owed,) = await _recorded_checks(ch) - check = {**_ID, "memory_key": _KEY, "checked": head, "tombstone": owed["tombstone"]} + check = { + **_ID, + "memory_key": _KEY, + "checked": head, + "tombstone": owed["tombstone"], + "checked_state_sha256": owed["checked_state_sha256"], + } first = await hm.rerun_delete_check(check) assert first is not None and first.tombstoned is True assert first.version is not None and first.version > int(owed["tombstone"]) diff --git a/tests/unit/test_digest_chain.py b/tests/unit/test_digest_chain.py index d7bbf4de07..dfe8dd01d9 100644 --- a/tests/unit/test_digest_chain.py +++ b/tests/unit/test_digest_chain.py @@ -23,6 +23,8 @@ def __init__(self, answers: list[list[dict[str, Any]]] | None = None, fail: bool self.fail = fail async def query_all(self, sql, params=None): + if "FROM loom.memory_refusals" in sql: + return [] if self.fail: raise RuntimeError("boom") self.queries.append((sql, dict(params or {}))) @@ -149,26 +151,33 @@ async def insert(self, table, rows, settings=None): self.inserts.append((table, list(rows))) -def _draft(): - from server.memory.app.session_summary import SessionSummaryDraft +def _wire_writer(monkeypatch, ch): + from types import SimpleNamespace + from unittest.mock import AsyncMock - return SessionSummaryDraft( - summary_text="on 2024-03-12, the user moved.", - embedding=[0.1] * 4, - embedding_model="fake-4", + from server.memory import store + from server.memory.app import session_summary + from tests.unit.test_memory_verify import _row + + source = _row(namespace="n", agent="a", user_id="u", session_id="s2", value="the user moved") + monkeypatch.setattr(store, "get_by_id", AsyncMock(return_value=source)) + monkeypatch.setattr( + "server.memory.store.summary_sources.memory_rows_by_identity_keys", + AsyncMock(return_value={store.identity_key(source): source}), ) + monkeypatch.setattr(session_summary, "get_client", lambda: ch) + monkeypatch.setattr(session_summary, "get_embedder", lambda: SimpleNamespace(model_name="fake")) + monkeypatch.setattr(session_summary, "document_vector", AsyncMock(return_value=[0.1] * 4)) + return _FakeState(written=[{"memory_id": source.memory_id, "version": source.version}]) @pytest.mark.asyncio async def test_digest_insert_then_chain_edges(monkeypatch) -> None: - from server.memory.app import bulk_set as _bulk_set_mod - from server.memory.app import session_summary as _session_summary_mod - from server.memory.app.session_summary import _insert_session_summary + from server.memory.app.session_summary import _write_session_summary ch = _WriterCH(answers=[[{"session_id": "s1"}], []]) - monkeypatch.setattr(_bulk_set_mod, "get_client", lambda: ch) - monkeypatch.setattr(_session_summary_mod, "get_client", lambda: ch) - await _insert_session_summary(_FakeInput(), _FakeState(), _draft()) + state = _wire_writer(monkeypatch, ch) + await _write_session_summary(_FakeInput(), state) tables = [t for t, _ in ch.inserts] assert tables == ["loom.session_summaries", "loom.operator_edges"] edge = ch.inserts[1][1][0] @@ -184,19 +193,21 @@ async def test_digest_insert_then_chain_edges(monkeypatch) -> None: @pytest.mark.asyncio async def test_chain_failure_leaves_the_digest(monkeypatch) -> None: - from server.memory.app import bulk_set as _bulk_set_mod - from server.memory.app import session_summary as _session_summary_mod - from server.memory.app.session_summary import _insert_session_summary + from server.memory.app.session_summary import _write_session_summary + + class _ChainUnavailable: + async def query_all(self, sql, params=None): + if "FROM loom.memory_refusals" in sql: + return [] + raise RuntimeError("chain unavailable") - class _NoQuery: def __init__(self): self.inserts = [] async def insert(self, table, rows, settings=None): self.inserts.append((table, list(rows))) - ch = _NoQuery() # no query_all at all: the chain raises, the digest stays - monkeypatch.setattr(_bulk_set_mod, "get_client", lambda: ch) - monkeypatch.setattr(_session_summary_mod, "get_client", lambda: ch) - await _insert_session_summary(_FakeInput(), _FakeState(), _draft()) + ch = _ChainUnavailable() + state = _wire_writer(monkeypatch, ch) + await _write_session_summary(_FakeInput(), state) assert [t for t, _ in ch.inserts] == ["loom.session_summaries"] diff --git a/tests/unit/test_embedding_reemit.py b/tests/unit/test_embedding_reemit.py index 83e7a2b26b..f525c43bb7 100644 --- a/tests/unit/test_embedding_reemit.py +++ b/tests/unit/test_embedding_reemit.py @@ -7,6 +7,7 @@ import asyncio import re import time +from contextlib import asynccontextmanager from datetime import UTC, datetime, timedelta from pathlib import Path from types import SimpleNamespace @@ -14,6 +15,7 @@ import pytest +from server.core.hashing import sha256_value from server.memory import store as M @@ -33,15 +35,60 @@ async def _none(*_a: Any, **_k: Any) -> dict[Any, int]: class _CH: def __init__(self, rows: list[dict[str, Any]]) -> None: self.rows = rows + self.heads = {} + self.execute_calls = [] + for row in rows: + if row.get("scope") != "shared": + source = _memory_row( + **{ + k: row[k] + for k in ( + "org", + "namespace", + "agent", + "user_id", + "session_id", + "memory_key", + ) + } + ) + self.heads[M.identity_key(source)] = M.build_memory_version_row(source) self.query_all_calls: list[tuple[str, dict[str, Any]]] = [] self.insert_calls: list[tuple[str, list[dict[str, Any]], dict[str, Any]]] = [] async def query_all(self, sql, params=None): self.query_all_calls.append((sql, params or {})) + if "memory_refusals" in sql: + return [] + if "FROM loom.memory FINAL" in sql: + return [r for k, r in self.heads.items() if k in (params or {}).get("keys", self.heads)] return list(self.rows) async def insert(self, table, rows, settings=None): self.insert_calls.append((table, rows, settings or {})) + if table == "loom.memory": + self.heads.update((M.identity_key(r), dict(r)) for r in rows) + + async def execute(self, sql, params=None, *, settings=None): + self.execute_calls.append((sql, params, settings)) + assert sql.startswith("DELETE FROM loom.memory_embeddings") + columns = ( + "org", + "namespace", + "agent", + "user_id", + "session_id", + "scope", + "embedding_model", + "memory_key", + ) + self.rows = [ + r + for r in self.rows + if not ( + all(r.get(c) == params[c] for c in columns) and r["version"] < params["version"] + ) + ] _LIVE = { @@ -59,6 +106,7 @@ async def insert(self, table, rows, settings=None): "memory_key_embedding": [0.4, 0.5, 0.6], "memory_key_embedding_dim": 3, "content_excerpt": "text", + "content_sha256": sha256_value("text"), "tags": ["t"], "confidence": 0.9, "expires_at": datetime(2026, 1, 1), @@ -130,8 +178,11 @@ async def test_shared_reemit_reads_the_scope_shared_rows_of_the_key() -> None: sql, params = ch.query_all_calls[0] assert "scope = 'shared'" in sql and "FINAL" in sql and "tombstoned_at IS NULL" in sql assert params == {"org": "o", "ns": "n", "k": "k"} - ((_, rows, _),) = ch.insert_calls - assert rows[0]["scope"] == "shared" and rows[0]["tombstoned_at"] == ts + assert not ch.insert_calls and not ch.rows + ((delete_sql, delete_params, settings),) = ch.execute_calls + assert delete_sql.startswith("DELETE FROM loom.memory_embeddings") + assert delete_params["scope"] == "shared" and delete_params["version"] > _LIVE["version"] + assert settings == {"lightweight_deletes_sync": 2} def _of_keys(rows: list[dict[str, Any]], params: dict[str, Any] | None) -> list[dict[str, Any]]: @@ -156,6 +207,8 @@ def __init__(self, rows: list[dict[str, Any]], entity_rows: list[dict[str, Any]] self.entity_rows = entity_rows async def query_all(self, sql, params=None): + if "memory_refusals" in sql or "FROM loom.memory FINAL" in sql: + return await super().query_all(sql, params) self.query_all_calls.append((sql, params or {})) if "loom.entity_vectors" in sql: return list(self.entity_rows) @@ -268,6 +321,48 @@ class _RMTCH: def __init__(self) -> None: self.emb: list[dict[str, Any]] = [] self.ent: list[dict[str, Any]] = [] + self.heads = {M.identity_key(_memory_row()): M.build_memory_version_row(_memory_row())} + self.execute_calls = [] + + def remember(self, rows): + for row in rows: + key = M.identity_key(row) + base = self.heads.get(key) or M.build_memory_version_row( + _memory_row( + **dict( + zip( + ("org", "namespace", "agent", "user_id", "session_id", "memory_key"), + key, + strict=True, + ) + ) + ) + ) + self.heads[key] = { + **base, + **{k: v for k, v in row.items() if k in M.MEMORY_PERSISTED_COLS}, + } + + async def execute(self, sql, params=None, *, settings=None): + self.execute_calls.append((sql, params, settings)) + assert sql.startswith("DELETE FROM loom.memory_embeddings") + columns = ( + "org", + "namespace", + "agent", + "user_id", + "session_id", + "scope", + "embedding_model", + "memory_key", + ) + self.emb = [ + r + for r in self.emb + if not ( + all(r.get(c) == params[c] for c in columns) and r["version"] < params["version"] + ) + ] @staticmethod def _final(rows: list[dict[str, Any]], key: Any, ver: Any) -> list[dict[str, Any]]: @@ -290,12 +385,50 @@ def live_ent(self) -> list[dict[str, Any]]: ) async def query_all(self, sql, params=None): + if "memory_refusals" in sql: + return [] + if "FROM loom.memory FINAL" in sql: + return [r for k, r in self.heads.items() if k in (params or {}).get("keys", self.heads)] if "entity_vectors" in sql: return [dict(r) for r in self.live_ent()] return [dict(r) for r in _of_keys(self.live_emb(), params)] async def insert(self, table, rows, settings=None): - (self.ent if table == "loom.entity_vectors" else self.emb).extend(dict(r) for r in rows) + if table == "loom.memory": + self.remember(rows) + elif table == "loom.entity_vectors": + self.ent.extend(dict(r) for r in rows) + elif table == "loom.memory_embeddings": + self.emb.extend(dict(r) for r in rows) + + +def _native_admission(monkeypatch, ch): + from server.core import pg + from server.memory.store import pg_memory + + async def execute(*args, **kwargs): + return None + + async def fetch(*args, **kwargs): + return [] + + async def fetchval(*args, **kwargs): + return None + + @asynccontextmanager + async def transaction(): + yield SimpleNamespace(execute=execute, fetch=fetch, fetchval=fetchval) + + async def heads(keys, **kwargs): + return {k: ch.heads[k] for k in keys if k in ch.heads} + + async def on_connection(conn, keys, **kwargs): + assert kwargs["lock"] + return await heads(keys) + + monkeypatch.setattr(pg, "transaction", transaction) + monkeypatch.setattr(pg_memory, "heads_by_keys", heads) + monkeypatch.setattr(pg_memory, "heads_on_connection", on_connection) class _Head(dict[str, Any]): @@ -320,21 +453,32 @@ async def test_a_reject_copies_the_heads_own_rows_and_retires_the_rest_bare(monk from server.core.ch import sequential_reads_active from server.memory.store import lifecycle, pg_memory, pg_publish - sha = "a" * 64 + sha = sha256_value("text") ch = _RMTCH() + _native_admission(monkeypatch, ch) ent = {"org": "o", "namespace": "n", "agent": "a", "user_id": "u", "memory_key": "k"} ch.ent.append({**ent, "entity_text": "e", "created_at": datetime(2026, 1, 1)}) ch.emb += [ {**_LIVE, "content_sha256": sha}, - {**_LIVE, "embedding_model": "m2", "content_sha256": "b" * 64, "content_excerpt": "old"}, - {**_LIVE, "session_id": "s2", "content_sha256": "c" * 64, "content_excerpt": "secret"}, + { + **_LIVE, + "embedding_model": "m2", + "content_sha256": sha256_value("old"), + "content_excerpt": "old", + }, + { + **_LIVE, + "session_id": "s2", + "content_sha256": sha256_value("secret"), + "content_excerpt": "secret", + }, ] heads = [ {**ent, "session_id": "s", "content_sha256": sha}, { **ent, "session_id": "s2", - "content_sha256": "c" * 64, + "content_sha256": sha256_value("secret"), "redacted_at": datetime(2026, 2, 1), }, ] @@ -343,6 +487,7 @@ async def _heads(*_: object) -> list[dict[str, Any]]: return heads async def _commit(rows: Any, **_: object) -> pg_memory.Committed: + ch.remember(rows) deadline = time.monotonic() + 30 targets = [0] * len(rows) return pg_memory.Committed( @@ -394,7 +539,8 @@ async def test_a_reject_whose_vector_copy_fails_still_stands_and_publishes( from server.memory.store import lifecycle, pg_memory, pg_publish ch = _RMTCH() - ch.emb.append({**_LIVE, "content_sha256": "a" * 64}) + _native_admission(monkeypatch, ch) + ch.emb.append({**_LIVE, "content_sha256": sha256_value("text")}) published: list[Any] = [] deadline = 0.0 if failure == "lease ended" else time.monotonic() + 30 @@ -405,9 +551,10 @@ async def _refused(*_: object, **__: object) -> None: monkeypatch.setattr(ch, "insert", _refused) async def _heads(*_: object) -> list[dict[str, Any]]: - return [{**_LIVE, "content_sha256": "a" * 64}] + return [{**_LIVE, "content_sha256": sha256_value("text")}] async def _commit(rows: Any, **_: object) -> pg_memory.Committed: + ch.remember(rows) return pg_memory.Committed(rows=rows, audits=[], seqs=[], targets=[0], deadline=deadline) async def _publish(committed: Any, **_: object) -> None: @@ -451,7 +598,7 @@ async def test_a_rejected_head_with_no_search_document_copies_no_row_of_its_text not landed: the reject copies none of them above it, and retires them with no text.""" from server.memory.store import lifecycle - sha = "a" * 64 + sha = sha256_value("text") ch = _RMTCH() ch.emb.append({**_LIVE, "content_sha256": sha}) rejected = {**_LIVE, "memory_type": "semantic", "content_sha256": sha, "version": 50} @@ -635,6 +782,7 @@ async def test_a_status_retire_writes_the_memory_rows_version_and_keeps_newer_ro {**_LIVE, "status": "superseded", "version": 30}, {**newer, "status": "superseded", "version": 30}, ] + ch.remember(rows) await M.mirror_status_to_embeddings(rows, ch=ch) by_agent = {r["agent"]: r for r in ch.live_emb()} assert (by_agent["a"]["status"], by_agent["a"]["version"]) == ("superseded", 30) @@ -674,13 +822,10 @@ class _SiteCH(_RMTCH): async def query_all(self, sql, params=None): if "embeddings_status" in sql: return [{**_LIVE, "status": "superseded", "invalid_at": None}] - if "entity_vectors" in sql or "memory_embeddings" in sql: - return await super().query_all(sql, params) - return [] + return await super().query_all(sql, params) async def insert(self, table, rows, settings=None): - if table in ("loom.memory_embeddings", "loom.entity_vectors"): - await super().insert(table, rows, settings) + await super().insert(table, rows, settings) def _memory_row(**over: Any) -> Any: @@ -689,7 +834,8 @@ def _memory_row(**over: Any) -> Any: **{c: _LIVE[c] for c in ("org", "namespace", "agent", "user_id", "session_id")}, "memory_key": "k", "memory_type": "episodic", # not searchable: no chunk side paths - "value": "v", + "value": "text", + "content_sha256": sha256_value("text"), "confidence": 0.9, "version": 1, "tags": [], @@ -709,7 +855,10 @@ def _patch_memory_io(mp: pytest.MonkeyPatch, ch: Any, row: Any) -> None: async def get_by_id(*_a, **_k): return row - async def no_insert(*_a, **_k): + ch.remember([M.build_memory_version_row(row)]) + + async def no_insert(rows, *_a, **_k): + ch.remember(rows) if on_inserted := _k.get("on_inserted"): on_inserted() return None @@ -892,6 +1041,7 @@ async def _site_dedup_collapse(ch, mp): from server.memory.api.models_graph import MemoryDedupCollapseInput from server.memory.app import dedup + ch.remember([M.build_memory_version_row(_memory_row(), status="duplicate")]) pair = SimpleNamespace(agent="a", user_id="u", session_id="s", memory_key="k", duplicate_of="c") async def pairs(*_a, **_k): @@ -982,8 +1132,9 @@ async def test_a_copy_under_a_high_or_deleted_head_is_retired_above_itself( async def _heads(ch: Any, keys: Any, sequential: bool) -> dict[Any, str]: return {k: ("high" if k[5] == "hi" else "") for k in keys if k[5] != "gone"} - async def _insert(rows: Any, **_k: Any) -> None: + async def _insert(rows: Any, **_k: Any) -> Any: inserted.extend(rows) + return rows monkeypatch.setattr(M.embeddings, "_head_sensitivity", _heads) monkeypatch.setattr(M.embeddings, "insert_embedding_rows", _insert) @@ -1035,8 +1186,8 @@ async def _check(rows: Any, **_k: Any) -> int: checked.append(list(rows)) return 0 - async def _insert(rows: Any, **_k: Any) -> None: - return None + async def _insert(rows: Any, **_k: Any) -> Any: + return rows monkeypatch.setattr(embeddings, "retire_copies_under_unembedded_heads", _check) monkeypatch.setattr(embeddings, "insert_embedding_rows", _insert) @@ -1095,7 +1246,12 @@ async def query_all(self, sql: str, params: Any = None) -> list[Any]: def _judged_memory_row() -> dict[str, Any]: # A deferred verdict lands at the judged write's version + 1 (`conflicts.judged_write`). - return {**_LIVE, "content_sha256": "a", "status": "duplicate", "version": _LIVE["version"] + 1} + return { + **_LIVE, + "content_sha256": sha256_value("text"), + "status": "duplicate", + "version": _LIVE["version"] + 1, + } @pytest.mark.asyncio @@ -1104,9 +1260,10 @@ async def test_a_judged_mirror_outranks_a_metadata_copy_that_landed_first() -> N verdict's mirror must not tie with it or skip it.""" ch = _RMTCH() ch.emb += [ - {**_LIVE, "content_sha256": "a"}, - {**_LIVE, "content_sha256": "a", "version": _LIVE["version"] + 1}, + {**_LIVE, "content_sha256": sha256_value("text")}, + {**_LIVE, "content_sha256": sha256_value("text"), "version": _LIVE["version"] + 1}, ] + ch.remember([_judged_memory_row()]) await M.mirror_status_to_embeddings([_judged_memory_row()], ch=ch, judged=True) (live,) = ch.live_emb() assert live["status"] == "duplicate" @@ -1115,10 +1272,13 @@ async def test_a_judged_mirror_outranks_a_metadata_copy_that_landed_first() -> N @pytest.mark.asyncio async def test_a_judged_mirror_outranks_a_metadata_copy_that_lands_after() -> None: ch = _RMTCH() - ch.emb.append({**_LIVE, "content_sha256": "a"}) + ch.emb.append({**_LIVE, "content_sha256": sha256_value("text")}) + ch.remember([_judged_memory_row()]) await M.mirror_status_to_embeddings([_judged_memory_row()], ch=ch, judged=True) # A copy that read write V before the mirror landed writes V + 1, still active. - ch.emb.append({**_LIVE, "content_sha256": "a", "version": _LIVE["version"] + 1}) + ch.emb.append( + {**_LIVE, "content_sha256": sha256_value("text"), "version": _LIVE["version"] + 1} + ) (live,) = ch.live_emb() assert live["status"] == "duplicate" @@ -1126,11 +1286,11 @@ async def test_a_judged_mirror_outranks_a_metadata_copy_that_lands_after() -> No @pytest.mark.asyncio async def test_a_judged_mirror_leaves_a_newer_revisions_vector_alone() -> None: ch = _RMTCH() - newer = {**_LIVE, "content_sha256": "b", "version": M.next_version()} - ch.emb += [{**_LIVE, "content_sha256": "a"}, newer] + newer = {**_LIVE, "content_sha256": sha256_value("old"), "version": M.next_version()} + ch.emb += [{**_LIVE, "content_sha256": sha256_value("text")}, newer] assert await M.mirror_status_to_embeddings([_judged_memory_row()], ch=ch, judged=True) == 0 (live,) = ch.live_emb() - assert live["content_sha256"] == "b" and live["status"] == "active" + assert live["content_sha256"] == sha256_value("old") and live["status"] == "active" @pytest.mark.asyncio @@ -1138,8 +1298,13 @@ async def test_a_judged_mirror_leaves_a_vector_a_supersede_hid_meanwhile() -> No """Another write's refines verdict superseded A between the settle's memory row and its mirror: the retire stands, A does not come back into search.""" ch = _RMTCH() - retired = {**_LIVE, "content_sha256": "a", "status": "superseded", "version": M.next_version()} - ch.emb += [{**_LIVE, "content_sha256": "a"}, retired] + retired = { + **_LIVE, + "content_sha256": sha256_value("text"), + "status": "superseded", + "version": M.next_version(), + } + ch.emb += [{**_LIVE, "content_sha256": sha256_value("text")}, retired] assert await M.mirror_status_to_embeddings([_judged_memory_row()], ch=ch, judged=True) == 0 (live,) = ch.live_emb() assert live["status"] == "superseded" @@ -1153,19 +1318,39 @@ async def test_a_reject_copies_only_rows_of_the_heads_hash() -> None: own, redacted = ("o", "n", "a", "u", "s", "k"), ("o", "n", "a", "u", "s2", "k") ch = _VectorsCH( [ - {**_LIVE, "content_sha256": "b" * 64}, - {**_LIVE, "embedding_model": "m2", "content_sha256": "a" * 64}, + {**_LIVE, "content_sha256": sha256_value("old")}, + {**_LIVE, "embedding_model": "m2", "content_sha256": sha256_value("text")}, {**_LIVE, "embedding_model": "m3", "content_sha256": ""}, - {**_LIVE, "embedding_model": "m4", "content_sha256": "c" * 64, "version": 99}, - {**_LIVE, "embedding_model": "m5", "content_sha256": "a" * 64, "version": 99}, - {**_LIVE, "session_id": "s2", "content_sha256": "a" * 64}, + { + **_LIVE, + "embedding_model": "m4", + "content_sha256": sha256_value("secret"), + "version": 99, + }, + { + **_LIVE, + "embedding_model": "m5", + "content_sha256": sha256_value("text"), + "version": 99, + }, + {**_LIVE, "session_id": "s2", "content_sha256": sha256_value("text")}, ], [], ) - assert await M.reject_embedding_rows([(own, 50, "a" * 64), (redacted, 50, None)], ch=ch) == 4 + ch.heads[own].update(status="rejected", version=50) + assert ( + await M.reject_embedding_rows( + [(own, 50, sha256_value("text")), (redacted, 50, None)], ch=ch + ) + == 4 + ) # One read for the hashed key, and one for the redacted key that fetches no text. - fetched = {params["s"][0]: _selected(sql) for sql, params in ch.query_all_calls} - assert len(ch.query_all_calls) == 2 and sorted(fetched) == ["s", "s2"] + vector_reads = [ + (sql, params) for sql, params in ch.query_all_calls if "FROM loom.memory_embeddings" in sql + ] + fetched = {params["s"][0]: _selected(sql) for sql, params in vector_reads} + assert len(vector_reads) == 2 and sorted(fetched) == ["s", "s2"] + assert len([sql for sql, _ in ch.query_all_calls if "FROM loom.memory FINAL" in sql]) == 2 assert sorted(fetched["s2"]) == sorted( [ "org", @@ -1285,10 +1470,12 @@ async def test_a_merge_retires_only_the_rows_still_holding_the_text_it_read(monk async def written(org, ids, update, *, skip, strong, **_k): asked["strong"] = strong heads = [ - _memory_row(memory_id="m-same", content_sha256="a" * 64), - _memory_row(memory_id="m-rewritten", content_sha256="b" * 64), - _memory_row(memory_id="m-pinned", content_sha256="a" * 64, pinned=True), - _memory_row(memory_id="m-locked", content_sha256="a" * 64, lock_mode="no_delete"), + _memory_row(memory_id="m-same", content_sha256=sha256_value("text")), + _memory_row(memory_id="m-rewritten", content_sha256=sha256_value("old")), + _memory_row(memory_id="m-pinned", content_sha256=sha256_value("text"), pinned=True), + _memory_row( + memory_id="m-locked", content_sha256=sha256_value("text"), lock_mode="no_delete" + ), ] rows = [update(h) for h in heads if not skip(h)] return len(rows) @@ -1306,10 +1493,22 @@ async def no_chunks(**_k): object(), MemoryConsolidateInput(org="o", namespace="n", agent="a", user_id="u", session_id="s"), [ - {"memory_id": "m-same", "memory_key": "k-same", "content_sha256": "a" * 64}, - {"memory_id": "m-rewritten", "memory_key": "k-rewritten", "content_sha256": "a" * 64}, - {"memory_id": "m-pinned", "memory_key": "k-pinned", "content_sha256": "a" * 64}, - {"memory_id": "m-locked", "memory_key": "k-locked", "content_sha256": "a" * 64}, + {"memory_id": "m-same", "memory_key": "k-same", "content_sha256": sha256_value("text")}, + { + "memory_id": "m-rewritten", + "memory_key": "k-rewritten", + "content_sha256": sha256_value("text"), + }, + { + "memory_id": "m-pinned", + "memory_key": "k-pinned", + "content_sha256": sha256_value("text"), + }, + { + "memory_id": "m-locked", + "memory_key": "k-locked", + "content_sha256": sha256_value("text"), + }, ], canonical_key="canon", now_naive=datetime.now(), @@ -1322,11 +1521,16 @@ async def test_a_copy_of_one_content_leaves_another_versions_vectors_alone() -> """`of_content` copies only the rows of the head's hash: a row of another version's text, which a redaction may be purging, is not carried forward.""" ch = _SiteCH() - ch.emb.append({**_LIVE, "content_sha256": "a" * 64, "version": 10}) - ch.emb.append({**_LIVE, "embedding_model": "m2", "content_sha256": "b" * 64, "version": 10}) + ch.emb.append({**_LIVE, "content_sha256": sha256_value("text"), "version": 10}) + ch.emb.append( + {**_LIVE, "embedding_model": "m2", "content_sha256": sha256_value("old"), "version": 10} + ) + ch.remember([M.build_memory_version_row(_memory_row(), status="conflicted", version=20)]) n = await M.reemit_embedding_rows_for_keys( - [(_KEY, {"status": "conflicted", "version": 20})], ch=ch, of_content={_KEY: "a" * 64} + [(_KEY, {"status": "conflicted", "version": 20})], + ch=ch, + of_content={_KEY: sha256_value("text")}, ) assert n == 1 copied = [r for r in ch.emb if r["version"] == 20] - assert [r["content_sha256"] for r in copied] == ["a" * 64] + assert [r["content_sha256"] for r in copied] == [sha256_value("text")] diff --git a/tests/unit/test_embedding_worker_fence.py b/tests/unit/test_embedding_worker_fence.py index c844e6b314..4ca63f8f81 100644 --- a/tests/unit/test_embedding_worker_fence.py +++ b/tests/unit/test_embedding_worker_fence.py @@ -84,6 +84,7 @@ async def find_stubs(**kw): async def insert_rows(rows, *a, **kw): inserted.append(list(rows)) + return rows async def none_missing(**kw): return [] diff --git a/tests/unit/test_entity_mirror_heal.py b/tests/unit/test_entity_mirror_heal.py index 3a54ef48ca..6644217df2 100644 --- a/tests/unit/test_entity_mirror_heal.py +++ b/tests/unit/test_entity_mirror_heal.py @@ -7,7 +7,9 @@ import asyncio from typing import Any +from server.core.hashing import sha256_value from server.memory import store as mr +from tests.unit.test_memory_verify import _row class _CH: @@ -15,11 +17,26 @@ class _CH: def __init__(self, answers: list[list[dict[str, Any]]]) -> None: self.answers = list(answers) + self.heads = {} + for pair in answers[0] if answers else []: + source = _row( + org="o", + namespace="n", + agent="a", + memory_key=pair["memory_key"], + value="x", + entities=pair["entities"], + ) + self.heads[mr.identity_key(source)] = mr.build_memory_version_row(source) self.queries: list[str] = [] self.inserts: list[tuple[str, list[dict[str, Any]]]] = [] async def query_all(self, sql, params=None): self.queries.append(" ".join(sql.split())[:80]) + if "memory_refusals" in sql: + return [] + if "FROM loom.memory FINAL" in sql and "PREWHERE" in sql: + return [r for key, r in self.heads.items() if key in params["keys"]] return self.answers.pop(0) if self.answers else [] async def insert(self, table, rows, settings=None): @@ -37,6 +54,7 @@ def _emb_row(entities: list[str]) -> dict[str, Any]: "memory_key": "k", "memory_type": "semantic", "content_excerpt": "x", + "content_sha256": sha256_value("x"), "embedding_model": "m", "embedding_dim": 3, "embedding": [0.1, 0.2, 0.3], diff --git a/tests/unit/test_erasure_no_content_columns.py b/tests/unit/test_erasure_no_content_columns.py index 080ab53bf0..c5a0cd7357 100644 --- a/tests/unit/test_erasure_no_content_columns.py +++ b/tests/unit/test_erasure_no_content_columns.py @@ -24,6 +24,9 @@ # table -> {column: why it is not subject content} _REVIEWED: dict[str, dict[str, str]] = { + "loom.memory_refusals": { + "scope_value": "the namespace or user id whose deletion consent the hash guards" + }, "loom.blacklist": {"reason": "the admin's reason for a refusal pattern"}, "loom.org_model_calls": {"reason": "the org admin's note on switching model calls off"}, "loom.runtime_overrides": {"value": "a rate limit or latency budget as JSON numbers"}, diff --git a/tests/unit/test_erasure_publication_tombstones.py b/tests/unit/test_erasure_publication_tombstones.py new file mode 100644 index 0000000000..d15272faf6 --- /dev/null +++ b/tests/unit/test_erasure_publication_tombstones.py @@ -0,0 +1,404 @@ +"""Delayed retirement cannot recreate plaintext or marker rows after erasure.""" + +from datetime import datetime + +import pytest + +from server.core.errors import BackendUnavailableError, ValidationError +from server.core.hashing import sha256_value +from server.memory.domain.refusals import fingerprint +from server.memory.store import refusals +from server.memory.store.inserts import insert_embedding_rows, insert_memory_versions +from server.memory.store.rows import build_embedding_row, build_memory_version_row +from tests.unit.test_memory_verify import _row + + +class CH: + def __init__(self, head, scope="", *, active=False, failure=False): + self.head = head + self.scope = scope + self.active = active + self.failure = failure + self.inserts = [] + self.deletes = [] + self.patterns = set(fingerprint("retired copper lark")) + + async def query_all(self, sql, params=None): + if "FROM loom.memory FINAL" in sql: + return [build_memory_version_row(self.head)] if self.head else [] + assert "memory_refusals" in sql + if self.failure: + raise BackendUnavailableError("policy unavailable") + if "SELECT DISTINCT" in sql: + return [{"units": -1 if self.active else 3}] + erasure_only = "('identity','erased_identity')" not in sql + match = self.scope and (not erasure_only or self.scope != "identity") + return [{"found": 1}] if match and self.patterns.intersection(params["candidates"]) else [] + + async def insert(self, table, rows, **kwargs): + self.inserts.extend(rows) + + async def execute(self, sql, params=None, *, settings=None): + assert sql.startswith("DELETE FROM loom.memory_embeddings") + assert "created_at" not in sql + assert settings == {"lightweight_deletes_sync": 2} + self.deletes.append(params) + + +def vector(head): + return build_embedding_row( + **{ + c: getattr(head, c) + for c in ("org", "namespace", "agent", "user_id", "session_id", "memory_key") + }, + content_excerpt="retired copper lark", + content_sha256=sha256_value("retired copper lark"), + embedding_model="test", + embedding_dim=1, + embedding=[0.5], + created_at=datetime(2026, 1, 1), + version=200, + tombstoned_at=datetime(2026, 1, 2), + ) + + +@pytest.mark.parametrize("scope", ["", "identity", "namespace", "user_id", "erased_identity"]) +async def test_retirement_preserves_only_permitted_cache_content(scope): + head = _row(value="retired copper lark", tombstoned_at=datetime(2026, 1, 2)) + ch = CH(head, scope) + row = vector(head) + acknowledged = await insert_embedding_rows([row], ch=ch) + assert len(acknowledged) == 1 + if scope in {"namespace", "user_id", "erased_identity"}: + assert not ch.inserts and len(ch.deletes) == 1 + else: + assert not ch.deletes and len(ch.inserts) == 1 + assert ch.inserts[0]["content_excerpt"] == ( + "" if scope == "identity" else row["content_excerpt"] + ) + + +@pytest.mark.parametrize("scope", ["", "identity", "user_id"]) +async def test_missing_source_is_cleaned_without_inserting_even_a_bare_marker(scope): + ch = CH(None, scope) + row = vector(_row(value="retired copper lark")) + assert await insert_embedding_rows([row], ch=ch) == [row] + assert not ch.inserts and len(ch.deletes) == 1 + assert ch.deletes[0]["version"] == 200 + assert "created_at" not in ch.deletes[0] + + +@pytest.mark.parametrize("active,failure", [(True, False), (False, True)]) +async def test_missing_source_still_requires_available_policy_and_no_active_erasure( + active, failure +): + ch = CH(None, active=active, failure=failure) + row = vector(_row(value="retired copper lark")) + with pytest.raises(BackendUnavailableError): + await insert_embedding_rows([row], ch=ch) + assert not ch.inserts and not ch.deletes + + +@pytest.mark.parametrize("scope", ["identity", "namespace", "user_id", "erased_identity"]) +async def test_canonical_tombstone_preserves_point_history_but_cannot_republish_erased_text(scope): + head = _row(value="retired copper lark", tombstoned_at=datetime(2026, 1, 2)) + ch = CH(head, scope) + rows = [build_memory_version_row(head)] + if scope == "identity": + await refusals.check_clickhouse(ch, rows) + else: + with pytest.raises(ValidationError, match="refused"): + await refusals.check_clickhouse(ch, rows) + + +@pytest.mark.parametrize("scope", ["", "identity", "namespace", "user_id", "erased_identity"]) +async def test_native_committed_history_delivers_after_point_deletion_but_not_erasure( + monkeypatch, scope +): + import json + from contextlib import asynccontextmanager + + from server.core import pg + from server.memory.app import extraction_queue + + class Connection: + async def execute(self, *args): + return None + + async def fetch(self, *args): + return [{"units": 3}] + + async def fetchval(self, sql, *args): + erasure_only = "('identity','erased_identity')" not in sql + return bool(scope and (not erasure_only or scope != "identity")) + + conn = Connection() + + @asynccontextmanager + async def transaction(): + yield conn + + monkeypatch.setattr(pg, "enabled", lambda: True) + monkeypatch.setattr(pg, "transaction", transaction) + rows = [build_memory_version_row(_row(value="retired copper lark"))] + + async def publish(): + ch = CH(None) + assert await insert_memory_versions(rows, ch=ch, published=True) == 1 + assert ch.inserts == rows + + if scope in {"", "identity"}: + await publish() + else: + with pytest.raises(ValidationError, match="refused"): + await publish() + # A new authority commit still checks point refusals. + if scope: + with pytest.raises(ValidationError, match="refused"): + await refusals.check_postgres(conn, rows) + with pytest.raises(ValidationError, match="refused"): + async with refusals.publication(CH(None), rows): + pytest.fail("Uncommitted content bypassed its refusal") + raw = { + **rows[0], + "content": "retired copper lark", + "payload": json.dumps( + {"messages": [{"role": "user", "content": "retired copper lark"}]} + ), + } + for table in ("loom.memory_evidence", "loom.memory_chunks", "queue"): + ch = CH(None) + with pytest.raises(ValidationError, match="refused"): + if table == "queue": + await extraction_queue._insert_rows(ch, [raw], skip_refused=False, create=True) + else: + await refusals.insert_content_rows(ch, table, [raw]) + assert ch.inserts == [] + else: + await refusals.check_postgres(conn, rows) + + +@pytest.mark.parametrize("scope", ["private", "shared"]) +async def test_orphan_cleanup_preserves_newer_vectors_and_removes_older_months(scope): + head = _row(value="retired copper lark") + requested = vector(head) | {"scope": scope} + old = requested | {"version": 50, "created_at": datetime(2025, 12, 1)} + recent = requested | {"version": 100, "created_at": datetime(2026, 1, 1)} + newer = requested | {"version": 250} + + class State(CH): + def __init__(self): + super().__init__(None) + self.rows = [old, recent, newer] + + async def execute(self, sql, params=None, *, settings=None): + await super().execute(sql, params, settings=settings) + self.rows = [r for r in self.rows if r["version"] >= params["version"]] + + ch = State() + await insert_embedding_rows([requested], ch=ch) + assert ch.rows == [newer] + assert not ch.inserts + + +@pytest.mark.parametrize( + "durable_present,current_present", [(True, False), (False, True), (False, False)] +) +async def test_native_retirement_rechecks_on_its_only_connection( + monkeypatch, durable_present, current_present +): + from contextlib import asynccontextmanager + + from server.core import pg + from server.memory.store import pg_memory + from server.memory.store.rows import identity_key + + head = _row(value="retired copper lark", tombstoned_at=datetime(2026, 1, 2)) + raw = build_memory_version_row(head) + heads = {identity_key(raw): raw} + borrowed = False + + class Connection: + async def execute(self, *args): + assert borrowed + + async def fetch(self, *args): + assert borrowed + return [] + + async def fetchval(self, *args): + assert borrowed + return None + + connection = Connection() + + async def durable(*args, **kwargs): + assert not borrowed and kwargs["durable"] + return heads if durable_present else {} + + async def current(conn, *args, **kwargs): + assert borrowed and conn is connection and kwargs["lock"] + return heads if current_present else {} + + @asynccontextmanager + async def transaction(): + nonlocal borrowed + assert not borrowed + borrowed = True + try: + yield connection + finally: + borrowed = False + + monkeypatch.setattr(pg, "enabled", lambda: True) + monkeypatch.setattr(pg, "transaction", transaction) + monkeypatch.setattr(pg_memory, "heads_by_keys", durable) + monkeypatch.setattr(pg_memory, "heads_on_connection", current) + ch = CH(None) + acknowledged = await insert_embedding_rows([vector(head)], ch=ch) + assert not ch.inserts + assert len(acknowledged) == len(ch.deletes) == int(not durable_present and not current_present) + + +async def test_a_purged_queue_source_is_not_recreated_by_a_state_transition(): + from server.memory.app import extraction_queue + from tests.unit.test_extraction_queue import _stored_row + + class Missing: + async def query_all(self, sql, params=None): + return [] + + async def insert(self, *args, **kwargs): + pytest.fail("A queue mirror recreated its missing source") + + await extraction_queue.mark_done(_stored_row(), written=1, ch=Missing()) + + +@pytest.mark.parametrize("scope", ["", "identity", "namespace", "user_id", "erased_identity"]) +@pytest.mark.parametrize("present", [False, True]) +@pytest.mark.parametrize("native", [False, True]) +@pytest.mark.parametrize("user", ["alice", "alice/team", ""]) +async def test_shared_canonical_retirement_cannot_recreate_erased_plaintext( + monkeypatch, scope, present, native, user +): + from server.memory.store import shared + + head = _row(value="retired copper lark", user_id=user) + row = { + **dict.fromkeys(shared.SHARED_PERSISTED_COLS), + "org": head.org, + "namespace": head.namespace, + "memory_key": head.memory_key, + "value": '"retired copper lark"', + "version": 200, + "tombstoned_at": datetime(2026, 1, 2), + "promoted_from": [ + "/".join( + str(getattr(head, c)) + for c in ("org", "namespace", "agent", "user_id", "session_id", "memory_key") + ) + ], + } + + class SharedCH(CH): + async def query_all(self, sql, params=None): + if params["uid"] != (user or "-"): + return [] + return await super().query_all(sql, params=params) + + async def query_one(self, sql, params=None): + assert "FROM loom.shared" in sql and "select_sequential_consistency" in sql + return {"version": 50} if present else None + + async def execute(self, sql, params=None, *, settings=None): + assert sql.startswith("DELETE FROM loom.shared") + assert "version <" in sql and params["version"] == 200 + assert settings == {"lightweight_deletes_sync": 2} + self.deletes.append(params) + + if native: + from contextlib import asynccontextmanager + + from server.core import pg + + borrowed = False + + class Connection: + async def execute(self, *args): + assert borrowed + + async def fetch(self, *args): + assert borrowed + return [{"units": 3}] if args[3] == (user or "-") else [] + + async def fetchval(self, *args): + assert borrowed + return args[3] == (user or "-") and scope in { + "namespace", + "user_id", + "erased_identity", + } + + @asynccontextmanager + async def transaction(): + nonlocal borrowed + assert not borrowed + borrowed = True + try: + yield Connection() + finally: + borrowed = False + + monkeypatch.setattr(pg, "enabled", lambda: True) + monkeypatch.setattr(pg, "transaction", transaction) + + ch = SharedCH(head, scope) + await shared.insert_shared_row(row, ch=ch) + purged = not present or scope in {"namespace", "user_id", "erased_identity"} + assert bool(ch.deletes) == purged + assert ch.inserts == ([] if purged else [row]) + + +async def test_shared_repair_cannot_borrow_an_ambiguous_neighbors_point_consent(): + import json + + from server.memory.domain.excerpt import build_excerpt + from server.memory.store import shared + + source = _row( + org="o", + namespace="n", + agent="a/alice", + user_id="team", + session_id="s", + memory_key="k", + value="retired copper lark", + ) + row = vector(source) | { + "agent": "_shared", + "user_id": "-", + "session_id": "-", + "scope": "shared", + "tombstoned_at": None, + "content_excerpt": build_excerpt(json.dumps(source.value), memory_key="k"), + } + canonical = { + "value": json.dumps(source.value), + "tombstoned_at": None, + "promoted_from": ["o/n/a/alice/team/s/k"], + "version": 100, + } + + class Shared(CH): + async def query_one(self, sql, params=None): + return canonical + + async def query_all(self, sql, params=None): + if "memory_refusals" in sql and params["uid"] != "alice/team": + return [] + return await super().query_all(sql, params=params) + + ch = Shared(source, "identity") + with pytest.raises(ValidationError, match="previous deletion"): + await shared.repair_shared_embeddings([row], ch=ch) + assert not ch.inserts diff --git a/tests/unit/test_erasure_step_order.py b/tests/unit/test_erasure_step_order.py index 588af3483f..1d58f7efba 100644 --- a/tests/unit/test_erasure_step_order.py +++ b/tests/unit/test_erasure_step_order.py @@ -58,6 +58,7 @@ def _push_delete(namespaces: int) -> tuple[str, tuple[tuple[str, str], ...]]: USER_TRACE: list[tuple[str, tuple[tuple[str, str], ...]]] = [ + ("memory.refusals", _SILENT), ("memory.tombstone", ((U, "loom.memory"),)), ("memory_embeddings.tombstone", ((U, "loom.memory_embeddings"),)), ( @@ -424,6 +425,7 @@ def _push_delete(namespaces: int) -> tuple[str, tuple[tuple[str, str], ...]]: # A user purge: the memory rows the later steps read the user's keys from are stamped # first and deleted, with the summaries built from them, in `memory.retire`. USER_PURGE_TRACE: list[tuple[str, tuple[tuple[str, str], ...]]] = [ + ("memory.refusals", _SILENT), ("memory.tombstone", ((U, "loom.memory"),)), ("memory_embeddings.tombstone", ((L, "loom.memory_embeddings"),)), ( @@ -625,6 +627,7 @@ async def test_namespace_saga_trace( run: Any, scope_type: str, scope_value: str, backends: list[str] ) -> None: want = [ + ("memory.refusals", _SILENT), _SOURCES_RETIRE, _push_delete(1 if scope_type == "namespace" else 2), ( @@ -646,6 +649,7 @@ async def test_namespace_purge_saga_trace(run: Any, external: bool) -> None: lightweight DELETE, the narrow ones by an ALTER DELETE. Same steps.""" got = await run("namespace_list", _ns_list(purge=True, external=external)) assert got == [ + ("memory.refusals", _SILENT), _SOURCES_RETIRE, _push_delete(2), ( diff --git a/tests/unit/test_export_evidence_holds_high.py b/tests/unit/test_export_evidence_holds_high.py index 81dfbe664a..8ad351d352 100644 --- a/tests/unit/test_export_evidence_holds_high.py +++ b/tests/unit/test_export_evidence_holds_high.py @@ -18,13 +18,13 @@ class _CH: + def __init__(self, digests): + self.digests = digests + async def query_all(self, sql: str, params: Any = None) -> list[dict[str, Any]]: if "session_summaries" in sql: - return [ - {"session_id": "s1", "summary_text": "a plain day", "derived_from": ["k"]}, - {"session_id": "s2", "summary_text": "plain", "derived_from": ["vault"]}, - {"session_id": "s3", "summary_text": f"the key is {_KEY}", "derived_from": []}, - ] + return self.digests + return [ {"session_id": "s", "chunk_index": 0, "content": "plain talk", "embedding_dim": 2}, {"session_id": "s", "chunk_index": 1, "content": "about the vault", "embedding_dim": 2}, @@ -44,10 +44,36 @@ async def _links(_ch: Any, **_k: Any) -> dict[tuple[str, int], list[str]]: return {("s", 0): ["k"], ("s", 1): ["vault"], ("s", 2): [], ("s", 3): []} async def _high_keys(_org: str, _ns: str, keys: Any, **_k: Any) -> set[str]: - # `vault` had a high version, and has since been deleted + # `vault` had a high version; its current source is a plain fact. return {"vault"} & set(keys) - monkeypatch.setattr(evidence_transfer, "get_client", lambda: _CH()) + from tests.unit.memory_store import seed_summary_sources + + digests = [ + { + "org": "o", + "namespace": "n", + "session_id": "s1", + "summary_text": "a plain day", + "derived_from": ["k"], + }, + { + "org": "o", + "namespace": "n", + "session_id": "s2", + "summary_text": "plain", + "derived_from": ["vault"], + }, + { + "org": "o", + "namespace": "n", + "session_id": "s3", + "summary_text": f"the key is {_KEY}", + "derived_from": ["credential"], + }, + ] + seed_summary_sources(monkeypatch, digests) + monkeypatch.setattr(evidence_transfer, "get_client", lambda: _CH(digests)) monkeypatch.setattr(chunk_links, "chunk_memory_keys", _links) monkeypatch.setattr(memory_repo, "high_keys", _high_keys) audits: list[list[dict[str, Any]]] = [] @@ -89,7 +115,7 @@ async def test_chunks_behind_a_high_key_holding_a_credential_or_unembedded_stay_ async def test_digests_behind_a_high_key_or_holding_a_credential_stay_home(evidence) -> None: out = await evidence_transfer.handle_memory_export_evidence(_input("digests")) assert [d.session_id for d in out.digests] == ["s1"] - assert out.held_high == 2 + assert out.held_high == 1 @pytest.mark.asyncio @@ -97,10 +123,9 @@ async def test_include_high_releases_held_digests_and_audits_each(evidence) -> N out = await evidence_transfer.handle_memory_export_evidence( _input("digests", include_high=True) ) - assert [d.session_id for d in out.digests] == ["s1", "s2", "s3"] and out.held_high == 0 + assert [d.session_id for d in out.digests] == ["s1", "s2"] and out.held_high == 0 assert evidence == [ [ {"kind": "digests", "session_id": "s2", "memory_keys": ["vault"]}, - {"kind": "digests", "session_id": "s3", "memory_keys": []}, ] ] diff --git a/tests/unit/test_extract_concurrent_projections.py b/tests/unit/test_extract_concurrent_projections.py index 48d4fa1c7a..4078d8b3c4 100644 --- a/tests/unit/test_extract_concurrent_projections.py +++ b/tests/unit/test_extract_concurrent_projections.py @@ -1,28 +1,26 @@ -"""The session digest is drafted while the extractor runs, and -the post-extraction projections are awaited together. +"""Extraction commits before overlapping projection writes. -CH-free and model-free: every projection is a stub that records when it ran. -The assertions are about ORDER and OVERLAP, not about row shapes (those are -covered by test_session_summaries.py and the chunk/evidence tests). +Projection stubs record execution order, overlap and isolated failures. """ from __future__ import annotations import asyncio import time +from types import SimpleNamespace from typing import Any +from unittest.mock import AsyncMock import pytest from fastapi import HTTPException from server.memory.api.models_bulk import MemorySetFromMessagesInput -from server.memory.app.session_summary import SessionSummaryDraft -_DRAFT = SessionSummaryDraft( - summary_text="User moved to Lisbon in 2021.", - embedding=[0.1, 0.2, 0.3], - embedding_model="fake-embed", -) + +@pytest.fixture(autouse=True) +def _no_refused_intake(monkeypatch): + # Pipeline and projection cases; actual refusal admission is tested separately. + monkeypatch.setattr("server.memory.store.refusals.check_intake", AsyncMock()) def _input() -> MemorySetFromMessagesInput: @@ -42,7 +40,6 @@ def __init__(self) -> None: self.spans: dict[str, tuple[float, float]] = {} self.calls: list[str] = [] self.failures: list[str] = [] - self.drafts_written: list[Any] = [] def stage(self, name: str, *, seconds: float = 0.05, result: Any = None, raises: bool = False): async def _run(*args: Any, **kwargs: Any) -> Any: @@ -52,8 +49,6 @@ async def _run(*args: Any, **kwargs: Any) -> Any: self.spans[name] = (t0, time.perf_counter()) if raises: raise RuntimeError(f"{name} exploded") - if name == "summary": - self.drafts_written.append(kwargs.get("draft")) return result return _run @@ -63,8 +58,7 @@ def _wire( monkeypatch: pytest.MonkeyPatch, log: _Log, *, - draft: Any = _DRAFT, - draft_raises: bool = False, + summary_raises: bool = False, chunks_raise: bool = False, ) -> None: import server.identity.app.namespace_ontology as ont @@ -72,6 +66,10 @@ def _wire( import server.memory.app.pipeline as pipe from server.memory.app import bulk_set as _bulk_set_mod + monkeypatch.setattr( + mx, "get_client", lambda: SimpleNamespace(query_all=AsyncMock(return_value=[])) + ) + async def _no_evidence(input: Any, msgs: Any, **kw: Any) -> list[str]: return ["ev-1"] @@ -92,14 +90,9 @@ async def _cold(org: str, ns: str) -> dict[str, Any]: monkeypatch.setattr(mx, "_write_source_evidence", _no_evidence) monkeypatch.setattr(pipe, "run_pipeline", _pipeline) - monkeypatch.setattr( - mx, - "_draft_session_summary", - log.stage("draft", seconds=0.02, result=draft, raises=draft_raises), - ) monkeypatch.setattr(mx, "_link_written_memories_to_evidence", log.stage("link", result=3)) monkeypatch.setattr(mx, "write_memory_chunks", log.stage("chunks", raises=chunks_raise)) - monkeypatch.setattr(mx, "_write_session_summary", log.stage("summary")) + monkeypatch.setattr(mx, "_write_session_summary", log.stage("summary", raises=summary_raises)) monkeypatch.setattr(mx, "record_input_projection_failure", _failure) monkeypatch.setattr(_bulk_set_mod, "record_input_projection_failure", _failure) monkeypatch.setattr(ont, "load_namespace_ontology", _cold) @@ -110,7 +103,7 @@ def _overlap(a: tuple[float, float], b: tuple[float, float]) -> bool: @pytest.mark.asyncio -async def test_draft_runs_during_extraction_and_projections_overlap( +async def test_committed_extraction_precedes_overlapping_projections( monkeypatch: pytest.MonkeyPatch, ) -> None: import server.memory.app.extract.handler as mx @@ -119,14 +112,10 @@ async def test_draft_runs_during_extraction_and_projections_overlap( _wire(monkeypatch, log) out = await mx.handle_memory_set_from_messages(_input()) - # The digest draft started before the pipeline finished. - assert log.spans["draft"][0] < log.spans["pipeline"][1] + assert log.spans["summary"][0] >= log.spans["pipeline"][1] # The three projections ran together, not back to back. assert _overlap(log.spans["link"], log.spans["chunks"]) assert _overlap(log.spans["chunks"], log.spans["summary"]) - # The pre-made draft reached the writer; nothing was drafted twice. - assert log.drafts_written == [_DRAFT] - assert log.calls.count("draft") == 1 assert out.belief_source_links == 3 assert [w.memory_key for w in out.written] == ["home"] assert log.failures == [] @@ -149,67 +138,31 @@ async def test_one_failing_projection_does_not_take_the_others_down( @pytest.mark.asyncio -async def test_failed_draft_is_recorded_and_not_redrafted(monkeypatch: pytest.MonkeyPatch) -> None: +async def test_a_summary_projection_failure_is_recorded(monkeypatch: pytest.MonkeyPatch) -> None: import server.memory.app.extract.handler as mx log = _Log() - _wire(monkeypatch, log, draft_raises=True) + _wire(monkeypatch, log, summary_raises=True) out = await mx.handle_memory_set_from_messages(_input()) assert log.failures == ["session_summaries"] - assert "summary" not in log.calls # no second model call on the inline path + assert log.calls.count("summary") == 1 assert out.written -@pytest.mark.asyncio -async def test_empty_draft_skips_the_row_without_a_second_call( - monkeypatch: pytest.MonkeyPatch, -) -> None: - import server.memory.app.extract.handler as mx - - log = _Log() - _wire(monkeypatch, log, draft=None) - await mx.handle_memory_set_from_messages(_input()) - - assert log.calls.count("draft") == 1 - assert "summary" not in log.calls - assert log.failures == [] - - -@pytest.mark.asyncio -async def test_extraction_failure_cancels_the_draft(monkeypatch: pytest.MonkeyPatch) -> None: +async def test_extraction_failure_never_starts_a_digest(monkeypatch: pytest.MonkeyPatch) -> None: import server.memory.app.extract.handler as mx import server.memory.app.pipeline as pipe from server.memory.app.extract import ExtractionError log = _Log() _wire(monkeypatch, log) - cancelled = asyncio.Event() - - async def _slow_draft(*args: Any, **kwargs: Any) -> Any: - try: - await asyncio.sleep(10) - except asyncio.CancelledError: - cancelled.set() - raise - async def _pipeline_raises(steps: Any, state: Any) -> Any: - await asyncio.sleep(0) # let the draft task start, as a real model call would + async def fail(*args: Any, **kwargs: Any) -> Any: raise ExtractionError("no JSON object in LLM response") - monkeypatch.setattr(mx, "_draft_session_summary", _slow_draft) - monkeypatch.setattr(pipe, "run_pipeline", _pipeline_raises) - - with pytest.raises(HTTPException) as ei: + monkeypatch.setattr(pipe, "run_pipeline", fail) + with pytest.raises(HTTPException) as error: await mx.handle_memory_set_from_messages(_input()) - assert ei.value.status_code == 502 - await asyncio.wait_for(cancelled.wait(), timeout=1.0) - - -def test_write_session_summary_uses_a_given_draft() -> None: - """The writer's `draft` parameter is the seam the concurrent path relies on.""" - import inspect - - from server.memory.app.session_summary import _write_session_summary - - assert "draft" in inspect.signature(_write_session_summary).parameters + assert error.value.status_code == 502 + assert "summary" not in log.calls diff --git a/tests/unit/test_extract_defer_allowlist.py b/tests/unit/test_extract_defer_allowlist.py index b9c2edec42..53b987049a 100644 --- a/tests/unit/test_extract_defer_allowlist.py +++ b/tests/unit/test_extract_defer_allowlist.py @@ -67,6 +67,7 @@ class _Auth: @pytest.mark.asyncio +@pytest.mark.usefixtures("no_blacklisted_values") async def test_handler_takes_the_queue_path_for_an_allowlisted_agent( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/tests/unit/test_extract_write_dates.py b/tests/unit/test_extract_write_dates.py index 780c07ccfd..61216e6805 100644 --- a/tests/unit/test_extract_write_dates.py +++ b/tests/unit/test_extract_write_dates.py @@ -84,6 +84,7 @@ async def test_triple_valid_from_null_when_undated() -> None: @pytest.mark.asyncio +@pytest.mark.usefixtures("no_blacklisted_values") async def test_schema_mode_entry_defaults_valid_at_from_observation_date( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/tests/unit/test_extraction_policy.py b/tests/unit/test_extraction_policy.py index 03b2d2b56e..83fb45603d 100644 --- a/tests/unit/test_extraction_policy.py +++ b/tests/unit/test_extraction_policy.py @@ -5,6 +5,7 @@ from __future__ import annotations from typing import Any +from unittest.mock import AsyncMock import pytest @@ -15,6 +16,13 @@ from server.settings import settings from tests.unit.switch_state import policies_read_now, policies_unread + +@pytest.fixture(autouse=True) +def _no_refused_intake(monkeypatch): + # Pipeline and projection cases; actual refusal admission is tested separately. + monkeypatch.setattr("server.memory.store.refusals.check_intake", AsyncMock()) + + _POLICY = "Never store credentials, payment card data or health data." _MSGS = [ {"role": "user", "content": "I moved to Lisbon in 2021 and I still live there today."}, @@ -182,29 +190,43 @@ async def test_the_typed_value_stage_ends_with_the_policy(monkeypatch) -> None: assert all(_POLICY not in s for s in llm.systems if s not in value_stage) -async def test_the_session_digest_ends_with_the_policy(monkeypatch) -> None: - import server.llm.client as llm_client - from server.memory.app import session_summary +async def test_the_session_digest_uses_committed_policy_filtered_facts_only(monkeypatch) -> None: + from types import SimpleNamespace + from unittest.mock import AsyncMock - llm = _Provider(text="On 2026-01-01 the user said they live in Lisbon.") - monkeypatch.setattr(llm_client, "get_llm_client", lambda: llm) + from server.memory import store + from server.memory.app import digest_chain, session_summary + from tests.unit.test_memory_verify import _row - class _Embedder: - model_name = "fake" - dim = 1 + source = _row(value="Lives in Lisbon.") + ch = SimpleNamespace(insert=AsyncMock(), query_all=AsyncMock(return_value=[])) - async def embed(self, text: str) -> list[float]: - return [1.0] + async def heads(keys, **kwargs): + from server.memory.store import identity_key - async def embed_many(self, texts: list[str]) -> list[list[float]]: - return [await self.embed(text) for text in texts] + return {identity_key(source): source} - monkeypatch.setattr(session_summary, "get_embedder", lambda: _Embedder()) - draft = await session_summary._draft_session_summary( - object(), _MSGS, "2026-01-01", operator_policy=_POLICY + monkeypatch.setattr("server.memory.store.summary_sources.memory_rows_by_identity_keys", heads) + monkeypatch.setattr(session_summary, "get_client", lambda: ch) + monkeypatch.setattr(session_summary, "get_embedder", lambda: SimpleNamespace(model_name="fake")) + monkeypatch.setattr(session_summary, "document_vector", AsyncMock(return_value=[1.0])) + monkeypatch.setattr(store, "get_by_id", AsyncMock(return_value=source)) + monkeypatch.setattr(digest_chain, "link_digest", AsyncMock()) + inp = SimpleNamespace( + org=source.org, + namespace=source.namespace, + agent=source.agent, + user_id=source.user_id, + session_id=source.session_id, + ) + state = SimpleNamespace( + written=[{"memory_id": source.memory_id, "version": source.version}], + extracted_facts=[{"value": "forbidden raw detail"}], + operator_policy=_POLICY, + observation_date=None, ) - assert draft is not None - assert llm.systems == [with_policy(session_summary._SESSION_SUMMARY_SYSTEM, _POLICY)] + await session_summary._write_session_summary(inp, state) + assert ch.insert.call_args.args[1][0]["summary_text"] == "Lives in Lisbon." async def test_trajectory_events_and_notes_end_with_the_policy( @@ -595,7 +617,7 @@ def who(org: str, *role: Role, ns: tuple[str, ...] = ()) -> AuthInfo: def _wire(monkeypatch: pytest.MonkeyPatch) -> dict[str, Any]: """The handler with every write stubbed: what reached the pipeline and the - digest draft, and whether the evidence write ran.""" + committed state, and whether the evidence write ran.""" import server.identity.app.namespace_ontology as ont import server.memory.app.extract.handler as mx import server.memory.app.pipeline as pipe @@ -610,9 +632,6 @@ async def _pipeline(steps: Any, state: Any) -> Any: seen["state_policy"] = state.operator_policy return state - async def _draft(*args: Any, **kw: Any) -> None: - seen["draft_policy"] = kw.get("operator_policy") - async def _nothing(*args: Any, **kw: Any) -> Any: return 0 @@ -621,7 +640,6 @@ async def _cold(org: str, ns: str) -> dict[str, Any]: monkeypatch.setattr(mx, "_write_source_evidence", _evidence) monkeypatch.setattr(pipe, "run_pipeline", _pipeline) - monkeypatch.setattr(mx, "_draft_session_summary", _draft) monkeypatch.setattr(mx, "_link_written_memories_to_evidence", _nothing) monkeypatch.setattr(mx, "write_memory_chunks", _nothing) monkeypatch.setattr(mx, "_write_session_summary", _nothing) @@ -654,7 +672,6 @@ async def test_set_from_messages_runs_under_the_stored_policy_whatever_it_asks( ) effective = f"{_POLICY}\n\nNo addresses." assert seen["state_policy"] == effective - assert seen["draft_policy"] == effective assert out.under_operator_policy is True policies_read_now({}) assert (await mx.handle_memory_set_from_messages(_request())).under_operator_policy is False @@ -1164,7 +1181,6 @@ async def no_conflict_policy(state: Any) -> bool: no_delete_in_flight(monkeypatch) monkeypatch.setattr(pipe, "_newest_wins", no_conflict_policy) monkeypatch.setattr(mx, "_write_source_evidence", no_evidence) - monkeypatch.setattr(mx, "_draft_session_summary", nothing) monkeypatch.setattr(mx, "_link_written_memories_to_evidence", nothing) monkeypatch.setattr(mx, "write_memory_chunks", nothing) monkeypatch.setattr(mx, "_write_session_summary", nothing) diff --git a/tests/unit/test_extraction_queue.py b/tests/unit/test_extraction_queue.py index 83f093b488..767ee7b418 100644 --- a/tests/unit/test_extraction_queue.py +++ b/tests/unit/test_extraction_queue.py @@ -49,6 +49,11 @@ async def insert(self, table, rows, settings=None): async def query_all(self, sql, params=None): self.query_all_calls.append((sql, params or {})) + if "memory_refusals" in sql: + return [] + if params and "source_version" in params: + # Transition tests supply a stored source; missing and changed sources have dedicated cases. + return [{"found": 1}] return self.query_all_returns.pop(0) if self.query_all_returns else [] async def query_one(self, sql, params=None): @@ -130,6 +135,7 @@ async def test_enqueue_writes_one_full_pending_row() -> None: @pytest.mark.asyncio +@pytest.mark.usefixtures("no_blacklisted_values") async def test_handler_defer_short_circuits_before_extraction( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -153,6 +159,7 @@ async def _boom(*a, **k): @pytest.mark.asyncio +@pytest.mark.usefixtures("no_blacklisted_values") async def test_handler_defer_enqueue_failure_is_503(monkeypatch: pytest.MonkeyPatch) -> None: import server.memory.app.extract.handler as mx @@ -429,8 +436,9 @@ class _Out: assert res.done_count == 1 and res.written_memories == 3 assert res.failures == ["bad: failed"] assert all(inp.defer is False for inp in seen) - # inserts: 1 claim (2 rows) + done + failed - statuses = [r["status"] for _, rows, _ in ch.inserts[1:] for r in rows] + statuses = [ + r["status"] for _, rows, _ in ch.inserts for r in rows if r["status"] != q.STATUS_RUNNING + ] assert statuses == [q.STATUS_DONE, q.STATUS_FAILED] snap = get_registry().snapshot()["extraction_queue"] assert snap["last_outcome"]["done"] == 1 @@ -994,6 +1002,11 @@ def __init__(self, rows: list[dict[str, Any]]) -> None: async def query_all(self, sql: str, params: dict[str, Any] | None = None) -> Any: params = params or {} + if "memory_refusals" in sql: + return [] + if "source_version" in params: + row = self.rows.get(params["queue_id"]) + return [{"found": 1}] if row and row["version"] == params["source_version"] else [] self.sql.append(sql) now = datetime.now(UTC) held = { @@ -1070,7 +1083,7 @@ async def test_a_payload_only_enqueuer_waits_without_holding_another_enqueuer( pytest.skip("clickhouse binary not installed") a, _, _ = q.payload_for(_input(), scope=WHOLE_NAMESPACE, enqueued_by="process:sa:t:agent-a") b, _, _ = q.payload_for(_input(), scope=WHOLE_NAMESPACE, enqueued_by="process:sa:t:agent-b") - setup = [ddl("memory_extraction_queue")] + setup = [ddl("memory_extraction_queue"), ddl("memory_refusals"), ddl("erasure_requests")] for qid, payload, status, lease, age, column in ( ("a-held", a, "running", "now64(3) + INTERVAL 10 MINUTE", 3, held_column), ("a-next", a, "pending", "toDateTime64(0, 3)", 2, ""), @@ -1119,7 +1132,7 @@ async def write_drops(items: list[Any], stage: str, reason: str, *, ch: Any = No ([("o", "n", "h")], "extraction_queue", "held_past_ttl"), ([("o", "n", "p")], "extraction_queue", "waited_past_ttl"), ] - marked = ch.inserts[-1][1] + marked = [r for _, rows, _ in ch.inserts for r in rows] assert [r["status"] for r in marked] == [q.STATUS_FAILED, q.STATUS_FAILED] assert "(held: refused)" in marked[0]["last_error"] @@ -1152,7 +1165,7 @@ async def write_drops(items: list[Any], stage: str, reason: str, *, ch: Any = No monkeypatch.setattr(drops, "write_drops", write_drops) assert await q.drop_expiring(ch=ch) == 4 - quoted = [r["last_error"].rsplit(" (", 1)[1] for r in ch.inserts[-1][1]] + quoted = [r["last_error"].rsplit(" (", 1)[1] for _, rows, _ in ch.inserts for r in rows] assert quoted == [ "RuntimeError)", "unreachable: ConnectError)", diff --git a/tests/unit/test_extraction_retry_mine.py b/tests/unit/test_extraction_retry_mine.py index d9a66de971..a850acdc88 100644 --- a/tests/unit/test_extraction_retry_mine.py +++ b/tests/unit/test_extraction_retry_mine.py @@ -37,8 +37,24 @@ def __init__(self, rows: list[dict[str, Any]]) -> None: async def query_all(self, sql: str, params: dict[str, Any] | None = None) -> list[Any]: self.reads.append((sql, dict(params or {}))) - if "system.mutations" in sql: + if "system.mutations" in sql or "FROM loom.memory_refusals" in sql: return [] + if params and "source_version" in params: + from server.core.hashing import sha256_text + + return ( + [{"found": 1}] + if any( + all( + row.get(k) == params[k] + for k in ("org", "namespace", "agent", "user_id", "session_id", "queue_id") + ) + and row["version"] == params["source_version"] + and sha256_text(row.get("payload") or "").upper() == params["payload_sha"] + for row in self.rows + ) + else [] + ) return [dict(r) for r in self.rows] async def execute(self, sql: str, params: dict[str, Any] | None = None) -> None: @@ -157,6 +173,7 @@ def test_a_failed_replay_ends_failed_again(max_attempts: int) -> None: ch = _StubCH([_failed_row(attempts=max_attempts)]) _requeue(ch, ["q1"], max_attempts=max_attempts) row = _failed_row() | {"status": "pending", "attempts": ch.commands[0][1]["attempts"]} + ch.rows = [row] # The in-place retry's current stored revision. status = asyncio.run(q.mark_error(row, error="again", max_attempts=max_attempts, ch=ch)) assert status == "failed" and ch.inserted[-1]["attempts"] == max_attempts diff --git a/tests/unit/test_file_rules_shared.py b/tests/unit/test_file_rules_shared.py index 6ca494d174..9312927c0d 100644 --- a/tests/unit/test_file_rules_shared.py +++ b/tests/unit/test_file_rules_shared.py @@ -124,16 +124,25 @@ def test_version_row_builders_carry_files() -> None: class _CH: - def __init__(self) -> None: + def __init__(self, existing=None) -> None: self.inserts: list[tuple[str, list[dict[str, Any]]]] = [] + self.heads = {} + if existing is not None: + row = memory_repo.build_memory_version_row(existing) + self.heads[memory_repo.identity_key(row)] = row async def insert(self, table: str, rows: list[dict[str, Any]], **_: Any) -> None: self.inserts.append((table, rows)) + if table == "loom.memory": + self.heads.update((memory_repo.identity_key(r), dict(r)) for r in rows) async def query_one(self, *_: Any, **__: Any) -> dict[str, Any] | None: return None - async def query_all(self, *_: Any, **__: Any) -> list[dict[str, Any]]: + async def query_all(self, sql, params=None) -> list[dict[str, Any]]: + if "FROM loom.memory FINAL" in sql: + keys = set((params or {}).get("keys", self.heads)) + return [r for key, r in self.heads.items() if key in keys] return [] async def execute(self, *_: Any, **__: Any) -> None: @@ -151,7 +160,7 @@ def _wire(monkeypatch: pytest.MonkeyPatch, existing: memory_repo.MemoryRow | Non from server.memory.app import restore as _restore_mod from server.memory.app import stack as _stack_mod - ch = _CH() + ch = _CH(existing) monkeypatch.setattr(mem_handlers, "get_client", lambda: ch) monkeypatch.setattr(_stack_mod, "get_client", lambda: ch) monkeypatch.setattr(_restore_mod, "get_client", lambda: ch) diff --git a/tests/unit/test_high_rows_sql.py b/tests/unit/test_high_rows_sql.py index ad77d64921..e6f335f574 100644 --- a/tests/unit/test_high_rows_sql.py +++ b/tests/unit/test_high_rows_sql.py @@ -661,9 +661,12 @@ def row( "period_digest_s26_s30", "period_digest_s31_s35", } - assert await period_digest._skipped_period_keys(ch, ident, rewrite=False) == every + candidates = sorted(every | {"period_digest_s16_s20", "period_digest_s21_s25", "summary:s1"}) + assert await period_digest._skipped_period_keys(ch, ident, candidates, rewrite=False) == every taken_out = {"period_digest_s1_s5", "period_digest_s6_s10", "period_digest_s31_s35"} - assert await period_digest._skipped_period_keys(ch, ident, rewrite=True) == taken_out + assert ( + await period_digest._skipped_period_keys(ch, ident, candidates, rewrite=True) == taken_out + ) @pytest.mark.asyncio @@ -779,24 +782,44 @@ async def _heads(keys, **_k): @pytest.mark.asyncio async def test_memory_digests_never_serves_a_stored_credential_key_fact(monkeypatch) -> None: - # A digest an earlier build wrote may hold one: the read leaves it out. + # One unsafe source withholds the whole digest; a proven neighbor remains available. from server.memory.api.models_temporal import MemoryDigestsInput from server.memory.app import curation token = "ghp_" + "a" * 36 - ch = _LocalCH( - "SET enable_quantized_codec = 1", # the summary embedding column's codec - ddl("session_summaries"), - "INSERT INTO loom.session_summaries (org, namespace, agent, user_id, session_id, " - "summary_text, key_facts, summary_embedding) VALUES ('o', 'n', 'a', '-', 's', " - f"'a session', ['the token is {token}', 'ran the tests'], " - "arrayWithConstant(3072, toFloat32(0)))", - ) + import json + + from clickhouse_connect.driver.binding import format_query_value + + statements = ["SET enable_quantized_codec = 1", ddl("session_summaries"), ddl("memory")] + for session, key, text in ( + ("s", "vault", f"the token is {token}"), + ("s", "tests", "ran the tests"), + ("safe", "safe", "safe fact"), + ): + statements.append( + "INSERT INTO loom.memory (org, namespace, agent, user_id, session_id, memory_key, " + "value, memory_type, version) VALUES ('o','n','a','-'," + f"{format_query_value(session)}, {format_query_value(key)}, " + f"{format_query_value(json.dumps(text))}, 'semantic', 1)" + ) + for session, facts, keys in ( + ("s", [f"the token is {token}", "ran the tests"], ["vault", "tests"]), + ("safe", ["safe fact"], ["safe"]), + ): + statements.append( + "INSERT INTO loom.session_summaries (org, namespace, agent, user_id, session_id, " + "summary_text, key_facts, derived_from, summary_embedding) VALUES ('o','n','a','-'," + f"{format_query_value(session)}, {format_query_value(chr(10).join(facts))}, " + f"{format_query_value(facts)}, {format_query_value(keys)}, " + "arrayWithConstant(3072, toFloat32(0)))" + ) + ch = _LocalCH(*statements) monkeypatch.setattr("server.core.ch.get_client", lambda: ch) out = await curation.handle_memory_digests( MemoryDigestsInput(org="o", namespace="n", agent="a") ) - assert [r.key_facts for r in out.rows] == [["ran the tests"]] + assert [(r.session_id, r.key_facts) for r in out.rows] == [("safe", ["safe fact"])] @pytest.mark.asyncio diff --git a/tests/unit/test_import_evidence_refuses_blacklisted.py b/tests/unit/test_import_evidence_refuses_blacklisted.py index ce388dc42c..7bdca47946 100644 --- a/tests/unit/test_import_evidence_refuses_blacklisted.py +++ b/tests/unit/test_import_evidence_refuses_blacklisted.py @@ -24,6 +24,11 @@ class _CH: def __init__(self) -> None: self.inserts: dict[str, list[dict[str, Any]]] = {} + async def query_all(self, sql, params=None): + if "memory_refusals" in sql: + return [] + raise AssertionError(sql) + async def insert(self, table: str, rows: list[dict[str, Any]], **_: Any) -> None: self.inserts.setdefault(table, []).extend(rows) @@ -33,7 +38,21 @@ def wired(monkeypatch: pytest.MonkeyPatch) -> tuple[_CH, list[str]]: from server import embedding from server.memory.app import blacklist_guard as blacklist from server.memory.app import evidence_transfer as mod - + from tests.unit.memory_store import seed_summary_sources + + seed_summary_sources( + monkeypatch, + [ + { + "org": "acme", + "namespace": "ns", + "agent": "a", + "user_id": "-", + "session_id": "s1", + "summary_text": "a parade happened", + } + ], + ) ch = _CH() hits: list[str] = [] @@ -85,7 +104,12 @@ async def test_blacklisted_chunks_and_digests_are_dropped_not_written(wired) -> ), ], digests=[ - EvidenceDigest(session_id="s1", summary_text="a parade happened"), + EvidenceDigest( + session_id="s1", + summary_text="a parade happened", + key_facts=["a parade happened"], + derived_from=["source:s1"], + ), EvidenceDigest(session_id="s2", summary_text=f"key {SECRET} shared"), # key_facts are stored on the row too, so they are checked as well. EvidenceDigest(session_id="s3", summary_text="ok", key_facts=[SECRET]), @@ -121,3 +145,22 @@ async def test_a_chunk_the_blacklist_could_not_check_is_counted_apart(wired) -> ) ) assert (out.chunks_written, out.skipped, out.unchecked) == (0, 2, 1) + + +async def test_an_unproven_plain_digest_is_counted_as_skipped(wired): + from server.memory.api.models_evidence_transfer import EvidenceDigest, MemoryImportEvidenceInput + from server.memory.app.evidence_transfer import handle_memory_import_evidence + + ch, hits = wired + output = await handle_memory_import_evidence( + MemoryImportEvidenceInput( + org="acme", + namespace="ns", + agent="a", + digests=[ + EvidenceDigest(session_id="missing", summary_text="A plain but unproven passage") + ], + ) + ) + assert (output.digests_written, output.skipped) == (0, 1) + assert not ch.inserts and not hits diff --git a/tests/unit/test_in_use_coverage.py b/tests/unit/test_in_use_coverage.py index 079eb227ed..7f1e4cc7c5 100644 --- a/tests/unit/test_in_use_coverage.py +++ b/tests/unit/test_in_use_coverage.py @@ -24,6 +24,7 @@ _TRANSIENT = "a short-lived claim or projection in Keeper" EXEMPT: dict[str, str] = { + "loom.memory_refusals": "hash-only refusal policy intentionally survives namespace reuse; no content or outside target", "loom.namespaces": "the registry row, which every path reads before the check", "loom.analysis_canary": "the analysis probe's one row, at org '' and namespace '', a scope no tenant uses", # derived @@ -96,6 +97,7 @@ } PG_EXEMPT: dict[str, str] = { + "loom.memory_refusals": "hash-only refusal policy intentionally survives namespace reuse; no content or outside target", "loom.memory_deliveries": "which targets a revision still owes; beside its revision", "loom.erased_memories": "evidence that rows were erased; holds no content", "loom.erased_documents": "evidence that documents were erased; holds no content", diff --git a/tests/unit/test_key_axis_text.py b/tests/unit/test_key_axis_text.py index 956fcefbea..db178ef7ae 100644 --- a/tests/unit/test_key_axis_text.py +++ b/tests/unit/test_key_axis_text.py @@ -100,6 +100,11 @@ async def embed_many(self, texts: list[str]) -> list[list[float]]: class _CH: rows: list[dict[str, Any]] = [] + async def query_all(self, sql, params=None): + if "FROM loom.memory FINAL" in sql: + return [row for key, row in heads.items() if key in params["keys"]] + return [] + async def insert(self, table: str, rows: list[dict[str, Any]], **_: Any) -> None: self.rows = rows @@ -115,6 +120,9 @@ async def _one_group(rows: Any) -> dict[Any, list[int]]: stored = { "org": "o", "namespace": "n", + "agent": "a", + "user_id": "-", + "session_id": "-", "embedding_model": "m1", "embedding_dim": 2, "memory_key_embedding": [0.0, 0.0], @@ -125,6 +133,15 @@ async def _one_group(rows: Any) -> dict[Any, list[int]]: {**stored, "memory_key": digest, "content_excerpt": '"Prefers dark roast."'}, {**stored, "memory_key": "coffee_order", "content_excerpt": '"Dark." (#coffee_order)'}, ] + from server.core.hashing import sha256_value + from server.memory.store.rows import build_memory_version_row, identity_key + from tests.unit.test_memory_verify import _row + + heads = {} + for row, value in zip(chunk, ("Prefers dark roast.", "Dark."), strict=True): + source = _row(org="o", namespace="n", agent="a", memory_key=row["memory_key"], value=value) + heads[identity_key(source)] = build_memory_version_row(source) + row["content_sha256"] = sha256_value(value) emb, ch = _Emb(), _CH() await reembed._rewrite_page(ch, emb, "memory_embeddings", chunk, key_only=False) assert emb.batches == [['"Prefers dark roast."', '"Dark." (#coffee_order)'], ["coffee order"]] diff --git a/tests/unit/test_legacy_vector_retirement.py b/tests/unit/test_legacy_vector_retirement.py new file mode 100644 index 0000000000..e6768f9e16 --- /dev/null +++ b/tests/unit/test_legacy_vector_retirement.py @@ -0,0 +1,59 @@ +"""Retirement removes unknown legacy content without treating it as a publishable copy.""" + +import pytest + +from server.core.hashing import sha256_value +from server.memory.store import embeddings +from server.memory.store.rows import build_embedding_row, build_memory_version_row +from tests.unit.test_memory_verify import _row + + +@pytest.mark.parametrize("status", ["superseded", "rejected", "duplicate"]) +@pytest.mark.parametrize("vector_version", [100, 250]) +@pytest.mark.parametrize("hash_kind", ["missing", "different", "bytes"]) +async def test_a_retirement_tombstones_unhashed_vectors_below_its_version( + status, vector_version, hash_kind +): + source = _row(value="current fact", status=status, version=200) + canonical = build_memory_version_row(source) + vector = build_embedding_row( + **{ + k: canonical[k] + for k in ("org", "namespace", "agent", "user_id", "session_id", "memory_key") + }, + content_excerpt="legacy text", + content_sha256=( + "" + if hash_kind == "missing" + else sha256_value("legacy text").encode() + if hash_kind == "bytes" + else sha256_value("legacy text") + ), + embedding_model="legacy", + embedding_dim=1, + embedding=[0.5], + version=vector_version, + ) + + class CH: + def __init__(self): + self.inserted = [] + + async def query_all(self, sql, params=None): + if "FROM loom.memory_embeddings" in sql: + return [vector] + if params and "keys" in params: + return [canonical] + return [] + + async def insert(self, table, rows, **kwargs): + assert table == "loom.memory_embeddings" + self.inserted.extend(rows) + + ch = CH() + count = await embeddings.mirror_status_to_embeddings([canonical], ch=ch, strict=True) + assert count == len(ch.inserted) == int(vector_version < 200) + if ch.inserted: + (retired,) = ch.inserted + assert retired["tombstoned_at"] is not None and retired["version"] == 200 + assert retired["content_excerpt"] == "" and retired["embedding"] == [] diff --git a/tests/unit/test_lock_guard_consistent_read.py b/tests/unit/test_lock_guard_consistent_read.py index ea9465b394..1f4da85e92 100644 --- a/tests/unit/test_lock_guard_consistent_read.py +++ b/tests/unit/test_lock_guard_consistent_read.py @@ -39,10 +39,10 @@ def _lagging_replica(monkeypatch: pytest.MonkeyPatch) -> tuple[_CH, list[dict[st stale = _row() # the version before the lock locked = _row(version=101, lock_mode="read_only", lock_reason="hold", locked_by="human:o") - async def head_versions(*args: Any, **kw: Any) -> list[dict[str, Any]]: + async def head_versions(sql, params=None) -> list[dict[str, Any]]: from server.memory.store import build_memory_version_row - return [build_memory_version_row(locked)] + return [build_memory_version_row(locked)] if "FROM loom.memory FINAL" in sql else [] monkeypatch.setattr(ch, "query_all", head_versions) reads: list[dict[str, Any]] = [] @@ -195,6 +195,13 @@ async def test_a_write_right_after_unlock_sees_the_unlock(monkeypatch: pytest.Mo ch = _CH() locked = _row(lock_mode="read_only", lock_reason="hold", locked_by="human:o") unlocked = _row(version=102) + + async def head_versions(sql, params=None): + from server.memory.store import build_memory_version_row + + return [build_memory_version_row(unlocked)] if "FROM loom.memory FINAL" in sql else [] + + monkeypatch.setattr(ch, "query_all", head_versions) reads: list[dict[str, Any]] = [] monkeypatch.setattr(mem_handlers, "get_client", lambda: ch) monkeypatch.setattr(_stack_mod, "get_client", lambda: ch) diff --git a/tests/unit/test_memory_apply.py b/tests/unit/test_memory_apply.py index 8b4ddb59ba..7bdaa42572 100644 --- a/tests/unit/test_memory_apply.py +++ b/tests/unit/test_memory_apply.py @@ -114,6 +114,7 @@ def __init__(self, filter_rows: list[dict[str, Any]] | None = None) -> None: self.executes: list[tuple[str, dict[str, Any]]] = [] self.query_alls: list[tuple[str, dict[str, Any]]] = [] self.filter_rows = filter_rows or [] + self.heads = {} async def insert(self, table: str, rows: list[dict[str, Any]], **_: Any) -> None: self.inserts.append((table, rows)) @@ -125,6 +126,10 @@ async def query_all( self, sql: str, params: dict[str, Any] | None = None, **_: Any ) -> list[dict[str, Any]]: self.query_alls.append((sql, params or {})) + if "FROM loom.memory_refusals" in sql: + return [] + if "FROM loom.memory FINAL" in sql and "keys" in (params or {}): + return [self.heads[key] for key in params["keys"] if key in self.heads] return self.filter_rows async def query_one(self, *_: Any, **__: Any) -> dict[str, Any] | None: @@ -132,6 +137,9 @@ async def query_one(self, *_: Any, **__: Any) -> dict[str, Any] | None: def _stub(monkeypatch: pytest.MonkeyPatch, rows_by_key: dict[str, MemoryRow], ch: _CH) -> None: + from server.memory.store.rows import build_memory_version_row, identity_key + + ch.heads = {identity_key(row): build_memory_version_row(row) for row in rows_by_key.values()} monkeypatch.setattr("server.core.ch.get_client", lambda: ch) monkeypatch.setattr("server.memory.app.handlers.get_client", lambda: ch) monkeypatch.setattr("server.memory.app.stack.get_client", lambda: ch) @@ -174,8 +182,9 @@ async def test_enforcing_delete_goes_through_the_handler(monkeypatch: pytest.Mon ) ) assert out.applied == 1 and out.rows[0].status == "applied" and out.rows[0].version - table, rows = ch.inserts[0] + table, rows = next((table, rows) for table, rows in ch.inserts if table == "loom.memory") assert table == "loom.memory" and rows[0]["tombstoned_at"] is not None + assert ch.inserts[0][0] == "loom.memory_refusals" @pytest.mark.asyncio diff --git a/tests/unit/test_memory_ask_deep_evidence.py b/tests/unit/test_memory_ask_deep_evidence.py index c1fd6df1e6..aff9f7b377 100644 --- a/tests/unit/test_memory_ask_deep_evidence.py +++ b/tests/unit/test_memory_ask_deep_evidence.py @@ -472,145 +472,13 @@ async def test_fetch_with_no_keys_does_not_query() -> None: assert ch.params is None -# ---- session-summary hits open their session, windowed by the question ------ - -from server.memory.app.deep_evidence import ( # noqa: E402 - fetch_session_turns, - session_ids_of, - window_turns_by_question, -) - - -def test_session_ids_of_reads_summary_keys_only() -> None: - assert session_ids_of(["m1", "summary:s1", "summary:s1", "summary:", "summary:s2"]) == [ - "s1", - "s2", - ] - - -def _turn(i: int, content: str) -> dict[str, Any]: - return { - "session_id": "s1", - "role": "user", - "observed_at": f"2026-01-01 00:00:{i:02d}", - "content": content, - } - - -def test_window_keeps_scoring_turns_with_neighbours_in_order() -> None: - rows = [ - _turn(0, "hello there"), - _turn(1, "we booked the Denver trip for March"), - _turn(2, "great, bring a jacket"), - _turn(3, "unrelated chatter about lunch"), - _turn(4, "also the concert in Denver was Brandon Flowers"), - _turn(5, "nice"), - ] - kept = window_turns_by_question(rows, "What did I do in Denver?", max_chars=10_000) - idx = [rows.index(r) for r in kept] - assert idx == [0, 1, 2, 3, 4, 5] or idx == sorted(idx) # chronological - assert 1 in idx and 4 in idx # both scoring turns - assert 3 in idx # neighbour of turn 4 - - -def test_window_respects_the_character_cap() -> None: - rows = [_turn(i, f"Denver turn {i} " + "x" * 200) for i in range(10)] - kept = window_turns_by_question(rows, "Denver", max_chars=700) - assert 0 < len(kept) <= 3 - assert sum(len(r["content"]) for r in kept) <= 700 - - -def test_window_returns_nothing_without_a_scoring_turn() -> None: - rows = [_turn(0, "hello"), _turn(1, "goodbye")] - assert window_turns_by_question(rows, "Denver trip", max_chars=1000) == [] - - -class _OneReadCH: - """A single statement answered with the rows given (the session-turn read).""" - - def __init__(self, rows: list[dict[str, Any]]) -> None: - self.rows = rows - self.params: dict[str, Any] | None = None - - async def query_all( - self, sql: str, params: dict[str, Any] | None = None - ) -> list[dict[str, Any]]: - self.params = params - return self.rows - - -async def test_fetch_session_turns_labels_with_the_summary_key_and_caps() -> None: - ch = _OneReadCH( - [ - { - "session_id": "s1", - "role": "user", - "observed_at": "2026-01-01 00:00:01", - "content": "Denver plan " + "a" * 100, - }, - { - "session_id": "s1", - "role": "assistant", - "observed_at": "2026-01-01 00:00:02", - "content": "ok", - }, - { - "session_id": "s2", - "role": "user", - "observed_at": "2026-01-02 00:00:01", - "content": "Denver again " + "b" * 100, - }, - ] - ) - turns = await fetch_session_turns( - org="o", - namespace="ns", - session_ids=["s1", "s2"], - question="Denver", - max_chars_per_session=1000, - max_chars_total=150, - ch=ch, - ) - assert ch.params["sids"] == ["s1", "s2"] - assert turns and turns[0].memory_key == "summary:s1" and turns[0].support_type == "session" - # 150 total: s1's window (114 + 2 chars) fits, s2 does not. - assert [t.memory_key for t in turns] == ["summary:s1", "summary:s1"] - - -async def test_deep_pass_opens_a_picked_session_summary(env) -> None: - """End to end through memory.ask: the selector picks the summary hit, its - session is fetched and rendered under the summary key.""" - env.search_output = _output( - [ - _hit("summary:s9", "Session about the Denver trip and a concert.", 0.9), - _hit("m1", "User's favorite color is blue.", 0.8), - ] - ) - env.set_llm( - _ScriptedLLM( - ["Blue [m1].", "Brandon Flowers in Denver [summary:s9]."], picks='{"picks": [0]}' - ) - ) - - async def fake_sessions(**kw: Any) -> list[EvidenceTurn]: - env.session_calls = kw - return [ - EvidenceTurn( - "summary:s9", - "user", - "2026-01-02 10:00:00", - "saw Brandon Flowers in Denver", - "session", - ) - ] - - env.monkeypatch.setattr(deep, "fetch_session_turns", fake_sessions) - out = await _run(query="who did I see in Denver", depth="raw") - assert env.session_calls["session_ids"] == ["s9"] - assert env.session_calls["question"] == "who did I see in Denver" - assert env.fetch_calls[0]["memory_keys"] == [] # no fact keys were picked - assert out.depth == "raw" - assert "[summary:s9] (user) saw Brandon Flowers" in env.llm.synth_calls[-1][1] +async def test_a_summary_hit_does_not_open_unproven_session_text(env) -> None: + env.search_output = _output([_hit("summary:s9", "a verified digest", 0.9)]) + env.set_llm(_ScriptedLLM(["A fact [summary:s9]."], picks='{"picks": [0]}')) + env.turns = [] + await _run(query="what happened", depth="raw") + assert env.fetch_calls[0]["memory_keys"] == [] + assert len(env.llm.synth_calls) == 1 # ---- the temporal trigger's selector is a setting ---- diff --git a/tests/unit/test_memory_clear.py b/tests/unit/test_memory_clear.py index df32308efd..bb8cdeccaf 100644 --- a/tests/unit/test_memory_clear.py +++ b/tests/unit/test_memory_clear.py @@ -530,15 +530,20 @@ def page(sql, params): .on("SELECT max(version) AS v", [{"v": 0}]) .on("SELECT count() AS n FROM loom.memory FINAL", [{"n": 0}], times=1) ) + ch.on("FROM loom.memory_refusals", []) ch.on("SELECT count() AS n", [{"n": 2}]) ch.on("SELECT * FROM loom.memory FINAL", page) ch.on("SELECT DISTINCT cityHash64", []) - ch.on( - "PREWHERE (org, namespace, agent, user_id, session_id, memory_key)", - lambda sql, params: [ - r for r in ch.inserted("memory") if lifecycle.identity_key(r) in params["keys"] - ], - ) + + def heads(sql, params): + current = {} + for row in [*rows, *ch.inserted("memory")]: + key = lifecycle.identity_key(row) + if key not in current or row["version"] >= current[key]["version"]: + current[key] = row + return [current[key] for key in params["keys"] if key in current] + + ch.on("PREWHERE (org, namespace, agent, user_id, session_id, memory_key)", heads) monkeypatch.setattr(lifecycle, "reemit_embedding_rows_for_keys", AsyncMock(return_value=0)) return ch.install(monkeypatch) diff --git a/tests/unit/test_memory_consolidate_handler.py b/tests/unit/test_memory_consolidate_handler.py index 96237e7278..c81cdd38ea 100644 --- a/tests/unit/test_memory_consolidate_handler.py +++ b/tests/unit/test_memory_consolidate_handler.py @@ -463,6 +463,7 @@ def test_tombstone_embeddings_for_key_mirrors_every_column_and_sets_the_stamp() async def fake_insert(rows, *, ch=None): inserted.extend(rows) + return list(rows) ts = datetime(2026, 9, 11, 12, 0, 0) with patch.object(embeddings, "insert_embedding_rows", new=fake_insert): diff --git a/tests/unit/test_memory_delete_tombstones_all_types.py b/tests/unit/test_memory_delete_tombstones_all_types.py index a6dc82748d..22f3bfca9b 100644 --- a/tests/unit/test_memory_delete_tombstones_all_types.py +++ b/tests/unit/test_memory_delete_tombstones_all_types.py @@ -21,7 +21,12 @@ from server.memory.app import handlers as mem_handlers from server.memory.app import restore as _restore_mod from server.memory.app import stack as _stack_mod -from server.memory.store import EMBEDDING_PERSISTED_COLS, MemoryRow +from server.memory.store import ( + EMBEDDING_PERSISTED_COLS, + MemoryRow, + build_memory_version_row, + identity_key, +) _NOW = datetime(2026, 9, 18, 12, 0, 0) @@ -43,11 +48,11 @@ def _mk_row(**overrides: Any) -> MemoryRow: "created_at": _NOW - timedelta(days=1), "expires_at": None, "valid_at": None, - "run_id": "", + "run_id": "-", "tags": ["alert"], "entities": [], "temporal_anchor": None, - "reinforcement_count": 0, + "reinforcement_count": 1, "last_reinforced_at": None, "source_resource_id": "", } @@ -98,6 +103,7 @@ def __init__( emb_rows: list[dict[str, Any]] | None = None, fail_inserts_into: tuple[str, ...] = (), ) -> None: + self.source: MemoryRow | None = None self.emb_rows = emb_rows or [] self.fail_inserts_into = fail_inserts_into self.insert_calls: list[tuple[str, list[dict[str, Any]], dict[str, Any]]] = [] @@ -107,6 +113,9 @@ async def query_all( self, sql: str, params: dict[str, Any] | None = None ) -> list[dict[str, Any]]: self.query_all_sqls.append(sql) + if params and "keys" in params and self.source is not None: + if identity_key(self.source) in params["keys"]: + return [build_memory_version_row(self.source)] if "FROM loom.memory_embeddings" in sql: return [dict(r) for r in self.emb_rows] return [] @@ -126,6 +135,7 @@ def rows_for(self, table: str) -> list[dict[str, Any]]: async def _delete(monkeypatch: pytest.MonkeyPatch, existing: MemoryRow, ch: _StubCH) -> Any: + ch.source = existing monkeypatch.setattr(mem_handlers, "get_client", lambda: ch) monkeypatch.setattr(_stack_mod, "get_client", lambda: ch) monkeypatch.setattr(_restore_mod, "get_client", lambda: ch) diff --git a/tests/unit/test_memory_digests_op.py b/tests/unit/test_memory_digests_op.py index 90d7de96cc..4e9fd0f213 100644 --- a/tests/unit/test_memory_digests_op.py +++ b/tests/unit/test_memory_digests_op.py @@ -21,7 +21,45 @@ async def query_all( self, sql: str, params: dict[str, Any] | None = None ) -> list[dict[str, Any]]: self.sql, self.params = sql, dict(params or {}) - return self.rows[: int(self.params["lim"])] + rows = self.rows + if after := self.params.get("after"): + rows = [ + row + for row in rows + if ( + row["created_at"], + row["session_id"], + row.get("agent", "-"), + row.get("user_id", "-"), + row.get("embedding_model", ""), + ) + > after + ] + return rows[: int(self.params["lim"])] + + +def _wire_sources(monkeypatch, rows): + from tests.unit.test_memory_verify import _row + + for row in rows: + row.update(agent="a", user_id="-", derived_from=["k1"], key_facts=[row["summary_text"]]) + + async def read(keys, **kwargs): + from server.memory.store.rows import identity_key + + parents = [ + _row( + namespace="n", + agent="a", + session_id=r["session_id"], + memory_key="k1", + value=r["summary_text"], + ) + for r in rows + ] + return {identity_key(r): r for r in parents if identity_key(r) in keys} + + monkeypatch.setattr("server.memory.store.summary_sources.memory_rows_by_identity_keys", read) def test_op_is_registered_for_viewers() -> None: @@ -45,6 +83,7 @@ async def test_wildcard_lists_all_sessions_and_pages(monkeypatch: pytest.MonkeyP } for i in range(3) ] + _wire_sources(monkeypatch, rows) ch = _FakeCH(rows) monkeypatch.setattr("server.core.ch.get_client", lambda: ch) out = await cur.handle_memory_digests( @@ -52,7 +91,7 @@ async def test_wildcard_lists_all_sessions_and_pages(monkeypatch: pytest.MonkeyP ) assert "session_id = {s:String}" not in ch.sql and ch.params["lim"] == 3 assert [r.session_id for r in out.rows] == ["ns:session_0", "ns:session_1"] and out.truncated - assert out.rows[0].key_facts == ["a"] and out.rows[0].derived_from == ["k1"] + assert out.rows[0].key_facts == ["digest 0"] and out.rows[0].derived_from == ["k1"] @pytest.mark.asyncio @@ -67,7 +106,7 @@ async def test_concrete_session_filters(monkeypatch: pytest.MonkeyPatch) -> None @pytest.mark.asyncio -async def test_a_digest_that_reads_as_a_credential_is_listed_without_its_text( +async def test_a_digest_that_reads_as_a_credential_is_withheld( monkeypatch: pytest.MonkeyPatch, ) -> None: from tests.unit._fake_credentials import OPENAI_KEY @@ -76,6 +115,30 @@ async def test_a_digest_that_reads_as_a_credential_is_listed_without_its_text( {"session_id": "ns:s1", "summary_text": f"The user pasted {OPENAI_KEY}.", "key_facts": []}, {"session_id": "ns:s2", "summary_text": "Planned the retreat.", "key_facts": []}, ] + _wire_sources(monkeypatch, rows) monkeypatch.setattr("server.core.ch.get_client", lambda: _FakeCH(rows)) out = await cur.handle_memory_digests(MemoryDigestsInput(org="o", namespace="n", agent="a")) - assert [r.summary_text for r in out.rows] == ["", "Planned the retreat."] + assert [r.summary_text for r in out.rows] == ["Planned the retreat."] + + +@pytest.mark.parametrize("withheld", [2, 270]) +@pytest.mark.parametrize("qualified", [1, 2]) +async def test_withheld_pages_do_not_hide_current_digests(monkeypatch, withheld, qualified): + rows = [ + { + "session_id": f"s{i:04}", + "agent": "a", + "user_id": "-", + "embedding_model": "test", + "created_at": datetime(2026, 1, 1), + "summary_text": f"fact {i}", + } + for i in range(withheld + qualified) + ] + _wire_sources(monkeypatch, rows[withheld:]) + monkeypatch.setattr("server.core.ch.get_client", lambda: _FakeCH(rows)) + out = await cur.handle_memory_digests( + MemoryDigestsInput(org="o", namespace="n", agent="a", limit=1) + ) + assert [row.session_id for row in out.rows] == [rows[withheld]["session_id"]] + assert out.truncated is (qualified > 1) diff --git a/tests/unit/test_memory_drops.py b/tests/unit/test_memory_drops.py index af9bfe1ed3..3e71f945b0 100644 --- a/tests/unit/test_memory_drops.py +++ b/tests/unit/test_memory_drops.py @@ -303,6 +303,7 @@ async def handle_consolidate(inp): @pytest.mark.asyncio +@pytest.mark.usefixtures("no_blacklisted_values") async def test_schema_mode_records_stubbed_fields(monkeypatch: pytest.MonkeyPatch) -> None: import server.memory.app.extract.handler as mx from server.llm.client import LLMResult diff --git a/tests/unit/test_memory_expiry.py b/tests/unit/test_memory_expiry.py index e258229c4e..3e78d92709 100644 --- a/tests/unit/test_memory_expiry.py +++ b/tests/unit/test_memory_expiry.py @@ -83,6 +83,8 @@ def __init__(self) -> None: async def query_all(self, sql: str, params: Any = None) -> list[dict[str, Any]]: assert sequential_reads_active() or "select_sequential_consistency = 1" in sql + if "FROM loom.memory_refusals" in sql: + return [] return [self.heads[tuple(k)] for k in params["keys"] if tuple(k) in self.heads] async def insert(self, table: str, rows: list[dict[str, Any]], **_: Any) -> None: @@ -477,6 +479,7 @@ async def test_batched_reemit_reads_and_inserts_once_per_batch( async def _insert(rows, *, ch=None): ch_.inserts.append(rows) + return list(rows) ch_ = ch monkeypatch.setattr(embeddings, "insert_embedding_rows", _insert) @@ -771,8 +774,9 @@ async def _live(ch, keys, **_: Any) -> list[dict[str, Any]]: inserted: list[dict[str, Any]] = [] - async def _insert(rows, *, ch=None) -> None: + async def _insert(rows, *, ch=None): inserted.extend(rows) + return list(rows) async def _none(*_a: Any, **_k: Any) -> int: return 0 diff --git a/tests/unit/test_memory_fence_publication.py b/tests/unit/test_memory_fence_publication.py index 39c36ab9c2..93699cdf66 100644 --- a/tests/unit/test_memory_fence_publication.py +++ b/tests/unit/test_memory_fence_publication.py @@ -45,6 +45,8 @@ async def test_contended_value_write_publishes_no_embedding( class CH: async def query_all(self, *args, **kwargs): + if "FROM loom.memory_refusals" in args[0]: + return [] return [_row().model_dump()] async def insert(self, table, rows, **kw): @@ -129,6 +131,8 @@ async def test_unanswered_high_memory_send_still_fences_search(monkeypatch): class LostAnswer: async def query_all(self, *args, **kwargs): + if "FROM loom.memory_refusals" in args[0]: + return [] return [] async def insert(self, *args, **kw): @@ -167,6 +171,8 @@ async def test_high_write_cancelled_during_holder_release_still_fences_search( class CH: async def query_all(self, *args, **kwargs): + if "FROM loom.memory_refusals" in args[0]: + return [] return [_row().model_dump()] async def insert(self, table, batch, **kw): @@ -214,6 +220,8 @@ async def test_cancelled_high_send_fences_and_preserves_the_cancellation(monkeyp class CH: async def query_all(self, *args, **kwargs): + if "FROM loom.memory_refusals" in args[0]: + return [] return [] async def insert(self, *args, **kw): @@ -245,6 +253,8 @@ async def test_copy_commit_mark_and_followups_survive_holder_release_cancellatio class CH: async def query_all(self, *args, **kwargs): + if "FROM loom.memory_refusals" in args[0]: + return [] return [_row().model_dump()] async def insert(self, *args, **kw): @@ -334,6 +344,8 @@ async def follow(name, *args, **kw): class CH: async def query_all(self, *args, **kwargs): + if "FROM loom.memory_refusals" in args[0]: + return [] return [_row().model_dump()] async def insert(self, table, batch, **kw): @@ -462,6 +474,8 @@ async def test_acknowledged_copy_finishes_followups_under_repeated_cancellation( class CH: async def query_all(self, *args, **kwargs): + if "FROM loom.memory_refusals" in args[0]: + return [] return [_row().model_dump()] async def insert(self, *args, **kw): @@ -518,6 +532,8 @@ async def test_bulk_embedding_failure_still_announces_acknowledged_memory(monkey class CH: async def query_all(self, *args, **kwargs): + if "FROM loom.memory_refusals" in args[0]: + return [] return [_row().model_dump()] async def insert(self, table, rows, **kw): @@ -547,6 +563,8 @@ async def test_bulk_followup_finishes_under_repeated_cancellation(monkeypatch, c class CH: async def query_all(self, *args, **kwargs): + if "FROM loom.memory_refusals" in args[0]: + return [] return [_row().model_dump()] async def insert(self, *args, **kw): diff --git a/tests/unit/test_memory_metadata_contention.py b/tests/unit/test_memory_metadata_contention.py index beb167e39b..4f53d5e5f7 100644 --- a/tests/unit/test_memory_metadata_contention.py +++ b/tests/unit/test_memory_metadata_contention.py @@ -32,6 +32,11 @@ async def insert(self, table: str, batch: list[dict[str, Any]], **kw: Any) -> No self.table.extend(batch) async def query_all(self, sql: str, params: Any = None) -> list[dict[str, Any]]: + if "FROM loom.memory_refusals" in sql: + from server.core.ch import sequential_reads_active + + assert sequential_reads_active() + return [] assert "loom.memory" in sql assert "select_sequential_consistency = 1" in sql heads = {} @@ -245,9 +250,11 @@ async def insert(self, *args: Any, **kw: Any) -> None: with pytest.raises(TimeoutError): await insert_memory_versions(batch, ch=LostAnswer(table)) - assert set(memory_insert_keeper.keys) == { - write_fence.lease_name(rows.identity_key(row)) for row in batch - } + from server.memory.store.refusals import fence_names + + assert set(memory_insert_keeper.keys) == ( + {write_fence.lease_name(rows.identity_key(row)) for row in batch} | set(fence_names(batch)) + ) with pytest.raises(lease_writes.FenceUnavailableError): await insert_memory_versions(batch[:1], ch=Replica(table)) assert len(table) == 2 @@ -386,11 +393,17 @@ async def pause(delay): monkeypatch.setattr(lease_writes.leases, "release_shielded", release) monkeypatch.setattr(write_fence, "_admission_wait_s", lambda: wait_s) row = memory_repo.build_memory_version_row(_row()) + from server.memory.store import refusals + + names = {write_fence.lease_name(rows.identity_key(row)), *refusals.fence_names([row])} + count = len(names) + assert count > 1 # Key and consent admission share this deadline. if admitted: await insert_memory_versions([row], ch=client) else: with pytest.raises(lease_writes.FenceUnavailableError): await insert_memory_versions([row], ch=client) assert client.insert.await_count == int(admitted) - assert attempts == (2 if stage == "retry" and admitted else 1) - assert release.await_count == int(admitted or stage == "take") + assert attempts == count * (2 if stage == "retry" and admitted else 1) + expected_releases = count if stage == "take" else count - 1 + (count if admitted else 0) + assert release.await_count == expected_releases diff --git a/tests/unit/test_memory_move.py b/tests/unit/test_memory_move.py index 445048acb7..aa35736b33 100644 --- a/tests/unit/test_memory_move.py +++ b/tests/unit/test_memory_move.py @@ -22,7 +22,7 @@ import server.memory.app.handlers as mem_handlers import server.memory.app.move as mv -from server.core.hashing import sha256_value_json +from server.core.hashing import sha256_value, sha256_value_json from server.identity.domain.read_scope import WHOLE_NAMESPACE from server.memory import store as memory_repo from server.memory.api.models_crud import MemoryGetInput, MemorySetOutput @@ -100,6 +100,7 @@ def _source(**over: Any) -> MemoryRow: "session_id": "-", "memory_key": _FROM, "content_excerpt": "ReplicaLag ch-3 42 (#alert_3f9c1a2b)", + "content_sha256": sha256_value(_OLD_VALUE), "embedding_model": "fake-test", "embedding_dim": 3, "embedding": [0.1, 0.2, 0.3], @@ -142,17 +143,24 @@ def __init__(self, routes: list[tuple[str, list[dict[str, Any]]]] | None = None) self.query_all_sqls: list[str] = [] self.routes = routes or [] self.fail_inserts_into: tuple[str, ...] = () + self.memory_heads: dict[Any, Any] = {} async def insert(self, table: str, rows: list[dict[str, Any]], settings: Any = None) -> None: if table in self.fail_inserts_into: raise RuntimeError(f"insert into {table} unavailable") self.inserts.append((table, [dict(r) for r in rows], settings or {})) + if table == "loom.memory": + from server.memory.store.rows import identity_key + + self.memory_heads.update({identity_key(row): row for row in rows}) async def query_one(self, *_: Any, **__: Any) -> dict[str, Any] | None: return None async def query_all(self, sql: str, params: Any = None, **_: Any) -> list[dict[str, Any]]: self.query_all_sqls.append(sql) + if "FROM loom.memory FINAL" in sql and "PREWHERE (org, namespace" in sql: + return [self.memory_heads[key] for key in params["keys"] if key in self.memory_heads] for needle, result in self.routes: if needle in sql: return [dict(r) for r in result] @@ -187,6 +195,10 @@ def __init__( set_out: MemorySetOutput | None = None, ) -> None: self.ch = ch or _ch_with_mirror_and_chunks() + if source is not None: + from server.memory.store.rows import identity_key + + self.ch.memory_heads[identity_key(source)] = source.model_dump() self.set_inputs: list[Any] = [] self.delete_inputs: list[Any] = [] self.get_by_key_calls: list[tuple[memory_repo.Identity, str]] = [] @@ -217,7 +229,7 @@ async def _delete(inp: Any) -> Any: monkeypatch.setattr( "server.memory.app.write.handle_memory_set", write_mod.handle_memory_set ) - monkeypatch.setattr(mem_handlers, "handle_memory_delete", _delete) + monkeypatch.setattr(mv, "_tombstone_source", _delete) monkeypatch.setattr(mv, "get_client", lambda: self.ch) @@ -342,7 +354,7 @@ def test_keep_redirect_false_takes_the_delete_path_and_writes_no_redirect( d = h.delete_inputs[0] assert (d.memory_key, d.agent, d.user_id, d.session_id) == (_FROM, "sre", "-", "-") assert out.redirected is False and out.memory_id == _DEST_ID - assert h.ch.inserts == [] # the delete path (stubbed) owns the source's rows + assert h.ch.inserts == [] # Internal retirement (stubbed) owns the source's rows. def test_a_merge_that_wrote_nothing_still_redirects_to_the_live_destination_row( @@ -622,3 +634,42 @@ def test_the_op_is_writer_audited_not_idempotent_and_listed_under_all_only() -> assert "memory.move" not in _MCP_CODING_OPS assert "memory.move" not in _MCP_ATOMIC_OPS assert op_toolsets("memory.move") == ["all"] + + +@pytest.mark.parametrize("cascade_fails", [False, True]) +async def test_internal_move_retirement_announces_its_landed_tombstone(monkeypatch, cascade_fails): + from unittest.mock import AsyncMock + + source = _source() + write = AsyncMock() + retire = AsyncMock(side_effect=RuntimeError("cascade unavailable") if cascade_fails else None) + announce = AsyncMock(return_value=False) + monkeypatch.setattr(mem_handlers, "_write_tombstone_row", write) + monkeypatch.setattr(mem_handlers, "_retire_derived", retire) + monkeypatch.setattr(mv.webhooks, "announce", announce) + if cascade_fails: + with pytest.raises(RuntimeError, match="cascade unavailable"): + await mv._tombstone_source(source) + else: + await mv._tombstone_source(source) + assert write.await_args.kwargs["exact"] is True + assert not write.await_args.kwargs.get("refuse_value", False) + assert announce.await_args.args[0][0][0] == "memory.tombstoned" + + +def test_a_move_preserves_a_source_written_while_the_destination_was_created(monkeypatch): + from server.memory.store.pg_memory import AdmissionContendedError + from server.memory.store.rows import build_memory_version_row, identity_key + + source = _source() + h = _Harness(monkeypatch, source=source) + newer = source.model_copy(update={"value": "a newer fact", "version": source.version + 100}) + + async def set_destination(inp): + h.ch.memory_heads[identity_key(newer)] = build_memory_version_row(newer) + return h.set_out + + monkeypatch.setattr(write_mod, "handle_memory_set", set_destination) + with pytest.raises(AdmissionContendedError): + _move(_inp()) + assert h.ch.rows_for("loom.memory") == [] diff --git a/tests/unit/test_memory_outline_backlinks_op.py b/tests/unit/test_memory_outline_backlinks_op.py index 613f06f143..a5d0796ef1 100644 --- a/tests/unit/test_memory_outline_backlinks_op.py +++ b/tests/unit/test_memory_outline_backlinks_op.py @@ -35,6 +35,16 @@ async def query_all( return [] +@pytest.fixture +def qualified_outline_sources(monkeypatch): + """Outline formatting receives qualified rows; source proof has separate cases.""" + + async def qualified(ch, rows, **kwargs): + return rows + + monkeypatch.setattr("server.memory.store.summary_sources.current_summary_sources", qualified) + + def test_ops_are_registered_for_viewers_and_classed_as_reads() -> None: from server.insights.app.op_mix import classify @@ -49,12 +59,13 @@ def test_ops_are_registered_for_viewers_and_classed_as_reads() -> None: @pytest.mark.asyncio +@pytest.mark.usefixtures("qualified_outline_sources") async def test_outline_composes_counts_sessions_hubs_communities_and_profile( monkeypatch: pytest.MonkeyPatch, ) -> None: ch = _RoutedCH( { - "loom.session_summaries FINAL WHERE org = {o:String} AND namespace = {n:String} AND tombstoned_at IS NULL ORDER BY": [ + "coalesce(valid_at, created_at)": [ { "session_id": "ns:s4", "summary_text": "Discussed the parish workshop. Then planned the retreat.", @@ -118,6 +129,15 @@ async def test_outline_skips_the_profile_read_for_the_dash_user( assert not any("memory_type = 'profile'" in sql for sql, _ in ch.calls) +async def test_outline_withholds_a_legacy_digest_without_source_proof(monkeypatch): + ch = _RoutedCH( + {"coalesce(valid_at, created_at)": [{"session_id": "s", "summary_text": "An old claim"}]} + ) + monkeypatch.setattr("server.core.ch.get_client", lambda: ch) + out = await cur.handle_memory_outline(MemoryOutlineInput(org="o", namespace="n", agent="a")) + assert out.sessions == [] + + def test_headline_takes_the_first_sentence_or_clips() -> None: assert cur._headline("One. Two.") == "One." assert cur._headline("x" * 300) == "x" * 160 @@ -187,6 +207,7 @@ async def test_backlinks_of_an_unknown_key_is_empty_not_an_error( @pytest.mark.asyncio +@pytest.mark.usefixtures("qualified_outline_sources") async def test_outline_shows_no_headline_for_a_digest_that_reads_as_a_credential( monkeypatch: pytest.MonkeyPatch, ) -> None: diff --git a/tests/unit/test_memory_pipeline.py b/tests/unit/test_memory_pipeline.py index f2838e1ed6..aec697e37f 100644 --- a/tests/unit/test_memory_pipeline.py +++ b/tests/unit/test_memory_pipeline.py @@ -240,6 +240,7 @@ async def test_run_pipeline_aborts_on_extraction_error( fail-open one: downstream steps do not run, the error reaches the caller (the handler maps it to 502).""" import server.memory.app.extract as mx + from server.memory.app import pipeline from server.memory.app.pipeline import register_step, run_pipeline from server.memory.app.pipeline_state import PipelineStep @@ -253,6 +254,7 @@ async def _sentinel(state, config): ran.append("after") return state + monkeypatch.setattr(pipeline, "_REGISTRY", dict(pipeline._REGISTRY)) register_step("test.after_extract", _sentinel) state = WorkflowState( org="dev", namespace="ns", agent="a", messages=[{"role": "user", "content": "x"}] diff --git a/tests/unit/test_memory_refusals.py b/tests/unit/test_memory_refusals.py new file mode 100644 index 0000000000..df201ec864 --- /dev/null +++ b/tests/unit/test_memory_refusals.py @@ -0,0 +1,462 @@ +"""Deleted content cannot return through another key or a containing passage.""" + +import pytest + +from server.memory.domain.refusals import fingerprint, matches, tokens + + +@pytest.mark.parametrize("native", [False, True]) +def test_erasure_captures_clickhouse_history_under_both_authorities(monkeypatch, native): + from server.core import pg + from server.erasure.store import refusals + + monkeypatch.setattr(pg, "enabled", lambda: native) + assert "loom.memory_versions" in dict(refusals._sources()) + + +@pytest.mark.parametrize("user_id", [None, "alice"]) +async def test_native_erasure_keeps_cleared_revision_identity_without_fingerprinting_null( + monkeypatch, user_id +): + from types import SimpleNamespace + + from server.erasure.store import consent, refusals, steps_pg + + identity = {"org": "o", "namespace": "n", "agent": "a", "user_id": "alice", "memory_key": "k"} + payload = {**identity, "value": '"current fact"', "value_json": "{}"} + recorded, dependents = [], [] + + class Connection: + async def cursor(self, sql, *args): + assert "coalesce(r.memory_key, r.row->>'memory_key'" in sql + yield {**identity, "row": None} + yield {**identity, "row": payload} + + async def record(conn, rows, **kwargs): + assert all(isinstance(row, dict) for row in rows) + recorded.extend(rows) + + async def capture(conn, rows, uid): + assert uid == "alice" + dependents.extend(rows) + + monkeypatch.setattr(consent, "current_policy", lambda: SimpleNamespace(record_postgres=record)) + monkeypatch.setattr(refusals, "capture_native_chunks", capture) + await steps_pg._capture_history(Connection(), "r.org=$1", "org=$1", ["o"], user_id) + assert recorded == [payload] + assert dependents == ([identity, identity] if user_id else []) + + +@pytest.mark.parametrize("value", [None, "", 0, False, {"b": 2, "a": 1}, [1, 2]]) +def test_canonical_values_are_refused(value): + assert matches(value, fingerprint(value)) + assert matches({"wrapped": value}, fingerprint(value)) + assert not matches("unrelated", fingerprint(value)) + + +def test_normalized_literal_is_refused_inside_later_text(): + policy = fingerprint("My code is ABC-123") + assert matches("Reminder: MY code is abc-123. Do not repeat it.", policy) + assert not matches("My code is abc-124", policy) + + +def test_oversized_scan_refuses_instead_of_allowing_a_prefix(): + from server.core.errors import ValidationError + + with pytest.raises(ValidationError): + matches("other " * 250_001, fingerprint("forgotten value")) + + +@pytest.mark.parametrize("value", ["alpha", None, False, 17, 0, [], {}, {"k": "v"}]) +def test_persisted_scalar_and_object_values_use_the_served_column(value): + import json + + from server.core.hashing import served_row_value + + stored = { + "value": json.dumps(value), + "value_json": json.dumps(value) if isinstance(value, dict) else "{}", + } + assert served_row_value(stored) == value + assert matches(served_row_value(stored), fingerprint(value)) + + +def test_absent_raw_metadata_does_not_reintroduce_a_deleted_null(): + from server.core.hashing import served_row_value + from server.memory.store.refusals import content_policy_row + + row = content_policy_row({"org": "o", "namespace": "n"}, ["an unrelated passage"]) + assert not matches(served_row_value(row), fingerprint(None)) + + +@pytest.mark.parametrize("metadata", [None, "", "{}", " { } ", {}, [], "null"]) +def test_default_raw_metadata_does_not_freeze_unrelated_intake(metadata): + from server.memory.store.refusals import _raw_content_values + + values = _raw_content_values({"content": "an unrelated passage", "metadata": metadata}) + assert values == ["an unrelated passage"] + + +@pytest.mark.parametrize("content", ["null", "{}", "[]"]) +def test_erasure_keeps_literal_content_distinct_from_empty_metadata(monkeypatch, content): + import json + + from server.memory.store import refusals as policy + + monkeypatch.setattr(policy, "_writer", lambda: "process:test") + row = {"org": "o", "namespace": "n", "value_json": "{}", "value": json.dumps(content)} + captured = policy._policy([row]) + patterns = [(r["kind"], r["units"], r["digest"]) for r in captured] + assert matches(policy._raw_content_values({"content": content}), patterns) + + +@pytest.mark.parametrize("metadata", [" { } ", " [ ] ", "null", '""', ""]) +def test_erasure_ignores_only_decoded_empty_metadata(monkeypatch, metadata): + from server.memory.store import refusals as policy + + monkeypatch.setattr(policy, "_writer", lambda: "process:test") + row = { + "org": "o", + "namespace": "n", + "value_json": "{}", + "value": metadata, + "capture_kind": "metadata", + } + assert policy._policy([row]) == [] + + +def test_erased_structured_metadata_matches_later_metadata(monkeypatch): + import json + + from server.memory.store import refusals as policy + + monkeypatch.setattr(policy, "_writer", lambda: "process:test") + metadata = json.dumps({"secret": "copper lark"}) + row = { + "org": "o", + "namespace": "n", + "value_json": "{}", + "value": metadata, + "capture_kind": "metadata", + } + captured = policy._policy([row]) + patterns = [(r["kind"], r["units"], r["digest"]) for r in captured] + assert matches( + policy._raw_content_values({"content": "unrelated", "metadata": metadata}), patterns + ) + + +async def test_candidate_lookups_fit_the_http_parameter_limit(): + from clickhouse_connect.driver.binding import format_bind_value + + from server.core.ch_params import FIELD_BYTES + from server.memory.store import refusals as policy + + batches = [] + + class CH: + async def query_all(self, sql, params=None): + if "SELECT DISTINCT" in sql: + return [{"units": 1}] + batch = params["candidates"] + assert len(format_bind_value(batch).encode()) <= FIELD_BYTES + batches.append(batch) + return [] + + value = " ".join(f"word{i}" for i in range(2100)) + await policy.check_clickhouse( + CH(), [policy.content_policy_row({"org": "o", "namespace": "n"}, value)] + ) + assert len(batches) >= 3 + assert sum(len(batch) for batch in batches) >= 2100 + + +async def test_a_current_source_hash_cannot_publish_a_refused_excerpt(monkeypatch): + from server.core.errors import ValidationError + from server.core.hashing import sha256_value + from server.memory.store import refusals as policy + + source = policy.content_policy_row({"org": "o", "namespace": "n"}, "current fact") + excerpt = "retired secret" + row = {**source, "content_sha256": sha256_value("current fact"), "content_excerpt": excerpt} + patterns = set(fingerprint(excerpt)) + + class CH: + async def query_all(self, sql, params=None): + if "SELECT DISTINCT" in sql: + return [{"units": 2}] + return [{"found": 1}] if patterns.intersection(params["candidates"]) else [] + + async def heads(*args, **kwargs): + return [row], [source] + + monkeypatch.setattr(policy, "_embedding_sources", heads) + with pytest.raises(ValidationError, match="refused"): + async with policy.publication(CH(), [row], embeddings=True): + pytest.fail("The refused excerpt reached its publisher") + + +@pytest.mark.parametrize("native_bytes", [False, True]) +async def test_embedding_admission_accepts_the_current_fixed_string_hash(monkeypatch, native_bytes): + from server.core.hashing import sha256_value + from server.memory.store import refusals as policy + from server.memory.store.rows import identity_key + from tests.unit.test_memory_verify import _row + + source = _row(value="current fact") + digest = sha256_value(source.value) + vector = { + **source.model_dump(), + "content_sha256": digest.encode() if native_bytes else digest, + } + + async def heads(*args, **kwargs): + return {identity_key(source): source} + + monkeypatch.setattr("server.memory.store.reads.memory_rows_by_identity_keys", heads) + admitted, sources = await policy._embedding_sources(object(), [vector]) + assert admitted == [vector] and len(sources) == 1 + + +@pytest.mark.parametrize( + "change", [{"value": "a replacement"}, {"status": "rejected"}, {"tombstoned_at": "2026-01-01"}] +) +async def test_embedding_admission_withholds_a_changed_or_deleted_source(monkeypatch, change): + from server.core.hashing import sha256_value + from server.memory.store import refusals as policy + from server.memory.store.rows import identity_key + from tests.unit.test_memory_verify import _row + + source = _row(value="current fact") + vector = {**source.model_dump(), "content_sha256": sha256_value(source.value)} + current = source.model_copy(update=change) + + async def heads(*args, **kwargs): + return {identity_key(current): current} + + monkeypatch.setattr("server.memory.store.reads.memory_rows_by_identity_keys", heads) + assert await policy._embedding_sources(object(), [vector]) == ([], []) + + +@pytest.mark.parametrize("entry", ["handler", "queue"]) +async def test_prior_transcript_messages_cannot_bypass_refusal(monkeypatch, entry): + from server.core.errors import ValidationError + from server.identity.domain.read_scope import WHOLE_NAMESPACE + from server.memory.api.models_bulk import MemorySetFromMessagesInput + from server.memory.app import extraction_queue + from server.memory.app.extract import handler + + secret = "The retired phrase is copper lark" + patterns = set(fingerprint(secret)) + + class CH: + async def query_all(self, sql, params=None): + if "SELECT DISTINCT" in sql: + return [{"units": len(tokens(secret))}] + return [{"found": 1}] if patterns.intersection(params["candidates"]) else [] + + async def insert(self, *args, **kwargs): + pytest.fail("Refused context reached persistent storage") + + ch = CH() + monkeypatch.setattr(handler, "get_client", lambda: ch) + request = MemorySetFromMessagesInput( + org="o", + namespace="n", + agent="a", + defer=True, + messages=[{"role": "user", "content": "hello"}], + last_messages=[{"role": "user", "content": secret}], + ) + with pytest.raises(ValidationError, match="refused"): + if entry == "handler": + await handler.handle_memory_set_from_messages(request) + else: + await extraction_queue.enqueue(request, scope=WHOLE_NAMESPACE, ch=ch) + + +def test_every_extraction_field_has_an_explicit_content_classification(): + from server.memory.api.models_bulk import MemorySetFromMessagesInput + from server.memory.domain.refusals import EXTRACTION_CONTENT_FIELDS, EXTRACTION_PROTOCOL_FIELDS + + assert not EXTRACTION_CONTENT_FIELDS & EXTRACTION_PROTOCOL_FIELDS + assert EXTRACTION_CONTENT_FIELDS | EXTRACTION_PROTOCOL_FIELDS == set( + MemorySetFromMessagesInput.model_fields + ) + + +@pytest.mark.parametrize( + "field,value", + [ + ("extraction_instructions", "retired copper lark"), + ("session_values", ["retired copper lark"]), + ("extra_tags", ["retired copper lark"]), + ("recalled_keys", ["retired copper lark"]), + ("record_schema", {"choice": ["retired copper lark"]}), + ("pipeline_steps", [{"step": "custom", "note": "retired copper lark"}]), + ("observation_date", "retired copper lark"), + ("messages", [{"role": "user", "name": "retired copper lark", "content": "hello"}]), + ("messages", [{"role": "retired copper lark", "content": "hello"}]), + ], +) +@pytest.mark.parametrize("entry", ["queue", "transition", "capture"]) +async def test_all_queued_text_obeys_the_same_refusal_policy(monkeypatch, field, value, entry): + import json + + from server.core.errors import ValidationError + from server.identity.domain.read_scope import WHOLE_NAMESPACE + from server.memory.api.models_bulk import MemorySetFromMessagesInput + from server.memory.app import extraction_queue as queue + from server.memory.store import refusals as policy + from tests.unit.test_extraction_queue import _stored_row + + patterns = set(fingerprint("retired copper lark")) + + class CH: + def __init__(self): + self.ended = [] + + async def query_all(self, sql, params=None): + if "SELECT DISTINCT" in sql: + return [{"units": 3}] + return [{"found": 1}] if patterns.intersection(params["candidates"]) else [] + + async def insert(self, *args, **kwargs): + pytest.fail("Refused queue text was copied") + + async def execute(self, sql, params=None): + assert "UPDATE status='failed'" in sql + assert "retired copper lark" not in sql + repr(params) + self.ended.append(params) + + request = MemorySetFromMessagesInput( + **{ + "org": "o", + "namespace": "n", + "agent": "a", + "messages": [{"role": "user", "content": "hello"}], + field: value, + } + ) + ch = CH() + if entry == "capture": + monkeypatch.setattr(policy, "_writer", lambda: "process:test") + rows = policy._policy( + [ + { + "org": "o", + "namespace": "n", + "value_json": "{}", + "value": request.model_dump_json(), + "capture_kind": "payload", + } + ] + ) + assert patterns <= {(r["kind"], r["units"], r["digest"]) for r in rows} + elif entry == "transition": + await queue.mark_done( + _stored_row(payload=json.dumps(request.model_dump())), written=0, ch=ch + ) + assert len(ch.ended) == 1 + else: + with pytest.raises(ValidationError, match="refused"): + await queue.enqueue(request, scope=WHOLE_NAMESPACE, ch=ch) + + +async def test_erasure_records_an_indexed_refusal_for_a_derived_rows_own_identity(monkeypatch): + from server.memory.store import refusals as policy + + monkeypatch.setattr(policy, "_writer", lambda: "process:test") + row = policy.content_policy_row( + {"org": "o", "namespace": "n", "user_id": "bob"}, "retired copper lark" + ) + recorded = policy._recording_policy([row], scope_kind="user_id", scope_value="alice") + assert {(r["scope_kind"], r["scope_value"], r["user_id"]) for r in recorded} == { + ("user_id", "alice", "bob"), + ("erased_identity", "n", "bob"), + } + + +def test_queue_protocol_defaults_do_not_freeze_unrelated_intake(): + from server.memory.domain.refusals import payload_fingerprints, transcript_content + + old = { + "messages": [{"role": "user", "content": "old passage"}], + "record_schema": {"old_field": "string"}, + "pipeline_steps": [{"step_id": "loom.llm_extract"}], + } + patterns = payload_fingerprints(old) + unrelated = { + "messages": [{"role": "user", "content": "unrelated passage"}], + "record_schema": {"new_field": "string"}, + "pipeline_steps": [{"step_id": "loom.llm_extract"}], + } + assert not matches(transcript_content(unrelated), patterns) + for default in ("user", "string", "loom.llm_extract", "role", "content", "step_id"): + assert not matches(default, patterns) + + +@pytest.mark.parametrize( + "value", [4417, False, {"private": [4417, "copper lark"]}, [4417, "copper lark"]] +) +def test_typed_queue_arguments_are_checked_and_captured_as_values(value): + from server.memory.domain.refusals import payload_fingerprints, transcript_content + + payload = {"pipeline_steps": [{"step_id": "loom.llm_extract", "config": {"arg": value}}]} + assert matches(transcript_content(payload), fingerprint(value)) + assert set(fingerprint(value)) <= payload_fingerprints(payload) + + +async def test_a_refused_stale_queue_transition_cannot_stop_a_newer_source_revision(): + from server.core.hashing import sha256_text + from server.memory.app import extraction_queue as queue + from tests.unit.test_extraction_queue import _stored_row + + original = _stored_row(version=10) + stale = queue._mirror(original, last_error="retired copper lark") + current = {**original, "version": 15} + + class CH: + async def execute(self, sql, params=None): + assert "version={source_version:UInt64}" in sql + assert "hex(SHA256(payload))={payload_sha:String}" in sql + assert params["source_version"] == 10 + assert params["payload_sha"] == sha256_text(original["payload"]).upper() + if current["version"] == params["source_version"]: + current["status"] = "failed" + + await queue._drop_refused(CH(), stale) + assert current["status"] == original["status"] + + +def test_known_pipeline_ids_are_classified_as_protocol(): + from server.memory.app.pipeline import registered_step_ids + from server.memory.domain.refusals import EXTRACTION_PROTOCOL_STEPS + + assert EXTRACTION_PROTOCOL_STEPS == set(registered_step_ids()) + + +@pytest.mark.parametrize( + "step", + [ + {"step_id": "retired copper lark"}, + {"step_id": "loom.hash_dedup", "retired copper lark": "ordinary"}, + ], +) +def test_unknown_step_ids_and_caller_argument_names_remain_content(step): + from server.memory.domain.refusals import payload_fingerprints, transcript_content + + payload = {"pipeline_steps": [step]} + patterns = set(fingerprint("retired copper lark")) + assert matches(transcript_content(payload), patterns) + assert patterns <= payload_fingerprints(payload) + + +@pytest.mark.parametrize("value", [None, False, 0, [], {}, ""]) +@pytest.mark.parametrize("field", ["step_id", "config"]) +def test_explicit_empty_and_typed_step_fields_are_not_protocol_defaults(field, value): + from server.memory.domain.refusals import payload_fingerprints, transcript_content + + payload = {"pipeline_steps": [{field: value}]} + assert matches(transcript_content(payload), fingerprint(value)) + assert set(fingerprint(value)) <= payload_fingerprints(payload) diff --git a/tests/unit/test_memory_reindex_mirror.py b/tests/unit/test_memory_reindex_mirror.py index d58ef5cd5d..cfd6acd38e 100644 --- a/tests/unit/test_memory_reindex_mirror.py +++ b/tests/unit/test_memory_reindex_mirror.py @@ -13,6 +13,7 @@ import pytest +from server.core.hashing import sha256_value from server.embedding._provider import MalformedEmbedding from server.memory import store as mr @@ -27,6 +28,26 @@ def __init__(self, rows: list[dict[str, Any]]) -> None: async def query_all(self, sql: str, params: dict[str, Any] | None = None) -> list[Any]: self.sql.append(sql) self.params.append(params or {}) + if "FROM loom.memory_refusals" in sql: + return [] + if "FROM loom.memory FINAL" in sql and "PREWHERE (org, namespace" in sql: + from server.memory.store import MemoryRow, build_memory_version_row, identity_key + + heads = [ + MemoryRow.from_snapshot( + { + **r, + "value": r["content_excerpt"], + "memory_id": "11111111-1111-1111-1111-111111111111", + } + ) + for r in self.rows + ] + return [ + build_memory_version_row(row) + for row in heads + if identity_key(row) in params["keys"] + ] if "max(version) AS ev" in sql: return [] # no embedding row of the key yet return self.rows @@ -57,6 +78,7 @@ def _memory_mirror_row() -> dict[str, Any]: "memory_key": "k", "memory_type": "preference", "content_excerpt": "x" * 600, + "content_sha256": sha256_value("x" * 600), "run_id": "r", "entities": ["e"], "tags": ["t"], @@ -125,6 +147,23 @@ def test_a_reindexed_row_carries_its_memory_rows_columns(monkeypatch: Any) -> No assert row["embedding_model"] == "fake-model" +def test_a_withheld_reindex_is_not_counted_or_indexed(monkeypatch: Any) -> None: + ch = _CH([_memory_mirror_row()]) + indexed: list[Any] = [] + + async def withheld(rows: Any, **_: Any) -> list[Any]: + assert len(rows) == 1 + return [] + + async def index(rows: Any, **_: Any) -> None: + indexed.extend(rows) + + monkeypatch.setattr(mr, "insert_embedding_rows", withheld) + monkeypatch.setattr(mr, "index_late_rows", index) + assert _run(monkeypatch, ch).processed == 0 + assert indexed == [] and ch.inserted == [] + + def test_the_read_skips_high_sensitivity_rows_and_current_keys_unless_forced( monkeypatch: Any, ) -> None: @@ -170,7 +209,15 @@ async def embed_many(self, texts: list[str]) -> list[list[float]]: def test_reindex_embeds_memory_sets_texts_and_pairs_the_key_vector(monkeypatch: Any) -> None: from server.memory.domain.embed_text import embedding_texts - rows = [dict(_memory_mirror_row(), memory_key=k, content_excerpt=f"v {k}") for k in "ab"] + rows = [ + dict( + _memory_mirror_row(), + memory_key=k, + content_excerpt=f"v {k}", + content_sha256=sha256_value(f"v {k}"), + ) + for k in "ab" + ] ch = _CH(rows) emb = _RecordingEmbedder() _run_with(monkeypatch, ch, emb) @@ -286,3 +333,12 @@ async def _index(rows: Any, **_k: Any) -> int: ch = _CH([dict(_memory_mirror_row())]) assert _run(monkeypatch, ch).processed == 1 assert len(handed) == 1 and handed[0][0]["memory_key"] == "k" + + +def test_reindex_repairs_an_unhashed_canonical_source_without_attesting_it(monkeypatch): + source = _memory_mirror_row() | {"content_sha256": "", "verified": 1} + ch = _CH([source]) + assert _run(monkeypatch, ch).processed == 1 + row = ch.inserted[0][1][0] + assert row["content_sha256"] == sha256_value(source["content_excerpt"]) + assert row["verified"] == 0 diff --git a/tests/unit/test_memory_set_chunks.py b/tests/unit/test_memory_set_chunks.py index a14e701f84..c3735111f0 100644 --- a/tests/unit/test_memory_set_chunks.py +++ b/tests/unit/test_memory_set_chunks.py @@ -30,6 +30,7 @@ from server.memory.domain.conflicts import ConflictResult from server.memory.domain.excerpt import EXCERPT_LIMIT from server.memory.store import MemoryRow +from server.memory.store.rows import build_memory_version_row, identity_key from server.settings import settings @@ -69,17 +70,24 @@ def __init__(self, routes: list[tuple[str, list[dict[str, Any]]]] | None = None) self.query_all_sqls: list[str] = [] self.routes = routes or [] self.fail_inserts_into: tuple[str, ...] = () + self.heads = {} async def insert(self, table: str, rows: list[dict[str, Any]], settings: Any = None) -> None: if table in self.fail_inserts_into: raise RuntimeError(f"insert into {table} unavailable") self.inserts.append((table, rows, settings or {})) + if table == "loom.memory": + self.heads.update({identity_key(row): dict(row) for row in rows}) async def query_one(self, *_: Any, **__: Any) -> dict[str, Any] | None: return None async def query_all(self, sql: str, params: Any = None, **_: Any) -> list[dict[str, Any]]: self.query_all_sqls.append(sql) + if "FROM loom.memory_refusals" in sql: + return [] + if "FROM loom.memory FINAL" in sql and "keys" in (params or {}): + return [self.heads[key] for key in params["keys"] if key in self.heads] for needle, result in self.routes: if needle in sql: return result @@ -463,6 +471,7 @@ async def test_memory_delete_tombstones_the_keys_chunks(monkeypatch: pytest.Monk async def _get_by_key(*_: Any, **__: Any) -> MemoryRow: return existing + ch.heads[identity_key(existing)] = build_memory_version_row(existing) monkeypatch.setattr("server.memory.store.get_by_key", _get_by_key) out = await mem_handlers.handle_memory_delete( MemoryDeleteInput(org="test", namespace="ops-alerts", agent="sre", memory_key=_KEY) @@ -507,6 +516,7 @@ async def test_memory_delete_of_an_episodic_row_reads_no_chunks( async def _get_by_key(*_: Any, **__: Any) -> MemoryRow: return existing + ch.heads[identity_key(existing)] = build_memory_version_row(existing) monkeypatch.setattr("server.memory.store.get_by_key", _get_by_key) await mem_handlers.handle_memory_delete( MemoryDeleteInput(org="test", namespace="ops-alerts", agent="sre", memory_key=_KEY) @@ -662,6 +672,7 @@ async def _announce(changes: Any) -> None: async def _retire(*_: Any, **__: Any) -> tuple[int, int]: raise RuntimeError("summary downgrade failed") + ch.heads[identity_key(existing)] = build_memory_version_row(existing) monkeypatch.setattr("server.memory.store.get_by_key", _get_by_key) monkeypatch.setattr("server.memory.app.webhooks.announce", _announce) monkeypatch.setattr(mem_handlers, "_retire_derived", _retire) @@ -752,7 +763,12 @@ async def test_a_writer_retires_only_its_own_and_legacy_chunks() -> None: ch=ch, embedder=_BatchEmbedder(), ) - assert [p["uid"] for p in ch.params if "uid" in p] == ["u-7"] + chunk_params = [ + params + for sql, params in zip(ch.query_all_sqls, ch.params, strict=True) + if "FROM loom.memory_chunks FINAL" in sql + ] + assert [params["uid"] for params in chunk_params] == ["u-7"] @pytest.mark.asyncio @@ -785,6 +801,7 @@ async def test_memory_delete_retires_the_deleted_rows_users_chunks( async def _get_by_key(*_: Any, **__: Any) -> MemoryRow: return existing + ch.heads[identity_key(existing)] = build_memory_version_row(existing) monkeypatch.setattr("server.memory.store.get_by_key", _get_by_key) await mem_handlers.handle_memory_delete( MemoryDeleteInput( @@ -811,6 +828,9 @@ async def _tombstone(**kw: Any) -> int: return 1 class _CH: + async def query_all(self, *args, **kwargs): + return [] + async def insert(self, *_a: Any, **_k: Any) -> None: return None @@ -847,6 +867,9 @@ async def _tombstone(**kw: Any) -> int: return 1 class _CH: + async def query_all(self, *args, **kwargs): + return [] + async def insert(self, *_a: Any, **_k: Any) -> None: return None diff --git a/tests/unit/test_memory_set_verdict.py b/tests/unit/test_memory_set_verdict.py index 7ce9de5a73..f0501cba83 100644 --- a/tests/unit/test_memory_set_verdict.py +++ b/tests/unit/test_memory_set_verdict.py @@ -25,6 +25,7 @@ from server.memory.app import write as write_mod from server.memory.domain.conflicts import ConflictPeer, ConflictResult from server.memory.store import MemoryRow +from server.memory.store.rows import identity_key from server.observability.event_args import project_result _NOW = datetime(2026, 9, 14, 12, 0, 0) @@ -34,16 +35,21 @@ class _CH: def __init__(self) -> None: self.inserts: list[tuple[str, list[dict[str, Any]]]] = [] + self.heads = {} async def insert(self, table: str, rows: list[dict[str, Any]], **_: Any) -> None: """The graph-write marks a scheduled projection flushes are not the write under test.""" if table not in ("loom.namespace_write_marks", "loom.graph_projections"): self.inserts.append((table, rows)) + if table == "loom.memory": + self.heads.update({identity_key(row): dict(row) for row in rows}) async def query_one(self, *_: Any, **__: Any) -> dict[str, Any] | None: return None - async def query_all(self, *_: Any, **__: Any) -> list[dict[str, Any]]: + async def query_all(self, sql: str, params: Any = None, **__: Any) -> list[dict[str, Any]]: + if "FROM loom.memory FINAL" in sql and "keys" in (params or {}): + return [self.heads[key] for key in params["keys"] if key in self.heads] return [] async def execute(self, *_: Any, **__: Any) -> None: diff --git a/tests/unit/test_memory_update_conflict.py b/tests/unit/test_memory_update_conflict.py index 188b65bf1e..399477e399 100644 --- a/tests/unit/test_memory_update_conflict.py +++ b/tests/unit/test_memory_update_conflict.py @@ -70,6 +70,8 @@ async def insert(self, table: str, rows: list[dict[str, Any]], **_: Any) -> None self.inserts.append((table, rows)) async def query_all(self, sql: str, params: Any = None, **kwargs: Any) -> list[dict[str, Any]]: + if "FROM loom.memory_refusals" in sql: + return [] if ( "FROM loom.memory FINAL" in sql and "PREWHERE (org, namespace, agent, user_id, session_id, memory_key)" in sql diff --git a/tests/unit/test_memory_verify.py b/tests/unit/test_memory_verify.py index 18905f76d6..ef13ff4bf1 100644 --- a/tests/unit/test_memory_verify.py +++ b/tests/unit/test_memory_verify.py @@ -453,9 +453,9 @@ async def boom(r: MemoryRow, *, verified: bool) -> None: async def test_the_mirror_re_emits_every_live_row_of_the_key( monkeypatch: pytest.MonkeyPatch, ) -> None: - """One row per (scope, embedding model): each is re-emitted whole with `verified` set, + """One private row per embedding model is re-emitted whole with `verified` set, keeping the columns the verify input never carried (reinforcement, mention times).""" - from server.memory.store import EMBEDDING_PERSISTED_COLS + from server.memory.store import EMBEDDING_PERSISTED_COLS, build_memory_version_row row = _row() stored = [ @@ -467,6 +467,7 @@ async def test_the_mirror_re_emits_every_live_row_of_the_key( "user_id": row.user_id, "session_id": row.session_id, "memory_key": row.memory_key, + "content_sha256": _sha(row.value), "scope": scope, "embedding_model": model, "embedding_dim": 2, @@ -475,12 +476,16 @@ async def test_the_mirror_re_emits_every_live_row_of_the_key( "mentioned_at": datetime(2026, 1, 3), "version": 7, } - for scope, model in (("private", "m1"), ("shared", "m2")) + for scope, model in (("private", "m1"), ("private", "m2")) ] inserted: list[dict[str, Any]] = [] class _CH: async def query_all(self, sql: str, params: dict[str, Any] | None = None) -> Any: + if "FROM loom.memory_refusals" in sql: + return [] + if "FROM loom.memory FINAL" in sql: + return [build_memory_version_row(row)] assert "FROM loom.memory_embeddings FINAL" in sql return [dict(r) for r in stored] @@ -492,7 +497,7 @@ async def insert(self, table: str, rows: list[dict[str, Any]], **_: Any) -> None await mv._mirror_verified_to_embeddings(row, verified=True) assert [(r["scope"], r["embedding_model"]) for r in inserted] == [ ("private", "m1"), - ("shared", "m2"), + ("private", "m2"), ] assert all(r["verified"] == 1 and r["version"] > 7 for r in inserted) assert all(r["mentioned_at"] == datetime(2026, 1, 3) for r in inserted) diff --git a/tests/unit/test_model_call_customer.py b/tests/unit/test_model_call_customer.py index 28f26927a9..8d1820f605 100644 --- a/tests/unit/test_model_call_customer.py +++ b/tests/unit/test_model_call_customer.py @@ -366,19 +366,29 @@ def test_the_fakes_check_the_customer_as_a_metered_provider_would(metered) -> No asyncio.run(FakeNGramEmbedder(dim=8).embed_many(["text"])) -def test_a_reflection_is_written_for_the_org_of_its_pair(metered) -> None: +def test_a_reflection_is_written_for_the_org_of_its_pair(metered, monkeypatch) -> None: # The contrast call and the reflection's write (which embeds) are both that org's. from server.memory.app import session_outcomes as so seen: list[tuple[str, str]] = [] - neg = {"org": "acme", "namespace": "ns", "session_id": "s1", "summary_embedding": [0.1]} + from tests.unit.memory_store import seed_summary_sources + + neg = { + "org": "acme", + "namespace": "ns", + "session_id": "s1", + "summary_embedding": [0.1], + "summary_text": "failed step", + } + pos = {**neg, "session_id": "s2", "similarity": 0.99, "summary_text": "ok"} + seed_summary_sources(monkeypatch, [neg, pos]) class _CH: async def query_all(self, sql: str, params: Any) -> list[Any]: return [neg] async def query_one(self, sql: str, params: Any) -> Any: - return {"session_id": "s2", "similarity": 0.99, "summary_text": "ok"} + return pos class _LLM: async def complete(self, *a: Any, **k: Any) -> Any: @@ -1084,11 +1094,15 @@ async def link(**kw: Any) -> None: return None monkeypatch.setattr("server.memory.app.evidence.link_belief_sources", link) + from tests.unit.memory_store import seed_summary_sources + + digests = _digests(1, 5) + seed_summary_sources(monkeypatch, digests) cfg = PeriodDigestConfig(sessions_per_period=5, namespace_filter="ns", require_opt_in=False) res = asyncio.run( run_period_digest( cfg, - ch=_FakeCH(_digests(1, 5)), + ch=_FakeCH(digests), llm=_MeteredLLM("Between 2023-07-01 and 2023-07-05, Caroline painted."), embedder=FakeNGramEmbedder(dim=3), ) diff --git a/tests/unit/test_model_call_holds.py b/tests/unit/test_model_call_holds.py index 8fa461293e..9f36a6bad0 100644 --- a/tests/unit/test_model_call_holds.py +++ b/tests/unit/test_model_call_holds.py @@ -82,12 +82,13 @@ async def find(self, *, batch_size: int, **_: Any) -> Any: candidates = sorted(live, key=lambda r: -r["version"])[: batch_size * 4] return [dict(r) for r in sorted(candidates, key=lambda r: r["created_at"])[:batch_size]] - async def insert(self, rows: list[dict[str, Any]], **_: Any) -> None: + async def insert(self, rows: list[dict[str, Any]], **_: Any) -> list[dict[str, Any]]: self.embedded.update(r["memory_key"] for r in rows if r["embedding_dim"] > 0) for row in rows: self.rows = [ dict(row) if old["memory_key"] == row["memory_key"] else old for old in self.rows ] + return rows async def writers(self, org: str, keys: Any, **_: Any) -> dict[Any, Any]: # a* were written by agent-a, b* by agent-b, into the shared namespace diff --git a/tests/unit/test_model_call_recovery.py b/tests/unit/test_model_call_recovery.py index 0e840741ce..4e4b5cee8b 100644 --- a/tests/unit/test_model_call_recovery.py +++ b/tests/unit/test_model_call_recovery.py @@ -30,7 +30,25 @@ async def embed_many(self, texts): seen.append(customer.headers(await customer.require())) return [[1.0, 0.0] for _ in texts] + from server.core.hashing import sha256_value + from server.memory.store.rows import build_memory_version_row + from tests.unit.test_memory_verify import _row + + source = _row( + org="acme", + namespace=AGENT_NS, + agent="a", + user_id="u1", + memory_key="k1", + value="agent-8 wrote this", + ) + class CH: + async def query_all(self, sql, params=None): + if "FROM loom.memory FINAL" in sql: + return [build_memory_version_row(source)] + return [] + async def insert(self, *args, **kwargs): pass @@ -42,6 +60,7 @@ async def insert(self, *args, **kwargs): session_id="-", memory_key="k1", content_excerpt="agent-8 wrote this", + content_sha256=sha256_value(source.value), embedding_model="old", embedding=[1.0, 0.0], memory_key_embedding=[1.0, 0.0], diff --git a/tests/unit/test_model_call_writers.py b/tests/unit/test_model_call_writers.py index 74142524cb..f39d19cb79 100644 --- a/tests/unit/test_model_call_writers.py +++ b/tests/unit/test_model_call_writers.py @@ -307,6 +307,10 @@ def test_a_reflection_judges_a_session_for_its_writer(writers: _Directory, monke "key_facts": [], } + from tests.unit.memory_store import seed_summary_sources + + seed_summary_sources(monkeypatch, [summary]) + class _CH: async def query_all(self, sql: str, params: Any = None) -> list[dict[str, Any]]: return [summary] diff --git a/tests/unit/test_model_calls_held_mid_pass.py b/tests/unit/test_model_calls_held_mid_pass.py index 314104e50d..7a0aa9b518 100644 --- a/tests/unit/test_model_calls_held_mid_pass.py +++ b/tests/unit/test_model_calls_held_mid_pass.py @@ -280,13 +280,11 @@ def test_the_period_digest_records_no_failure_when_its_call_is_held(monkeypatch, async def idents(ch: Any, cfg: Any) -> list[dict[str, Any]]: return [{"org": "acme", "namespace": "n", "agent": "a", "user_id": "u"}] - async def nothing(*a: Any, **k: Any) -> Any: - return set() + async def planned(*a: Any, **k: Any) -> Any: + return [period], 0 monkeypatch.setattr(pd, "_fetch_namespaces", idents) - monkeypatch.setattr(pd, "_fetch_digests", nothing) - monkeypatch.setattr(pd, "_skipped_period_keys", nothing) - monkeypatch.setattr(pd, "plan_periods", lambda *a, **k: ([period], 0)) + monkeypatch.setattr(pd, "_plan_digest_pages", planned) monkeypatch.setattr(pd, "build_period_prompt", lambda p: ("system", "user")) llm = SimpleNamespace(complete=_raise(error)) result = asyncio.run(pd.run_period_digest(ch=object(), llm=llm, embedder=object())) diff --git a/tests/unit/test_model_calls_switch.py b/tests/unit/test_model_calls_switch.py index 8f902b2127..eb95a51245 100644 --- a/tests/unit/test_model_calls_switch.py +++ b/tests/unit/test_model_calls_switch.py @@ -435,8 +435,9 @@ async def find_stubs(*, exclude_orgs: frozenset[str] = frozenset(), **_: Any) -> dict(s) for s in stubs if s["org"] not in exclude_orgs and s["memory_key"] not in done ] - async def insert_rows(rows: list[dict[str, Any]], *a: Any, **kw: Any) -> None: + async def insert_rows(rows: list[dict[str, Any]], *a: Any, **kw: Any) -> list[dict[str, Any]]: inserted.extend(rows) + return rows async def none(**_: Any) -> list[Any]: return [] @@ -662,6 +663,24 @@ def __init__(self, rows: list[dict[str, Any]], *, fail_drops: bool = False) -> N self.inserts: list[tuple[str, list[dict[str, Any]]]] = [] async def query_all(self, sql: str, params: dict[str, Any]) -> list[dict[str, Any]]: + if "FROM loom.memory_refusals" in sql: + return [] + if "source_version" in params: + from server.core.hashing import sha256_text + + return ( + [{"found": 1}] + if any( + all( + row.get(k) == params[k] + for k in ("org", "namespace", "agent", "user_id", "session_id", "queue_id") + ) + and row["version"] == params["source_version"] + and sha256_text(row.get("payload") or "").upper() == params["payload_sha"] + for row in self.rows + ) + else [] + ) self.sql, self.params = sql, params return self.rows @@ -671,7 +690,19 @@ async def insert(self, table: str, rows: list[dict[str, Any]], **_: Any) -> None self.inserts.append((table, rows)) -_WAITING = [{"org": "acme", "namespace": "n", "queue_id": "q1", "status": "pending"}] +_WAITING = [ + { + "org": "acme", + "namespace": "n", + "agent": "a", + "user_id": "-", + "session_id": "-", + "version": 1, + "payload": '{"messages":[]}', + "queue_id": "q1", + "status": "pending", + } +] def test_a_dropped_request_is_marked_failed_after_its_drop_record_is_written() -> None: diff --git a/tests/unit/test_model_calls_switch_workers.py b/tests/unit/test_model_calls_switch_workers.py index de049a53b2..27fcf98293 100644 --- a/tests/unit/test_model_calls_switch_workers.py +++ b/tests/unit/test_model_calls_switch_workers.py @@ -303,8 +303,9 @@ async def picked(**kw: Any) -> list[dict[str, Any]]: async def served(rows: list[dict[str, Any]], *a: Any) -> tuple[list[Any], list[Any]]: return rows, [] - async def insert(rows: list[dict[str, Any]], **_: Any) -> None: + async def insert(rows: list[dict[str, Any]], **_: Any) -> list[dict[str, Any]]: written.extend(rows) + return rows async def nothing(*a: Any, **k: Any) -> None: return None @@ -384,8 +385,9 @@ def test_a_stale_switch_stops_the_embedding_worker_before_any_write(monkeypatch) written: list[Any] = [] - async def insert(rows: list[dict[str, Any]], **_: Any) -> None: + async def insert(rows: list[dict[str, Any]], **_: Any) -> list[dict[str, Any]]: written.extend(rows) + return rows async def none(*a: Any, **k: Any) -> list[Any]: return [] @@ -438,8 +440,9 @@ async def picked(**kw: Any) -> list[dict[str, Any]]: async def served(rows: list[dict[str, Any]], *a: Any) -> tuple[list[Any], list[Any]]: return rows, [] - async def insert(rows: list[dict[str, Any]], **_: Any) -> None: + async def insert(rows: list[dict[str, Any]], **_: Any) -> list[dict[str, Any]]: written.extend(rows) + return rows async def nothing(*a: Any, **k: Any) -> None: return None diff --git a/tests/unit/test_namespace_rollback_unembedded.py b/tests/unit/test_namespace_rollback_unembedded.py index 671c98824c..1125554a3e 100644 --- a/tests/unit/test_namespace_rollback_unembedded.py +++ b/tests/unit/test_namespace_rollback_unembedded.py @@ -84,8 +84,9 @@ async def _memory(rows: list[dict[str, Any]], **_: Any) -> None: if on_inserted := _.get("on_inserted"): on_inserted() - async def _stubs(rows: list[dict[str, Any]], **_: Any) -> None: + async def _stubs(rows: list[dict[str, Any]], **_: Any) -> list[dict[str, Any]]: seen["stubs"].extend(rows) + return rows async def _retire(items: list[tuple[Any, int]], **_: Any) -> int: seen["retire_calls"].append(items) @@ -465,10 +466,15 @@ async def _published(*_a: Any, **_k: Any) -> None: async def _complete(seqs: list[Any], done: int, **_: Any) -> None: settled.append((list(seqs), done, list(calls["stubs"]))) + async def _held(org: str, ns: str, ids: list[Any], **_: Any) -> set[Any]: + assert (org, ns) == ("test", "personal") + return {pg_publish.document_id(row) for row in calls["stubs"]} & set(ids) + _authority_holds(monkeypatch, calls) monkeypatch.setattr(pg_publish, "commit", _commit) monkeypatch.setattr(pg_publish, "publish", _published) monkeypatch.setattr(pg_memory, "complete", _complete) + monkeypatch.setattr(pg_publish, "held_documents", _held) plan = RollbackPlan( reinstate=[_action("plain", "reinstate", 10), _action("secret", "reinstate", 11)], remove=[], diff --git a/tests/unit/test_negative_escalation.py b/tests/unit/test_negative_escalation.py index ce1af3ea63..ade8d41186 100644 --- a/tests/unit/test_negative_escalation.py +++ b/tests/unit/test_negative_escalation.py @@ -13,7 +13,10 @@ import pytest import server.memory.app.ops as ops +from server.core.datetimes import naive_utc_now +from server.core.hashing import sha256_value from server.memory.api.models_temporal import MemoryOutcomeInput +from server.memory.store.rows import MemoryRow, _hydrate, build_memory_version_row, identity_key @pytest.fixture(autouse=True) @@ -60,6 +63,21 @@ class _QuarantineCH(_StubCH): def __init__(self, ids): super().__init__([]) self.ids = ids + self.heads = {} + + def current(self): + heads = { + identity_key(h): build_memory_version_row(h) + for h in self.heads.values() + if h is not None + } + for table, rows in self.inserts: + if table == "loom.memory": + for row in rows: + key = identity_key(row) + if key not in heads or int(heads[key]["version"]) < int(row["version"]): + heads[key] = row + return heads async def query_one(self, sql, params=None): if "event_type = 'rejection'" in sql: @@ -70,7 +88,11 @@ async def query_one(self, sql, params=None): async def query_all(self, sql, params=None): if "FROM loom.memory FINAL" in sql: - return [{"memory_id": mid} for mid in self.ids] + return [ + r + for key, r in self.current().items() + if not params or "keys" not in params or key in params["keys"] + ] if "FROM loom.memory_embeddings FINAL" in sql: p = params or {} return [ @@ -86,6 +108,7 @@ async def query_all(self, sql, params=None): "embedding_dim": 2, "status": "active", "version": 2, # copied at read + 1 since its memory row + "content_sha256": sha256_value("session fact"), } for o, n, a, u, s, k in zip( p["o"], p["n"], p["a"], p["u"], p["s"], p["k"], strict=True @@ -95,7 +118,13 @@ async def query_all(self, sql, params=None): def _session_row(session_id, lock_mode="", version=1): - return SimpleNamespace( + return MemoryRow( + value="session fact", + pinned=False, + confidence=1, + created_at=naive_utc_now(), + expires_at=None, + content_sha256=sha256_value("session fact"), org="o", namespace="n", agent="a", @@ -121,6 +150,7 @@ def _wire_quarantine(monkeypatch, heads, outcome_reads=None): from server.settings import settings ch = _QuarantineCH(ids=list(heads)) + ch.heads = heads first = outcome_reads or heads async def get_by_id(memory_id, **kw): @@ -129,28 +159,17 @@ async def get_by_id(memory_id, **kw): async def get_first(memory_id, **kw): return first[memory_id] - def build(row, **overrides): - keys = ("org", "namespace", "agent", "user_id", "session_id", "memory_key", "status") - return {**{k: getattr(row, k) for k in keys}, "invalid_at": None, **overrides} - monkeypatch.setattr(settings, "memory_negative_demote_threshold", 3) monkeypatch.setattr("server.core.ch.get_client", lambda: ch) monkeypatch.setattr(memory_repo, "get_by_id", get_first) monkeypatch.setattr(lifecycle, "get_by_id", get_by_id) - monkeypatch.setattr(lifecycle, "build_memory_version_row", build) async def current_heads(keys, **kwargs): - columns = ("org", "namespace", "agent", "user_id", "session_id", "memory_key") - return { - tuple(getattr(head, c) for c in columns): head - for head in heads.values() - if tuple(getattr(head, c) for c in columns) in keys - } + return {key: _hydrate(row) for key, row in ch.current().items() if key in keys} monkeypatch.setattr( "server.memory.store.write_fence.memory_rows_by_identity_keys", current_heads ) - monkeypatch.setattr("server.memory.store.inserts.check_memory_rows", lambda rows: None) return ch @@ -194,8 +213,8 @@ async def test_a_locked_memory_is_not_quarantined(monkeypatch, mode): async def test_a_session_copy_never_quarantines_the_namespace_definition(monkeypatch): copy = _session_row("s1") shared = _session_row("-") - for row in (copy, shared): - row.agent, row.memory_type = "-", "definition" + copy = copy.model_copy(update={"agent": "-", "memory_type": "definition"}) + shared = shared.model_copy(update={"agent": "-", "memory_type": "definition"}) ch = _wire_quarantine(monkeypatch, {"mid-s1": copy, "mid--": shared}) out = await ops.handle_memory_outcome(_negative()) @@ -207,8 +226,8 @@ async def test_a_session_copy_never_quarantines_the_namespace_definition(monkeyp async def test_rating_the_namespace_definition_quarantines_nothing(monkeypatch): shared = _session_row("-") copy = _session_row("s1") - for row in (copy, shared): - row.agent, row.memory_type = "-", "definition" + copy = copy.model_copy(update={"agent": "-", "memory_type": "definition"}) + shared = shared.model_copy(update={"agent": "-", "memory_type": "definition"}) ch = _wire_quarantine(monkeypatch, {"mid--": shared, "mid-s1": copy}) out = await ops.handle_memory_outcome( diff --git a/tests/unit/test_observation_date_default.py b/tests/unit/test_observation_date_default.py index c92e070fa2..f9fe7a5b48 100644 --- a/tests/unit/test_observation_date_default.py +++ b/tests/unit/test_observation_date_default.py @@ -5,12 +5,19 @@ from datetime import UTC, datetime from typing import Any +from unittest.mock import AsyncMock import pytest from server.memory.domain.observation_date import resolve_observation_date +@pytest.fixture(autouse=True) +def _no_refused_intake(monkeypatch): + # Pipeline and projection cases; actual refusal admission is tested separately. + monkeypatch.setattr("server.memory.store.refusals.check_intake", AsyncMock()) + + def test_caller_value_wins_and_is_trimmed() -> None: assert resolve_observation_date(" 2023-05-30 ") == ("2023-05-30", "caller") assert resolve_observation_date("2023-05-30T10:00:00", default="none") == ( @@ -62,7 +69,6 @@ async def _cold(org: str, ns: str) -> dict[str, Any]: monkeypatch.setattr(mx, "_link_written_memories_to_evidence", _zero) monkeypatch.setattr(mx, "write_memory_chunks", _none) monkeypatch.setattr(mx, "_write_session_summary", _none) - monkeypatch.setattr(mx, "_draft_session_summary", _none) monkeypatch.setattr(mx, "record_input_projection_failure", _none) monkeypatch.setattr(_bulk_set_mod, "record_input_projection_failure", _none) monkeypatch.setattr(ont, "load_namespace_ontology", _cold) diff --git a/tests/unit/test_passage_sensitivity.py b/tests/unit/test_passage_sensitivity.py index 05d8346efc..051d83d216 100644 --- a/tests/unit/test_passage_sensitivity.py +++ b/tests/unit/test_passage_sensitivity.py @@ -489,7 +489,7 @@ def test_a_private_key_header_without_a_body_holds_only_its_own_chunk() -> None: @pytest.mark.asyncio -async def test_import_evidence_stores_a_credential_chunk_and_digest_without_a_vector( +async def test_import_holds_a_credential_chunk_and_withholds_an_unproven_digest( monkeypatch: pytest.MonkeyPatch, ) -> None: import server.embedding as embmod @@ -525,9 +525,8 @@ async def not_refused(*_: Any, **__: Any) -> bool: assert set(chunks) == {0, 1} assert (chunks[0]["embedding_dim"], chunks[1]["embedding_dim"]) == (2, 0) assert chunks[1]["content"] == f"key {OPENAI_KEY}" # stored as written - [digest] = ch.inserts["loom.session_summaries"] - assert digest["embedding_dim"] == 0 and digest["summary_text"] == f"Shared {AWS_KEY_ID}." - assert (out.chunks_written, out.digests_written, out.skipped) == (2, 1, 1) + assert "loom.session_summaries" not in ch.inserts + assert (out.chunks_written, out.digests_written, out.skipped) == (2, 0, 2) assert out.embedding_dim == 2 assert emb.seen == [_PLAIN] @@ -659,24 +658,33 @@ def test_a_chunk_key_with_a_credential_version_is_held_back() -> None: @pytest.mark.asyncio -async def test_the_summary_leg_serves_no_digest_that_reads_as_a_credential() -> None: +async def test_the_summary_leg_serves_no_digest_that_reads_as_a_credential(monkeypatch) -> None: + from unittest.mock import AsyncMock + + from server.memory import store from server.memory.app.retrieval.summaries import fetch_session_summaries + from tests.unit.test_memory_verify import _row + from tests.unit.test_summary_sources import digest + + secret = _row( + namespace="n", + agent="a", + user_id="u", + memory_key="secret", + session_id="s1", + value="x" * 1195 + f" {AWS_KEY_ID}.", + ) + plain = _row( + namespace="n", agent="a", user_id="u", memory_key="plain", session_id="s2", value=_PLAIN + ) + monkeypatch.setattr( + "server.memory.store.summary_sources.memory_rows_by_identity_keys", + AsyncMock(return_value={store.identity_key(r): r for r in [secret, plain]}), + ) class _Rows: async def query_all(self, *_: Any, **__: Any) -> list[dict[str, Any]]: - return [ - { - # The key straddles the served excerpt's 1200-character cut. - "session_id": "s1", - "summary_text": "x" * 1195 + f" {AWS_KEY_ID}.", - "distance": 0.1, - }, - { - "session_id": "s2", - "summary_text": _PLAIN, - "distance": 0.2, - }, - ] + return [digest(secret), digest(plain, distance=0.2)] hits = await fetch_session_summaries( _Rows(), @@ -710,29 +718,30 @@ async def find(*, limit: int, **_: Any) -> list[dict[str, Any]]: assert asked == [2 + _HELD_MARGIN] -@pytest.mark.asyncio -async def test_a_digest_that_reads_as_a_credential_is_drafted_without_a_vector( - monkeypatch: pytest.MonkeyPatch, -) -> None: - import server.llm.client as llm_client - from server.memory.app import session_summary +async def test_a_credential_source_never_enters_a_digest(monkeypatch: pytest.MonkeyPatch) -> None: + from unittest.mock import AsyncMock - class _LLM: - async def complete(self, *_: Any, **__: Any) -> Any: - return SimpleNamespace(text=f"The user pasted {OPENAI_KEY}.", tokens_in=1, tokens_out=1) - - emb = _Embedder() - monkeypatch.setattr(llm_client, "get_llm_client", lambda: _LLM()) - monkeypatch.setattr(session_summary, "get_embedder", lambda: emb) - draft = await session_summary._draft_session_summary( - SimpleNamespace(session_id="s"), - [{"role": "user", "content": "x"}], - None, - operator_policy="", + from server.memory import store + from server.memory.app import session_summary + from tests.unit.test_memory_verify import _row + + source = _row(value=f"The user pasted {OPENAI_KEY}.", sensitivity="") + ch = SimpleNamespace(insert=AsyncMock()) + vector = AsyncMock() + monkeypatch.setattr(session_summary, "get_client", lambda: ch) + monkeypatch.setattr(session_summary, "document_vector", vector) + monkeypatch.setattr(store, "get_by_id", AsyncMock(return_value=source)) + inp = SimpleNamespace( + org=source.org, + namespace=source.namespace, + agent=source.agent, + user_id=source.user_id, + session_id=source.session_id, ) - assert draft is not None - assert draft.embedding == [] and emb.seen == [] - assert OPENAI_KEY in draft.summary_text # stored as written + state = SimpleNamespace(written=[{"memory_id": source.memory_id, "version": source.version}]) + await session_summary._write_session_summary(inp, state) + vector.assert_not_awaited() + ch.insert.assert_not_awaited() @pytest.mark.asyncio @@ -801,31 +810,6 @@ async def test_the_deep_evidence_pass_opens_no_turn_that_reads_as_a_credential() assert [t.content for t in turns] == [_PLAIN] -@pytest.mark.asyncio -async def test_a_summary_hit_opens_no_session_turn_that_reads_as_a_credential() -> None: - from server.memory.app.deep_evidence import fetch_session_turns - - rows = [ - {"session_id": "s1", "role": "user", "observed_at": "", "content": c} - for c in (f"the standup key is {AWS_KEY_ID}", _PLAIN) - ] - - class _CH: - async def query_all(self, sql: str, params: dict[str, Any] | None = None) -> list[Any]: - return rows - - turns = await fetch_session_turns( - org="o", - namespace="n", - session_ids=["s1"], - question="When is the standup?", - max_chars_per_session=1000, - max_chars_total=5000, - ch=_CH(), - ) - assert [t.content for t in turns] == [_PLAIN] - - @pytest.mark.asyncio async def test_the_when_evidence_pass_recalls_no_turn_that_reads_as_a_credential( monkeypatch: pytest.MonkeyPatch, diff --git a/tests/unit/test_period_digest.py b/tests/unit/test_period_digest.py index f177c25a12..e01bfd4809 100644 --- a/tests/unit/test_period_digest.py +++ b/tests/unit/test_period_digest.py @@ -28,6 +28,16 @@ ) +@pytest.fixture(autouse=True) +def verified_period_inputs(monkeypatch): + """Period-policy tests start after source qualification, covered in test_summary_sources.""" + + async def qualified(ch, rows, **kwargs): + return rows + + monkeypatch.setattr("server.memory.store.summary_sources.current_summary_sources", qualified) + + def _digest(n: int, agent: str = "a") -> dict[str, Any]: return { "org": "o", @@ -149,13 +159,33 @@ def __init__(self, digests: list[dict[str, Any]], existing: set[str] | None = No self.link_calls: list[dict[str, Any]] = [] async def query_all(self, sql: str, params: Any = None) -> list[dict[str, Any]]: + if "FROM loom.memory FINAL" in sql and "PREWHERE (org, namespace" in sql: + from server.memory.store.rows import identity_key + + heads = {} + for table, rows in self.inserts: + if table == "loom.memory": + heads.update({identity_key(row): row for row in rows}) + return [heads[key] for key in params["keys"] if key in heads] if "uniqExact(s.session_id)" in sql: sessions = len({d["session_id"] for d in self._digests}) return [ {"org": "o", "namespace": "ns", "agent": "a", "user_id": "-", "sessions": sessions} ] if "FROM loom.session_summaries FINAL" in sql: - return list(self._digests) + rows = self._digests + if "LIMIT 1 BY session_id" in sql: + latest = {} + for row in sorted( + rows, key=lambda d: (-d.get("version", 1), d.get("embedding_model", "")) + ): + latest.setdefault(row["session_id"], row) + rows = list(latest.values()) + ordered = sorted(rows, key=lambda d: (d["at"], d["session_id"])) + after = params.get("after") + if after: + ordered = [d for d in ordered if (d["at"], d["session_id"]) > after] + return ordered[: params["limit"]] if "startsWith(memory_key" in sql: return [{"memory_key": k, "taken_out": False} for k in self._existing] if "memory_key IN" in sql: @@ -335,7 +365,7 @@ async def _heads(keys: Any, **_: Any) -> Any: # every source has a newer head async def test_a_failed_source_read_fails_the_period_before_its_model_call() -> None: class _SourcesDown(_FakeCH): async def query_all(self, sql: str, params: Any = None) -> list[dict[str, Any]]: - if "memory_key IN" in sql: + if "memory_key IN" in sql and "startsWith" not in sql: raise ConnectionError("clickhouse unreachable") return await super().query_all(sql, params) @@ -693,3 +723,82 @@ async def _run(cfg: Any, **_: Any) -> Any: ) assert app._dispatch(args) == 0 assert seen == ["process:cli:period-digest"] + + +async def test_unqualified_session_text_never_reaches_the_period_model(monkeypatch): + from unittest.mock import AsyncMock + + from server.memory.app.period_digest import _plan_digest_pages + + qualification = AsyncMock(return_value=[]) + monkeypatch.setattr( + "server.memory.store.summary_sources.current_summary_sources", qualification + ) + ch = _FakeCH(_digests(1, 5)) + assert await _plan_digest_pages( + ch, {"org": "o", "namespace": "ns", "agent": "a", "user_id": "-"}, _CFG + ) == ([], 0) + qualification.assert_awaited_once() + + +async def test_period_paging_selects_one_latest_digest_per_session_before_its_cursor(): + from server.memory.app.period_digest import _fetch_digests + + rows = [dict(_digest(1), embedding_model=f"m{i}", version=i) for i in (1, 2, 3)] + rows.extend(_digests(2, 3)) + identity = {"org": "o", "namespace": "ns", "agent": "a", "user_id": "-"} + found = [ + row async for page in _fetch_digests(_FakeCH(rows), identity, page_size=2) for row in page + ] + assert [row["session_id"] for row in found] == [f"ns:session_{i}" for i in (1, 2, 3)] + assert found[0]["embedding_model"] == "m3" + + +@pytest.mark.asyncio +async def test_streamed_windows_skip_old_keys_before_proof_and_reach_later_pages(monkeypatch): + from datetime import timedelta + from unittest.mock import AsyncMock + + from server.memory.app.period_digest import _plan_digest_pages + + digests = [ + dict(_digest(1), session_id=f"ns:session_{n}", at=datetime(2023, 1, 1) + timedelta(days=n)) + for n in range(1, 401) + ] + existing = {period_key(f"ns:session_{n}", f"ns:session_{n + 4}") for n in range(1, 301, 5)} + qualified = AsyncMock(side_effect=lambda ch, rows, **kw: rows) + monkeypatch.setattr("server.memory.store.summary_sources.current_summary_sources", qualified) + ident = {"org": "o", "namespace": "ns", "agent": "a", "user_id": "-"} + cfg = replace(_CFG, max_periods_per_identity=2) + periods, skipped = await _plan_digest_pages(_FakeCH(digests, existing), ident, cfg) + assert [p.memory_key for p in periods] == [ + period_key("ns:session_301", "ns:session_305"), + period_key("ns:session_306", "ns:session_310"), + ] + assert skipped == 60 + qualified.assert_awaited_once() + checked = qualified.call_args.args[1] + assert len(checked) == 100 and all( + int(d["session_id"].rsplit("_", 1)[1]) > 300 for d in checked + ) + existing.update(p.memory_key for p in periods) + later, _ = await _plan_digest_pages(_FakeCH(digests, existing), ident, cfg) + assert later[0].memory_key == period_key("ns:session_311", "ns:session_315") + + +@pytest.mark.asyncio +async def test_an_unproven_digest_withholds_its_whole_window_without_shifting_later_windows( + monkeypatch, +): + from server.memory.app.period_digest import _plan_digest_pages + + async def qualify(ch, rows, **kwargs): + return [d for d in rows if d["session_id"] != "ns:session_2"] + + monkeypatch.setattr("server.memory.store.summary_sources.current_summary_sources", qualify) + periods, _ = await _plan_digest_pages( + _FakeCH(_digests(1, 10)), + {"org": "o", "namespace": "ns", "agent": "a", "user_id": "-"}, + _CFG, + ) + assert [p.memory_key for p in periods] == [_KEY_6_10] diff --git a/tests/unit/test_pg_authority.py b/tests/unit/test_pg_authority.py index 3a4ae4f60a..9dd0cc75f0 100644 --- a/tests/unit/test_pg_authority.py +++ b/tests/unit/test_pg_authority.py @@ -1365,6 +1365,118 @@ def _claim(seq: int, targets: int, *, ns: str = "n", attempts: int = 1) -> pg_me ) +async def test_delivery_retains_only_the_document_admission_withheld(monkeypatch) -> None: + from dataclasses import replace + + from server.memory.app import after_write, pg_delivery + from server.memory.store import pg_publish + + claims = [ + replace( + _claim(i, pg_memory.SEARCH_DOC), + lease_token=10 + i, + deadline=time.monotonic() + 60, + row={ + "org": "o", + "namespace": "n", + "agent": "a", + "user_id": "u", + "session_id": "s", + "memory_key": f"k{i}", + "version": i, + }, + ) + for i in range(1, 5) + ] + completed, released, indexed = [], [], [] + + async def none(*a, **kw): + return None + + async def held(*a, **kw): + return {pg_publish.document_id(claims[0].row)} + + async def heads(owed, deadline): + assert [c.seq for c in owed] == [2, 3, 4] + return [replace(claims[2], row={**claims[2].row, "version": 30}), claims[3]] + + async def documents(built, **kw): + return [dict(c.row) for c in built] + + async def insert(rows, **kw): + assert [r["version"] for r in rows] == [30, 4] + return rows[:1] + + async def complete(seqs, target, *, tokens): + completed.append((seqs, target, tokens)) + + async def release(owed): + released.extend(c.seq for c in owed) + + monkeypatch.setattr("server.core.ch.get_client", lambda: object()) + monkeypatch.setattr(pg_publish, "publish_rows", none) + monkeypatch.setattr(pg_publish, "held_documents", held) + monkeypatch.setattr(pg_publish, "retire_decision_records", none) + monkeypatch.setattr(pg_delivery, "_still_heads", heads) + monkeypatch.setattr(pg_delivery, "_search_documents", documents) + monkeypatch.setattr(pg_delivery, "_retire_old_vectors", none) + monkeypatch.setattr("server.memory.store.insert_embedding_rows", insert) + monkeypatch.setattr(after_write, "index_late_rows", lambda rows, **kw: indexed.extend(rows)) + monkeypatch.setattr(pg_memory, "complete", complete) + monkeypatch.setattr(pg_memory, "release", release) + assert await pg_delivery._publish(claims) == 3 + assert completed == [ + ([("o", "n", 1), ("o", "n", 2), ("o", "n", 3)], pg_memory.SEARCH_DOC, [11, 12, 13]) + ] + assert released == [4] and [r["version"] for r in indexed] == [30] + + +async def test_a_delivery_withheld_by_admission_counts_no_success_or_failure(monkeypatch) -> None: + from server.memory.app import pg_delivery + + async def renew(claims): + return claims + + async def withheld(claims, **kw): + return 0 + + async def unexpected(*a, **kw): + raise AssertionError("withholding counts no failure") + + monkeypatch.setattr(pg_memory, "renew", renew) + monkeypatch.setattr(pg_memory, "record_failure", unexpected) + monkeypatch.setattr(pg_delivery, "_publish", withheld) + assert await pg_delivery.publish_claims([_claim(1, pg_memory.SEARCH_DOC)]) == (0, 0) + + +@pytest.mark.parametrize("present", [False, True]) +async def test_a_copy_settles_only_an_exact_document_held_in_clickhouse(monkeypatch, present): + from server.memory.store import lifecycle, pg_publish + + row = { + "org": "o", + "namespace": "n", + "agent": "a", + "user_id": "u", + "session_id": "s", + "memory_key": "k", + "version": 7, + } + seq, completed = ("o", "n", 3), [] + + async def held(org, ns, ids, **kw): + assert (org, ns, ids) == ("o", "n", [pg_publish.document_id(row)]) + return set(ids) if present else {(*ids[0][:-1], 6)} + + async def complete(seqs, target): + completed.append((seqs, target)) + + monkeypatch.setattr(pg_publish, "held_documents", held) + monkeypatch.setattr(pg_memory, "complete", complete) + await lifecycle._settle_documents(SimpleNamespace(rows=[row], seqs=[seq]), [True], ch=object()) + assert completed == [([seq] if present else [], pg_memory.SEARCH_DOC)] + + @pytest.mark.asyncio async def test_a_revision_clickhouse_keeps_refusing_holds_back_only_itself( monkeypatch: pytest.MonkeyPatch, @@ -1376,10 +1488,11 @@ async def test_a_revision_clickhouse_keeps_refusing_holds_back_only_itself( published: list[tuple[list[int], bool]] = [] failed: list[int] = [] - async def _publish(claims: list[pg_memory.Claimed], *, stub: bool = False) -> None: + async def _publish(claims: list[pg_memory.Claimed], *, stub: bool = False) -> int: if any(c.seq == 2 for c in claims) and not stub: raise RuntimeError("refused") published.append(([c.seq for c in claims], stub)) + return len(claims) async def _record_failure( claims: list[pg_memory.Claimed], error: BaseException, *, park: bool = False @@ -1406,10 +1519,11 @@ async def _renew(claims: list[pg_memory.Claimed]) -> list[pg_memory.Claimed]: assert published == [([2], True)] # A stub refused too parks that claim alone; the rest of the pass is published. - async def _refuse_2(claims: list[pg_memory.Claimed], *, stub: bool = False) -> None: + async def _refuse_2(claims: list[pg_memory.Claimed], *, stub: bool = False) -> int: if any(c.seq == 2 for c in claims): raise RuntimeError("refused") published.append(([c.seq for c in claims], stub)) + return len(claims) monkeypatch.setattr(pg_delivery, "_publish", _refuse_2) published.clear() @@ -1459,10 +1573,14 @@ def test_the_authoritys_erasure_step_leads_the_erasing_steps_and_runs_only_under from server.settings import settings monkeypatch.setattr(settings, "storage_authority", "postgres") - assert authority_steps(USER_STEPS)[0][0] == "pg_memory.delete" + assert [name for name, _ in authority_steps(USER_STEPS)][:2] == [ + "memory.refusals", + "pg_memory.delete", + ] # A namespace's sources stop first, so no sync turn writes into what the steps erase. names = [name for name, _ in authority_steps(NAMESPACE_STEPS)] - assert names[:4] == [ + assert names[:5] == [ + "memory.refusals", "knowledge_sources.retire", "knowledge_source_push.delete", "pg_memory.delete", @@ -1859,10 +1977,11 @@ async def test_a_refused_statement_is_the_claims_attempt_and_the_rest_still_publ published: list[int] = [] refused = pg.PostgresRefusedError(asyncpg.exceptions.CheckViolationError("check")) - async def _publish(claims: list[pg_memory.Claimed], *, stub: bool = False) -> None: + async def _publish(claims: list[pg_memory.Claimed], *, stub: bool = False) -> int: if any(c.seq == 1 for c in claims): raise refused published.extend(c.seq for c in claims) + return len(claims) async def _record( claims: list[pg_memory.Claimed], error: BaseException, *, park: bool = False @@ -1925,8 +2044,9 @@ async def test_a_namespace_whose_claims_are_all_gone_leaves_the_rest_of_the_pass published: list[str] = [] - async def _publish(claims: list[pg_memory.Claimed], *, stub: bool = False) -> None: + async def _publish(claims: list[pg_memory.Claimed], *, stub: bool = False) -> int: published.extend(c.namespace for c in claims) + return len(claims) async def _renew(claims: list[pg_memory.Claimed]) -> list[pg_memory.Claimed]: return [c for c in claims if c.namespace != "erased"] @@ -1952,11 +2072,12 @@ async def test_a_last_attempt_its_lease_ran_out_on_publishes_its_stub_under_a_ne stubs: list[pg_memory.Claimed] = [] renewals: list[int] = [] - async def _publish(claims: list[pg_memory.Claimed], *, stub: bool = False) -> None: + async def _publish(claims: list[pg_memory.Claimed], *, stub: bool = False) -> int: pg_lease.check_lease(claims[0].deadline) if not stub: raise pg_lease.PublicationPending() # the embedder outlasted the lease stubs.extend(claims) + return len(claims) async def _renew(claims: list[pg_memory.Claimed]) -> list[pg_memory.Claimed]: renewals.append(len(claims)) @@ -2057,10 +2178,11 @@ async def _held(*_: object, **__: object) -> set[object]: return set() async def _docs(claims: list[object], *, stub: bool = False) -> list[dict[str, object]]: - return [{"memory_key": "k"}] + return [{"org": "o", "namespace": "n", "memory_key": "k"}] - async def _insert(docs: list[object], ch: object = None) -> None: + async def _insert(docs: list[object], ch: object = None) -> list[object]: steps.append("documents") + return docs async def _complete(seqs: object, done: int, *, tokens: object = 1) -> None: steps.append(f"complete {done}") @@ -3001,9 +3123,13 @@ async def execute(self, sql: str, *args: object) -> None: return None async def fetch(self, sql: str, *args: Any) -> list[dict[str, Any]]: + if "loom.memory_refusals" in sql: + return [] return [{"org": "o", "scope": "n", "generation": 5}] async def fetchval(self, sql: str, *args: Any) -> bool: + if "loom.memory_refusals" in sql: + return False asked.append(list(args[2])) return uuid.UUID(erased) in args[2] @@ -3136,9 +3262,10 @@ async def _renew(batch: list[Any]) -> list[Any]: # New objects, as the store's `replace` returns. return [SimpleNamespace(**vars(c)) for c in batch] - async def _publish(batch: list[Any], **_: Any) -> None: + async def _publish(batch: list[Any], **_: Any) -> int: seen["published"].append([c.seq for c in batch]) await publish(batch) + return len(batch) async def _release(batch: list[Any]) -> None: seen["released"].extend(c.seq for c in batch) @@ -5821,6 +5948,8 @@ async def execute(self, sql: str, *args: object) -> None: return None async def fetch(self, sql: str, *args: Any) -> list[dict[str, Any]]: + if "loom.memory_refusals" in sql: + return [] if "loom.memory_current c" in sql: return [{**row, "row": pg_memory.encode_row(head)}] return [{"org": "o", "scope": "n", "generation": 5}] diff --git a/tests/unit/test_pg_batch.py b/tests/unit/test_pg_batch.py index bb7d1344bf..7c9d36fedf 100644 --- a/tests/unit/test_pg_batch.py +++ b/tests/unit/test_pg_batch.py @@ -56,6 +56,7 @@ def __init__(self, monkeypatch: pytest.MonkeyPatch, *, interval_ms: int = 0) -> self.release = asyncio.Event() self.alone: BaseException | None = None self.refuse_documents: BaseException | None = None + self.withhold_documents: set[str] = set() self.refuse_retirement = False # Each retirement's keys; `retire_alone` is what a retirement of one commit raises. self.retired: list[list[str]] = [] @@ -91,11 +92,13 @@ async def _memory(rows: Any, *, ch: Any, published: bool) -> int: self.memory.append(keys) return len(rows) - async def _documents(rows: Any, *, ch: Any) -> None: + async def _documents(rows: Any, *, ch: Any) -> list[dict[str, Any]]: if self.refuse_documents is not None: raise self.refuse_documents + rows = [r for r in rows if r["memory_key"] not in self.withhold_documents] self.documents.append([r["memory_key"] for r in rows]) self.document_rows += rows + return rows async def _audits(rows: Any, *, ch: Any, retried: bool) -> None: self.audits.append(len(rows)) @@ -175,6 +178,34 @@ async def submit( await pg_batch.submit(committed, documents=docs, ch=self.ch, wait=wait) +async def test_canonical_and_vector_sends_do_not_compete_for_the_same_fences(monkeypatch): + batch = _Batching(monkeypatch) + memory_send = pg_batch.inserts.insert_memory_versions + document_send = pg_batch.inserts.insert_embedding_rows + publishing_memory = False + overlap = [] + + async def memory(rows, **kwargs): + nonlocal publishing_memory + publishing_memory = True + try: + await asyncio.sleep(0) + return await memory_send(rows, **kwargs) + finally: + publishing_memory = False + + async def documents(rows, **kwargs): + overlap.append(publishing_memory) + return await document_send(rows, **kwargs) + + monkeypatch.setattr(pg_batch.inserts, "insert_memory_versions", memory) + monkeypatch.setattr(pg_batch.inserts, "insert_embedding_rows", documents) + await batch.submit("k", 1) + assert overlap == [False] + assert batch.memory == batch.documents == [["k"]] + assert batch.landed == [(("o", "n", 1), _FULL)] + + async def test_commits_queued_together_are_one_insert_per_table( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -387,6 +418,47 @@ async def test_a_document_whose_key_moved_on_is_not_sent_and_stays_owed( assert batch.handed_over == [("o", "n", 1)] +async def test_a_document_withheld_after_the_heads_read_stays_owed( + monkeypatch: pytest.MonkeyPatch, +) -> None: + batch = _Batching(monkeypatch) + batch.withhold_documents = {"k1"} + await asyncio.gather(batch.submit("k1", 1), batch.submit("k2", 2)) + assert batch.documents == [["k2"]] + assert sorted(batch.landed) == [ + (("o", "n", 1), MEMORY_ROW | AUDIT_ROW), + (("o", "n", 2), _FULL), + ] + assert batch.handed_over == [("o", "n", 1)] + + +@pytest.mark.parametrize("field", ["org", "namespace", "agent", "user_id", "session_id"]) +async def test_a_receipt_cannot_settle_another_identity(monkeypatch, field) -> None: + from server.memory.store.rows import identity_key + + batch = _Batching(monkeypatch) + first, other = batch.committed("k", 1), batch.committed("k", 2) + other.rows[0][field] = "other" + other.seqs[0] = (other.rows[0]["org"], other.rows[0]["namespace"], 2) + rows = [first.rows[0], other.rows[0]] + + async def heads(keys, **kw): + return {identity_key(row): (row, 0) for row in rows} + + async def insert(documents, **kw): + assert len(documents) == 2 + return documents[:1] + + monkeypatch.setattr(pg_memory, "heads_owing", heads) + monkeypatch.setattr("server.memory.store.inserts.insert_embedding_rows", insert) + await asyncio.gather( + pg_batch.submit(first, documents=rows[:1], ch=batch.ch), + pg_batch.submit(other, documents=rows[1:], ch=batch.ch), + ) + assert batch.landed == [(first.seqs[0], _FULL), (other.seqs[0], MEMORY_ROW | AUDIT_ROW)] + assert batch.handed_over == other.seqs + + @pytest.mark.parametrize("head_owes", [0, SEARCH_DOC]) async def test_a_document_whose_row_was_copied_is_sent_onto_the_copy( monkeypatch: pytest.MonkeyPatch, head_owes: int diff --git a/tests/unit/test_pg_durable_reads.py b/tests/unit/test_pg_durable_reads.py index 750e1ba6c6..2a7c7a6c70 100644 --- a/tests/unit/test_pg_durable_reads.py +++ b/tests/unit/test_pg_durable_reads.py @@ -112,8 +112,11 @@ "server/memory/store/records.py": "a retirement reads its writers durably; " "`live_writer_record` filters a request's read, which commits through `memory.set`", "server/memory/store/redaction.py": "decides a redaction, which commits", - "server/memory/store/source_review.py": "holds the exact source after a durable read " - "through shared publication", + "server/memory/store/refusals.py": "reads durable sources before publication, then " + "rechecks their heads on the admission transaction's connection", + "server/memory/store/shared.py": "qualifies a durable source before its fenced promotion", + "server/memory/store/summary_sources.py": "reads durable source proofs before publication, " + "then rechecks them under the same transaction's admission locks", "server/memory/workers/pg_payload_clear.py": "clears through its own commit, and only " "revisions delivered a horizon ago that ClickHouse holds", } diff --git a/tests/unit/test_pipeline_actor_sensitivity_gauges.py b/tests/unit/test_pipeline_actor_sensitivity_gauges.py index 8dd948450e..477ecd16c2 100644 --- a/tests/unit/test_pipeline_actor_sensitivity_gauges.py +++ b/tests/unit/test_pipeline_actor_sensitivity_gauges.py @@ -241,6 +241,8 @@ def __init__(self, rows: list[dict[str, Any]] | None = None) -> None: async def query_all(self, sql, params=None): self.calls.append((sql, params or {})) + if "FROM loom.memory_refusals" in sql: + return [] return list(self.rows) async def query_one(self, sql, params=None): @@ -341,6 +343,7 @@ async def _live(ch, **ident): async def _ins(rows, *, ch=None): inserted.append(rows) + return rows monkeypatch.setattr(memory_repo, "retire_vectors_for_keys", _retire) monkeypatch.setattr(memory_repo, "live_embedding_rows_for_key", _live) @@ -1292,6 +1295,7 @@ async def test_integrity_counts_and_the_dim0_repair(monkeypatch) -> None: async def _ins(rows, *, ch=None): inserted.append(rows) + return rows monkeypatch.setattr(memory_repo, "insert_embedding_rows", _ins) rows = [ @@ -1490,7 +1494,7 @@ async def _live(ch, **ident): return [{**ident, "tags": ["t"], "embedding": [0.1], "embedding_dim": 1, "version": 3}] async def _ins(rows, *, ch=None): - return None + return rows async def _check(rows, *, ch=None): checked.append(rows) diff --git a/tests/unit/test_profile_leg.py b/tests/unit/test_profile_leg.py index f8b6b1caa8..9c9a62eb60 100644 --- a/tests/unit/test_profile_leg.py +++ b/tests/unit/test_profile_leg.py @@ -230,8 +230,9 @@ def test_the_stub_embeds_label_and_value_where_the_old_vector_lives() -> None: class _Repo: pass - async def _insert(rows: list, ch: Any = None) -> None: + async def _insert(rows: list, ch: Any = None) -> list: written.extend(rows) + return rows import server.memory.store as mr diff --git a/tests/unit/test_promote_definition.py b/tests/unit/test_promote_definition.py index 3b91481d4d..ec25be3a6e 100644 --- a/tests/unit/test_promote_definition.py +++ b/tests/unit/test_promote_definition.py @@ -21,6 +21,7 @@ from server.memory.app import promote from server.memory.domain.definitions import definition_names, fold_name from server.memory.store import MemoryRow, shared +from server.memory.store.rows import identity_key from server.settings import settings from tests.unit.clickhouse_local import CLICKHOUSE, LocalCH, ddl @@ -50,7 +51,7 @@ def _row(**over: Any) -> MemoryRow: @pytest.fixture -def store(monkeypatch: pytest.MonkeyPatch) -> dict[str, Any]: +def store(monkeypatch: pytest.MonkeyPatch, no_blacklisted_values: None) -> dict[str, Any]: """The source row, the live shared definitions (key -> value) and what was written, with the claim: `free` (False: another holds it; None: the lease store does not answer), `claims` (the names taken) and `held` (inside it); `heads` and @@ -69,6 +70,10 @@ def store(monkeypatch: pytest.MonkeyPatch) -> dict[str, Any]: async def get_by_key(*_a: Any, **_k: Any) -> MemoryRow: return state["row"] + async def sources(*args, **kwargs): + row = state["row"] + return {identity_key(row): row} + async def definitions(org: str, namespace: str, limit: int, **_k: Any) -> list[dict]: # The two fields the store extracts, as its statement reads them. assert state["held"], "the check runs under the namespace's claim" @@ -99,6 +104,7 @@ async def holding(name: str, **_k: Any) -> AsyncIterator[bool]: state["held"] = False monkeypatch.setattr(memory_repo, "get_by_key", get_by_key) + monkeypatch.setattr("server.memory.store.reads.memory_rows_by_identity_keys", sources) monkeypatch.setattr(promote, "get_embedder", lambda: None) monkeypatch.setattr(promote, "writer_actor", lambda fallback="": "user:admin") monkeypatch.setattr(promote, "naive_utc_now", lambda: NOW) @@ -176,6 +182,20 @@ def test_a_definitions_names(value: Any, names: set[str] | None) -> None: assert definition_names(value) == names +async def test_definition_claim_precedes_the_source_connection(store, monkeypatch): + """A demote holds the claim while borrowing a connection; promotion uses that order.""" + original = promote.refusals.source_publication + + @asynccontextmanager + async def admission(*args, **kwargs): + assert store["held"], "the definition claim must precede the source connection" + async with original(*args, **kwargs) as mark: + yield mark + + monkeypatch.setattr(promote.refusals, "source_publication", admission) + await promote.handle_memory_promote(_input()) + + async def test_an_approved_definition_is_promoted_by_the_tokens_actor( store: dict[str, Any], ) -> None: diff --git a/tests/unit/test_promote_refuses_high.py b/tests/unit/test_promote_refuses_high.py index 0e87de20fa..37e609b11e 100644 --- a/tests/unit/test_promote_refuses_high.py +++ b/tests/unit/test_promote_refuses_high.py @@ -73,8 +73,10 @@ async def get(*args, **kwargs): ) +@pytest.mark.usefixtures("no_blacklisted_values") async def test_promotion_rechecks_the_source_after_preparing_its_projection(monkeypatch): from server.memory.store.pg_memory import AdmissionContendedError + from server.memory.store.rows import identity_key source = _row("") changed = source.model_copy( @@ -89,7 +91,11 @@ async def get(*args, **kwargs): async def write(*args, **kwargs): writes.append(args) + async def heads(*args, **kwargs): + return {identity_key(changed): changed} + monkeypatch.setattr(memory_repo, "get_by_key", get) + monkeypatch.setattr("server.memory.store.reads.memory_rows_by_identity_keys", heads) monkeypatch.setattr(promote, "get_embedder", lambda: None) monkeypatch.setattr(shared, "insert_shared_row", write) with pytest.raises(AdmissionContendedError): diff --git a/tests/unit/test_promote_source_identity.py b/tests/unit/test_promote_source_identity.py index db5dc8a790..f8f40c7e7b 100644 --- a/tests/unit/test_promote_source_identity.py +++ b/tests/unit/test_promote_source_identity.py @@ -15,6 +15,7 @@ @pytest.mark.asyncio +@pytest.mark.usefixtures("no_blacklisted_values") async def test_the_copy_names_the_row_it_was_read_from(monkeypatch: pytest.MonkeyPatch) -> None: """A lookup by `agent="*"` reads one row; the copy names that row's identity, by which an erasure finds it, not the wildcard.""" @@ -41,7 +42,14 @@ async def _get(*_a: Any, **_k: Any) -> MemoryRow: async def _write(row: dict[str, Any]) -> None: rows.append(row) + async def sources(*args, **kwargs): + from server.memory.store.rows import identity_key + + row = await _get() + return {identity_key(row): row} + monkeypatch.setattr(memory_repo, "get_by_key", _get) + monkeypatch.setattr("server.memory.store.reads.memory_rows_by_identity_keys", sources) monkeypatch.setattr(promote, "get_embedder", lambda: None) monkeypatch.setattr(shared, "insert_shared_row", _write) await promote.handle_memory_promote( diff --git a/tests/unit/test_promotion_source.py b/tests/unit/test_promotion_source.py index 9371c3ffdf..681d4eaafd 100644 --- a/tests/unit/test_promotion_source.py +++ b/tests/unit/test_promotion_source.py @@ -30,3 +30,61 @@ def test_an_identity_part_holding_a_slash_is_read_into_the_key() -> None: parsed = parse_promotion_source("o/n/-/-/run/7/definition/arr") assert parsed is not None assert (parsed["session_id"], parsed["memory_key"]) == ("run", "7/definition/arr") + + +@pytest.mark.parametrize("field", ["agent", "user_id", "session_id", "memory_key"]) +def test_consent_covers_every_ambiguous_user_interpretation(field): + from server.memory.domain.promotion_source import promotion_users + + identity = { + "org": "o/div", + "namespace": "n/team", + "agent": "bot", + "user_id": "alice", + "session_id": "session", + "memory_key": "definition/arr", + } + identity[field] += "/segment" + users = promotion_users([promotion_source(**identity)], identity["org"], identity["namespace"]) + assert users is not None and identity["user_id"] in users + if field == "user_id": + assert "alice" in users and "alice/segment" in users + + +@pytest.mark.parametrize( + "entries", [[], ["wrong/prefix/a/alice/s/k"], ["o/n/a/alice/s/"], ["o/n/" + "x/" * 1000 + "k"]] +) +def test_incomplete_source_denial_proof_cannot_authorize_plaintext(entries): + from server.memory.domain.promotion_source import promotion_users + + assert promotion_users(entries, "o", "n") is None + + +@pytest.mark.parametrize( + "field", ["org", "namespace", "agent", "user_id", "session_id", "memory_key"] +) +def test_positive_proof_finds_the_actual_identity_despite_slashes(field): + from server.memory.domain.promotion_source import promotion_keys + + identity = dict( + org="o", + namespace="n", + agent="bot", + user_id="alice", + session_id="s", + memory_key="definition/arr", + ) + identity[field] += "/team" + keys = promotion_keys([promotion_source(**identity)], identity["org"], identity["namespace"]) + assert keys is not None and tuple(identity.values()) in keys + + +@pytest.mark.parametrize("key", ["path/", "/", "a//", "//", "a//b/"]) +@pytest.mark.parametrize("user", ["", "alice/team", "alice"]) +def test_complete_source_key_may_end_in_a_slash(key, user): + from server.memory.domain.promotion_source import promotion_keys, promotion_users + + identity = ("o", "n", "agent", user, "session", key) + entry = promotion_source(*identity) + assert user in promotion_users([entry], "o", "n") + assert identity in promotion_keys([entry], "o", "n") diff --git a/tests/unit/test_proposal_commit_review.py b/tests/unit/test_proposal_commit_review.py index a267318e0e..26a3148b33 100644 --- a/tests/unit/test_proposal_commit_review.py +++ b/tests/unit/test_proposal_commit_review.py @@ -163,6 +163,8 @@ async def test_native_commit_checks_decoded_review_state_before_publishing(monke class Conn: async def fetch(self, sql, *args): + if "loom.memory_refusals" in sql: + return [] if "loom.memory_current c" in sql: return [{**head, "row": head}] return [{"org": source.org, "scope": source.namespace, "generation": 5}] diff --git a/tests/unit/test_proposals.py b/tests/unit/test_proposals.py index 2f23f40175..5235d09979 100644 --- a/tests/unit/test_proposals.py +++ b/tests/unit/test_proposals.py @@ -82,6 +82,9 @@ def _proposal_row(key: str, status: str, conflict_with: list[str] | None = None) class _CH: + async def query_all(self, *args, **kwargs): + return [] + def __init__(self) -> None: self.inserts: list[list[dict[str, Any]]] = [] diff --git a/tests/unit/test_read_scope_admission.py b/tests/unit/test_read_scope_admission.py index 61bee06cf1..fec2803bc0 100644 --- a/tests/unit/test_read_scope_admission.py +++ b/tests/unit/test_read_scope_admission.py @@ -614,17 +614,6 @@ async def test_deep_evidence_opens_only_the_scopes_links_turns_and_sessions() -> # The links come only from the memory rows the scope reaches. assert "memory_id IN (" in ch.calls[0][0] _bound_to_u1(ch) - ch = _CH() - await deep_evidence.fetch_session_turns( - org="o", - namespace="n", - session_ids=["sess-1"], - question="q", - max_chars_per_session=99, - max_chars_total=99, - ch=ch, - ) - _bound_to_u1(ch) def _peer(mid: str, uid: str, *, pinned: bool = False) -> Any: diff --git a/tests/unit/test_read_your_write_inserts.py b/tests/unit/test_read_your_write_inserts.py index ec14afffcf..2d33987614 100644 --- a/tests/unit/test_read_your_write_inserts.py +++ b/tests/unit/test_read_your_write_inserts.py @@ -147,7 +147,7 @@ def test_default_read_path_tables_are_written_synchronously(rel_path: str, table ("server/observability/audit.py", "loom.audit_log", "switched"), ("server/observability/events.py", "loom.agent_events", "switched"), ("server/memory/store/webhooks.py", "loom.memory_webhook_outbox", "switched"), - ("server/memory/app/evidence.py", "loom.memory_evidence", "switched"), + ("server/memory/store/refusals.py", "loom.memory_evidence", "switched"), ("server/memory/app/evidence.py", "loom.memory_belief_sources", "switched"), ("server/core/rmt_carry.py", "loom.carry_acks", "request"), ("server/auth/attempts.py", "loom.auth_attempts", "request"), diff --git a/tests/unit/test_reembed_in_database.py b/tests/unit/test_reembed_in_database.py index 6c55cc8393..2e196b3f00 100644 --- a/tests/unit/test_reembed_in_database.py +++ b/tests/unit/test_reembed_in_database.py @@ -132,21 +132,72 @@ async def _one(sql, params=None): return {"n": 7} async def _all(sql, params=None): - return [{"name": n} for n in COLS] + return [ + {"name": n} + for n in ("org", "entity_text", "entity_embedding", "embedding_model", "embedding_dim") + ] monkeypatch.setattr(c, "query_one", _one) monkeypatch.setattr(c, "query_all", _all) monkeypatch.setattr(ch_mod, "guard_pending_schema", lambda *a, **k: _noop()) n = await rb.reembed_table_in_database( - c, "memory_embeddings", model="m2", dim=4, credentials="coll", log=lambda *_: None + c, "entity_vectors", model="m2", dim=4, credentials="coll", log=lambda *_: None ) assert n == 7 and len(sent) == 1 got = sent[0] - assert got["sql"].startswith("INSERT INTO loom.memory_embeddings") + assert got["sql"].startswith("INSERT INTO loom.entity_vectors") assert got["settings"]["allow_experimental_ai_functions"] == 1 assert got["settings"]["ai_function_max_api_calls_per_query"] == 1000 +async def test_database_memory_vectors_publish_through_admission_and_count_only_acks(monkeypatch): + import server.memory.store as store + + rows = [ + {"org": org, "memory_key": "fact", "embedding_model": "m2", "_k": (org,) * 8} + for org in ("kept", "refused") + ] + queries = [] + + class CH: + async def query_one(self, sql, params=None): + return {"n": 2} + + async def query_all(self, sql, params=None, *, settings=None): + if "system.columns" in sql: + return [{"name": name} for name in COLS] + queries.append(sql) + assert settings["allow_experimental_ai_functions"] == 1 + assert params["page"] == 200 + assert "AS embedding_model" in sql.split(" FROM (", 1)[0] + assert "embedding_model != 'm2'" in sql.split(" FROM (", 1)[1] + assert "LIMIT {page:UInt32}" in sql + return rows + + async def execute(self, *args, **kwargs): + pytest.fail("A memory vector bypassed protected publication") + + published = [] + + async def admit(out, *, ch): + published.extend(out) + return out[:1] + + monkeypatch.setattr(store, "insert_embedding_rows", admit) + held = set() + written = await rb.reembed_table_in_database( + CH(), + "memory_embeddings", + model="m2", + dim=4, + credentials="coll", + left_orgs=held, + log=lambda *_: None, + ) + assert written == 1 and held == {"refused"} + assert len(queries) == 1 and all("_k" not in row for row in published) + + async def _noop() -> None: return None diff --git a/tests/unit/test_reembed_paging.py b/tests/unit/test_reembed_paging.py index 75b09d8fe4..992212f8b0 100644 --- a/tests/unit/test_reembed_paging.py +++ b/tests/unit/test_reembed_paging.py @@ -10,7 +10,9 @@ import pytest from server.core.ch import sequential_reads_active +from server.core.hashing import sha256_value from server.memory.app import reembed as rb +from server.memory.store.rows import MemoryRow, build_memory_version_row from tests.unit._fake_credentials import OPENAI_KEY from tests.unit.clickhouse_local import CLICKHOUSE, EMBEDDINGS, VECTOR, run @@ -22,6 +24,19 @@ async def embed_many(self, texts: list[str]) -> list[list[float]]: return [[0.1, 0.2] for _ in texts] +def _canonical(row): + return build_memory_version_row( + MemoryRow.from_snapshot( + { + **row, + "memory_id": "11111111-1111-1111-1111-111111111111", + "memory_type": "fact", + "value": row["content_excerpt"], + } + ) + ) + + def key(r: dict[str, Any]) -> tuple[str, ...]: return (r["org"], r["namespace"], r["embedding_model"], r["thread_id"], r["msg_id"]) @@ -137,6 +152,7 @@ async def test_a_memory_row_is_re_embedded_whatever_its_text() -> None: "embedding_model": "m1", "memory_key": "fixture_key", "content_excerpt": excerpt, + "content_sha256": sha256_value(excerpt), "tags": ["sensitivity:low"], "embedding": [0.0], "embedding_dim": 1, @@ -147,6 +163,10 @@ async def test_a_memory_row_is_re_embedded_whatever_its_text() -> None: class _MemCH: async def query_all(self, sql: str, params: Any = None) -> list[dict]: + if "FROM loom.memory_refusals" in sql: + return [] + if "FROM loom.memory FINAL" in sql: + return [_canonical(row)] if "system.columns" in sql: return [{"name": c} for c in row] if "keys" in (params or {}): # by the key the row was read with, not the new model's @@ -324,6 +344,7 @@ def row(k: str) -> dict[str, Any]: "embedding_model": "m2", "memory_key": "k", "content_excerpt": "x", + "content_sha256": sha256_value("x"), "embedding": [0.0, 0.0], "embedding_dim": 2, "memory_key_embedding": [0.0, 0.0], @@ -340,6 +361,10 @@ def __init__(self, stored_now: dict[str, Any]) -> None: self.written: list[dict[str, Any]] = [] async def query_all(self, sql: str, params: Any = None) -> list[dict]: + if "FROM loom.memory_refusals" in sql: + return [] + if "FROM loom.memory FINAL" in sql: + return [_canonical(self.stored_now)] if "system.columns" in sql: return [{"name": c} for c in _MEMORY_ROW] if "keys" in (params or {}): @@ -407,3 +432,18 @@ async def test_a_key_only_rewrite_keeps_the_value_vector_stored_now() -> None: n = await rb.reembed_table(ch, _Emb(), "memory_embeddings", key_only=True, log=lambda *_: None) assert n == 1 and ch.written[0]["embedding"] == [0.5, 0.5] and ch.written[0]["version"] == 8 assert ch.written[0]["memory_key_embedding"] == [0.1, 0.2] + + +async def test_a_withheld_publication_is_not_counted_or_deleted_as_replaced(monkeypatch): + from server.memory import store + + async def hold(rows, **kwargs): + return [] + + monkeypatch.setattr(store, "insert_embedding_rows", hold) + ch = _KeyOnlyCH(dict(_MEMORY_ROW)) + held = set() + count = await rb.reembed_table( + ch, _Emb(), "memory_embeddings", key_only=True, left_orgs=held, log=lambda *_: None + ) + assert count == 0 and held == {"o"} and ch.written == [] diff --git a/tests/unit/test_restore_mirrors_all_types.py b/tests/unit/test_restore_mirrors_all_types.py index ffa21f44b9..63cc57e51a 100644 --- a/tests/unit/test_restore_mirrors_all_types.py +++ b/tests/unit/test_restore_mirrors_all_types.py @@ -18,6 +18,7 @@ import pytest from server.core import leases +from server.core.hashing import sha256_value from server.memory import store as memory_repo from server.memory.api.models_crud import MemoryRestoreInput from server.memory.app import handlers as mem_handlers @@ -67,6 +68,7 @@ def _row(memory_type: str, **over: Any) -> MemoryRow: "memory_key": "alert:disk-full:host-7", "memory_type": memory_type, "value": {"state": "firing"}, + "content_sha256": sha256_value({"state": "firing"}), "value_text": '{"state": "firing"}', "confidence": 1.0, "version": 100, @@ -101,6 +103,7 @@ def _row(memory_type: str, **over: Any) -> MemoryRow: class _CH: def __init__(self, emb_rows: list[dict[str, Any]], *, fail: tuple[str, ...] = ()) -> None: + self.heads = {} self.emb_rows = emb_rows self.fail = fail self.inserts: list[tuple[str, list[dict[str, Any]], dict[str, Any]]] = [] @@ -112,6 +115,8 @@ async def query_one(self, *_: Any, **__: Any) -> dict[str, Any] | None: async def query_all(self, sql: str, params: Any = None, **_: Any) -> list[dict[str, Any]]: self.sqls.append(sql) + if params and "keys" in params and "FROM loom.memory FINAL" in sql: + return [self.heads[k] for k in params["keys"] if k in self.heads] if "FROM loom.memory_embeddings" in sql: return [dict(r) for r in self.emb_rows] return [] @@ -120,6 +125,8 @@ async def insert(self, table: str, rows: list[dict[str, Any]], settings: Any = N if table in self.fail: raise RuntimeError(f"insert into {table} unavailable") self.inserts.append((table, [dict(r) for r in rows], settings or {})) + if table == "loom.memory": + self.heads.update({memory_repo.identity_key(r): dict(r) for r in rows}) async def execute(self, *_: Any, **__: Any) -> None: return None @@ -151,6 +158,9 @@ def _query_one_row(existing: MemoryRow, **over: Any) -> dict[str, Any]: def _wire(monkeypatch: pytest.MonkeyPatch, existing: MemoryRow, ch: _CH) -> None: + ch.heads[memory_repo.identity_key(existing)] = memory_repo.build_memory_version_row(existing) + for row in ch.emb_rows: + row.setdefault("content_sha256", existing.content_sha256) monkeypatch.setattr(mem_handlers, "get_client", lambda: ch) monkeypatch.setattr(_stack_mod, "get_client", lambda: ch) monkeypatch.setattr(_restore_mod, "get_client", lambda: ch) diff --git a/tests/unit/test_schema_baseline.py b/tests/unit/test_schema_baseline.py index 76fbf08655..f0aff2c8dd 100644 --- a/tests/unit/test_schema_baseline.py +++ b/tests/unit/test_schema_baseline.py @@ -170,6 +170,7 @@ def test_the_journal_files() -> None: "112_code_repo_git_proxy_identity.sql", "113_oauth_verified_email.sql", "114_memory_proposal_review.sql", + "115_memory_refusals.sql", ] diff --git a/tests/unit/test_session_digest_prompt.py b/tests/unit/test_session_digest_prompt.py index b3640c69ca..a8b8b6758b 100644 --- a/tests/unit/test_session_digest_prompt.py +++ b/tests/unit/test_session_digest_prompt.py @@ -1,76 +1,53 @@ -"""The session digest's prompt and output budget, and the transcript cap -behind `memory_session_digest_max_input_chars`.""" +"""Digest text is extractive, bounded and independent of model-generated prose.""" from __future__ import annotations -from types import SimpleNamespace +from datetime import datetime from typing import Any import pytest -from server.memory.app import session_summary as ss +from server.memory.domain.summary_provenance import render_summary, source_text +from tests.unit import test_session_summaries +from tests.unit.test_memory_verify import _row +writer = test_session_summaries.writer -class _LLM: - def __init__(self) -> None: - self.calls: list[tuple[str, str, int]] = [] - async def complete(self, system: str, user: str, max_tokens: int = 0, **kw: Any): - self.calls.append((system, user, max_tokens)) - return SimpleNamespace( - text="Events:\n- on 2024-03-11 the user ran a 5k", tokens_in=10, tokens_out=5 - ) +def test_renderer_preserves_source_text_without_inventing_facts() -> None: + facts = ["Lives in Lisbon.", "Deployments use Tuesday and Thursday."] + assert render_summary(facts) == "Lives in Lisbon.\nDeployments use Tuesday and Thursday." + assert render_summary([]) == "" -class _Emb: - model_name = "fake-embed" - dim = 2 - - async def embed(self, text: str) -> list[float]: - return [0.1, 0.2] - - async def embed_many(self, texts: list[str]) -> list[list[float]]: - return [await self.embed(text) for text in texts] +def test_structured_source_values_have_a_stable_rendering() -> None: + first = _row(value={"city": "Lisbon", "age": 30}) + second = _row(value={"age": 30, "city": "Lisbon"}) + now = datetime.now() + assert ( + source_text(first, now=now) + == source_text(second, now=now) + == '{"age": 30, "city": "Lisbon"}' + ) -@pytest.fixture -def env(monkeypatch: pytest.MonkeyPatch) -> SimpleNamespace: - import server.llm.client as llm_client - from server.memory.app import bulk_set as _bulk_set_mod - from server.memory.app import session_summary as _session_summary_mod - from server.settings import settings +async def test_digest_generation_never_calls_a_model(writer: Any, monkeypatch: Any) -> None: + from server.memory.app.session_summary import _write_session_summary - llm = _LLM() - monkeypatch.setattr(llm_client, "get_llm_client", lambda: llm) - monkeypatch.setattr(_bulk_set_mod, "get_embedder", lambda: _Emb()) - monkeypatch.setattr(_session_summary_mod, "get_embedder", lambda: _Emb()) - monkeypatch.setattr(settings, "memory_session_digest_max_input_chars", 3000) - return SimpleNamespace(llm=llm, settings=settings, monkeypatch=monkeypatch) + def unexpected() -> Any: + raise AssertionError("a model could introduce text without a source") + monkeypatch.setattr("server.llm.client.get_llm_client", unexpected) + await _write_session_summary(writer.input, writer.state) + writer.ch.insert.assert_awaited_once() -def _msgs(n: int = 3, size: int = 100) -> list[dict[str, str]]: - return [{"role": "user", "content": ("x" * size)} for _ in range(n)] +@pytest.mark.parametrize("basis", ["created_at", "temporal_anchor", "valid_at"]) +def test_a_source_cannot_lend_text_before_its_valid_time_lower_bound(basis): + from datetime import timedelta -@pytest.mark.asyncio -async def test_the_digest_is_dated_prose_under_its_budget(env) -> None: - draft = await ss._draft_session_summary( - SimpleNamespace(session_id="s"), _msgs(), "2024-03-11", operator_policy="" - ) - system, user, max_tokens = env.llm.calls[0] - assert system == ss._SESSION_SUMMARY_SYSTEM and max_tokens == 160 - assert "Session date: 2024-03-11" in user - assert draft is not None and draft.embedding_model == "fake-embed" - - -def test_transcript_cap_follows_the_setting(env) -> None: - long = _msgs(n=10, size=1000) # ~10k chars - p = ss._build_session_summary_user_prompt(long, "2024-01-01") - assert len(p) < 3000 + 200 - env.monkeypatch.setattr(env.settings, "memory_session_digest_max_input_chars", 12000) - p2 = ss._build_session_summary_user_prompt(long, "2024-01-01") - assert len(p2) > 9000 - env.monkeypatch.setattr(env.settings, "memory_session_digest_max_input_chars", 0) - assert len(ss._build_session_summary_user_prompt(long, "2024-01-01")) < 3000 + 200, ( - "0 = default" - ) + now = datetime(2026, 1, 2) + source = _row(value="a later fact", created_at=now) + if basis != "created_at": + source = source.model_copy(update={basis: now, "created_at": now - timedelta(days=2)}) + assert source_text(source, now=now, as_of=now - timedelta(days=1)) is None diff --git a/tests/unit/test_session_outcomes.py b/tests/unit/test_session_outcomes.py index c9067f85c0..6fd1e74c0c 100644 --- a/tests/unit/test_session_outcomes.py +++ b/tests/unit/test_session_outcomes.py @@ -16,6 +16,16 @@ from server.memory.store.session_outcomes import OUTCOMES_TABLE +@pytest.fixture +def qualified_sessions(monkeypatch): + """Outcome policy cases start after source proof, tested separately below.""" + + async def qualified(ch, rows): + return rows + + monkeypatch.setattr("server.memory.store.summary_sources.verified_summary_batch", qualified) + + class FakeCH: def __init__(self) -> None: self.all_returns: list[list[dict[str, Any]]] = [] @@ -140,6 +150,7 @@ def test_build_outcome_row_validates_and_stamps_a_version() -> None: @pytest.mark.asyncio +@pytest.mark.usefixtures("qualified_sessions") async def test_label_sessions_judges_unlabelled_summaries_and_isolates_failures() -> None: ch = FakeCH() ch.all_returns.append( @@ -237,6 +248,7 @@ def _neg(sid="bad", conf=0.9): @pytest.mark.asyncio +@pytest.mark.usefixtures("qualified_sessions") async def test_distil_pairs_a_failed_session_with_its_nearest_success_and_writes_one_reflection() -> ( None ): @@ -295,9 +307,10 @@ async def writer(**kw): assert "s.user_id = {uid:String}" in sql and params["uid"] == "u" assert "o.agent = s.agent" in sql and "o.user_id = s.user_id" in sql assert params["q"] == [0.1, 0.2, 0.3] - # the contrast prompt carried both digests and the judged reason + # The contrast prompt carries proven digests; an unproven judged reason stays out. system, user = llm.calls[0] - assert "Never ran the tests." in user and "ran the suite" in user and "declared done" in user + assert "Never ran the tests." not in user + assert "ran the suite" in user and "declared done" in user w = written[0] assert (w["neg_session"], w["pos_session"]) == ("bad", "good") assert w["lessons"] == ["Run the test suite before declaring a parser change done."] @@ -307,6 +320,7 @@ async def writer(**kw): @pytest.mark.asyncio +@pytest.mark.usefixtures("qualified_sessions") async def test_distil_skips_when_no_neighbour_is_close_enough_or_nothing_was_learned() -> None: ch = FakeCH() ch.all_returns.append([_neg("a"), _neg("b"), _neg("c")]) @@ -497,3 +511,19 @@ async def _scoped(table, *, org, scope_where, params, ch, lightweight=False): dict(saga_steps.USER_STEPS)["session_outcomes.delete"] is memory_steps.delete_session_outcomes_by_user ) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("contrast", [False, True]) +async def test_an_unproven_session_never_reaches_judgement_or_contrast(contrast): + ch = FakeCH() + ch.all_returns.append([_neg()]) + llm = FakeLLM(["{}"]) + result = so.DistillResult() + if contrast: + await so.distil_contrasts(ch, llm, so.DistillConfig(), result) + assert result.contrast_candidates == 0 + else: + await so.label_sessions(ch, llm, so.DistillConfig(), result) + assert result.sessions_scanned == 0 + assert llm.calls == [] and ch.inserted == [] diff --git a/tests/unit/test_session_summaries.py b/tests/unit/test_session_summaries.py index a192155144..76f4ba8c2b 100644 --- a/tests/unit/test_session_summaries.py +++ b/tests/unit/test_session_summaries.py @@ -5,16 +5,17 @@ loom.session_summaries INSERT rows. Guards the RMT(version) silent-data-loss class (a write omits a column -> FINAL surfaces the schema DEFAULT over a prior real value). - - _write_session_summary — the inline ingest writer. Stubs the LLM, embedder, - and CH client so the row shape (mirror-every-column), the date-label carriage, - and the fail-open skips run CH-free / network-free. + - _write_session_summary — the inline ingest writer. Stubs authoritative source + heads, the embedder and the CH client to check complete fact rendering, + row shape, date labels and skipped writes without network access. """ from __future__ import annotations -from dataclasses import dataclass, field from datetime import datetime +from types import SimpleNamespace from typing import Any +from unittest.mock import ANY, AsyncMock import pytest @@ -81,264 +82,156 @@ def test_defaults_match_schema_for_omitted_columns() -> None: assert row["user_id"] == "-" -# ---------------------------------------------------------------------- -# Pure helpers — _top_key_facts + _build_session_summary_user_prompt -# ---------------------------------------------------------------------- - - -def test_top_key_facts_never_carry_a_credential(monkeypatch: pytest.MonkeyPatch) -> None: - # The digest is listed by memory.digests and sent to the outcome judge; a fact whose - # text reads as high stays out of its key facts, whatever the autodetect setting. - from server.memory.app.extract.handler import _top_key_facts - from server.settings import settings - - monkeypatch.setattr(settings, "memory_sensitivity_autodetect", False) - token = "ghp_" + "a" * 36 - facts = [ - {"value": f"the deploy token is {token}"}, - {"value": {"note": "deploy token:\n" + token}}, # JSON-escaped, the newline hides it - {"value": "user lives in Berlin"}, - ] - assert _top_key_facts(facts) == ["user lives in Berlin"] - - -def test_the_judge_prompt_drops_a_stored_credential_key_fact() -> None: - # Digests an earlier build wrote may hold one; the outcome judge's prompt drops it. - from server.memory.app.session_outcomes import judge_prompt - - token = "ghp_" + "a" * 36 - _system, user = judge_prompt("a session", [f"the token is {token}", "ran the tests"]) - assert token not in user and "ran the tests" in user - - -def test_top_key_facts_dedups_and_caps() -> None: - from server.memory.app.extract.handler import ( - _SESSION_SUMMARY_MAX_KEY_FACTS, - _top_key_facts, +@pytest.fixture +def writer(monkeypatch: Any) -> Any: + from server.memory import store + from server.memory.app import digest_chain, session_summary + from tests.unit.test_memory_verify import _row + + source = _row( + namespace="n", + agent="a", + user_id="u", + session_id="s", + memory_key="known", + value="current fact", ) - - facts = [ - {"value": "user lives in Berlin"}, - {"value": "user lives in Berlin"}, # dup -> dropped - {"value": {"city": "Berlin"}}, # dict -> json string surface - {"value": ""}, # empty -> dropped - {"value": "fact a"}, - {"value": "fact b"}, - {"value": "fact c"}, - {"value": "fact d"}, - {"value": "fact e"}, - ] - out = _top_key_facts(facts) - assert len(out) <= _SESSION_SUMMARY_MAX_KEY_FACTS - assert out[0] == "user lives in Berlin" - # The dict value coerces to a JSON string surface, not dropped. - assert any("Berlin" in s and "{" in s for s in out) - - -def test_user_prompt_carries_date_and_truncates() -> None: - from server.memory.app.extract.handler import ( - _SESSION_SUMMARY_MAX_INPUT_CHARS, - _build_session_summary_user_prompt, + state = SimpleNamespace( + written=[ + { + "memory_id": source.memory_id, + "memory_key": source.memory_key, + "version": source.version, + } + ], + extracted_facts=[{"memory_key": "uncommitted", "value": "retired transcript fact"}], + messages=[{"role": "user", "content": "retired transcript fact"}], + observation_date=None, ) + ch = SimpleNamespace(insert=AsyncMock(), query_all=AsyncMock(return_value=[])) + from server.memory.store.rows import identity_key - msgs = [{"role": "user", "content": "x" * 10000}] - prompt = _build_session_summary_user_prompt(msgs, "2024-03-11") - assert "Session date: 2024-03-11" in prompt - assert "[user]" in prompt - # The transcript body is bounded; the header + framing add a little. - assert len(prompt) < _SESSION_SUMMARY_MAX_INPUT_CHARS + 200 - - -def test_user_prompt_unknown_date() -> None: - from server.memory.app.extract.handler import _build_session_summary_user_prompt - - prompt = _build_session_summary_user_prompt([{"role": "user", "content": "hi"}], None) - assert "Session date: unknown" in prompt + monkeypatch.setattr( + "server.memory.store.summary_sources.memory_rows_by_identity_keys", + AsyncMock(return_value={identity_key(source): source}), + ) + read = AsyncMock(return_value=source) + vector = AsyncMock(return_value=[1.0, 0.0]) + link = AsyncMock() + monkeypatch.setattr(store, "get_by_id", read) + monkeypatch.setattr(session_summary, "get_client", lambda: ch) + monkeypatch.setattr(session_summary, "get_embedder", lambda: SimpleNamespace(model_name="fake")) + monkeypatch.setattr(session_summary, "document_vector", vector) + monkeypatch.setattr(digest_chain, "link_digest", link) + inp = SimpleNamespace( + org="o", namespace="n", agent="a", user_id="u", session_id="s", run_id="run" + ) + return SimpleNamespace( + input=inp, state=state, source=source, ch=ch, read=read, vector=vector, link=link + ) -# ---------------------------------------------------------------------- -# _write_session_summary — end-to-end with stubs -# ---------------------------------------------------------------------- +async def test_digest_contains_only_committed_current_sources(writer: Any) -> None: + from server.memory.app.session_summary import _write_session_summary + await _write_session_summary(writer.input, writer.state) + table, rows = writer.ch.insert.call_args.args + assert table == "loom.session_summaries" + row = rows[0] + assert set(row) == set(SESSION_SUMMARY_PERSISTED_COLS) + assert row["summary_text"] == "current fact" + assert row["key_facts"] == ["current fact"] + assert row["derived_from"] == ["known"] + assert "retired transcript fact" not in str(row) + assert row["run_id"] == "run" + writer.vector.assert_awaited_once_with(ANY, "current fact") + writer.link.assert_awaited_once() + + +@pytest.mark.parametrize( + "change", + [ + {"status": "rejected"}, + {"status": "proposed"}, + {"sensitivity": "high"}, + {"version": 101}, + {"session_id": "another-session"}, + {"value_truncated": True}, + ], +) +async def test_unqualified_sources_are_not_embedded_or_written(writer: Any, change: Any) -> None: + from server.memory.app.session_summary import _write_session_summary -@dataclass -class _FakeInput: - org: str = "o" - namespace: str = "n" - agent: str = "a" - user_id: str = "u" - session_id: str = "sess-1" + writer.read.return_value = writer.source.model_copy(update=change) + await _write_session_summary(writer.input, writer.state) + writer.ch.insert.assert_not_awaited() + writer.vector.assert_not_awaited() -@dataclass -class _FakeState: - extracted_facts: list[dict[str, Any]] = field(default_factory=list) - observation_date: str | None = None - operator_policy: str = "" +async def test_no_committed_fact_produces_no_digest(writer: Any) -> None: + from server.memory.app.session_summary import _write_session_summary + writer.state.written = [] + await _write_session_summary(writer.input, writer.state) + writer.read.assert_not_awaited() + writer.ch.insert.assert_not_awaited() -class _FakeLLM: - provider = "fake" - model = "fake-echo-1" - def __init__(self, text: str = "on 2024-03-11, the user moved to Berlin.") -> None: - self._text = text - self.calls: list[tuple[str, str, int]] = [] +async def test_a_source_read_failure_is_reported_to_the_projection_caller(writer: Any) -> None: + from server.memory.app.session_summary import _write_session_summary - async def complete(self, system, user, max_tokens=1024, response_format=None): - from server.llm.client import LLMResult + writer.read.side_effect = TimeoutError("authority unavailable") + with pytest.raises(TimeoutError): + await _write_session_summary(writer.input, writer.state) + writer.ch.insert.assert_not_awaited() - self.calls.append((system, user, max_tokens)) - return LLMResult( - text=self._text, - tokens_in=10, - tokens_out=5, - provider=self.provider, - model=self.model, - ) +async def test_digest_cap_keeps_whole_facts_instead_of_partial_provenance( + writer: Any, monkeypatch: Any +) -> None: + from server.memory.app.session_summary import _write_session_summary + from server.settings import settings -class _FakeEmbedder: - def __init__(self, vec: list[float] | None = None) -> None: - self.model_name = "fake-ngram-8" - self.dim = 8 - self._vec = [0.1] * 8 if vec is None else vec + monkeypatch.setattr(settings, "memory_session_digest_max_input_chars", 500) + writer.read.return_value = writer.source.model_copy(update={"value": "x" * 501}) + await _write_session_summary(writer.input, writer.state) + writer.ch.insert.assert_not_awaited() + writer.vector.assert_not_awaited() - async def embed(self, text: str) -> list[float]: - return list(self._vec) - async def embed_many(self, texts: list[str]) -> list[list[float]]: - return [await self.embed(text) for text in texts] +async def test_a_missing_observation_date_stays_undated(writer: Any) -> None: + from server.memory.app.session_summary import _write_session_summary + await _write_session_summary(writer.input, writer.state) + assert writer.ch.insert.call_args.args[1][0]["valid_at"] is None -class _FakeCH: - def __init__(self) -> None: - self.inserts: list[tuple[str, list[dict[str, Any]], dict[str, Any]]] = [] - async def insert(self, table, rows, settings=None): - self.inserts.append((table, rows, settings or {})) +async def test_a_source_removed_while_embedding_is_not_published(writer, monkeypatch): + from server.memory.app.session_summary import _write_session_summary + async def removed_during_embedding(*args, **kwargs): + monkeypatch.setattr( + "server.memory.store.summary_sources.memory_rows_by_identity_keys", + AsyncMock(return_value={}), + ) + return [1.0, 0.0] -@pytest.fixture -def _patch_clients(monkeypatch): - """Patch get_llm_client / get_embedder / get_client used by the writer.""" - import server.llm.client as llm_mod - from server.memory.app import bulk_set as _bulk_set_mod - from server.memory.app import session_summary as _session_summary_mod - - llm = _FakeLLM() - embedder = _FakeEmbedder() - ch = _FakeCH() - # _write_session_summary imports get_llm_client lazily from server.llm.client - monkeypatch.setattr(llm_mod, "get_llm_client", lambda: llm) - monkeypatch.setattr(_bulk_set_mod, "get_embedder", lambda: embedder) - monkeypatch.setattr(_session_summary_mod, "get_embedder", lambda: embedder) - monkeypatch.setattr(_bulk_set_mod, "get_client", lambda: ch) - monkeypatch.setattr(_session_summary_mod, "get_client", lambda: ch) - return llm, embedder, ch - - -@pytest.mark.asyncio -async def test_write_summary_inserts_full_row(_patch_clients) -> None: - from server.memory.app.extract.handler import _write_session_summary - - llm, embedder, ch = _patch_clients - state = _FakeState( - extracted_facts=[{"value": "user moved to Berlin"}, {"value": "likes jazz"}], - observation_date="2024-03-11", + monkeypatch.setattr( + "server.memory.app.session_summary.document_vector", removed_during_embedding ) - await _write_session_summary(_FakeInput(), [{"role": "user", "content": "hi"}], state) - - assert len(ch.inserts) == 1 - table, rows, settings = ch.inserts[0] - assert table == "loom.session_summaries" - # Synchronous insert so the immediate post-ingest read sees it. - assert settings.get("async_insert") == 0 - assert settings.get("wait_for_async_insert") == 1 - assert len(rows) == 1 - row = rows[0] - # Mirror-every-column: every persisted column is present in the insert dict. - assert set(row) == set(SESSION_SUMMARY_PERSISTED_COLS) - assert row["summary_text"] - assert row["summary_embedding"] == [0.1] * 8 - assert row["embedding_model"] == "fake-ngram-8" - assert row["embedding_dim"] == 8 - # key_facts derived from extracted_facts (no extra LLM call). - assert row["key_facts"] == ["user moved to Berlin", "likes jazz"] - # valid_at carries the parsed session date (LABEL). - assert isinstance(row["valid_at"], datetime) - assert row["valid_at"].year == 2024 and row["valid_at"].month == 3 - # Exactly one cheap LLM call, max_tokens bounded. - assert len(llm.calls) == 1 - assert llm.calls[0][2] == 160 - - -@pytest.mark.asyncio -async def test_write_summary_null_valid_at_when_undated(_patch_clients) -> None: - from server.memory.app.extract.handler import _write_session_summary - - _llm, _embedder, ch = _patch_clients - state = _FakeState(extracted_facts=[{"value": "f"}], observation_date=None) - await _write_session_summary(_FakeInput(), [{"role": "user", "content": "hi"}], state) - - row = ch.inserts[0][1][0] - assert row["valid_at"] is None - + await _write_session_summary(writer.input, writer.state) + writer.ch.insert.assert_not_awaited() + writer.link.assert_not_awaited() -@pytest.mark.asyncio -async def test_write_summary_skips_when_no_facts(_patch_clients) -> None: - from server.memory.app.extract.handler import _write_session_summary - llm, _embedder, ch = _patch_clients - state = _FakeState(extracted_facts=[], observation_date="2024-03-11") - await _write_session_summary(_FakeInput(), [{"role": "user", "content": "hi"}], state) +async def test_a_current_source_after_many_missing_candidates_can_still_form_a_digest(writer): + from server.memory.app.session_summary import _write_session_summary - # Nothing to summarize -> no LLM call, no insert. - assert llm.calls == [] - assert ch.inserts == [] - - -@pytest.mark.asyncio -async def test_write_summary_skips_on_empty_digest(monkeypatch, _patch_clients) -> None: - from server.memory.app.extract.handler import _write_session_summary - - llm, _embedder, ch = _patch_clients - llm._text = " " # whitespace-only digest - state = _FakeState(extracted_facts=[{"value": "f"}], observation_date="2024-03-11") - await _write_session_summary(_FakeInput(), [{"role": "user", "content": "hi"}], state) - - assert ch.inserts == [] # empty digest -> no row - - -@pytest.mark.asyncio -@pytest.mark.parametrize("vec", [[], [0.1], [float("nan")] * 8, [float("inf")] * 8]) -async def test_write_summary_skips_on_invalid_embedding(monkeypatch, _patch_clients, vec) -> None: - from server.memory.app import bulk_set as _bulk_set_mod - from server.memory.app import session_summary as _session_summary_mod - from server.memory.app.extract.handler import _write_session_summary - - llm, _embedder, ch = _patch_clients - monkeypatch.setattr(_bulk_set_mod, "get_embedder", lambda: _FakeEmbedder(vec=vec)) - monkeypatch.setattr(_session_summary_mod, "get_embedder", lambda: _FakeEmbedder(vec=vec)) - state = _FakeState(extracted_facts=[{"value": "f"}], observation_date="2024-03-11") - await _write_session_summary(_FakeInput(), [{"role": "user", "content": "hi"}], state) - - assert ch.inserts == [] # Invalid vectors never become summary rows. - - -@pytest.mark.asyncio -async def test_write_summary_propagates_for_caller_failopen(monkeypatch, _patch_clients) -> None: - """The writer itself raises on LLM error; the CALLER wraps it (the - set_from_messages try/except records a projection failure). So the writer's - contract is: raise, don't swallow — the fail-open boundary is one level up.""" - import server.llm.client as llm_mod - from server.memory.app.extract.handler import _write_session_summary - - class _BoomLLM: - async def complete(self, *a, **k): - raise RuntimeError("llm down") - - monkeypatch.setattr(llm_mod, "get_llm_client", lambda: _BoomLLM()) - state = _FakeState(extracted_facts=[{"value": "f"}], observation_date="2024-03-11") - with pytest.raises(RuntimeError, match="llm down"): - await _write_session_summary(_FakeInput(), [{"role": "user", "content": "hi"}], state) + committed = writer.state.written[0] + writer.state.written = [dict(memory_id=f"missing-{i}", version=100) for i in range(48)] + [ + committed + ] + writer.read.side_effect = [None] * 48 + [writer.source] + await _write_session_summary(writer.input, writer.state) + assert writer.read.await_count == 49 + assert writer.ch.insert.call_args.args[1][0]["key_facts"] == [writer.source.value] diff --git a/tests/unit/test_session_summary_retrieval.py b/tests/unit/test_session_summary_retrieval.py index e25c5645e4..927eb4ee93 100644 --- a/tests/unit/test_session_summary_retrieval.py +++ b/tests/unit/test_session_summary_retrieval.py @@ -18,6 +18,15 @@ ) +@pytest.fixture(autouse=True) +def _qualified_sources(monkeypatch): + # These cases isolate ranking and hit mapping; source proof has its own suite. + async def qualified(ch, rows, **kwargs): + return rows + + monkeypatch.setattr("server.memory.store.summary_sources.current_summary_sources", qualified) + + class StubCH: def __init__(self, returns: list[dict[str, Any]] | None = None, raise_on_query: bool = False): self.returns = returns or [] diff --git a/tests/unit/test_set_from_messages_fail_closed.py b/tests/unit/test_set_from_messages_fail_closed.py index f175782886..d39a96479a 100644 --- a/tests/unit/test_set_from_messages_fail_closed.py +++ b/tests/unit/test_set_from_messages_fail_closed.py @@ -12,6 +12,7 @@ from __future__ import annotations from typing import Any +from unittest.mock import AsyncMock import pytest from fastapi import HTTPException @@ -19,6 +20,12 @@ from server.memory.api.models_bulk import REFUSED, MemorySetFromMessagesInput, _BulkSetResult +@pytest.fixture(autouse=True) +def _no_refused_intake(monkeypatch): + # Pipeline and projection cases; actual refusal admission is tested separately. + monkeypatch.setattr("server.memory.store.refusals.check_intake", AsyncMock()) + + def _stub_projections(monkeypatch: pytest.MonkeyPatch) -> None: import server.memory.app.extract.handler as mx from server.memory.app import bulk_set as _bulk_set_mod @@ -36,7 +43,6 @@ async def _none(*args, **kwargs): monkeypatch.setattr(mx, "_link_written_memories_to_evidence", _zero) monkeypatch.setattr(mx, "write_memory_chunks", _none) monkeypatch.setattr(mx, "_write_session_summary", _none) - monkeypatch.setattr(mx, "_draft_session_summary", _none) monkeypatch.setattr(mx, "record_input_projection_failure", _none) monkeypatch.setattr(_bulk_set_mod, "record_input_projection_failure", _none) @@ -153,37 +159,25 @@ async def _fake_typed( assert bool(out.degraded) is not projected -@pytest.mark.asyncio -async def test_any_pipeline_failure_stops_the_digest_draft( +async def test_any_pipeline_failure_prevents_digest_generation( monkeypatch: pytest.MonkeyPatch, ) -> None: - """The digest draft (an LLM call and an embed) runs beside the extractor; a pipeline - failure that is not an ExtractionError must stop it too, not leave it spending.""" - import asyncio + from unittest.mock import AsyncMock import server.memory.app.extract.handler as mx import server.memory.app.pipeline as pipe _stub_projections(monkeypatch) - cancelled: list[bool] = [] + summary = AsyncMock() - async def _slow_draft(*args, **kwargs): - try: - await asyncio.sleep(30) - except asyncio.CancelledError: - cancelled.append(True) - raise - - async def _pipeline_raises(steps, state): - await asyncio.sleep(0) # the draft has started + async def fail(*args, **kwargs): raise RuntimeError("bulk write refused") - monkeypatch.setattr(mx, "_draft_session_summary", _slow_draft) - monkeypatch.setattr(pipe, "run_pipeline", _pipeline_raises) + monkeypatch.setattr(mx, "_write_session_summary", summary) + monkeypatch.setattr(pipe, "run_pipeline", fail) with pytest.raises(RuntimeError, match="bulk write refused"): await mx.handle_memory_set_from_messages(_input()) - await asyncio.sleep(0) - assert cancelled == [True] + summary.assert_not_awaited() @pytest.mark.asyncio diff --git a/tests/unit/test_set_from_messages_refusals.py b/tests/unit/test_set_from_messages_refusals.py index 95ad782017..e89b169025 100644 --- a/tests/unit/test_set_from_messages_refusals.py +++ b/tests/unit/test_set_from_messages_refusals.py @@ -12,11 +12,18 @@ async def test_a_schema_request_without_a_key_writes_no_evidence( ) -> None: """The 400 comes before the evidence write starts, so a refused request leaves no evidence rows behind.""" + from types import SimpleNamespace + from unittest.mock import AsyncMock + from fastapi import HTTPException from server.memory.api.models_bulk import MemorySetFromMessagesInput from server.memory.app.extract import handler as hm + monkeypatch.setattr( + hm, "get_client", lambda: SimpleNamespace(query_all=AsyncMock(return_value=[])) + ) + started: list[object] = [] async def _evidence(*a: object, **k: object) -> list[str]: diff --git a/tests/unit/test_shared_and_documents_list_repo.py b/tests/unit/test_shared_and_documents_list_repo.py index 5f40c82053..1ac7b835d1 100644 --- a/tests/unit/test_shared_and_documents_list_repo.py +++ b/tests/unit/test_shared_and_documents_list_repo.py @@ -10,14 +10,18 @@ from __future__ import annotations +from contextlib import asynccontextmanager from datetime import UTC, datetime from typing import Any import pytest +from server.core.hashing import sha256_value from server.core.tag_sql import TagFilter, tag_where from server.knowledge.store import documents as documents_repo from server.memory.store import shared as shared_repo +from server.memory.store.rows import build_memory_version_row +from tests.unit.test_memory_verify import _row # The any-case tag matches, as the list reads render them. _FOLDED_ANY = tag_where(TagFilter(("x",), "folded_any"), {}) @@ -26,6 +30,7 @@ class StubCH: def __init__(self) -> None: + self.source = None self.query_one_calls: list[tuple[str, dict[str, Any]]] = [] self.query_one_returns: list[dict[str, Any] | None] = [] self.query_all_calls: list[tuple[str, dict[str, Any]]] = [] @@ -39,15 +44,30 @@ async def query_one(self, sql, params=None): async def query_all(self, sql, params=None): self.query_all_calls.append((sql, params or {})) + if "memory_refusals" in sql: + return [] + if "FROM loom.memory FINAL" in sql: + return [build_memory_version_row(self.source)] if self.source else [] return self.query_all_returns.pop(0) if self.query_all_returns else [] - async def execute(self, sql, params=None): + async def execute(self, sql, params=None, *, settings=None): self.execute_calls.append((sql, params or {})) async def insert(self, table, rows, settings=None): self.insert_calls.append((table, rows, settings or {})) +@asynccontextmanager +async def _approved_source(ch, value="source fact"): + from server.memory.store.refusals import source_publication + + ch.source = _row( + org="clickhouse", namespace="ns", agent="author", memory_key="source", value=value + ) + async with source_publication(ch, ch.source, "k"): + yield ch.source + + # ---------------------------------------------------------------------- # shared_repo # ---------------------------------------------------------------------- @@ -56,15 +76,17 @@ async def insert(self, table, rows, settings=None): @pytest.mark.asyncio async def test_insert_shared_row_forces_synchronous_visibility() -> None: ch = StubCH() - await shared_repo.insert_shared_row( - { - **dict.fromkeys(shared_repo.SHARED_PERSISTED_COLS), - "org": "clickhouse", - "namespace": "ns", - "memory_key": "k", - }, - ch=ch, - ) + async with _approved_source(ch, "source fact"): + await shared_repo.insert_shared_row( + { + **dict.fromkeys(shared_repo.SHARED_PERSISTED_COLS), + "org": "clickhouse", + "namespace": "ns", + "memory_key": "k", + "value": "source fact", + }, + ch=ch, + ) table, _, settings = ch.insert_calls[0] assert table == "loom.shared" # search-visibility contract: next FINAL read sees the row. @@ -98,10 +120,18 @@ async def test_insert_shared_embedding_uses_memory_embeddings_table() -> None: """The shared embedding lives on loom.memory_embeddings (the same table as private embeddings) — scope='shared' is the marker.""" ch = StubCH() - await shared_repo.insert_shared_embedding( - {"scope": "shared", "agent": "_shared"}, - ch=ch, - ) + async with _approved_source(ch, "source fact"): + await shared_repo.insert_shared_embedding( + { + "org": "clickhouse", + "namespace": "ns", + "memory_key": "k", + "content_sha256": sha256_value("source fact"), + "scope": "shared", + "agent": "_shared", + }, + ch=ch, + ) table, rows, _ = ch.insert_calls[0] assert table == "loom.memory_embeddings" assert rows[0]["agent"] == "_shared" @@ -116,10 +146,19 @@ async def test_insert_shared_embedding_writes_a_complete_builder_row() -> None: from server.memory.store.rows import EMBEDDING_PERSISTED_COLS ch = StubCH() - await shared_repo.insert_shared_embedding( - {"scope": "shared", "agent": "_shared", "content_excerpt": "dogs were barking"}, - ch=ch, - ) + async with _approved_source(ch, "dogs were barking"): + await shared_repo.insert_shared_embedding( + { + "org": "clickhouse", + "namespace": "ns", + "memory_key": "k", + "content_sha256": sha256_value("dogs were barking"), + "scope": "shared", + "agent": "_shared", + "content_excerpt": "dogs were barking", + }, + ch=ch, + ) row = ch.insert_calls[0][1][0] assert set(row) == set(EMBEDDING_PERSISTED_COLS) assert row["content_excerpt_lemma"] == "dog be bark" @@ -140,13 +179,16 @@ async def test_promote_carries_the_source_provenance_onto_the_shared_embedding( from server.memory.app import promote as _memory_promote written: list[dict[str, Any]] = [] + ch = StubCH() + insert_shared_embedding = shared_repo.insert_shared_embedding + monkeypatch.setattr(_memory_promote, "get_client", lambda: ch) class _Emb: model_name = "fake" dim = 2 - async def embed(self, text: str) -> list[float]: - return [0.1, 0.2] + async def embed_many(self, texts: list[str]) -> list[list[float]]: + return [[0.1, 0.2] for _ in texts] async def _no_peers(**kw: Any) -> list[Any]: return [] @@ -155,6 +197,7 @@ async def _shared_row(row: dict[str, Any], **kw: Any) -> None: return None async def _shared_emb(row: dict[str, Any], **kw: Any) -> None: + await insert_shared_embedding(row, ch=ch) written.append(row) monkeypatch.setattr(_memory_promote, "get_embedder", lambda: _Emb()) @@ -162,13 +205,13 @@ async def _shared_emb(row: dict[str, Any], **kw: Any) -> None: monkeypatch.setattr(shared_repo, "insert_shared_row", _shared_row) monkeypatch.setattr(shared_repo, "insert_shared_embedding", _shared_emb) - for verified_sha, want in (("a" * 64, 1), ("b" * 64, 0)): + for verified_sha, want in ((sha256_value({"v": "external fact"}), 1), ("b" * 64, 0)): src = memory_repo.MemoryRow.from_snapshot( dict( memory_id="11111111-1111-1111-1111-111111111111", sensitivity="", org="o", - namespace="team", + namespace="n", agent="a", user_id="-", session_id="-", @@ -180,7 +223,7 @@ async def _shared_emb(row: dict[str, Any], **kw: Any) -> None: source_class="external", verified_by="human:alice", verified_sha256=verified_sha, - content_sha256="a" * 64, + content_sha256=sha256_value({"v": "external fact"}), ) ) @@ -188,6 +231,7 @@ async def _get(*a: Any, _src: Any = src, **kw: Any) -> Any: return _src monkeypatch.setattr(memory_repo, "get_by_key", _get) + ch.source = src written.clear() await _memory_promote.handle_memory_promote( MemoryPromoteInput(org="o", namespace="n", agent="a", memory_key="k") @@ -216,13 +260,8 @@ async def test_fetch_shared_none_when_missing() -> None: @pytest.mark.asyncio -async def test_tombstone_shared_embedding_re_emits_a_versioned_row() -> None: - """memory.search(scope='shared') filters on tombstoned_at IS NULL; the - demote path lands the tombstone as a committed re-emit of the live shared - rows (every column carried, tombstoned_at set, fresh version) so the next - search does not see the row — and never as an ALTER UPDATE, whose patch - part would cost the fact leg its codes-first read.""" - from server.memory.store import EMBEDDING_PERSISTED_COLS +async def test_tombstone_shared_embedding_deletes_older_cached_versions() -> None: + """Shared retirement deletes older cache versions without publishing another copy.""" ch = StubCH() when = datetime(2026, 5, 17, tzinfo=UTC) @@ -243,19 +282,14 @@ async def test_tombstone_shared_embedding_re_emits_a_versioned_row() -> None: ] ) await shared_repo.tombstone_shared_embedding("clickhouse", "ns", "k", when, version=9, ch=ch) - assert ch.execute_calls == [] + assert len(ch.execute_calls) == 1 and not ch.insert_calls read_sql, read_params = ch.query_all_calls[0] assert "scope = 'shared'" in read_sql and "tombstoned_at IS NULL" in read_sql assert "FINAL" in read_sql and read_params["k"] == "k" - ((table, rows, settings),) = ch.insert_calls - assert table == "loom.memory_embeddings" and settings == { - "async_insert": 0, - "wait_for_async_insert": 1, - } - (row,) = rows - assert set(row) >= set(EMBEDDING_PERSISTED_COLS) - assert row["tombstoned_at"] == when and row["scope"] == "shared" - assert row["embedding"] == [0.1, 0.2, 0.3] and row["version"] == 9 # the demote's + delete_sql, delete_params = ch.execute_calls[0] + assert delete_sql.startswith("DELETE FROM loom.memory_embeddings") + assert "version < {version:UInt64}" in delete_sql + assert delete_params["scope"] == "shared" and delete_params["version"] == 9 @pytest.mark.asyncio @@ -368,3 +402,26 @@ async def test_list_with_chunk_counts_requires_every_tag_when_asked() -> None: sql, params = ch.query_all_calls[0] assert _FOLDED_ALL in sql assert params["tags"] == ["Specs", "onboarding"] + + +def test_promoted_vector_binds_verification_to_the_complete_served_value(): + from types import SimpleNamespace + + from server.memory.app.promote import _shared_vector_row + + source = _row( + value="complete served fact", + content_sha256="a" * 64, + verified_by="human:reviewer", + verified_sha256="a" * 64, + ) + row = _shared_vector_row( + source, + "k", + "display excerpt", + SimpleNamespace(model_name="m", dim=1), + [0.5], + datetime.now(UTC), + ) + assert row["content_sha256"] == sha256_value(source.value) + assert row["verified"] == 0 diff --git a/tests/unit/test_status_mirror.py b/tests/unit/test_status_mirror.py index 22b899216a..964956648d 100644 --- a/tests/unit/test_status_mirror.py +++ b/tests/unit/test_status_mirror.py @@ -9,21 +9,40 @@ import pytest +from server.core.hashing import sha256_value from server.memory import store as memory_repo from server.memory.store import lifecycle +@pytest.fixture +def mirror_publication(monkeypatch): + """Reconcile selection uses an acknowledged publisher; consent has separate cases.""" + + async def publish(rows, *, ch): + await ch.insert("loom.memory_embeddings", rows) + return rows + + monkeypatch.setattr("server.memory.store.embeddings.insert_embedding_rows", publish) + + class _CH: """Records inserts; answers the embeddings read with one live row per key.""" def __init__(self, emb_status: str = "active") -> None: self.inserts: list[tuple[str, list[dict[str, Any]]]] = [] self.emb_status = emb_status + self.heads: dict[Any, Any] = {} async def query_all( self, sql: str, params: dict[str, Any] | None = None ) -> list[dict[str, Any]]: p = params or {} + if "FROM loom.memory FINAL" in sql: + return [ + memory_repo.build_memory_version_row(self.heads[key]) + for key in p.get("keys", []) + if key in self.heads + ] if "FROM loom.memory_embeddings FINAL" in sql: # the keyed read: one identity per position of the six arrays return [ @@ -42,6 +61,7 @@ async def query_all( "status": self.emb_status, "version": 1, "content_excerpt": "x", + "content_sha256": sha256_value("x"), } for o, n, a, u, s_, k in zip( p["o"], p["n"], p["a"], p["u"], p["s"], p["k"], strict=True @@ -53,6 +73,13 @@ async def insert( self, table: str, rows: list[dict[str, Any]], settings: dict[str, Any] | None = None ) -> None: self.inserts.append((table, rows)) + if table == "loom.memory": + from server.memory.store.rows import identity_key + + for row in rows: + key = identity_key(row) + if key in self.heads: + self.heads[key] = self.heads[key].model_copy(update=row) def _existing(status: str = "active") -> Any: @@ -72,6 +99,13 @@ def _existing(status: str = "active") -> Any: def _wire(monkeypatch: pytest.MonkeyPatch, ch: _CH, existing: Any) -> None: + from server.memory.store.rows import MemoryRow, identity_key + + source = MemoryRow.from_snapshot( + {**vars(existing), "value": "x", "content_sha256": sha256_value("x")} + ) + ch.heads[identity_key(source)] = source + async def _get_by_id( memory_id: str, *, @@ -159,6 +193,7 @@ async def test_already_mirrored_row_is_not_rewritten(monkeypatch: pytest.MonkeyP @pytest.mark.asyncio +@pytest.mark.usefixtures("mirror_publication") async def test_reconcile_holds_back_a_key_the_authority_has_moved_past( monkeypatch: pytest.MonkeyPatch, ) -> None: @@ -229,6 +264,7 @@ async def _reconcile(*_: Any, **__: Any) -> dict[str, int]: @pytest.mark.asyncio +@pytest.mark.usefixtures("mirror_publication") async def test_reconcile_counts_on_dry_run_and_mirrors_otherwise() -> None: class _RCH(_CH): async def query_all( @@ -278,6 +314,7 @@ async def query_all( @pytest.mark.asyncio +@pytest.mark.usefixtures("mirror_publication") async def test_reconcile_reads_the_mirror_by_the_full_identity() -> None: """One memory_key lives under many identities, each with its own status. The divergence is computed per identity row and @@ -322,6 +359,7 @@ async def query_all( @pytest.mark.asyncio +@pytest.mark.usefixtures("mirror_publication") async def test_reconcile_reads_the_mirror_in_one_statement_per_batch() -> None: """`loom reconcile-status` hands up to 5,000 divergent identities to the mirror: one keyed read per 500 of them, not one read per identity.""" @@ -643,9 +681,12 @@ async def query_all( self, sql: str, params: dict[str, Any] | None = None ) -> list[dict[str, Any]]: if "FROM loom.memory FINAL" in sql: + if "keys" in (params or {}): + return await super().query_all(sql, params) assert (params or {})["key"] == "home" return [{"memory_id": "m1"}, {"memory_id": "m2"}] - self.vector_reads.append(sql) + if "FROM loom.memory_embeddings FINAL" in sql: + self.vector_reads.append(sql) rows = await super().query_all(sql, params) # Each vector row was copied at read + 1 since its memory row (version 1). return [{**r, "version": 2} for r in rows] @@ -668,10 +709,20 @@ async def _get_by_id(memory_id: str, **_: Any) -> Any: return heads[memory_id] async def _current(keys, **kwargs): - from server.memory.store.rows import MemoryRow, identity_key - rows = [MemoryRow.from_snapshot(vars(row)) for row in heads.values()] - return {identity_key(row): row for row in rows if identity_key(row) in keys} + return {key: row for key, row in ch.heads.items() if key in keys} + + from server.memory.store.rows import MemoryRow, identity_key + + ch.heads = { + identity_key(row): row + for row in ( + MemoryRow.from_snapshot( + {**vars(head), "value": "x", "content_sha256": sha256_value("x")} + ) + for head in heads.values() + ) + } monkeypatch.setattr(lifecycle, "get_by_id", _get_by_id) monkeypatch.setattr("server.memory.store.write_fence.memory_rows_by_identity_keys", _current) diff --git a/tests/unit/test_summary_leg_links.py b/tests/unit/test_summary_leg_links.py index 2d3a2dcbf8..b596e1ab48 100644 --- a/tests/unit/test_summary_leg_links.py +++ b/tests/unit/test_summary_leg_links.py @@ -5,6 +5,17 @@ from typing import Any +import pytest + + +@pytest.fixture(autouse=True) +def _qualified_sources(monkeypatch): + # These cases isolate ranking and hit mapping; source proof has its own suite. + async def qualified(ch, rows, **kwargs): + return rows + + monkeypatch.setattr("server.memory.store.summary_sources.current_summary_sources", qualified) + def test_digest_row_carries_links_and_session_into_the_hit() -> None: """The summary leg's row → SearchHit mapping (memory.app.retrieval).""" diff --git a/tests/unit/test_summary_pages.py b/tests/unit/test_summary_pages.py new file mode 100644 index 0000000000..03bf7ce270 --- /dev/null +++ b/tests/unit/test_summary_pages.py @@ -0,0 +1,119 @@ +"""Source filtering cannot make an unfinished digest scan look complete.""" + +from __future__ import annotations + +from datetime import datetime + +import pytest + +from server.core.errors import BackendUnavailableError +from server.memory.store import summary_pages +from server.memory.store.rows import identity_key +from tests.unit.test_memory_verify import _row + + +class Pages: + def __init__(self, rows, order): + self.rows, self.order, self.calls = rows, order, [] + + def key(self, row): + if self.order == "session": + return row["session_id"] + return ( + row["created_at"], + row["session_id"], + row["agent"], + row["user_id"], + row["embedding_model"], + ) + + async def query_all(self, sql, params): + self.calls.append((sql, params)) + assert "org={o:String}" in sql and params["o"] == "o" + assert params["uid"] == "subject" and "user_id={uid:String}" in sql + rows = self.rows + if params["after"] is not None: + assert "{after:" in sql + rows = [ + row + for row in rows + if ( + self.key(row) < params["after"] + if self.order == "newest" + else self.key(row) > params["after"] + ) + ] + return rows[: params["lim"]] + + +async def listing(ch, limit=1): + return await summary_pages.verified_summary_page( + ch, + org="o", + namespace="n", + limit=limit, + order=ch.order, + where="org={o:String} AND namespace={n:String} AND user_id={uid:String}", + params={"o": "o", "n": "n", "uid": "subject"}, + ) + + +def candidates(count, order): + rows = [ + dict( + agent="a", + user_id="subject", + session_id=f"s{i:04}", + embedding_model="test", + created_at=datetime(2026, 1, 1), + d=datetime(2026, 1, 1), + summary_text=f"fact {i}", + key_facts=[f"fact {i}"], + derived_from=["k"], + ) + for i in range(count) + ] + return rows[::-1] if order == "newest" else rows + + +@pytest.mark.parametrize("order", ["oldest", "newest", "session"]) +@pytest.mark.parametrize("withheld", [2, 270]) +@pytest.mark.parametrize("qualified", [0, 1, 2]) +@pytest.mark.parametrize("page_size", [1, 256]) +async def test_pages_fill_from_current_sources(monkeypatch, order, withheld, qualified, page_size): + monkeypatch.setattr(summary_pages.settings, "memory_digest_listing_page_size", page_size) + monkeypatch.setattr(summary_pages.settings, "memory_digest_listing_max_pages", 512) + rows = candidates(withheld + qualified, order) + heads = [ + _row( + namespace="n", + agent="a", + user_id="subject", + session_id=row["session_id"], + memory_key="k", + value=row["summary_text"], + ) + for row in rows[withheld:] + ] + + async def read(keys, **kwargs): + return {identity_key(row): row for row in heads if identity_key(row) in keys} + + monkeypatch.setattr("server.memory.store.summary_sources.memory_rows_by_identity_keys", read) + ch = Pages(rows, order) + result = await listing(ch) + assert result == rows[withheld:] + assert len(ch.calls) >= 2 + if order == "session": + assert all("LIMIT 1 BY session_id" in sql for sql, _ in ch.calls) + + +async def test_scan_budget_refuses_instead_of_reporting_no_more_digests(monkeypatch): + monkeypatch.setattr(summary_pages.settings, "memory_digest_listing_max_pages", 2) + rows = candidates(300, "oldest") + for row in rows: + row["derived_from"] = [] + ch = Pages(rows, "oldest") + with pytest.raises(BackendUnavailableError, match="scan budget"): + await listing(ch) + assert len(ch.calls) == 2 diff --git a/tests/unit/test_summary_sources.py b/tests/unit/test_summary_sources.py index 536493dd7a..12dc71a5f1 100644 --- a/tests/unit/test_summary_sources.py +++ b/tests/unit/test_summary_sources.py @@ -1,36 +1,205 @@ -"""An unchecked source never makes a session digest eligible for retrieval.""" +"""A digest is served only when every byte can be reconstructed from live facts.""" from __future__ import annotations +import json +from datetime import timedelta +from types import SimpleNamespace from typing import Any +from unittest.mock import AsyncMock +import pytest + +from server.core.datetimes import naive_utc_now +from server.memory import store from server.memory.app.retrieval.summaries import fetch_session_summaries +from server.memory.store.summary_sources import current_summary_sources +from tests.unit.test_memory_verify import _row + + +def digest(source: Any, **overrides: Any) -> dict[str, Any]: + return { + "agent": source.agent, + "user_id": source.user_id, + "session_id": source.session_id, + "derived_from": [source.memory_key], + "key_facts": [source.value], + "summary_text": source.value, + "version": 200, + "distance": 0.1, + **overrides, + } + + +@pytest.fixture +def sources(monkeypatch: Any) -> dict[str, Any]: + values = {"source": _row(value="current fact", memory_key="source")} + + async def read(keys, **kwargs): + from server.memory.store.rows import identity_key + + assert kwargs["sequential"] + return {identity_key(r): r for r in values.values() if identity_key(r) in keys} + + monkeypatch.setattr("server.memory.store.summary_sources.memory_rows_by_identity_keys", read) + return values + + +@pytest.mark.parametrize("lineage", [None, []]) +async def test_a_summary_without_source_lineage_is_not_retrievable(lineage: Any) -> None: + row = {"summary_text": "unknown fact", "key_facts": ["unknown fact"]} + if lineage is not None: + row["derived_from"] = lineage + assert await current_summary_sources(None, [row], org="o", namespace="ns") == [] + + +@pytest.mark.parametrize( + "change", + [ + {"status": "rejected"}, + {"status": "proposed"}, + {"status": "duplicate"}, + {"status": "superseded"}, + {"sensitivity": "high"}, + {"value": "replacement fact"}, + {"value_truncated": True}, + {"requires_review": True}, + {"user_id": "another-user"}, + {"session_id": "another-session"}, + {"agent": "another-agent"}, + ], +) +async def test_changed_or_unserved_sources_withhold_the_digest(sources: Any, change: Any) -> None: + row = digest(sources["source"]) + sources["source"] = sources["source"].model_copy(update=change) + assert await current_summary_sources(None, [row], org="o", namespace="ns") == [] + + +@pytest.mark.parametrize("field", ["expires_at", "valid_to", "tombstoned_at"]) +async def test_retired_sources_withhold_the_digest(sources: Any, field: str) -> None: + row = digest(sources["source"]) + sources["source"] = sources["source"].model_copy( + update={field: naive_utc_now() - timedelta(seconds=1)} + ) + assert await current_summary_sources(None, [row], org="o", namespace="ns") == [] + + +async def test_a_touch_does_not_invalidate_verifiable_unchanged_text(sources: Any) -> None: + row = digest(sources["source"]) + sources["source"] = sources["source"].model_copy(update={"version": 999, "pinned": True}) + assert await current_summary_sources(None, [row], org="o", namespace="ns") == [row] + + +async def test_a_missing_source_and_a_forged_sentence_do_not_hide_a_valid_neighbour( + sources: Any, +) -> None: + valid = digest(sources["source"]) + missing = digest(sources["source"], derived_from=["missing"]) + forged = digest(sources["source"], summary_text="current fact\nretired fact") + assert await current_summary_sources( + None, [missing, forged, valid], org="o", namespace="ns" + ) == [valid] + + +async def test_a_current_model_digest_does_not_qualify_a_stale_model_row(sources): + from server.memory.app.period_digest import PeriodDigestConfig, _plan_identity + + source = sources["source"] = sources["source"].model_copy(update={"session_id": "session"}) + scope = {c: getattr(source, c) for c in ("org", "namespace")} + stale = digest( + source, + **scope, + embedding_model="old", + summary_text="retired fact", + key_facts=["retired fact"], + ) + current = digest(source, **scope, embedding_model="current") + ch = SimpleNamespace(query_all=AsyncMock(return_value=[])) + identity = {c: getattr(source, c) for c in ("org", "namespace", "agent", "user_id")} + assert await _plan_identity( + ch, [stale, current], identity, PeriodDigestConfig(sessions_per_period=2) + ) == ([], 0) + + +async def test_source_state_failure_withholds_the_summary(monkeypatch: Any) -> None: + source = _row(value="current fact", memory_key="source") + + async def unavailable(*args: Any, **kwargs: Any) -> Any: + raise TimeoutError("source state unavailable") + + class CH: + async def query_all(self, *args: Any, **kwargs: Any) -> Any: + return [digest(source)] + + monkeypatch.setattr( + "server.memory.store.summary_sources.memory_rows_by_identity_keys", unavailable + ) + assert ( + await fetch_session_summaries( + CH(), + org="o", + namespace="ns", + agent=source.agent, + user_id="-", + query_vec=[1.0, 0.0], + embedding_model="test-model", + ) + == [] + ) + + +async def test_postgres_authority_overrides_a_stale_clickhouse_source(monkeypatch: Any) -> None: + from server.memory.store import pg_memory + from server.settings import settings + + source = _row(value="current fact", memory_key="source") + rejected = store.build_memory_version_row(source, status="rejected", version=201) + monkeypatch.setattr(settings, "storage_authority", "postgres") + monkeypatch.setattr( + pg_memory, "heads_by_keys", AsyncMock(return_value={store.identity_key(source): rejected}) + ) + ch = SimpleNamespace(query_all=AsyncMock(side_effect=AssertionError("must read the authority"))) + assert await current_summary_sources(ch, [digest(source)], org="o", namespace="ns") == [] + ch.query_all.assert_not_awaited() + + +@pytest.mark.parametrize("fact", ["文" * 600, "x" * 3000, "🌱" * 600]) +async def test_encoded_source_size_does_not_drop_a_complete_fact(monkeypatch, fact): + from server.memory.store.rows import identity_key + + source = _row(value=fact, memory_key="source") + + async def read(keys, **kwargs): + assert kwargs["max_value_chars"] >= len(json.dumps(fact)) + return {identity_key(source): source} + + monkeypatch.setattr("server.memory.store.summary_sources.memory_rows_by_identity_keys", read) + row = digest(source) + assert await current_summary_sources(None, [row], org="o", namespace="ns") == [row] + + +@pytest.mark.parametrize("count", [6, 50]) +@pytest.mark.parametrize("retired_last", [False, True]) +async def test_many_sources_are_qualified_in_one_exact_key_read(monkeypatch, count, retired_last): + from server.memory.store.rows import identity_key + + sources = [_row(value="文" * 400, memory_key=f"source-{n}") for n in range(count)] + calls = [] + async def read(keys, **kwargs): + calls.append(keys) + return {identity_key(source): source for source in sources} -async def test_source_state_failure_withholds_the_summary() -> None: - class UnavailableSources: - async def query_all(self, sql: str, params: Any = None) -> list[dict[str, Any]]: - if "FROM loom.memory FINAL" in sql: - raise TimeoutError("source state unavailable") - return [ - { - "agent": "writer", - "user_id": "reader", - "session_id": "session", - "version": 200, - "derived_from": ["source"], - "summary_text": "old fact", - "distance": 0.1, - } - ] - - hits = await fetch_session_summaries( - UnavailableSources(), - org="org", - namespace="namespace", - agent="writer", - user_id="reader", - query_vec=[1.0, 0.0], - embedding_model="test-model", + monkeypatch.setattr("server.memory.store.summary_sources.memory_rows_by_identity_keys", read) + facts = [source.value for source in sources] + row = digest( + sources[0], + derived_from=[source.memory_key for source in sources], + key_facts=facts, + summary_text="\n".join(facts), ) - assert hits == [] + if retired_last: + sources[-1] = sources[-1].model_copy(update={"status": "superseded"}) + expected = [] if retired_last else [row] + assert await current_summary_sources(None, [row], org="o", namespace="ns") == expected + assert len(calls) == 1 and len(calls[0]) == count diff --git a/tests/unit/test_value_updates.py b/tests/unit/test_value_updates.py index 776d5867b4..275c9f1a08 100644 --- a/tests/unit/test_value_updates.py +++ b/tests/unit/test_value_updates.py @@ -202,8 +202,15 @@ async def boom(*a: Any, **k: Any) -> int: def test_enabled_precedence_request_over_contextvar_over_setting( monkeypatch: pytest.MonkeyPatch, ) -> None: + from types import SimpleNamespace + from unittest.mock import AsyncMock + from server.memory.app import bulk_set as bulk_set from server.memory.app.extract import handler as hm + + monkeypatch.setattr( + hm, "get_client", lambda: SimpleNamespace(query_all=AsyncMock(return_value=[])) + ) from server.settings import settings monkeypatch.setattr(settings, "memory_extract_value_updates", False) @@ -242,6 +249,9 @@ async def test_set_from_messages_starts_its_link_count_at_zero( ) -> None: """A worker replays queued requests one after another in one task: a request that writes nothing must not report the previous request's value-update links.""" + from types import SimpleNamespace + from unittest.mock import AsyncMock + from server.memory.api.models_bulk import ( MemorySetFromMessagesInput, MemorySetFromMessagesOutput, @@ -249,6 +259,10 @@ async def test_set_from_messages_starts_its_link_count_at_zero( from server.memory.app import bulk_set as bulk_set from server.memory.app.extract import handler as hm + monkeypatch.setattr( + hm, "get_client", lambda: SimpleNamespace(query_all=AsyncMock(return_value=[])) + ) + async def _enqueued(_inp: object, _rule_agent: str) -> MemorySetFromMessagesOutput: return MemorySetFromMessagesOutput(written=[]) diff --git a/tests/unit/test_version_row_value_bytes.py b/tests/unit/test_version_row_value_bytes.py index 1fd1cd8ab4..6846041f04 100644 --- a/tests/unit/test_version_row_value_bytes.py +++ b/tests/unit/test_version_row_value_bytes.py @@ -87,16 +87,21 @@ def _same_bytes_and_hash(written: str, stored: str, parsed: Any) -> None: class _CH: def __init__(self, routes: list[tuple[str, list[dict[str, Any]]]] | None = None) -> None: self.inserts: list[tuple[str, list[dict[str, Any]], dict[str, Any]]] = [] + self.heads = {} self.routes = routes or [] self.query_one_result: dict[str, Any] | None = None async def insert(self, table: str, rows: list[dict[str, Any]], settings: Any = None) -> None: self.inserts.append((table, [dict(r) for r in rows], settings or {})) + if table == "loom.memory": + self.heads.update({mr.identity_key(r): dict(r) for r in rows}) async def query_one(self, *_: Any, **__: Any) -> dict[str, Any] | None: return self.query_one_result async def query_all(self, sql: str, params: Any = None, **_: Any) -> list[dict[str, Any]]: + if params and "keys" in params and "FROM loom.memory FINAL" in sql: + return [self.heads[k] for k in params["keys"] if k in self.heads] for needle, result in self.routes: if needle in sql: return [dict(r) for r in result] @@ -120,7 +125,15 @@ def rows_for(self, table: str) -> list[dict[str, Any]]: def _wire(monkeypatch: pytest.MonkeyPatch, existing: MemoryRow) -> _CH: - ch = _CH(routes=[("FROM loom.memory_embeddings", [dict(_EMB_ROW)])]) + ch = _CH( + routes=[ + ( + "FROM loom.memory_embeddings", + [dict(_EMB_ROW, content_sha256=existing.content_sha256)], + ) + ] + ) + ch.heads[mr.identity_key(existing)] = mr.build_memory_version_row(existing) monkeypatch.setattr(mem_handlers, "get_client", lambda: ch) monkeypatch.setattr(_stack_mod, "get_client", lambda: ch) monkeypatch.setattr(_restore_mod, "get_client", lambda: ch) diff --git a/tests/unit/test_workers_poll_the_work.py b/tests/unit/test_workers_poll_the_work.py index 03f46b9fad..5be1087d37 100644 --- a/tests/unit/test_workers_poll_the_work.py +++ b/tests/unit/test_workers_poll_the_work.py @@ -7,8 +7,7 @@ exact, reads the full rows by exact key only for what the scan returned, writes one insert per table per round, and scans from 0 on its slow clock; - entity_extractor reads the cost cap once per pass for every namespace of - its rows and writes the pass's rows in one batch (six statements, not - six per row); the batch write emits the same rows as N single writes; + its rows and writes the pass's rows in batches with bounded source checks; the batch write emits the same rows as N single writes; - the cost-cap verdict for N namespaces is one statement, fail-open; - embedding_worker marks itself idle only when all three legs found nothing; - every pass records its counts as outcome scalars. @@ -23,6 +22,7 @@ import pytest from server.code.store.index import CodeRepo as _RealCodeRepo +from server.core.hashing import sha256_value from server.settings import settings from server.workers import pass_context from server.workers import supervisor as sup @@ -565,6 +565,32 @@ def __init__(self, answers: list[list[dict[str, Any]]]) -> None: self.inserts: list[tuple[str, list[dict[str, Any]]]] = [] self.heads: dict[Any, Any] = {} + for batch in answers: + for row in batch: + if "entities" in row and "embedding_model" not in row: + from server.memory.store.rows import identity_key + + head = {**_mem_row(row["memory_key"]), **row} + self.heads[identity_key(head)] = head + + async def query_one(self, sql, params=None): + import json + + from server.core.hashing import served_row_value + from server.memory.domain.promotion_source import promotion_source + + self.queries.append(" ".join(sql.split())) + assert "FROM loom.shared FINAL" in sql + key = (params["org"], params["ns"], "a", "-", "-", params["memory_key"]) + source = self.heads.get(key) + if source is None: + return None + return { + "value": json.dumps(served_row_value(source)), + "tombstoned_at": None, + "promoted_from": [promotion_source(*key)], + } + async def query_all(self, sql, params=None): from server.memory.store.rows import identity_key @@ -606,6 +632,7 @@ def _mem_row(key: str) -> dict[str, Any]: "memory_type": "semantic", "value": '{"a": 1}', "value_json": "{}", + "content_sha256": sha256_value({"a": 1}), "confidence": 1.0, "version": 1, "pinned": 0, @@ -619,7 +646,10 @@ def _mem_row(key: str) -> dict[str, Any]: def _emb_row(key: str, scope: str = "private") -> dict[str, Any]: + import json + from server.memory import store as mr + from server.memory.domain.excerpt import build_excerpt return { **mr.EMBEDDING_PERSISTED_COLS, @@ -631,7 +661,10 @@ def _emb_row(key: str, scope: str = "private") -> dict[str, Any]: "memory_key": key, "memory_type": "semantic", "scope": scope, - "content_excerpt": "x", + "content_excerpt": build_excerpt(json.dumps({"a": 1}), memory_key=key) + if scope == "shared" + else "x", + "content_sha256": sha256_value({"a": 1}), "embedding_model": "m", "embedding_dim": 3, "embedding": [0.1, 0.2, 0.3], @@ -656,9 +689,7 @@ def _item( } -def test_apply_entities_batch_is_six_statements_for_n_rows_and_emits_the_same_rows_as_n_single_writes() -> ( - None -): +def test_apply_entities_batch_bounds_reads_and_preserves_every_mirror() -> None: from server.memory import store as mr ch = _CH( @@ -703,27 +734,34 @@ def test_apply_entities_batch_is_six_statements_for_n_rows_and_emits_the_same_ro mr.apply_entities_batch([_item("k1", ["alice"]), _item("k2", ["bob", "eve"])], ch=ch) ) assert written == {"memory": 2, "memory_embeddings": 3} - assert len(ch.queries) == 4 and [t for t, _ in ch.inserts] == [ + assert len(ch.queries) == 17 and [t for t, _ in ch.inserts] == [ "loom.memory", "loom.memory_embeddings", + "loom.memory_embeddings", ] + mirrors = [q for q in ch.queries if "FROM loom.memory_embeddings" in q] + heads = [q for q in ch.queries if "FROM loom.memory FINAL" in q] + assert len(mirrors) == 2 and len(heads) == 5 + assert len([q for q in ch.queries if "FROM loom.shared FINAL" in q]) == 2 + assert len([q for q in ch.queries if "memory_refusals" in q]) == 8 assert ( ch.queries[0].startswith("SELECT") and "FROM loom.memory FINAL PREWHERE (org, namespace, agent, user_id, session_id, memory_key) IN" in ch.queries[0] ) - assert ( - "PREWHERE (org, namespace" in ch.queries[1] - ) # Review state is checked under the write fence. - assert ch.queries[2].startswith( + assert "PREWHERE (org, namespace" in heads[1] # Review state is checked under the write fence. + assert mirrors[0].startswith( "SELECT DISTINCT org, namespace, agent, user_id, session_id, memory_key, scope, embedding_model FROM loom.memory_embeddings" ) - assert ch.queries[3].startswith( + assert mirrors[1].startswith( "SELECT * FROM loom.memory_embeddings FINAL PREWHERE (org, namespace, agent, user_id, session_id, scope, embedding_model, memory_key) IN" ) mem = {r["memory_key"]: r for r in ch.inserts[0][1]} assert mem["k1"]["entities"] == ["alice"] and mem["k2"]["entities"] == ["bob", "eve"] - emb = ch.inserts[1][1] + emb = sorted( + (r for table, rows in ch.inserts if table == "loom.memory_embeddings" for r in rows), + key=lambda r: (r["memory_key"], r["scope"]), + ) assert [(r["memory_key"], r["scope"], r["entities"]) for r in emb] == [ ("k1", "private", ["alice"]), ("k2", "private", ["bob", "eve"]), @@ -736,7 +774,11 @@ def test_apply_entities_batch_is_six_statements_for_n_rows_and_emits_the_same_ro assert all(set(mr.EMBEDDING_PERSISTED_COLS) <= set(r) for r in emb), ( "every persisted column mirrored" ) - assert all(r["embedding"] == [0.1, 0.2, 0.3] and r["content_excerpt"] == "x" for r in emb) + assert all( + r["embedding"] == [0.1, 0.2, 0.3] + and r["content_excerpt"] == _emb_row(r["memory_key"], r["scope"])["content_excerpt"] + for r in emb + ) def test_apply_entities_batch_writes_only_onto_the_head_the_entities_came_from() -> None: @@ -771,12 +813,12 @@ def test_apply_entities_batch_keeps_its_work_over_a_copy_of_the_same_text() -> N takes the entities; one whose text changed does not.""" from server.memory import store as mr - read = "a" * 64 + read = sha256_value({"a": 1}) ch = _CH( answers=[ [ _mem_row("k1") | {"version": 3, "content_sha256": read}, - _mem_row("k2") | {"version": 3, "content_sha256": "b" * 64}, + _mem_row("k2") | {"version": 3, "content_sha256": sha256_value({"a": 2})}, ] ] ) @@ -796,7 +838,7 @@ def test_a_copy_of_the_same_text_redacted_or_made_high_since_keeps_no_entities( ) -> None: from server.memory import store as mr - read = "a" * 64 + read = sha256_value({"a": 1}) ch = _CH(answers=[[_mem_row("k1") | {"version": 3, "content_sha256": read} | since]]) written = asyncio.run( mr.apply_entities_batch([_item("k1", ["alice"], content_sha256=read)], ch=ch) @@ -818,7 +860,8 @@ def test_apply_entities_batch_copies_every_search_document_of_a_head_written() - ) written = asyncio.run(mr.apply_entities_batch([_item("k1", ["alice"])], ch=ch)) assert written == {"memory": 1, "memory_embeddings": 2} - assert [(r["scope"], r["version"]) for r in ch.inserts[1][1]] == [("private", 2), ("shared", 3)] + vectors = [r for table, rows in ch.inserts if table == "loom.memory_embeddings" for r in rows] + assert sorted((r["scope"], r["version"]) for r in vectors) == [("private", 2), ("shared", 3)] def test_apply_entities_batch_leaves_a_newer_writes_search_document( @@ -835,7 +878,7 @@ async def sleep(s: float) -> None: slept.append(s) monkeypatch.setattr(entities.asyncio, "sleep", sleep) - head, newer = "a" * 64, "b" * 64 + head, newer = sha256_value({"a": 1}), sha256_value({"a": 2}) ch = _CH( answers=[ [_mem_row("k1") | {"content_sha256": head}], @@ -875,7 +918,7 @@ def test_a_restatement_with_its_own_entities_keeps_them() -> None: pass's to tag.""" from server.memory import store as mr - read = "a" * 64 + read = sha256_value({"a": 1}) restated = _mem_row("k1") | {"version": 3, "content_sha256": read, "entities": ["acme"]} ch = _CH(answers=[[restated]]) written = asyncio.run( @@ -890,7 +933,7 @@ def test_apply_entities_batch_writes_above_a_copy_newer_than_its_draw() -> None: from server.memory import store as mr from server.memory.store.rows import next_version - read = "a" * 64 + read = sha256_value({"a": 1}) far = next_version() + 10**9 # a lock from a replica whose clock is a second ahead ch = _CH(answers=[[_mem_row("k1") | {"version": far, "content_sha256": read}]]) asyncio.run(mr.apply_entities_batch([_item("k1", ["alice"], content_sha256=read)], ch=ch)) @@ -1053,7 +1096,10 @@ async def fake_sleep(s): written = asyncio.run(mr.apply_entities_batch([_item("k1", ["a"]), _item("k2", ["b"])], ch=ch)) assert slept == [mr.ENTITY_MIRROR_RETRY_DELAY_S], "exactly one bounded retry for the batch" assert written == {"memory": 2, "memory_embeddings": 2} - assert len(ch.queries) == 6 and len(ch.inserts) == 2 + assert len(ch.queries) == 11 and len(ch.inserts) == 2 + assert len([q for q in ch.queries if "FROM loom.memory_embeddings" in q]) == 4 + assert len([q for q in ch.queries if "FROM loom.memory FINAL" in q]) == 3 + assert len([q for q in ch.queries if "memory_refusals" in q]) == 4 assert [r["memory_key"] for r in ch.inserts[1][1]] == ["k1", "k2"] @@ -1075,7 +1121,7 @@ async def fake_sleep(s): assert asyncio.run(mr.apply_entities_batch([], ch=ch)) == {"memory": 0, "memory_embeddings": 0} -def test_heal_reads_every_pair_in_one_keyed_final_read_and_one_insert() -> None: +def test_heal_batches_mirror_reads_and_checks_shared_sources() -> None: from server.memory import store as mr pairs = [ @@ -1100,9 +1146,12 @@ def test_heal_reads_every_pair_in_one_keyed_final_read_and_one_insert() -> None: ] ch = _CH(answers=[pairs, [_emb_row("k1"), _emb_row("k2"), _emb_row("k2", "shared")]]) healed = asyncio.run(mr.heal_divergent_entities(limit=50, ch=ch)) - assert healed == 3 and len(ch.queries) == 2 and len(ch.inserts) == 1 + assert healed == 3 and len(ch.queries) == 13 and len(ch.inserts) == 2 + assert len([q for q in ch.queries if "FROM loom.shared FINAL" in q]) == 2 + assert len([q for q in ch.queries if "memory_refusals" in q]) == 6 assert "IN {keys:Array(Tuple(String, String, String, String, String, String))}" in ch.queries[1] - assert [(r["memory_key"], r["entities"]) for r in ch.inserts[0][1]] == [ + vectors = [r for table, rows in ch.inserts if table == "loom.memory_embeddings" for r in rows] + assert sorted((r["memory_key"], r["entities"]) for r in vectors) == [ ("k1", ["x"]), ("k2", ["y"]), ("k2", ["y"]), @@ -1172,6 +1221,7 @@ async def find_stubs(**kw): async def insert_rows(rows, *a, **kw): env["inserted"].append(list(rows)) + return [] if env.get("withheld") else rows async def orphans(**kw): return [dict(o) for o in env["orphans"]] @@ -1361,3 +1411,42 @@ async def query_all(self, sql, params=None): ) assert params["keys"] == [("o", "n1", "m", "p", "c")] assert out == [{"org": "o", "namespace": "n1", "path": "p", "chunk_id": "c", "version": 7}] + + +async def test_embedding_worker_counts_only_acknowledged_fills(emb_env): + from server.memory.workers import embedding as worker + from server.workers import status + + emb_env.update(stubs=[_stub_row()], withheld=True) + result = await worker.run_pass(worker.EmbeddingWorkerConfig(batch_size=10)) + assert result.embedded_count == 0 and result.reconciled == [] + assert status.get_registry().snapshot()["embedding_worker"]["last_outcome"]["rows_written"] == 0 + + +@pytest.mark.copy_head_check +async def test_embedding_worker_withholds_an_unbacked_shared_stub_before_embedding( + emb_env, monkeypatch +): + from server.memory.store import shared + from server.memory.workers import embedding as worker + + seen = [] + + async def missing(row, **kwargs): + seen.append(row) + return None + + class Provider: + model_name, dim = "fake-ngram-3", 3 + + async def embed_many(self, texts): + pytest.fail("An unbacked shared stub reached the provider") + + async def embed(self, text): + pytest.fail("An unbacked shared stub reached the provider") + + emb_env["stubs"] = [_stub_row() | {"scope": "shared"}] + monkeypatch.setattr(shared, "shared_embedding_source", missing) + monkeypatch.setattr(worker, "get_embedder", lambda: Provider()) + result = await worker.run_pass(worker.EmbeddingWorkerConfig(batch_size=10)) + assert len(seen) == 1 and result.embedded_count == 0 and not emb_env["inserted"] diff --git a/tests/unit/tunable_constants.txt b/tests/unit/tunable_constants.txt index 737af69e9b..eae60243ee 100644 --- a/tests/unit/tunable_constants.txt +++ b/tests/unit/tunable_constants.txt @@ -430,7 +430,6 @@ server/memory/app/search/query_spelling.py::_MAX_DOUBLINGS server/memory/app/search_llm.py::_QUERY_ENTITY_CACHE_MAX server/memory/app/search_llm.py::_VERB_SYNONYM_CACHE_MAX server/memory/app/session_brief.py::_MAX_SOURCE_SESSIONS -server/memory/app/session_summary.py::_SESSION_SUMMARY_MAX_INPUT_CHARS server/memory/app/session_summary.py::_SESSION_SUMMARY_MAX_KEY_FACTS server/memory/app/set_chunks.py::_MAX_CHUNKS_FALLBACK server/memory/app/set_entities.py::_SET_ENTITY_MAX_CANDIDATES