Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
18 commits
Select commit Hold shift + click to select a range
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions docs/architecture/memory-architecture.md
Original file line number Diff line number Diff line change
Expand Up @@ -1008,6 +1008,10 @@ Session digests carry the keys they compressed in `derived_from`. Before dedupli
the summary leg checks those known sources under the digest's full identity. A source
that is missing, unserved, deleted, expired, outside its validity, high-sensitivity or at a newer
version withholds the digest. An unreadable source check withholds all unchecked digests.
Under Postgres authority, the leg reads each exact source key again after reading its
projection. The authoritative version, content hash and serving metadata must match
the projection; a missing head or content hash withholds the digest. This check reads
metadata only, and never falls back to the projection when Postgres is unavailable.
Without source revision hashes, a later source version conservatively withholds a digest
during replay too. Lineage covers extracted facts, not every sentence of the transcript:
a digest without a known source remains session evidence, and does not prove that its
Expand Down
6 changes: 4 additions & 2 deletions scripts/gates/testsupport/cell_trigger/capture.py
Original file line number Diff line number Diff line change
Expand Up @@ -256,8 +256,10 @@ def expected(self, row, audit, context, *, api, metadata, driver):
"workload semantic/active",
)
need(row["confidence"] == 1.0, "workload confidence")
for field, default in self.p["sparse_defaults"].items():
equal(row[field], default)
equal(
{field: row[field] for field in self.p["sparse_defaults"]},
self.p["sparse_defaults"],
)
need(
row["value"] == json.dumps(payload(key)) and row["content_sha256"] == content_hash(key),
"exact stored payload/hash",
Expand Down
52 changes: 52 additions & 0 deletions server/memory/store/pg_memory.py
Original file line number Diff line number Diff line change
Expand Up @@ -493,6 +493,37 @@ def _key_arrays(keys: Sequence[IdentityKey]) -> list[list[str]]:

_HEADS_BY_KEYS = f"SELECT c.row {_EXACT_KEYS}"

# A digest may use the projection's eligibility decision only while these fields match
# its authority. Read no source values: a shortlist can name many source keys.
SUMMARY_SOURCE_STATE_COLUMNS = (
"version",
"content_sha256",
"status",
"sensitivity",
"tombstoned_at",
"redacted_at",
"expires_at",
"valid_to",
"created_at",
"valid_at",
"temporal_anchor",
"invalid_at",
)
_SUMMARY_SOURCE_COLUMNS = (
"org",
"namespace",
"agent",
"user_id",
"session_id",
"memory_key",
*SUMMARY_SOURCE_STATE_COLUMNS,
)
_SUMMARY_SOURCE_HEADS = (
"SELECT jsonb_strip_nulls(jsonb_build_object("
+ ", ".join(f"'{col}', c.row->'{col}'" for col in _SUMMARY_SOURCE_COLUMNS)
+ f")) AS row {_EXACT_KEYS}"
)

_HEADS_OWING = f"""
SELECT c.row, coalesce(d.targets, 0) AS owed {_EXACT_KEYS}
LEFT JOIN loom.memory_deliveries d
Expand Down Expand Up @@ -1633,6 +1664,27 @@ async def heads_by_keys(
return {identity_key(r["row"]): decode_row(r["row"]) for r in rows}


async def summary_source_heads(
keys: Sequence[IdentityKey],
) -> dict[IdentityKey, dict[str, Any]]:
"""Exact-key digest source metadata, including deleted and unserved heads.

A missing or unreadable head cannot validate a projected source. Values and other
payloads stay in the authority; the caller compares only the serving state.
"""
keys = [k for k in keys if not pg.unstorable(k)]
if not keys:
return {}
rows = await _read(_SUMMARY_SOURCE_HEADS, *_key_arrays(keys), durable=False)
heads = {}
for row in rows:
decoded = decode_row(row["row"])
heads[identity_key(decoded)] = {
col: decoded.get(col) for col in SUMMARY_SOURCE_STATE_COLUMNS
}
return heads


@asynccontextmanager
async def locked_source(source: MemoryRow) -> AsyncIterator[None]:
"""Hold an unchanged durable source while its shared projection is published."""
Expand Down
40 changes: 38 additions & 2 deletions server/memory/store/summary_sources.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,20 @@

from __future__ import annotations

from datetime import datetime
from typing import Any

from server.core import pg
from server.core.datetimes import to_naive_utc
from server.core.hashing import hash_text
from server.core.repo_base import _CHClient, query_all_sliced
from server.memory.domain.redaction_lineage import is_content_hash
from server.memory.domain.temporal_sql import (
as_of_predicate,
status_predicate,
tombstone_predicate,
)
from server.memory.store import pg_memory

SourceKey = tuple[str, str, str, str]

Expand All @@ -21,6 +27,18 @@ def _sources(row: dict[str, Any]) -> list[SourceKey]:
return [(agent, user, session, str(key)) for key in row.get("derived_from", []) if key]


def _source_state(row: dict[str, Any]) -> tuple[Any, ...]:
return tuple(
hash_text(value)
if col == "content_sha256"
else to_naive_utc(value)
if isinstance(value, datetime)
else value
for col in pg_memory.SUMMARY_SOURCE_STATE_COLUMNS
for value in [row.get(col)]
)


async def current_summary_sources(
ch: _CHClient,
rows: list[dict[str, Any]],
Expand All @@ -43,6 +61,7 @@ async def current_summary_sources(
tombstone_predicate(as_of_basis, replaying=replaying),
status_predicate(as_of_basis, replaying=replaying),
"sensitivity != 'high'",
"redacted_at IS NULL",
"(expires_at IS NULL OR expires_at > now())",
"(valid_to IS NULL OR valid_to > now())",
]
Expand All @@ -55,7 +74,8 @@ async def current_summary_sources(
f"""
SELECT agent, if(user_id = '', '-', user_id) AS source_user,
if(session_id = '', '-', session_id) AS source_session, memory_key,
version, {" AND ".join(state)} AS source_served
{", ".join(pg_memory.SUMMARY_SOURCE_STATE_COLUMNS)},
{" AND ".join(state)} AS source_served
FROM loom.memory FINAL
WHERE org = {{org:String}} AND namespace = {{ns:String}}
AND (agent, if(user_id = '', '-', user_id), if(session_id = '', '-', session_id),
Expand All @@ -64,6 +84,11 @@ async def current_summary_sources(
params,
"sources",
)
authority = (
await pg_memory.summary_source_heads([(org, namespace, *key) for key in sources])
if pg.enabled()
else None
)
heads: dict[SourceKey, tuple[bool, int]] = {}
for p in parents:
key = (
Expand All @@ -73,7 +98,18 @@ async def current_summary_sources(
str(p["memory_key"]),
)
served, version = heads.get(key, (True, 0))
heads[key] = (served and bool(p["source_served"]), max(version, int(p["version"])))
current = True
if authority is not None:
head = authority.get((org, namespace, *key))
current = (
head is not None
and is_content_hash(str(head.get("content_sha256") or ""))
and _source_state(head) == _source_state(p)
)
heads[key] = (
served and bool(p["source_served"]) and current,
max(version, int(p["version"])),
)
return [
row
for row in rows
Expand Down
15 changes: 15 additions & 0 deletions tests/analyzer/cases/legs.py
Original file line number Diff line number Diff line change
Expand Up @@ -713,6 +713,21 @@ def cases() -> list[Case]:
"as_of": (False, True),
"basis": ("valid_time", "knowledge_time"),
},
rows=rows_by_substring(
{
"FROM loom.session_summaries FINAL": [
{
"agent": AGENT,
"user_id": "alice",
"session_id": "session",
"version": 1,
"derived_from": ["source"],
"summary_text": "a source fact",
"distance": 0.1,
}
]
}
),
),
Case(
"memory_repo.reads",
Expand Down
3 changes: 3 additions & 0 deletions tests/analyzer/shapes.txt
Original file line number Diff line number Diff line change
Expand Up @@ -995,6 +995,7 @@
764e0e259701 SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, e.status AS status, e.version AS version, -- Multi-vector axis: min(value_dist, key_dist). CH?t g
765404bd5d8d SELECT toDate(coalesce(temporal_anchor, valid_at)) AS event_date, cosineDistance(embedding, {q:Array(Float32)}) AS distance, substring(content_excerpt, ?, ?) AS excerpt, memory_key, temporal_anchor IS
7662b4971175 SELECT org, namespace, agent, user_id, session_id, memory_id, memory_key, memory_type, value, confidence, source_trace, version, tombstoned_at, created_at, expires_at, sensitivity FROM loom.memory FIN
769c165248dc SELECT agent, if(user_id = ?, ?, user_id) AS source_user, if(session_id = ?, ?, session_id) AS source_session, memory_key, version, content_sha256, status, sensitivity, tombstoned_at, redacted_at, exp
76ed76b066ae SELECT e.memory_key AS memory_key, e.memory_type AS memory_type, e.content_excerpt AS content_excerpt, ? - cosineDistance(e.embedding, {qv:Array(Float32)}) AS similarity, toString(coalesce(e.temporal_
76f47ab9df72 SELECT ? AS present FROM loom.code_caller_guards WHERE org = {org:String} AND namespace = {ns:String} LIMIT ?
77424b91d322 SELECT count() AS n FROM loom.memory FINAL WHERE org = {org:String} AND namespace IN {ns:Array(String)}
Expand Down Expand Up @@ -1652,6 +1653,7 @@ bd4266cbfe15 SELECT count() AS n FROM loom.conflict_allowlist FINAL WHERE org =
bda43b1f632f WITH arrayConcat(splitByNonAlpha(context_lc), splitByNonAlpha(content_lc)) AS words SELECT namespace, source_uri, chunk_idx, doc_version, JSONExtractUInt(chunk_metadata, ?) AS revision, cosineDistance
bde7c432bca1 ALTER TABLE loom.entity_communities DELETE WHERE (org = {org:String} AND (cityHash64(namespace, entity) IN {entities:Array(UInt64)} OR cityHash64(namespace, label) IN {entities:Array(UInt64)})) AND ({
bde98d358a2b SELECT path, uniqExact(facet_kind, name) AS matched, groupArray((facet_kind, name, ordinal, line, confidence)) AS hits, length(arrayIntersect(groupUniqArrayIf(ordinal, facet_kind = {k0:String} AND nam
be08a1021716 SELECT agent, if(user_id = ?, ?, user_id) AS source_user, if(session_id = ?, ?, session_id) AS source_session, memory_key, version, content_sha256, status, sensitivity, tombstoned_at, redacted_at, exp
be29d7515a0a SELECT count() AS n FROM loom.memory_doc_stats WHERE org = {org:String} AND namespace = {ns:String}
be6f152846ae SELECT arrayFirst( u -> startsWith(source_resource_id, concat({prefix:String}, u, ?)), {uris:Array(String)} ) AS source_uri, toUInt8(has(tags, {proposed_tag:String})) AS proposed, count() AS total, co
be735bd4b1cf ALTER TABLE loom.namespace_guides DELETE WHERE (org = {org:String} AND namespace IN {ns:Array(String)}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync = ?
Expand Down Expand Up @@ -1935,6 +1937,7 @@ dbd04ba47fb3 SELECT agent, user_id, session_id, memory_key, memory_type, content
dbe0f33a89a7 SELECT toString(memory_id) AS memory_id, org, namespace, agent, user_id, session_id, memory_key, value, created_at, valid_at, version, redacted_at, toString(content_sha256) AS content_sha256 FROM loom
dbe745784521 SELECT memory_type, toString(status) AS status, source_class, count() AS rows, min(updated_at) AS first_at, max(updated_at) AS last_at FROM loom.memory FINAL WHERE org = {org:String} AND namespace = {
dbefbb4f200e SELECT concat(?, toString(c.session_id), ?, toString(c.chunk_index)) AS memory_key, ? AS memory_type, substring(c.content, ?, ?) AS content_excerpt, c.content AS content_full, c.agent AS agent, c.user
dbf0d4bbb01d SELECT agent, if(user_id = ?, ?, user_id) AS source_user, if(session_id = ?, ?, session_id) AS source_session, memory_key, version, content_sha256, status, sensitivity, tombstoned_at, redacted_at, exp
dbf737bf992e ALTER TABLE loom.knowledge_uploads DELETE WHERE (org = {org:String} AND user_id = {uid:String}) AND ({erasure_mutation_tag:String} = {erasure_mutation_tag:String}) SETTINGS mutations_sync = ?
dbfe0a193e5b SELECT count() AS n FROM loom.documents FINAL WHERE org = {org:String} AND namespace IN {ns:Array(String)} AND superseded_at IS NULL
dc003d9f3eb2 SELECT toString(failure_id) AS failure_id, org, namespace, agent, projection, source_operation, source_resource_id, memory_keys, error_type, error_message, payload, status, created_at, resolved_at FRO
Expand Down
37 changes: 37 additions & 0 deletions tests/integration/test_pg_authority.py
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,31 @@ async def landed() -> bool:
await eventually(landed, description="the org's deliveries published")


async def test_digest_source_proof_reads_exact_authoritative_metadata(
call: Any, org: str, pg_conn: Any
) -> None:
from server.core import pg
from server.memory.store import pg_memory

ns = f"pg-{uuid.uuid4().hex[:8]}"
response = await call("memory.set", _set(ns, "source", "a source fact"))
assert response.status_code == 200, response.text
key = (org, ns, "a", "-", "-", "source")
other_key = (org, ns, "another-agent", "-", "-", "source")
stored = await pg_conn.fetchval(
"SELECT row FROM loom.memory_current WHERE org = $1 AND namespace = $2", org, ns
)
try:
heads = await pg_memory.summary_source_heads([key, other_key])
decoded = pg_memory.decode_row(stored)
assert heads == {
key: {col: decoded.get(col) for col in pg_memory.SUMMARY_SOURCE_STATE_COLUMNS}
}
assert "value" not in heads[key] and "value_json" not in heads[key]
finally:
await pg.close_pool()


@pytest.mark.usefixtures("pg_authority")
async def test_a_write_is_listed_and_rolled_back_right_after_it_answers(
call: Any, org: str
Expand Down Expand Up @@ -2421,6 +2446,18 @@ async def test_a_reject_is_the_authoritys_head_and_clickhouses_row(
ns = f"pg-{uuid.uuid4().hex[:8]}"
assert (await call("memory.set", _set(ns, "k", "a fact to reject"))).status_code == 200
before = int((await _head_row(pg_conn, org, ns, "k"))["version"])

# A new set answers at commit; reject copies the vectors already published.
async def document_published() -> bool:
rows = ch_client.query(
"SELECT status, version FROM loom.memory_embeddings FINAL "
"WHERE org = {org:String} AND namespace = {ns:String} AND memory_key = 'k'",
settings=_SEQUENTIAL,
parameters={"org": org, "ns": ns},
).result_rows
return rows == [("active", before)]

await eventually(document_published, description="the initial vector is published")
body = {"namespace": ns, "agent": "a", "memory_key": "k", "rejection_reason": "wrong"}
rejected = await call("memory.reject", body)
assert rejected.status_code == 200 and rejected.json()["rejected"] is True, rejected.text
Expand Down
72 changes: 71 additions & 1 deletion tests/unit/test_cell_attach_cleanup.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
from unittest.mock import AsyncMock

import pytest
import pytest_asyncio

from server.cli import cell_attach, cell_control
from tests.unit.cell_control_owner import owner
Expand Down Expand Up @@ -96,7 +97,8 @@ async def work():
await asyncio.gather(*owned_tasks, return_exceptions=True)


@pytest.mark.asyncio
@pytest.mark.asyncio(loop_scope="function")
@pytest.mark.usefixtures("_cleanup_clock")
async def test_lock_owner_bounds_close_inside_the_same_cleanup_allowance():
closing = asyncio.Event()
conn = AsyncMock()
Expand Down Expand Up @@ -152,3 +154,71 @@ async def open(owned, what):
if failed:
assert caught.value is original
conn.close.assert_awaited_once()


@pytest.mark.parametrize("scheduler_s", [0.0, 0.16])
@pytest.mark.asyncio(loop_scope="function")
@pytest.mark.usefixtures("_cleanup_clock")
async def test_cleanup_allowance_ignores_independent_scheduler_time(monkeypatch, scheduler_s):
import time as wall_time

real_monotonic = wall_time.monotonic
elapsed = 0.0

def monotonic():
return real_monotonic() + elapsed

actual_close = cell_control.Owner.close

async def close(owned):
nonlocal elapsed
try:
return await actual_close(owned)
finally:
elapsed += scheduler_s

monkeypatch.setattr(wall_time, "monotonic", monotonic)
monkeypatch.setattr(cell_control.Owner, "close", close)
await test_lock_owner_bounds_close_inside_the_same_cleanup_allowance()


@pytest_asyncio.fixture(loop_scope="function")
async def _cleanup_clock(monkeypatch):
from server.core import pg
from tests.unit.clock_fake import ManualClock

before = set(asyncio.all_tasks())
loop = asyncio.get_running_loop()
clock = ManualClock(t=loop.time())
selector = loop._selector
real_select = selector.select

def select(timeout=None):
if timeout is not None:
clock.advance(max(0.0, timeout))
timeout = 0
return real_select(timeout)

# The cleanup protocol spends idle time, independently of CPU scheduling.
monkeypatch.setattr(loop, "time", clock.monotonic)
monkeypatch.setattr(selector, "select", select)
monkeypatch.setattr(cell_control, "time", clock)
monkeypatch.setattr(pg, "time", clock)
monkeypatch.setattr(f"{__name__}.time", clock)
yield clock
leaked = asyncio.all_tasks() - before - {asyncio.current_task()}
for task in leaked:
task.cancel()
await asyncio.gather(*leaked, return_exceptions=True)
assert not leaked, "cell cleanup leaves no task on its private loop"


@pytest.mark.asyncio(loop_scope="function")
async def test_cleanup_clock_preserves_the_close_deadline(_cleanup_clock):
from server.core import pg

loop = asyncio.get_running_loop()
assert loop.time == time.monotonic == cell_control.time.monotonic == pg.time.monotonic
began = _cleanup_clock.monotonic()
await test_lock_owner_bounds_close_inside_the_same_cleanup_allowance()
assert _cleanup_clock.monotonic() - began == pytest.approx(0.03)
Loading