fix(log): panel round 1 for #765/#766 — keyset redaction, hashed secret set, one log-off seam

docs/adr/0057-sql-is-seen-through-the-statement-log-and-a-terminals-sql.md

@@ -35,4 +35,6 @@ On Postgres the SQL text depends on the live table: ferro casts bound values (`$
 
 A bound value shows as `***`, a sentinel whose `repr` is `***` so a test can tell it from the string, when it is a `SecretStr`/`SecretBytes` or when it is bound to a column whose field is declared one. Either rule alone leaks: a plain `"hunter2"` compared against a secret column, or a `SecretStr` passed to `raw.execute` where no column is known. The mask is decided in Python, from the model's declared secret fields and the value's type, and travels to Rust beside the binds; Rust never re-derives it, the one-decision pattern I-1 uses. `.sql()`'s `params` are redacted the same way: one rule, applied in one place, and `print(rendered)` in a bug report is the common use. `SET LOCAL` takes no bind parameters on Postgres, so a session setting's value appears inline, unredacted; settings are tenancy scope, not secrets by declaration. Depends on #760, which makes a secret field store its raw value and hydrate as a secret type.
 
+Amended 2026-10-21 (panel F11 on #771): the mask is a set of values, not a positional flag. The binds are made by the query builder in Rust and never pass through Python one by one, so Python cannot mark positions. Instead each operation has one *secret-value set*, built from two facts. Python collects what every `SecretStr` / `SecretBytes` it unwraps hides (a query's literals as it compiles, a write's field values, a raw statement's arguments) through one collector. Rust adds every value the operation binds to a column whose field is declared secret: predicate literals, literal `SET`s, keyset positions (`after` / `before`) on a secret order key, root or traversed, and a write's secret-column values. A bind is masked when it equals a member of the set, one hash lookup per bind, decided before the GIL is taken to write the record. So `Account.where(lambda account: account.token == "sk_live_1")` logs `params=(***,)` though `"sk_live_1"` is a plain `str`. A plain parameter that happens to equal a secret of the same operation is masked too (`bulk_create` of one row whose email equals another row's token shows both as `***`): over-redaction, in the safe direction, never a leak. The switch is still read once per operation, by the engine at the operation's entry. A write's or raw statement's Python-collected values reach the engine as a lazy iterable it reads only when the log is on; a query's are collected as it compiles, because its `.sql()` masks with the same values.
+
 A pin holds the two doors together: for every terminal on both dialects, `.sql()` equals the record the executed call wrote.

src/ferro/_bind_payload.py

@@ -10,6 +10,7 @@
 
 import json
 import uuid
+from collections.abc import Iterable, Iterator
 from datetime import date, datetime, time
 from decimal import Decimal
 from enum import Enum
@@ -41,39 +42,61 @@ def unwrap_secret(value: Any) -> Any:
     return value
 
 
-def secret_values_of(fields: Mapping[str, object]) -> tuple[object, ...]:
-    """Return what each `SecretStr` / `SecretBytes` among `fields`' values hides.
+def secret_values_in(values: Iterable[object]) -> tuple[object, ...]:
+    """Return what each `SecretStr` / `SecretBytes` among `values` hides.
 
-    The statement log masks a bound parameter equal to any of these, so a
-    secret written through this mapping never reaches a record.
+    The one collector of an operation's secret values: a query's compile, a
+    write and a raw statement each call it once. The statement log and a
+    rendered statement mask a bound parameter equal to any of these.
 
     Args:
-        fields: Field or column names mapped to the values about to be bound.
+        values: The values about to be bound. A list, tuple or set among them
+            is searched too, so a secret inside an `IN` list is found.
 
     Returns:
-        The unwrapped value of every secret in `fields`, in order; empty when
+        The unwrapped value of every secret in `values`, in order; empty when
         there is none.
     """
-    return tuple(
-        unwrap_secret(value)
-        for value in fields.values()
-        if isinstance(value, (SecretStr, SecretBytes))
-    )
+    found: list[object] = []
+    pending = list(values)
+    pending.reverse()
+    while pending:
+        value = pending.pop()
+        if isinstance(value, (SecretStr, SecretBytes)):
+            found.append(unwrap_secret(value))
+        elif isinstance(value, (list, tuple, set, frozenset)):
+            pending.extend(reversed(list(value)))
+    return tuple(found)
 
 
-def secret_values(instance: Any) -> tuple[object, ...]:
-    """Return what each secret field of a model instance hides.
+def deferred_secret_values(values: Iterable[object]) -> Iterator[object]:
+    """Yield the secret values in `values`, collected on first read.
+
+    A write or a raw statement hands this to the engine, which reads it only
+    when the statement log is on: the engine's one read of the log's switch
+    decides whether anything is collected.
 
     Args:
-        instance: The model instance about to be saved.
+        values: The values about to be bound, read lazily.
 
-    Returns:
-        The unwrapped value of every field holding a `SecretStr` or
-        `SecretBytes`, in field order; empty when there is none.
+    Yields:
+        The unwrapped value of every secret in `values`, in order.
+    """
+    yield from secret_values_in(values)
+
+
+def field_values(instances: Iterable[Any]) -> Iterator[object]:
+    """Yield every field value of each model instance, in field order.
+
+    Args:
+        instances: The model instances about to be saved.
+
+    Yields:
+        Each instance's field values, as the instance holds them.
     """
-    return secret_values_of(
-        {name: getattr(instance, name, None) for name in type(instance).model_fields}
-    )
+    for instance in instances:
+        for name in type(instance).model_fields:
+            yield getattr(instance, name, None)
 
 
 def _raw_bound_fields(instance: Any) -> dict[str, Any]:

src/ferro/_core.pyi

@@ -1,4 +1,4 @@
-from collections.abc import Callable
+from collections.abc import Callable, Iterable
 from pathlib import Path
 from typing import Any, Literal, Optional
 
@@ -651,14 +651,14 @@ async def fetch_filtered(
     record_cls: type | None = None,
     hop_classes: dict[str, type] | None = None,
     *,
-    secret_values: list[object] = ...,
+    secret_values: Iterable[object] = ...,
 ) -> list[Any]: ...
 async def count_filtered(
     name: str,
     query_ir_json: str,
     route: RouteHandle,
     *,
-    secret_values: list[object] = ...,
+    secret_values: Iterable[object] = ...,
     operation: Literal["count", "exists"] = "count",
 ) -> int: ...
 async def fetch_one(
@@ -672,21 +672,21 @@ async def save_record(
     route: RouteHandle,
     mode: str = "insert",
     *,
-    secret_values: list[object] = ...,
+    secret_values: Iterable[object] = ...,
 ) -> int | None: ...
 async def update_record(
     name: str,
     data: dict[str, Any],
     route: RouteHandle,
     *,
-    secret_values: list[object] = ...,
+    secret_values: Iterable[object] = ...,
 ) -> int: ...
 async def save_bulk_records(
     name: str,
     rows: list[dict[str, Any]],
     route: RouteHandle,
     *,
-    secret_values: list[object] = ...,
+    secret_values: Iterable[object] = ...,
 ) -> int: ...
 async def delete_record(
     name: str,
@@ -698,14 +698,14 @@ async def delete_filtered(
     query_ir_json: str,
     route: RouteHandle,
     *,
-    secret_values: list[object] = ...,
+    secret_values: Iterable[object] = ...,
 ) -> int: ...
 async def update_filtered(
     name: str,
     query_ir_json: str,
     route: RouteHandle,
     *,
-    secret_values: list[object] = ...,
+    secret_values: Iterable[object] = ...,
 ) -> int: ...
 
 RenderedParts = tuple[str, tuple[object, ...], tuple[bool, ...], str]
@@ -719,29 +719,29 @@ async def render_fetch(
     record_cls: type | None = None,
     hop_classes: dict[str, type] | None = None,
     *,
-    secret_values: list[object] = ...,
+    secret_values: Iterable[object] = ...,
 ) -> RenderedParts: ...
 async def render_count(
     name: str,
     query_ir_json: str,
     route: RouteHandle,
     *,
-    secret_values: list[object] = ...,
+    secret_values: Iterable[object] = ...,
     operation: Literal["count", "exists"] = "count",
 ) -> RenderedParts: ...
 async def render_update(
     name: str,
     query_ir_json: str,
     route: RouteHandle,
     *,
-    secret_values: list[object] = ...,
+    secret_values: Iterable[object] = ...,
 ) -> RenderedParts: ...
 async def render_delete(
     name: str,
     query_ir_json: str,
     route: RouteHandle,
     *,
-    secret_values: list[object] = ...,
+    secret_values: Iterable[object] = ...,
 ) -> RenderedParts: ...
 async def add_m2m_links(
     join_table: str,
@@ -791,21 +791,21 @@ async def raw_execute(
     route: RouteHandle,
     autocommit: bool = False,
     *,
-    secret_values: list[object] = ...,
+    secret_values: Iterable[object] = ...,
 ) -> int: ...
 async def raw_fetch_all(
     sql: str,
     args: list[Any],
     route: RouteHandle,
     *,
-    secret_values: list[object] = ...,
+    secret_values: Iterable[object] = ...,
 ) -> list[dict[str, Any]]: ...
 async def raw_fetch_one(
     sql: str,
     args: list[Any],
     route: RouteHandle,
     *,
-    secret_values: list[object] = ...,
+    secret_values: Iterable[object] = ...,
 ) -> dict[str, Any] | None: ...
 def register_instance(
     name: str,

src/ferro/models.py

@@ -1,6 +1,5 @@
 """Define the core ORM model base and transaction helpers for Ferro."""
 
-import logging
 from contextlib import asynccontextmanager
 from enum import Enum
 from typing import (
@@ -25,11 +24,11 @@
 from ._bind_payload import (
     apply_save_exclude,
     apply_save_only,
+    deferred_secret_values,
+    field_values,
     normalize_save_columns,
     save_bind_payload,
-    secret_values,
 )
-from ._sql_log import LOGGER as _SQL_LOG
 from ._core import (
     begin_transaction,
     commit_transaction,
@@ -58,17 +57,6 @@
 )
 
 
-def _secrets_to_mask(instances: "list[Model]") -> list[object]:
-    """The secret field values of `instances` the statement log must mask.
-
-    Collected only while `ferro.sql` is enabled for `DEBUG`: with the log off
-    no record is written, so there is nothing to mask.
-    """
-    if not _SQL_LOG.isEnabledFor(logging.DEBUG):
-        return []
-    return [value for instance in instances for value in secret_values(instance)]
-
-
 _FERRO_CONNECTION_ATTR = "__ferro_connection_name"
 _FERRO_PERSISTED_ATTR = "__ferro_persisted"
 
@@ -474,7 +462,7 @@ async def save(
                     "Use Query.update() for set-oriented or expression writes."
                 )
         route, identity_using = await _instance_transaction_route(self, using, session)
-        secrets = _secrets_to_mask([self])
+        secrets = deferred_secret_values(field_values([self]))
         new_id = None
         if on_conflict == "update":
             new_id = await save_record(
@@ -827,7 +815,7 @@ async def bulk_create(
             cls.__ferro_identity__,
             data,
             route,
-            secret_values=_secrets_to_mask(list(instances)),
+            secret_values=deferred_secret_values(field_values(instances)),
         )
 
     @classmethod

src/ferro/query/builder.py

@@ -1305,7 +1305,7 @@ async def all(self: "Query[T]") -> list[T]:
             compiled.wire_json,
             route,
             hop_classes=compiled.hop_classes,
-            secret_values=list(compiled.secret_values),
+            secret_values=compiled.secret_values,
         )
         if self._before is not None:
             rows.reverse()
@@ -1333,7 +1333,7 @@ async def _count(self, operation: Literal["count", "exists"]) -> int:
             model_identity(self.model_cls),
             compiled.wire_json,
             route,
-            secret_values=list(compiled.secret_values),
+            secret_values=compiled.secret_values,
             operation=operation,
         )
 
@@ -1401,7 +1401,7 @@ async def update(
             model_identity(self.model_cls),
             compiled.wire_json,
             route,
-            secret_values=list(compiled.secret_values),
+            secret_values=compiled.secret_values,
         )
 
     @terminal
@@ -1445,7 +1445,7 @@ async def delete(self: "Query[T]") -> int:
             model_identity(self.model_cls),
             compiled.wire_json,
             route,
-            secret_values=list(compiled.secret_values),
+            secret_values=compiled.secret_values,
         )
 
     @terminal
@@ -1881,7 +1881,7 @@ async def all(self: "ProjectedQuery[T]") -> Rows[Row]:  # type: ignore[override]
             route,
             record_cls=Row,
             hop_classes=compiled.hop_classes,
-            secret_values=list(compiled.secret_values),
+            secret_values=compiled.secret_values,
         )
         if self._before is not None:
             records.reverse()