From e18d7e0aef7b795c94b45880a9a65e37e31d6014 Mon Sep 17 00:00:00 2001 From: Taylor Date: Wed, 21 Oct 2026 09:26:04 -0400 Subject: [PATCH 1/4] test(docs): regenerate every SQL block on the query and mutation guides tests/test_docs_sql.py reads each sql fence's marker line (-- .py: ()), runs the example once per dialect, records what every snippet region sends (a terminal's .sql(), the statement log for anything else) and compares byte for byte. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01Hb9U9A7Zu9ypdHTsDvdScP --- tests/test_docs_sql.py | 633 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 633 insertions(+) create mode 100644 tests/test_docs_sql.py diff --git a/tests/test_docs_sql.py b/tests/test_docs_sql.py new file mode 100644 index 00000000..8714221a --- /dev/null +++ b/tests/test_docs_sql.py @@ -0,0 +1,633 @@ +"""The SQL the query and mutation guides show is the SQL Ferro sends. + +A page never carries SQL a person typed. Every ```sql block on the Queries, +Aggregations, Relationships and Mutations guides opens with a marker line +naming where its SQL comes from:: + + -- predicates.py:first-query (sqlite) + +that is, an example script in ``docs/examples/``, one of its snippet regions +(``# --8<-- [start:first-query]`` ... ``[end:first-query]``) and the dialect. +The block holds every statement that region sends, in order: a query +terminal's statement as its ``.sql()`` renders it, and any other statement +(``save()``, ``create()``, ``bulk_create()``, ``delete()`` on an instance, a +many-to-many ``add()``) as the statement log records it. Schema statements +(``migrate``), Postgres catalog probes (``catalog``), session settings and a +``transaction()`` block's own ``BEGIN`` / ``COMMIT`` are not shown. + +Each statement is laid out with one clause per line (``_layout``) and ends +with ``;``. Joining a statement's lines with single spaces gives back the +rendered SQL byte for byte, so the layout adds line breaks and nothing else. + +The test runs each named example once per dialect in a subprocess +(``python -m tests.test_docs_sql [url]``, the capture +below), records what every snippet region sent and compares each marked +block with that record. A block marked ``(postgres)`` runs under the +``postgres_only`` marker, against the database ``--db-backends`` selects. +""" + +from __future__ import annotations + +import io +import json +import logging +import os +import re +import subprocess +import sys +from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import pytest + +REPO_ROOT = Path(__file__).resolve().parents[1] +DOCS_PAGES = REPO_ROOT / "docs" / "pages" +DOCS_EXAMPLES = REPO_ROOT / "docs" / "examples" + +PAGES = [ + "guide/queries.md", + "guide/aggregations.md", + "guide/relationships.md", + "guide/mutations.md", +] +"""The pages whose every ```sql block is regenerated by this test.""" + +MARKER = re.compile( + r"^-- (?P[A-Za-z0-9_]+\.py):(?P[A-Za-z0-9_-]+) " + r"\((?Psqlite|postgres)\)$" +) +"""The first line of a marked block: ``-- .py: ()``.""" + +SNIPPET = re.compile(r"#\s*--8<--\s*\[(?Pstart|end):(?P[A-Za-z0-9_-]+)\]") +"""A snippet region's start or end line in an example script.""" + +HIDDEN_OPERATIONS = frozenset({"migrate", "catalog", "settings", "transaction"}) +"""Statement-log operations a block does not show: schema statements, the +Postgres catalog probe, session settings and a transaction block's own +statements.""" + +CLAUSES = ( + "FROM", + "INNER JOIN", + "LEFT JOIN", + "WHERE", + "GROUP BY", + "HAVING", + "ORDER BY", + "LIMIT", + "OFFSET", + "SET", + "VALUES", + "ON CONFLICT", + "RETURNING", +) +"""The keywords that start a new line of a laid-out statement, at the top +level of the statement only (never inside parentheses or quotes).""" + + +# -- the layout ------------------------------------------------------------------ + + +def _layout(sql: str) -> str: + """``sql`` laid out one clause per line. + + A line break replaces the single space before each top-level clause + keyword (``CLAUSES``). A ``CREATE TABLE`` statement also puts each column + and constraint of its parenthesised body on its own line, indented two + spaces. Removing each line's indentation and joining the lines with + single spaces gives back ``sql``. + """ + table = sql.startswith("CREATE TABLE") + breaks: dict[int, str] = {} + depth = 0 + quote: str | None = None + for index, char in enumerate(sql): + if quote is not None: + if char == quote: + quote = None + continue + if char in "\"'": + quote = char + elif char == "(": + depth += 1 + elif char == ")": + depth -= 1 + elif char != " ": + continue + elif depth == 0: + rest = sql[index + 1 :] + if any( + rest.startswith(keyword) + and (len(rest) == len(keyword) or rest[len(keyword)] in " (") + for keyword in CLAUSES + ): + breaks[index] = "\n" + elif table and depth == 1: + # At depth 1 the body's own "(" or a "," between its items comes + # before the space; its closing ")" comes after it. + if sql[index - 1] in "(,": + breaks[index] = "\n " + elif sql[index + 1 : index + 2] == ")": + breaks[index] = "\n" + return "".join(breaks.get(index, char) for index, char in enumerate(sql)) + + +def _unlaid(laid: str) -> str: + """The statement ``_layout`` laid out: its lines unindented and joined by spaces.""" + return " ".join(line.lstrip(" ") for line in laid.splitlines()) + + +def _statements(sqls: list[str]) -> str: + """The body a marked block holds for ``sqls``: each laid out, ending in ``;``.""" + return "".join(f"{_layout(sql)};\n" for sql in sqls) + + +# -- the pages' blocks ----------------------------------------------------------- + + +@dataclass(frozen=True) +class SqlBlock: + """One ```sql block on a page.""" + + page: str + """The page, relative to ``docs/pages``.""" + line: int + """The line of the block's opening fence.""" + body: str + """Everything between the fences, the fence's indentation removed.""" + + @property + def where(self) -> str: + return f"{self.page}:{self.line}" + + @property + def marker(self) -> re.Match[str] | None: + return MARKER.match(self.body.partition("\n")[0]) + + @property + def shown(self) -> str: + """The block's statements: its body after the marker line.""" + return self.body.partition("\n")[2] + + +FENCE = re.compile( + r"^(?P[ \t]*)```(?P[^\n]*)\n(?P.*?)^(?P=indent)```", + flags=re.DOTALL | re.MULTILINE, +) + + +def _sql_blocks(page: str, text: str | None = None) -> list[SqlBlock]: + """Every ```sql block of ``page`` (or of ``text``, read as that page).""" + if text is None: + text = (DOCS_PAGES / page).read_text(encoding="utf-8") + blocks = [] + for match in FENCE.finditer(text): + if match["info"].strip().split(" ")[0] != "sql": + continue + body = "".join( + line.removeprefix(match["indent"]) + for line in match["body"].splitlines(keepends=True) + ) + blocks.append(SqlBlock(page, text.count("\n", 0, match.start()) + 1, body)) + return blocks + + +def _all_blocks() -> list[SqlBlock]: + return [block for page in PAGES for block in _sql_blocks(page)] + + +def _snippet_ranges(source: str) -> dict[str, tuple[int, int]]: + """Each snippet region of an example's ``source``, as its (start, end) lines.""" + starts: dict[str, int] = {} + ranges: dict[str, tuple[int, int]] = {} + for number, line in enumerate(source.splitlines(), start=1): + match = SNIPPET.search(line) + if match is None: + continue + if match["edge"] == "start": + starts[match["name"]] = number + elif match["name"] in starts: + ranges[match["name"]] = (starts.pop(match["name"]), number) + return ranges + + +def check_block(block: SqlBlock, sent: Callable[[str, str], dict[str, list[str]]]) -> None: + """Assert ``block`` is marked and holds what its example's region sends. + + Args: + block: The page's block. + sent: Given an example script's name and a dialect, what each of its + snippet regions sends, as ``{region: [sql, ...]}``. + + Raises: + AssertionError: The block has no marker, names an example or region + that does not exist, or differs from what the region sends. The + message names the page, the line and the example. + """ + marker = block.marker + assert marker is not None, ( + f"{block.where}: a ```sql block must open with a marker line " + f"'-- .py: ()' naming where its SQL comes " + f"from; this one opens with {block.body.partition(chr(10))[0]!r}" + ) + example, snippet, dialect = marker["example"], marker["snippet"], marker["dialect"] + named = f"{block.where} ({example}:{snippet}, {dialect})" + path = DOCS_EXAMPLES / example + assert path.is_file(), f"{named}: docs/examples/{example} does not exist" + ranges = _snippet_ranges(path.read_text(encoding="utf-8")) + assert snippet in ranges, f"{named}: {example} has no snippet region {snippet!r}" + statements = sent(example, dialect).get(snippet, []) + assert statements, f"{named}: the region {snippet!r} of {example} sends nothing" + expected = _statements(statements) + assert block.shown == expected, ( + f"{named}: the block differs from what {example}:{snippet} sends on " + f"{dialect}. It sends:\n{expected}" + ) + + +# -- the tests -------------------------------------------------------------------- + + +def test_layout_only_adds_line_breaks() -> None: + sql = ( + 'SELECT "t"."id" FROM "t" INNER JOIN "a" AS "x" ON "x"."id" = "t"."a_id" ' + 'WHERE EXISTS (SELECT 1 FROM "s" WHERE "s"."t_id" = "t"."id") ' + 'AND "t"."memo" = \'FROM x\' ORDER BY "t"."id" ASC LIMIT ? OFFSET ?' + ) + laid = _layout(sql) + assert laid.splitlines() == [ + 'SELECT "t"."id"', + 'FROM "t"', + 'INNER JOIN "a" AS "x" ON "x"."id" = "t"."a_id"', + 'WHERE EXISTS (SELECT 1 FROM "s" WHERE "s"."t_id" = "t"."id") ' + "AND \"t\".\"memo\" = 'FROM x'", + 'ORDER BY "t"."id" ASC', + "LIMIT ?", + "OFFSET ?", + ] + assert _unlaid(laid) == sql + + +def test_layout_puts_each_column_of_a_create_table_on_its_own_line() -> None: + sql = ( + 'CREATE TABLE IF NOT EXISTS "t" ( "id" integer NOT NULL, "a_id" integer, ' + 'CONSTRAINT "fk_t_a_id_a" FOREIGN KEY ("a_id") REFERENCES "a" ("id") ' + "ON DELETE CASCADE )" + ) + laid = _layout(sql) + assert laid.splitlines() == [ + 'CREATE TABLE IF NOT EXISTS "t" (', + ' "id" integer NOT NULL,', + ' "a_id" integer,', + ' CONSTRAINT "fk_t_a_id_a" FOREIGN KEY ("a_id") REFERENCES "a" ("id") ' + "ON DELETE CASCADE", + ")", + ] + assert _unlaid(laid) == sql + + +SENT = {"predicates.py": {"first-query": ['SELECT "id" FROM "transaction" WHERE "amount" <= ?']}} + + +def _fake_sent(example: str, dialect: str) -> dict[str, list[str]]: + return SENT.get(example, {}) + + +def _page_with(body: str) -> SqlBlock: + [block] = _sql_blocks( + "guide/queries.md", f"# Queries\n\nIt sends:\n\n```sql\n{body}```\n" + ) + return block + + +def test_a_block_holding_what_its_region_sends_passes() -> None: + check_block( + _page_with( + "-- predicates.py:first-query (sqlite)\n" + 'SELECT "id"\nFROM "transaction"\nWHERE "amount" <= ?;\n' + ), + _fake_sent, + ) + + +@pytest.mark.parametrize( + ("body", "message"), + [ + ( + "-- predicates.py:first-query (sqlite)\n" + 'SELECT "id"\nFROM "transaction"\nWHERE "amount" < ?;\n', + "guide/queries.md:5 (predicates.py:first-query, sqlite): the block differs", + ), + ( + 'SELECT "id"\nFROM "transaction"\nWHERE "amount" <= ?;\n', + "guide/queries.md:5: a ```sql block must open with a marker line", + ), + ( + "-- predicate.py:first-query (sqlite)\nSELECT 1;\n", + "guide/queries.md:5 (predicate.py:first-query, sqlite): " + "docs/examples/predicate.py does not exist", + ), + ( + "-- predicates.py:first-queries (sqlite)\nSELECT 1;\n", + "guide/queries.md:5 (predicates.py:first-queries, sqlite): " + "predicates.py has no snippet region 'first-queries'", + ), + ( + "-- predicates.py:typo (sqlite)\nSELECT 1;\n", + "guide/queries.md:5 (predicates.py:typo, sqlite): " + "the region 'typo' of predicates.py sends nothing", + ), + ], + ids=["edited", "unmarked", "unknown-example", "unknown-region", "sends-nothing"], +) +def test_a_block_that_is_not_what_its_region_sends_fails_naming_it( + body: str, message: str +) -> None: + with pytest.raises(AssertionError) as raised: + check_block(_page_with(body), _fake_sent) + assert str(raised.value).startswith(message), str(raised.value) + + +def test_every_page_shows_sql() -> None: + """Each of the four pages shows the SQL of its queries; none is left without.""" + for page in PAGES: + assert _sql_blocks(page), f"{page} shows no SQL" + + +PLACEHOLDER = re.compile(r"\$\d+|\?") +"""A bind placeholder: SQLite's ``?`` or PostgreSQL's ``$1``.""" + + +def check_same_on_postgres( + block: SqlBlock, sent: Callable[[str, str], dict[str, list[str]]] +) -> None: + """Assert a SQLite block stands for PostgreSQL too: the region sends the + same statements there once each placeholder is read as ``?``. + + A region whose SQL differs in more than its placeholders shows a + ``(postgres)`` block of its own, and this check does not apply to it. + + Raises: + AssertionError: PostgreSQL's statements differ in more than their + placeholders. The message names the page, the line and the example. + """ + marker = block.marker + assert marker is not None, f"{block.where}: a ```sql block has no marker line" + example, snippet = marker["example"], marker["snippet"] + statements = sent(example, "postgres").get(snippet, []) + expected = PLACEHOLDER.sub("?", _statements(statements)) + assert PLACEHOLDER.sub("?", block.shown) == expected, ( + f"{block.where} ({example}:{snippet}, sqlite): the block stands for both " + f"databases, but on postgres the region sends more than other " + f"placeholders; show a (postgres) block beside it. It sends:\n" + f"{_statements(statements)}" + ) + + +def test_a_block_differing_on_postgres_only_in_placeholders_stands_for_both() -> None: + block = _page_with( + "-- predicates.py:first-query (sqlite)\n" + 'SELECT "id"\nFROM "transaction"\nWHERE "amount" <= ?;\n' + ) + numbered = {"first-query": ['SELECT "id" FROM "transaction" WHERE "amount" <= $1']} + check_same_on_postgres(block, lambda example, dialect: numbered) + + cast = { + "first-query": [ + 'SELECT "id" FROM "transaction" WHERE "amount" <= CAST($1 AS numeric)' + ] + } + with pytest.raises(AssertionError) as raised: + check_same_on_postgres(block, lambda example, dialect: cast) + assert str(raised.value).startswith( + "guide/queries.md:5 (predicates.py:first-query, sqlite): the block stands " + "for both databases" + ) + + +def _capture(example: str, dialect: str, url: str | None) -> dict[str, list[str]]: + """What each snippet region of ``example`` sends on ``dialect``. + + The example runs with ``PYTHONHASHSEED=0``: ``save(only=...)`` and + ``save(exclude=...)`` write their columns in the order of a Python set, so + the order of their ``SET`` list follows string hashing. + """ + command = [sys.executable, "-m", "tests.test_docs_sql", example, dialect] + if url is not None: + command.append(url) + result = subprocess.run( + command, + capture_output=True, + text=True, + timeout=180, + cwd=REPO_ROOT, + env={**os.environ, "PYTHONHASHSEED": "0"}, + ) + assert result.returncode == 0, ( + f"capturing {example} on {dialect} failed\n" + f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}" + ) + return json.loads(result.stdout.strip().splitlines()[-1]) + + +_captured: dict[tuple[str, str], dict[str, list[str]]] = {} +"""Each example's capture per dialect, made once per test session.""" + + +def _sent_on(url: str | None) -> Callable[[str, str], dict[str, list[str]]]: + def sent(example: str, dialect: str) -> dict[str, list[str]]: + if (example, dialect) not in _captured: + if not (DOCS_EXAMPLES / example).is_file(): + return {} + _captured[example, dialect] = _capture(example, dialect, url) + return _captured[example, dialect] + + return sent + + +def _dialect(block: SqlBlock) -> str: + """The dialect a block is marked for; an unmarked block counts as SQLite's, + so it fails there naming itself.""" + return block.marker["dialect"] if block.marker else "sqlite" + + +def _postgres_twins() -> set[tuple[str, str, str]]: + """The (page, example, region) of every block marked ``(postgres)``.""" + return { + (block.page, block.marker["example"], block.marker["snippet"]) + for block in _all_blocks() + if block.marker and block.marker["dialect"] == "postgres" + } + + +@pytest.mark.parametrize( + "block", + [block for block in _all_blocks() if _dialect(block) == "sqlite"], + ids=lambda block: block.where, +) +def test_a_sqlite_block_is_what_its_example_sends(block: SqlBlock) -> None: + check_block(block, _sent_on(None)) + + +def _postgres_checked() -> list[SqlBlock]: + """Every block but a SQLite one whose region has a ``(postgres)`` twin.""" + twins = _postgres_twins() + return [ + block + for block in _all_blocks() + if not ( + block.marker + and block.marker["dialect"] == "sqlite" + and (block.page, block.marker["example"], block.marker["snippet"]) in twins + ) + ] + + +@pytest.mark.postgres_only +@pytest.mark.parametrize("block", _postgres_checked(), ids=lambda block: block.where) +def test_a_block_is_what_its_example_sends_on_postgres( + block: SqlBlock, db_url: str +) -> None: + """A ``(postgres)`` block is what its region sends on PostgreSQL; a SQLite + block with no ``(postgres)`` twin is too, once placeholders are read alike.""" + if _dialect(block) == "postgres": + check_block(block, _sent_on(db_url)) + return + check_same_on_postgres(block, _sent_on(db_url)) + + +# -- the capture ------------------------------------------------------------------- +# +# ``python -m tests.test_docs_sql [url]`` runs one example +# script as ``__main__`` and prints, as its last line, a JSON object mapping +# each snippet region to the statements sent while a line inside it ran. + + +class _Capture: + """What one example run sends, attributed to the example line that sent it.""" + + def __init__(self, path: Path, dialect: str) -> None: + self.path = str(path) + self.dialect = dialect + self.examples = str(DOCS_EXAMPLES) + self.where: tuple[str, int] | None = None + """The docs example file and line running now.""" + self.depth = 0 + """How many terminals are running; their own statements are rendered.""" + self.sent: list[tuple[int, str]] = [] + self.printed: list[tuple[int, str]] = [] + self.pending = "" + + def on_line(self, code: Any, line: int) -> Any: + if code.co_filename.startswith(self.examples): + self.where = (code.co_filename, line) + return None + return sys.monitoring.DISABLE + + def record(self, sql: str) -> None: + if self.where is not None and self.where[0] == self.path: + self.sent.append((self.where[1], sql)) + + def write(self, text: str) -> None: + """Record each line the example prints, at the example line printing it.""" + self.pending += text + *lines, self.pending = self.pending.split("\n") + if self.where is not None and self.where[0] == self.path: + self.printed.extend((self.where[1], line) for line in lines) + + def on_statement(self, record: logging.LogRecord) -> None: + operation = getattr(record, "operation", None) + if self.depth == 0 and operation not in HIDDEN_OPERATIONS: + assert record.dialect == self.dialect, record.dialect # type: ignore[attr-defined] + self.record(record.sql) # type: ignore[attr-defined] + + def by_region(self) -> dict[str, list[str]]: + ranges = _snippet_ranges(Path(self.path).read_text(encoding="utf-8")) + regions = {} + for name, (start, end) in ranges.items(): + sent = [sql for line, sql in self.sent if start < line < end] + printed = [text for line, text in self.printed if start < line < end] + regions[name] = sent or printed + return regions + + +def _run_capture(example: str, dialect: str, url: str | None) -> dict[str, list[str]]: + import asyncio # noqa: F401 (the example's own import, made early) + import runpy + + import ferro + from ferro.query.rendered import Terminal + + capture = _Capture(DOCS_EXAMPLES / example, dialect) + + class _Handler(logging.Handler): + def emit(self, record: logging.LogRecord) -> None: + capture.on_statement(record) + + logger = logging.getLogger("ferro.sql") + logger.addHandler(_Handler()) + logger.setLevel(logging.DEBUG) + + call = Terminal.__call__ + + def render_then_call(self: Terminal, *args: Any, **kwargs: Any) -> Any: + # Called first, so a terminal that refuses its query before returning + # an awaitable raises here, where the example expects it. + awaitable = call(self, *args, **kwargs) + if capture.depth: + return awaitable + + async def run() -> Any: + capture.depth += 1 + try: + try: + rendered = await self.sql(*args, **kwargs) + except Exception: # the call raises it too, and sends nothing + rendered = None + if rendered is not None: + assert rendered.dialect == dialect, rendered.dialect + capture.record(str(rendered)) + return await awaitable + finally: + capture.depth -= 1 + + return run() + + Terminal.__call__ = render_then_call # type: ignore[method-assign] + + if url is not None: + connect = ferro.connect + + async def connect_to(_url: str, *args: Any, **kwargs: Any) -> Any: + return await connect(url, *args, **kwargs) + + ferro.connect = connect_to # type: ignore[assignment] + + tool = 4 + sys.monitoring.use_tool_id(tool, "docs-sql") + sys.monitoring.register_callback(tool, sys.monitoring.events.LINE, capture.on_line) + sys.monitoring.set_events(tool, sys.monitoring.events.LINE) + sys.path.insert(0, str(DOCS_EXAMPLES)) + stdout = sys.stdout + + class _Tee(io.TextIOBase): + def write(self, text: str) -> int: + capture.write(text) + return stdout.write(text) + + sys.stdout = _Tee() + try: + runpy.run_path(str(DOCS_EXAMPLES / example), run_name="__main__") + finally: + sys.stdout = stdout + sys.monitoring.set_events(tool, 0) + sys.monitoring.free_tool_id(tool) + return capture.by_region() + + +if __name__ == "__main__": + example, dialect, *rest = sys.argv[1:] + regions = _run_capture(example, dialect, rest[0] if rest else None) + print(json.dumps(regions)) From 6a7f459653673bfd7f6b6b06739d0bc7fffa797a Mon Sep 17 00:00:00 2001 From: Taylor Date: Wed, 21 Oct 2026 09:33:36 -0400 Subject: [PATCH 2/4] docs: make the query and mutation examples run on both databases The SQL the guides show is captured on SQLite and PostgreSQL, so every example they name must run on both: the populated-relations raw update uses $1 placeholders, the mutations example stops asserting a SQLite id, the relationships DDL asserts check names rather than SQLite's text, the aggregations partition moves into the script, and the merge example runs when the session is on PostgreSQL. The capture renders a terminal once, however many terminals it runs inside. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01Hb9U9A7Zu9ypdHTsDvdScP --- docs/examples/aggregations.py | 11 +++++++ docs/examples/mutations.py | 29 ++++++++++++++++++- docs/examples/mutations_annotated.py | 9 ++++++ docs/examples/populated_relations.py | 3 +- docs/examples/relationships.py | 43 +++++++++------------------- tests/test_docs_sql.py | 13 +++++---- 6 files changed, 71 insertions(+), 37 deletions(-) diff --git a/docs/examples/aggregations.py b/docs/examples/aggregations.py index 83c5a6c9..8981ede0 100644 --- a/docs/examples/aggregations.py +++ b/docs/examples/aggregations.py @@ -8,6 +8,7 @@ """ import asyncio +from collections import defaultdict from decimal import Decimal from typing import Annotated @@ -237,6 +238,16 @@ async def main() -> None: assert row.avg_price == Decimal("3") # --8<-- [end:decimal-contract] + # --8<-- [start:partition] + # Whole instances bucketed by a key: partition a full query in Python. + by_account: dict[int | None, list[Transaction]] = defaultdict(list) + for transaction in await Transaction.select().all(): + by_account[transaction.account_id].append(transaction) + + sizes = {key: len(rows) for key, rows in by_account.items()} + assert sizes == {1: 2, 2: 1, None: 1} + # --8<-- [end:partition] + # --8<-- [start:error-family] # sum() over a string column has no portable meaning. try: diff --git a/docs/examples/mutations.py b/docs/examples/mutations.py index 9cacdfc0..5cd0cca8 100644 --- a/docs/examples/mutations.py +++ b/docs/examples/mutations.py @@ -22,6 +22,15 @@ class Customer(Model): # --8<-- [end:models] +# --8<-- [start:conversation] +class Conversation(Model): + id: int | None = Field(default=None, primary_key=True) + turns: dict = Field(default_factory=dict) + + +# --8<-- [end:conversation] + + async def main() -> None: # --8<-- [start:session] await connect("sqlite::memory:", auto_migrate=True) @@ -68,7 +77,7 @@ async def main() -> None: # --8<-- [start:save-insert-update] frank = Customer(email="frank@example.com", name="Frank") await frank.save() # never persisted: INSERT - assert frank.id == 6 + assert frank.id is not None # the database assigned it frank.plan = "pro" await frank.save() # persisted: UPDATE by primary key @@ -133,6 +142,24 @@ async def main() -> None: assert alice.views == 1 and alice.updated_at is not None # --8<-- [end:recipe-update] + # .merge() exists on PostgreSQL only; the rendered statement says + # which database this session runs on. + on_postgres = (await Customer.select().count.sql()).dialect == "postgres" + if on_postgres: + # --8<-- [start:merge] + chat = await Conversation.create() + await Conversation.where( + lambda conversation: conversation.id == chat.id + ).update( + lambda conversation: { + "turns": conversation.turns.merge({"r1": {"role": "user"}}) + } + ) + + await chat.refresh() + assert chat.turns == {"r1": {"role": "user"}} + # --8<-- [end:merge] + # --8<-- [start:mutation-guard] trial = Customer.where(lambda customer: customer.plan == "trial") try: diff --git a/docs/examples/mutations_annotated.py b/docs/examples/mutations_annotated.py index ee8e85d5..0a2c3527 100644 --- a/docs/examples/mutations_annotated.py +++ b/docs/examples/mutations_annotated.py @@ -28,6 +28,15 @@ class Customer(Model): # --8<-- [end:models] +# --8<-- [start:conversation] +class Conversation(Model): + id: Annotated[int | None, Field(primary_key=True)] = None + turns: Annotated[dict, Field(default_factory=dict)] + + +# --8<-- [end:conversation] + + async def main() -> None: await connect("sqlite::memory:", auto_migrate=True) diff --git a/docs/examples/populated_relations.py b/docs/examples/populated_relations.py index b09f4910..5d3d49f5 100644 --- a/docs/examples/populated_relations.py +++ b/docs/examples/populated_relations.py @@ -140,7 +140,8 @@ async def main() -> None: assert from_checking.account.label == "Savings" # Move the row to another account behind the ORM's back, then fetch it. - await execute('UPDATE "transaction" SET account_id = ? WHERE id = ?', 1, 5) + # ($1 and $2 are placeholders SQLite and PostgreSQL both accept.) + await execute('UPDATE "transaction" SET account_id = $1 WHERE id = $2', 1, 5) await Transaction.get(5) # refreshes the instance in place pending = from_checking.account # awaitable again diff --git a/docs/examples/relationships.py b/docs/examples/relationships.py index 0b7b3d56..44784725 100644 --- a/docs/examples/relationships.py +++ b/docs/examples/relationships.py @@ -101,15 +101,12 @@ async def main() -> None: if statement.subject == "player": print(statement.sql) # --8<-- [end:player-ddl] - assert table_ddl(report, "player") == [ - 'CREATE TABLE IF NOT EXISTS "player" ( "id" integer NOT NULL PRIMARY KEY ' - 'AUTOINCREMENT, "mentor_id" integer, "name" varchar NOT NULL, "team_id" ' - 'integer NOT NULL, CONSTRAINT "fk_player_mentor_id_player" FOREIGN KEY ' - '("mentor_id") REFERENCES "player" ("id") ON DELETE SET NULL, CONSTRAINT ' - '"fk_player_team_id_team" FOREIGN KEY ("team_id") REFERENCES "team" ("id") ' - "ON DELETE CASCADE )", - 'CREATE INDEX IF NOT EXISTS "idx_player_team_id" ON "player" ("team_id")', - ] + # The page shows these statements whole, on both databases + # (tests/test_docs_sql.py); here, the names each relation gives them. + player_ddl = "\n".join(table_ddl(report, "player")) + assert 'CONSTRAINT "fk_player_mentor_id_player"' in player_ddl + assert 'CONSTRAINT "fk_player_team_id_team"' in player_ddl + assert '"idx_player_team_id" ON "player" ("team_id")' in player_ddl # --8<-- [start:populated] players = ( @@ -134,14 +131,11 @@ async def main() -> None: assert profile.bio == "Plays in the front row" assert await corro.profile is None # Corro has no profile # --8<-- [end:one-to-one] - assert table_ddl(report, "profile") == [ - 'CREATE TABLE IF NOT EXISTS "profile" ( "bio" varchar NOT NULL, "id" integer ' - 'NOT NULL PRIMARY KEY AUTOINCREMENT, "player_id" integer NOT NULL, ' - 'CONSTRAINT "fk_profile_player_id_player" FOREIGN KEY ("player_id") ' - 'REFERENCES "player" ("id") ON DELETE CASCADE )', + # unique=True gives the shadow column a unique index. + assert ( 'CREATE UNIQUE INDEX IF NOT EXISTS "uq_profile_player_id" ON "profile" ' - '("player_id")', - ] + '("player_id")' + ) in table_ddl(report, "profile") # --8<-- [start:many-to-many] spring = await Tournament.create(name="Spring Open") @@ -177,19 +171,10 @@ async def main() -> None: if statement.subject == "player_tournaments": print(statement.sql) # --8<-- [end:join-table-ddl] - assert table_ddl(report, "player_tournaments") == [ - 'CREATE TABLE IF NOT EXISTS "player_tournaments" ( "player_id" integer NOT ' - 'NULL, "tournament_id" integer NOT NULL, CONSTRAINT ' - '"fk_player_tournaments_player_id_player" FOREIGN KEY ("player_id") ' - 'REFERENCES "player" ("id") ON DELETE CASCADE, CONSTRAINT ' - '"fk_player_tournaments_tournament_id_tournament" FOREIGN KEY ' - '("tournament_id") REFERENCES "tournament" ("id") ON DELETE CASCADE )', - 'CREATE INDEX IF NOT EXISTS "idx_player_tournaments_tournament_id_player_id" ' - 'ON "player_tournaments" ("tournament_id", "player_id")', - "CREATE UNIQUE INDEX IF NOT EXISTS " - '"uq_player_tournaments_player_id_tournament_id" ON "player_tournaments" ' - '("player_id", "tournament_id")', - ] + link_ddl = "\n".join(table_ddl(report, "player_tournaments")) + assert 'CONSTRAINT "fk_player_tournaments_player_id_player"' in link_ddl + assert '"idx_player_tournaments_tournament_id_player_id"' in link_ddl + assert '"uq_player_tournaments_player_id_tournament_id"' in link_ddl # --8<-- [start:self-referential] crabby = await Player.create(name="Crabby", team=rustaceans, mentor=ferris) diff --git a/tests/test_docs_sql.py b/tests/test_docs_sql.py index 8714221a..06b08c96 100644 --- a/tests/test_docs_sql.py +++ b/tests/test_docs_sql.py @@ -15,8 +15,8 @@ (``migrate``), Postgres catalog probes (``catalog``), session settings and a ``transaction()`` block's own ``BEGIN`` / ``COMMIT`` are not shown. -Each statement is laid out with one clause per line (``_layout``) and ends -with ``;``. Joining a statement's lines with single spaces gives back the +Each statement is laid out with one clause per line (``_layout``), ends +with ``;`` and is set off from the next by a blank line. Joining a statement's lines with single spaces gives back the rendered SQL byte for byte, so the layout adds line breaks and nothing else. The test runs each named example once per dialect in a subprocess @@ -140,8 +140,9 @@ def _unlaid(laid: str) -> str: def _statements(sqls: list[str]) -> str: - """The body a marked block holds for ``sqls``: each laid out, ending in ``;``.""" - return "".join(f"{_layout(sql)};\n" for sql in sqls) + """The body a marked block holds for ``sqls``: each laid out and ending in + ``;``, with a blank line between two statements.""" + return "\n".join(f"{_layout(sql)};\n" for sql in sqls) # -- the pages' blocks ----------------------------------------------------------- @@ -576,10 +577,10 @@ def render_then_call(self: Terminal, *args: Any, **kwargs: Any) -> Any: # Called first, so a terminal that refuses its query before returning # an awaitable raises here, where the example expects it. awaitable = call(self, *args, **kwargs) - if capture.depth: - return awaitable async def run() -> Any: + if capture.depth: # a terminal another terminal runs: not rendered again + return await awaitable capture.depth += 1 try: try: From 64875792d1ef49eaeb889440eb69fa9dc0f9ee41 Mon Sep 17 00:00:00 2001 From: Taylor Date: Wed, 21 Oct 2026 09:33:36 -0400 Subject: [PATCH 3/4] docs: show the SQL each query and mutation sends on the four guides Every sql fence on Queries, Aggregations, Relationships and Mutations opens with a -- .py: () marker and is regenerated by tests/test_docs_sql.py; PostgreSQL gets its own tab where it differs in more than placeholders. The hand-laid DDL blocks are replaced, the false exists() tip is removed, and the conventions page records the marker recipe. The CI docs job runs the new test. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01Hb9U9A7Zu9ypdHTsDvdScP --- .github/workflows/ci.yml | 1 + docs/pages/guide/aggregations.md | 158 ++- docs/pages/guide/mutations.md | 537 +++++++- docs/pages/guide/queries.md | 1087 ++++++++++++++++- docs/pages/guide/relationships.md | 431 ++++++- .../patterns/docs-page-conventions.md | 60 +- 6 files changed, 2186 insertions(+), 88 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b4f326ad..a7b45acc 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -108,6 +108,7 @@ jobs: tests/test_docs_examples.py \ tests/test_docstring_examples.py \ tests/test_docs_migrations.py \ + tests/test_docs_sql.py \ tests/test_docs_testing_fixtures.py \ tests/test_docs_checks.py \ tests/test_docs_lint.py \ diff --git a/docs/pages/guide/aggregations.md b/docs/pages/guide/aggregations.md index f32c58c9..08d5fd93 100644 --- a/docs/pages/guide/aggregations.md +++ b/docs/pages/guide/aggregations.md @@ -14,6 +14,18 @@ Here is a whole program. It adds four transactions and asks for the sum of the a The single record holds four numbers over the four rows: they total 115, so the average is 28.75. Every query on this page runs inside a session like this one; see [Sessions](connections.md#sessions) for why and how to open one. +Every example on this page is followed by the SQL it sends, captured from Ferro when the example runs. `first()` sends this statement; on PostgreSQL only the placeholders differ, `$1` where SQLite has `?`: + +```sql +-- aggregations.py:global (sqlite) +SELECT COUNT("transaction"."id") AS "n", SUM("transaction"."amount") AS "total", AVG("transaction"."amount") AS "average", MIN("transaction"."amount") AS "smallest", MAX("transaction"."amount") AS "largest" +FROM "transaction" +WHERE TRUE +LIMIT ?; +``` + +Each dict key names an aggregate with `AS`, and `first()` adds a `LIMIT` of 1. + ## The ledger Every example on this page queries one small ledger. An account holds transactions, and a transaction may have no account: @@ -53,6 +65,19 @@ Aggregates measure whatever `where()` leaves in, relation traversal included: Only the two Checking transactions are measured, so the total is 30. The source column can traverse as well (`transaction.account.label.min()`): the traversal keeps rows where the relation exists, the way a `where()` on the same path does, and shares that path's join with any other clause that uses it. +The query sends this; on PostgreSQL only the placeholders differ: + +```sql +-- aggregations.py:global-where (sqlite) +SELECT SUM("transaction"."amount") AS "total" +FROM "transaction" +INNER JOIN "account" AS "j1_account" ON "transaction"."account_id" = "j1_account"."id" +WHERE "j1_account"."label" = ? +LIMIT ?; +``` + +The traversal is an `INNER JOIN` on `account`, so `SUM` measures only the joined rows. + ### Result types The result type comes from the source column's Python type, so a query returns the same Python types on SQLite and Postgres, even where the databases differ (Postgres widens `SUM(bigint)` to `numeric`; SQLite averages everything as a float): @@ -73,6 +98,18 @@ For example, the average of the three prices stays a `Decimal`: The prices total 9.00 over three rows, so the average is `Decimal("3")`. Every aggregate result except `count()` can be `None`, as the next section shows. +The query sends this; on PostgreSQL only the placeholders differ: + +```sql +-- aggregations.py:decimal-contract (sqlite) +SELECT AVG("transaction"."price") AS "avg_price" +FROM "transaction" +WHERE "transaction"."price" IS NOT NULL +LIMIT ?; +``` + +SQL's `AVG` returns a float on SQLite and a `numeric` on PostgreSQL; Ferro reads either back as a `Decimal`, because the source column is one. + ### Empty input SQL answers an aggregate over zero rows with `NULL` (`COUNT` answers `0`), and Ferro passes that through. "Sum of no rows" and "sum of rows that total zero" are different facts, and zero would be the wrong answer for `min()` and `max()` anyway: @@ -83,6 +120,18 @@ SQL answers an aggregate over zero rows with `NULL` (`COUNT` answers `0`), and F No transaction is over 10,000, so the query still returns one record: `n` is `0` and `total` is `None`. +The query sends this; on PostgreSQL only the placeholders differ: + +```sql +-- aggregations.py:empty (sqlite) +SELECT COUNT("transaction"."id") AS "n", SUM("transaction"."amount") AS "total" +FROM "transaction" +WHERE "transaction"."amount" > ? +LIMIT ?; +``` + +With no `GROUP BY`, SQL answers an aggregate-only `SELECT` with exactly one row, even over zero rows. + ### Counting a column `transaction.price.count()` is SQL's `COUNT(price)`: it counts the rows where that column is not `NULL`. To count rows whatever else they hold, count the primary key (`transaction.id.count()`): @@ -93,6 +142,18 @@ No transaction is over 10,000, so the query still returns one record: `n` is `0` All four transactions are counted by `id`, and three by `price`, because the orphan has no price. +The query sends this; on PostgreSQL only the placeholders differ: + +```sql +-- aggregations.py:count-nulls (sqlite) +SELECT COUNT("transaction"."id") AS "rows", COUNT("transaction"."price") AS "priced" +FROM "transaction" +WHERE TRUE +LIMIT ?; +``` + +`COUNT("transaction"."price")` skips the `NULL` price; `COUNT("transaction"."id")` counts every row. + ## Grouped queries Mix a plain field into an aggregate projection and the query is grouped. Every plain field is a group key, and each group collapses to one record: @@ -103,6 +164,19 @@ Mix a plain field into an aggregate projection and the query is grouped. Every p Account 1 totals 30 and account 2 totals 45, one record each. There is no `group_by()`: the record shape is the grouping. SQL already requires every selected bare column to be a group key, so a separate `group_by()` could only repeat the projection or contradict it. Postgres rejects the contradiction at run time, and SQLite answers with an arbitrary row's value. Deriving the grouping from the projection removes that whole class of error. +The query sends this; on PostgreSQL only the placeholders differ: + +```sql +-- aggregations.py:grouped (sqlite) +SELECT "transaction"."account_id" AS "acct", SUM("transaction"."amount") AS "total" +FROM "transaction" +WHERE "transaction"."account_id" IS NOT NULL +GROUP BY "transaction"."account_id" +ORDER BY "acct" ASC NULLS LAST; +``` + +The plain field became the `GROUP BY` key, and `order_by("acct")` sorts by its output name. + Group keys can traverse, and traversal narrows rows the way a predicate does: ```python @@ -111,6 +185,20 @@ Group keys can traverse, and traversal narrows rows the way a predicate does: The orphan has no account, so it drops out and the two labelled groups remain. +The query sends this; on PostgreSQL only the placeholders differ: + +```sql +-- aggregations.py:grouped-traversed (sqlite) +SELECT "j1_account"."label" AS "account_label", SUM("transaction"."amount") AS "total" +FROM "transaction" +INNER JOIN "account" AS "j1_account" ON "transaction"."account_id" = "j1_account"."id" +WHERE TRUE +GROUP BY "j1_account"."label" +ORDER BY "account_label" ASC NULLS LAST; +``` + +The `INNER JOIN` drops the orphan before grouping, and the joined `label` is the group key. + ### The `None` group for missing relations Add `left_join()` and rows without a relation stay in, grouped under a `None` key instead of being dropped: @@ -121,6 +209,19 @@ Add `left_join()` and rows without a relation stay in, grouped under a `None` ke The orphan's 40 now appears as its own group. +The query sends this; on PostgreSQL only the placeholders differ: + +```sql +-- aggregations.py:none-bucket (sqlite) +SELECT "j1_account"."label" AS "account_label", SUM("transaction"."amount") AS "total" +FROM "transaction" +LEFT JOIN "account" AS "j1_account" ON "transaction"."account_id" = "j1_account"."id" +WHERE TRUE +GROUP BY "j1_account"."label"; +``` + +The `LEFT JOIN` keeps the orphan, whose `label` is `NULL`, so it forms the `None` group. + ### Grouped queries over zero rows A grouped query over zero matching rows returns zero records. Unlike a global aggregate, there is no group to report on: @@ -129,21 +230,36 @@ A grouped query over zero matching rows returns zero records. Unlike a global ag --8<-- "docs/examples/aggregations.py:zero-groups" ``` +No transaction is over 10,000, so `rows` is empty. The query sends this; on PostgreSQL only the placeholders differ: + +```sql +-- aggregations.py:zero-groups (sqlite) +SELECT "transaction"."account_id" AS "acct", SUM("transaction"."amount") AS "total" +FROM "transaction" +WHERE "transaction"."amount" > ? +GROUP BY "transaction"."account_id"; +``` + +With a `GROUP BY` and no rows, SQL returns no groups at all. + ### Grouping and partitioning Grouping collapses rows: each group becomes one record of keys and measures, and the individual rows are gone. If you want whole model instances bucketed by a key, every transaction arranged per account, do that in Python over a full query: -```python {test=run prelude=aggregations} -from collections import defaultdict +```python +--8<-- "docs/examples/aggregations.py:partition" +``` -by_account: dict[int | None, list[Transaction]] = defaultdict(list) -for transaction in await Transaction.select().all(): - by_account[transaction.account_id].append(transaction) +Account 1 holds two transactions, account 2 holds one, and the orphan sits under `None`. The query sends this; on PostgreSQL only the placeholders differ: -assert {key: len(rows) for key, rows in by_account.items()} == {1: 2, 2: 1, None: 1} +```sql +-- aggregations.py:partition (sqlite) +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."id", "transaction"."memo", "transaction"."price" +FROM "transaction" +WHERE TRUE; ``` -Account 1 holds two transactions, account 2 holds one, and the orphan sits under `None`. Reach for aggregation when you want facts about groups, and partition in Python when you want the rows themselves, organized. This block runs inside a session, as [Sessions](connections.md#sessions) describes. +It is a plain `SELECT` of whole rows, with no `GROUP BY`: the bucketing happens in Python. Reach for aggregation when you want facts about groups, and partition in Python when you want the rows themselves, organized. ## Ordering grouped results @@ -159,6 +275,21 @@ A string resolves output field names first, then root columns. `order_by("total" Savings has the largest total, so it is the one group returned. `limit()` and `offset()` always act on groups in a grouped query, so pagination pages through the summary, not the underlying rows. +The query sends this; on PostgreSQL only the placeholders differ: + +```sql +-- aggregations.py:top-n (sqlite) +SELECT "j1_account"."label" AS "account_label", SUM("transaction"."amount") AS "total" +FROM "transaction" +INNER JOIN "account" AS "j1_account" ON "transaction"."account_id" = "j1_account"."id" +WHERE TRUE +GROUP BY "j1_account"."label" +ORDER BY "total" DESC NULLS LAST +LIMIT ?; +``` + +`ORDER BY "total"` names the output field, and `LIMIT` applies to the groups. + ### Sorting by an aggregate expression Where strings name outputs, a lambda writes the source expression itself, aggregates included: @@ -169,6 +300,19 @@ Where strings name outputs, a lambda writes the source expression itself, aggreg The groups come back with the largest total first. An aggregate sort key must match a projected aggregate (same function, same column, same path). An expression you did not project raises when you build the query and tells you to name it in the projection. +The query sends this; on PostgreSQL only the placeholders differ: + +```sql +-- aggregations.py:order-by-lambda (sqlite) +SELECT "transaction"."account_id" AS "acct", SUM("transaction"."amount") AS "total" +FROM "transaction" +WHERE "transaction"."account_id" IS NOT NULL +GROUP BY "transaction"."account_id" +ORDER BY "total" DESC NULLS LAST; +``` + +The lambda's `transaction.amount.sum()` matched the projected aggregate, so the sort reads its output name, `"total"`. + ### Allowed sort keys On an aggregate projection, every sort key must be a group key or an aggregate. Sorting groups by any other column is the arbitrary-row trap again, so Ferro rejects it when you build the query, whether the key is a string, a lambda, or was chained before the `select()`. On a plain, non-aggregate projection nothing changes: unselected root columns still sort. diff --git a/docs/pages/guide/mutations.md b/docs/pages/guide/mutations.md index ec8fcbb9..5ddc688c 100644 --- a/docs/pages/guide/mutations.md +++ b/docs/pages/guide/mutations.md @@ -14,6 +14,27 @@ Every call runs inside a session. Connect, open one, and create a customer: `create()` returns the saved instance, with the primary key the database assigned and `plan` filled from its default. The blocks below run inside this session; see [Sessions](connections.md#sessions). +Every example on this page is followed by the SQL it sends, captured from Ferro when the example runs. `create()` sends one `INSERT`. PostgreSQL also casts the `datetime` parameter to `timestamptz` and asks for the new `id` back with `RETURNING`: + +=== "SQLite" + + ```sql + -- mutations.py:session (sqlite) + INSERT INTO "customer" ("email", "name", "plan", "views", "updated_at") + VALUES (?, ?, ?, ?, ?); + ``` + +=== "PostgreSQL" + + ```sql + -- mutations.py:session (postgres) + INSERT INTO "customer" ("email", "name", "plan", "views", "updated_at") + VALUES ($1, $2, $3, $4, CAST($5 AS timestamptz)) + RETURNING "id"; + ``` + +On SQLite the driver reports the new row's `id` without a `RETURNING` clause. + The examples use one model, a customer with a unique email: === "Assignment" @@ -49,6 +70,41 @@ Ferro has four ways to write a row, and each does only what its name says: The table now holds four customers. Fields you leave out take their defaults, so Carol's `plan` is `"free"`. +`create()`, `bulk_create()` and `count()` send these statements. PostgreSQL also casts the `datetime` parameter to `timestamptz`, and its single-row insert returns the new `id`: + +=== "SQLite" + + ```sql + -- mutations.py:create (sqlite) + INSERT INTO "customer" ("email", "name", "plan", "views", "updated_at") + VALUES (?, ?, ?, ?, ?); + + INSERT INTO "customer" ("email", "name", "plan", "views", "updated_at") + VALUES (?, ?, ?, ?, ?), (?, ?, ?, ?, ?); + + SELECT COUNT(*) + FROM "customer" + WHERE TRUE; + ``` + +=== "PostgreSQL" + + ```sql + -- mutations.py:create (postgres) + INSERT INTO "customer" ("email", "name", "plan", "views", "updated_at") + VALUES ($1, $2, $3, $4, CAST($5 AS timestamptz)) + RETURNING "id"; + + INSERT INTO "customer" ("email", "name", "plan", "views", "updated_at") + VALUES ($1, $2, $3, $4, CAST($5 AS timestamptz)), ($6, $7, $8, $9, CAST($10 AS timestamptz)); + + SELECT COUNT(*) + FROM "customer" + WHERE TRUE; + ``` + +`bulk_create()` sends one multi-row `INSERT` with no `RETURNING`, which is why it writes no IDs back. + To create a row with a foreign key, see [Relationships](relationships.md). Pydantic validates each instance when it is built, so invalid data raises before Ferro sends anything to the database. A batch can be any size. SQLite binds at most 32,766 parameters in one statement and PostgreSQL 65,535, so Ferro splits a large batch across as many `INSERT` statements as the database needs. The batch stays atomic: inside a [`transaction()`](transactions.md) block the transaction is the boundary, and outside one, a failing row (a unique violation late in the batch, say) leaves none of the batch inserted. @@ -62,6 +118,36 @@ A batch can be any size. SQLite binds at most 32,766 parameters in one statement The instance passed to `bulk_create()` still has `id = None`; the one loaded back has the database's ID, `5`. +The batch and the lookup in that note send these statements. PostgreSQL also casts the `datetime` parameter to `timestamptz`: + +=== "SQLite" + + ```sql + -- mutations.py:bulk-identity (sqlite) + INSERT INTO "customer" ("email", "name", "plan", "views", "updated_at") + VALUES (?, ?, ?, ?, ?); + + SELECT "customer"."email", "customer"."id", "customer"."name", "customer"."plan", "customer"."updated_at", "customer"."views" + FROM "customer" + WHERE "customer"."email" = ? + LIMIT ?; + ``` + +=== "PostgreSQL" + + ```sql + -- mutations.py:bulk-identity (postgres) + INSERT INTO "customer" ("email", "name", "plan", "views", "updated_at") + VALUES ($1, $2, $3, $4, CAST($5 AS timestamptz)); + + SELECT "customer"."email", "customer"."id", "customer"."name", "customer"."plan", "customer"."updated_at", "customer"."views" + FROM "customer" + WHERE "customer"."email" = $1 + LIMIT $2; + ``` + +The batch's `INSERT` returns no `id` on either database. + `create()` always inserts. A duplicate primary key or unique value raises `UniqueViolationError` and leaves the existing row as it was: ```python @@ -70,6 +156,37 @@ A batch can be any size. SQLite binds at most 32,766 parameters in one statement Alice's row still holds the name `"Alice"`. Use [`upsert()`](#upsert) when you want insert-or-update instead. +The refused insert and the lookup send these statements. PostgreSQL also casts the `datetime` parameter to `timestamptz` and adds `RETURNING "id"`: + +=== "SQLite" + + ```sql + -- mutations.py:create-strict (sqlite) + INSERT INTO "customer" ("email", "name", "plan", "views", "updated_at") + VALUES (?, ?, ?, ?, ?); + + SELECT "customer"."email", "customer"."id", "customer"."name", "customer"."plan", "customer"."updated_at", "customer"."views" + FROM "customer" + WHERE "customer"."id" = ? + LIMIT ?; + ``` + +=== "PostgreSQL" + + ```sql + -- mutations.py:create-strict (postgres) + INSERT INTO "customer" ("email", "name", "plan", "views", "updated_at") + VALUES ($1, $2, $3, $4, CAST($5 AS timestamptz)) + RETURNING "id"; + + SELECT "customer"."email", "customer"."id", "customer"."name", "customer"."plan", "customer"."updated_at", "customer"."views" + FROM "customer" + WHERE "customer"."id" = $1 + LIMIT $2; + ``` + +The database refused the `INSERT`; `UniqueViolationError` carries its report. + ## Saving: insert or update An instance is either **transient** (built with `Customer(...)` and never saved) or **persisted** (loaded from the database, or saved). `save()` reads that state: it inserts a transient instance and updates a persisted one by primary key: @@ -80,6 +197,43 @@ An instance is either **transient** (built with `Customer(...)` and never saved) The first `save()` inserted Frank and set his `id`; the second updated the same row, so there is still one Frank. A transient instance with a duplicate primary key or unique value raises `UniqueViolationError`, as `create()` does. +The two saves and the count send these statements. PostgreSQL also casts the `datetime` parameter to `timestamptz` and returns the new `id`: + +=== "SQLite" + + ```sql + -- mutations.py:save-insert-update (sqlite) + INSERT INTO "customer" ("email", "name", "plan", "views", "updated_at") + VALUES (?, ?, ?, ?, ?); + + UPDATE "customer" + SET "email" = ?, "name" = ?, "plan" = ?, "views" = ?, "updated_at" = ? + WHERE "id" = ?; + + SELECT COUNT(*) + FROM "customer" + WHERE "customer"."name" = ?; + ``` + +=== "PostgreSQL" + + ```sql + -- mutations.py:save-insert-update (postgres) + INSERT INTO "customer" ("email", "name", "plan", "views", "updated_at") + VALUES ($1, $2, $3, $4, CAST($5 AS timestamptz)) + RETURNING "id"; + + UPDATE "customer" + SET "email" = $1, "name" = $2, "plan" = $3, "views" = $4, "updated_at" = CAST($5 AS timestamptz) + WHERE "id" = $6; + + SELECT COUNT(*) + FROM "customer" + WHERE "customer"."name" = $1; + ``` + +The second `save()` wrote every column, by primary key. + If the row behind a persisted instance is gone (another writer deleted it, or you changed the primary key before saving), the update matches nothing. `save()` then raises `ModelDoesNotExist` instead of inserting the row again: ```python @@ -88,6 +242,36 @@ If the row behind a persisted instance is gone (another writer deleted it, or yo Frank stays deleted. `instance.delete()` makes an instance transient again, so a later `save()` on it inserts a new row. +The batch delete and the save send these statements. PostgreSQL also casts the `datetime` parameter to `timestamptz`: + +=== "SQLite" + + ```sql + -- mutations.py:save-missing (sqlite) + DELETE + FROM "customer" + WHERE "customer"."id" = ?; + + UPDATE "customer" + SET "email" = ?, "name" = ?, "plan" = ?, "views" = ?, "updated_at" = ? + WHERE "id" = ?; + ``` + +=== "PostgreSQL" + + ```sql + -- mutations.py:save-missing (postgres) + DELETE + FROM "customer" + WHERE "customer"."id" = $1; + + UPDATE "customer" + SET "email" = $1, "name" = $2, "plan" = $3, "views" = $4, "updated_at" = CAST($5 AS timestamptz) + WHERE "id" = $6; + ``` + +The `UPDATE` matched no row, so `save()` raised instead of sending an `INSERT`. + !!! note "Copies share persistence state" `model_copy()` copies the persistence state too, so saving a copy of a persisted instance updates the same row. To clone a row, build a new instance. @@ -101,6 +285,48 @@ Frank stays deleted. `instance.delete()` makes an instance transient again, so a The row kept `plan = "pro"`, while the instance still held `"enterprise"` until `refresh()` reloaded it. Ferro runs no `SELECT` after the update, which is why the instance and the row can differ. +The fetch, the partial save and the refresh send these statements. PostgreSQL also casts the `datetime` parameter to `timestamptz`: + +=== "SQLite" + + ```sql + -- mutations.py:save-only (sqlite) + SELECT "customer"."email", "customer"."id", "customer"."name", "customer"."plan", "customer"."updated_at", "customer"."views" + FROM "customer" + WHERE "customer"."id" = ? + LIMIT ?; + + UPDATE "customer" + SET "updated_at" = ?, "name" = ? + WHERE "id" = ?; + + SELECT "customer"."email", "customer"."id", "customer"."name", "customer"."plan", "customer"."updated_at", "customer"."views" + FROM "customer" + WHERE "customer"."id" = ? + LIMIT ?; + ``` + +=== "PostgreSQL" + + ```sql + -- mutations.py:save-only (postgres) + SELECT "customer"."email", "customer"."id", "customer"."name", "customer"."plan", "customer"."updated_at", "customer"."views" + FROM "customer" + WHERE "customer"."id" = $1 + LIMIT $2; + + UPDATE "customer" + SET "updated_at" = CAST($1 AS timestamptz), "name" = $2 + WHERE "id" = $3; + + SELECT "customer"."email", "customer"."id", "customer"."name", "customer"."plan", "customer"."updated_at", "customer"."views" + FROM "customer" + WHERE "customer"."id" = $1 + LIMIT $2; + ``` + +The `UPDATE` sets only `name` and `updated_at`. Ferro writes the columns of a partial save in no fixed order, so a log may show them the other way round. + `save(exclude=...)` writes every column except the ones you name: ```python @@ -109,6 +335,38 @@ The row kept `plan = "pro"`, while the instance still held `"enterprise"` until The name changed and the plan did not. +The save and the refresh send these statements. PostgreSQL also casts the `datetime` parameter to `timestamptz`: + +=== "SQLite" + + ```sql + -- mutations.py:save-exclude (sqlite) + UPDATE "customer" + SET "updated_at" = ?, "name" = ?, "views" = ?, "email" = ? + WHERE "id" = ?; + + SELECT "customer"."email", "customer"."id", "customer"."name", "customer"."plan", "customer"."updated_at", "customer"."views" + FROM "customer" + WHERE "customer"."id" = ? + LIMIT ?; + ``` + +=== "PostgreSQL" + + ```sql + -- mutations.py:save-exclude (postgres) + UPDATE "customer" + SET "updated_at" = CAST($1 AS timestamptz), "name" = $2, "views" = $3, "email" = $4 + WHERE "id" = $5; + + SELECT "customer"."email", "customer"."id", "customer"."name", "customer"."plan", "customer"."updated_at", "customer"."views" + FROM "customer" + WHERE "customer"."id" = $1 + LIMIT $2; + ``` + +Every column but `plan` and the primary key is in the `SET`. + Two rules matter most. A partial save is an update, so it needs a persisted instance: on a transient one, `only=` and `exclude=` raise `ValueError`. And a call takes one write-set, not both. The [`Model.save`](../api/model.md#ferro.Model.save) reference lists the rest: which names count as columns, and which write-sets are empty and refused. ## Batch updates and deletes @@ -121,6 +379,21 @@ Two rules matter most. A partial save is an update, so it needs a persisted inst Three free customers moved to the `"trial"` plan, and Erin's row is gone. +The update and the delete send these statements; on PostgreSQL only the placeholders differ: + +```sql +-- mutations.py:update-delete (sqlite) +UPDATE "customer" +SET "plan" = ? +WHERE "customer"."plan" = ?; + +DELETE +FROM "customer" +WHERE "customer"."name" = ?; +``` + +Each is one statement over every matching row, with the query's filter as its `WHERE`. + !!! warning "Batch updates do not touch instances you hold" `where(...).update(...)` writes to the database only. Instances already in memory, including those in the identity map, keep their old values until you call `refresh()` on them. @@ -134,47 +407,63 @@ Three free customers moved to the `"trial"` plan, and Erin's row is gone. The database added 1 to Alice's `views` and stamped `updated_at` with its own clock. -`+` and `-` follow SQL, so `NULL + 1` is `NULL`. `now` is a value you import (`from ferro import now`), not a function you call, and it can only be assigned to a `datetime` column. +The update and the refresh send these statements; on PostgreSQL only the placeholders differ: -On PostgreSQL, `.merge()` merges a dict into a JSON object column, with the patch winning where keys overlap. It replaces top-level keys only. A `NULL` column counts as `{}`. This block needs a PostgreSQL database: +```sql +-- mutations.py:recipe-update (sqlite) +UPDATE "customer" +SET "views" = "views" + ?, "updated_at" = CURRENT_TIMESTAMP +WHERE "customer"."id" = ?; -=== "Assignment" +SELECT "customer"."email", "customer"."id", "customer"."name", "customer"."plan", "customer"."updated_at", "customer"."views" +FROM "customer" +WHERE "customer"."id" = ? +LIMIT ?; +``` - ```python {test=compile reason="needs a PostgreSQL database"} - from ferro import Field, Model +`customer.views + 1` became `"views" + ?`, and `now` became `CURRENT_TIMESTAMP`. +`+` and `-` follow SQL, so `NULL + 1` is `NULL`. `now` is a value you import (`from ferro import now`), not a function you call, and it can only be assigned to a `datetime` column. - class Conversation(Model): - id: int | None = Field(default=None, primary_key=True) - turns: dict = Field(default_factory=dict) +On PostgreSQL, `.merge()` merges a dict into a JSON object column, with the patch winning where keys overlap. It replaces top-level keys only. A `NULL` column counts as `{}`. The examples use a conversation whose `turns` column holds a JSON object: +=== "Assignment" - chat = await Conversation.create() - await Conversation.where(lambda conversation: conversation.id == chat.id).update( - lambda conversation: {"turns": conversation.turns.merge({"r1": {"role": "user"}})} - ) + ```python + --8<-- "docs/examples/mutations.py:conversation" ``` === "Annotated" - ```python {test=compile reason="needs a PostgreSQL database"} - from typing import Annotated + ```python + --8<-- "docs/examples/mutations_annotated.py:conversation" + ``` + +This block needs a PostgreSQL database. It creates a conversation and merges one turn into it: - from ferro import Field, Model +```python +--8<-- "docs/examples/mutations.py:merge" +``` +After the update, `turns` holds `{"r1": {"role": "user"}}`. The create, the update and the refresh send these statements: - class Conversation(Model): - id: Annotated[int | None, Field(primary_key=True)] = None - turns: Annotated[dict, Field(default_factory=dict)] +```sql +-- mutations.py:merge (postgres) +INSERT INTO "conversation" ("turns") +VALUES (CAST($1 AS json)) +RETURNING "id"; +UPDATE "conversation" +SET "turns" = (COALESCE("turns"::jsonb, '{}'::jsonb)) || CAST($1 AS jsonb) +WHERE "conversation"."id" = $2; - chat = await Conversation.create() - await Conversation.where(lambda conversation: conversation.id == chat.id).update( - lambda conversation: {"turns": conversation.turns.merge({"r1": {"role": "user"}})} - ) - ``` +SELECT "conversation"."id", "conversation"."turns" +FROM "conversation" +WHERE "conversation"."id" = $1 +LIMIT $2; +``` -After the update, `turns` holds `{"r1": {"role": "user"}}`. `.merge()` raises when the query is built if the connection is SQLite, the column holds a list, or the patch is a list. +The `||` operator merges the patch into the column read as `jsonb`, and `COALESCE` stands in `{}` for a `NULL` column. `.merge()` raises when the query is built if the connection is SQLite, the column holds a list, or the patch is a list. ### No `limit()` on batch writes @@ -198,6 +487,23 @@ To change a bounded set of rows, fetch their primary keys first, then filter on Only Alice, the oldest trial customer, was deleted. +The fetch and the delete send these statements; on PostgreSQL only the placeholders differ: + +```sql +-- mutations.py:mutation-by-key (sqlite) +SELECT "customer"."email", "customer"."id", "customer"."name", "customer"."plan", "customer"."updated_at", "customer"."views" +FROM "customer" +WHERE "customer"."plan" = ? +ORDER BY "customer"."id" ASC NULLS LAST +LIMIT ?; + +DELETE +FROM "customer" +WHERE "customer"."id" IN (?); +``` + +The `DELETE` filters on the fetched keys with `IN`. + ## Deleting one row `instance.delete()` deletes the row behind an instance: @@ -208,6 +514,27 @@ Only Alice, the oldest trial customer, was deleted. Carol's row is gone. Deleting a parent row applies the `on_delete` rule of every foreign key that points at it, `CASCADE` by default; read [Delete behavior](relationships.md#delete-behavior) before deleting rows that have children. +The lookup, the delete and the check send these statements; on PostgreSQL only the placeholders differ: + +```sql +-- mutations.py:delete-one (sqlite) +SELECT "customer"."email", "customer"."id", "customer"."name", "customer"."plan", "customer"."updated_at", "customer"."views" +FROM "customer" +WHERE "customer"."email" = ? +LIMIT ?; + +DELETE +FROM "customer" +WHERE "customer"."id" = ?; + +SELECT "customer"."email", "customer"."id", "customer"."name", "customer"."plan", "customer"."updated_at", "customer"."views" +FROM "customer" +WHERE "customer"."id" = ? +LIMIT ?; +``` + +`carol.delete()` deletes one row by primary key. + ## `upsert()` `upsert(**fields)` inserts the row, or updates the existing row when the primary key already exists: @@ -218,6 +545,58 @@ Carol's row is gone. Deleting a parent row applies the `on_delete` rule of every The second call updated row 100 in place. It wrote the whole row, so `name` went back to its default `""`: pass every field you want to keep. +The two upserts, the count and the lookup send these statements. PostgreSQL also casts the `datetime` parameter to `timestamptz`: + +=== "SQLite" + + ```sql + -- mutations.py:upsert (sqlite) + INSERT INTO "customer" ("id", "email", "name", "plan", "views", "updated_at") + VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT ("id") DO UPDATE + SET "email" = "excluded"."email", "name" = "excluded"."name", "plan" = "excluded"."plan", "views" = "excluded"."views", "updated_at" = "excluded"."updated_at"; + + INSERT INTO "customer" ("id", "email", "name", "plan", "views", "updated_at") + VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT ("id") DO UPDATE + SET "email" = "excluded"."email", "name" = "excluded"."name", "plan" = "excluded"."plan", "views" = "excluded"."views", "updated_at" = "excluded"."updated_at"; + + SELECT COUNT(*) + FROM "customer" + WHERE "customer"."id" = ?; + + SELECT "customer"."email", "customer"."id", "customer"."name", "customer"."plan", "customer"."updated_at", "customer"."views" + FROM "customer" + WHERE "customer"."id" = ? + LIMIT ?; + ``` + +=== "PostgreSQL" + + ```sql + -- mutations.py:upsert (postgres) + INSERT INTO "customer" ("id", "email", "name", "plan", "views", "updated_at") + VALUES ($1, $2, $3, $4, $5, CAST($6 AS timestamptz)) + ON CONFLICT ("id") DO UPDATE + SET "email" = "excluded"."email", "name" = "excluded"."name", "plan" = "excluded"."plan", "views" = "excluded"."views", "updated_at" = "excluded"."updated_at"; + + INSERT INTO "customer" ("id", "email", "name", "plan", "views", "updated_at") + VALUES ($1, $2, $3, $4, $5, CAST($6 AS timestamptz)) + ON CONFLICT ("id") DO UPDATE + SET "email" = "excluded"."email", "name" = "excluded"."name", "plan" = "excluded"."plan", "views" = "excluded"."views", "updated_at" = "excluded"."updated_at"; + + SELECT COUNT(*) + FROM "customer" + WHERE "customer"."id" = $1; + + SELECT "customer"."email", "customer"."id", "customer"."name", "customer"."plan", "customer"."updated_at", "customer"."views" + FROM "customer" + WHERE "customer"."id" = $1 + LIMIT $2; + ``` + +Each upsert is one `INSERT ... ON CONFLICT ("id") DO UPDATE` that sets every column from the row it tried to insert, which is why `name` was reset. + The conflict target is the primary key only, so a conflict on another unique column still raises `UniqueViolationError`. Without a primary key there is nothing to conflict on, and the call inserts a new row. `upsert()` is shorthand for `instance.save(on_conflict="update")`, which does the same for an instance you already hold. ## `get_or_create()` and `update_or_create()` @@ -230,6 +609,47 @@ The conflict target is the primary key only, so a conflict on another unique col The first call created Heidi; the second found the same row. +The two calls send these statements. PostgreSQL also casts the `datetime` parameter to `timestamptz` and returns the new `id`: + +=== "SQLite" + + ```sql + -- mutations.py:get-or-create (sqlite) + SELECT "customer"."email", "customer"."id", "customer"."name", "customer"."plan", "customer"."updated_at", "customer"."views" + FROM "customer" + WHERE "customer"."email" = ? + LIMIT ?; + + INSERT INTO "customer" ("email", "name", "plan", "views", "updated_at") + VALUES (?, ?, ?, ?, ?); + + SELECT "customer"."email", "customer"."id", "customer"."name", "customer"."plan", "customer"."updated_at", "customer"."views" + FROM "customer" + WHERE "customer"."email" = ? + LIMIT ?; + ``` + +=== "PostgreSQL" + + ```sql + -- mutations.py:get-or-create (postgres) + SELECT "customer"."email", "customer"."id", "customer"."name", "customer"."plan", "customer"."updated_at", "customer"."views" + FROM "customer" + WHERE "customer"."email" = $1 + LIMIT $2; + + INSERT INTO "customer" ("email", "name", "plan", "views", "updated_at") + VALUES ($1, $2, $3, $4, CAST($5 AS timestamptz)) + RETURNING "id"; + + SELECT "customer"."email", "customer"."id", "customer"."name", "customer"."plan", "customer"."updated_at", "customer"."views" + FROM "customer" + WHERE "customer"."email" = $1 + LIMIT $2; + ``` + +The first call looked the row up, found nothing and inserted it; the second stopped after the lookup. + `update_or_create(defaults={...}, **filters)` takes the same arguments, but when a row matches it applies `defaults` to it and saves it: ```python @@ -238,6 +658,38 @@ The first call created Heidi; the second found the same row. Heidi's row now has the `"pro"` plan. +The call sends these statements. PostgreSQL also casts the `datetime` parameter to `timestamptz`: + +=== "SQLite" + + ```sql + -- mutations.py:update-or-create (sqlite) + SELECT "customer"."email", "customer"."id", "customer"."name", "customer"."plan", "customer"."updated_at", "customer"."views" + FROM "customer" + WHERE "customer"."email" = ? + LIMIT ?; + + UPDATE "customer" + SET "email" = ?, "name" = ?, "plan" = ?, "views" = ?, "updated_at" = ? + WHERE "id" = ?; + ``` + +=== "PostgreSQL" + + ```sql + -- mutations.py:update-or-create (postgres) + SELECT "customer"."email", "customer"."id", "customer"."name", "customer"."plan", "customer"."updated_at", "customer"."views" + FROM "customer" + WHERE "customer"."email" = $1 + LIMIT $2; + + UPDATE "customer" + SET "email" = $1, "name" = $2, "plan" = $3, "views" = $4, "updated_at" = CAST($5 AS timestamptz) + WHERE "id" = $6; + ``` + +It looked the row up, then saved it with an `UPDATE` by primary key. + !!! note "Not atomic" Both helpers read, then write. Two writers can both miss the row and both try to insert it; with a unique constraint on the filter columns, the second insert raises [`UniqueViolationError`](../api/exceptions.md#ferro.UniqueViolationError), so catch it and look the row up again. For an atomic insert-or-update on the primary key, use [`upsert()`](#upsert). @@ -251,6 +703,22 @@ Heidi's row now has the `"pro"` plan. The batch update changed the row but not the instance; `refresh()` brought the new name in. It raises `RuntimeError` if the instance has no primary key or its row no longer exists. +The update and the refresh send these statements; on PostgreSQL only the placeholders differ: + +```sql +-- mutations.py:refresh (sqlite) +UPDATE "customer" +SET "name" = ? +WHERE "customer"."id" = ?; + +SELECT "customer"."email", "customer"."id", "customer"."name", "customer"."plan", "customer"."updated_at", "customer"."views" +FROM "customer" +WHERE "customer"."id" = ? +LIMIT ?; +``` + +`refresh()` is a `SELECT` by primary key. + ## Database errors Every database failure raises a typed exception under `FerroError`, so you never match on driver message text. Constraint violations are subclasses of `IntegrityError` and keep the driver's details as attributes: @@ -269,6 +737,27 @@ UNIQUE constraint failed: customer.email `sqlstate` is the database's error code (`"23505"` on PostgreSQL, `"2067"` on SQLite), `constraint` names the violated constraint on PostgreSQL and is `None` on SQLite, and `driver_message` is the driver's own text, for logs. The [exceptions reference](../api/exceptions.md) lists the whole hierarchy. +The refused statement is this `INSERT`. PostgreSQL also casts the `datetime` parameter to `timestamptz` and adds `RETURNING "id"`: + +=== "SQLite" + + ```sql + -- mutations.py:handling-errors (sqlite) + INSERT INTO "customer" ("email", "name", "plan", "views", "updated_at") + VALUES (?, ?, ?, ?, ?); + ``` + +=== "PostgreSQL" + + ```sql + -- mutations.py:handling-errors (postgres) + INSERT INTO "customer" ("email", "name", "plan", "views", "updated_at") + VALUES ($1, $2, $3, $4, CAST($5 AS timestamptz)) + RETURNING "id"; + ``` + +The database rejected it, and the exception carries the database's own report. + ## See also - [Transactions](transactions.md): group several writes so they commit or roll back together diff --git a/docs/pages/guide/queries.md b/docs/pages/guide/queries.md index 068dea9e..bf3bda48 100644 --- a/docs/pages/guide/queries.md +++ b/docs/pages/guide/queries.md @@ -21,6 +21,18 @@ To savings -500 `where()` takes a lambda whose parameter stands for one row, and `order_by()` takes a lambda naming the column to sort by. Nothing reaches the database until you await `all()`. The values in a query are always sent as parameters, never pasted into the SQL. Every query on this page runs inside a session like this one; see [Sessions](connections.md#sessions) for why and how to open one. +Every example on this page is followed by the SQL it sends, captured from Ferro when the example runs. Awaiting `all()` sends this statement; on PostgreSQL only the placeholders differ, `$1` where SQLite has `?`: + +```sql +-- predicates.py:first-query (sqlite) +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" +FROM "transaction" +WHERE "transaction"."amount" <= ? +ORDER BY "transaction"."amount" ASC NULLS LAST; +``` + +The query names every column of `transaction`, the filter compares `amount` with a placeholder, and `NULLS LAST` fixes where `NULL` values sort on both databases. + ## The ledger Every example on this page queries one small ledger. An owner holds accounts, an account holds transactions, a transaction may have a category and split lines, a transfer links the two halves of a move between accounts, and tags attach to transactions many-to-many: @@ -65,6 +77,17 @@ Ada Lovelace owns Checking and Savings, Grace Hopper owns Travel, and Petty cash The three transactions with no `cleared_on` come back. `Model.where(...)` starts a query and the `await ... .all()` runs it; `Model.select()` starts a query with no filter, for when you only want to order, page or count. +The query sends this; on PostgreSQL only the placeholders differ: + +```sql +-- predicates.py:filtering (sqlite) +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" +FROM "transaction" +WHERE "transaction"."cleared_on" IS NULL; +``` + +`== None` became `IS NULL`: in SQL, `= NULL` matches no row. + Ferro checks every column name when you build the query, before anything reaches the database. A misspelled column raises `AttributeError` with the closest match and every column and relation you can use: ```python @@ -108,6 +131,36 @@ Each comparison maps to one SQL operator: `%` in a `LIKE` pattern matches any run of characters, so `"%coffee"` matches Office coffee, and the two dates match the two Lyon rows. +The two queries send these statements. PostgreSQL also casts each date parameter to `date`: + +=== "SQLite" + + ```sql + -- predicates.py:operators (sqlite) + SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" + FROM "transaction" + WHERE "transaction"."memo" LIKE ?; + + SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" + FROM "transaction" + WHERE "transaction"."posted_on" IN (?, ?); + ``` + +=== "PostgreSQL" + + ```sql + -- predicates.py:operators (postgres) + SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" + FROM "transaction" + WHERE "transaction"."memo" LIKE $1; + + SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" + FROM "transaction" + WHERE "transaction"."posted_on" IN (CAST($1 AS date), CAST($2 AS date)); + ``` + +`.like()` became `LIKE`, and `.in_()` became `IN` with one placeholder per value. + ## Combining conditions Join conditions with `&` (and) and `|` (or), or chain several `where()` calls, which are joined with and: @@ -118,6 +171,36 @@ Join conditions with `&` (and) and `|` (or), or chain several `where()` calls, w Only the two Lyon rows are uncleared and above -90; Weekly shop is uncleared but spent -95.00. The chained query keeps the debits posted on or after 10 March. +The two queries send these statements. PostgreSQL also casts the date parameter to `date`: + +=== "SQLite" + + ```sql + -- predicates.py:combining (sqlite) + SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" + FROM "transaction" + WHERE "transaction"."cleared_on" IS NULL AND "transaction"."amount" > ?; + + SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" + FROM "transaction" + WHERE "transaction"."amount" < ? AND "transaction"."posted_on" >= ?; + ``` + +=== "PostgreSQL" + + ```sql + -- predicates.py:combining (postgres) + SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" + FROM "transaction" + WHERE "transaction"."cleared_on" IS NULL AND "transaction"."amount" > $1; + + SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" + FROM "transaction" + WHERE "transaction"."amount" < $1 AND "transaction"."posted_on" >= CAST($2 AS date); + ``` + +`&` and the chained `where()` both became `AND`. + !!! warning "Parenthesize each side of `&` and `|`" In Python, `&` and `|` bind tighter than comparisons, so `transaction.amount < 0 & transaction.cleared_on == None` parses as `transaction.amount < (0 & transaction.cleared_on) == None`. Wrap each condition in parentheses: `(transaction.amount < 0) & (transaction.cleared_on == None)`. @@ -133,6 +216,17 @@ Put `~` in front of any condition to negate it: a comparison, `.like()`, `.in_() The four debits below -50 come back, minus Train to Lyon, whose memo matches `"%Lyon"`. Each `~` becomes SQL `NOT (...)` around the condition it wraps, so negated conditions combine with `&` and `|` like any other, and `~~` cancels out. +The query sends this; on PostgreSQL only the placeholders differ: + +```sql +-- predicates.py:negation (sqlite) +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" +FROM "transaction" +WHERE (NOT "transaction"."memo" LIKE ?) AND "transaction"."amount" < ?; +``` + +`~` wrapped the `LIKE` in `NOT`, kept in parentheses beside the `AND`. + ### Negation and NULL values SQL compares `NULL` to nothing: `cleared_on < '2026-03-05'` is neither true nor false for a row whose `cleared_on` is `NULL`, and `WHERE` keeps only the rows where the condition is true. `NOT` of an unknown is still unknown, so a negated comparison drops the `NULL` rows too: @@ -143,6 +237,36 @@ SQL compares `NULL` to nothing: `cleared_on < '2026-03-05'` is neither true nor Two transactions cleared before 5 March and four cleared later, which leaves out the three uncleared ones from both lists. `!=` behaves the same way: `transaction.cleared_on != date(2026, 3, 2)` also leaves out the uncleared rows. +The two queries send these statements. PostgreSQL also casts the date parameter to `date`: + +=== "SQLite" + + ```sql + -- predicates.py:three-valued (sqlite) + SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" + FROM "transaction" + WHERE "transaction"."cleared_on" < ?; + + SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" + FROM "transaction" + WHERE NOT "transaction"."cleared_on" < ?; + ``` + +=== "PostgreSQL" + + ```sql + -- predicates.py:three-valued (postgres) + SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" + FROM "transaction" + WHERE "transaction"."cleared_on" < CAST($1 AS date); + + SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" + FROM "transaction" + WHERE NOT "transaction"."cleared_on" < CAST($1 AS date); + ``` + +Neither `WHERE` mentions `NULL`, so neither keeps the uncleared rows. + To keep the `NULL` rows, ask for them with an `== None` branch: ```python @@ -151,6 +275,28 @@ To keep the `NULL` rows, ask for them with an `== None` branch: The four later rows and the three uncleared rows come back. +The query sends this. PostgreSQL also casts the date parameter to `date`: + +=== "SQLite" + + ```sql + -- predicates.py:keep-nulls (sqlite) + SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" + FROM "transaction" + WHERE (NOT "transaction"."cleared_on" < ?) OR "transaction"."cleared_on" IS NULL; + ``` + +=== "PostgreSQL" + + ```sql + -- predicates.py:keep-nulls (postgres) + SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" + FROM "transaction" + WHERE (NOT "transaction"."cleared_on" < CAST($1 AS date)) OR "transaction"."cleared_on" IS NULL; + ``` + +The `IS NULL` branch is what brings the uncleared rows back. + ## Fetching by primary key [`Model.get(pk)`][ferro.models.Model.get] loads one row by its primary key and returns the model, never `None`. A missing row raises `ModelDoesNotExist`, a `LookupError` that carries `.model` and `.pk`, so an HTTP layer can turn it into a 404. When a missing row is normal, use [`get_or_none()`][ferro.models.Model.get_or_none]: @@ -161,6 +307,28 @@ The four later rows and the three uncleared rows come back. `get(2)` returns Rent, `get(99)` raises, and `get_or_none(99)` returns `None`. Both methods also exist on `Model.using("name")` for [named connections](connections.md#named-connections). +The three calls send one statement each, the same statement; on PostgreSQL only the placeholders differ: + +```sql +-- predicates.py:get (sqlite) +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" +FROM "transaction" +WHERE "transaction"."id" = ? +LIMIT ?; + +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" +FROM "transaction" +WHERE "transaction"."id" = ? +LIMIT ?; + +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" +FROM "transaction" +WHERE "transaction"."id" = ? +LIMIT ?; +``` + +Each is a primary-key filter with a `LIMIT` of 1. `get(99)` sent its statement too; the error comes from the empty result. + ## Ordering, limit & offset [`order_by()`][ferro.query.builder.Query.order_by] takes a lambda naming the column, and an optional direction (`"asc"`, the default, or `"desc"`). `limit()` and `offset()` cut a slice out of the ordered rows: @@ -171,6 +339,26 @@ The four later rows and the three uncleared rows come back. The first three are the largest debits; `offset(3)` skips them and returns the next three. Chain `order_by()` again to break ties on a second column. +The two queries send these statements; on PostgreSQL only the placeholders differ: + +```sql +-- predicates.py:ordering-slicing (sqlite) +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" +FROM "transaction" +WHERE TRUE +ORDER BY "transaction"."amount" ASC NULLS LAST +LIMIT ?; + +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" +FROM "transaction" +WHERE TRUE +ORDER BY "transaction"."amount" ASC NULLS LAST +LIMIT ? +OFFSET ?; +``` + +`WHERE TRUE` stands in for the missing filter, and `limit()` and `offset()` became `LIMIT` and `OFFSET`. + `order_by()` also accepts a column name as a string (`order_by("amount", "desc")`). The lambda is the documented form because only a lambda can [order by a related column](#ordering-by-a-related-column). `NULL` values sort last on every database unless you say otherwise. Pass `nulls="first"` to put them first, or `nulls="native"` for each database's own default, which differs between PostgreSQL and SQLite for `"desc"`: @@ -181,6 +369,19 @@ The first three are the largest debits; `offset(3)` skips them and returns the n The three uncleared transactions lead, then the latest cleared one, Office coffee. +The query sends this; on PostgreSQL only the placeholders differ: + +```sql +-- predicates.py:nulls-ordering (sqlite) +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" +FROM "transaction" +WHERE TRUE +ORDER BY "transaction"."cleared_on" DESC NULLS FIRST, "transaction"."id" ASC NULLS LAST +LIMIT ?; +``` + +`nulls="first"` became `NULLS FIRST`; the tie-breaker on `id` keeps the default, `NULLS LAST`. + ## Paging with `after()` and `before()` [`after()`][ferro.query.builder.Query.after] starts a page right after a row you already have, in the order the query declares. Pass it the last row of the page you showed: @@ -191,6 +392,44 @@ The three uncleared transactions lead, then the latest cleared one, Office coffe The second page starts at To savings, the row after Market run. Unlike `offset()`, it does not skip or repeat rows when rows are added or removed between the two requests. +The two pages send these statements. PostgreSQL also casts the date parameters to `date`: + +=== "SQLite" + + ```sql + -- predicates.py:after-paging (sqlite) + SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" + FROM "transaction" + WHERE TRUE + ORDER BY "transaction"."posted_on" ASC NULLS LAST, "transaction"."id" ASC NULLS LAST + LIMIT ?; + + SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" + FROM "transaction" + WHERE "transaction"."posted_on" > ? OR "transaction"."posted_on" IS NULL OR ("transaction"."posted_on" = ? AND ("transaction"."id" > ? OR "transaction"."id" IS NULL)) + ORDER BY "transaction"."posted_on" ASC NULLS LAST, "transaction"."id" ASC NULLS LAST + LIMIT ?; + ``` + +=== "PostgreSQL" + + ```sql + -- predicates.py:after-paging (postgres) + SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" + FROM "transaction" + WHERE TRUE + ORDER BY "transaction"."posted_on" ASC NULLS LAST, "transaction"."id" ASC NULLS LAST + LIMIT $1; + + SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" + FROM "transaction" + WHERE "transaction"."posted_on" > CAST($1 AS date) OR "transaction"."posted_on" IS NULL OR ("transaction"."posted_on" = CAST($2 AS date) AND ("transaction"."id" > $3 OR "transaction"."id" IS NULL)) + ORDER BY "transaction"."posted_on" ASC NULLS LAST, "transaction"."id" ASC NULLS LAST + LIMIT $4; + ``` + +The second page starts with a condition on the order keys instead of an `OFFSET`: a later `posted_on`, or Market run's `posted_on` with a larger `id`. The `IS NULL` branches match the `NULLS LAST` order. + `after()` follows four rules: - The order keys must include the primary key, so every row has exactly one place in the order. Ferro does not add it for you: `after()` raises `ValueError` if `id` is missing. Here, To savings and From checking share a `posted_on`, and `id` settles which comes first. @@ -206,6 +445,32 @@ The second page starts at To savings, the row after Market run. Unlike `offset() Market run's position is its `posted_on` and its `id`, and paging after that tuple returns the same page as paging after the row. `query.after(row)` is `query.after(query.position_of(row))`. +Paging after the tuple sends the same statement as paging after the row. PostgreSQL also casts the date parameters to `date`: + +=== "SQLite" + + ```sql + -- predicates.py:position-of (sqlite) + SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" + FROM "transaction" + WHERE "transaction"."posted_on" > ? OR "transaction"."posted_on" IS NULL OR ("transaction"."posted_on" = ? AND ("transaction"."id" > ? OR "transaction"."id" IS NULL)) + ORDER BY "transaction"."posted_on" ASC NULLS LAST, "transaction"."id" ASC NULLS LAST + LIMIT ?; + ``` + +=== "PostgreSQL" + + ```sql + -- predicates.py:position-of (postgres) + SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" + FROM "transaction" + WHERE "transaction"."posted_on" > CAST($1 AS date) OR "transaction"."posted_on" IS NULL OR ("transaction"."posted_on" = CAST($2 AS date) AND ("transaction"."id" > $3 OR "transaction"."id" IS NULL)) + ORDER BY "transaction"."posted_on" ASC NULLS LAST, "transaction"."id" ASC NULLS LAST + LIMIT $4; + ``` + +`position_of()` itself sends nothing; it reads the values off the row. + [`before()`][ferro.query.builder.Query.before] pages backwards. With a limit it returns the page immediately before the bound, still in the declared order; without one it returns every earlier row: ```python @@ -214,6 +479,54 @@ Market run's position is its `posted_on` and its `id`, and paging after that tup `before(...).limit(2)` returns the two rows before To savings, and `before(...)` alone returns all three earlier rows. `first()` returns the row immediately before the bound, while `all()[0]` returns the earliest row, so use `first()` for "the previous row". +The three queries send these statements. PostgreSQL also casts the date parameters to `date`: + +=== "SQLite" + + ```sql + -- predicates.py:before-paging (sqlite) + SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" + FROM "transaction" + WHERE "transaction"."posted_on" < ? OR ("transaction"."posted_on" = ? AND "transaction"."id" < ?) + ORDER BY "transaction"."posted_on" DESC NULLS FIRST, "transaction"."id" DESC NULLS FIRST + LIMIT ?; + + SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" + FROM "transaction" + WHERE "transaction"."posted_on" < ? OR ("transaction"."posted_on" = ? AND "transaction"."id" < ?) + ORDER BY "transaction"."posted_on" DESC NULLS FIRST, "transaction"."id" DESC NULLS FIRST; + + SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" + FROM "transaction" + WHERE "transaction"."posted_on" < ? OR ("transaction"."posted_on" = ? AND "transaction"."id" < ?) + ORDER BY "transaction"."posted_on" DESC NULLS FIRST, "transaction"."id" DESC NULLS FIRST + LIMIT ?; + ``` + +=== "PostgreSQL" + + ```sql + -- predicates.py:before-paging (postgres) + SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" + FROM "transaction" + WHERE "transaction"."posted_on" < CAST($1 AS date) OR ("transaction"."posted_on" = CAST($2 AS date) AND "transaction"."id" < $3) + ORDER BY "transaction"."posted_on" DESC NULLS FIRST, "transaction"."id" DESC NULLS FIRST + LIMIT $4; + + SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" + FROM "transaction" + WHERE "transaction"."posted_on" < CAST($1 AS date) OR ("transaction"."posted_on" = CAST($2 AS date) AND "transaction"."id" < $3) + ORDER BY "transaction"."posted_on" DESC NULLS FIRST, "transaction"."id" DESC NULLS FIRST; + + SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" + FROM "transaction" + WHERE "transaction"."posted_on" < CAST($1 AS date) OR ("transaction"."posted_on" = CAST($2 AS date) AND "transaction"."id" < $3) + ORDER BY "transaction"."posted_on" DESC NULLS FIRST, "transaction"."id" DESC NULLS FIRST + LIMIT $4; + ``` + +`before()` flips each comparison and reverses each sort key with its `NULLS` placement, so the `LIMIT` keeps the rows nearest the bound; Ferro puts them back in the declared order. + A `None` order-key value is how paging continues through `NULL`s. Ordered by `cleared_on` descending, the uncleared rows sort last; to continue after Train to Lyon, pass `None` for its `cleared_on`: ```python @@ -222,6 +535,18 @@ A `None` order-key value is how paging continues through `NULL`s. Ordered by `cl The two uncleared rows after Train to Lyon come back. The same methods page a query ordered by a [related column](#ordering-by-a-related-column) or a [projected query](#composing-projections). [Pagination](../howto/pagination.md) shows how to wire this into an API. +The query sends this; on PostgreSQL only the placeholders differ: + +```sql +-- predicates.py:after-null-paging (sqlite) +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" +FROM "transaction" +WHERE "transaction"."cleared_on" IS NULL AND ("transaction"."id" > ? OR "transaction"."id" IS NULL) +ORDER BY "transaction"."cleared_on" DESC NULLS LAST, "transaction"."id" ASC NULLS LAST; +``` + +With `None` for `cleared_on`, the condition stays among the `NULL` rows and continues by `id`. + ## Executing queries A query runs when you await one of its terminals: [`all()`][ferro.query.builder.Query.all] returns a list of instances, [`first()`][ferro.query.builder.Query.first] returns the first row or `None`, [`count()`][ferro.query.builder.Query.count] returns how many rows match, and [`exists()`][ferro.query.builder.Query.exists] returns whether any does: @@ -232,8 +557,29 @@ A query runs when you await one of its terminals: [`all()`][ferro.query.builder. `count()` and `exists()` hydrate no instances. `Model.all()` is short for `Model.select().all()`. -!!! tip "Prefer `exists()` to `count() > 0`" - `exists()` lets the database stop at the first match instead of counting every row. +The four calls send these statements; on PostgreSQL only the placeholders differ: + +```sql +-- predicates.py:terminals (sqlite) +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" +FROM "transaction"; + +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" +FROM "transaction" +WHERE TRUE +ORDER BY "transaction"."posted_on" DESC NULLS LAST +LIMIT ?; + +SELECT COUNT(*) +FROM "transaction" +WHERE "transaction"."cleared_on" IS NULL; + +SELECT COUNT(*) +FROM "transaction" +WHERE "transaction"."memo" LIKE ?; +``` + +`count()` and `exists()` both send `SELECT COUNT(*)`; `exists()` compares the count with zero. ## Query immutability @@ -245,6 +591,31 @@ A query runs when you await one of its terminals: [`all()`][ferro.query.builder. The three queries built from `debits` are independent, and `debits` still counts all seven debits afterwards. +The four calls send these statements; on PostgreSQL only the placeholders differ: + +```sql +-- predicates.py:immutability (sqlite) +SELECT COUNT(*) +FROM "transaction" +WHERE "transaction"."amount" < ?; + +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" +FROM "transaction" +WHERE "transaction"."amount" < ? +ORDER BY "transaction"."amount" ASC NULLS LAST +LIMIT ?; + +SELECT COUNT(*) +FROM "transaction" +WHERE "transaction"."amount" < ? AND "transaction"."cleared_on" IS NULL; + +SELECT COUNT(*) +FROM "transaction" +WHERE "transaction"."amount" < ?; +``` + +Every statement carries the `amount < ?` filter of `debits`; only the branches add to it. + ## Querying across relationships A `where()` or `order_by()` lambda can follow a foreign key to the related model's columns. Write the relation, then the column: @@ -253,7 +624,17 @@ A `where()` or `order_by()` lambda can follow a foreign key to the related model --8<-- "docs/examples/traversal.py:basic" ``` -`transaction.account.label` reads the label of each transaction's account, and From checking is the one transaction on Savings. You do not write the join: following `transaction.account` is the join, and the query still returns `Transaction` instances. These examples run inside the session from [the first block](#queries). +`transaction.account.label` reads the label of each transaction's account, and From checking is the one transaction on Savings. You do not write the join: following `transaction.account` is the join, and the query still returns `Transaction` instances. These examples run inside the session from [the first block](#queries). The query sends this; on PostgreSQL only the placeholders differ: + +```sql +-- traversal.py:basic (sqlite) +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" +FROM "transaction" +INNER JOIN "account" AS "j1_account" ON "transaction"."account_id" = "j1_account"."id" +WHERE "j1_account"."label" = ?; +``` + +Following `transaction.account` became one `INNER JOIN` on `account`, under the alias `j1_account`. ### Traversing at any depth @@ -263,7 +644,18 @@ Each step resolves against the next model, so you can chain as far as the foreig --8<-- "docs/examples/traversal.py:multi-hop" ``` -Grace Hopper owns Travel, so both Travel transactions come back. Ferro checks each step against its own model when you build the query, and a misspelled step names that model and the closest match: +Grace Hopper owns Travel, so both Travel transactions come back. The query sends this; on PostgreSQL only the placeholders differ: + +```sql +-- traversal.py:multi-hop (sqlite) +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" +FROM "transaction" +INNER JOIN "account" AS "j1_account" ON "transaction"."account_id" = "j1_account"."id" +INNER JOIN "owner" AS "j2_owner" ON "j1_account"."owner_id" = "j2_owner"."id" +WHERE "j2_owner"."email" = ?; +``` + +Each step is one more join: `j2_owner` joins on the account's `owner_id`. Ferro checks each step against its own model when you build the query, and a misspelled step names that model and the closest match: ```python --8<-- "docs/examples/traversal.py:typo" @@ -281,6 +673,19 @@ Every step is an inner join, whether or not the foreign key is nullable. A row w Only Grace's two transactions come back. Office coffee is on Petty cash, which has no owner; it has no owner name to compare, so the join drops it. The join type never depends on nullability, so making a foreign key nullable later never changes what an existing query returns. To keep the rows without a relation, use [`left_join()`](#keeping-rows-that-have-no-relation). +The query sends this; on PostgreSQL only the placeholders differ: + +```sql +-- traversal.py:inner-narrows (sqlite) +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" +FROM "transaction" +INNER JOIN "account" AS "j1_account" ON "transaction"."account_id" = "j1_account"."id" +INNER JOIN "owner" AS "j2_owner" ON "j1_account"."owner_id" = "j2_owner"."id" +WHERE "j2_owner"."name" <> ?; +``` + +The `INNER JOIN` on `owner` drops Office coffee: Petty cash has no `owner_id` to join on. + The join applies to the whole query, not to one side of an `|`: ```python @@ -289,6 +694,19 @@ The join applies to the whole query, not to one side of an `|`: Office coffee matches the second branch, but the join on `account.owner` has already dropped it. +The query sends this; on PostgreSQL only the placeholders differ: + +```sql +-- traversal.py:inner-or (sqlite) +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" +FROM "transaction" +INNER JOIN "account" AS "j1_account" ON "transaction"."account_id" = "j1_account"."id" +INNER JOIN "owner" AS "j2_owner" ON "j1_account"."owner_id" = "j2_owner"."id" +WHERE "j2_owner"."name" = ? OR "transaction"."memo" = ?; +``` + +The `OR` sits in the `WHERE`, after both joins. + `count()` counts the same rows. Following a foreign key reaches one related row per row, so the join never multiplies results: ```python @@ -297,6 +715,19 @@ Office coffee matches the second branch, but the join on `account.owner` has alr Ada's two accounts hold six transactions. +`count()` sends this; on PostgreSQL only the placeholders differ: + +```sql +-- traversal.py:count (sqlite) +SELECT COUNT(*) +FROM "transaction" +INNER JOIN "account" AS "j1_account" ON "transaction"."account_id" = "j1_account"."id" +INNER JOIN "owner" AS "j2_owner" ON "j1_account"."owner_id" = "j2_owner"."id" +WHERE "j2_owner"."name" = ?; +``` + +`count()` counts over the same two joins. + ### Ordering by a related column `order_by()` follows relations the same way: @@ -305,13 +736,37 @@ Ada's two accounts hold six transactions. --8<-- "docs/examples/traversal.py:order-by" ``` -Accounts sort by label, descending: Travel, Savings, Petty cash, then Checking after the limit. Sorting is an inner join too, so ordering by a column that some rows cannot reach drops those rows: +Accounts sort by label, descending: Travel, Savings, Petty cash, then Checking after the limit. The query sends this; on PostgreSQL only the placeholders differ: + +```sql +-- traversal.py:order-by (sqlite) +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" +FROM "transaction" +INNER JOIN "account" AS "j1_account" ON "transaction"."account_id" = "j1_account"."id" +WHERE TRUE +ORDER BY "j1_account"."label" DESC NULLS LAST, "transaction"."id" ASC NULLS LAST +LIMIT ?; +``` + +The sort reads the joined `label`, then `id` breaks ties. Sorting is an inner join too, so ordering by a column that some rows cannot reach drops those rows: ```python --8<-- "docs/examples/traversal.py:order-by-narrows" ``` -Eight of the nine transactions come back. `after()` and `before()` take a tuple of the related values, so you can page without loading the relation: +Eight of the nine transactions come back. The query sends this; on PostgreSQL only the placeholders differ: + +```sql +-- traversal.py:order-by-narrows (sqlite) +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" +FROM "transaction" +INNER JOIN "account" AS "j1_account" ON "transaction"."account_id" = "j1_account"."id" +INNER JOIN "owner" AS "j2_owner" ON "j1_account"."owner_id" = "j2_owner"."id" +WHERE TRUE +ORDER BY "j2_owner"."name" ASC NULLS LAST, "transaction"."id" ASC NULLS LAST; +``` + +Office coffee drops out at the `owner` join. `after()` and `before()` take a tuple of the related values, so you can page without loading the relation: ```python --8<-- "docs/examples/traversal.py:traversed-paging" @@ -319,6 +774,20 @@ Eight of the nine transactions come back. `after()` and `before()` take a tuple The page starts after Office coffee, the last Petty cash row. `position_of()` and `after(row)` read the related value off the instance, so they need the relation [populated with `include()`](#populating-relations-with-include); a tuple does not. +The query sends this; on PostgreSQL only the placeholders differ: + +```sql +-- traversal.py:traversed-paging (sqlite) +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" +FROM "transaction" +INNER JOIN "account" AS "j1_account" ON "transaction"."account_id" = "j1_account"."id" +WHERE "j1_account"."label" > ? OR "j1_account"."label" IS NULL OR ("j1_account"."label" = ? AND ("transaction"."id" > ? OR "transaction"."id" IS NULL)) +ORDER BY "j1_account"."label" ASC NULLS LAST, "transaction"."id" ASC NULLS LAST +LIMIT ?; +``` + +The position compares the joined `label` and then `id`, as a position on the model's own columns would. + ### One join per relation path The path decides the join. Use `transaction.account` in two `where()` calls, in both sides of an `|`, or in `where()` and `order_by()`, and the query still joins `account` once. You never name or manage an alias: @@ -329,6 +798,21 @@ The path decides the join. Use `transaction.account` in two `where()` calls, in The traversal filter, a plain column filter, an ordering and a limit run as one statement, and Ada's two largest debits come back. +The query sends this; on PostgreSQL only the placeholders differ: + +```sql +-- traversal.py:composed (sqlite) +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" +FROM "transaction" +INNER JOIN "account" AS "j1_account" ON "transaction"."account_id" = "j1_account"."id" +INNER JOIN "owner" AS "j2_owner" ON "j1_account"."owner_id" = "j2_owner"."id" +WHERE "j2_owner"."name" = ? AND "transaction"."amount" < ? +ORDER BY "transaction"."amount" ASC NULLS LAST +LIMIT ?; +``` + +`account` and `owner` are joined once each; the plain filter and the ordering read `transaction` itself. + ### Comparing to a related instance Each foreign key adds a `_id` column, and filtering on it needs no join: @@ -337,19 +821,61 @@ Each foreign key adds a `_id` column, and filtering on it needs no joi --8<-- "docs/examples/traversal.py:shadow-column" ``` -From checking is the one transaction whose `account_id` is Savings' `id`. Comparing the relation to a saved instance means the same thing, with the same join-free query: +From checking is the one transaction whose `account_id` is Savings' `id`. Fetching Savings and the query send these statements; on PostgreSQL only the placeholders differ: + +```sql +-- traversal.py:shadow-column (sqlite) +SELECT "account"."id", "account"."label", "account"."owner_id" +FROM "account" +WHERE "account"."id" = ? +LIMIT ?; + +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" +FROM "transaction" +WHERE "transaction"."account_id" = ?; +``` + +The filter reads `account_id` on `transaction`, with no join. Comparing the relation to a saved instance means the same thing, with the same join-free query: ```python --8<-- "docs/examples/traversal.py:instance-eq" ``` -`transaction.account == savings` is `transaction.account_id == savings.id`; `!=` returns the other eight. At the end of a longer path, the comparison uses the last step's foreign key column: +`transaction.account == savings` is `transaction.account_id == savings.id`; `!=` returns the other eight. The two queries send these statements; on PostgreSQL only the placeholders differ: + +```sql +-- traversal.py:instance-eq (sqlite) +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" +FROM "transaction" +WHERE "transaction"."account_id" = ?; + +SELECT COUNT(*) +FROM "transaction" +WHERE "transaction"."account_id" <> ?; +``` + +Both compare `account_id`, the second with `<>`. At the end of a longer path, the comparison uses the last step's foreign key column: ```python --8<-- "docs/examples/traversal.py:deep-instance-eq" ``` -`transaction.account.owner == grace` joins `account` and compares its `owner_id`. An instance that has not been saved has no primary key to compare, so Ferro refuses it when you build the query: +`transaction.account.owner == grace` joins `account` and compares its `owner_id`. Fetching Grace and the query send these statements; on PostgreSQL only the placeholders differ: + +```sql +-- traversal.py:deep-instance-eq (sqlite) +SELECT "owner"."email", "owner"."id", "owner"."name" +FROM "owner" +WHERE "owner"."id" = ? +LIMIT ?; + +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" +FROM "transaction" +INNER JOIN "account" AS "j1_account" ON "transaction"."account_id" = "j1_account"."id" +WHERE "j1_account"."owner_id" = ?; +``` + +The first statement fetched Grace; the query joins `account` and never `owner`. An instance that has not been saved has no primary key to compare, so Ferro refuses it when you build the query: ```python --8<-- "docs/examples/traversal.py:unpersisted" @@ -365,7 +891,17 @@ A bare `join()` with no condition keeps the rows whose relation exists: --8<-- "docs/examples/traversal.py:existence" ``` -Petty cash has no owner and is left out. To ask the opposite, compare the relation to `None`: `== None` and `!= None` become `IS NULL` and `IS NOT NULL` on the foreign key column, with no join: +Petty cash has no owner and is left out. The query sends this; on PostgreSQL only the placeholders differ: + +```sql +-- traversal.py:existence (sqlite) +SELECT "account"."id", "account"."label", "account"."owner_id" +FROM "account" +INNER JOIN "owner" AS "j1_owner" ON "account"."owner_id" = "j1_owner"."id" +WHERE TRUE; +``` + +The bare `join()` is an `INNER JOIN` with nothing added to the `WHERE`. To ask the opposite, compare the relation to `None`: `== None` and `!= None` become `IS NULL` and `IS NOT NULL` on the foreign key column, with no join: ```python --8<-- "docs/examples/traversal.py:is-null" @@ -373,6 +909,21 @@ Petty cash has no owner and is left out. To ask the opposite, compare the relati Both forms ask about a foreign key on the model you query. To ask whether a row has related rows on another table, such as "has at least one split line", use an [existence test](#existence-tests-on-reverse-many-to-many-relations). +The two queries send these statements; on PostgreSQL only the placeholders differ: + +```sql +-- traversal.py:is-null (sqlite) +SELECT "account"."id", "account"."label", "account"."owner_id" +FROM "account" +WHERE "account"."owner_id" IS NULL; + +SELECT COUNT(*) +FROM "account" +WHERE "account"."owner_id" IS NOT NULL; +``` + +Neither statement joins `owner`. + ### Keeping rows that have no relation [`left_join()`][ferro.query.builder.Query.left_join] keeps the rows whose relation is missing. It makes every step of its path a left join, and a `where()` or `order_by()` on the same path uses it too: @@ -383,6 +934,19 @@ Both forms ask about a foreign key on the model you query. To ask whether a row All four accounts come back, Petty cash last because its owner name is `NULL`. Without `left_join()`, ordering by `account.owner.name` would drop it. Declaring `join()` and `left_join()` on the same path raises `ValueError`. +The query sends this; on PostgreSQL only the placeholders differ: + +```sql +-- traversal.py:left-join (sqlite) +SELECT "account"."id", "account"."label", "account"."owner_id" +FROM "account" +LEFT JOIN "owner" AS "j1_owner" ON "account"."owner_id" = "j1_owner"."id" +WHERE TRUE +ORDER BY "j1_owner"."name" ASC NULLS LAST, "account"."id" ASC NULLS LAST; +``` + +The `LEFT JOIN` keeps Petty cash with `NULL` in every `j1_owner` column, and `NULLS LAST` sorts it to the end. + ### Two foreign keys to the same table Two foreign keys to one model are two paths, so they are two joins: @@ -393,6 +957,21 @@ Two foreign keys to one model are two paths, so they are two joins: `transfer.outflow` and `transfer.inflow` both point at `Transaction`, and each reaches its own account. The one transfer runs from transaction 4 on Checking to transaction 5 on Savings. +The query sends this; on PostgreSQL only the placeholders differ: + +```sql +-- traversal.py:two-fk-query (sqlite) +SELECT "transfer"."id", "transfer"."inflow_id", "transfer"."outflow_id" +FROM "transfer" +INNER JOIN "transaction" AS "j1_outflow" ON "transfer"."outflow_id" = "j1_outflow"."id" +INNER JOIN "account" AS "j2_account" ON "j1_outflow"."account_id" = "j2_account"."id" +INNER JOIN "transaction" AS "j3_inflow" ON "transfer"."inflow_id" = "j3_inflow"."id" +INNER JOIN "account" AS "j4_account" ON "j3_inflow"."account_id" = "j4_account"."id" +WHERE "j2_account"."label" = ? AND "j4_account"."label" = ?; +``` + +Each path has its own aliases, `j1_outflow` and `j3_inflow`, each with its own join on `account`. + ### Self-referential traversal A foreign key to the same model traverses like any other: @@ -403,6 +982,18 @@ A foreign key to the same model traverses like any other: Groceries and Dining have Food as their parent. +The query sends this; on PostgreSQL only the placeholders differ: + +```sql +-- traversal.py:self-fk-query (sqlite) +SELECT "category"."id", "category"."name", "category"."parent_id" +FROM "category" +INNER JOIN "category" AS "j1_parent" ON "category"."parent_id" = "j1_parent"."id" +WHERE "j1_parent"."name" = ?; +``` + +The `category` table joins itself under the alias `j1_parent`. + ### Traversing from a many-to-many A many-to-many relation on an instance is a query, and its lambda can follow foreign keys on the other model: @@ -413,6 +1004,24 @@ A many-to-many relation on an instance is a query, and its lambda can follow for Of the two reimbursable transactions, only Dinner in Lyon is in Dining. +Fetching the tag and the query send these statements; on PostgreSQL only the placeholders differ: + +```sql +-- traversal.py:m2m-query (sqlite) +SELECT "tag"."id", "tag"."name" +FROM "tag" +WHERE "tag"."name" = ? +LIMIT ?; + +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" +FROM "transaction" +INNER JOIN "tag_transactions" ON "transaction"."id" = "tag_transactions"."transaction_id" +INNER JOIN "category" AS "j1_category" ON "transaction"."category_id" = "j1_category"."id" +WHERE "tag_transactions"."tag_id" = ? AND "j1_category"."name" = ?; +``` + +The many-to-many query joins the link table, `tag_transactions`, and `transaction.category` adds a join on `category`. + ### Filtering reverse relations A reverse relation on an instance (`account.transactions`) is a query too. Filter, order and count it before you await it: @@ -423,6 +1032,28 @@ A reverse relation on an instance (`account.transactions`) is a query too. Filte Checking holds five transactions, and its two largest debits are Rent and To savings. That query starts from one account. To filter accounts by what their transactions hold, use an [existence test](#existence-tests-on-reverse-many-to-many-relations). +Fetching Checking and the two queries send these statements; on PostgreSQL only the placeholders differ: + +```sql +-- traversal.py:reverse-filter (sqlite) +SELECT "account"."id", "account"."label", "account"."owner_id" +FROM "account" +WHERE "account"."id" = ? +LIMIT ?; + +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" +FROM "transaction" +WHERE "transaction"."account_id" = ? AND "transaction"."amount" < ? +ORDER BY "transaction"."amount" ASC NULLS LAST +LIMIT ?; + +SELECT COUNT(*) +FROM "transaction" +WHERE "transaction"."account_id" = ?; +``` + +The reverse relation became a filter on `account_id`. + ### Result type of traversed queries A traversed query returns the model you queried: filtering `Transaction` through `transaction.account.owner` still returns `Transaction` instances, and it does not load the accounts. `await transaction.account` still runs its own query. To load related rows with the results, use [`include()`](#populating-relations-with-include). @@ -443,6 +1074,38 @@ The message names the fix. Fetch the primary keys through the join first, then u Both Travel transactions are marked cleared. A join-free relation filter, such as `transaction.account == travel` or `transaction.category == None`, is allowed on `update()` and `delete()` as it is. +The fetch and the update send these statements. PostgreSQL also casts the date parameter to `date`: + +=== "SQLite" + + ```sql + -- traversal.py:mutate-limitation (sqlite) + SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" + FROM "transaction" + INNER JOIN "account" AS "j1_account" ON "transaction"."account_id" = "j1_account"."id" + WHERE "j1_account"."label" = ?; + + UPDATE "transaction" + SET "cleared_on" = ? + WHERE "transaction"."id" IN (?, ?); + ``` + +=== "PostgreSQL" + + ```sql + -- traversal.py:mutate-limitation (postgres) + SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" + FROM "transaction" + INNER JOIN "account" AS "j1_account" ON "transaction"."account_id" = "j1_account"."id" + WHERE "j1_account"."label" = $1; + + UPDATE "transaction" + SET "cleared_on" = CAST($1 AS date) + WHERE "transaction"."id" IN ($2, $3); + ``` + +The join stays in the `SELECT`; the `UPDATE` filters on the fetched keys alone. + ### Unsupported predicate forms A lambda that returns a bare relation (`lambda transaction: transaction.account`) filters nothing, so it raises `TypeError` and names the forms that do: `== None`, `== instance` or a column comparison. Python's `and` and `or` raise `TypeError` too: @@ -463,6 +1126,17 @@ A foreign key points from the row you query to one related row. A reverse relati Weekly shop is the only transaction with split lines. `.exists()` becomes a correlated `EXISTS` subquery, not a join, so each matching row comes back once however many related rows it has. It works the same for a one-to-one reverse relation, a one-to-many one and a many-to-many one, so changing a relation's cardinality never breaks a query. These examples run inside the session from [the first block](#queries). +The query sends this; on PostgreSQL only the placeholders differ: + +```sql +-- existence_tests.py:first (sqlite) +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" +FROM "transaction" +WHERE EXISTS(SELECT ? FROM "splitline" AS "x1_lines" WHERE "x1_lines"."transaction_id" = "transaction"."id"); +``` + +The `EXISTS` subquery over `splitline` refers back to the outer `transaction` row. What it selects does not matter to `EXISTS`, so it selects a bound constant. + ### Bare tests and negation A bare `.exists()` asks whether any related row is there, and `~` asks whether none is: @@ -473,6 +1147,21 @@ A bare `.exists()` asks whether any related row is there, and `~` asks whether n `transfer_out` and `transfer_in` are the one-to-one reverse relations of `Transfer.outflow` and `Transfer.inflow`. The two halves of the transfer come back, and the other seven rows are in no transfer. +The two queries send these statements; on PostgreSQL only the placeholders differ: + +```sql +-- existence_tests.py:bare (sqlite) +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" +FROM "transaction" +WHERE EXISTS(SELECT ? FROM "transfer" AS "x1_transfer_out" WHERE "x1_transfer_out"."outflow_id" = "transaction"."id") OR EXISTS(SELECT ? FROM "transfer" AS "x2_transfer_in" WHERE "x2_transfer_in"."inflow_id" = "transaction"."id"); + +SELECT COUNT(*) +FROM "transaction" +WHERE (NOT EXISTS(SELECT ? FROM "transfer" AS "x1_transfer_out" WHERE "x1_transfer_out"."outflow_id" = "transaction"."id")) AND (NOT EXISTS(SELECT ? FROM "transfer" AS "x2_transfer_in" WHERE "x2_transfer_in"."inflow_id" = "transaction"."id")); +``` + +`~` became `NOT EXISTS`, and each relation has its own subquery. + ### Scoping with an inner predicate Pass `.exists()` a lambda to choose which related rows count. The inner lambda is a full predicate over the related model, with every operator, `&`, `|`, `~`, traversal and further existence tests: @@ -481,7 +1170,16 @@ Pass `.exists()` a lambda to choose which related rows count. The inner lambda i --8<-- "docs/examples/existence_tests.py:scoped" ``` -Market run has Groceries itself, and Weekly shop has it on a split line. The result has one row per transaction, so ordering and paging work as on any query: +Market run has Groceries itself, and Weekly shop has it on a split line. The query sends this; on PostgreSQL only the placeholders differ: + +```sql +-- existence_tests.py:scoped (sqlite) +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" +FROM "transaction" +WHERE "transaction"."category_id" = ? OR EXISTS(SELECT ? FROM "splitline" AS "x1_lines" WHERE "x1_lines"."transaction_id" = "transaction"."id" AND "x1_lines"."category_id" = ?); +``` + +The inner lambda's condition joined the subquery's `WHERE`. The result has one row per transaction, so ordering and paging work as on any query: ```python --8<-- "docs/examples/existence_tests.py:composes" @@ -489,6 +1187,19 @@ Market run has Groceries itself, and Weekly shop has it on a split line. The res Weekly shop, at -95.00, is the larger of the two. +The query sends this; on PostgreSQL only the placeholders differ: + +```sql +-- existence_tests.py:composes (sqlite) +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" +FROM "transaction" +WHERE "transaction"."category_id" = ? OR EXISTS(SELECT ? FROM "splitline" AS "x1_lines" WHERE "x1_lines"."transaction_id" = "transaction"."id" AND "x1_lines"."category_id" = ?) +ORDER BY "transaction"."amount" ASC NULLS LAST +LIMIT ?; +``` + +`ORDER BY` and `LIMIT` apply to the outer query. + ### Grouping conditions Two conditions on the same relation can ask two different questions: is there one related row that matches both, or a row for each? Where you put the conditions decides which: @@ -499,6 +1210,21 @@ Two conditions on the same relation can ask two different questions: is there on No single split line is both Dining and below -50, but Weekly shop has a Dining line and a different line below -50. Writing `transaction.lines.category_id` directly raises an error, because nothing on the page would say which of the two questions it asks. +The two queries send these statements; on PostgreSQL only the placeholders differ: + +```sql +-- existence_tests.py:grouping (sqlite) +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" +FROM "transaction" +WHERE EXISTS(SELECT ? FROM "splitline" AS "x1_lines" WHERE "x1_lines"."transaction_id" = "transaction"."id" AND ("x1_lines"."category_id" = ? AND "x1_lines"."amount" < ?)); + +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" +FROM "transaction" +WHERE EXISTS(SELECT ? FROM "splitline" AS "x1_lines" WHERE "x1_lines"."transaction_id" = "transaction"."id" AND "x1_lines"."category_id" = ?) AND EXISTS(SELECT ? FROM "splitline" AS "x2_lines" WHERE "x2_lines"."transaction_id" = "transaction"."id" AND "x2_lines"."amount" < ?); +``` + +One subquery holding both conditions asks for one line that matches both; two subqueries ask for a line for each. + ### Traversal inside the test, and nesting The inner lambda can follow foreign keys on the related model. Those joins run inside the `EXISTS` subquery, with the same [inner-join rule](#join-type-for-traversed-relations), so they filter the related rows and never the outer ones: @@ -507,7 +1233,16 @@ The inner lambda can follow foreign keys on the related model. Those joins run i --8<-- "docs/examples/existence_tests.py:traversal-inside" ``` -Weekly shop's lines are in Groceries and Dining, both under Food. Existence tests nest to any depth: +Weekly shop's lines are in Groceries and Dining, both under Food. The query sends this; on PostgreSQL only the placeholders differ: + +```sql +-- existence_tests.py:traversal-inside (sqlite) +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" +FROM "transaction" +WHERE EXISTS(SELECT ? FROM "splitline" AS "x1_lines" INNER JOIN "category" AS "x2_category" ON "x1_lines"."category_id" = "x2_category"."id" INNER JOIN "category" AS "x3_parent" ON "x2_category"."parent_id" = "x3_parent"."id" WHERE "x1_lines"."transaction_id" = "transaction"."id" AND "x3_parent"."name" = ?); +``` + +Both `category` joins sit inside the subquery. Existence tests nest to any depth: ```python --8<-- "docs/examples/existence_tests.py:nested" @@ -515,6 +1250,17 @@ Weekly shop's lines are in Groceries and Dining, both under Food. Existence test Only Ada has an account holding a split transaction. +The query sends this; on PostgreSQL only the placeholders differ: + +```sql +-- existence_tests.py:nested (sqlite) +SELECT "owner"."email", "owner"."id", "owner"."name" +FROM "owner" +WHERE EXISTS(SELECT ? FROM "account" AS "x1_accounts" WHERE "x1_accounts"."owner_id" = "owner"."id" AND EXISTS(SELECT ? FROM "transaction" AS "x2_transactions" WHERE "x2_transactions"."account_id" = "x1_accounts"."id" AND EXISTS(SELECT ? FROM "splitline" AS "x3_lines" WHERE "x3_lines"."transaction_id" = "x2_transactions"."id"))); +``` + +Each level is an `EXISTS` inside the one before, tied to its parent's row. + ### Many-to-many A many-to-many relation reads the same from either side. The test goes through the link table, and the inner lambda scopes over the other model: @@ -525,6 +1271,25 @@ A many-to-many relation reads the same from either side. The test goes through t Rent carries `tax`, `reimbursable` is on the uncleared Lyon rows, and six transactions have no tag. +The three queries send these statements; on PostgreSQL only the placeholders differ: + +```sql +-- existence_tests.py:m2m (sqlite) +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" +FROM "transaction" +WHERE EXISTS(SELECT ? FROM "tag_transactions" AS "x1_tags" INNER JOIN "tag" AS "x2_tags" ON "x1_tags"."tag_id" = "x2_tags"."id" WHERE "x1_tags"."transaction_id" = "transaction"."id" AND "x2_tags"."name" = ?); + +SELECT "tag"."id", "tag"."name" +FROM "tag" +WHERE EXISTS(SELECT ? FROM "tag_transactions" AS "x1_transactions" INNER JOIN "transaction" AS "x2_transactions" ON "x1_transactions"."transaction_id" = "x2_transactions"."id" WHERE "x1_transactions"."tag_id" = "tag"."id" AND "x2_transactions"."cleared_on" IS NULL); + +SELECT COUNT(*) +FROM "transaction" +WHERE NOT EXISTS(SELECT ? FROM "tag_transactions" AS "x1_tags" INNER JOIN "tag" AS "x2_tags" ON "x1_tags"."tag_id" = "x2_tags"."id" WHERE "x1_tags"."transaction_id" = "transaction"."id"); +``` + +Each test reaches the other model through the link table, `tag_transactions`, from either side. + ### Unsupported reverse-relation forms A reverse or many-to-many relation is tested, never traversed. Every other use in a query fails when you build it, and the message names the `.exists()` form to use: @@ -564,6 +1329,20 @@ Dinner in Lyon | Travel The query still returns `list[Transaction]`, and `transaction.account` is now the `Account` instance itself, as the field's annotation (`account: Account`) says. Reading it needs no `await` and runs no query. These examples run inside the session from [the first block](#queries). +The query sends this; on PostgreSQL only the placeholders differ: + +```sql +-- populated_relations.py:basic (sqlite) +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on", "j1_account"."id" AS "j1_account__id", "j1_account"."label" AS "j1_account__label", "j1_account"."owner_id" AS "j1_account__owner_id" +FROM "transaction" +LEFT JOIN "account" AS "j1_account" ON "transaction"."account_id" = "j1_account"."id" +WHERE TRUE +ORDER BY "transaction"."id" DESC NULLS LAST +LIMIT ?; +``` + +The account's columns come back in the same rows, through a `LEFT JOIN`, under `j1_account__` names. + ### The population contract A relation on an instance is either populated or not: @@ -577,6 +1356,23 @@ A relation on an instance is either populated or not: `await transaction.account` fetched the Travel account with a second query. +The fetch and the await send these statements; on PostgreSQL only the placeholders differ: + +```sql +-- populated_relations.py:awaitable (sqlite) +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" +FROM "transaction" +WHERE "transaction"."id" = ? +LIMIT ?; + +SELECT "account"."id", "account"."label", "account"."owner_id" +FROM "account" +WHERE "account"."id" = ? +LIMIT ?; +``` + +The second statement is the one `await transaction.account` ran. + !!! warning "Awaiting a populated relation raises" On a populated instance, `await transaction.account` raises `TypeError: 'Account' object can't be awaited`, because the attribute is the instance, not a coroutine. Decide at the query: if you will read the relation, include it. @@ -590,13 +1386,33 @@ There is no separate "loaded" model type: an included query returns `list[Transa --8<-- "docs/examples/populated_relations.py:membership" ``` -All nine transactions come back, and Salary's `category` is `None`. A relation only `include()` uses becomes a left join; a relation a `where()` or `order_by()` also follows keeps that clause's inner join, so the filter still decides membership: +All nine transactions come back, and Salary's `category` is `None`. The query sends this; on PostgreSQL only the placeholders differ: + +```sql +-- populated_relations.py:membership (sqlite) +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on", "j1_category"."id" AS "j1_category__id", "j1_category"."name" AS "j1_category__name", "j1_category"."parent_id" AS "j1_category__parent_id" +FROM "transaction" +LEFT JOIN "category" AS "j1_category" ON "transaction"."category_id" = "j1_category"."id" +WHERE TRUE; +``` + +A relation only `include()` uses becomes a `LEFT JOIN`, which keeps Salary. A relation a `where()` or `order_by()` also follows keeps that clause's inner join, so the filter still decides membership: ```python --8<-- "docs/examples/populated_relations.py:interplay" ``` -The `where()` keeps the two Travel rows, and `include()` attaches their account. `count()` and `exists()` measure the same rows with or without `include()`: +The `where()` keeps the two Travel rows, and `include()` attaches their account. The query sends this; on PostgreSQL only the placeholders differ: + +```sql +-- populated_relations.py:interplay (sqlite) +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on", "j1_account"."id" AS "j1_account__id", "j1_account"."label" AS "j1_account__label", "j1_account"."owner_id" AS "j1_account__owner_id" +FROM "transaction" +INNER JOIN "account" AS "j1_account" ON "transaction"."account_id" = "j1_account"."id" +WHERE "j1_account"."label" = ?; +``` + +The filter and the population share one `INNER JOIN`. `count()` and `exists()` measure the same rows with or without `include()`: ```python --8<-- "docs/examples/populated_relations.py:count" @@ -604,6 +1420,21 @@ The `where()` keeps the two Travel rows, and `include()` attaches their account. Both count the seven debits. +The two calls send the same statement; on PostgreSQL only the placeholders differ: + +```sql +-- populated_relations.py:count (sqlite) +SELECT COUNT(*) +FROM "transaction" +WHERE "transaction"."amount" < ?; + +SELECT COUNT(*) +FROM "transaction" +WHERE "transaction"."amount" < ?; +``` + +`include()` adds nothing to a count. + ### Multi-hop paths Including a path populates every step of it. A `NULL` along the way ends the chain with `None`, and the row stays: @@ -614,6 +1445,20 @@ Including a path populates every step of it. A `NULL` along the way ends the cha Salary's account and its owner are both loaded. Office coffee's account is Petty cash, whose owner is `None`. +The query sends this; on PostgreSQL only the placeholders differ: + +```sql +-- populated_relations.py:multi-hop (sqlite) +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on", "j1_account"."id" AS "j1_account__id", "j1_account"."label" AS "j1_account__label", "j1_account"."owner_id" AS "j1_account__owner_id", "j2_owner"."email" AS "j2_owner__email", "j2_owner"."id" AS "j2_owner__id", "j2_owner"."name" AS "j2_owner__name" +FROM "transaction" +LEFT JOIN "account" AS "j1_account" ON "transaction"."account_id" = "j1_account"."id" +LEFT JOIN "owner" AS "j2_owner" ON "j1_account"."owner_id" = "j2_owner"."id" +WHERE TRUE +ORDER BY "transaction"."id" ASC NULLS LAST; +``` + +Each step is one more `LEFT JOIN`, and each related row's columns come back under its alias. + Includes add up, in any order, and repeating one changes nothing: `.include(lambda transaction: transaction.account)` followed by `.include(lambda transaction: transaction.account.owner)` is the same query as the second one alone. ### Identity of populated instances @@ -624,7 +1469,22 @@ In a session, each row is one object. A populated account is the same object a d --8<-- "docs/examples/populated_relations.py:identity" ``` -Both Travel transactions point at the `travel` object fetched first. Populations also accumulate on the objects a session holds: a later query's `include()` attaches to instances an earlier query returned: +Both Travel transactions point at the `travel` object fetched first. Fetching Travel and the query send these statements; on PostgreSQL only the placeholders differ: + +```sql +-- populated_relations.py:identity (sqlite) +SELECT "account"."id", "account"."label", "account"."owner_id" +FROM "account" +WHERE "account"."id" = ? +LIMIT ?; + +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on", "j1_account"."id" AS "j1_account__id", "j1_account"."label" AS "j1_account__label", "j1_account"."owner_id" AS "j1_account__owner_id" +FROM "transaction" +LEFT JOIN "account" AS "j1_account" ON "transaction"."account_id" = "j1_account"."id" +WHERE "transaction"."account_id" = ?; +``` + +The account's columns come back with each row, and the session hands back the object it already holds. Populations also accumulate on the objects a session holds: a later query's `include()` attaches to instances an earlier query returned: ```python --8<-- "docs/examples/populated_relations.py:accumulate" @@ -632,6 +1492,28 @@ Both Travel transactions point at the `travel` object fetched first. Populations `market_run` was fetched with nothing populated, and the two later queries populated its account and its category. Outside a session there is no identity map, so each row of an included query gets its own related instance. [Identity map](../concepts/identity-map.md) covers sessions and shared instances. +The three queries send these statements; on PostgreSQL only the placeholders differ: + +```sql +-- populated_relations.py:accumulate (sqlite) +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" +FROM "transaction" +WHERE "transaction"."id" = ? +LIMIT ?; + +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on", "j1_account"."id" AS "j1_account__id", "j1_account"."label" AS "j1_account__label", "j1_account"."owner_id" AS "j1_account__owner_id" +FROM "transaction" +LEFT JOIN "account" AS "j1_account" ON "transaction"."account_id" = "j1_account"."id" +WHERE TRUE; + +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on", "j1_category"."id" AS "j1_category__id", "j1_category"."name" AS "j1_category__name", "j1_category"."parent_id" AS "j1_category__parent_id" +FROM "transaction" +LEFT JOIN "category" AS "j1_category" ON "transaction"."category_id" = "j1_category"."id" +WHERE TRUE; +``` + +Neither later query names `market_run`; the session matched its row by primary key. + ### Population after a refresh Every fetch refreshes the instances a session already holds. A refresh keeps a population only while the row's foreign key still points at it; if the foreign key changed, the population is dropped and the relation is awaitable again: @@ -642,6 +1524,33 @@ Every fetch refreshes the instances a session already holds. A refresh keeps a p After the row moved to Checking, `from_checking.account` no longer claims Savings; awaiting it fetches Checking. +The four statements are the same on PostgreSQL, apart from the placeholders Ferro writes: + +```sql +-- populated_relations.py:refresh-rule (sqlite) +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on", "j1_account"."id" AS "j1_account__id", "j1_account"."label" AS "j1_account__label", "j1_account"."owner_id" AS "j1_account__owner_id" +FROM "transaction" +LEFT JOIN "account" AS "j1_account" ON "transaction"."account_id" = "j1_account"."id" +WHERE "transaction"."id" = ? +LIMIT ?; + +UPDATE "transaction" +SET account_id = $1 +WHERE id = $2; + +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" +FROM "transaction" +WHERE "transaction"."id" = ? +LIMIT ?; + +SELECT "account"."id", "account"."label", "account"."owner_id" +FROM "account" +WHERE "account"."id" = ? +LIMIT ?; +``` + +Ferro sends the raw `UPDATE` as written. The last `SELECT` is the await, fetching Checking. + ### Unsupported `include()` forms Ferro refuses these when you build the query, before any SQL: @@ -669,6 +1578,19 @@ A query loads whole rows into whole model instances. When a view only reads two Each record has a `memo` and an `amount`, read as attributes like a model's. These examples run inside the session from [the first block](#queries). +The query sends this; on PostgreSQL only the placeholders differ: + +```sql +-- partial_selects.py:basic (sqlite) +SELECT "transaction"."memo" AS "memo", "transaction"."amount" AS "amount" +FROM "transaction" +WHERE TRUE +ORDER BY "transaction"."id" ASC NULLS LAST +LIMIT ?; +``` + +Only the two selected columns are in the `SELECT`, each named with `AS`. + ### The `Row` result type A selected query does not return `Transaction` instances. It returns [`Row`][ferro.query.Row] records in a [`Rows`][ferro.query.Rows] list: @@ -687,6 +1609,18 @@ One column needs no tuple: Each record holds only the `memo` of an uncleared transaction. +The query sends this; on PostgreSQL only the placeholders differ: + +```sql +-- partial_selects.py:single (sqlite) +SELECT "transaction"."memo" AS "memo" +FROM "transaction" +WHERE "transaction"."cleared_on" IS NULL +ORDER BY "transaction"."id" ASC NULLS LAST; +``` + +The statement selects `memo` alone. + ### The `Rows` container `Rows` behaves like a list: index it, slice it, iterate it and take its `len()`. `model_dump()` returns a list of dicts, and `Rows[Row]` works as a FastAPI `response_model`: @@ -697,6 +1631,18 @@ Each record holds only the `memo` of an uncleared transaction. A slice of a `Rows` is a `Rows` too, with the two newest records. +The query sends this; on PostgreSQL only the placeholders differ: + +```sql +-- partial_selects.py:container (sqlite) +SELECT "transaction"."id" AS "id", "transaction"."memo" AS "memo" +FROM "transaction" +WHERE TRUE +ORDER BY "id" DESC NULLS LAST; +``` + +Indexing, slicing and `model_dump()` run in Python, over the records this one statement returned. + ### Column-name strings `select()` also takes column names as strings, with the same rules as `order_by()`: columns on the queried model only, the `_id` columns included, checked when you build the query: @@ -707,6 +1653,19 @@ A slice of a `Rows` is a `Rows` too, with the two newest records. The lambda form is the documented one because only a lambda can reach across a relation. +The query sends this; on PostgreSQL only the placeholders differ: + +```sql +-- partial_selects.py:strings (sqlite) +SELECT "transaction"."id" AS "id", "transaction"."memo" AS "memo" +FROM "transaction" +WHERE TRUE +ORDER BY "id" ASC NULLS LAST +LIMIT ?; +``` + +Each string became a column of `transaction`, named after itself. + ### Reaching across a relation A selected column can follow a foreign key, at any depth, as in a `where()` lambda. A column reached this way is named after its last step: @@ -717,6 +1676,20 @@ A selected column can follow a foreign key, at any depth, as in a `where()` lamb `transaction.account.label` becomes the field `label`. Strings never follow relations: `select("account.label")` raises. +The query sends this; on PostgreSQL only the placeholders differ: + +```sql +-- partial_selects.py:traversed (sqlite) +SELECT "transaction"."memo" AS "memo", "j1_account"."label" AS "label" +FROM "transaction" +INNER JOIN "account" AS "j1_account" ON "transaction"."account_id" = "j1_account"."id" +WHERE TRUE +ORDER BY "transaction"."id" ASC NULLS LAST +LIMIT ?; +``` + +The traversal joins `account`, and `AS "label"` names the field after the last step. + ### Naming output fields Return a dict from the lambda to name the record's fields. The keys name the fields, in their order: @@ -725,7 +1698,20 @@ Return a dict from the lambda to name the record's fields. The keys name the fie --8<-- "docs/examples/partial_selects.py:aliases" ``` -The record has the three names the dict gave it. The dict form also settles a name clash: `transaction.id` and `transaction.account.id` would both be `id`, and selecting both without names raises `ValueError` that suggests the dict: +The record has the three names the dict gave it. The query sends this; on PostgreSQL only the placeholders differ: + +```sql +-- partial_selects.py:aliases (sqlite) +SELECT "transaction"."memo" AS "memo", "j1_account"."label" AS "account", "j2_owner"."email" AS "owner_email" +FROM "transaction" +INNER JOIN "account" AS "j1_account" ON "transaction"."account_id" = "j1_account"."id" +INNER JOIN "owner" AS "j2_owner" ON "j1_account"."owner_id" = "j2_owner"."id" +WHERE TRUE +ORDER BY "transaction"."id" ASC NULLS LAST +LIMIT ?; +``` + +Each dict key became an `AS` name. The dict form also settles a name clash: `transaction.id` and `transaction.account.id` would both be `id`, and selecting both without names raises `ValueError` that suggests the dict: ```python --8<-- "docs/examples/partial_selects.py:collision" @@ -743,6 +1729,26 @@ A selected column that follows a relation uses the same [inner join](#join-type- Office coffee drops out of the first query and comes back in the second with `owner_email` set to `None`. A field filled from a missing relation is `None` even when its column is not nullable: the record describes the row you got. +The two queries send these statements; on PostgreSQL only the placeholders differ: + +```sql +-- partial_selects.py:traversal-narrows (sqlite) +SELECT "transaction"."memo" AS "memo", "j2_owner"."email" AS "owner_email" +FROM "transaction" +INNER JOIN "account" AS "j1_account" ON "transaction"."account_id" = "j1_account"."id" +INNER JOIN "owner" AS "j2_owner" ON "j1_account"."owner_id" = "j2_owner"."id" +WHERE TRUE; + +SELECT "transaction"."memo" AS "memo", "j2_owner"."email" AS "owner_email" +FROM "transaction" +LEFT JOIN "account" AS "j1_account" ON "transaction"."account_id" = "j1_account"."id" +LEFT JOIN "owner" AS "j2_owner" ON "j1_account"."owner_id" = "j2_owner"."id" +WHERE TRUE +ORDER BY "transaction"."id" ASC NULLS LAST; +``` + +`left_join()` turned both joins into `LEFT JOIN`s. + ### Composing projections A selected query takes `where()` (traversal included), `order_by()` (even on a column you did not select), `limit()`, `offset()` and `first()`: @@ -751,18 +1757,63 @@ A selected query takes `where()` (traversal included), `order_by()` (even on a c --8<-- "docs/examples/partial_selects.py:compose" ``` -The Travel rows come back ordered by `amount`, which is not in the records. `after()` and `before()` page a selected query too. `after(row)` reads the order keys from the record, so every order key must be selected; otherwise pass a tuple: +The Travel rows come back ordered by `amount`, which is not in the records. The query sends this; on PostgreSQL only the placeholders differ: + +```sql +-- partial_selects.py:compose (sqlite) +SELECT "transaction"."id" AS "id", "transaction"."memo" AS "memo" +FROM "transaction" +INNER JOIN "account" AS "j1_account" ON "transaction"."account_id" = "j1_account"."id" +WHERE "j1_account"."label" = ? +ORDER BY "transaction"."amount" ASC NULLS LAST; +``` + +`ORDER BY` reads `transaction.amount`, a column the `SELECT` does not list. `after()` and `before()` page a selected query too. `after(row)` reads the order keys from the record, so every order key must be selected; otherwise pass a tuple: ```python --8<-- "docs/examples/partial_selects.py:projected-paging" ``` -The first page holds the five Checking rows, so the next starts at Petty cash. `count()` and `exists()` count the same rows as the query without `select()`: +The first page holds the five Checking rows, so the next starts at Petty cash. The two pages send these statements; on PostgreSQL only the placeholders differ: + +```sql +-- partial_selects.py:projected-paging (sqlite) +SELECT "j1_account"."label" AS "account", "transaction"."id" AS "id", "transaction"."memo" AS "memo" +FROM "transaction" +INNER JOIN "account" AS "j1_account" ON "transaction"."account_id" = "j1_account"."id" +WHERE TRUE +ORDER BY "j1_account"."label" ASC NULLS LAST, "id" ASC NULLS LAST +LIMIT ?; + +SELECT "j1_account"."label" AS "account", "transaction"."id" AS "id", "transaction"."memo" AS "memo" +FROM "transaction" +INNER JOIN "account" AS "j1_account" ON "transaction"."account_id" = "j1_account"."id" +WHERE "j1_account"."label" > ? OR "j1_account"."label" IS NULL OR ("j1_account"."label" = ? AND ("transaction"."id" > ? OR "transaction"."id" IS NULL)) +ORDER BY "j1_account"."label" ASC NULLS LAST, "id" ASC NULLS LAST +LIMIT ?; +``` + +The second page's position compares the selected `account` and `id`. `count()` and `exists()` count the same rows as the query without `select()`: ```python --8<-- "docs/examples/partial_selects.py:count" ``` +Both count the three uncleared transactions. The two calls send the same statement; on PostgreSQL only the placeholders differ: + +```sql +-- partial_selects.py:count (sqlite) +SELECT COUNT(*) +FROM "transaction" +WHERE "transaction"."cleared_on" IS NULL; + +SELECT COUNT(*) +FROM "transaction" +WHERE "transaction"."cleared_on" IS NULL; +``` + +A projection changes no count: both calls send `COUNT(*)` with the query's filter. + ### Unsupported projection forms A misspelled column fails when you build the query, as in `where()`; a column that follows a relation is checked at each step: diff --git a/docs/pages/guide/relationships.md b/docs/pages/guide/relationships.md index fd21d510..61730143 100644 --- a/docs/pages/guide/relationships.md +++ b/docs/pages/guide/relationships.md @@ -20,6 +20,52 @@ Rustaceans ['Ferris'] `await ferris.team` runs one query and returns the `Team`. `rustaceans.players` is a query that runs when you call a terminal such as `.all()`. Every example on this page runs inside the session this program opens; see [Sessions](connections.md#sessions) for why and how to open one. +Every example on this page is followed by the SQL it sends, captured from Ferro when the example runs. The two creates and the two reads send these statements. PostgreSQL also asks each single-row `INSERT` for the new `id` with `RETURNING`: + +=== "SQLite" + + ```sql + -- relationships.py:first (sqlite) + INSERT INTO "team" ("name") + VALUES (?); + + INSERT INTO "player" ("name", "team_id", "mentor_id") + VALUES (?, ?, ?); + + SELECT "team"."id", "team"."name" + FROM "team" + WHERE "team"."id" = ? + LIMIT ?; + + SELECT "player"."id", "player"."mentor_id", "player"."name", "player"."team_id" + FROM "player" + WHERE "player"."team_id" = ?; + ``` + +=== "PostgreSQL" + + ```sql + -- relationships.py:first (postgres) + INSERT INTO "team" ("name") + VALUES ($1) + RETURNING "id"; + + INSERT INTO "player" ("name", "team_id", "mentor_id") + VALUES ($1, $2, $3) + RETURNING "id"; + + SELECT "team"."id", "team"."name" + FROM "team" + WHERE "team"."id" = $1 + LIMIT $2; + + SELECT "player"."id", "player"."mentor_id", "player"."name", "player"."team_id" + FROM "player" + WHERE "player"."team_id" = $1; + ``` + +`await ferris.team` is a `SELECT` on `team` by primary key; `rustaceans.players.all()` is a `SELECT` on `player` filtered by `team_id`. + ## The roster Every example on this page uses one roster. A team has players, a player may have a profile and a mentor, players enter tournaments, and a tournament may have a winning team: @@ -50,6 +96,51 @@ Each relation has two sides. The forward side is a [`ForeignKey`][ferro.ForeignK Gordon plays for the Gophers, so the Rustaceans' roster is Corro and Ferris. `rustaceans.players` is a query, so you can filter, order and page it before you run it. +The three creates and the roster query send these statements. PostgreSQL also asks each single-row `INSERT` for the new `id` with `RETURNING`: + +=== "SQLite" + + ```sql + -- relationships.py:one-to-many (sqlite) + INSERT INTO "player" ("name", "team_id", "mentor_id") + VALUES (?, ?, ?); + + INSERT INTO "team" ("name") + VALUES (?); + + INSERT INTO "player" ("name", "team_id", "mentor_id") + VALUES (?, ?, ?); + + SELECT "player"."id", "player"."mentor_id", "player"."name", "player"."team_id" + FROM "player" + WHERE "player"."team_id" = ? + ORDER BY "player"."name" ASC NULLS LAST; + ``` + +=== "PostgreSQL" + + ```sql + -- relationships.py:one-to-many (postgres) + INSERT INTO "player" ("name", "team_id", "mentor_id") + VALUES ($1, $2, $3) + RETURNING "id"; + + INSERT INTO "team" ("name") + VALUES ($1) + RETURNING "id"; + + INSERT INTO "player" ("name", "team_id", "mentor_id") + VALUES ($1, $2, $3) + RETURNING "id"; + + SELECT "player"."id", "player"."mentor_id", "player"."name", "player"."team_id" + FROM "player" + WHERE "player"."team_id" = $1 + ORDER BY "player"."name" ASC NULLS LAST; + ``` + +The reverse relation is a filter on the shadow column, `team_id`. + ### Shadow foreign-key columns For each `ForeignKey` field, Ferro adds a column and a model field named after it with `_id` (`team_id` for `team`), holding the related row's primary key. Its Python type follows the target model's primary key. Read it or filter on it without loading the team: @@ -58,29 +149,58 @@ For each `ForeignKey` field, Ferro adds a column and a model field named after i --8<-- "docs/examples/relationships.py:shadow-column" ``` -`ferris.team_id` is already on the instance, and the count runs one query with no join. The `player` table shows the shadow columns and their constraints: +`ferris.team_id` is already on the instance, and the count runs one query with no join. It sends this; on PostgreSQL only the placeholders differ: + +```sql +-- relationships.py:shadow-column (sqlite) +SELECT COUNT(*) +FROM "player" +WHERE "player"."team_id" = ?; +``` + +The `player` table shows the shadow columns and their constraints: ```python --8<-- "docs/examples/relationships.py:player-ddl" ``` -It prints, with line breaks added: +It prints these statements, laid out one column per line. PostgreSQL declares `id` as `serial` and runs the statements under a lock timeout: -```sql -CREATE TABLE IF NOT EXISTS "player" ( - "id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, - "mentor_id" integer, - "name" varchar NOT NULL, - "team_id" integer NOT NULL, - CONSTRAINT "fk_player_mentor_id_player" FOREIGN KEY ("mentor_id") - REFERENCES "player" ("id") ON DELETE SET NULL, - CONSTRAINT "fk_player_team_id_team" FOREIGN KEY ("team_id") - REFERENCES "team" ("id") ON DELETE CASCADE -) -CREATE INDEX IF NOT EXISTS "idx_player_team_id" ON "player" ("team_id") -``` +=== "SQLite" + + ```sql + -- relationships.py:player-ddl (sqlite) + CREATE TABLE IF NOT EXISTS "player" ( + "id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, + "mentor_id" integer, + "name" varchar NOT NULL, + "team_id" integer NOT NULL, + CONSTRAINT "fk_player_mentor_id_player" FOREIGN KEY ("mentor_id") REFERENCES "player" ("id") ON DELETE SET NULL, + CONSTRAINT "fk_player_team_id_team" FOREIGN KEY ("team_id") REFERENCES "team" ("id") ON DELETE CASCADE + ); + + CREATE INDEX IF NOT EXISTS "idx_player_team_id" ON "player" ("team_id"); + ``` -Each foreign key becomes a constraint named `fk___`. `idx_player_team_id` comes from `index=True`; see [Indexing foreign keys](#indexing-foreign-keys). On PostgreSQL the statements are the same except the `id` column, which is `"id" serial PRIMARY KEY NOT NULL`. +=== "PostgreSQL" + + ```sql + -- relationships.py:player-ddl (postgres) + SET LOCAL lock_timeout = '5000ms'; + + CREATE TABLE IF NOT EXISTS "player" ( + "id" serial PRIMARY KEY NOT NULL, + "mentor_id" integer, + "name" varchar NOT NULL, + "team_id" integer NOT NULL, + CONSTRAINT "fk_player_mentor_id_player" FOREIGN KEY ("mentor_id") REFERENCES "player" ("id") ON DELETE SET NULL, + CONSTRAINT "fk_player_team_id_team" FOREIGN KEY ("team_id") REFERENCES "team" ("id") ON DELETE CASCADE + ); + + CREATE INDEX IF NOT EXISTS "idx_player_team_id" ON "player" ("team_id"); + ``` + +Each foreign key becomes a constraint named `fk_
__`. `idx_player_team_id` comes from `index=True`; see [Indexing foreign keys](#indexing-foreign-keys). To filter or order by a column of the related model, a lambda can follow the relation, as in `Player.where(lambda player: player.team.name == "Rustaceans")`. [Querying across relationships](queries.md#querying-across-relationships) covers it. @@ -102,6 +222,19 @@ Gordon | Gophers `player.team` is now the `Team` instance itself: reading it needs no `await` and runs no query. `include()` covers forward foreign keys only; load a `BackRef` or many-to-many collection with `await rustaceans.players.all()`. [Populating relations with `include()`](queries.md#populating-relations-with-include) covers nullable relations, multi-hop paths and how long a relation stays populated. +The query sends this; on PostgreSQL only the placeholders differ: + +```sql +-- relationships.py:populated (sqlite) +SELECT "player"."id", "player"."mentor_id", "player"."name", "player"."team_id", "j1_team"."id" AS "j1_team__id", "j1_team"."name" AS "j1_team__name" +FROM "player" +LEFT JOIN "team" AS "j1_team" ON "player"."team_id" = "j1_team"."id" +WHERE TRUE +ORDER BY "player"."name" ASC NULLS LAST; +``` + +`include()` added a `LEFT JOIN` on `team`, whose columns come back under `j1_team__` names. + The [`ForeignKey`][ferro.ForeignKey], [`BackRef`][ferro.BackRef] and [`Relation`][ferro.Relation] reference lists every parameter. ## One-to-one @@ -114,6 +247,47 @@ Add `unique=True` to a `ForeignKey` to allow at most one child per parent. `Prof Ferris's profile comes back as one `Profile`. Corro has none, so `await corro.profile` returns `None`. Ferro backs `unique=True` with a unique index on the shadow column, `uq_profile_player_id`, so a second profile for Ferris would fail. That index also serves lookups, so `index=True` next to `unique=True` is redundant, and Ferro warns with a `UserWarning` and ignores it. +The create and the two reads send these statements. PostgreSQL also asks each single-row `INSERT` for the new `id` with `RETURNING`: + +=== "SQLite" + + ```sql + -- relationships.py:one-to-one (sqlite) + INSERT INTO "profile" ("bio", "player_id") + VALUES (?, ?); + + SELECT "profile"."bio", "profile"."id", "profile"."player_id" + FROM "profile" + WHERE "profile"."player_id" = ? + LIMIT ?; + + SELECT "profile"."bio", "profile"."id", "profile"."player_id" + FROM "profile" + WHERE "profile"."player_id" = ? + LIMIT ?; + ``` + +=== "PostgreSQL" + + ```sql + -- relationships.py:one-to-one (postgres) + INSERT INTO "profile" ("bio", "player_id") + VALUES ($1, $2) + RETURNING "id"; + + SELECT "profile"."bio", "profile"."id", "profile"."player_id" + FROM "profile" + WHERE "profile"."player_id" = $1 + LIMIT $2; + + SELECT "profile"."bio", "profile"."id", "profile"."player_id" + FROM "profile" + WHERE "profile"."player_id" = $1 + LIMIT $2; + ``` + +Each read of the one-to-one side is a `SELECT` on `profile` by `player_id` with a `LIMIT` of 1. + ## Many-to-many Declare [`ManyToMany(related_name=...)`][ferro.ManyToMany] on one side and a `BackRef()` collection on the other. `Player.tournaments` and `Tournament.players` are the two sides. Enter players in tournaments, then read both directions: @@ -122,7 +296,69 @@ Declare [`ManyToMany(related_name=...)`][ferro.ManyToMany] on one side and a `Ba --8<-- "docs/examples/relationships.py:many-to-many" ``` -Ferris entered both tournaments, and the Spring Open has Ferris and Gordon. Both sides are queries with three link methods, `add(*instances)`, `remove(*instances)` and `clear()`: +Ferris entered both tournaments, and the Spring Open has Ferris and Gordon. The creates, the links and the two reads send these statements. PostgreSQL also asks each single-row `INSERT` for the new `id` with `RETURNING`: + +=== "SQLite" + + ```sql + -- relationships.py:many-to-many (sqlite) + INSERT INTO "tournament" ("name", "winner_id") + VALUES (?, ?); + + INSERT INTO "tournament" ("name", "winner_id") + VALUES (?, ?); + + INSERT INTO "player_tournaments" ("player_id", "tournament_id") + VALUES (?, ?), (?, ?); + + INSERT INTO "player_tournaments" ("player_id", "tournament_id") + VALUES (?, ?); + + SELECT "tournament"."id", "tournament"."name", "tournament"."winner_id" + FROM "tournament" + INNER JOIN "player_tournaments" ON "tournament"."id" = "player_tournaments"."tournament_id" + WHERE "player_tournaments"."player_id" = ? + ORDER BY "tournament"."name" ASC NULLS LAST; + + SELECT "player"."id", "player"."mentor_id", "player"."name", "player"."team_id" + FROM "player" + INNER JOIN "player_tournaments" ON "player"."id" = "player_tournaments"."player_id" + WHERE "player_tournaments"."tournament_id" = ? + ORDER BY "player"."name" ASC NULLS LAST; + ``` + +=== "PostgreSQL" + + ```sql + -- relationships.py:many-to-many (postgres) + INSERT INTO "tournament" ("name", "winner_id") + VALUES ($1, $2) + RETURNING "id"; + + INSERT INTO "tournament" ("name", "winner_id") + VALUES ($1, $2) + RETURNING "id"; + + INSERT INTO "player_tournaments" ("player_id", "tournament_id") + VALUES ($1, $2), ($3, $4); + + INSERT INTO "player_tournaments" ("player_id", "tournament_id") + VALUES ($1, $2); + + SELECT "tournament"."id", "tournament"."name", "tournament"."winner_id" + FROM "tournament" + INNER JOIN "player_tournaments" ON "tournament"."id" = "player_tournaments"."tournament_id" + WHERE "player_tournaments"."player_id" = $1 + ORDER BY "tournament"."name" ASC NULLS LAST; + + SELECT "player"."id", "player"."mentor_id", "player"."name", "player"."team_id" + FROM "player" + INNER JOIN "player_tournaments" ON "player"."id" = "player_tournaments"."player_id" + WHERE "player_tournaments"."tournament_id" = $1 + ORDER BY "player"."name" ASC NULLS LAST; + ``` + +Each `add()` is one `INSERT` into the join table, and each side reads through it with an `INNER JOIN`. Both sides are queries with three link methods, `add(*instances)`, `remove(*instances)` and `clear()`: ```python --8<-- "docs/examples/relationships.py:m2m-remove" @@ -130,6 +366,36 @@ Ferris entered both tournaments, and the Spring Open has Ferris and Gordon. Both `remove()` drops one of Ferris's links and `clear()` drops the rest. The tournaments themselves stay, and so does Gordon's entry. `add()` and `remove()` read each instance's `id` attribute, so they work only when the related model's primary key is named `id`. +The calls send these statements; on PostgreSQL only the placeholders differ: + +```sql +-- relationships.py:m2m-remove (sqlite) +DELETE +FROM "player_tournaments" +WHERE "player_id" = ? AND "tournament_id" IN (?); + +SELECT "tournament"."id", "tournament"."name", "tournament"."winner_id" +FROM "tournament" +INNER JOIN "player_tournaments" ON "tournament"."id" = "player_tournaments"."tournament_id" +WHERE "player_tournaments"."player_id" = ?; + +DELETE +FROM "player_tournaments" +WHERE "player_id" = ?; + +SELECT COUNT(*) +FROM "tournament" +INNER JOIN "player_tournaments" ON "tournament"."id" = "player_tournaments"."tournament_id" +WHERE "player_tournaments"."player_id" = ?; + +SELECT COUNT(*) +FROM "player" +INNER JOIN "player_tournaments" ON "player"."id" = "player_tournaments"."player_id" +WHERE "player_tournaments"."tournament_id" = ?; +``` + +`remove()` and `clear()` delete rows of the join table only. + ### The join table Ferro creates the join table itself. It names the table after the side that declares `ManyToMany`, as `
_`, here `player_tournaments`, with one `
_id` column for each side. Print its statements from the `create_tables()` report: @@ -138,24 +404,43 @@ Ferro creates the join table itself. It names the table after the side that decl --8<-- "docs/examples/relationships.py:join-table-ddl" ``` -It prints, with line breaks added: +It prints these statements, laid out one column per line. PostgreSQL runs them under a lock timeout: -```sql -CREATE TABLE IF NOT EXISTS "player_tournaments" ( - "player_id" integer NOT NULL, - "tournament_id" integer NOT NULL, - CONSTRAINT "fk_player_tournaments_player_id_player" FOREIGN KEY ("player_id") - REFERENCES "player" ("id") ON DELETE CASCADE, - CONSTRAINT "fk_player_tournaments_tournament_id_tournament" FOREIGN KEY ("tournament_id") - REFERENCES "tournament" ("id") ON DELETE CASCADE -) -CREATE INDEX IF NOT EXISTS "idx_player_tournaments_tournament_id_player_id" - ON "player_tournaments" ("tournament_id", "player_id") -CREATE UNIQUE INDEX IF NOT EXISTS "uq_player_tournaments_player_id_tournament_id" - ON "player_tournaments" ("player_id", "tournament_id") -``` - -Both link columns cascade, so deleting a player or a tournament deletes its links. The unique index stores each link once, and the index on `(tournament_id, player_id)` serves lookups from the tournament side. The statements are the same on PostgreSQL. +=== "SQLite" + + ```sql + -- relationships.py:join-table-ddl (sqlite) + CREATE TABLE IF NOT EXISTS "player_tournaments" ( + "player_id" integer NOT NULL, + "tournament_id" integer NOT NULL, + CONSTRAINT "fk_player_tournaments_player_id_player" FOREIGN KEY ("player_id") REFERENCES "player" ("id") ON DELETE CASCADE, + CONSTRAINT "fk_player_tournaments_tournament_id_tournament" FOREIGN KEY ("tournament_id") REFERENCES "tournament" ("id") ON DELETE CASCADE + ); + + CREATE INDEX IF NOT EXISTS "idx_player_tournaments_tournament_id_player_id" ON "player_tournaments" ("tournament_id", "player_id"); + + CREATE UNIQUE INDEX IF NOT EXISTS "uq_player_tournaments_player_id_tournament_id" ON "player_tournaments" ("player_id", "tournament_id"); + ``` + +=== "PostgreSQL" + + ```sql + -- relationships.py:join-table-ddl (postgres) + SET LOCAL lock_timeout = '5000ms'; + + CREATE TABLE IF NOT EXISTS "player_tournaments" ( + "player_id" integer NOT NULL, + "tournament_id" integer NOT NULL, + CONSTRAINT "fk_player_tournaments_player_id_player" FOREIGN KEY ("player_id") REFERENCES "player" ("id") ON DELETE CASCADE, + CONSTRAINT "fk_player_tournaments_tournament_id_tournament" FOREIGN KEY ("tournament_id") REFERENCES "tournament" ("id") ON DELETE CASCADE + ); + + CREATE INDEX IF NOT EXISTS "idx_player_tournaments_tournament_id_player_id" ON "player_tournaments" ("tournament_id", "player_id"); + + CREATE UNIQUE INDEX IF NOT EXISTS "uq_player_tournaments_player_id_tournament_id" ON "player_tournaments" ("player_id", "tournament_id"); + ``` + +Both link columns cascade, so deleting a player or a tournament deletes its links. The unique index stores each link once, and the index on `(tournament_id, player_id)` serves lookups from the tournament side. `ManyToMany` takes three parameters: @@ -177,6 +462,45 @@ A model can point at itself, for mentors, org charts, threads or category trees. Crabby's mentor is Ferris, and Ferris's mentees are Crabby. Ferris has no mentor, which the `mentor_id` column allows only because of `nullable=True`. Writing `"Player | None"` instead fails at `connect()` with `RuntimeError: Relationship resolution failed: 'Player | None' not found`. +The create and the two reads send these statements. PostgreSQL also asks each single-row `INSERT` for the new `id` with `RETURNING`: + +=== "SQLite" + + ```sql + -- relationships.py:self-referential (sqlite) + INSERT INTO "player" ("name", "team_id", "mentor_id") + VALUES (?, ?, ?); + + SELECT "player"."id", "player"."mentor_id", "player"."name", "player"."team_id" + FROM "player" + WHERE "player"."id" = ? + LIMIT ?; + + SELECT "player"."id", "player"."mentor_id", "player"."name", "player"."team_id" + FROM "player" + WHERE "player"."mentor_id" = ?; + ``` + +=== "PostgreSQL" + + ```sql + -- relationships.py:self-referential (postgres) + INSERT INTO "player" ("name", "team_id", "mentor_id") + VALUES ($1, $2, $3) + RETURNING "id"; + + SELECT "player"."id", "player"."mentor_id", "player"."name", "player"."team_id" + FROM "player" + WHERE "player"."id" = $1 + LIMIT $2; + + SELECT "player"."id", "player"."mentor_id", "player"."name", "player"."team_id" + FROM "player" + WHERE "player"."mentor_id" = $1; + ``` + +`await crabby.mentor` reads `player` by primary key, and `ferris.mentees` filters `player` on `mentor_id`. + ## Nullable relations For an optional relation to another model, put `| None` inside `Annotated` and default the field to `None`. `Tournament.winner` is `Annotated[Team | None, ForeignKey(...)] = None`, so a tournament can exist before anyone wins it: @@ -187,6 +511,21 @@ For an optional relation to another model, put `| None` inside `Annotated` and d The Autumn Cup has no winner, so its `winner_id` is `None`. Setting `winner_id` and saving records the Rustaceans' Spring Open win, and `rustaceans.titles` finds it. A relation field cannot be assigned on an instance (`spring.winner = rustaceans` raises `AttributeError`), so set the shadow column. +The save and the query send these statements; on PostgreSQL only the placeholders differ: + +```sql +-- relationships.py:nullable (sqlite) +UPDATE "tournament" +SET "name" = ?, "winner_id" = ? +WHERE "id" = ?; + +SELECT "tournament"."id", "tournament"."name", "tournament"."winner_id" +FROM "tournament" +WHERE "tournament"."winner_id" = ?; +``` + +`save()` wrote `winner_id` with the rest of the row, and `rustaceans.titles` filters on it. + With the default `nullable="infer"`, the column's nullability follows the annotation. `on_delete="SET NULL"` needs a nullable column, so combining it with `nullable=False` raises `ValueError`. !!! note "Put `| None` inside `Annotated`" @@ -212,6 +551,30 @@ Delete the Rustaceans, and each rule in the roster runs: Corro, Ferris and Crabby go with their team (`CASCADE` on `Player.team`), and Ferris's profile goes with Ferris (`CASCADE` on `Profile.player`). The Spring Open stays, with no winner (`SET NULL` on `Tournament.winner`). Gordon plays for the Gophers and stays. Had only Ferris been deleted, `SET NULL` on `Player.mentor` would have kept Crabby, with no mentor. +The delete and the checks send these statements; on PostgreSQL only the placeholders differ: + +```sql +-- relationships.py:on-delete (sqlite) +DELETE +FROM "team" +WHERE "team"."id" = ?; + +SELECT "player"."id", "player"."mentor_id", "player"."name", "player"."team_id" +FROM "player" +WHERE TRUE; + +SELECT COUNT(*) +FROM "profile" +WHERE TRUE; + +SELECT "tournament"."id", "tournament"."name", "tournament"."winner_id" +FROM "tournament" +WHERE "tournament"."id" = ? +LIMIT ?; +``` + +Ferro sends one `DELETE`; the database applies each `on_delete` rule itself. + Because `CASCADE` is the default, deleting a parent removes its children unless you choose otherwise. Pick `RESTRICT` when deleting a parent that still has children would be a bug you want to surface. ## Indexing foreign keys diff --git a/docs/solutions/patterns/docs-page-conventions.md b/docs/solutions/patterns/docs-page-conventions.md index 8fa7a8fc..548eccaf 100644 --- a/docs/solutions/patterns/docs-page-conventions.md +++ b/docs/solutions/patterns/docs-page-conventions.md @@ -7,6 +7,7 @@ related_files: - docs/examples/ - docs/includes/abbreviations.md - tests/test_docs_examples.py + - tests/test_docs_sql.py - zensical.toml related_issues: [698, 700, 701, 703, 704, 725] related_prs: [] @@ -79,11 +80,60 @@ session (`{test=run prelude=}`, from #704). ### Generated SQL beside each query -Each query and each schema feature shows the SQL Ferro sends, for each -dialect where it differs. The SQL is captured by the example's own test run, -never typed by hand; a hand-simplified block drifts (unquoted reserved -words, invented aliases). This lands once #700 ships a public way to read a -query's SQL (#725 applies it to the guides). +Each query and each mutation an example runs shows the SQL Ferro sends, in a +`sql` fence below the example's result sentence. A page never carries SQL a +person typed: a hand-simplified block drifts (unquoted reserved words, +invented aliases), and the four such blocks the audit found on the Queries +guide had. The block's first line is a marker naming where its SQL comes from: + +```sql +-- predicates.py:first-query (sqlite) +SELECT "transaction"."account_id", ... +FROM "transaction" +WHERE "transaction"."amount" <= ? +ORDER BY "transaction"."amount" ASC NULLS LAST; +``` + +`.py` is a script in `docs/examples/`, `first-query` one of its +snippet regions (`# --8<-- [start:first-query]`), and `(sqlite)` or +`(postgres)` the dialect. The block holds every statement the region sends, +in order: a query terminal's statement as its `.sql()` renders it, and any +other statement (`save()`, `create()`, `bulk_create()`, an instance's +`delete()`, a many-to-many `add()`, a raw `execute()`) as the statement log +records it. Schema statements, the Postgres catalog probe, session settings +and a `transaction()` block's own `BEGIN` / `COMMIT` are left out. A region +that sends nothing but prints statements (a `create_tables()` report's +`statement.sql`) shows what it prints. + +`tests/test_docs_sql.py` regenerates every `sql` fence on the pages it lists +(`PAGES`: Queries, Aggregations, Relationships, Mutations) and compares byte +for byte, failing with the page, the line and the example. To add a block: + +1. Write the query in a snippet region of an example script; the region is + what the block shows, so keep one example per region. +2. Below the example's result sentence, write a lead-in naming the call and + saying whether only the placeholders differ on PostgreSQL ("The query + sends this; on PostgreSQL only the placeholders differ:"), then a `sql` + fence holding just the marker line. +3. Fill the fence from the test's failure message, which prints what the + region sends, then write the sentence after it: what in the SQL the reader + should notice. +4. Where PostgreSQL differs in more than placeholders (casts, `RETURNING`, + `serial`), show both as `=== "SQLite"` / `=== "PostgreSQL"` tabs, the + second marked `(postgres)`. Under `--db-backends=sqlite,postgres` the test + checks a `(postgres)` block exactly and a lone SQLite block once both + sides read each placeholder as `?`, so "only the placeholders differ" + stays true. + +Each statement is laid out one clause per line (`FROM`, `JOIN`, `WHERE`, +`GROUP BY`, `ORDER BY`, `LIMIT` and the rest, at the top level only), a +`CREATE TABLE` one column per line, and ends with `;`. Unindenting the lines +and joining them with single spaces gives back the statement Ferro sent. The +example runs with `PYTHONHASHSEED=0`, because a partial `save(only=...)` +writes its `SET` columns in set order. An example that a `(postgres)` block +or a lone SQLite block names must run on PostgreSQL too: `ferro.connect` is +pointed at the test database, so raw SQL in it uses `$1` placeholders, which +SQLite also accepts. ### Quoted migration files are regenerated, never pasted From 45e4f82db6655b7e7868a97bc947d9de8092380b Mon Sep 17 00:00:00 2001 From: Taylor Date: Wed, 21 Oct 2026 09:41:43 -0400 Subject: [PATCH 4/4] test(docs): fail the SQL capture when .sql() and the call disagree A terminal whose .sql() raises must raise the same exception type when called, having sent nothing; one that then succeeds, or fails otherwise, fails the capture naming the example line instead of losing its statement behind the depth guard. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_01Hb9U9A7Zu9ypdHTsDvdScP --- tests/test_docs_sql.py | 151 +++++++++++++++++++++++++++++++++++------ 1 file changed, 129 insertions(+), 22 deletions(-) diff --git a/tests/test_docs_sql.py b/tests/test_docs_sql.py index 06b08c96..9fa99e1f 100644 --- a/tests/test_docs_sql.py +++ b/tests/test_docs_sql.py @@ -28,6 +28,7 @@ from __future__ import annotations +import asyncio import io import json import logging @@ -35,7 +36,7 @@ import re import subprocess import sys -from collections.abc import Callable +from collections.abc import Awaitable, Callable from dataclasses import dataclass from pathlib import Path from typing import Any @@ -214,7 +215,9 @@ def _snippet_ranges(source: str) -> dict[str, tuple[int, int]]: return ranges -def check_block(block: SqlBlock, sent: Callable[[str, str], dict[str, list[str]]]) -> None: +def check_block( + block: SqlBlock, sent: Callable[[str, str], dict[str, list[str]]] +) -> None: """Assert ``block`` is marked and holds what its example's region sends. Args: @@ -263,7 +266,7 @@ def test_layout_only_adds_line_breaks() -> None: 'FROM "t"', 'INNER JOIN "a" AS "x" ON "x"."id" = "t"."a_id"', 'WHERE EXISTS (SELECT 1 FROM "s" WHERE "s"."t_id" = "t"."id") ' - "AND \"t\".\"memo\" = 'FROM x'", + 'AND "t"."memo" = \'FROM x\'', 'ORDER BY "t"."id" ASC', "LIMIT ?", "OFFSET ?", @@ -289,7 +292,11 @@ def test_layout_puts_each_column_of_a_create_table_on_its_own_line() -> None: assert _unlaid(laid) == sql -SENT = {"predicates.py": {"first-query": ['SELECT "id" FROM "transaction" WHERE "amount" <= ?']}} +SENT = { + "predicates.py": { + "first-query": ['SELECT "id" FROM "transaction" WHERE "amount" <= ?'] + } +} def _fake_sent(example: str, dialect: str) -> dict[str, list[str]]: @@ -531,6 +538,56 @@ def record(self, sql: str) -> None: if self.where is not None and self.where[0] == self.path: self.sent.append((self.where[1], sql)) + @property + def line(self) -> str: + """The example line running now, as ``:``.""" + if self.where is None: + return f"{Path(self.path).name}:?" + return f"{Path(self.where[0]).name}:{self.where[1]}" + + async def run_terminal( + self, render: Callable[[], Awaitable[Any]], call: Awaitable[Any] + ) -> Any: + """Record what a terminal sends, rendered by its ``.sql()``, then run it. + + A terminal another terminal runs is not rendered again: the outer + rendering already holds its statement. Its own statements reach the + statement log while ``depth`` is up, and are not recorded twice. + + When ``.sql()`` raises, the call must raise the same exception type, + having sent nothing; a call that then succeeds, or fails otherwise, + would send a statement no block could show, so it fails the capture + naming the example line. + """ + if self.depth: + return await call + self.depth += 1 + try: + line = self.line + try: + rendered = await render() + except Exception as refused: + try: + result = await call + except type(refused): + raise + except Exception as other: + raise AssertionError( + f"{line}: .sql() raised {refused!r} but the call raised " + f"{other!r}; a terminal must refuse the same way in both" + ) from other + raise AssertionError( + f"{line}: .sql() raised {refused!r} but the call succeeded " + f"(returned {result!r}), so what it sent cannot be shown" + ) from refused + assert rendered.dialect == self.dialect, ( + f"{line}: rendered for {rendered.dialect}, ran on {self.dialect}" + ) + self.record(str(rendered)) + return await call + finally: + self.depth -= 1 + def write(self, text: str) -> None: """Record each line the example prints, at the example line printing it.""" self.pending += text @@ -554,8 +611,74 @@ def by_region(self) -> dict[str, list[str]]: return regions +class _Rendered: + def __init__(self, sql: str, dialect: str = "sqlite") -> None: + self.sql, self.dialect = sql, dialect + + def __str__(self) -> str: + return self.sql + + +def _capture_at(line: int) -> _Capture: + capture = _Capture(DOCS_EXAMPLES / "predicates.py", "sqlite") + capture.where = (capture.path, line) + return capture + + +async def _returns(value: Any) -> Any: + return value + + +async def _raises(error: Exception) -> Any: + raise error + + +def test_a_terminal_whose_sql_raises_but_whose_call_succeeds_fails_the_capture() -> ( + None +): + capture = _capture_at(42) + with pytest.raises(AssertionError) as raised: + asyncio.run( + capture.run_terminal( + lambda: _raises(ValueError("cannot render")), _returns([1, 2]) + ) + ) + assert str(raised.value).startswith( + "predicates.py:42: .sql() raised ValueError('cannot render') but the call " + "succeeded" + ), str(raised.value) + + +def test_a_terminal_whose_sql_and_call_refuse_differently_fails_the_capture() -> None: + capture = _capture_at(7) + with pytest.raises(AssertionError, match=r"^predicates\.py:7: \.sql\(\) raised"): + asyncio.run( + capture.run_terminal( + lambda: _raises(ValueError("no")), _raises(TypeError("no")) + ) + ) + + +def test_a_terminal_that_refuses_in_both_raises_its_error_and_records_nothing() -> None: + capture = _capture_at(7) + with pytest.raises(ValueError, match="refused"): + asyncio.run( + capture.run_terminal( + lambda: _raises(ValueError("cannot")), _raises(ValueError("refused")) + ) + ) + assert capture.sent == [] and capture.depth == 0 + + +def test_a_terminal_records_its_rendering_and_returns_its_result() -> None: + capture = _capture_at(9) + result = asyncio.run( + capture.run_terminal(lambda: _returns(_Rendered("SELECT 1")), _returns(3)) + ) + assert (result, capture.sent, capture.depth) == (3, [(9, "SELECT 1")], 0) + + def _run_capture(example: str, dialect: str, url: str | None) -> dict[str, list[str]]: - import asyncio # noqa: F401 (the example's own import, made early) import runpy import ferro @@ -578,23 +701,7 @@ def render_then_call(self: Terminal, *args: Any, **kwargs: Any) -> Any: # an awaitable raises here, where the example expects it. awaitable = call(self, *args, **kwargs) - async def run() -> Any: - if capture.depth: # a terminal another terminal runs: not rendered again - return await awaitable - capture.depth += 1 - try: - try: - rendered = await self.sql(*args, **kwargs) - except Exception: # the call raises it too, and sends nothing - rendered = None - if rendered is not None: - assert rendered.dialect == dialect, rendered.dialect - capture.record(str(rendered)) - return await awaitable - finally: - capture.depth -= 1 - - return run() + return capture.run_terminal(lambda: self.sql(*args, **kwargs), awaitable) Terminal.__call__ = render_then_call # type: ignore[method-assign]