diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ccb0a374..84c2d87c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -59,6 +59,54 @@ jobs: run: | uv run --no-sync --python 3.13 prek run --all-files + docs: + name: Docs build and docs tests + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.13' + + - name: Install UV + uses: astral-sh/setup-uv@v5 + with: + enable-cache: true + + - name: Set up Rust + uses: dtolnay/rust-toolchain@stable + + - name: Cache Rust build + uses: Swatinem/rust-cache@v2 + with: + prefix-key: v1 + cache-on-failure: true + + - name: Install documentation and test dependencies + run: | + uv sync --only-group ci-test --only-group docs --no-install-project --python 3.13 + + - name: Build Rust extension + run: | + uv run maturin develop + + - name: Build Zensical site + run: | + uv run --no-sync zensical build --clean --strict + + - name: Run the docs lint + run: | + uv run --no-sync python -m tests._docs_checks docs/pages docs/examples src/ferro + + - name: Run the docs tests + # Every docs test module is named test_doc*.py, which + # tests/test_docs_site.py pins, so a new one runs here unlisted. + run: | + uv run --no-sync pytest --no-cov -p no:cacheprovider tests/test_doc*.py + test-rust: name: Rust tests runs-on: ubuntu-latest @@ -119,7 +167,7 @@ jobs: - name: Install dependencies run: | - uv sync --only-group ci-test --no-install-project --python 3.13 + uv sync --only-group ci-test --only-group docs --no-install-project --python 3.13 - name: Build Rust extension run: | @@ -170,7 +218,7 @@ jobs: - name: Install dependencies run: | - uv sync --only-group ci-test --no-install-project --python ${{ matrix.python-version }} + uv sync --only-group ci-test --only-group docs --no-install-project --python ${{ matrix.python-version }} - name: Build Rust extension run: | @@ -296,7 +344,7 @@ jobs: all-checks: name: All Checks Passed - needs: [lint-and-format, test-python-pr, test-python-main, test-python-backend-matrix, test-rust] + needs: [lint-and-format, docs, test-python-pr, test-python-main, test-python-backend-matrix, test-rust] runs-on: ubuntu-latest if: always() steps: @@ -310,5 +358,6 @@ jobs: if ! ok "${{ needs.test-python-main.result }}"; then exit 1; fi if ! ok "${{ needs.test-python-backend-matrix.result }}"; then exit 1; fi if ! ok "${{ needs.test-rust.result }}"; then exit 1; fi + if ! ok "${{ needs.docs.result }}"; then exit 1; fi echo "All checks passed!" diff --git a/.github/workflows/publish-docs.yml b/.github/workflows/publish-docs.yml index b3fbe971..79ed5c3e 100644 --- a/.github/workflows/publish-docs.yml +++ b/.github/workflows/publish-docs.yml @@ -42,7 +42,7 @@ jobs: - name: Build Zensical site run: | - uv run --no-sync zensical build --clean + uv run --no-sync zensical build --clean --strict - name: Upload Pages artifact uses: actions/upload-pages-artifact@v3 diff --git a/AGENTS.md b/AGENTS.md index d87ca665..eb6a9b18 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -422,38 +422,47 @@ preferred declaration style. --- -## I-8: Lambda predicates are the official query style +## I-8: Queries are lambda predicates named after the model -Documentation and examples use the lambda predicate style for all queries: +A query filters with a lambda whose parameter is named after the model, in +lowercase singular: ```python adults = await User.where(lambda user: user.age >= 18).all() +recent = await Post.order_by(lambda post: post.published_at, "desc").all() ``` +Lambdas are the only predicate form. `where()` raises `TypeError` on anything +that is not a callable; the `col()` and operator-style predicates were removed +in 0.14. + Rules: -- **Every query example** in docs, docstring `Examples:` sections, and - `docs/examples/` scripts uses lambda predicates. -- Name the lambda parameter after the model in **lowercase singular** - (`user` for `User`, `post` for `Post`) so filters read naturally. -- When the predicate styles themselves are documented, present them in - order **lambda > `col()` > operator**, with lambda labeled the officially - recommended style. -- **Operator style** (`User.where(User.age >= 18)`) is compatible today but - is slated for deprecation in a future release and fails static type - checking (`User.age >= 18` types as `bool`; `where()` expects - `QueryNode | Predicate`). Docs say so explicitly wherever the style is - shown. -- **`order_by` is not a predicate**, but its lambda selector - (`order_by(lambda u: u.age, "desc")`) is validated and is the documented - style — and it is the ONLY way to order by a related column - (`order_by(lambda t: t.account.label)`), which relation traversal requires. - Attribute style is not accepted — `order_by` takes a column-name string or - a lambda selector (anything else raises `TypeError`). - -The canonical comparisons live in `docs/pages/guide/queries.md` -("Predicate Styles") and `docs/pages/concepts/query-typing.md`; everywhere -else uses lambda without restating the trade-offs. +- **Every query example** in docs pages, docstring `Examples:` sections and + `docs/examples/` scripts passes lambdas to query methods (`where`, + `select`, `order_by`, `include`, `exists`, `count`, `aggregate`, + `group_by`, `after`, `before` and every other method that takes a + callable). +- **Name the parameter after the model in lowercase singular**: `user` for + `User`, `post` for `Post`, `line_item` for `LineItem`, so the predicate + reads as the sentence it means. A traversal keeps the root model's name + (`lambda transaction: transaction.account.label`). +- **`order_by`** takes a column-name string (`order_by("created_at", + "desc")`) or a lambda selector (`order_by(lambda post: post.created_at, + "desc")`); anything else raises `TypeError`. The lambda is the documented + form, and the only one that orders by a related column + (`order_by(lambda transaction: transaction.account.label)`). +- **The lint** `uv run --no-sync python -m tests._docs_checks PATH...` lists + every lambda passed to a query method whose parameter is one character, + as `path:line: message`, and exits 1. It reads Python files, Python fences + in Markdown and docstrings. `tests/test_docs_lint.py` runs it over + `docs/pages`, `docs/examples` and `src/ferro`, so a violation fails the + suite; run it yourself over every docs page, example and docstring you + touch. + +`docs/pages/guide/queries.md` and `docs/pages/concepts/query-typing.md` +explain how lambda predicates are evaluated and typed; every other page uses +them without restating that. --- @@ -528,8 +537,8 @@ key: pinned_at = datetime(2026, 3, 1, 15, 0, tzinfo=UTC) await Message(id=2, pinned_at=pinned_at, ...).save() page = await ( - Message.order_by(lambda m: m.pinned_at) - .order_by(lambda m: m.id) + Message.order_by(lambda message: message.pinned_at) + .order_by(lambda message: message.id) .after((pinned_at, 2)) .all() ) @@ -556,6 +565,72 @@ Architecture review: I-22 in #428. See PRD #430 and --- +## I-14: Docstrings are short Google-style reference entries + +The API reference is rendered from docstrings by mkdocstrings and griffe, so +a docstring is a reference entry, written once in one form: + +```python +async def get(cls, pk: Any, *, session: Session | None = None) -> Self: + """Fetch one record by primary key. + + Args: + pk: The primary key value to look up. + session: The session to run in. Defaults to the one opened by the + enclosing `async with engines.session():`. + + Returns: + The record whose primary key is `pk`. + + Raises: + ModelDoesNotExist: No row has that key. Use + [`get_or_none`][ferro.Model.get_or_none] to get `None` instead. + + Examples: + ```python {test=run prelude=reference} + author = await Author.get(1) + print(author) + ``` + """ +``` + +Rules: + +- **Google style as griffe parses it**: `Args:`, `Returns:`, `Yields:`, + `Raises:`, `Attributes:`, `Examples:`, each a section griffe recognises, + with entries indented under it. A section griffe drops or splits into + table rows is a bug in the docstring. +- **The summary is one line ending with a period.** A function or method + summary is an infinitive ("Fetch one record by primary key."); a class + summary is a noun phrase ("A lazy query over one model's rows."). The + body starts after a blank line. +- **Every parameter is in `Args:`**, keyword-only ones included + (`session`, `using`, `nulls`), with the default stated when it is not + obvious from the signature. +- **Every public dataclass or Pydantic field** has a one-line docstring: a + string literal on the line below the field. +- **Cross-references are Markdown autorefs**: + `` [`Model.where`][ferro.Model.where] `` or `[Model][ferro.Model]`. + Sphinx roles (`:class:`, `:func:`, `:meth:`, `:data:`) render as literal + text. +- **Every reference is one the reader can follow.** Docstrings carry no ADR + numbers, issue numbers, internal codenames or Rust internals (crate, + function and FFI names); a link to a docs page carries the reasoning. +- **Examples use lambda predicates (I-8) against the shared reference + prelude** (`docs/examples/_preludes/reference.py`: `Author` and `Post`, + connected, inside a session), in a fence marked + `{test=run prelude=reference}` so the docs harness + (`tests/test_docstring_examples.py`) runs it. +- **An explanation longer than one paragraph lives in a guide or concepts + page**, and the docstring links to it. + +`docs/solutions/patterns/docstring-conventions.md` shows each rule as a +before/after pair. Page-level conventions (running examples, snippet +markers, the session wrapper, the Annotated idiom for I-7 twins) are in +`docs/solutions/patterns/docs-page-conventions.md`. + +--- + ## Agent skills ### Issue tracker diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2bd38ea5..a14ab02c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,7 +2,7 @@ -We welcome contributions to Ferro! This guide will help you get started with developing Ferro locally. +To develop Ferro locally you need Python 3.13 or later, a Rust toolchain and uv. This guide takes you from a clone to a pull request. ## Prerequisites @@ -18,29 +18,30 @@ Before starting, ensure you have: curl -LsSf https://astral.sh/uv/install.sh | sh ``` -## Getting Started +## Getting started -### 1. Clone the Repository +### 1. Clone the repository ```bash git clone https://github.com/syn54x/ferro-orm.git cd ferro-orm ``` -### 2. Install Dependencies +### 2. Install dependencies ```bash uv sync --group dev ``` -This will install all development dependencies including: +This installs every development dependency, including: + - Testing tools (pytest, pytest-asyncio, pytest-cov) - Linting and formatting tools (ruff, prek) - Build tools (maturin) - Documentation tools (zensical) - Release tools (commitizen, python-semantic-release) -### 3. Install Pre-commit Hooks +### 3. Install pre-commit hooks ```bash # Install all hooks (file checks, linting, formatting) @@ -50,7 +51,8 @@ uv run prek install uv run prek install --hook-type commit-msg ``` -These hooks will automatically: +These hooks: + - Check for trailing whitespace - Fix end-of-file issues - Validate YAML, TOML, and JSON files @@ -59,7 +61,7 @@ These hooks will automatically: - Lint Rust code with clippy - Validate conventional commit messages -### 4. Build the Rust Extension +### 4. Build the Rust extension ```bash uv run maturin develop @@ -67,14 +69,17 @@ uv run maturin develop This compiles the Rust core and installs it in development mode. You'll need to re-run this command after making changes to Rust code. -## Development Workflow +## Development workflow -### Running Tests +### Running tests ```bash # Run all tests with coverage uv run pytest +# Run all tests in parallel, without coverage +uv run pytest -n 8 --no-cov + # Run specific test file uv run pytest tests/test_models.py @@ -85,7 +90,7 @@ uv run pytest -v uv run pytest --cov=src --cov-report=html ``` -### Running Linters +### Running linters ```bash # Run all pre-commit hooks @@ -98,7 +103,7 @@ cargo fmt # Rust formatting cargo clippy --workspace --all-targets -- -D warnings # Rust linting ``` -### Building Documentation +### Building documentation ```bash # Serve documentation locally (with live reload) @@ -107,10 +112,16 @@ uv run zensical serve # Build documentation uv run zensical build -# Documentation will be available at http://127.0.0.1:8000/ +# The site is served at http://127.0.0.1:8000/ ``` -### Testing Your Changes +Before you open a documentation pull request, run the strict build. It fails on a broken link or a snippet that no longer exists: + +```bash +uv run zensical build --clean --strict +``` + +### Testing your changes Before submitting a PR, ensure: @@ -126,7 +137,7 @@ Before submitting a PR, ensure: 3. **Rust tests pass:** ```bash - cargo test + cargo test --no-default-features --features testing ``` 4. **Code builds successfully:** @@ -134,7 +145,7 @@ Before submitting a PR, ensure: uv run maturin develop ``` -## Conventional Commits +## Conventional commits Ferro uses [Conventional Commits](https://www.conventionalcommits.org/) for automated version bumping and changelog generation. All commit messages **must** follow this format: @@ -146,7 +157,7 @@ Ferro uses [Conventional Commits](https://www.conventionalcommits.org/) for auto [optional footer(s)] ``` -### Commit Types +### Commit types - **feat**: New feature (triggers minor version bump) - **fix**: Bug fix (triggers patch version bump) @@ -181,7 +192,7 @@ git commit -m "feat: redesign query API BREAKING CHANGE: Query.filter() now requires Q objects instead of kwargs" ``` -### Commit Validation +### Commit validation The pre-commit hook will automatically validate your commit message format. Invalid commits will be rejected with an error message. @@ -190,7 +201,7 @@ If you need to bypass the hook (not recommended), use: git commit --no-verify -m "message" ``` -## Pull Request Process +## Pull request process 1. **Create a feature branch:** ```bash @@ -208,56 +219,70 @@ git commit --no-verify -m "message" git push origin feat/my-new-feature ``` -4. **Open a Pull Request** on GitHub +4. **Open a pull request** on GitHub. Put `Closes #N` in the body for each issue the pull request completes (see [Project rules](#project-rules)). 5. **Wait for CI checks to pass:** - All linters must pass - All tests must pass - Code must build on all platforms -6. **Address review feedback** if any +6. **Address review feedback**, if any. + +A maintainer merges the pull request once it is approved. + +### PR requirements -7. **Merge** once approved! +- All CI checks pass +- Conventional commit format followed +- Tests added for new features +- Documentation updated +- No merge conflicts with `main` +- `Closes #N` in the body for every issue the pull request completes +- `CHANGELOG.md` untouched -### PR Requirements +## Project rules -- ✅ All CI checks pass -- ✅ Conventional commit format followed -- ✅ Tests added for new features -- ✅ Documentation updated -- ✅ No merge conflicts with main +[`AGENTS.md`](https://github.com/syn54x/ferro-orm/blob/main/AGENTS.md) lists the invariants every change must keep. They bind human contributors and AI agents alike. The ones a pull request meets most often: -## Release Process +- **Issues close with the pull request (I-9).** Put a GitHub keyword in the body for each completed issue (`Closes #89`, `Fixes #90`), so nobody has to triage it after the merge. +- **Never edit `CHANGELOG.md` (I-10).** The release workflow writes it from your conventional commit messages, so describe the change in the pull request title and body instead. +- **Docs show both field styles (I-7).** An example that declares fields with `Field()` appears twice, as an Assignment tab and an Annotated tab, each backed by a runnable script in `docs/examples/`. +- **Docs queries use lambda predicates (I-8).** Name the lambda parameter after the model in lowercase singular: `User.where(lambda user: user.age >= 18)`. +- **Every migration door produces the same DDL (I-1).** A schema change goes through the one planner, so `connect(auto_migrate=True)`, `ferro migrate new` and the Alembic bridge emit the same statements. +- **No `unwrap()` across the FFI boundary (I-3).** A PyO3 function returns a `PyResult` instead of panicking. + +## Release process Ferro uses automated releases. You don't need to manually bump versions or update the changelog. -### How Releases Work +### How releases work 1. **Commits are merged to main** - - Changelog is automatically updated with unreleased changes + - Changelog is automatically updated with unreleased changes 2. **Maintainer triggers the release workflow** - The release process is fully automated. To trigger a new release: + The release process is fully automated. To trigger a new release: + + - **Via GitHub CLI (Recommended):** + ```bash + gh workflow run release.yml + ``` + - **Via GitHub Web UI:** Go to the **Actions** tab, select the **Release** workflow, and click **Run workflow**. - - **Via GitHub CLI (Recommended):** - ```bash - gh workflow run release.yml - ``` - - **Via GitHub Web UI:** Go to the **Actions** tab, select the **Release** workflow, and click **Run workflow**. + This will: - This will: - - Automatically determine the next version from conventional commits. - - Update `pyproject.toml` and `Cargo.toml`. - - Finalize `CHANGELOG.md`. - - Create and push the Git tag. - - Trigger the **Build & Publish** workflow to upload wheels to PyPI. + - Automatically determine the next version from conventional commits. + - Update `pyproject.toml` and `Cargo.toml`. + - Finalize `CHANGELOG.md`. + - Create and push the Git tag. + - Trigger the **Build & Publish** workflow to upload wheels to PyPI. 3. **Package is automatically published to PyPI** - - Cross-platform wheels are built - - Package is uploaded using trusted publishing + - Cross-platform wheels are built + - Package is uploaded using trusted publishing -### Version Bumping +### Version bumping Version bumps are determined by commit types: @@ -265,7 +290,7 @@ Version bumps are determined by commit types: - **Minor** (1.0.0 → 1.1.0): Commits with `feat:` type - **Patch** (1.0.0 → 1.0.1): Commits with `fix:` or `perf:` type -## Code Style +## Code style ### Python @@ -285,66 +310,74 @@ Version bumps are determined by commit types: ## Testing -### Python Tests +### Python tests -Located in `tests/` directory. Use pytest with async support: +Tests that cross the Python and Rust bridge, or exercise Pydantic models, live in `tests/`. A test connects with the `db_url` fixture, opens a session and asserts on what comes back: -```python +```python {test=compile reason="a test for this repository's suite, which supplies its db_url fixture"} import pytest -from ferro import Model, FerroField, connect + +from ferro import Field, Model, connect, engines + + +class Author(Model): + id: int | None = Field(default=None, primary_key=True) + name: str + @pytest.mark.asyncio -async def test_create_model(): - await connect("sqlite::memory:") - user = await User.create(name="Alice") - assert user.name == "Alice" +async def test_create_author(db_url): + await connect(db_url, auto_migrate=True) + async with engines.session(): + author = await Author.create(name="Ada Lovelace") + assert author.id is not None ``` -### Rust Tests +The `db_url` fixture gives each test its own SQLite file. Mark a test `backend_matrix` to run it against Postgres too when a server is available. -Located alongside Rust code with `#[cfg(test)]`: +### Rust tests -```rust -#[cfg(test)] -mod tests { - use super::*; +Pure SQL and schema generation logic is tested in Rust, in a `#[cfg(test)]` module next to the code it covers. Run all of them with: - #[test] - fn test_sql_generation() { - let query = generate_select_query("users"); - assert_eq!(query, "SELECT * FROM users"); - } -} +```bash +cargo test --no-default-features --features testing ``` -## Project Structure +The flags skip the extension-module linking so the test binary can resolve Python symbols, and start the interpreter for tests that call into Python. A feature is not done until both the Rust and the Python tests pass. + +## Project structure ``` ferro/ ├── src/ -│ ├── ferro/ # Python package -│ │ ├── __init__.py -│ │ ├── models.py -│ │ ├── queries.py -│ │ └── ... -│ └── lib.rs # Rust core +│ ├── ferro/ # Python package (models, query/, relations/, migrations/, cli/) +│ └── *.rs # Rust core, exposed to Python through PyO3 +├── crates/ +│ ├── ferro-ddl-lowering/ # column types, names and rendering rules +│ ├── ferro-migrate/ # the one schema planner and its renderer +│ └── ferro-schema-ir/ # the schema intermediate representation ├── tests/ # Python tests -├── docs/ # Documentation +├── docs/ +│ ├── pages/ # the documentation site +│ ├── examples/ # runnable scripts the pages embed +│ ├── adr/ # architecture decision records +│ └── solutions/ # patterns and known footguns ├── .github/ │ └── workflows/ # CI/CD workflows -├── Cargo.toml # Rust dependencies -├── pyproject.toml # Python dependencies +├── AGENTS.md # project invariants +├── GLOSSARY.md # domain vocabulary +├── Cargo.toml # Rust workspace +├── pyproject.toml # Python package └── README.md ``` -## Getting Help +## Getting help - **Issues**: [GitHub Issues](https://github.com/syn54x/ferro-orm/issues) -- **Discussions**: [GitHub Discussions](https://github.com/syn54x/ferro-orm/discussions) -- **Documentation**: [https://syn54x.github.io/ferro-orm](https://syn54x.github.io/ferro-orm) +- **Documentation**: [https://ferro-orm.x54.sh/](https://ferro-orm.x54.sh/) ## License -By contributing to Ferro, you agree that your contributions will be licensed under the same license as the project (Apache 2.0 OR MIT). +By contributing to Ferro, you agree that your contributions will be licensed under the project's license, [Apache 2.0](https://github.com/syn54x/ferro-orm/blob/main/LICENSE). diff --git a/README.md b/README.md index af755807..90ee1aec 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,5 @@ # Ferro: The Pydantic x Rust ORM - - [![PyTest](https://img.shields.io/badge/Pytest-0A9EDC?style=for-the-badge&logo=pytest&logoColor=white)](https://docs.pytest.org/) [![Ruff](https://img.shields.io/badge/Ruff-FFC107?style=for-the-badge&logo=python&logoColor=black)](https://docs.astral.sh/ruff/) [![Zensical](https://img.shields.io/badge/Zensical-4051B5?style=for-the-badge&logo=markdown&logoColor=white)](https://zensical.org/) @@ -14,60 +12,66 @@ [![Codecov](https://img.shields.io/codecov/c/github/syn54x/ferro-orm?style=for-the-badge&logo=codecov&logoColor=white)](https://codecov.io/gh/syn54x/ferro-orm) [![Issues](https://img.shields.io/github/issues/syn54x/ferro-orm?style=for-the-badge&logo=github&logoColor=white)](https://github.com/syn54x/ferro-orm/issues) -Ferro is a high-performance, asynchronous ORM for Python, powered by a core engine written in Rust. Designed for simplicity and it only has one dependency: Pydantic. Delivering ergonomics familiar to the modern Pythonista and the speed and safety of Rust's SQLx and Sea-Query. + +Ferro is an async ORM for Python: you declare Pydantic models, and a Rust engine builds the SQL, runs it and turns rows into model instances. Its runtime dependencies are Pydantic and pydantic-settings. + -## Key Features +Here is a model, a query and the result: -- **High-Performance Core**: All SQL generation and row hydration are handled by a dedicated Rust engine, minimizing "Python Tax" on data-heavy operations. -- **Async First**: Built from the ground up for asynchronous applications, utilizing `pyo3-async-runtimes` for non-blocking I/O. -- **Pydantic Integration**: Leverages Pydantic V2 for schema definition and data validation, providing full IDE support and type safety. -- **Zero-Copy Intent**: Designed with zero-copy principles to maximize throughput during large-scale data retrieval. -- **Identity Map**: Ensures object consistency across your application by tracking active model instances in a thread-safe registry. +```python +import asyncio -## Architecture +from ferro import Field, Model, connect, engines -Ferro operates through a dual-layer architecture connected via a high-performance FFI (Foreign Function Interface) bridge: -1. **Python Layer**: Developers define models using standard Python classes; a metaclass registers them with the backend. -2. **Rust Engine**: Built on `SQLx` and `Sea-Query` for GIL-free row parsing and object instantiation. +class User(Model): + id: int | None = Field(default=None, primary_key=True) + username: str + is_active: bool = True -## Installation -Ferro is distributed as pre-compiled wheels for macOS, Linux, and Windows. +async def main(): + await connect("sqlite::memory:", auto_migrate=True) -```bash -pip install ferro-orm -# Or with the migrations CLI (ferro migrate) -pip install "ferro-orm[cli]" -# Or with the Alembic bridge -pip install "ferro-orm[alembic]" + async with engines.session(): + await User.create(username="alice") + await User.create(username="bob", is_active=False) + + active_users = await User.where(lambda user: user.is_active == True).all() + print(f"Found {len(active_users)} active users.") + + +asyncio.run(main()) ``` -Schema changes go through auto-migrate while you develop and reviewed migrations (`ferro migrate new`, `ferro migrate up`) in production — see [Schema Management](https://ferro-orm.x54.sh/guide/schema/overview/). Ferro currently supports SQLite and PostgreSQL. Register multiple named connections with `connect(..., name="...")` when a process needs more than one database — see the [connections guide](https://ferro-orm.x54.sh/guide/connections/). +Running it prints: -## Quick Start +```text +Found 1 active users. +``` -```python -import asyncio -from ferro import Field, Model, connect +Every ORM call runs inside `async with engines.session():`, which routes it to the connection you opened. `auto_migrate=True` creates the table on connect; it suits development and tests. -class User(Model): - id: int | None = Field(default=None, primary_key=True) - username: str - is_active: bool = True +## What Ferro does -async def main(): - await connect("sqlite:example.db?mode=rwc", auto_migrate=True) +- The Rust engine generates the SQL and parses rows outside the GIL, then fills each model's fields directly, without calling `__init__`. +- Everything is async, on `pyo3-async-runtimes` and SQLx. +- Models are Pydantic v2 models, so validation and IDE support work as they do for any Pydantic class. +- Within a session, one database row is one Python object (the identity map). +- It supports SQLite and PostgreSQL. Both are compiled into the engine, so no driver packages are needed. - # Create - alice = await User.create(username="alice") +## Installation - # Query (lambda predicates — name the parameter after the model, e.g. user for User) - active_users = await User.where(lambda user: user.is_active == True).all() - print(f"Found {len(active_users)} active users.") +Ferro ships pre-compiled wheels for macOS, Linux and Windows: -if __name__ == "__main__": - asyncio.run(main()) +```bash +pip install ferro-orm +# With the migrations CLI (ferro migrate) +pip install "ferro-orm[cli]" +# With the Alembic bridge +pip install "ferro-orm[alembic]" ``` - +## Schema changes + +Auto-migrate keeps a development database current. For production, `ferro migrate new` writes reviewed migration files and `ferro migrate up` applies them; see [Schema management](https://ferro-orm.x54.sh/guide/schema/overview/). To use more than one database, register named connections with `connect(..., name="...")`; see [Connections](https://ferro-orm.x54.sh/guide/connections/). diff --git a/docs/examples/_ledger_seed.py b/docs/examples/_ledger_seed.py new file mode 100644 index 00000000..82f204ed --- /dev/null +++ b/docs/examples/_ledger_seed.py @@ -0,0 +1,107 @@ +"""The rows of the Queries guide's ledger, shared by its example scripts. + +The guide (docs/pages/guide/queries.md) declares one ledger in +``predicates.py`` and its Annotated twin ``predicates_annotated.py``. Both +twins, the other query examples that import ``predicates``, and the +``queries`` prelude add these same rows, so every result the guide shows is +the same wherever it runs. + +Two owners, four accounts (``Petty cash`` has no owner), four categories +(``Groceries`` and ``Dining`` sit under ``Food``), nine transactions, one +split transaction with two lines, one transfer between ``Checking`` and +``Savings`` and two tags. +""" + +from __future__ import annotations + +from collections.abc import Mapping +from datetime import date +from decimal import Decimal +from typing import Any + + +async def seed(models: Mapping[str, Any]) -> None: + """Add the ledger's rows through the model classes in ``models``. + + ``models`` maps each class name (``Owner``, ``Account``, ``Category``, + ``Transaction``, ``SplitLine``, ``Transfer``, ``Tag``) to the class, so + either declaration style's classes can be seeded. + """ + Owner = models["Owner"] + Account = models["Account"] + Category = models["Category"] + Transaction = models["Transaction"] + SplitLine = models["SplitLine"] + Transfer = models["Transfer"] + Tag = models["Tag"] + + ada = await Owner.create(name="Ada Lovelace", email="ada@example.com") + grace = await Owner.create(name="Grace Hopper", email="grace@example.com") + + checking = await Account.create(label="Checking", owner=ada) + savings = await Account.create(label="Savings", owner=ada) + travel = await Account.create(label="Travel", owner=grace) + petty_cash = await Account.create(label="Petty cash") + + food = await Category.create(name="Food") + groceries = await Category.create(name="Groceries", parent=food) + dining = await Category.create(name="Dining", parent=food) + housing = await Category.create(name="Housing") + + async def add( + memo: str, + amount: str, + posted_on: date, + cleared_on: date | None, + account: Any, + category: Any = None, + ) -> Any: + return await Transaction.create( + memo=memo, + amount=Decimal(amount), + posted_on=posted_on, + cleared_on=cleared_on, + account=account, + category=category, + ) + + await add("Salary", "3200.00", date(2026, 3, 1), date(2026, 3, 2), checking) + rent = await add( + "Rent", "-1450.00", date(2026, 3, 2), date(2026, 3, 3), checking, housing + ) + await add( + "Market run", "-86.40", date(2026, 3, 5), date(2026, 3, 6), checking, groceries + ) + to_savings = await add( + "To savings", "-500.00", date(2026, 3, 7), date(2026, 3, 8), checking + ) + from_checking = await add( + "From checking", "500.00", date(2026, 3, 7), date(2026, 3, 8), savings + ) + train = await add("Train to Lyon", "-64.00", date(2026, 3, 9), None, travel) + dinner = await add( + "Dinner in Lyon", "-48.50", date(2026, 3, 10), None, travel, dining + ) + await add( + "Office coffee", + "-12.75", + date(2026, 3, 11), + date(2026, 3, 12), + petty_cash, + dining, + ) + weekly_shop = await add("Weekly shop", "-95.00", date(2026, 3, 12), None, checking) + + await SplitLine.create( + amount=Decimal("-72.00"), transaction=weekly_shop, category=groceries + ) + await SplitLine.create( + amount=Decimal("-23.00"), transaction=weekly_shop, category=dining + ) + + await Transfer.create(outflow=to_savings, inflow=from_checking) + + reimbursable = await Tag.create(name="reimbursable") + tax = await Tag.create(name="tax") + await reimbursable.transactions.add(train, dinner) + await tax.transactions.add(rent) diff --git a/docs/examples/_migrations_project.py b/docs/examples/_migrations_project.py index 8ddcd524..0ed9c679 100644 --- a/docs/examples/_migrations_project.py +++ b/docs/examples/_migrations_project.py @@ -34,6 +34,10 @@ from contextlib import contextmanager from pathlib import Path +# The docs harness's region reader, so a region reads the same here as on a page. +sys.path.insert(0, str(Path(__file__).resolve().parents[2])) +from tests._docs_shared import snippet_regions # noqa: E402 + __all__ = ["Project", "project"] FERRO_TOML = 'models = ["blog.models"]\ndialects = ["postgres", "sqlite"]\n' @@ -42,10 +46,10 @@ def _region(script: Path, name: str = "models") -> str: - source = script.read_text() - opening, closing = f"# --8<-- [start:{name}]", f"# --8<-- [end:{name}]" - start = source.index(opening) + len(opening) - return source[start : source.index(closing)].strip("\n") + "\n" + regions = snippet_regions(script.read_text(), where=str(script)) + if name not in regions: + raise ValueError(f"{script} has no snippet region {name!r}") + return regions[name].body.strip("\n") + "\n" def _before(region: str) -> str: diff --git a/docs/examples/_preludes/__init__.py b/docs/examples/_preludes/__init__.py new file mode 100644 index 00000000..8f92078a --- /dev/null +++ b/docs/examples/_preludes/__init__.py @@ -0,0 +1,102 @@ +"""Preludes for runnable docs blocks, and the runner that uses them. + +A prelude is a module in this directory. It declares the models a docs block +uses and defines ``opened()``, an async context manager that connects to an +in-memory SQLite database, adds a few rows and opens a session. A block marked +``{test=run prelude=}`` sees every name the prelude defines and runs +inside ``opened()``, with top-level ``await``. + +Run a block by piping its source to this file: + + python docs/examples/_preludes/__init__.py + +``path`` and ``first line`` are where the block's source sits, so a traceback +points at the docs page or docstring, not at this runner. With +``FERRO_DOCS_DATABASE_URL`` set (a ``backend=postgres`` fence), every +``connect()`` the prelude or the block makes goes to that database instead of +the URL it names, keeping the PostgreSQL query options that URL carries +(``tests._docs_shared.point_connect_at``). +""" + +from __future__ import annotations + +import ast +import asyncio +import importlib.util +import inspect +import os +import sys +from pathlib import Path +from types import ModuleType + +PRELUDES = Path(__file__).resolve().parent + +# The docs harness's connect redirect, the one the SQL capture installs too. +sys.path.insert(0, str(PRELUDES.parents[2])) +from tests._docs_shared import point_connect_at # noqa: E402 + + +def load_prelude(name: str) -> ModuleType: + """Import ``.py`` from this directory, refusing a name that has none.""" + path = PRELUDES / f"{name}.py" + if name.startswith("_") or not path.is_file(): + available = sorted( + p.stem for p in PRELUDES.glob("*.py") if not p.stem.startswith("_") + ) + raise SystemExit( + f"no prelude named {name!r}; add docs/examples/_preludes/{name}.py " + f"or use one of: {', '.join(available)}" + ) + spec = importlib.util.spec_from_file_location(f"_ferro_prelude_{name}", path) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +async def run_block(prelude: str, source: str, path: str, first_line: int) -> None: + """Run ``source`` inside the prelude's ``opened()``, with its names in scope.""" + module = load_prelude(prelude) + # The block runs as a module of its own whose file is the page or + # docstring it came from. A model class the block declares then finds its + # source there: pydantic reads field docstrings from the class's source + # (``use_attribute_docstrings``), and under ``__main__`` it would read + # this runner instead. + block = ModuleType("_ferro_docs_block") + block.__file__ = path + sys.modules[block.__name__] = block + namespace = vars(block) + namespace.update( + (name, value) + for name, value in vars(module).items() + if not name.startswith("__") + ) + # Pad the source so line numbers in a traceback are the page's own. + # ``dont_inherit``: the block must not pick up this file's + # ``from __future__ import annotations``, which would turn a declared + # model's annotations into strings a reader's own module never has. + code = compile( + "\n" * (first_line - 1) + source, + path, + "exec", + flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT, + dont_inherit=True, + ) + async with module.opened(): + result = eval(code, namespace) # noqa: S307 - the docs block is ours + if inspect.iscoroutine(result): + await result + + +def main(argv: list[str]) -> None: + prelude, path, first_line = argv + url = os.environ.get("FERRO_DOCS_DATABASE_URL") + if url: + # Before the prelude imports connect, so its name is this one too. + point_connect_at(url) + asyncio.run(run_block(prelude, sys.stdin.read(), path, int(first_line))) + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/docs/examples/_preludes/auto_migrate.py b/docs/examples/_preludes/auto_migrate.py new file mode 100644 index 00000000..523db7d6 --- /dev/null +++ b/docs/examples/_preludes/auto_migrate.py @@ -0,0 +1,56 @@ +"""The running example of the Auto-migrate page: ``Author`` after it gained +``nickname``, and the database from before. + +``opened()`` does not connect: each block on that page connects itself, with +the flag it is about. It runs the block in a scratch directory holding +``dev.db``, the SQLite file an earlier release built: ``author`` without +``nickname``, with a ``bio`` column the model has since dropped, and one row, +Ada Lovelace's. A block that wants a brand-new database connects to another +file in the same directory. +""" + +import os +import sqlite3 +import tempfile +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from pathlib import Path + +import ferro +from ferro import Field, Model, engines # noqa: F401 - names the blocks use + + +class Author(Model): + id: int | None = Field(default=None, primary_key=True) + name: str + email: str = Field(unique=True) + nickname: str | None = Field(default=None, index=True) + + +BEFORE = """ +CREATE TABLE "author" ( + "bio" varchar, + "email" varchar NOT NULL, + "id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, + "name" varchar NOT NULL +); +CREATE UNIQUE INDEX "uq_author_email" ON "author" ("email"); +INSERT INTO "author" ("name", "email", "bio") +VALUES ('Ada Lovelace', 'ada@example.com', 'Wrote the first program.'); +""" + + +@asynccontextmanager +async def opened() -> AsyncIterator[None]: + """Run the block in a scratch directory holding the earlier release's ``dev.db``.""" + cwd = os.getcwd() + with tempfile.TemporaryDirectory() as scratch: + connection = sqlite3.connect(Path(scratch) / "dev.db") + connection.executescript(BEFORE) + connection.close() + os.chdir(scratch) + try: + yield + finally: + ferro.reset_engine() + os.chdir(cwd) diff --git a/docs/examples/_preludes/concepts.py b/docs/examples/_preludes/concepts.py new file mode 100644 index 00000000..19257e67 --- /dev/null +++ b/docs/examples/_preludes/concepts.py @@ -0,0 +1,27 @@ +"""The running example of the Concepts pages. + +The ``User`` model declared in ``docs/examples/concepts.py``, with its four +users (ada, grace, alan and edsger, who is inactive), in an in-memory SQLite +database. A block marked ``{test=run prelude=concepts}`` runs inside +``opened()``'s session and can use ``User`` directly. +""" + +import sys +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from concepts import User, seed # noqa: E402,F401 + +from ferro import connect, engines # noqa: E402 + + +@asynccontextmanager +async def opened() -> AsyncIterator[None]: + """Connect, add the four users and hold a session open around the block.""" + await connect("sqlite::memory:", auto_migrate=True) + await seed() + async with engines.session(): + yield diff --git a/docs/examples/_preludes/connections.py b/docs/examples/_preludes/connections.py new file mode 100644 index 00000000..199d8255 --- /dev/null +++ b/docs/examples/_preludes/connections.py @@ -0,0 +1,35 @@ +"""The quickstart's models, with no connection: for the Connections guide. + +A block marked ``{test=run prelude=connections}`` can use ``Author`` and +``Post`` and connects on its own, because connecting and opening sessions is +what the Connections guide shows. ``opened()`` connects nothing. +""" + +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from datetime import UTC, datetime +from typing import Annotated + +from ferro import BackRef, Field, ForeignKey, Model, Relation + + +class Author(Model): + id: int | None = Field(default=None, primary_key=True) + name: str + email: str = Field(unique=True) + posts: Relation[list["Post"]] = BackRef() + + +class Post(Model): + id: int | None = Field(default=None, primary_key=True) + title: str + body: str + published: bool = False + created_at: datetime = Field(default_factory=lambda: datetime.now(UTC)) + author: Annotated[Author, ForeignKey(related_name="posts")] + + +@asynccontextmanager +async def opened() -> AsyncIterator[None]: + """Run the block as it stands: it connects and opens its own sessions.""" + yield diff --git a/docs/examples/_preludes/invoices.py b/docs/examples/_preludes/invoices.py new file mode 100644 index 00000000..284b9c14 --- /dev/null +++ b/docs/examples/_preludes/invoices.py @@ -0,0 +1,43 @@ +"""Invoices of two tenants, for the docstrings of session settings. + +A block marked ``{test=run prelude=invoices backend=postgres}`` runs on +PostgreSQL, where a session can send settings (``set_config``). ``opened()`` +connects, adds three invoices and does not open a session: the block opens +its own, with the settings it is about. ``resolve_tenant(request)`` stands for +the application's own lookup and returns ``"acme"``. +""" + +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager + +import ferro +from ferro import Field, Model, connect, engines + + +class Invoice(Model): + id: int | None = Field(default=None, primary_key=True) + tenant: str + total: float + + +request = object() +"""The incoming request a web handler would hold.""" + + +async def resolve_tenant(request: object) -> str: + """The tenant the request belongs to.""" + return "acme" + + +@asynccontextmanager +async def opened() -> AsyncIterator[None]: + """Connect and add the invoices; the block opens its own session.""" + await connect("postgresql://localhost/app", auto_migrate=True) + try: + async with engines.session(): + await Invoice.create(tenant="acme", total=250.0) + await Invoice.create(tenant="acme", total=40.0) + await Invoice.create(tenant="globex", total=900.0) + yield + finally: + ferro.reset_engine() diff --git a/docs/examples/_preludes/jwt_claims.py b/docs/examples/_preludes/jwt_claims.py new file mode 100644 index 00000000..48c88993 --- /dev/null +++ b/docs/examples/_preludes/jwt_claims.py @@ -0,0 +1,34 @@ +"""The Row-level security guide's JSON claims: a Postgres connection named +``"app"`` and the ``auth_uid()`` function the guide creates. + +A block marked ``{test=run prelude=jwt_claims backend=postgres}`` runs on +PostgreSQL. ``opened()`` connects under the name ``"app"`` and creates +``auth_uid()`` with the guide's own statement; the block opens its own +session, with the claims it sends. +""" + +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager + +import ferro +from ferro import engines, execute + +AUTH_UID = """\ +CREATE OR REPLACE FUNCTION auth_uid() RETURNS uuid +LANGUAGE sql STABLE +AS $$ + SELECT (NULLIF(current_setting('request.jwt.claims', true), '')::json ->> 'sub')::uuid +$$""" +"""The function the guide creates in a migration, as the guide shows it.""" + + +@asynccontextmanager +async def opened() -> AsyncIterator[None]: + """Connect as ``"app"`` and create ``auth_uid()``.""" + await ferro.connect("postgresql://localhost/app", name="app") + async with engines.session("app"): + await execute(AUTH_UID) + try: + yield + finally: + ferro.reset_engine() diff --git a/docs/examples/_preludes/label_addition.py b/docs/examples/_preludes/label_addition.py new file mode 100644 index 00000000..9d8d4378 --- /dev/null +++ b/docs/examples/_preludes/label_addition.py @@ -0,0 +1,44 @@ +"""The Auto-migrate page's label addition: ``Plan`` gained ``PRO``, and the +database is from before. + +The models are ``docs/examples/enum_labels.py``'s. A block marked +``{test=run prelude=label_addition backend=postgres}`` runs on PostgreSQL, +where ``opened()`` first builds what an earlier release built: the native +type ``plan`` with only ``'free'``, and ``author`` with a ``plan`` column of +it, as that release's create pass left them. It does not leave a +connection open: the block connects itself, with the flag it is about. +""" + +import sys +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +import ferro # noqa: E402 +from enum_labels import Author, Plan # noqa: E402, F401 - names the blocks use +from ferro import engines, execute, fetch_all # noqa: E402, F401 + +BEFORE = [ + """CREATE TYPE "plan" AS ENUM ('free')""", + """CREATE TABLE "author" ( "email" varchar NOT NULL, "id" serial PRIMARY KEY""" + """ NOT NULL, "name" varchar NOT NULL, "nickname" varchar, "plan" plan NOT NULL )""", + """CREATE INDEX "idx_author_nickname" ON "author" ("nickname")""", + """CREATE UNIQUE INDEX "uq_author_email" ON "author" ("email")""", +] +"""What the create pass sends for ``Author`` while ``Plan`` has only ``FREE``.""" + + +@asynccontextmanager +async def opened() -> AsyncIterator[None]: + """Build the earlier release's schema, then run the block.""" + await ferro.connect("postgresql://localhost/app") + async with engines.session(): + for statement in BEFORE: + await execute(statement) + ferro.reset_engine() + try: + yield + finally: + ferro.reset_engine() diff --git a/docs/examples/_preludes/queries.py b/docs/examples/_preludes/queries.py new file mode 100644 index 00000000..24041e9a --- /dev/null +++ b/docs/examples/_preludes/queries.py @@ -0,0 +1,40 @@ +"""The running example of the Queries guide and the typed-predicates page. + +The ledger declared in ``docs/examples/predicates.py``, with the rows of +``docs/examples/_ledger_seed.py``, in an in-memory SQLite database. A block +marked ``{test=run prelude=queries}`` runs inside ``opened()`` and can use +``Owner``, ``Account``, ``Category``, ``Transaction``, ``SplitLine``, +``Transfer``, ``Tag``, ``date`` and ``Decimal`` directly. +""" + +import sys +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +import predicates # noqa: E402 +from _ledger_seed import seed # noqa: E402 +from predicates import ( # noqa: E402,F401 + Account, + Category, + Owner, + SplitLine, + Tag, + Transaction, + Transfer, + date, + Decimal, +) + +from ferro import connect, engines # noqa: E402 + + +@asynccontextmanager +async def opened() -> AsyncIterator[None]: + """Connect, add the ledger's rows and hold a session open around the block.""" + await connect("sqlite::memory:", auto_migrate=True) + async with engines.session(): + await seed(vars(predicates)) + yield diff --git a/docs/examples/_preludes/raw_sql.py b/docs/examples/_preludes/raw_sql.py new file mode 100644 index 00000000..8e1f4fbc --- /dev/null +++ b/docs/examples/_preludes/raw_sql.py @@ -0,0 +1,35 @@ +"""The running example of the Raw SQL guide. + +The ``Event`` model declared in ``docs/examples/raw_sql.py``, with its three +events, in an in-memory SQLite database. A block marked +``{test=run prelude=raw_sql}`` runs inside ``opened()`` and can use +``Event``, ``execute``, ``fetch_all``, ``fetch_one`` and ``transaction`` +directly. +""" + +import sys +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from raw_sql import Event, add_events # noqa: E402,F401 + +from ferro import ( # noqa: E402,F401 + connect, + engines, + execute, + fetch_all, + fetch_one, + transaction, +) + + +@asynccontextmanager +async def opened() -> AsyncIterator[None]: + """Connect, add the three events and hold a session open around the block.""" + await connect("sqlite::memory:", auto_migrate=True) + async with engines.session(): + await add_events() + yield diff --git a/docs/examples/_preludes/reference.py b/docs/examples/_preludes/reference.py new file mode 100644 index 00000000..8d56de96 --- /dev/null +++ b/docs/examples/_preludes/reference.py @@ -0,0 +1,49 @@ +"""The running example for docstrings and the API reference. + +Two authors and their posts, in an in-memory SQLite database. A block marked +``{test=run prelude=reference}`` runs inside ``opened()`` and can use +``Author`` and ``Post`` directly. +""" + +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from typing import Annotated + +from ferro import BackRef, Field, ForeignKey, Model, Relation, connect, engines + + +class Author(Model): + id: int | None = Field(default=None, primary_key=True) + name: str + email: str = Field(unique=True) + posts: Relation[list["Post"]] = BackRef() + + +class Post(Model): + id: int | None = Field(default=None, primary_key=True) + title: str + published: bool = False + views: int = 0 + author: Annotated[Author, ForeignKey(related_name="posts")] + + +@asynccontextmanager +async def opened() -> AsyncIterator[None]: + """Connect, add the rows and hold a session open around the block.""" + await connect("sqlite::memory:", auto_migrate=True) + async with engines.session(): + ada = await Author.create(name="Ada Lovelace", email="ada@example.com") + grace = await Author.create(name="Grace Hopper", email="grace@example.com") + await Post.create( + title="Notes on the Analytical Engine", + published=True, + views=1843, + author=ada, + ) + await Post.create( + title="The Education of a Computer", + published=True, + views=952, + author=grace, + ) + yield diff --git a/docs/examples/_preludes/upgrade.py b/docs/examples/_preludes/upgrade.py new file mode 100644 index 00000000..d4511ca5 --- /dev/null +++ b/docs/examples/_preludes/upgrade.py @@ -0,0 +1,53 @@ +"""The running example of the Upgrade guide. + +The ``Account`` model of ``docs/examples/upgrade_account.py`` and four rows, in +an in-memory SQLite database. A block marked ``{test=run prelude=upgrade}`` runs +inside ``opened()`` and can use ``Account``, ``Role``, ``datetime``, ``UTC``, +``ferro`` and ``engines`` directly. + +Ada Lovelace (36, admin, last seen 2026-03-01), Grace Hopper (45, never seen), +Alan Turing (41, inactive, last seen 2026-02-01) and Linus Torvalds (28, last +seen 2026-03-05). Every other account is an active member. +""" + +import sys +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from datetime import UTC, datetime # noqa: F401 - names the blocks use +from pathlib import Path + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) + +from upgrade_account import Account, Role # noqa: E402,F401 + +import ferro # noqa: E402,F401 +from ferro import connect, engines # noqa: E402,F401 + + +@asynccontextmanager +async def opened() -> AsyncIterator[None]: + """Connect, add the four accounts and hold a session open around the block.""" + await connect("sqlite::memory:", auto_migrate=True) + async with engines.session(): + await Account.create( + name="Ada Lovelace", + email="ada@example.com", + role=Role.ADMIN, + age=36, + seen_at=datetime(2026, 3, 1, 15, 0, tzinfo=UTC), + ) + await Account.create(name="Grace Hopper", email="grace@example.com", age=45) + await Account.create( + name="Alan Turing", + email="alan@example.com", + age=41, + active=False, + seen_at=datetime(2026, 2, 1, 9, 30, tzinfo=UTC), + ) + await Account.create( + name="Linus Torvalds", + email="linus@example.com", + age=28, + seen_at=datetime(2026, 3, 5, 18, 0, tzinfo=UTC), + ) + yield diff --git a/docs/examples/aggregations.py b/docs/examples/aggregations.py index 90bf3cfb..4bc79add 100644 --- a/docs/examples/aggregations.py +++ b/docs/examples/aggregations.py @@ -1,15 +1,16 @@ -"""Runnable companion to the "Aggregations & Grouped Queries" guide +"""Runnable companion to the "Aggregations and grouped queries" guide (docs/pages/guide/aggregations.md). -Seeds a small transactions table, then runs every aggregate query the guide -shows and asserts the exact records that come back. +Seeds a small ledger of four transactions, then runs every aggregate query +the guide shows and asserts the exact records that come back. """ import asyncio +from collections import defaultdict from decimal import Decimal from typing import Annotated -from ferro import BackRef, Field, ForeignKey, Model, Relation, connect, engines +from ferro import BackRef, Field, ForeignKey, Model, Relation # --8<-- [start:schema] @@ -22,7 +23,7 @@ class Account(Model): class Transaction(Model): id: int | None = Field(default=None, primary_key=True) amount: int - price: Decimal | None = Field(default=None) + price: Decimal | None = None memo: str account: Annotated[Account | None, ForeignKey(related_name="transactions")] = None @@ -30,64 +31,72 @@ class Transaction(Model): # --8<-- [end:schema] +async def seed() -> None: + """Add the guide's rows. Call inside a session.""" + checking = await Account.create(id=1, label="Checking") + savings = await Account.create(id=2, label="Savings") + await Transaction.create( + id=1, amount=10, price=Decimal("1.25"), memo="coffee", account=checking + ) + await Transaction.create( + id=2, amount=20, price=Decimal("2.75"), memo="lunch", account=checking + ) + await Transaction.create( + id=3, amount=45, price=Decimal("5.00"), memo="dinner", account=savings + ) + await Transaction.create(id=4, amount=40, memo="orphan") # no account, no price + + async def main() -> None: - await connect("sqlite::memory:", auto_migrate=True) + # --8<-- [start:global] + from ferro import connect, engines + await connect("sqlite::memory:", auto_migrate=True) async with engines.session(): - a1 = await Account.create(id=1, label="a1") - b1 = await Account.create(id=2, label="b1") - await Transaction.create( - id=1, amount=10, price=Decimal("1.25"), memo="coffee", account=a1 - ) - await Transaction.create( - id=2, amount=20, price=Decimal("2.75"), memo="lunch", account=a1 - ) - await Transaction.create( - id=3, amount=30, price=Decimal("6.00"), memo="dinner", account=b1 - ) - await Transaction.create(id=4, amount=40, memo="orphan") # no account + await seed() # adds the ledger's four transactions - # --8<-- [start:global] - # An aggregate-only projection collapses to exactly one record: - # the global aggregate, read idiomatically with first(). + # An aggregate-only projection collapses to exactly one record, + # read idiomatically with first(). row = await Transaction.select( - lambda t: { - "n": t.id.count(), - "total": t.amount.sum(), - "average": t.amount.avg(), - "smallest": t.amount.min(), - "largest": t.amount.max(), + lambda transaction: { + "n": transaction.id.count(), + "total": transaction.amount.sum(), + "average": transaction.amount.avg(), + "smallest": transaction.amount.min(), + "largest": transaction.amount.max(), } ).first() assert row is not None assert row.model_dump() == { "n": 4, - "total": 100, - "average": 25.0, + "total": 115, + "average": 28.75, "smallest": 10, - "largest": 40, + "largest": 45, } # --8<-- [end:global] # --8<-- [start:global-where] - # Aggregates measure whatever where() leaves in — traversal included. + # Aggregates measure whatever where() leaves in, traversal included. row = await ( - Transaction.select(lambda t: {"total": t.amount.sum()}) - .where(lambda t: t.account.label == "a1") + Transaction.select(lambda transaction: {"total": transaction.amount.sum()}) + .where(lambda transaction: transaction.account.label == "Checking") .first() ) assert row is not None and row.total == 30 # --8<-- [end:global-where] # --8<-- [start:empty] - # Over zero matching rows, SQL's own empty-input semantics pass - # through verbatim: one record, None for sum/avg/min/max, 0 for - # count. No hidden COALESCE — "sum of no rows" and "sum of rows - # totaling zero" stay distinguishable. + # Over zero matching rows: one record, None for sum, 0 for count. row = await ( - Transaction.select(lambda t: {"n": t.id.count(), "total": t.amount.sum()}) - .where(lambda t: t.amount > 10_000) + Transaction.select( + lambda transaction: { + "n": transaction.id.count(), + "total": transaction.amount.sum(), + } + ) + .where(lambda transaction: transaction.amount > 10_000) .first() ) assert row is not None @@ -96,10 +105,13 @@ async def main() -> None: # --8<-- [end:empty] # --8<-- [start:count-nulls] - # COUNT(column) counts non-NULL values of that column — SQL - # semantics, verbatim. Only one of four transactions has a price... + # COUNT(column) counts the non-NULL values of that column. Three of + # the four transactions have a price; the orphan has none. row = await Transaction.select( - lambda t: {"rows": t.id.count(), "priced": t.price.count()} + lambda transaction: { + "rows": transaction.id.count(), + "priced": transaction.price.count(), + } ).first() assert row is not None assert row.rows == 4 @@ -107,148 +119,207 @@ async def main() -> None: # --8<-- [end:count-nulls] # --8<-- [start:grouped] - # Mix a plain field in and the query becomes GROUPED: every - # non-aggregate field is a group key. There is no group_by() — - # the record shape IS the grouping. + # Mix a plain field in and the query becomes grouped: every + # non-aggregate field is a group key. There is no group_by(). rows = await ( Transaction.select( - lambda t: {"acct": t.account_id, "total": t.amount.sum()} + lambda transaction: { + "acct": transaction.account_id, + "total": transaction.amount.sum(), + } ) - .where(lambda t: t.account_id != None) # noqa: E711 + .where(lambda transaction: transaction.account_id != None) .order_by("acct") .all() ) assert rows.model_dump() == [ {"acct": 1, "total": 30}, - {"acct": 2, "total": 30}, + {"acct": 2, "total": 45}, ] # --8<-- [end:grouped] # --8<-- [start:grouped-traversed] - # Group keys may traverse. Traversal narrows (the account-less - # transaction drops out) exactly like a where() predicate would. + # Group keys may traverse. The account-less transaction drops out, + # exactly as it would from a where() predicate on the same path. rows = await ( Transaction.select( - lambda t: {"account_label": t.account.label, "total": t.amount.sum()} + lambda transaction: { + "account_label": transaction.account.label, + "total": transaction.amount.sum(), + } ) .order_by("account_label") .all() ) assert rows.model_dump() == [ - {"account_label": "a1", "total": 30}, - {"account_label": "b1", "total": 30}, + {"account_label": "Checking", "total": 30}, + {"account_label": "Savings", "total": 45}, ] # --8<-- [end:grouped-traversed] # --8<-- [start:none-bucket] - # left_join() keeps relation-less rows, so "has no account" becomes - # a visible None-keyed group instead of a dropped row. + # left_join() keeps the account-less transaction, so "no account" + # becomes a group keyed None instead of a dropped row. rows = await ( Transaction.select( - lambda t: {"account_label": t.account.label, "total": t.amount.sum()} + lambda transaction: { + "account_label": transaction.account.label, + "total": transaction.amount.sum(), + } ) - .left_join(lambda t: t.account) + .left_join(lambda transaction: transaction.account) .all() ) buckets = {row.account_label: row.total for row in rows} - assert buckets == {"a1": 30, "b1": 30, None: 40} + assert buckets == {"Checking": 30, "Savings": 45, None: 40} # --8<-- [end:none-bucket] # --8<-- [start:top-n] - # The top-N idiom: keys + aggregates, order by the aggregate's - # output name, limit the groups. + # The top-N idiom: keys and aggregates, ordered by the aggregate's + # output name, with limit() applied to the groups. top = await ( Transaction.select( - lambda t: {"account_label": t.account.label, "total": t.amount.sum()} + lambda transaction: { + "account_label": transaction.account.label, + "total": transaction.amount.sum(), + } ) .order_by("total", "desc") .limit(1) .all() ) - assert len(top) == 1 + assert top.model_dump() == [{"account_label": "Savings", "total": 45}] # --8<-- [end:top-n] # --8<-- [start:order-by-lambda] - # order_by's lambda form spells the source expression — including - # the aggregate itself. + # order_by()'s lambda form spells the source expression, aggregate + # included. The largest total comes first. rows = await ( Transaction.select( - lambda t: {"acct": t.account_id, "total": t.amount.sum()} + lambda transaction: { + "acct": transaction.account_id, + "total": transaction.amount.sum(), + } ) - .where(lambda t: t.account_id != None) # noqa: E711 - .order_by(lambda t: t.amount.sum(), "desc") - .order_by("acct") + .where(lambda transaction: transaction.account_id != None) + .order_by(lambda transaction: transaction.amount.sum(), "desc") .all() ) - assert [row.acct for row in rows] == [1, 2] + assert [(row.acct, row.total) for row in rows] == [(2, 45), (1, 30)] # --8<-- [end:order-by-lambda] # --8<-- [start:zero-groups] - # A grouped query over zero rows returns zero records — unlike a - # global aggregate, there is no group to report on. + # A grouped query over zero rows returns zero records. rows = await ( Transaction.select( - lambda t: {"acct": t.account_id, "total": t.amount.sum()} + lambda transaction: { + "acct": transaction.account_id, + "total": transaction.amount.sum(), + } ) - .where(lambda t: t.amount > 10_000) + .where(lambda transaction: transaction.amount > 10_000) .all() ) assert len(rows) == 0 # --8<-- [end:zero-groups] # --8<-- [start:decimal-contract] - # Result types are a pinned cross-backend contract derived from the - # source column: avg over a Decimal column stays Decimal (never a - # silently lossy float), on SQLite and Postgres alike. + # avg() over a Decimal column stays Decimal, on SQLite and Postgres. row = await ( - Transaction.select(lambda t: {"avg_price": t.price.avg()}) - .where(lambda t: t.price != None) # noqa: E711 + Transaction.select( + lambda transaction: {"avg_price": transaction.price.avg()} + ) + .where(lambda transaction: transaction.price != None) .first() ) assert row is not None assert isinstance(row.avg_price, Decimal) + assert row.avg_price == Decimal("3") # --8<-- [end:decimal-contract] - # --8<-- [start:errors] - # The loud limits, all at build time — before any SQL: + # --8<-- [start:partition] + # Whole instances bucketed by a key: partition a full query in Python. + by_account: dict[int | None, list[Transaction]] = defaultdict(list) + for transaction in await Transaction.select().all(): + by_account[transaction.account_id].append(transaction) - # Aggregates over families with no portable cross-backend meaning. + sizes = {key: len(rows) for key, rows in by_account.items()} + assert sizes == {1: 2, 2: 1, None: 1} + # --8<-- [end:partition] + + # --8<-- [start:error-family] + # sum() over a string column has no portable meaning. try: - Transaction.select(lambda t: {"x": t.memo.sum()}) + Transaction.select(lambda transaction: {"x": transaction.memo.sum()}) except TypeError as exc: - assert "string-typed" in str(exc) + print(f"{type(exc).__name__}: {exc}") + else: + raise AssertionError("expected TypeError") + # --8<-- [end:error-family] - # Aggregates inside where() — filtering after aggregation is - # having(), which is not built yet. + # --8<-- [start:error-where] + # An aggregate inside where() raises and points at having(). try: - Transaction.where(lambda t: t.amount.sum() > 100) + Transaction.where(lambda transaction: transaction.amount.sum() > 100) + except TypeError as exc: + print(f"{type(exc).__name__}: {exc}") + else: + raise AssertionError("expected TypeError") + # --8<-- [end:error-where] + + # --8<-- [start:error-builtin-sum] + # Python's builtin sum() over a column reference raises. + try: + Transaction.select( + lambda transaction: { + "x": sum(transaction.amount) # type: ignore[arg-type] + } + ) except TypeError as exc: - assert "having()" in str(exc) + print(f"{type(exc).__name__}: {exc}") + else: + raise AssertionError("expected TypeError") + # --8<-- [end:error-builtin-sum] - # The builtin-sum trap: aggregation is a method on the column. + # --8<-- [start:error-unnamed] + # An aggregate outside the dict form has no output name. try: - Transaction.select(lambda t: {"x": sum(t.amount)}) # type: ignore[arg-type] + Transaction.select(lambda transaction: transaction.amount.sum()) except TypeError as exc: - assert "did you mean t.amount.sum()?" in str(exc) + print(f"{type(exc).__name__}: {exc}") + else: + raise AssertionError("expected TypeError") + # --8<-- [end:error-unnamed] - # Sorting a grouped query by a column that is not a group key would - # let each group answer with an arbitrary row's value. + # --8<-- [start:error-order-key] + # Sorting groups by a column that is not a group key raises. try: Transaction.select( - lambda t: {"acct": t.account_id, "total": t.amount.sum()} + lambda transaction: { + "acct": transaction.account_id, + "total": transaction.amount.sum(), + } ).order_by("memo") except ValueError as exc: - assert "group key or an aggregate" in str(exc) + print(f"{type(exc).__name__}: {exc}") + else: + raise AssertionError("expected ValueError") + # --8<-- [end:error-order-key] - # count()/exists() on an aggregate projection are ambiguous between - # rows and groups; both spellings are in the error. + # --8<-- [start:error-count] + # count() on an aggregate projection is ambiguous: rows or groups? try: Transaction.select( - lambda t: {"acct": t.account_id, "total": t.amount.sum()} + lambda transaction: { + "acct": transaction.account_id, + "total": transaction.amount.sum(), + } ).count() except ValueError as exc: - assert "len(await q.all())" in str(exc) - # --8<-- [end:errors] + print(f"{type(exc).__name__}: {exc}") + else: + raise AssertionError("expected ValueError") + # --8<-- [end:error-count] print("aggregations example ran successfully") diff --git a/docs/examples/aggregations_annotated.py b/docs/examples/aggregations_annotated.py index 4b8739ee..d764b64f 100644 --- a/docs/examples/aggregations_annotated.py +++ b/docs/examples/aggregations_annotated.py @@ -1,28 +1,29 @@ -"""Annotated-style companion to aggregations.py (AGENTS.md I-7). +"""Runnable companion to the "Aggregations and grouped queries" guide +(docs/pages/guide/aggregations.md). -Field options move into ``Annotated[...]``. Aggregation is independent of the -declaration style — the queries are identical in both — so only the schema and -one round-trip live here. +Seeds a small ledger of four transactions, then runs every aggregate query +the guide shows and asserts the exact records that come back. """ import asyncio +from collections import defaultdict from decimal import Decimal from typing import Annotated -from ferro import BackRef, Field, ForeignKey, Model, Relation, connect, engines +from ferro import BackRef, Field, ForeignKey, Model, Relation # --8<-- [start:schema] class Account(Model): - id: Annotated[int | None, Field(default=None, primary_key=True)] + id: Annotated[int | None, Field(primary_key=True)] = None label: str transactions: Relation[list["Transaction"]] = BackRef() class Transaction(Model): - id: Annotated[int | None, Field(default=None, primary_key=True)] + id: Annotated[int | None, Field(primary_key=True)] = None amount: int - price: Annotated[Decimal | None, Field(default=None)] + price: Decimal | None = None memo: str account: Annotated[Account | None, ForeignKey(related_name="transactions")] = None @@ -30,22 +31,297 @@ class Transaction(Model): # --8<-- [end:schema] +async def seed() -> None: + """Add the guide's rows. Call inside a session.""" + checking = await Account.create(id=1, label="Checking") + savings = await Account.create(id=2, label="Savings") + await Transaction.create( + id=1, amount=10, price=Decimal("1.25"), memo="coffee", account=checking + ) + await Transaction.create( + id=2, amount=20, price=Decimal("2.75"), memo="lunch", account=checking + ) + await Transaction.create( + id=3, amount=45, price=Decimal("5.00"), memo="dinner", account=savings + ) + await Transaction.create(id=4, amount=40, memo="orphan") # no account, no price + + async def main() -> None: - await connect("sqlite::memory:", auto_migrate=True) + # --8<-- [start:global] + from ferro import connect, engines + await connect("sqlite::memory:", auto_migrate=True) async with engines.session(): - a1 = await Account.create(id=1, label="a1") - await Transaction.create( - id=1, amount=10, price=Decimal("1.25"), memo="coffee", account=a1 + await seed() # adds the ledger's four transactions + + # An aggregate-only projection collapses to exactly one record, + # read idiomatically with first(). + row = await Transaction.select( + lambda transaction: { + "n": transaction.id.count(), + "total": transaction.amount.sum(), + "average": transaction.amount.avg(), + "smallest": transaction.amount.min(), + "largest": transaction.amount.max(), + } + ).first() + + assert row is not None + assert row.model_dump() == { + "n": 4, + "total": 115, + "average": 28.75, + "smallest": 10, + "largest": 45, + } + # --8<-- [end:global] + + # --8<-- [start:global-where] + # Aggregates measure whatever where() leaves in, traversal included. + row = await ( + Transaction.select(lambda transaction: {"total": transaction.amount.sum()}) + .where(lambda transaction: transaction.account.label == "Checking") + .first() + ) + assert row is not None and row.total == 30 + # --8<-- [end:global-where] + + # --8<-- [start:empty] + # Over zero matching rows: one record, None for sum, 0 for count. + row = await ( + Transaction.select( + lambda transaction: { + "n": transaction.id.count(), + "total": transaction.amount.sum(), + } + ) + .where(lambda transaction: transaction.amount > 10_000) + .first() + ) + assert row is not None + assert row.n == 0 + assert row.total is None + # --8<-- [end:empty] + + # --8<-- [start:count-nulls] + # COUNT(column) counts the non-NULL values of that column. Three of + # the four transactions have a price; the orphan has none. + row = await Transaction.select( + lambda transaction: { + "rows": transaction.id.count(), + "priced": transaction.price.count(), + } + ).first() + assert row is not None + assert row.rows == 4 + assert row.priced == 3 + # --8<-- [end:count-nulls] + + # --8<-- [start:grouped] + # Mix a plain field in and the query becomes grouped: every + # non-aggregate field is a group key. There is no group_by(). + rows = await ( + Transaction.select( + lambda transaction: { + "acct": transaction.account_id, + "total": transaction.amount.sum(), + } + ) + .where(lambda transaction: transaction.account_id != None) + .order_by("acct") + .all() + ) + assert rows.model_dump() == [ + {"acct": 1, "total": 30}, + {"acct": 2, "total": 45}, + ] + # --8<-- [end:grouped] + + # --8<-- [start:grouped-traversed] + # Group keys may traverse. The account-less transaction drops out, + # exactly as it would from a where() predicate on the same path. + rows = await ( + Transaction.select( + lambda transaction: { + "account_label": transaction.account.label, + "total": transaction.amount.sum(), + } + ) + .order_by("account_label") + .all() + ) + assert rows.model_dump() == [ + {"account_label": "Checking", "total": 30}, + {"account_label": "Savings", "total": 45}, + ] + # --8<-- [end:grouped-traversed] + + # --8<-- [start:none-bucket] + # left_join() keeps the account-less transaction, so "no account" + # becomes a group keyed None instead of a dropped row. + rows = await ( + Transaction.select( + lambda transaction: { + "account_label": transaction.account.label, + "total": transaction.amount.sum(), + } + ) + .left_join(lambda transaction: transaction.account) + .all() ) - await Transaction.create(id=2, amount=20, memo="lunch", account=a1) + buckets = {row.account_label: row.total for row in rows} + assert buckets == {"Checking": 30, "Savings": 45, None: 40} + # --8<-- [end:none-bucket] + + # --8<-- [start:top-n] + # The top-N idiom: keys and aggregates, ordered by the aggregate's + # output name, with limit() applied to the groups. + top = await ( + Transaction.select( + lambda transaction: { + "account_label": transaction.account.label, + "total": transaction.amount.sum(), + } + ) + .order_by("total", "desc") + .limit(1) + .all() + ) + assert top.model_dump() == [{"account_label": "Savings", "total": 45}] + # --8<-- [end:top-n] + + # --8<-- [start:order-by-lambda] + # order_by()'s lambda form spells the source expression, aggregate + # included. The largest total comes first. + rows = await ( + Transaction.select( + lambda transaction: { + "acct": transaction.account_id, + "total": transaction.amount.sum(), + } + ) + .where(lambda transaction: transaction.account_id != None) + .order_by(lambda transaction: transaction.amount.sum(), "desc") + .all() + ) + assert [(row.acct, row.total) for row in rows] == [(2, 45), (1, 30)] + # --8<-- [end:order-by-lambda] + + # --8<-- [start:zero-groups] + # A grouped query over zero rows returns zero records. + rows = await ( + Transaction.select( + lambda transaction: { + "acct": transaction.account_id, + "total": transaction.amount.sum(), + } + ) + .where(lambda transaction: transaction.amount > 10_000) + .all() + ) + assert len(rows) == 0 + # --8<-- [end:zero-groups] + + # --8<-- [start:decimal-contract] + # avg() over a Decimal column stays Decimal, on SQLite and Postgres. + row = await ( + Transaction.select( + lambda transaction: {"avg_price": transaction.price.avg()} + ) + .where(lambda transaction: transaction.price != None) + .first() + ) + assert row is not None + assert isinstance(row.avg_price, Decimal) + assert row.avg_price == Decimal("3") + # --8<-- [end:decimal-contract] + + # --8<-- [start:partition] + # Whole instances bucketed by a key: partition a full query in Python. + by_account: dict[int | None, list[Transaction]] = defaultdict(list) + for transaction in await Transaction.select().all(): + by_account[transaction.account_id].append(transaction) + + sizes = {key: len(rows) for key, rows in by_account.items()} + assert sizes == {1: 2, 2: 1, None: 1} + # --8<-- [end:partition] + + # --8<-- [start:error-family] + # sum() over a string column has no portable meaning. + try: + Transaction.select(lambda transaction: {"x": transaction.memo.sum()}) + except TypeError as exc: + print(f"{type(exc).__name__}: {exc}") + else: + raise AssertionError("expected TypeError") + # --8<-- [end:error-family] + + # --8<-- [start:error-where] + # An aggregate inside where() raises and points at having(). + try: + Transaction.where(lambda transaction: transaction.amount.sum() > 100) + except TypeError as exc: + print(f"{type(exc).__name__}: {exc}") + else: + raise AssertionError("expected TypeError") + # --8<-- [end:error-where] + + # --8<-- [start:error-builtin-sum] + # Python's builtin sum() over a column reference raises. + try: + Transaction.select( + lambda transaction: { + "x": sum(transaction.amount) # type: ignore[arg-type] + } + ) + except TypeError as exc: + print(f"{type(exc).__name__}: {exc}") + else: + raise AssertionError("expected TypeError") + # --8<-- [end:error-builtin-sum] + + # --8<-- [start:error-unnamed] + # An aggregate outside the dict form has no output name. + try: + Transaction.select(lambda transaction: transaction.amount.sum()) + except TypeError as exc: + print(f"{type(exc).__name__}: {exc}") + else: + raise AssertionError("expected TypeError") + # --8<-- [end:error-unnamed] + + # --8<-- [start:error-order-key] + # Sorting groups by a column that is not a group key raises. + try: + Transaction.select( + lambda transaction: { + "acct": transaction.account_id, + "total": transaction.amount.sum(), + } + ).order_by("memo") + except ValueError as exc: + print(f"{type(exc).__name__}: {exc}") + else: + raise AssertionError("expected ValueError") + # --8<-- [end:error-order-key] - rows = await Transaction.select( - lambda t: {"acct": t.account_id, "total": t.amount.sum()} - ).all() - assert rows.model_dump() == [{"acct": 1, "total": 30}] + # --8<-- [start:error-count] + # count() on an aggregate projection is ambiguous: rows or groups? + try: + Transaction.select( + lambda transaction: { + "acct": transaction.account_id, + "total": transaction.amount.sum(), + } + ).count() + except ValueError as exc: + print(f"{type(exc).__name__}: {exc}") + else: + raise AssertionError("expected ValueError") + # --8<-- [end:error-count] - print("aggregations_annotated example ran successfully") + print("aggregations example ran successfully") if __name__ == "__main__": diff --git a/docs/examples/auto_migrate_errors.py b/docs/examples/auto_migrate_errors.py new file mode 100644 index 00000000..37baa5a9 --- /dev/null +++ b/docs/examples/auto_migrate_errors.py @@ -0,0 +1,163 @@ +"""Runnable companion for the errors and warnings the Auto-migrate guide quotes +(docs/pages/guide/schema/auto-migrate.md, docs/pages/reference/configuration.md). + +Each region brings a database to the state the page describes, connects, and +prints what Ferro raises or warns. With no arguments it runs on SQLite files in +a temporary directory. ``python docs/examples/auto_migrate_errors.py `` +runs the Postgres scenario instead: a server default no model declares. +""" + +import asyncio +import os +import sys +import tempfile +import warnings +from enum import StrEnum +from pathlib import Path + +from ferro import ( + Field, + Model, + clear_registry, + connect, + engines, + execute, + reset_engine, +) + + +class Kind(StrEnum): + PERSON = "person" + COMPANY = "company" + + +def declare_author(release: str) -> type[Model]: + """Declare ``Author`` as the release ``release`` has it. + + Every release declares the same model, so each declaration replaces the + last, as a new deploy of the app would. + """ + reset_engine() + clear_registry() + if release == "base": + + class Author(Model): + id: int | None = Field(default=None, primary_key=True) + name: str + + elif release == "required-bio": + + class Author(Model): + id: int | None = Field(default=None, primary_key=True) + name: str + bio: str # required, with no default + + elif release == "kind": + + class Author(Model): + id: int | None = Field(default=None, primary_key=True) + name: str + kind: Kind = Field(default=Kind.PERSON, db_type="text") + + elif release == "checked-kind": + + class Author(Model): + id: int | None = Field(default=None, primary_key=True) + name: str + kind: Kind = Field(default=Kind.PERSON, db_type="text", db_check=True) + + elif release == "slug": + + class Author(Model): + id: int | None = Field(default=None, primary_key=True) + name: str + slug: str + + else: + raise ValueError(release) + return Author + + +async def required_column(url: str) -> None: + declare_author("base") + await connect(url, auto_migrate=True) + declare_author("required-bio") + # --8<-- [start:required-column] + try: + await connect(url, migrate_updates=True) + except ValueError as error: + print(f"{type(error).__name__}: {error}") + # --8<-- [end:required-column] + + +async def sqlite_check(url: str) -> None: + declare_author("kind") + await connect(url, auto_migrate=True) + declare_author("checked-kind") + # --8<-- [start:sqlite-check] + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + await connect(url, migrate_updates=True) + for warning in caught: + print(warning.message) + # --8<-- [end:sqlite-check] + + +async def run_lock(root: Path, url: str) -> None: + import fcntl + + declare_author("base") + (root / "ferro.toml").write_text( + 'models = ["__main__"]\ndialects = ["sqlite"]\nlock_timeout = "1s"\n' + ) + # Another process's migration run holds the SQLite run lock: an OS lock on + # the database's .ferro-migrate.lock file. + with open(root / "app.db.ferro-migrate.lock", "a+") as held: + fcntl.flock(held, fcntl.LOCK_EX | fcntl.LOCK_NB) + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + try: + await connect(url, auto_migrate=True) + except Exception as error: # noqa: BLE001 - printed below + gave_up = error + else: + raise AssertionError("the pass ran while the lock was held") + # --8<-- [start:lock-wait] + for warning in caught: + print(warning.message) + # --8<-- [end:lock-wait] + # --8<-- [start:lock-timeout] + print(gave_up) + # --8<-- [end:lock-timeout] + (root / "ferro.toml").unlink() + + +async def foreign_default(url: str) -> None: + declare_author("slug") + await connect(url, auto_migrate=True) + async with engines.session(): + await execute('ALTER TABLE "author" ALTER COLUMN "slug" SET DEFAULT \'\'') + declare_author("slug") + # --8<-- [start:foreign-default] + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + await connect(url, migrate_updates=True) + for warning in caught: + print(warning.message) + # --8<-- [end:foreign-default] + + +async def main(argv: list[str]) -> None: + if argv: + await foreign_default(argv[0]) + return + with tempfile.TemporaryDirectory() as tmp: + root = Path(tmp) + os.chdir(root) + await required_column(f"sqlite:{root / 'required.db'}?mode=rwc") + await sqlite_check(f"sqlite:{root / 'check.db'}?mode=rwc") + await run_lock(root, f"sqlite:{root / 'app.db'}?mode=rwc") + + +if __name__ == "__main__": + asyncio.run(main(sys.argv[1:])) diff --git a/docs/examples/concepts.py b/docs/examples/concepts.py new file mode 100644 index 00000000..b8a7a982 --- /dev/null +++ b/docs/examples/concepts.py @@ -0,0 +1,158 @@ +"""Runnable companion to the Concepts pages (docs/pages/concepts/). + +Declares the ``User`` model that the architecture, identity map, type safety +and performance pages share, adds four users and runs the blocks those pages +embed: the query the architecture page follows through each layer, one row +resolving to one instance, sessionless loads, a connection with the +identity map turned off, and the backends page's connect-and-query block. The ``concepts`` prelude +(``_preludes/concepts.py``) imports the model and the rows from here for the +pages' inline blocks. +""" + +import asyncio + +# --8<-- [start:models] +from ferro import Field, Model + + +class User(Model): + id: int | None = Field(default=None, primary_key=True) + username: str = Field(unique=True) + email: str + age: int + active: bool = True + + +# --8<-- [end:models] + +from ferro import connect, engines, reset_engine # noqa: E402 + +USERS = [ + {"username": "ada", "email": "ada@example.com", "age": 36}, + {"username": "grace", "email": "grace@example.com", "age": 45}, + {"username": "alan", "email": "alan@example.com", "age": 17}, + {"username": "edsger", "email": "edsger@example.com", "age": 52, "active": False}, +] + + +async def seed(model: type[User] = User) -> None: + """Add the four users, in their own session.""" + async with engines.session(): + await model.bulk_create([model(**row) for row in USERS]) + + +async def query() -> None: + # --8<-- [start:query] + from ferro import engines + + async with engines.session(): + users = await ( + User.where(lambda user: user.active == True) + .order_by(lambda user: user.age, "desc") + .limit(10) + .all() + ) + assert [user.username for user in users] == ["grace", "ada", "alan"] + # --8<-- [end:query] + + +async def refresh_on_load() -> None: + # --8<-- [start:refresh-on-load] + from ferro import engines, execute + + async with engines.session(): + ada = await User.get(1) + + # Something else (another request, a script, raw SQL) changes the + # row without going through `ada`. + await execute('UPDATE "user" SET email = ? WHERE id = ?', "ada@lovelace.dev", 1) + + again = await User.get(1) + assert again is ada # the same object + assert ada.email == "ada@lovelace.dev" # refreshed in place + + # An unsaved edit does not survive a re-fetch: the database wins. + ada.email = "unsaved edit" + await User.get(1) + assert ada.email == "ada@lovelace.dev" + # --8<-- [end:refresh-on-load] + + +async def sessionless() -> None: + # --8<-- [start:sessionless] + # `using=` alone: no session, so no identity map. + first = await User.using("default").get(2) + second = await User.using("default").get(2) + assert first is not second + assert first.email == second.email == "grace@example.com" + + # No session and no `using=`: Ferro has no route for the call. + try: + await User.get(2) + except RuntimeError as error: + assert str(error).startswith("No database route for this operation.") + else: + raise AssertionError("a call with no route must raise RuntimeError") + # --8<-- [end:sessionless] + + +async def without_identity_map() -> None: + reset_engine() + # --8<-- [start:opt-out] + from ferro import connect, engines + + await connect("sqlite::memory:", auto_migrate=True, identity_map=False) + + async with engines.session(): + ada = await User.create(username="ada", email="ada@example.com", age=36) + first = await User.get(ada.id) + second = await User.get(ada.id) + assert first is not ada + assert first is not second # no map, even inside a session + # --8<-- [end:opt-out] + + +async def backends(url: str = "sqlite::memory:") -> None: + reset_engine() + # --8<-- [start:backends] + from ferro import connect, engines + + await connect(url, auto_migrate=True) + + async with engines.session(): + await User.create(username="ada", email="ada@example.com", age=36) + adults = await User.where(lambda user: user.age >= 18).all() + assert [user.username for user in adults] == ["ada"] + # --8<-- [end:backends] + + +async def unsupported_scheme() -> None: + reset_engine() + # --8<-- [start:unsupported-scheme] + from ferro import InterfaceError, connect + + try: + await connect("mysql://user:password@localhost:3306/app") + except InterfaceError as error: + print(f"InterfaceError: {error}") + else: + raise AssertionError("expected InterfaceError") + # --8<-- [end:unsupported-scheme] + + +async def main() -> None: + # --8<-- [start:connect] + await connect("sqlite::memory:", auto_migrate=True) + # --8<-- [end:connect] + await seed() + await query() + await refresh_on_load() + await sessionless() + await without_identity_map() + await backends() + await unsupported_scheme() + print("concepts: ok") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/docs/examples/concepts_annotated.py b/docs/examples/concepts_annotated.py new file mode 100644 index 00000000..1bc47841 --- /dev/null +++ b/docs/examples/concepts_annotated.py @@ -0,0 +1,159 @@ +"""Runnable companion to the Concepts pages (docs/pages/concepts/). + +Declares the ``User`` model that the architecture, identity map, type safety +and performance pages share, adds four users and runs the blocks those pages +embed: the query the architecture page follows through each layer, one row +resolving to one instance, sessionless loads, a connection with the +identity map turned off, and the backends page's connect-and-query block. The ``concepts`` prelude +(``_preludes/concepts.py``) imports the model and the rows from here for the +pages' inline blocks. +""" + +import asyncio +from typing import Annotated + +# --8<-- [start:models] +from ferro import Field, Model + + +class User(Model): + id: Annotated[int | None, Field(primary_key=True)] = None + username: Annotated[str, Field(unique=True)] + email: str + age: int + active: bool = True + + +# --8<-- [end:models] + +from ferro import connect, engines, reset_engine # noqa: E402 + +USERS = [ + {"username": "ada", "email": "ada@example.com", "age": 36}, + {"username": "grace", "email": "grace@example.com", "age": 45}, + {"username": "alan", "email": "alan@example.com", "age": 17}, + {"username": "edsger", "email": "edsger@example.com", "age": 52, "active": False}, +] + + +async def seed(model: type[User] = User) -> None: + """Add the four users, in their own session.""" + async with engines.session(): + await model.bulk_create([model(**row) for row in USERS]) + + +async def query() -> None: + # --8<-- [start:query] + from ferro import engines + + async with engines.session(): + users = await ( + User.where(lambda user: user.active == True) + .order_by(lambda user: user.age, "desc") + .limit(10) + .all() + ) + assert [user.username for user in users] == ["grace", "ada", "alan"] + # --8<-- [end:query] + + +async def refresh_on_load() -> None: + # --8<-- [start:refresh-on-load] + from ferro import engines, execute + + async with engines.session(): + ada = await User.get(1) + + # Something else (another request, a script, raw SQL) changes the + # row without going through `ada`. + await execute('UPDATE "user" SET email = ? WHERE id = ?', "ada@lovelace.dev", 1) + + again = await User.get(1) + assert again is ada # the same object + assert ada.email == "ada@lovelace.dev" # refreshed in place + + # An unsaved edit does not survive a re-fetch: the database wins. + ada.email = "unsaved edit" + await User.get(1) + assert ada.email == "ada@lovelace.dev" + # --8<-- [end:refresh-on-load] + + +async def sessionless() -> None: + # --8<-- [start:sessionless] + # `using=` alone: no session, so no identity map. + first = await User.using("default").get(2) + second = await User.using("default").get(2) + assert first is not second + assert first.email == second.email == "grace@example.com" + + # No session and no `using=`: Ferro has no route for the call. + try: + await User.get(2) + except RuntimeError as error: + assert str(error).startswith("No database route for this operation.") + else: + raise AssertionError("a call with no route must raise RuntimeError") + # --8<-- [end:sessionless] + + +async def without_identity_map() -> None: + reset_engine() + # --8<-- [start:opt-out] + from ferro import connect, engines + + await connect("sqlite::memory:", auto_migrate=True, identity_map=False) + + async with engines.session(): + ada = await User.create(username="ada", email="ada@example.com", age=36) + first = await User.get(ada.id) + second = await User.get(ada.id) + assert first is not ada + assert first is not second # no map, even inside a session + # --8<-- [end:opt-out] + + +async def backends(url: str = "sqlite::memory:") -> None: + reset_engine() + # --8<-- [start:backends] + from ferro import connect, engines + + await connect(url, auto_migrate=True) + + async with engines.session(): + await User.create(username="ada", email="ada@example.com", age=36) + adults = await User.where(lambda user: user.age >= 18).all() + assert [user.username for user in adults] == ["ada"] + # --8<-- [end:backends] + + +async def unsupported_scheme() -> None: + reset_engine() + # --8<-- [start:unsupported-scheme] + from ferro import InterfaceError, connect + + try: + await connect("mysql://user:password@localhost:3306/app") + except InterfaceError as error: + print(f"InterfaceError: {error}") + else: + raise AssertionError("expected InterfaceError") + # --8<-- [end:unsupported-scheme] + + +async def main() -> None: + # --8<-- [start:connect] + await connect("sqlite::memory:", auto_migrate=True) + # --8<-- [end:connect] + await seed() + await query() + await refresh_on_load() + await sessionless() + await without_identity_map() + await backends() + await unsupported_scheme() + print("concepts: ok") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/docs/examples/enum_labels.py b/docs/examples/enum_labels.py new file mode 100644 index 00000000..620f6c98 --- /dev/null +++ b/docs/examples/enum_labels.py @@ -0,0 +1,46 @@ +# ruff: noqa: E402 - the models come first, as the docs page shows them +"""Runnable companion for label addition (docs/pages/guide/schema/auto-migrate.md). + +``Plan`` after it gained ``PRO``. The page's connect block runs these models +on PostgreSQL against a database whose ``plan`` type has only ``'free'`` +(``docs/examples/_preludes/enum_labels.py``). On its own the script saves an +author on SQLite, where an enum is stored as text and has no type to grow. +""" + +# --8<-- [start:models] +from enum import StrEnum + +from ferro import Field, Model + + +class Plan(StrEnum): + FREE = "free" + PRO = "pro" # new: the live type has only 'free' + + +class Author(Model): + id: int | None = Field(default=None, primary_key=True) + name: str + email: str = Field(unique=True) + nickname: str | None = Field(default=None, index=True) + plan: Plan = Plan.FREE + + +# --8<-- [end:models] + +import asyncio + +from ferro import connect, engines + + +async def main() -> None: + await connect("sqlite::memory:", auto_migrate=True) + async with engines.session(): + author = await Author.create( + name="Ada Lovelace", email="ada@example.com", plan=Plan.PRO + ) + assert (await Author.get(author.id)).plan is Plan.PRO + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/docs/examples/enum_labels_annotated.py b/docs/examples/enum_labels_annotated.py new file mode 100644 index 00000000..07bce26a --- /dev/null +++ b/docs/examples/enum_labels_annotated.py @@ -0,0 +1,47 @@ +# ruff: noqa: E402 - the models come first, as the docs page shows them +"""Runnable companion for label addition (docs/pages/guide/schema/auto-migrate.md). + +``Plan`` after it gained ``PRO``. The page's connect block runs these models +on PostgreSQL against a database whose ``plan`` type has only ``'free'`` +(``docs/examples/_preludes/enum_labels.py``). On its own the script saves an +author on SQLite, where an enum is stored as text and has no type to grow. +""" + +# --8<-- [start:models] +from enum import StrEnum +from typing import Annotated + +from ferro import Field, Model + + +class Plan(StrEnum): + FREE = "free" + PRO = "pro" # new: the live type has only 'free' + + +class Author(Model): + id: Annotated[int | None, Field(primary_key=True)] = None + name: str + email: Annotated[str, Field(unique=True)] + nickname: Annotated[str | None, Field(index=True)] = None + plan: Plan = Plan.FREE + + +# --8<-- [end:models] + +import asyncio + +from ferro import connect, engines + + +async def main() -> None: + await connect("sqlite::memory:", auto_migrate=True) + async with engines.session(): + author = await Author.create( + name="Ada Lovelace", email="ada@example.com", plan=Plan.PRO + ) + assert (await Author.get(author.id)).plan is Plan.PRO + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/docs/examples/existence_tests.py b/docs/examples/existence_tests.py index aaa6d0d7..01c3dbe9 100644 --- a/docs/examples/existence_tests.py +++ b/docs/examples/existence_tests.py @@ -1,211 +1,179 @@ -"""Runnable companion to the Existence Tests guide (docs/pages/guide/queries.md). +"""Runnable companion to "Existence tests on reverse & many-to-many +relations" in the Queries guide (docs/pages/guide/queries.md). -Seeds the transfer/split-line domain the guide walks through (one-to-one and -to-many BackRefs plus a category both layers point at) and a tagged-user M2M -pair, then runs every existence test the guide shows and asserts the exact -rows that come back. +Runs every existence test the guide shows against the guide's ledger +(declared in predicates.py, rows from _ledger_seed.py) and asserts the rows +that come back. """ import asyncio -from typing import Annotated -from ferro import ( - BackRef, - Field, - ForeignKey, - ManyToMany, - Model, - Relation, - connect, - engines, -) +import predicates +from _ledger_seed import seed +from predicates import Account, Category, Owner, Tag, Transaction - -# --8<-- [start:schema] -class Category(Model): - id: int | None = Field(default=None, primary_key=True) - name: str - transactions: Relation[list["Transaction"]] = BackRef() - lines: Relation[list["SplitLine"]] = BackRef() - - -class Transaction(Model): - id: int | None = Field(default=None, primary_key=True) - amount: int - category: Annotated[ - Category | None, ForeignKey(related_name="transactions", on_delete="SET NULL") - ] = None - # One-to-one BackRefs: a transfer references a transaction via a unique FK - transfer_out: "Transfer" = BackRef() - transfer_in: "Transfer" = BackRef() - # To-many BackRef: a split transaction carries its lines - lines: Relation[list["SplitLine"]] = BackRef() - - -class Transfer(Model): - id: int | None = Field(default=None, primary_key=True) - outflow_transaction: Annotated[ - Transaction | None, ForeignKey(related_name="transfer_out", unique=True) - ] = None - inflow_transaction: Annotated[ - Transaction | None, ForeignKey(related_name="transfer_in", unique=True) - ] = None - - -class SplitLine(Model): - id: int | None = Field(default=None, primary_key=True) - txn: Annotated[Transaction, ForeignKey(related_name="lines", on_delete="CASCADE")] - category: Annotated[ - Category | None, ForeignKey(related_name="lines", on_delete="SET NULL") - ] = None - amount: int = 0 - - -# --8<-- [end:schema] - - -# --8<-- [start:m2m-schema] -class Tag(Model): - id: int | None = Field(default=None, primary_key=True) - name: str - users: Relation[list["User"]] = ManyToMany(related_name="tags") - - -class User(Model): - id: int | None = Field(default=None, primary_key=True) - username: str - tags: Relation[list["Tag"]] = BackRef() - - -# --8<-- [end:m2m-schema] +from ferro import connect, engines async def main() -> None: await connect("sqlite::memory:", auto_migrate=True) async with engines.session(): - outflow = await Transaction.create(amount=-100) - inflow = await Transaction.create(amount=100) - plain = await Transaction.create(amount=-20) - assert plain.amount == -20 - await Transfer.create(outflow_transaction=outflow) - await Transfer.create(inflow_transaction=inflow) + await seed(vars(predicates)) - # --8<-- [start:bare] - # "Is this transaction part of any transfer?" — membership via either - # FK column, one EXISTS per side, each matching row exactly once. - in_transfer = await Transaction.where( - lambda t: t.transfer_out.exists() | t.transfer_in.exists() + # --8<-- [start:first] + split = await Transaction.where( + lambda transaction: transaction.lines.exists() ).all() + assert [transaction.memo for transaction in split] == ["Weekly shop"] + # --8<-- [end:first] - # The negated branch: ~ renders NOT EXISTS - not_in_transfer = await Transaction.where( - lambda t: ~t.transfer_out.exists() & ~t.transfer_in.exists() + # --8<-- [start:bare] + in_a_transfer = await Transaction.where( + lambda transaction: ( + transaction.transfer_out.exists() | transaction.transfer_in.exists() + ) ).all() + assert {transaction.memo for transaction in in_a_transfer} == { + "To savings", + "From checking", + } + + not_in_a_transfer = await Transaction.where( + lambda transaction: ( + ~transaction.transfer_out.exists() & ~transaction.transfer_in.exists() + ) + ).count() + assert not_in_a_transfer == 7 # --8<-- [end:bare] - assert {t.amount for t in in_transfer} == {-100, 100} - assert {t.amount for t in not_in_transfer} == {-20} - groceries = await Category.create(name="Groceries") - split = await Transaction.create(amount=-70) # category vacated while split - await SplitLine.create(txn=split, category=groceries, amount=-30) - await SplitLine.create(txn=split, category=groceries, amount=-40) - await Transaction.create(amount=-10, category=groceries) - ids = [groceries.id] + groceries = await Category.where( + lambda category: category.name == "Groceries" + ).first() + dining = await Category.where( + lambda category: category.name == "Dining" + ).first() + assert groceries is not None and dining is not None # --8<-- [start:scoped] - # The inner lambda is a full ferro predicate over the related model: - # "transactions carrying the category at the root OR on any line". - # A line-less transaction survives through the OR's root branch, and - # a transaction with several matching lines comes back exactly once. - matching = await Transaction.where( - lambda t: ( - t.category_id.in_(ids) - | t.lines.exists(lambda line: line.category_id.in_(ids)) + # Groceries on the transaction itself, or on any of its split lines. + grocery_spend = await Transaction.where( + lambda transaction: ( + (transaction.category_id == groceries.id) + | transaction.lines.exists( + lambda split_line: split_line.category_id == groceries.id + ) ) ).all() + assert {transaction.memo for transaction in grocery_spend} == { + "Market run", + "Weekly shop", + } # --8<-- [end:scoped] - assert {t.amount for t in matching} == {-70, -10} # --8<-- [start:composes] - # Root-shaped results: existence tests compose with every other - # predicate, ordering, and paging — nothing about the query changes. - page = ( + largest_grocery_spend = ( await Transaction.where( - lambda t: ( - t.category_id.in_(ids) - | t.lines.exists(lambda line: line.category_id.in_(ids)) + lambda transaction: ( + (transaction.category_id == groceries.id) + | transaction.lines.exists( + lambda split_line: split_line.category_id == groceries.id + ) ) ) - .order_by("amount", "desc") - .limit(1) - .all() + .order_by(lambda transaction: transaction.amount) + .first() ) + assert largest_grocery_spend is not None + assert largest_grocery_spend.memo == "Weekly shop" # --8<-- [end:composes] - assert [t.amount for t in page] == [-10] - - fuel_run = await Transaction.create(amount=-55) - await SplitLine.create(txn=fuel_run, category=groceries, amount=-55) - spread = await Transaction.create(amount=-60) - await SplitLine.create(txn=spread, category=groceries, amount=-5) - await SplitLine.create(txn=spread, amount=-55) # --8<-- [start:grouping] - # Grouping is YOUR choice, spelled explicitly — the two shapes below - # are different questions with different answers. - - # One line matches BOTH conditions: + # One split line that is both Dining and below -50: one_line_both = await Transaction.where( - lambda t: t.lines.exists( - lambda line: line.category_id.in_(ids) & (line.amount <= -50) + lambda transaction: transaction.lines.exists( + lambda split_line: ( + (split_line.category_id == dining.id) & (split_line.amount < -50) + ) ) ).all() + assert one_line_both == [] - # SOME line matches each condition (possibly different lines): + # A Dining line, and a line below -50 (here, two different lines): some_line_each = await Transaction.where( - lambda t: ( - t.lines.exists(lambda line: line.category_id.in_(ids)) - & t.lines.exists(lambda line: line.amount <= -50) + lambda transaction: ( + transaction.lines.exists( + lambda split_line: split_line.category_id == dining.id + ) + & transaction.lines.exists(lambda split_line: split_line.amount < -50) ) ).all() + assert [transaction.memo for transaction in some_line_each] == ["Weekly shop"] # --8<-- [end:grouping] - assert {t.amount for t in one_line_both} == {-55} - assert {t.amount for t in some_line_each} == {-55, -60} # --8<-- [start:traversal-inside] - # Forward traversal works inside the test (joins render INSIDE the - # EXISTS subquery, ADR-0006 semantics unchanged), and tests nest. - by_name = await Transaction.where( - lambda t: t.lines.exists(lambda line: line.category.name == "Groceries") - ).all() - - active_categories = await Category.where( - lambda c: c.lines.exists(lambda line: line.txn.amount < -50) + food_lines = await Transaction.where( + lambda transaction: transaction.lines.exists( + lambda split_line: split_line.category.parent.name == "Food" + ) ).all() + assert [transaction.memo for transaction in food_lines] == ["Weekly shop"] # --8<-- [end:traversal-inside] - assert {t.amount for t in by_name} == {-70, -55, -60} - assert {c.name for c in active_categories} == {"Groceries"} - admin = await Tag.create(name="admin") - beta = await Tag.create(name="beta") - alice = await User.create(username="alice") - bob = await User.create(username="bob") - await User.create(username="carol") - await admin.users.add(alice) - await beta.users.add(alice, bob) + # --8<-- [start:nested] + owners_with_split_spending = await Owner.where( + lambda owner: owner.accounts.exists( + lambda account: account.transactions.exists( + lambda transaction: transaction.lines.exists() + ) + ) + ).all() + assert [owner.name for owner in owners_with_split_spending] == ["Ada Lovelace"] + # --8<-- [end:nested] # --8<-- [start:m2m] - # Many-to-many spells identically — the test correlates through the - # join table, and the inner lambda scopes over the target model. - admins = await User.where( - lambda u: u.tags.exists(lambda tag: tag.name == "admin") + taxed = await Transaction.where( + lambda transaction: transaction.tags.exists(lambda tag: tag.name == "tax") ).all() - tagged = await User.where(lambda u: u.tags.exists()).all() - untagged = await User.where(lambda u: ~u.tags.exists()).all() + assert [transaction.memo for transaction in taxed] == ["Rent"] + + tags_on_uncleared = await Tag.where( + lambda tag: tag.transactions.exists( + lambda transaction: transaction.cleared_on == None + ) + ).all() + assert [tag.name for tag in tags_on_uncleared] == ["reimbursable"] + + untagged = await Transaction.where( + lambda transaction: ~transaction.tags.exists() + ).count() + assert untagged == 6 # --8<-- [end:m2m] - assert {u.username for u in admins} == {"alice"} - assert {u.username for u in tagged} == {"alice", "bob"} - assert {u.username for u in untagged} == {"carol"} + + # --8<-- [start:dead-ends] + try: + Transaction.where( + lambda transaction: transaction.lines.category_id == groceries.id + ) + except AttributeError as error: + assert "lines.exists(" in str(error) + else: + raise AssertionError("column access on a reverse relation must raise") + + try: + Transaction.where(lambda transaction: transaction.transfer_out != None) + except TypeError as error: + assert "transfer_out.exists()" in str(error) + else: + raise AssertionError("comparing a reverse relation must raise") + + try: + Account.select().join(lambda account: account.transactions) + except TypeError as error: + assert "transactions.exists(" in str(error) + else: + raise AssertionError("joining a reverse relation must raise") + # --8<-- [end:dead-ends] print("existence_tests example ran successfully") diff --git a/docs/examples/existence_tests_annotated.py b/docs/examples/existence_tests_annotated.py deleted file mode 100644 index 62e36b05..00000000 --- a/docs/examples/existence_tests_annotated.py +++ /dev/null @@ -1,116 +0,0 @@ -"""Annotated-style companion to existence_tests.py (AGENTS.md I-7).""" - -import asyncio -from typing import Annotated - -from ferro import ( - BackRef, - Field, - ForeignKey, - ManyToMany, - Model, - Relation, - connect, - engines, -) - - -# --8<-- [start:schema] -class Category(Model): - id: Annotated[int | None, Field(default=None, primary_key=True)] - name: str - transactions: Relation[list["Transaction"]] = BackRef() - lines: Relation[list["SplitLine"]] = BackRef() - - -class Transaction(Model): - id: Annotated[int | None, Field(default=None, primary_key=True)] - amount: int - category: Annotated[ - Category | None, ForeignKey(related_name="transactions", on_delete="SET NULL") - ] = None - # One-to-one BackRefs: a transfer references a transaction via a unique FK - transfer_out: "Transfer" = BackRef() - transfer_in: "Transfer" = BackRef() - # To-many BackRef: a split transaction carries its lines - lines: Relation[list["SplitLine"]] = BackRef() - - -class Transfer(Model): - id: Annotated[int | None, Field(default=None, primary_key=True)] - outflow_transaction: Annotated[ - Transaction | None, ForeignKey(related_name="transfer_out", unique=True) - ] = None - inflow_transaction: Annotated[ - Transaction | None, ForeignKey(related_name="transfer_in", unique=True) - ] = None - - -class SplitLine(Model): - id: Annotated[int | None, Field(default=None, primary_key=True)] - txn: Annotated[Transaction, ForeignKey(related_name="lines", on_delete="CASCADE")] - category: Annotated[ - Category | None, ForeignKey(related_name="lines", on_delete="SET NULL") - ] = None - amount: int = 0 - - -# --8<-- [end:schema] - - -# --8<-- [start:m2m-schema] -class Tag(Model): - id: Annotated[int | None, Field(default=None, primary_key=True)] - name: str - users: Relation[list["User"]] = ManyToMany(related_name="tags") - - -class User(Model): - id: Annotated[int | None, Field(default=None, primary_key=True)] - username: str - tags: Relation[list["Tag"]] = BackRef() - - -# --8<-- [end:m2m-schema] - - -async def main() -> None: - await connect("sqlite::memory:", auto_migrate=True) - - async with engines.session(): - outflow = await Transaction.create(amount=-100) - await Transaction.create(amount=-20) - await Transfer.create(outflow_transaction=outflow) - - in_transfer = await Transaction.where( - lambda t: t.transfer_out.exists() | t.transfer_in.exists() - ).all() - assert {t.amount for t in in_transfer} == {-100} - - groceries = await Category.create(name="Groceries") - split = await Transaction.create(amount=-70) - await SplitLine.create(txn=split, category=groceries, amount=-70) - - line_aware = await Transaction.where( - lambda t: ( - t.category_id.in_([groceries.id]) - | t.lines.exists(lambda line: line.category_id.in_([groceries.id])) - ) - ).all() - assert {t.amount for t in line_aware} == {-70} - - admin = await Tag.create(name="admin") - alice = await User.create(username="alice") - await User.create(username="bob") - await admin.users.add(alice) - - admins = await User.where( - lambda u: u.tags.exists(lambda tag: tag.name == "admin") - ).all() - assert {u.username for u in admins} == {"alice"} - - print("existence_tests_annotated example ran successfully") - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/docs/examples/faq.py b/docs/examples/faq.py new file mode 100644 index 00000000..501c5b2c --- /dev/null +++ b/docs/examples/faq.py @@ -0,0 +1,52 @@ +"""Runnable companion to the FAQ (docs/pages/faq.md): the Author and Post models.""" + +import asyncio + +# --8<-- [start:models] +from typing import Annotated + +from ferro import BackRef, Field, ForeignKey, Model, Relation, connect, engines + + +class Author(Model): + id: int | None = Field(default=None, primary_key=True) + name: str + email: str = Field(unique=True) + posts: Relation[list["Post"]] = BackRef() + + +class Post(Model): + id: int | None = Field(default=None, primary_key=True) + title: str + published: bool = False + views: int = 0 + author: Annotated[Author, ForeignKey(related_name="posts")] + + +# --8<-- [end:models] + + +async def main() -> None: + await connect("sqlite::memory:", auto_migrate=True) + async with engines.session(): + ada = await Author.create(name="Ada Lovelace", email="ada@example.com") + await Post.create(title="Notes", published=True, views=10, author=ada) + authors = await Author.where(lambda author: author.name == "Ada Lovelace").all() + assert [author.email for author in authors] == ["ada@example.com"] + posts = await Post.where(lambda post: post.views == 10).all() + assert len(posts) == 1 + + # --8<-- [start:databases] + await connect("sqlite::memory:", name="analytics", auto_migrate=True) + async with engines.session(): + authors = await Author.all() # the default connection + analytics_authors = await Author.using("analytics").all() # explicit routing + print(len(authors), len(analytics_authors)) # 1 0 + # --8<-- [end:databases] + assert (len(authors), len(analytics_authors)) == (1, 0) + + print("faq example ran successfully") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/docs/examples/faq_annotated.py b/docs/examples/faq_annotated.py new file mode 100644 index 00000000..38e40032 --- /dev/null +++ b/docs/examples/faq_annotated.py @@ -0,0 +1,52 @@ +"""Runnable companion to the FAQ (docs/pages/faq.md): the Author and Post models.""" + +import asyncio + +# --8<-- [start:models] +from typing import Annotated + +from ferro import BackRef, Field, ForeignKey, Model, Relation, connect, engines + + +class Author(Model): + id: Annotated[int | None, Field(primary_key=True)] = None + name: str + email: Annotated[str, Field(unique=True)] + posts: Relation[list["Post"]] = BackRef() + + +class Post(Model): + id: Annotated[int | None, Field(primary_key=True)] = None + title: str + published: bool = False + views: int = 0 + author: Annotated[Author, ForeignKey(related_name="posts")] + + +# --8<-- [end:models] + + +async def main() -> None: + await connect("sqlite::memory:", auto_migrate=True) + async with engines.session(): + ada = await Author.create(name="Ada Lovelace", email="ada@example.com") + await Post.create(title="Notes", published=True, views=10, author=ada) + authors = await Author.where(lambda author: author.name == "Ada Lovelace").all() + assert [author.email for author in authors] == ["ada@example.com"] + posts = await Post.where(lambda post: post.views == 10).all() + assert len(posts) == 1 + + # --8<-- [start:databases] + await connect("sqlite::memory:", name="analytics", auto_migrate=True) + async with engines.session(): + authors = await Author.all() # the default connection + analytics_authors = await Author.using("analytics").all() # explicit routing + print(len(authors), len(analytics_authors)) # 1 0 + # --8<-- [end:databases] + assert (len(authors), len(analytics_authors)) == (1, 0) + + print("faq example ran successfully") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/docs/examples/from_sqlalchemy.py b/docs/examples/from_sqlalchemy.py new file mode 100644 index 00000000..9c5c8856 --- /dev/null +++ b/docs/examples/from_sqlalchemy.py @@ -0,0 +1,107 @@ +"""Runnable companion to the SQLAlchemy migration how-to (docs/pages/howto/migrate-from-sqlalchemy.md).""" + +import asyncio + +from ferro import ModelDoesNotExist, connect, engines, transaction + +# --8<-- [start:models] +from typing import Annotated + +from ferro import BackRef, Field, ForeignKey, Model, Relation + + +class User(Model): + id: int | None = Field(default=None, primary_key=True) + username: str = Field(unique=True) + email: str + age: int + posts: Relation[list["Post"]] = BackRef() + + +class Post(Model): + id: int | None = Field(default=None, primary_key=True) + title: str + author: Annotated[User, ForeignKey(related_name="posts")] + + +# --8<-- [end:models] + + +async def main() -> None: + await connect("sqlite::memory:", auto_migrate=True) + + # --8<-- [start:session] + async with engines.session(): + ada = await User.create(username="ada", email="ada@example.com", age=36) + assert ada.id is not None # inserted now; the key is set + # --8<-- [end:session] + + await User.create(username="grace", email="grace@example.com", age=45) + await User.create(username="linus", email="linus@example.com", age=17) + await User.create(username="alan", email="alan@example.com", age=41) + + # --8<-- [start:fetch-all] + users = await User.all() + assert [user.username for user in users] == ["ada", "grace", "linus", "alan"] + # --8<-- [end:fetch-all] + + # --8<-- [start:filter] + adults = ( + await User.where(lambda user: user.age >= 18) + .order_by(lambda user: user.age) + .limit(2) + .all() + ) + assert [user.username for user in adults] == ["ada", "alan"] + # --8<-- [end:filter] + + # --8<-- [start:get] + grace = await User.get(2) + assert grace.username == "grace" + + try: + await User.get(99) + except ModelDoesNotExist: + missing = None + else: + raise AssertionError("User.get(99) should raise ModelDoesNotExist") + assert missing is None + + assert await User.get_or_none(99) is None + # --8<-- [end:get] + + # --8<-- [start:upserts] + user, created = await User.get_or_create( + username="ada", defaults={"email": "other@example.com", "age": 1} + ) + assert (created, user.id) == (False, ada.id) + # --8<-- [end:upserts] + + # --8<-- [start:relationships] + post = await Post.create(title="Notes on the engine", author=ada) + await Post.create(title="Poetical science", author=ada) + + author = await post.author + assert author.username == "ada" + + recent = await ada.posts.order_by(lambda post: post.id, "desc").limit(5).all() + assert [row.title for row in recent] == [ + "Poetical science", + "Notes on the engine", + ] + # --8<-- [end:relationships] + + # --8<-- [start:transactions] + try: + async with transaction(): + await User.create(username="margaret", email="m@example.com", age=30) + raise RuntimeError("abort") + except RuntimeError: + ... # the block raised, so the insert rolled back + + assert await User.where(lambda user: user.username == "margaret").count() == 0 + # --8<-- [end:transactions] + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/docs/examples/from_sqlalchemy_annotated.py b/docs/examples/from_sqlalchemy_annotated.py new file mode 100644 index 00000000..4f2863f8 --- /dev/null +++ b/docs/examples/from_sqlalchemy_annotated.py @@ -0,0 +1,107 @@ +"""Runnable companion to the SQLAlchemy migration how-to (docs/pages/howto/migrate-from-sqlalchemy.md).""" + +import asyncio + +from ferro import ModelDoesNotExist, connect, engines, transaction + +# --8<-- [start:models] +from typing import Annotated + +from ferro import BackRef, Field, ForeignKey, Model, Relation + + +class User(Model): + id: Annotated[int | None, Field(primary_key=True)] = None + username: Annotated[str, Field(unique=True)] + email: str + age: int + posts: Relation[list["Post"]] = BackRef() + + +class Post(Model): + id: Annotated[int | None, Field(primary_key=True)] = None + title: str + author: Annotated[User, ForeignKey(related_name="posts")] + + +# --8<-- [end:models] + + +async def main() -> None: + await connect("sqlite::memory:", auto_migrate=True) + + # --8<-- [start:session] + async with engines.session(): + ada = await User.create(username="ada", email="ada@example.com", age=36) + assert ada.id is not None # inserted now; the key is set + # --8<-- [end:session] + + await User.create(username="grace", email="grace@example.com", age=45) + await User.create(username="linus", email="linus@example.com", age=17) + await User.create(username="alan", email="alan@example.com", age=41) + + # --8<-- [start:fetch-all] + users = await User.all() + assert [user.username for user in users] == ["ada", "grace", "linus", "alan"] + # --8<-- [end:fetch-all] + + # --8<-- [start:filter] + adults = ( + await User.where(lambda user: user.age >= 18) + .order_by(lambda user: user.age) + .limit(2) + .all() + ) + assert [user.username for user in adults] == ["ada", "alan"] + # --8<-- [end:filter] + + # --8<-- [start:get] + grace = await User.get(2) + assert grace.username == "grace" + + try: + await User.get(99) + except ModelDoesNotExist: + missing = None + else: + raise AssertionError("User.get(99) should raise ModelDoesNotExist") + assert missing is None + + assert await User.get_or_none(99) is None + # --8<-- [end:get] + + # --8<-- [start:upserts] + user, created = await User.get_or_create( + username="ada", defaults={"email": "other@example.com", "age": 1} + ) + assert (created, user.id) == (False, ada.id) + # --8<-- [end:upserts] + + # --8<-- [start:relationships] + post = await Post.create(title="Notes on the engine", author=ada) + await Post.create(title="Poetical science", author=ada) + + author = await post.author + assert author.username == "ada" + + recent = await ada.posts.order_by(lambda post: post.id, "desc").limit(5).all() + assert [row.title for row in recent] == [ + "Poetical science", + "Notes on the engine", + ] + # --8<-- [end:relationships] + + # --8<-- [start:transactions] + try: + async with transaction(): + await User.create(username="margaret", email="m@example.com", age=30) + raise RuntimeError("abort") + except RuntimeError: + ... # the block raised, so the insert rolled back + + assert await User.where(lambda user: user.username == "margaret").count() == 0 + # --8<-- [end:transactions] + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/docs/examples/migration_errors.py b/docs/examples/migration_errors.py new file mode 100644 index 00000000..9391ad84 --- /dev/null +++ b/docs/examples/migration_errors.py @@ -0,0 +1,113 @@ +# ruff: noqa: E402, F811 - the models come first, before then after, as the other migrations examples +"""Runnable companion for the errors the migrations pages quote +(docs/pages/guide/schema/deploying.md, docs/pages/guide/schema/overview.md). + +A project on SQLite walks through the two-release change the Deploying guide +tabulates: release 1 adds a nullable ``slug`` (one expand step), release 2 +makes it required (backfill, add-constraint, contract). On the way it reaches +the states those errors describe: a database behind its migrations, one ahead +of its checkout by an expand step and then by a contract step, and a tracked +database that ``connect()`` is asked to auto-migrate. Each region prints the +error Ferro raises. +""" + +# --8<-- [start:models] +from ferro import Field, Model + + +class Author(Model): + id: int | None = Field(default=None, primary_key=True) + name: str + slug: str | None = None # new + + +# --8<-- [end:models] + +# --8<-- [start:slug-required] +from ferro import Field, Model + + +class Author(Model): + id: int | None = Field(default=None, primary_key=True) + name: str + slug: str + + +# --8<-- [end:slug-required] + +import asyncio +import os +import shutil +import sys + +import ferro +import ferro.migrations +from _migrations_project import project + +TODO = ' author.slug = todo("the slug for an existing author")\n' +WRITTEN = ' author.slug = author.name.lower().replace(" ", "-")\n' + + +async def main() -> None: + with project(__file__) as app: + app.models_before() + app.ferro("migrate", "new", "create_author") + app.ferro("migrate", "up") + # Release 1: a nullable slug, one expand step. + app.models_after() + app.ferro("migrate", "new", "author_slug_nullable") + os.chdir(app.root) + sys.path.insert(0, str(app.root)) + await ferro.connect(app.url) + + # --8<-- [start:behind] + try: + await ferro.migrations.require_applied() + except ferro.migrations.PendingMigrationsError as error: + print(f"{type(error).__name__}: {error}") + # --8<-- [end:behind] + + # Release 1 applies 0002; release 0's checkout lacks it. + app.ferro("migrate", "up") + release_1 = app.migrations / "0002_author_slug_nullable" + kept = app.root / release_1.name + shutil.move(release_1, kept) + + # --8<-- [start:ahead] + try: + await ferro.migrations.require_applied() + except ferro.migrations.DatabaseAheadError as error: + print(f"{type(error).__name__}: {error}") + # --8<-- [end:ahead] + + # An expand step lets release 0 through. + await ferro.migrations.require_applied(allow_ahead="expand-only") + + # Release 2 makes slug required; release 1's checkout lacks 0003. + shutil.move(kept, release_1) + app.models_region("slug-required") + app.ferro("migrate", "new", "author_slug_required") + release_2 = app.migrations / "0003_author_slug_required" + backfill = release_2 / "01_backfill_author.py" + backfill.write_text(backfill.read_text().replace(TODO, WRITTEN)) + app.ferro("migrate", "up") + shutil.rmtree(release_2) + + # --8<-- [start:ahead-by-contract] + try: + await ferro.migrations.require_applied(allow_ahead="expand-only") + except ferro.migrations.DatabaseAheadError as error: + print(f"{type(error).__name__}: {error}") + # --8<-- [end:ahead-by-contract] + + ferro.reset_engine() + # --8<-- [start:tracked] + try: + await ferro.connect(app.url, auto_migrate=True) + except ferro.migrations.MigrationRefused as error: + print(f"{type(error).__name__}: {error}") + # --8<-- [end:tracked] + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/docs/examples/migrations_adopting.py b/docs/examples/migrations_adopting.py index 6d7640a0..966b5b80 100644 --- a/docs/examples/migrations_adopting.py +++ b/docs/examples/migrations_adopting.py @@ -1,6 +1,6 @@ # ruff: noqa: E402 - the models come first, as the other migrations examples """Runnable companion for adopting migrations at start-up -(docs/pages/howto/adopting-migrations.md, "An app that migrates at start-up"). +(docs/pages/guide/schema/adopting-migrations.md, "An app that migrates at start-up"). The previous release built its SQLite file with auto-migrate; this release generates ``0001`` from the same models and runs the page's ``start()``: @@ -35,10 +35,7 @@ class Author(Model): async def start(database_url: str) -> None: await ferro.connect(database_url) try: - # A file the previous release built matches 0001 and is recorded; a - # brand-new, empty file matches nothing and stays unrecorded, and - # up() creates it. - await ferro.migrations.baseline(target="0001") + await ferro.migrations.baseline(target="0001") # the previous release's schema except ferro.migrations.AlreadyTrackedError: pass # already tracked: an earlier start, or another instance, got here first await ferro.migrations.up() diff --git a/docs/examples/migrations_backfill_annotated.py b/docs/examples/migrations_backfill_annotated.py index f551a0d9..7023203a 100644 --- a/docs/examples/migrations_backfill_annotated.py +++ b/docs/examples/migrations_backfill_annotated.py @@ -1,6 +1,5 @@ # ruff: noqa: E402 - the models come first, as the docs page shows them -"""Runnable companion for backfills and guards, Annotated style -(docs/pages/guide/schema/data-steps.md). +"""Runnable companion for backfills and guards (docs/pages/guide/schema/data-steps.md). A required column added to a table that has rows: the guard refuses, the backfill is written where it says todo(...), the late-row recovery is @@ -10,12 +9,12 @@ # --8<-- [start:models] from typing import Annotated -from ferro import FerroField, Model +from ferro import Field, Model class Author(Model): - id: Annotated[int | None, FerroField(primary_key=True)] = None - name: Annotated[str, FerroField(index=True)] + id: Annotated[int | None, Field(primary_key=True)] = None + name: Annotated[str, Field(index=True)] slug: str # new diff --git a/docs/examples/migrations_quickstart_annotated.py b/docs/examples/migrations_quickstart_annotated.py index 17709966..d0e04318 100644 --- a/docs/examples/migrations_quickstart_annotated.py +++ b/docs/examples/migrations_quickstart_annotated.py @@ -1,6 +1,5 @@ # ruff: noqa: E402 - the models come first, as the docs page shows them -"""Runnable companion for the Migrations loop, Annotated style -(docs/pages/guide/schema/migrations.md). +"""Runnable companion for the Migrations loop (docs/pages/guide/schema/migrations.md). init, new, up; then a model edit, check, new, up, drift, down, on SQLite. """ @@ -8,14 +7,14 @@ # --8<-- [start:models] from typing import Annotated -from ferro import FerroField, Model +from ferro import Field, Model class Author(Model): - id: Annotated[int | None, FerroField(primary_key=True)] = None + id: Annotated[int | None, Field(primary_key=True)] = None name: str - email: Annotated[str, FerroField(unique=True)] - nickname: Annotated[str | None, FerroField(index=True)] = None # new + email: Annotated[str, Field(unique=True)] + nickname: Annotated[str | None, Field(index=True)] = None # new # --8<-- [end:models] diff --git a/docs/examples/migrations_rename.py b/docs/examples/migrations_rename.py index 57e451d2..0d719f4b 100644 --- a/docs/examples/migrations_rename.py +++ b/docs/examples/migrations_rename.py @@ -1,10 +1,34 @@ -# ruff: noqa: E402 - the models come first, as the docs page shows them -"""Runnable companion for rename hints (docs/pages/guide/schema/migrations.md). +# ruff: noqa: E402, F811 - the models come first, as the docs page shows them: before, then after +"""Runnable companion for rename hints +(docs/pages/guide/schema/renames-and-retirements.md). A table, a column and an enum label renamed in one migration, applied over a row and reverted, on SQLite. """ +# --8<-- [start:base] +from enum import StrEnum + +from ferro import Field, Model + + +class Status(StrEnum): + ACTIVE = "active" + PAUSED = "paused" + CANCELED = "canceled" + + +class Author(Model): + id: int | None = Field(default=None, primary_key=True) + name: str + email: str = Field(unique=True) + nickname: str | None = Field(default=None, index=True) + bio: str = Field(index=True) + status: Status = Status.ACTIVE + + +# --8<-- [end:base] + # --8<-- [start:models] from enum import StrEnum @@ -13,16 +37,20 @@ class Status(StrEnum): __ferro_renamed_labels__ = {"cancelled": "canceled"} # new - DRAFT = "draft" + ACTIVE = "active" + PAUSED = "paused" CANCELLED = "cancelled" # before: CANCELED = "canceled" class Writer(Model): # before: class Author(Model): __ferro_renamed_from__ = "author" # new id: int | None = Field(default=None, primary_key=True) - # before: nickname: str = Field(index=True) - handle: str = Field(index=True, renamed_from="nickname") - status: Status = Status.DRAFT + name: str + email: str = Field(unique=True) + # before: nickname: str | None = Field(default=None, index=True) + handle: str | None = Field(default=None, index=True, renamed_from="nickname") + bio: str = Field(index=True) + status: Status = Status.ACTIVE # --8<-- [end:models] @@ -36,7 +64,10 @@ def main() -> None: app.models_before() app.ferro("migrate", "new", "create_author") app.ferro("migrate", "up") - app.sql("INSERT INTO author (nickname, status) VALUES ('ada', 'canceled')") + app.sql( + "INSERT INTO author (name, email, nickname, bio, status) VALUES" + " ('Ada Lovelace', 'ada@example.com', 'ada', 'Analyst', 'canceled')" + ) app.models_after() # the three rename hints written = app.ferro("migrate", "new", "rename_writer") diff --git a/docs/examples/migrations_rename_annotated.py b/docs/examples/migrations_rename_annotated.py index f8881be9..ebaaeab3 100644 --- a/docs/examples/migrations_rename_annotated.py +++ b/docs/examples/migrations_rename_annotated.py @@ -1,30 +1,58 @@ -# ruff: noqa: E402 - the models come first, as the docs page shows them -"""Runnable companion for rename hints, Annotated style -(docs/pages/guide/schema/migrations.md). +# ruff: noqa: E402, F811 - the models come first, as the docs page shows them: before, then after +"""Runnable companion for rename hints +(docs/pages/guide/schema/renames-and-retirements.md). A table, a column and an enum label renamed in one migration, applied over a row and reverted, on SQLite. """ +# --8<-- [start:base] +from enum import StrEnum +from typing import Annotated + +from ferro import Field, Model + + +class Status(StrEnum): + ACTIVE = "active" + PAUSED = "paused" + CANCELED = "canceled" + + +class Author(Model): + id: Annotated[int | None, Field(primary_key=True)] = None + name: str + email: Annotated[str, Field(unique=True)] + nickname: Annotated[str | None, Field(index=True)] = None + bio: Annotated[str, Field(index=True)] + status: Status = Status.ACTIVE + + +# --8<-- [end:base] + # --8<-- [start:models] from enum import StrEnum from typing import Annotated -from ferro import FerroField, Model +from ferro import Field, Model class Status(StrEnum): __ferro_renamed_labels__ = {"cancelled": "canceled"} # new - DRAFT = "draft" + ACTIVE = "active" + PAUSED = "paused" CANCELLED = "cancelled" # before: CANCELED = "canceled" class Writer(Model): # before: class Author(Model): __ferro_renamed_from__ = "author" # new - id: Annotated[int | None, FerroField(primary_key=True)] = None - # before: nickname: Annotated[str, FerroField(index=True)] - handle: Annotated[str, FerroField(index=True, renamed_from="nickname")] - status: Status = Status.DRAFT + id: Annotated[int | None, Field(primary_key=True)] = None + name: str + email: Annotated[str, Field(unique=True)] + # before: nickname: Annotated[str | None, Field(index=True)] = None + handle: Annotated[str | None, Field(index=True, renamed_from="nickname")] = None + bio: Annotated[str, Field(index=True)] + status: Status = Status.ACTIVE # --8<-- [end:models] @@ -38,7 +66,10 @@ def main() -> None: app.models_before() app.ferro("migrate", "new", "create_author") app.ferro("migrate", "up") - app.sql("INSERT INTO author (nickname, status) VALUES ('ada', 'canceled')") + app.sql( + "INSERT INTO author (name, email, nickname, bio, status) VALUES" + " ('Ada Lovelace', 'ada@example.com', 'ada', 'Analyst', 'canceled')" + ) app.models_after() # the three rename hints written = app.ferro("migrate", "new", "rename_writer") diff --git a/docs/examples/migrations_retire.py b/docs/examples/migrations_retire.py index 91b500c6..3f9a7aed 100644 --- a/docs/examples/migrations_retire.py +++ b/docs/examples/migrations_retire.py @@ -1,5 +1,6 @@ # ruff: noqa: E402 - the models come first, as the docs page shows them -"""Runnable companion for retiring a column (docs/pages/guide/schema/migrations.md). +"""Runnable companion for retiring a column +(docs/pages/guide/schema/renames-and-retirements.md). Release A declares ``bio`` required. Release B retires it: the migration only relaxes ``NOT NULL``, so A's code keeps writing ``bio`` while B's never names @@ -7,14 +8,25 @@ """ # --8<-- [start:models] +from enum import StrEnum + from ferro import Field, Model +class Status(StrEnum): + ACTIVE = "active" + PAUSED = "paused" + CANCELED = "canceled" + + class Author(Model): id: int | None = Field(default=None, primary_key=True) name: str + email: str = Field(unique=True) + nickname: str | None = Field(default=None, index=True) # before: bio: str = Field(index=True) - bio: str | None = Field(index=True, retired=True) + bio: str | None = Field(default=None, index=True, retired=True) + status: Status = Status.ACTIVE # --8<-- [end:models] @@ -29,7 +41,10 @@ def main() -> None: app.models_before() app.ferro("migrate", "new", "create_author") app.ferro("migrate", "up") - app.sql("INSERT INTO author (name, bio) VALUES ('ada', 'pioneer')") + app.sql( + "INSERT INTO author (name, email, bio, status) VALUES" + " ('Ada Lovelace', 'ada@example.com', 'Analyst', 'active')" + ) # Release B retires bio: one change, the relaxation. app.models_after() @@ -41,8 +56,14 @@ def main() -> None: app.ferro("migrate", "up") assert "no drift" in app.ferro("migrate", "drift") # Release A's write (names bio) and B's (does not) both succeed. - app.sql("INSERT INTO author (name, bio) VALUES ('grace', 'admiral')") - app.sql("INSERT INTO author (name) VALUES ('linus')") + app.sql( + "INSERT INTO author (name, email, bio, status) VALUES" + " ('Grace Hopper', 'grace@example.com', 'Admiral', 'active')" + ) + app.sql( + "INSERT INTO author (name, email, status) VALUES" + " ('Linus Torvalds', 'linus@example.com', 'active')" + ) # Release C deletes the declaration: the destructive drop. (app.root / "blog" / "models.py").write_text( @@ -61,9 +82,9 @@ def main() -> None: app.ferro("migrate", "up") assert "no drift" in app.ferro("migrate", "drift") assert app.sql("SELECT name FROM author ORDER BY id") == [ - ("ada",), - ("grace",), - ("linus",), + ("Ada Lovelace",), + ("Grace Hopper",), + ("Linus Torvalds",), ] diff --git a/docs/examples/migrations_retire_annotated.py b/docs/examples/migrations_retire_annotated.py index 1f4d0cf1..defb179c 100644 --- a/docs/examples/migrations_retire_annotated.py +++ b/docs/examples/migrations_retire_annotated.py @@ -1,6 +1,6 @@ # ruff: noqa: E402 - the models come first, as the docs page shows them -"""Runnable companion for retiring a column, Annotated style -(docs/pages/guide/schema/migrations.md). +"""Runnable companion for retiring a column +(docs/pages/guide/schema/renames-and-retirements.md). Release A declares ``bio`` required. Release B retires it: the migration only relaxes ``NOT NULL``, so A's code keeps writing ``bio`` while B's never names @@ -8,16 +8,26 @@ """ # --8<-- [start:models] +from enum import StrEnum from typing import Annotated -from ferro import FerroField, Model +from ferro import Field, Model + + +class Status(StrEnum): + ACTIVE = "active" + PAUSED = "paused" + CANCELED = "canceled" class Author(Model): - id: Annotated[int | None, FerroField(primary_key=True)] = None + id: Annotated[int | None, Field(primary_key=True)] = None name: str - # before: bio: Annotated[str, FerroField(index=True)] - bio: Annotated[str | None, FerroField(index=True, retired=True)] = None + email: Annotated[str, Field(unique=True)] + nickname: Annotated[str | None, Field(index=True)] = None + # before: bio: Annotated[str, Field(index=True)] + bio: Annotated[str | None, Field(index=True, retired=True)] = None + status: Status = Status.ACTIVE # --8<-- [end:models] @@ -32,7 +42,10 @@ def main() -> None: app.models_before() app.ferro("migrate", "new", "create_author") app.ferro("migrate", "up") - app.sql("INSERT INTO author (name, bio) VALUES ('ada', 'pioneer')") + app.sql( + "INSERT INTO author (name, email, bio, status) VALUES" + " ('Ada Lovelace', 'ada@example.com', 'Analyst', 'active')" + ) # Release B retires bio: one change, the relaxation. app.models_after() @@ -44,8 +57,14 @@ def main() -> None: app.ferro("migrate", "up") assert "no drift" in app.ferro("migrate", "drift") # Release A's write (names bio) and B's (does not) both succeed. - app.sql("INSERT INTO author (name, bio) VALUES ('grace', 'admiral')") - app.sql("INSERT INTO author (name) VALUES ('linus')") + app.sql( + "INSERT INTO author (name, email, bio, status) VALUES" + " ('Grace Hopper', 'grace@example.com', 'Admiral', 'active')" + ) + app.sql( + "INSERT INTO author (name, email, status) VALUES" + " ('Linus Torvalds', 'linus@example.com', 'active')" + ) # Release C deletes the declaration: the destructive drop. (app.root / "blog" / "models.py").write_text( @@ -64,9 +83,9 @@ def main() -> None: app.ferro("migrate", "up") assert "no drift" in app.ferro("migrate", "drift") assert app.sql("SELECT name FROM author ORDER BY id") == [ - ("ada",), - ("grace",), - ("linus",), + ("Ada Lovelace",), + ("Grace Hopper",), + ("Linus Torvalds",), ] diff --git a/docs/examples/migrations_retire_label.py b/docs/examples/migrations_retire_label.py index 79d7ddaf..83655159 100644 --- a/docs/examples/migrations_retire_label.py +++ b/docs/examples/migrations_retire_label.py @@ -1,8 +1,8 @@ # ruff: noqa: E402 - the models come first, as the docs page shows them """Runnable companion for retiring an enum label -(docs/pages/guide/schema/migrations.md). +(docs/pages/guide/schema/renames-and-retirements.md). -Release A writes ``canceled``. Release B retires it: the member stays so rows +Release A writes ``paused``. Release B retires it: the member stays so rows holding it still load, the label stays in the database, and the migration says "no schema change". On SQLite. """ @@ -14,15 +14,19 @@ class Status(StrEnum): - __ferro_retired_labels__ = frozenset({"canceled"}) # new - PAID = "paid" + __ferro_retired_labels__ = frozenset({"paused"}) # new + ACTIVE = "active" + PAUSED = "paused" CANCELED = "canceled" -class Order(Model): +class Author(Model): id: int | None = Field(default=None, primary_key=True) - status: Status = Status.PAID - note: str = "" + name: str + email: str = Field(unique=True) + nickname: str | None = Field(default=None, index=True) + bio: str = Field(index=True) + status: Status = Status.ACTIVE # --8<-- [end:models] @@ -34,18 +38,20 @@ class Order(Model): from _migrations_project import project PROBE = """ -from blog.models import Order, Status +from blog.models import Author, Status # A row release A wrote hydrates as the member and can change another field. -order = Order.model_construct(id=1, status=Status.CANCELED) -order.note = "b" -assert order.status is Status.CANCELED +author = Author.model_construct(id=1, status=Status.PAUSED) +author.bio = "Mathematician" +assert author.status is Status.PAUSED # This release does not write the label. try: - Order(status="canceled") + Author( + name="Grace Hopper", email="grace@example.com", bio="Admiral", status="paused" + ) except ValueError as error: - assert "Status.CANCELED ('canceled') is retired in this release" in str(error) + assert "Status.PAUSED ('paused') is retired in this release" in str(error) else: raise SystemExit("the retired label was accepted") """ @@ -53,22 +59,37 @@ class Order(Model): def main() -> None: with project(__file__) as app: - # Release A creates the table; its code writes "canceled". + # Release A creates the table; its code writes "paused". app.models_before() - app.ferro("migrate", "new", "create_order") + app.ferro("migrate", "new", "create_author") app.ferro("migrate", "up") - app.sql("INSERT INTO \"order\" (status, note) VALUES ('canceled', 'a')") + app.sql( + "INSERT INTO author (name, email, bio, status) VALUES" + " ('Ada Lovelace', 'ada@example.com', 'Analyst', 'paused')" + ) # Release B retires the label: nothing to migrate, nothing to check. app.models_after() - assert "no schema change" in app.ferro("migrate", "new", "retire_canceled") - assert not (app.migrations / "0002_retire_canceled").exists() + assert "no schema change" in app.ferro("migrate", "new", "retire_paused") + assert not (app.migrations / "0002_retire_paused").exists() app.ferro("migrate", "check") + # --8<-- [start:retired-write] + try: + Author( + name="Grace Hopper", + email="grace@example.com", + bio="Admiral", + status="paused", + ) + except ValueError as error: + print(f"{type(error).__name__}: {error}") + # --8<-- [end:retired-write] + subprocess.run( [sys.executable, "-c", PROBE], cwd=app.root, check=True, timeout=120 ) - assert app.sql('SELECT status FROM "order"') == [("canceled",)] + assert app.sql("SELECT status FROM author") == [("paused",)] if __name__ == "__main__": diff --git a/docs/examples/migrations_retire_label_annotated.py b/docs/examples/migrations_retire_label_annotated.py index e7fa3cca..ceca5d0a 100644 --- a/docs/examples/migrations_retire_label_annotated.py +++ b/docs/examples/migrations_retire_label_annotated.py @@ -1,8 +1,8 @@ # ruff: noqa: E402 - the models come first, as the docs page shows them -"""Runnable companion for retiring an enum label, Annotated style -(docs/pages/guide/schema/migrations.md). +"""Runnable companion for retiring an enum label +(docs/pages/guide/schema/renames-and-retirements.md). -Release A writes ``canceled``. Release B retires it: the member stays so rows +Release A writes ``paused``. Release B retires it: the member stays so rows holding it still load, the label stays in the database, and the migration says "no schema change". On SQLite. """ @@ -11,19 +11,23 @@ from enum import StrEnum from typing import Annotated -from ferro import FerroField, Model +from ferro import Field, Model class Status(StrEnum): - __ferro_retired_labels__ = frozenset({"canceled"}) # new - PAID = "paid" + __ferro_retired_labels__ = frozenset({"paused"}) # new + ACTIVE = "active" + PAUSED = "paused" CANCELED = "canceled" -class Order(Model): - id: Annotated[int | None, FerroField(primary_key=True)] = None - status: Annotated[Status, FerroField()] = Status.PAID - note: str = "" +class Author(Model): + id: Annotated[int | None, Field(primary_key=True)] = None + name: str + email: Annotated[str, Field(unique=True)] + nickname: Annotated[str | None, Field(index=True)] = None + bio: Annotated[str, Field(index=True)] + status: Status = Status.ACTIVE # --8<-- [end:models] @@ -35,18 +39,20 @@ class Order(Model): from _migrations_project import project PROBE = """ -from blog.models import Order, Status +from blog.models import Author, Status # A row release A wrote hydrates as the member and can change another field. -order = Order.model_construct(id=1, status=Status.CANCELED) -order.note = "b" -assert order.status is Status.CANCELED +author = Author.model_construct(id=1, status=Status.PAUSED) +author.bio = "Mathematician" +assert author.status is Status.PAUSED # This release does not write the label. try: - Order(status="canceled") + Author( + name="Grace Hopper", email="grace@example.com", bio="Admiral", status="paused" + ) except ValueError as error: - assert "Status.CANCELED ('canceled') is retired in this release" in str(error) + assert "Status.PAUSED ('paused') is retired in this release" in str(error) else: raise SystemExit("the retired label was accepted") """ @@ -54,22 +60,37 @@ class Order(Model): def main() -> None: with project(__file__) as app: - # Release A creates the table; its code writes "canceled". + # Release A creates the table; its code writes "paused". app.models_before() - app.ferro("migrate", "new", "create_order") + app.ferro("migrate", "new", "create_author") app.ferro("migrate", "up") - app.sql("INSERT INTO \"order\" (status, note) VALUES ('canceled', 'a')") + app.sql( + "INSERT INTO author (name, email, bio, status) VALUES" + " ('Ada Lovelace', 'ada@example.com', 'Analyst', 'paused')" + ) # Release B retires the label: nothing to migrate, nothing to check. app.models_after() - assert "no schema change" in app.ferro("migrate", "new", "retire_canceled") - assert not (app.migrations / "0002_retire_canceled").exists() + assert "no schema change" in app.ferro("migrate", "new", "retire_paused") + assert not (app.migrations / "0002_retire_paused").exists() app.ferro("migrate", "check") + # --8<-- [start:retired-write] + try: + Author( + name="Grace Hopper", + email="grace@example.com", + bio="Admiral", + status="paused", + ) + except ValueError as error: + print(f"{type(error).__name__}: {error}") + # --8<-- [end:retired-write] + subprocess.run( [sys.executable, "-c", PROBE], cwd=app.root, check=True, timeout=120 ) - assert app.sql('SELECT status FROM "order"') == [("canceled",)] + assert app.sql("SELECT status FROM author") == [("paused",)] if __name__ == "__main__": diff --git a/docs/examples/migrations_squash.py b/docs/examples/migrations_squash.py index 4f3c2018..ccb35c16 100644 --- a/docs/examples/migrations_squash.py +++ b/docs/examples/migrations_squash.py @@ -1,8 +1,8 @@ # ruff: noqa: E402, F811 - the models come first, as the docs page shows them, once per task -"""Runnable companion for squashing (docs/pages/guide/schema/migrations.md). +"""Runnable companion for squashing (docs/pages/guide/schema/squash.md). Two tasks on one branch: the first renames ``author.nickname`` to ``handle`` -with a rename hint, the second deletes the hint, now inert, and adds ``bio``. +with a rename hint, the second deletes the hint, now inert, and adds ``Team``. Squashed, the two migrations are one with one ``RENAME COLUMN``; a database that applied both inherits it and runs nothing; on SQLite. """ @@ -13,8 +13,10 @@ class Author(Model): id: int | None = Field(default=None, primary_key=True) - # before: nickname: str | None = None - handle: str | None = Field(default=None, renamed_from="nickname") + name: str + email: str = Field(unique=True) + # before: nickname: str | None = Field(default=None, index=True) + handle: str | None = Field(default=None, index=True, renamed_from="nickname") # --8<-- [end:models] @@ -25,8 +27,15 @@ class Author(Model): class Author(Model): id: int | None = Field(default=None, primary_key=True) - handle: str | None = None - bio: str | None = None + name: str + email: str = Field(unique=True) + handle: str | None = Field(default=None, index=True) + + +class Team(Model): + id: int | None = Field(default=None, primary_key=True) + name: str + slug: str # --8<-- [end:later] @@ -40,39 +49,40 @@ def main() -> None: app.models_before() app.ferro("migrate", "new", "create_author") app.ferro("migrate", "up") - app.sql("INSERT INTO author (nickname) VALUES ('ada')") + app.sql( + "INSERT INTO author (name, email, nickname) VALUES" + " ('Ada Lovelace', 'ada@example.com', 'ada')" + ) app.models_after() # task one: the rename hint app.ferro("migrate", "new", "rename_handle") app.ferro("migrate", "up") - app.models_region("later") # task two: the hint deleted, bio added - app.ferro("migrate", "new", "add_bio") + app.models_region("later") # task two: the hint deleted, Team added + app.ferro("migrate", "new", "add_teams") app.ferro("migrate", "up") - written = app.ferro("migrate", "squash", "author_profile", "--from", "0002") - assert "replaces 0002_rename_handle … 0003_add_bio, now deleted" in written + written = app.ferro("migrate", "squash", "teams", "--from", "0002") + assert "replaces 0002_rename_handle … 0003_add_teams, now deleted" in written assert sorted(path.name for path in app.migrations.glob("0*")) == [ "0001_create_author", - "0002_author_profile", + "0002_teams", ] - up = ( - app.migrations / "0002_author_profile" / "01_schema.up.postgres.sql" - ).read_text() + up = (app.migrations / "0002_teams" / "01_schema.up.postgres.sql").read_text() assert 'ALTER TABLE "author" RENAME COLUMN "nickname" TO "handle";' in up - assert 'ADD COLUMN "bio"' in up + assert 'CREATE TABLE IF NOT EXISTS "team"' in up assert "DROP COLUMN" not in up - # This database applied 0002_rename_handle and 0003_add_bio: it inherits. + # This database applied 0002_rename_handle and 0003_add_teams: it inherits. status = app.ferro("migrate", "status", exit_code=3) - assert "0002_author_profile pending (inheritable)" in status + assert "0002_teams pending (inheritable)" in status inherited = app.ferro("migrate", "up") assert ( - "inherited 0002_author_profile: this database had applied " - "0002_rename_handle … 0003_add_bio; nothing was run" + "inherited 0002_teams: this database had applied " + "0002_rename_handle … 0003_add_teams; nothing was run" ) in inherited assert "applied (inherited)" in app.ferro("migrate", "status") - assert "no drift against 0002_author_profile" in app.ferro("migrate", "drift") + assert "no drift against 0002_teams" in app.ferro("migrate", "drift") assert app.sql("SELECT handle FROM author") == [("ada",)] # An inherited migration's down runs: the rename is reverted. @@ -82,8 +92,9 @@ def main() -> None: # Run for real, the squash renames the column and keeps the row. app.ferro("migrate", "up") - assert "no drift against 0002_author_profile" in app.ferro("migrate", "drift") - assert app.sql("SELECT handle, bio FROM author") == [("ada", None)] + assert "no drift against 0002_teams" in app.ferro("migrate", "drift") + assert app.sql("SELECT handle FROM author") == [("ada",)] + assert app.sql("SELECT count(*) FROM team") == [(0,)] if __name__ == "__main__": diff --git a/docs/examples/migrations_squash_annotated.py b/docs/examples/migrations_squash_annotated.py index 124ac24f..5e2266df 100644 --- a/docs/examples/migrations_squash_annotated.py +++ b/docs/examples/migrations_squash_annotated.py @@ -1,9 +1,8 @@ # ruff: noqa: E402, F811 - the models come first, as the docs page shows them, once per task -"""Runnable companion for squashing, Annotated style -(docs/pages/guide/schema/migrations.md). +"""Runnable companion for squashing (docs/pages/guide/schema/squash.md). Two tasks on one branch: the first renames ``author.nickname`` to ``handle`` -with a rename hint, the second deletes the hint, now inert, and adds ``bio``. +with a rename hint, the second deletes the hint, now inert, and adds ``Team``. Squashed, the two migrations are one with one ``RENAME COLUMN``; a database that applied both inherits it and runs nothing; on SQLite. """ @@ -11,13 +10,15 @@ # --8<-- [start:models] from typing import Annotated -from ferro import FerroField, Model +from ferro import Field, Model class Author(Model): - id: Annotated[int | None, FerroField(primary_key=True)] = None - # before: nickname: str | None = None - handle: Annotated[str | None, FerroField(renamed_from="nickname")] = None + id: Annotated[int | None, Field(primary_key=True)] = None + name: str + email: Annotated[str, Field(unique=True)] + # before: nickname: Annotated[str | None, Field(index=True)] = None + handle: Annotated[str | None, Field(index=True, renamed_from="nickname")] = None # --8<-- [end:models] @@ -25,13 +26,20 @@ class Author(Model): # --8<-- [start:later] from typing import Annotated -from ferro import FerroField, Model +from ferro import Field, Model class Author(Model): - id: Annotated[int | None, FerroField(primary_key=True)] = None - handle: str | None = None - bio: str | None = None + id: Annotated[int | None, Field(primary_key=True)] = None + name: str + email: Annotated[str, Field(unique=True)] + handle: Annotated[str | None, Field(index=True)] = None + + +class Team(Model): + id: Annotated[int | None, Field(primary_key=True)] = None + name: str + slug: str # --8<-- [end:later] @@ -45,39 +53,40 @@ def main() -> None: app.models_before() app.ferro("migrate", "new", "create_author") app.ferro("migrate", "up") - app.sql("INSERT INTO author (nickname) VALUES ('ada')") + app.sql( + "INSERT INTO author (name, email, nickname) VALUES" + " ('Ada Lovelace', 'ada@example.com', 'ada')" + ) app.models_after() # task one: the rename hint app.ferro("migrate", "new", "rename_handle") app.ferro("migrate", "up") - app.models_region("later") # task two: the hint deleted, bio added - app.ferro("migrate", "new", "add_bio") + app.models_region("later") # task two: the hint deleted, Team added + app.ferro("migrate", "new", "add_teams") app.ferro("migrate", "up") - written = app.ferro("migrate", "squash", "author_profile", "--from", "0002") - assert "replaces 0002_rename_handle … 0003_add_bio, now deleted" in written + written = app.ferro("migrate", "squash", "teams", "--from", "0002") + assert "replaces 0002_rename_handle … 0003_add_teams, now deleted" in written assert sorted(path.name for path in app.migrations.glob("0*")) == [ "0001_create_author", - "0002_author_profile", + "0002_teams", ] - up = ( - app.migrations / "0002_author_profile" / "01_schema.up.postgres.sql" - ).read_text() + up = (app.migrations / "0002_teams" / "01_schema.up.postgres.sql").read_text() assert 'ALTER TABLE "author" RENAME COLUMN "nickname" TO "handle";' in up - assert 'ADD COLUMN "bio"' in up + assert 'CREATE TABLE IF NOT EXISTS "team"' in up assert "DROP COLUMN" not in up - # This database applied 0002_rename_handle and 0003_add_bio: it inherits. + # This database applied 0002_rename_handle and 0003_add_teams: it inherits. status = app.ferro("migrate", "status", exit_code=3) - assert "0002_author_profile pending (inheritable)" in status + assert "0002_teams pending (inheritable)" in status inherited = app.ferro("migrate", "up") assert ( - "inherited 0002_author_profile: this database had applied " - "0002_rename_handle … 0003_add_bio; nothing was run" + "inherited 0002_teams: this database had applied " + "0002_rename_handle … 0003_add_teams; nothing was run" ) in inherited assert "applied (inherited)" in app.ferro("migrate", "status") - assert "no drift against 0002_author_profile" in app.ferro("migrate", "drift") + assert "no drift against 0002_teams" in app.ferro("migrate", "drift") assert app.sql("SELECT handle FROM author") == [("ada",)] # An inherited migration's down runs: the rename is reverted. @@ -87,8 +96,9 @@ def main() -> None: # Run for real, the squash renames the column and keeps the row. app.ferro("migrate", "up") - assert "no drift against 0002_author_profile" in app.ferro("migrate", "drift") - assert app.sql("SELECT handle, bio FROM author") == [("ada", None)] + assert "no drift against 0002_teams" in app.ferro("migrate", "drift") + assert app.sql("SELECT handle FROM author") == [("ada",)] + assert app.sql("SELECT count(*) FROM team") == [(0,)] if __name__ == "__main__": diff --git a/docs/examples/migrations_testing.py b/docs/examples/migrations_testing.py index 1e55e423..a9026f81 100644 --- a/docs/examples/migrations_testing.py +++ b/docs/examples/migrations_testing.py @@ -1,8 +1,11 @@ -# ruff: noqa: E402 - the models come first, as the docs page shows them +# ruff: noqa: E402, F811 - the models come first, and each of the page's files imports what it uses """Runnable companion for testing migrations (docs/pages/guide/schema/testing.md). Generates two migrations (the second backfills a new required column), then -runs the page's two harness tests against fresh SQLite files. +runs the page's tests under pytest, in the project, the way a reader runs +them: its ``conftest.py`` fixture, its test module and its pytest settings. +Then it declares the backfill's down irreversible and runs the page's round +trip that stops there. """ # --8<-- [start:models] @@ -18,6 +21,21 @@ class Author(Model): # --8<-- [end:models] +# --8<-- [start:fixture] +import pytest + +import ferro + + +@pytest.fixture +def database_url(tmp_path): + yield f"sqlite:{tmp_path / 'test.db'}?mode=rwc" + ferro.reset_engine() # close the test's connection + + +# --8<-- [end:fixture] + + # --8<-- [start:tests] import ferro from ferro.migrations.testing import harness @@ -25,17 +43,17 @@ class Author(Model): async def test_the_slug_backfill(database_url): await ferro.connect(database_url) - h = harness() - await h.apply_through("0001") - async with h.models_at("0001") as models: # the table as 0001 left it + migrations = harness() + await migrations.apply_through("0001") + async with migrations.models_at("0001") as models: # the table as 0001 left it await models.Author(name="Ada Lovelace").save() - await h.apply("0002") # refused unless the database stands at 0001 - async with h.models_at("0002") as models: + await migrations.apply("0002") # refused unless the database stands at 0001 + async with migrations.models_at("0002") as models: ada = await models.Author.where( lambda author: author.name == "Ada Lovelace" ).first() assert ada.slug == "ada-lovelace" - await h.revert_to("0001") + await migrations.revert_to("0001") async def test_every_migration_round_trips(database_url): @@ -47,16 +65,58 @@ async def test_every_migration_round_trips(database_url): # --8<-- [end:tests] -import asyncio +# --8<-- [start:irreversible] +import ferro +from ferro.migrations.testing import RoundTripResult, harness + + +async def test_the_round_trip_stops_at_the_irreversible_step(database_url): + await ferro.connect(database_url) + result = await harness().round_trip() + assert result == RoundTripResult( + applied=["0001_create_author", "0002_author_slug"], + reverted_to="0002_author_slug", + irreversible=( + "0002_author_slug", + "02_backfill_author", + "the old slugs are gone", + ), + ) + + +# --8<-- [end:irreversible] + + import os +import subprocess +import sys +from pathlib import Path -from _migrations_project import project +from _migrations_project import _region, project TODO = ' author.slug = todo("the slug for an existing author")\n' WRITTEN = ' author.slug = author.name.lower().replace(" ", "-")\n' +NOTHING_TO_REVERSE = '@nothing_to_reverse("01_expand.down.sql drops the column")' +IRREVERSIBLE = '@irreversible("the old slugs are gone")' +PYTEST_SETTINGS = '[tool.pytest.ini_options]\nasyncio_mode = "auto"\n' + + +def run_pytest(app, *modules: str, passed: int) -> None: + """``pytest`` over ``modules`` in the project, as a reader runs it.""" + result = subprocess.run( + [sys.executable, "-m", "pytest", "-q", "-p", "no:cacheprovider", *modules], + capture_output=True, + text=True, + timeout=300, + cwd=app.root, + env={**os.environ, "PYTHONDONTWRITEBYTECODE": "1"}, + ) + assert result.returncode == 0, result.stdout + result.stderr + assert f"{passed} passed" in result.stdout, result.stdout def main() -> None: + script = Path(__file__) with project(__file__) as app: app.models_before() app.ferro("migrate", "new", "create_author") @@ -65,18 +125,24 @@ def main() -> None: backfill = app.migrations / "0002_author_slug" / "02_backfill_author.py" backfill.write_text(backfill.read_text().replace(TODO, WRITTEN)) - # The harness reads the project's config from the working directory, - # as it does under pytest run from the project root. - cwd = os.getcwd() - os.chdir(app.root) - try: - for number, test in enumerate( - (test_the_slug_backfill, test_every_migration_round_trips) - ): - asyncio.run(test(f"sqlite:{app.root / f'test{number}.db'}?mode=rwc")) - ferro.reset_engine() - finally: - os.chdir(cwd) + # The page's files, in the project: pytest runs from its root, where the + # harness reads the project's configuration, as every ferro command does. + (app.root / "pyproject.toml").write_text(PYTEST_SETTINGS) + (app.root / "conftest.py").write_text(_region(script, "fixture")) + (app.root / "test_migrations.py").write_text(_region(script, "tests")) + run_pytest(app, "test_migrations.py", passed=2) + + # A backfill whose down is declared irreversible ends the downward walk. + source = backfill.read_text() + assert NOTHING_TO_REVERSE in source + backfill.write_text( + source.replace(NOTHING_TO_REVERSE, IRREVERSIBLE).replace( + "import chunked, nothing_to_reverse, todo", + "import chunked, irreversible, todo", + ) + ) + (app.root / "test_round_trip.py").write_text(_region(script, "irreversible")) + run_pytest(app, "test_round_trip.py", passed=1) if __name__ == "__main__": diff --git a/docs/examples/migrations_testing_annotated.py b/docs/examples/migrations_testing_annotated.py index 3395c575..0336c55e 100644 --- a/docs/examples/migrations_testing_annotated.py +++ b/docs/examples/migrations_testing_annotated.py @@ -1,26 +1,43 @@ -# ruff: noqa: E402 - the models come first, as the docs page shows them -"""Runnable companion for testing migrations, Annotated style -(docs/pages/guide/schema/testing.md). +# ruff: noqa: E402, F811 - the models come first, and each of the page's files imports what it uses +"""Runnable companion for testing migrations (docs/pages/guide/schema/testing.md). Generates two migrations (the second backfills a new required column), then -runs the page's two harness tests against fresh SQLite files. +runs the page's tests under pytest, in the project, the way a reader runs +them: its ``conftest.py`` fixture, its test module and its pytest settings. +Then it declares the backfill's down irreversible and runs the page's round +trip that stops there. """ # --8<-- [start:models] from typing import Annotated -from ferro import FerroField, Model +from ferro import Field, Model class Author(Model): - id: Annotated[int | None, FerroField(primary_key=True)] = None - name: Annotated[str, FerroField(index=True)] + id: Annotated[int | None, Field(primary_key=True)] = None + name: Annotated[str, Field(index=True)] slug: str # new # --8<-- [end:models] +# --8<-- [start:fixture] +import pytest + +import ferro + + +@pytest.fixture +def database_url(tmp_path): + yield f"sqlite:{tmp_path / 'test.db'}?mode=rwc" + ferro.reset_engine() # close the test's connection + + +# --8<-- [end:fixture] + + # --8<-- [start:tests] import ferro from ferro.migrations.testing import harness @@ -28,17 +45,17 @@ class Author(Model): async def test_the_slug_backfill(database_url): await ferro.connect(database_url) - h = harness() - await h.apply_through("0001") - async with h.models_at("0001") as models: # the table as 0001 left it + migrations = harness() + await migrations.apply_through("0001") + async with migrations.models_at("0001") as models: # the table as 0001 left it await models.Author(name="Ada Lovelace").save() - await h.apply("0002") # refused unless the database stands at 0001 - async with h.models_at("0002") as models: + await migrations.apply("0002") # refused unless the database stands at 0001 + async with migrations.models_at("0002") as models: ada = await models.Author.where( lambda author: author.name == "Ada Lovelace" ).first() assert ada.slug == "ada-lovelace" - await h.revert_to("0001") + await migrations.revert_to("0001") async def test_every_migration_round_trips(database_url): @@ -50,16 +67,58 @@ async def test_every_migration_round_trips(database_url): # --8<-- [end:tests] -import asyncio +# --8<-- [start:irreversible] +import ferro +from ferro.migrations.testing import RoundTripResult, harness + + +async def test_the_round_trip_stops_at_the_irreversible_step(database_url): + await ferro.connect(database_url) + result = await harness().round_trip() + assert result == RoundTripResult( + applied=["0001_create_author", "0002_author_slug"], + reverted_to="0002_author_slug", + irreversible=( + "0002_author_slug", + "02_backfill_author", + "the old slugs are gone", + ), + ) + + +# --8<-- [end:irreversible] + + import os +import subprocess +import sys +from pathlib import Path -from _migrations_project import project +from _migrations_project import _region, project TODO = ' author.slug = todo("the slug for an existing author")\n' WRITTEN = ' author.slug = author.name.lower().replace(" ", "-")\n' +NOTHING_TO_REVERSE = '@nothing_to_reverse("01_expand.down.sql drops the column")' +IRREVERSIBLE = '@irreversible("the old slugs are gone")' +PYTEST_SETTINGS = '[tool.pytest.ini_options]\nasyncio_mode = "auto"\n' + + +def run_pytest(app, *modules: str, passed: int) -> None: + """``pytest`` over ``modules`` in the project, as a reader runs it.""" + result = subprocess.run( + [sys.executable, "-m", "pytest", "-q", "-p", "no:cacheprovider", *modules], + capture_output=True, + text=True, + timeout=300, + cwd=app.root, + env={**os.environ, "PYTHONDONTWRITEBYTECODE": "1"}, + ) + assert result.returncode == 0, result.stdout + result.stderr + assert f"{passed} passed" in result.stdout, result.stdout def main() -> None: + script = Path(__file__) with project(__file__) as app: app.models_before() app.ferro("migrate", "new", "create_author") @@ -68,18 +127,24 @@ def main() -> None: backfill = app.migrations / "0002_author_slug" / "02_backfill_author.py" backfill.write_text(backfill.read_text().replace(TODO, WRITTEN)) - # The harness reads the project's config from the working directory, - # as it does under pytest run from the project root. - cwd = os.getcwd() - os.chdir(app.root) - try: - for number, test in enumerate( - (test_the_slug_backfill, test_every_migration_round_trips) - ): - asyncio.run(test(f"sqlite:{app.root / f'test{number}.db'}?mode=rwc")) - ferro.reset_engine() - finally: - os.chdir(cwd) + # The page's files, in the project: pytest runs from its root, where the + # harness reads the project's configuration, as every ferro command does. + (app.root / "pyproject.toml").write_text(PYTEST_SETTINGS) + (app.root / "conftest.py").write_text(_region(script, "fixture")) + (app.root / "test_migrations.py").write_text(_region(script, "tests")) + run_pytest(app, "test_migrations.py", passed=2) + + # A backfill whose down is declared irreversible ends the downward walk. + source = backfill.read_text() + assert NOTHING_TO_REVERSE in source + backfill.write_text( + source.replace(NOTHING_TO_REVERSE, IRREVERSIBLE).replace( + "import chunked, nothing_to_reverse, todo", + "import chunked, irreversible, todo", + ) + ) + (app.root / "test_round_trip.py").write_text(_region(script, "irreversible")) + run_pytest(app, "test_round_trip.py", passed=1) if __name__ == "__main__": diff --git a/docs/examples/models_and_fields.py b/docs/examples/models_and_fields.py new file mode 100644 index 00000000..c6293645 --- /dev/null +++ b/docs/examples/models_and_fields.py @@ -0,0 +1,271 @@ +"""Runnable companion to the Models and fields guide (docs/pages/guide/models-and-fields.md). + +A product catalog. ``Product`` is the running example; ``Category``, +``Order`` and ``Review`` join it in the sections that need a model of their +own. Each SQL block on the page is a section of ``DDL`` below, and ``main()`` +checks ``DDL`` against the statements ``create_tables()`` ran. +""" + +# --8<-- [start:program] +import asyncio +from datetime import UTC, datetime +from decimal import Decimal +from enum import StrEnum +from typing import ClassVar + +from ferro import Field, Model, connect, create_tables, engines, varchar + + +class Status(StrEnum): + DRAFT = "draft" + ACTIVE = "active" + RETIRED = "retired" + + +class Product(Model): + # --8<-- [start:composite-uniques] + __ferro_composite_uniques__: ClassVar[tuple[tuple[str, ...], ...]] = ( + ("category", "name"), + ) + # --8<-- [end:composite-uniques] + # --8<-- [start:composite-indexes] + __ferro_composite_indexes__: ClassVar[tuple[tuple[str, ...], ...]] = ( + ("status", "created_at"), + ) + # --8<-- [end:composite-indexes] + + # --8<-- [start:primary-key] + id: int | None = Field(default=None, primary_key=True) + # --8<-- [end:primary-key] + sku: str = Field(unique=True) + # --8<-- [start:db-type] + name: str = Field(db_type=varchar(200), max_length=200) + # --8<-- [end:db-type] + category: str + # --8<-- [start:validation-fields] + price: Decimal = Field(ge=0, decimal_places=2) + # --8<-- [end:validation-fields] + # --8<-- [start:enum] + status: Status = Status.DRAFT + # --8<-- [end:enum] + # --8<-- [start:json] + attributes: dict[str, str] = Field(default_factory=dict) + tags: list[str] = Field(default_factory=list) + # --8<-- [end:json] + created_at: datetime = Field(default_factory=lambda: datetime.now(UTC)) + + +async def main() -> None: + await connect("sqlite::memory:") + report = await create_tables() + + async with engines.session(): + lamp = await Product.create( + sku="LMP-001", + name="Desk lamp", + category="lighting", + price=Decimal("39.90"), + ) + assert lamp.id == 1 # the database assigned it + assert lamp.status is Status.DRAFT + # --8<-- [end:program] + assert _statements(DDL) == sorted( + _normalized(statement.sql) for statement in report.statements + ) + await tour(lamp) + + print("models_and_fields example ran successfully") + + +import re # noqa: E402 +import uuid # noqa: E402 + +from pydantic import ValidationError # noqa: E402 + +from ferro import UniqueViolationError, fetch_one # noqa: E402 + + +# --8<-- [start:category] +class Category(Model): + # --8<-- [start:table-name] + __ferro_table__: ClassVar[str] = "categories" + # --8<-- [end:table-name] + + code: str = Field(primary_key=True) # a natural key, such as "lighting" + title: str + + +# --8<-- [end:category] + + +# --8<-- [start:order] +class Order(Model): + id: uuid.UUID = Field(default_factory=uuid.uuid4, primary_key=True) + total: Decimal + + +# --8<-- [end:order] + + +# --8<-- [start:base-class] +class Timestamped(Model): + created_at: datetime = Field(default_factory=lambda: datetime.now(UTC)) + + +class Review(Timestamped): + id: int | None = Field(default=None, primary_key=True) + product_sku: str + rating: int + + +# --8<-- [end:base-class] + + +async def tour(lamp: Product) -> None: + # --8<-- [start:save] + floor_lamp = Product( + sku="LMP-002", + name="Floor lamp", + category="lighting", + price=Decimal("129.00"), + ) + assert floor_lamp.id is None + await floor_lamp.save() + assert floor_lamp.id == 2 + # --8<-- [end:save] + + # --8<-- [start:json-round-trip] + shade = await Product.create( + sku="SHD-001", + name="Linen shade", + category="lighting", + price=Decimal("24.50"), + attributes={"colour": "oat", "fabric": "linen"}, + tags=["bedroom", "living room"], + ) + stored = await Product.where(lambda product: product.sku == "SHD-001").first() + assert stored.attributes == {"colour": "oat", "fabric": "linen"} + assert stored.tags == ["bedroom", "living room"] + # --8<-- [end:json-round-trip] + assert stored is shade + + # --8<-- [start:enum-value] + lamp.status = Status.ACTIVE + await lamp.save() + row = await fetch_one('SELECT "status" FROM "product" WHERE "id" = ?', lamp.id) + assert row == {"status": "active"} + # --8<-- [end:enum-value] + + await Category.create(code="lighting", title="Lighting") + assert (await Category.get("lighting")).title == "Lighting" + + # --8<-- [start:uuid-value] + order = await Order.create(total=Decimal("68.40")) + row = await fetch_one('SELECT "id" FROM "order"') + assert row == {"id": str(order.id)} # 36 characters, hyphens included + # --8<-- [end:uuid-value] + + # --8<-- [start:composite-unique-violation] + try: + await Product.create( + sku="LMP-003", + name="Desk lamp", + category="lighting", + price=Decimal("42.00"), + ) + except UniqueViolationError as exc: + assert "UNIQUE constraint failed: product.category, product.name" in str(exc) + # --8<-- [end:composite-unique-violation] + else: + raise AssertionError("expected UniqueViolationError") + + # --8<-- [start:validation] + try: + await Product.create( + sku="LMP-004", + name="Clamp lamp", + category="lighting", + price=Decimal("-5.00"), + ) + except ValidationError as exc: + [error] = exc.errors() + assert error["loc"] == ("price",) + assert error["msg"] == "Input should be greater than or equal to 0" + # --8<-- [end:validation] + else: + raise AssertionError("expected ValidationError") + + # --8<-- [start:base-class-rows] + await Review.create(product_sku="LMP-001", rating=5) + assert await Review.select().count() == 1 + assert await Timestamped.select().count() == 0 # a table of its own + # --8<-- [end:base-class-rows] + + +DDL = """ +# --8<-- [start:product-ddl] +# --8<-- [start:product-table] +CREATE TABLE IF NOT EXISTS "product" ( + "attributes" JSON NOT NULL, + "category" varchar NOT NULL, + "created_at" DATETIME NOT NULL, + "id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, +# --8<-- [start:name-column] + "name" varchar(200) NOT NULL, +# --8<-- [end:name-column] + "price" NUMERIC NOT NULL, + "sku" varchar NOT NULL, +# --8<-- [start:status-column] + "status" varchar(7) NOT NULL, +# --8<-- [end:status-column] + "tags" JSON NOT NULL +); +# --8<-- [end:product-table] +# --8<-- [start:sku-unique] +CREATE UNIQUE INDEX IF NOT EXISTS "uq_product_sku" ON "product" ("sku"); +# --8<-- [end:sku-unique] +# --8<-- [start:composite-unique-ddl] +CREATE UNIQUE INDEX IF NOT EXISTS "uq_product_category_name" ON "product" ("category", "name"); +# --8<-- [end:composite-unique-ddl] +# --8<-- [start:composite-index-ddl] +CREATE INDEX IF NOT EXISTS "idx_product_status_created_at" ON "product" ("status", "created_at"); +# --8<-- [end:composite-index-ddl] +# --8<-- [end:product-ddl] +# --8<-- [start:category-ddl] +CREATE TABLE IF NOT EXISTS "categories" ( + "code" varchar NOT NULL PRIMARY KEY, + "title" varchar NOT NULL +); +# --8<-- [end:category-ddl] +# --8<-- [start:order-ddl] +CREATE TABLE IF NOT EXISTS "order" ( + "id" CHAR(32) NOT NULL PRIMARY KEY, + "total" NUMERIC NOT NULL +); +# --8<-- [end:order-ddl] +# --8<-- [start:base-class-ddl] +CREATE TABLE IF NOT EXISTS "timestamped" ( + "created_at" DATETIME NOT NULL +); +CREATE TABLE IF NOT EXISTS "review" ( + "created_at" DATETIME NOT NULL, + "id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, + "product_sku" varchar NOT NULL, + "rating" integer NOT NULL +); +# --8<-- [end:base-class-ddl] +""" + + +def _normalized(statement: str) -> str: + return re.sub(r"\s+", " ", statement).strip() + + +def _statements(ddl: str) -> list[str]: + """The statements of ``ddl``, snippet markers left out, one line each.""" + text = "\n".join(line for line in ddl.splitlines() if "--8<--" not in line) + return sorted(_normalized(sql) for sql in text.split(";") if sql.strip()) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/docs/examples/models_and_fields_annotated.py b/docs/examples/models_and_fields_annotated.py new file mode 100644 index 00000000..bc131371 --- /dev/null +++ b/docs/examples/models_and_fields_annotated.py @@ -0,0 +1,274 @@ +"""Runnable companion to the Models and fields guide (docs/pages/guide/models-and-fields.md). + +A product catalog. ``Product`` is the running example; ``Category``, +``Order`` and ``Review`` join it in the sections that need a model of their +own. Each SQL block on the page is a section of ``DDL`` below, and ``main()`` +checks ``DDL`` against the statements ``create_tables()`` ran. +""" + +# --8<-- [start:program] +import asyncio +from datetime import UTC, datetime +from decimal import Decimal +from enum import StrEnum +from typing import Annotated, ClassVar + +from ferro import Field, Model, connect, create_tables, engines, varchar + + +class Status(StrEnum): + DRAFT = "draft" + ACTIVE = "active" + RETIRED = "retired" + + +class Product(Model): + # --8<-- [start:composite-uniques] + __ferro_composite_uniques__: ClassVar[tuple[tuple[str, ...], ...]] = ( + ("category", "name"), + ) + # --8<-- [end:composite-uniques] + # --8<-- [start:composite-indexes] + __ferro_composite_indexes__: ClassVar[tuple[tuple[str, ...], ...]] = ( + ("status", "created_at"), + ) + # --8<-- [end:composite-indexes] + + # --8<-- [start:primary-key] + id: Annotated[int | None, Field(primary_key=True)] = None + # --8<-- [end:primary-key] + sku: Annotated[str, Field(unique=True)] + # --8<-- [start:db-type] + name: Annotated[str, Field(db_type=varchar(200), max_length=200)] + # --8<-- [end:db-type] + category: str + # --8<-- [start:validation-fields] + price: Annotated[Decimal, Field(ge=0, decimal_places=2)] + # --8<-- [end:validation-fields] + # --8<-- [start:enum] + status: Status = Status.DRAFT + # --8<-- [end:enum] + # --8<-- [start:json] + attributes: dict[str, str] = Field(default_factory=dict) + tags: list[str] = Field(default_factory=list) + # --8<-- [end:json] + created_at: datetime = Field(default_factory=lambda: datetime.now(UTC)) + + +async def main() -> None: + await connect("sqlite::memory:") + report = await create_tables() + + async with engines.session(): + lamp = await Product.create( + sku="LMP-001", + name="Desk lamp", + category="lighting", + price=Decimal("39.90"), + ) + assert lamp.id == 1 # the database assigned it + assert lamp.status is Status.DRAFT + # --8<-- [end:program] + assert _statements(DDL) == sorted( + _normalized(statement.sql) for statement in report.statements + ) + await tour(lamp) + + print("models_and_fields example ran successfully") + + +import re # noqa: E402 +import uuid # noqa: E402 +from typing import Annotated # noqa: E402 + +from pydantic import ValidationError # noqa: E402 + +from ferro import UniqueViolationError, fetch_one # noqa: E402 + + +# --8<-- [start:category] +class Category(Model): + # --8<-- [start:table-name] + __ferro_table__: ClassVar[str] = "categories" + # --8<-- [end:table-name] + + code: Annotated[str, Field(primary_key=True)] # a natural key, such as "lighting" + title: str + + +# --8<-- [end:category] + + +# --8<-- [start:order] +class Order(Model): + id: Annotated[uuid.UUID, Field(primary_key=True)] = Field( + default_factory=uuid.uuid4 + ) + total: Decimal + + +# --8<-- [end:order] + + +# --8<-- [start:base-class] +class Timestamped(Model): + created_at: datetime = Field(default_factory=lambda: datetime.now(UTC)) + + +class Review(Timestamped): + id: Annotated[int | None, Field(primary_key=True)] = None + product_sku: str + rating: int + + +# --8<-- [end:base-class] + + +async def tour(lamp: Product) -> None: + # --8<-- [start:save] + floor_lamp = Product( + sku="LMP-002", + name="Floor lamp", + category="lighting", + price=Decimal("129.00"), + ) + assert floor_lamp.id is None + await floor_lamp.save() + assert floor_lamp.id == 2 + # --8<-- [end:save] + + # --8<-- [start:json-round-trip] + shade = await Product.create( + sku="SHD-001", + name="Linen shade", + category="lighting", + price=Decimal("24.50"), + attributes={"colour": "oat", "fabric": "linen"}, + tags=["bedroom", "living room"], + ) + stored = await Product.where(lambda product: product.sku == "SHD-001").first() + assert stored.attributes == {"colour": "oat", "fabric": "linen"} + assert stored.tags == ["bedroom", "living room"] + # --8<-- [end:json-round-trip] + assert stored is shade + + # --8<-- [start:enum-value] + lamp.status = Status.ACTIVE + await lamp.save() + row = await fetch_one('SELECT "status" FROM "product" WHERE "id" = ?', lamp.id) + assert row == {"status": "active"} + # --8<-- [end:enum-value] + + await Category.create(code="lighting", title="Lighting") + assert (await Category.get("lighting")).title == "Lighting" + + # --8<-- [start:uuid-value] + order = await Order.create(total=Decimal("68.40")) + row = await fetch_one('SELECT "id" FROM "order"') + assert row == {"id": str(order.id)} # 36 characters, hyphens included + # --8<-- [end:uuid-value] + + # --8<-- [start:composite-unique-violation] + try: + await Product.create( + sku="LMP-003", + name="Desk lamp", + category="lighting", + price=Decimal("42.00"), + ) + except UniqueViolationError as exc: + assert "UNIQUE constraint failed: product.category, product.name" in str(exc) + # --8<-- [end:composite-unique-violation] + else: + raise AssertionError("expected UniqueViolationError") + + # --8<-- [start:validation] + try: + await Product.create( + sku="LMP-004", + name="Clamp lamp", + category="lighting", + price=Decimal("-5.00"), + ) + except ValidationError as exc: + [error] = exc.errors() + assert error["loc"] == ("price",) + assert error["msg"] == "Input should be greater than or equal to 0" + # --8<-- [end:validation] + else: + raise AssertionError("expected ValidationError") + + # --8<-- [start:base-class-rows] + await Review.create(product_sku="LMP-001", rating=5) + assert await Review.select().count() == 1 + assert await Timestamped.select().count() == 0 # a table of its own + # --8<-- [end:base-class-rows] + + +DDL = """ +# --8<-- [start:product-ddl] +# --8<-- [start:product-table] +CREATE TABLE IF NOT EXISTS "product" ( + "attributes" JSON NOT NULL, + "category" varchar NOT NULL, + "created_at" DATETIME NOT NULL, + "id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, +# --8<-- [start:name-column] + "name" varchar(200) NOT NULL, +# --8<-- [end:name-column] + "price" NUMERIC NOT NULL, + "sku" varchar NOT NULL, +# --8<-- [start:status-column] + "status" varchar(7) NOT NULL, +# --8<-- [end:status-column] + "tags" JSON NOT NULL +); +# --8<-- [end:product-table] +# --8<-- [start:sku-unique] +CREATE UNIQUE INDEX IF NOT EXISTS "uq_product_sku" ON "product" ("sku"); +# --8<-- [end:sku-unique] +# --8<-- [start:composite-unique-ddl] +CREATE UNIQUE INDEX IF NOT EXISTS "uq_product_category_name" ON "product" ("category", "name"); +# --8<-- [end:composite-unique-ddl] +# --8<-- [start:composite-index-ddl] +CREATE INDEX IF NOT EXISTS "idx_product_status_created_at" ON "product" ("status", "created_at"); +# --8<-- [end:composite-index-ddl] +# --8<-- [end:product-ddl] +# --8<-- [start:category-ddl] +CREATE TABLE IF NOT EXISTS "categories" ( + "code" varchar NOT NULL PRIMARY KEY, + "title" varchar NOT NULL +); +# --8<-- [end:category-ddl] +# --8<-- [start:order-ddl] +CREATE TABLE IF NOT EXISTS "order" ( + "id" CHAR(32) NOT NULL PRIMARY KEY, + "total" NUMERIC NOT NULL +); +# --8<-- [end:order-ddl] +# --8<-- [start:base-class-ddl] +CREATE TABLE IF NOT EXISTS "timestamped" ( + "created_at" DATETIME NOT NULL +); +CREATE TABLE IF NOT EXISTS "review" ( + "created_at" DATETIME NOT NULL, + "id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, + "product_sku" varchar NOT NULL, + "rating" integer NOT NULL +); +# --8<-- [end:base-class-ddl] +""" + + +def _normalized(statement: str) -> str: + return re.sub(r"\s+", " ", statement).strip() + + +def _statements(ddl: str) -> list[str]: + """The statements of ``ddl``, snippet markers left out, one line each.""" + text = "\n".join(line for line in ddl.splitlines() if "--8<--" not in line) + return sorted(_normalized(sql) for sql in text.split(";") if sql.strip()) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/docs/examples/multiple_databases.py b/docs/examples/multiple_databases.py index e14d1b3c..125948b8 100644 --- a/docs/examples/multiple_databases.py +++ b/docs/examples/multiple_databases.py @@ -1,45 +1,82 @@ -"""Runnable companion to the Multiple Databases how-to (docs/pages/howto/multiple-databases.md).""" +"""Runnable companion to the multiple-databases how-to (docs/pages/howto/multiple-databases.md).""" import asyncio -from ferro import Field, Model, connect, engines, transaction +from ferro import connect, create_tables, engines, transaction + +# --8<-- [start:models] +from ferro import Field, Model class Metric(Model): id: int | None = Field(default=None, primary_key=True) name: str - value: float = 0.0 + value: int = 0 + + +# --8<-- [end:models] async def main() -> None: # --8<-- [start:connect] await connect("sqlite::memory:", name="app", default=True, auto_migrate=True) await connect("sqlite::memory:", name="analytics", auto_migrate=True) - # --8<-- [end:connect] - # --8<-- [start:routing] async with engines.session(): - # Writes go to the default ("app") connection unless routed - await Metric.create(name="signups", value=1) + await Metric.create(name="signups", value=12) # the default, "app" app_metrics = await Metric.all() - # Route reads and writes to a named connection with .using() — this - # doesn't need a session; a `using=` call is its own explicit route - await Metric.using("analytics").create(name="page_views", value=100) + await Metric.using("analytics").create(name="page_views", value=1840) analytics_metrics = await Metric.using("analytics").all() - # --8<-- [end:routing] - assert len(app_metrics) == 1 - assert len(analytics_metrics) == 1 - assert analytics_metrics[0].name == "page_views" + + print(len(app_metrics), len(analytics_metrics)) + # --8<-- [end:connect] + assert [metric.name for metric in app_metrics] == ["signups"] + assert [metric.name for metric in analytics_metrics] == ["page_views"] + + # --8<-- [start:session] + async with engines.session("analytics") as session: + busy = ( + await session.query(Metric).where(lambda metric: metric.value > 100).all() + ) + + print([metric.name for metric in busy]) + # --8<-- [end:session] + assert [metric.name for metric in busy] == ["page_views"] + + # --8<-- [start:conflict] + async with engines.session(): + try: + await Metric.using("analytics").all() + except ValueError as error: + print(f"{type(error).__name__}: {error}") + else: + raise AssertionError("expected ValueError") + # --8<-- [end:conflict] # --8<-- [start:transaction] async with transaction(using="analytics"): - await Metric.using("analytics").create(name="clicks", value=42) + await Metric.create(name="clicks", value=42) # runs on "analytics" + try: + await Metric.using("app").create(name="clicks", value=42) + except ValueError as error: + print(f"{type(error).__name__}: {error}") + else: + raise AssertionError("expected ValueError") + + print(len(await Metric.using("analytics").all())) # --8<-- [end:transaction] - assert len(await Metric.using("analytics").all()) == 2 async with engines.session(): assert len(await Metric.all()) == 1 + # --8<-- [start:create-tables] + await connect("sqlite::memory:", name="archive") + report = await create_tables(using="archive") + + print([statement.subject for statement in report.statements]) + # --8<-- [end:create-tables] + assert len(await Metric.using("archive").all()) == 0 + print("multiple_databases example ran successfully") diff --git a/docs/examples/multiple_databases_annotated.py b/docs/examples/multiple_databases_annotated.py new file mode 100644 index 00000000..bac13592 --- /dev/null +++ b/docs/examples/multiple_databases_annotated.py @@ -0,0 +1,85 @@ +"""Runnable companion to the multiple-databases how-to (docs/pages/howto/multiple-databases.md).""" + +import asyncio +from typing import Annotated + +from ferro import connect, create_tables, engines, transaction + +# --8<-- [start:models] +from ferro import Field, Model + + +class Metric(Model): + id: Annotated[int | None, Field(primary_key=True)] = None + name: str + value: int = 0 + + +# --8<-- [end:models] + + +async def main() -> None: + # --8<-- [start:connect] + await connect("sqlite::memory:", name="app", default=True, auto_migrate=True) + await connect("sqlite::memory:", name="analytics", auto_migrate=True) + + async with engines.session(): + await Metric.create(name="signups", value=12) # the default, "app" + app_metrics = await Metric.all() + + await Metric.using("analytics").create(name="page_views", value=1840) + analytics_metrics = await Metric.using("analytics").all() + + print(len(app_metrics), len(analytics_metrics)) + # --8<-- [end:connect] + assert [metric.name for metric in app_metrics] == ["signups"] + assert [metric.name for metric in analytics_metrics] == ["page_views"] + + # --8<-- [start:session] + async with engines.session("analytics") as session: + busy = ( + await session.query(Metric).where(lambda metric: metric.value > 100).all() + ) + + print([metric.name for metric in busy]) + # --8<-- [end:session] + assert [metric.name for metric in busy] == ["page_views"] + + # --8<-- [start:conflict] + async with engines.session(): + try: + await Metric.using("analytics").all() + except ValueError as error: + print(f"{type(error).__name__}: {error}") + else: + raise AssertionError("expected ValueError") + # --8<-- [end:conflict] + + # --8<-- [start:transaction] + async with transaction(using="analytics"): + await Metric.create(name="clicks", value=42) # runs on "analytics" + try: + await Metric.using("app").create(name="clicks", value=42) + except ValueError as error: + print(f"{type(error).__name__}: {error}") + else: + raise AssertionError("expected ValueError") + + print(len(await Metric.using("analytics").all())) + # --8<-- [end:transaction] + async with engines.session(): + assert len(await Metric.all()) == 1 + + # --8<-- [start:create-tables] + await connect("sqlite::memory:", name="archive") + report = await create_tables(using="archive") + + print([statement.subject for statement in report.statements]) + # --8<-- [end:create-tables] + assert len(await Metric.using("archive").all()) == 0 + + print("multiple_databases example ran successfully") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/docs/examples/mutations.py b/docs/examples/mutations.py index 3b2bcda8..02515c54 100644 --- a/docs/examples/mutations.py +++ b/docs/examples/mutations.py @@ -2,9 +2,12 @@ import asyncio +from ferro import ModelDoesNotExist, UniqueViolationError, connect, engines, now + +# --8<-- [start:models] from datetime import datetime -from ferro import Field, Model, UniqueViolationError, connect, engines, now +from ferro import Field, Model class Customer(Model): @@ -16,136 +19,224 @@ class Customer(Model): updated_at: datetime | None = None +# --8<-- [end:models] + + +# --8<-- [start:conversation] +class Conversation(Model): + id: int | None = Field(default=None, primary_key=True) + turns: dict = Field(default_factory=dict) + + +# --8<-- [end:conversation] + + async def main() -> None: + # --8<-- [start:session] await connect("sqlite::memory:", auto_migrate=True) async with engines.session(): - # --8<-- [start:get-or-create] - customer, created = await Customer.get_or_create( - email="alice@example.com", - defaults={"name": "Alice"}, + alice = await Customer.create(email="alice@example.com", name="Alice") + assert (alice.id, alice.plan) == (1, "free") + # --8<-- [end:session] + + # --8<-- [start:create] + bob = await Customer.create(email="bob@example.com", name="Bob", plan="pro") + + inserted = await Customer.bulk_create( + [ + Customer(email="carol@example.com", name="Carol"), + Customer(email="dave@example.com", name="Dave", plan="pro"), + ] ) - assert created is True + assert inserted == 2 + assert await Customer.select().count() == 4 + # --8<-- [end:create] - # Second call finds the existing row instead of inserting - same, created = await Customer.get_or_create(email="alice@example.com") - assert created is False and same.id == customer.id - # --8<-- [end:get-or-create] + # --8<-- [start:bulk-identity] + erin = Customer(email="erin@example.com", name="Erin") + await Customer.bulk_create([erin]) + assert erin.id is None # not written back - # --8<-- [start:update-or-create] - customer, created = await Customer.update_or_create( - email="alice@example.com", - defaults={"plan": "pro"}, - ) - assert created is False and customer.plan == "pro" - # --8<-- [end:update-or-create] - - # --8<-- [start:refresh] - # Reload an instance from the database, discarding local state - await Customer.where( - lambda customer: customer.email == "alice@example.com" - ).update(name="Alice L.") - await customer.refresh() - assert customer.name == "Alice L." - # --8<-- [end:refresh] + erin = await Customer.where( + lambda customer: customer.email == "erin@example.com" + ).first() + assert erin.id == 5 + # --8<-- [end:bulk-identity] # --8<-- [start:create-strict] - alice = await Customer.get(customer.id) - - # create() never updates an existing row — a duplicate is an error try: - await Customer.create(id=alice.id, email="alice.2@example.com") + await Customer.create(email="alice@example.com", name="Alice again") except UniqueViolationError: ... # the existing row is untouched + else: + raise AssertionError("expected UniqueViolationError") + assert (await Customer.get(alice.id)).name == "Alice" # --8<-- [end:create-strict] # --8<-- [start:save-insert-update] - bob = Customer(email="bob@example.com", name="Bob") - await bob.save() # never persisted: INSERT (bob.id is now set) + frank = Customer(email="frank@example.com", name="Frank") + await frank.save() # never persisted: INSERT + assert frank.id is not None # the database assigned it - bob.plan = "pro" - await bob.save() # persisted: UPDATE ... WHERE id = ? + frank.plan = "pro" + await frank.save() # persisted: UPDATE by primary key - bobs = await Customer.where(lambda customer: customer.name == "Bob").count() - assert bobs == 1 # two saves, one row + franks = await Customer.where(lambda customer: customer.name == "Frank").count() + assert franks == 1 # two saves, one row # --8<-- [end:save-insert-update] + # --8<-- [start:save-missing] + await Customer.where(lambda customer: customer.id == frank.id).delete() + frank.name = "Franklin" + try: + await frank.save() + except ModelDoesNotExist: + ... # the row is gone; save() does not insert it again + else: + raise AssertionError("expected ModelDoesNotExist") + # --8<-- [end:save-missing] + # --8<-- [start:save-only] row = await Customer.get(bob.id) - row.name = "Bobby" - row.updated_at = datetime.now() - row.plan = "enterprise" # omitted: left untouched in the database + row.name = "Robert" + row.updated_at = datetime(2026, 3, 1, 9, 30) + row.plan = "enterprise" # not in only=, so not written await row.save(only={"name", "updated_at"}) - # instance is not refreshed — omitted in-memory fields may diverge - assert row.plan == "enterprise" - # --8<-- [end:save-only] + assert row.plan == "enterprise" # the instance keeps its change await row.refresh() - assert row.name == "Bobby" - assert row.plan == "pro" + assert (row.name, row.plan) == ("Robert", "pro") # the row kept "pro" + # --8<-- [end:save-only] # --8<-- [start:save-exclude] - row.name = "Robert" - row.plan = "enterprise" # excluded: left untouched in the database + row.name = "Bobby" + row.plan = "enterprise" # excluded, so not written await row.save(exclude={"plan"}) - # --8<-- [end:save-exclude] await row.refresh() - assert row.name == "Robert" - assert row.plan == "pro" + assert (row.name, row.plan) == ("Bobby", "pro") + # --8<-- [end:save-exclude] - # --8<-- [start:upsert] - # No row with this primary key: INSERT - carol = await Customer.upsert(id=100, email="carol@example.com", name="Carol") + # --8<-- [start:update-delete] + upgraded = await Customer.where( + lambda customer: customer.plan == "free" + ).update(plan="trial") + assert upgraded == 3 # Alice, Carol and Erin - # Same primary key again: UPDATE of the existing row - carol = await Customer.upsert(id=100, email="carol@example.com", plan="pro") - assert carol.plan == "pro" - assert await Customer.where(lambda customer: customer.id == 100).count() == 1 - - # The whole row is written: `name` was left unset above, so the stored - # "Carol" reverted to the field default — pass every field you care about. - stored = await Customer.get(100) - assert stored.name == "" - # --8<-- [end:upsert] - - # --8<-- [start:handling-errors] - try: - await Customer.create(email="alice@example.com") # email already taken - except UniqueViolationError as exc: - print(exc.sqlstate) # "23505" (SQLSTATE) on Postgres, "2067" on SQLite - print(exc.constraint) # violated constraint name (Postgres only) - print(exc.driver_message) # original driver text, for logs - # --8<-- [end:handling-errors] - - dora = await Customer.create( - email="dora@example.com" - ) # plan defaults to "free" + removed = await Customer.where( + lambda customer: customer.name == "Erin" + ).delete() + assert removed == 1 + # --8<-- [end:update-delete] # --8<-- [start:recipe-update] - await Customer.where(lambda customer: customer.id == dora.id).update( + await Customer.where(lambda customer: customer.id == alice.id).update( lambda customer: { "views": customer.views + 1, "updated_at": now, } ) + + await alice.refresh() + assert alice.views == 1 and alice.updated_at is not None # --8<-- [end:recipe-update] + # .merge() exists on PostgreSQL only; the rendered statement says + # which database this session runs on. + on_postgres = (await Customer.select().count.sql()).dialect == "postgres" + if on_postgres: + # --8<-- [start:merge] + chat = await Conversation.create() + await Conversation.where( + lambda conversation: conversation.id == chat.id + ).update( + lambda conversation: { + "turns": conversation.turns.merge({"r1": {"role": "user"}}) + } + ) + + await chat.refresh() + assert chat.turns == {"r1": {"role": "user"}} + # --8<-- [end:merge] + # --8<-- [start:mutation-guard] - free = Customer.where(lambda customer: customer.plan == "free") + trial = Customer.where(lambda customer: customer.plan == "trial") try: - await free.limit(10).delete() - except ValueError: - # Portable SQL has no DELETE ... LIMIT. Fetch primary keys first, - # then mutate by primary-key set: - doomed = await free.limit(10).all() - ids = [customer.id for customer in doomed] - removed = await Customer.where( - lambda customer: customer.id.in_(ids) - ).delete() - assert removed == 1 + await trial.limit(1).delete() + except ValueError as error: + print(f"{type(error).__name__}: {error}") + else: + raise AssertionError("expected ValueError") # --8<-- [end:mutation-guard] + # --8<-- [start:mutation-by-key] + oldest = await trial.order_by(lambda customer: customer.id).limit(1).all() + ids = [customer.id for customer in oldest] + removed = await Customer.where(lambda customer: customer.id.in_(ids)).delete() + assert removed == 1 # Alice, the oldest trial customer + # --8<-- [end:mutation-by-key] + + # --8<-- [start:delete-one] + carol = await Customer.where( + lambda customer: customer.email == "carol@example.com" + ).first() + await carol.delete() + + assert await Customer.get_or_none(carol.id) is None + # --8<-- [end:delete-one] + + # --8<-- [start:upsert] + grace = await Customer.upsert(id=100, email="grace@example.com", name="Grace") + + grace = await Customer.upsert(id=100, email="grace@example.com", plan="pro") + assert grace.plan == "pro" + assert await Customer.where(lambda customer: customer.id == 100).count() == 1 + + stored = await Customer.get(100) + assert (stored.name, stored.plan) == ("", "pro") # name reset to its default + # --8<-- [end:upsert] + + # --8<-- [start:get-or-create] + heidi, created = await Customer.get_or_create( + email="heidi@example.com", + defaults={"name": "Heidi"}, + ) + assert created is True + + same, created = await Customer.get_or_create(email="heidi@example.com") + assert created is False and same.id == heidi.id + # --8<-- [end:get-or-create] + + # --8<-- [start:update-or-create] + heidi, created = await Customer.update_or_create( + email="heidi@example.com", + defaults={"plan": "pro"}, + ) + assert created is False and heidi.plan == "pro" + # --8<-- [end:update-or-create] + + # --8<-- [start:refresh] + await Customer.where(lambda customer: customer.id == heidi.id).update( + name="Heidi L." + ) + assert heidi.name == "Heidi" # the instance has not seen the update + + await heidi.refresh() + assert heidi.name == "Heidi L." + # --8<-- [end:refresh] + + # --8<-- [start:handling-errors] + try: + await Customer.create(email="heidi@example.com") # email already taken + except UniqueViolationError as error: + print(error.sqlstate) + print(error.constraint) + print(error.driver_message) + else: + raise AssertionError("expected UniqueViolationError") + # --8<-- [end:handling-errors] + print("mutations example ran successfully") diff --git a/docs/examples/mutations_annotated.py b/docs/examples/mutations_annotated.py new file mode 100644 index 00000000..fe46f550 --- /dev/null +++ b/docs/examples/mutations_annotated.py @@ -0,0 +1,245 @@ +"""Runnable companion to the Mutations guide (docs/pages/guide/mutations.md).""" + +import asyncio + +from ferro import ModelDoesNotExist, UniqueViolationError, connect, engines, now + +# --8<-- [start:models] +from datetime import datetime +from typing import Annotated + +from ferro import Field, Model + + +class Customer(Model): + id: Annotated[int | None, Field(primary_key=True)] = None + email: Annotated[str, Field(unique=True)] + name: str = "" + plan: str = "free" + views: int = 0 + updated_at: datetime | None = None + + +# --8<-- [end:models] + + +# --8<-- [start:conversation] +class Conversation(Model): + id: Annotated[int | None, Field(primary_key=True)] = None + turns: dict = Field(default_factory=dict) + + +# --8<-- [end:conversation] + + +async def main() -> None: + # --8<-- [start:session] + await connect("sqlite::memory:", auto_migrate=True) + + async with engines.session(): + alice = await Customer.create(email="alice@example.com", name="Alice") + assert (alice.id, alice.plan) == (1, "free") + # --8<-- [end:session] + + # --8<-- [start:create] + bob = await Customer.create(email="bob@example.com", name="Bob", plan="pro") + + inserted = await Customer.bulk_create( + [ + Customer(email="carol@example.com", name="Carol"), + Customer(email="dave@example.com", name="Dave", plan="pro"), + ] + ) + assert inserted == 2 + assert await Customer.select().count() == 4 + # --8<-- [end:create] + + # --8<-- [start:bulk-identity] + erin = Customer(email="erin@example.com", name="Erin") + await Customer.bulk_create([erin]) + assert erin.id is None # not written back + + erin = await Customer.where( + lambda customer: customer.email == "erin@example.com" + ).first() + assert erin.id == 5 + # --8<-- [end:bulk-identity] + + # --8<-- [start:create-strict] + try: + await Customer.create(email="alice@example.com", name="Alice again") + except UniqueViolationError: + ... # the existing row is untouched + else: + raise AssertionError("expected UniqueViolationError") + assert (await Customer.get(alice.id)).name == "Alice" + # --8<-- [end:create-strict] + + # --8<-- [start:save-insert-update] + frank = Customer(email="frank@example.com", name="Frank") + await frank.save() # never persisted: INSERT + assert frank.id is not None # the database assigned it + + frank.plan = "pro" + await frank.save() # persisted: UPDATE by primary key + + franks = await Customer.where(lambda customer: customer.name == "Frank").count() + assert franks == 1 # two saves, one row + # --8<-- [end:save-insert-update] + + # --8<-- [start:save-missing] + await Customer.where(lambda customer: customer.id == frank.id).delete() + frank.name = "Franklin" + try: + await frank.save() + except ModelDoesNotExist: + ... # the row is gone; save() does not insert it again + else: + raise AssertionError("expected ModelDoesNotExist") + # --8<-- [end:save-missing] + + # --8<-- [start:save-only] + row = await Customer.get(bob.id) + row.name = "Robert" + row.updated_at = datetime(2026, 3, 1, 9, 30) + row.plan = "enterprise" # not in only=, so not written + await row.save(only={"name", "updated_at"}) + assert row.plan == "enterprise" # the instance keeps its change + + await row.refresh() + assert (row.name, row.plan) == ("Robert", "pro") # the row kept "pro" + # --8<-- [end:save-only] + + # --8<-- [start:save-exclude] + row.name = "Bobby" + row.plan = "enterprise" # excluded, so not written + await row.save(exclude={"plan"}) + + await row.refresh() + assert (row.name, row.plan) == ("Bobby", "pro") + # --8<-- [end:save-exclude] + + # --8<-- [start:update-delete] + upgraded = await Customer.where( + lambda customer: customer.plan == "free" + ).update(plan="trial") + assert upgraded == 3 # Alice, Carol and Erin + + removed = await Customer.where( + lambda customer: customer.name == "Erin" + ).delete() + assert removed == 1 + # --8<-- [end:update-delete] + + # --8<-- [start:recipe-update] + await Customer.where(lambda customer: customer.id == alice.id).update( + lambda customer: { + "views": customer.views + 1, + "updated_at": now, + } + ) + + await alice.refresh() + assert alice.views == 1 and alice.updated_at is not None + # --8<-- [end:recipe-update] + + # .merge() exists on PostgreSQL only; the rendered statement says + # which database this session runs on. + on_postgres = (await Customer.select().count.sql()).dialect == "postgres" + if on_postgres: + # --8<-- [start:merge] + chat = await Conversation.create() + await Conversation.where( + lambda conversation: conversation.id == chat.id + ).update( + lambda conversation: { + "turns": conversation.turns.merge({"r1": {"role": "user"}}) + } + ) + + await chat.refresh() + assert chat.turns == {"r1": {"role": "user"}} + # --8<-- [end:merge] + + # --8<-- [start:mutation-guard] + trial = Customer.where(lambda customer: customer.plan == "trial") + try: + await trial.limit(1).delete() + except ValueError as error: + print(f"{type(error).__name__}: {error}") + else: + raise AssertionError("expected ValueError") + # --8<-- [end:mutation-guard] + + # --8<-- [start:mutation-by-key] + oldest = await trial.order_by(lambda customer: customer.id).limit(1).all() + ids = [customer.id for customer in oldest] + removed = await Customer.where(lambda customer: customer.id.in_(ids)).delete() + assert removed == 1 # Alice, the oldest trial customer + # --8<-- [end:mutation-by-key] + + # --8<-- [start:delete-one] + carol = await Customer.where( + lambda customer: customer.email == "carol@example.com" + ).first() + await carol.delete() + + assert await Customer.get_or_none(carol.id) is None + # --8<-- [end:delete-one] + + # --8<-- [start:upsert] + grace = await Customer.upsert(id=100, email="grace@example.com", name="Grace") + + grace = await Customer.upsert(id=100, email="grace@example.com", plan="pro") + assert grace.plan == "pro" + assert await Customer.where(lambda customer: customer.id == 100).count() == 1 + + stored = await Customer.get(100) + assert (stored.name, stored.plan) == ("", "pro") # name reset to its default + # --8<-- [end:upsert] + + # --8<-- [start:get-or-create] + heidi, created = await Customer.get_or_create( + email="heidi@example.com", + defaults={"name": "Heidi"}, + ) + assert created is True + + same, created = await Customer.get_or_create(email="heidi@example.com") + assert created is False and same.id == heidi.id + # --8<-- [end:get-or-create] + + # --8<-- [start:update-or-create] + heidi, created = await Customer.update_or_create( + email="heidi@example.com", + defaults={"plan": "pro"}, + ) + assert created is False and heidi.plan == "pro" + # --8<-- [end:update-or-create] + + # --8<-- [start:refresh] + await Customer.where(lambda customer: customer.id == heidi.id).update( + name="Heidi L." + ) + assert heidi.name == "Heidi" # the instance has not seen the update + + await heidi.refresh() + assert heidi.name == "Heidi L." + # --8<-- [end:refresh] + + # --8<-- [start:handling-errors] + try: + await Customer.create(email="heidi@example.com") # email already taken + except UniqueViolationError as error: + print(error.sqlstate) + print(error.constraint) + print(error.driver_message) + else: + raise AssertionError("expected UniqueViolationError") + # --8<-- [end:handling-errors] + + print("mutations example ran successfully") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/docs/examples/pagination.py b/docs/examples/pagination.py index 14cf7590..024569d1 100644 --- a/docs/examples/pagination.py +++ b/docs/examples/pagination.py @@ -1,20 +1,33 @@ -"""Runnable companion to the Pagination how-to (docs/pages/howto/pagination.md).""" +"""Runnable companion to the pagination how-to (docs/pages/howto/pagination.md). + +Seven articles, newest first, paged three at a time: by page number with +``offset()``, by position with ``after()`` and ``before()``, and through +``article_page()``, the function the page's FastAPI endpoint calls. +""" import asyncio +# --8<-- [start:model] +from datetime import date + from ferro import Field, Model, connect, engines class Article(Model): id: int | None = Field(default=None, primary_key=True) title: str + published_on: date = Field(index=True) + + +# --8<-- [end:model] # --8<-- [start:offset] -async def get_page(page: int, per_page: int = 20) -> list[Article]: +async def get_page(page: int, per_page: int) -> list[Article]: return ( await Article.select() - .order_by(lambda article: article.id) + .order_by(lambda article: article.published_on, "desc") + .order_by(lambda article: article.id, "desc") .limit(per_page) .offset((page - 1) * per_page) .all() @@ -24,31 +37,115 @@ async def get_page(page: int, per_page: int = 20) -> list[Article]: # --8<-- [end:offset] -# --8<-- [start:keyset] -async def get_after(after_id: int | None, limit: int = 20) -> list[Article]: - query = ( - Article.select() - if after_id is None - else Article.where(lambda article: article.id > after_id) - ) - return await query.order_by(lambda article: article.id).limit(limit).all() +# --8<-- [start:cursor] +NEWEST_FIRST = ( + Article.select() + .order_by(lambda article: article.published_on, "desc") + .order_by(lambda article: article.id, "desc") +) + + +def encode_cursor(article: Article) -> str: + published_on, article_id = NEWEST_FIRST.position_of(article) + return f"{published_on.isoformat()}_{article_id}" -# --8<-- [end:keyset] +def decode_cursor(cursor: str) -> tuple[date, int]: + published_on, article_id = cursor.split("_") + return date.fromisoformat(published_on), int(article_id) + + +async def article_page(cursor: str | None = None, limit: int = 20) -> dict: + query = NEWEST_FIRST.limit(limit + 1) + if cursor is not None: + query = query.after(decode_cursor(cursor)) + rows = await query.all() + items = rows[:limit] + return { + "items": items, + "next_cursor": encode_cursor(items[-1]) if len(rows) > limit else None, + } + + +# --8<-- [end:cursor] async def main() -> None: + # --8<-- [start:session] await connect("sqlite::memory:", auto_migrate=True) async with engines.session(): - await Article.bulk_create([Article(title=f"Article {i}") for i in range(1, 51)]) - - page_two = await get_page(page=2, per_page=10) - assert [a.title for a in page_two][:2] == ["Article 11", "Article 12"] - - first_batch = await get_after(after_id=None, limit=10) - next_batch = await get_after(after_id=first_batch[-1].id, limit=10) - assert next_batch[0].title == "Article 11" + await Article.bulk_create( + [ + Article(title="Choosing a primary key", published_on=date(2026, 1, 5)), + Article(title="Indexes that pay off", published_on=date(2026, 1, 12)), + Article(title="Reading a query plan", published_on=date(2026, 1, 12)), + Article(title="Keyset paging", published_on=date(2026, 1, 19)), + Article(title="Soft deletes", published_on=date(2026, 1, 26)), + Article(title="Storing UTC", published_on=date(2026, 2, 2)), + Article(title="Testing with SQLite", published_on=date(2026, 2, 9)), + ] + ) + # --8<-- [end:session] + + # --8<-- [start:offset-call] + page_two = await get_page(page=2, per_page=3) + assert [article.title for article in page_two] == [ + "Keyset paging", + "Reading a query plan", + "Indexes that pay off", + ] + # --8<-- [end:offset-call] + + # --8<-- [start:after] + newest = ( + Article.select() + .order_by(lambda article: article.published_on, "desc") + .order_by(lambda article: article.id, "desc") + .limit(3) + ) + + first_page = await newest.all() + assert [article.title for article in first_page] == [ + "Testing with SQLite", + "Storing UTC", + "Soft deletes", + ] + + second_page = await newest.after(first_page[-1]).all() + assert [article.title for article in second_page] == [ + "Keyset paging", + "Reading a query plan", + "Indexes that pay off", + ] + # --8<-- [end:after] + + # --8<-- [start:before] + previous_page = await newest.before(second_page[0]).all() + assert previous_page == first_page + # --8<-- [end:before] + + # --8<-- [start:cursor-calls] + first = await article_page(limit=3) + assert first["next_cursor"] == "2026-01-26_5" + + second = await article_page(cursor=first["next_cursor"], limit=3) + assert second["next_cursor"] == "2026-01-12_2" + + last = await article_page(cursor=second["next_cursor"], limit=3) + assert [article.title for article in last["items"]] == [ + "Choosing a primary key" + ] + assert last["next_cursor"] is None + # --8<-- [end:cursor-calls] + + # A page that ends exactly on the last row reports no next page. + exact = await article_page(cursor="2026-01-12_3", limit=2) + assert [article.title for article in exact["items"]] == [ + "Indexes that pay off", + "Choosing a primary key", + ] + assert exact["next_cursor"] is None print("pagination example ran successfully") diff --git a/docs/examples/pagination_annotated.py b/docs/examples/pagination_annotated.py new file mode 100644 index 00000000..24d81fb8 --- /dev/null +++ b/docs/examples/pagination_annotated.py @@ -0,0 +1,155 @@ +"""Runnable companion to the pagination how-to (docs/pages/howto/pagination.md). + +Seven articles, newest first, paged three at a time: by page number with +``offset()``, by position with ``after()`` and ``before()``, and through +``article_page()``, the function the page's FastAPI endpoint calls. +""" + +import asyncio + +# --8<-- [start:model] +from datetime import date +from typing import Annotated + +from ferro import Field, Model, connect, engines + + +class Article(Model): + id: Annotated[int | None, Field(primary_key=True)] = None + title: str + published_on: Annotated[date, Field(index=True)] + + +# --8<-- [end:model] + + +# --8<-- [start:offset] +async def get_page(page: int, per_page: int) -> list[Article]: + return ( + await Article.select() + .order_by(lambda article: article.published_on, "desc") + .order_by(lambda article: article.id, "desc") + .limit(per_page) + .offset((page - 1) * per_page) + .all() + ) + + +# --8<-- [end:offset] + + +# --8<-- [start:cursor] +NEWEST_FIRST = ( + Article.select() + .order_by(lambda article: article.published_on, "desc") + .order_by(lambda article: article.id, "desc") +) + + +def encode_cursor(article: Article) -> str: + published_on, article_id = NEWEST_FIRST.position_of(article) + return f"{published_on.isoformat()}_{article_id}" + + +def decode_cursor(cursor: str) -> tuple[date, int]: + published_on, article_id = cursor.split("_") + return date.fromisoformat(published_on), int(article_id) + + +async def article_page(cursor: str | None = None, limit: int = 20) -> dict: + query = NEWEST_FIRST.limit(limit + 1) + if cursor is not None: + query = query.after(decode_cursor(cursor)) + rows = await query.all() + items = rows[:limit] + return { + "items": items, + "next_cursor": encode_cursor(items[-1]) if len(rows) > limit else None, + } + + +# --8<-- [end:cursor] + + +async def main() -> None: + # --8<-- [start:session] + await connect("sqlite::memory:", auto_migrate=True) + + async with engines.session(): + await Article.bulk_create( + [ + Article(title="Choosing a primary key", published_on=date(2026, 1, 5)), + Article(title="Indexes that pay off", published_on=date(2026, 1, 12)), + Article(title="Reading a query plan", published_on=date(2026, 1, 12)), + Article(title="Keyset paging", published_on=date(2026, 1, 19)), + Article(title="Soft deletes", published_on=date(2026, 1, 26)), + Article(title="Storing UTC", published_on=date(2026, 2, 2)), + Article(title="Testing with SQLite", published_on=date(2026, 2, 9)), + ] + ) + # --8<-- [end:session] + + # --8<-- [start:offset-call] + page_two = await get_page(page=2, per_page=3) + assert [article.title for article in page_two] == [ + "Keyset paging", + "Reading a query plan", + "Indexes that pay off", + ] + # --8<-- [end:offset-call] + + # --8<-- [start:after] + newest = ( + Article.select() + .order_by(lambda article: article.published_on, "desc") + .order_by(lambda article: article.id, "desc") + .limit(3) + ) + + first_page = await newest.all() + assert [article.title for article in first_page] == [ + "Testing with SQLite", + "Storing UTC", + "Soft deletes", + ] + + second_page = await newest.after(first_page[-1]).all() + assert [article.title for article in second_page] == [ + "Keyset paging", + "Reading a query plan", + "Indexes that pay off", + ] + # --8<-- [end:after] + + # --8<-- [start:before] + previous_page = await newest.before(second_page[0]).all() + assert previous_page == first_page + # --8<-- [end:before] + + # --8<-- [start:cursor-calls] + first = await article_page(limit=3) + assert first["next_cursor"] == "2026-01-26_5" + + second = await article_page(cursor=first["next_cursor"], limit=3) + assert second["next_cursor"] == "2026-01-12_2" + + last = await article_page(cursor=second["next_cursor"], limit=3) + assert [article.title for article in last["items"]] == [ + "Choosing a primary key" + ] + assert last["next_cursor"] is None + # --8<-- [end:cursor-calls] + + # A page that ends exactly on the last row reports no next page. + exact = await article_page(cursor="2026-01-12_3", limit=2) + assert [article.title for article in exact["items"]] == [ + "Indexes that pay off", + "Choosing a primary key", + ] + assert exact["next_cursor"] is None + + print("pagination example ran successfully") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/docs/examples/partial_selects.py b/docs/examples/partial_selects.py index cd1f049a..17fe4039 100644 --- a/docs/examples/partial_selects.py +++ b/docs/examples/partial_selects.py @@ -1,192 +1,207 @@ -"""Runnable companion to the "Selecting a Column Subset" guide section +"""Runnable companion to "Selecting a column subset" in the Queries guide (docs/pages/guide/queries.md). -Seeds a small transactions table, then runs every partial-select query the -guide shows and asserts the exact records that come back. +Runs every projection the guide shows against the guide's ledger (declared in +predicates.py, rows from _ledger_seed.py) and asserts the records that come +back. """ import asyncio -from typing import Annotated +from decimal import Decimal -from ferro import BackRef, Field, ForeignKey, Model, Relation, Rows, connect, engines +import predicates +from _ledger_seed import seed +from predicates import Transaction - -# --8<-- [start:schema] -class Owner(Model): - id: int | None = Field(default=None, primary_key=True) - email: str - accounts: Relation[list["Account"]] = BackRef() - - -class Account(Model): - id: int | None = Field(default=None, primary_key=True) - label: str - owner: Annotated[Owner | None, ForeignKey(related_name="accounts")] = None - transactions: Relation[list["Transaction"]] = BackRef() - - -class Transaction(Model): - id: int | None = Field(default=None, primary_key=True) - amount: int - memo: str - account: Annotated[Account | None, ForeignKey(related_name="transactions")] = None - - -# --8<-- [end:schema] +from ferro import Rows, connect, engines async def main() -> None: await connect("sqlite::memory:", auto_migrate=True) async with engines.session(): - alice = await Owner.create(id=1, email="alice@example.com") - a1 = await Account.create(id=1, label="a1", owner=alice) - b1 = await Account.create(id=2, label="b1") - await Transaction.create(id=1, amount=10, memo="coffee", account=a1) - await Transaction.create(id=2, amount=20, memo="lunch", account=a1) - await Transaction.create(id=3, amount=30, memo="dinner", account=b1) + await seed(vars(predicates)) # --8<-- [start:basic] - rows = await Transaction.select(lambda t: (t.id, t.amount)).all() - - rows[0].amount # attribute access, like a model - rows.model_dump() # [{"id": 1, "amount": 10}, ...] + rows = ( + await Transaction.select( + lambda transaction: (transaction.memo, transaction.amount) + ) + .order_by(lambda transaction: transaction.id) + .limit(2) + .all() + ) + assert rows[1].memo == "Rent" # attribute access, like a model + assert rows.model_dump() == [ + {"memo": "Salary", "amount": Decimal("3200")}, + {"memo": "Rent", "amount": Decimal("-1450")}, + ] # --8<-- [end:basic] - assert {(r.id, r.amount) for r in rows} == {(1, 10), (2, 20), (3, 30)} - assert rows[0].model_dump() == {"id": rows[0].id, "amount": rows[0].amount} # --8<-- [start:not-a-model] row = rows[0] - assert not isinstance(row, Transaction) # a Row, not a Transaction - assert not hasattr(row, "save") # records cannot be persisted + assert not isinstance(row, Transaction) + assert not hasattr(row, "save") # --8<-- [end:not-a-model] # --8<-- [start:single] - amounts = await Transaction.select(lambda t: t.amount).order_by("amount").all() - assert [r.amount for r in amounts] == [10, 20, 30] + memos = ( + await Transaction.select(lambda transaction: transaction.memo) + .where(lambda transaction: transaction.cleared_on == None) + .order_by(lambda transaction: transaction.id) + .all() + ) + assert [row.memo for row in memos] == [ + "Train to Lyon", + "Dinner in Lyon", + "Weekly shop", + ] # --8<-- [end:single] # --8<-- [start:container] - newest = await ( - Transaction.select(lambda t: (t.id, t.memo)).order_by("id", "desc").all() + newest = ( + await Transaction.select( + lambda transaction: (transaction.id, transaction.memo) + ) + .order_by(lambda transaction: transaction.id, "desc") + .all() ) + assert len(newest) == 9 + assert newest[0].memo == "Weekly shop" - assert len(newest) == 3 # list-like: len ... - assert newest[0].memo == "dinner" # ... index ... - top_two = newest[:2] # ... and slice (a Rows again) + top_two = newest[:2] # a slice is a Rows too assert isinstance(top_two, Rows) assert top_two.model_dump() == [ - {"id": 3, "memo": "dinner"}, - {"id": 2, "memo": "lunch"}, + {"id": 9, "memo": "Weekly shop"}, + {"id": 8, "memo": "Office coffee"}, ] # --8<-- [end:container] # --8<-- [start:strings] - rows = await Transaction.select("id", "amount").order_by("id").all() - assert rows[0].model_dump() == {"id": 1, "amount": 10} + rows = await Transaction.select("id", "memo").order_by("id").limit(1).all() + assert rows.model_dump() == [{"id": 1, "memo": "Salary"}] # --8<-- [end:strings] # --8<-- [start:traversed] - # A selected field may reach across a relation, at any depth. - # Unaliased, the field takes the bare leaf column name. - rows = await ( - Transaction.select(lambda t: (t.memo, t.account.label)).order_by("id").all() + rows = ( + await Transaction.select( + lambda transaction: (transaction.memo, transaction.account.label) + ) + .order_by(lambda transaction: transaction.id) + .limit(1) + .all() ) - assert rows[0].model_dump() == {"memo": "coffee", "label": "a1"} + assert rows.model_dump() == [{"memo": "Salary", "label": "Checking"}] # --8<-- [end:traversed] # --8<-- [start:aliases] - # A dict-returning selector names the output fields — keys are the - # record's field names, values may traverse. - rows = await ( - Transaction.select( - lambda t: { - "memo": t.memo, - "account_label": t.account.label, - "owner_email": t.account.owner.email, + rows = ( + await Transaction.select( + lambda transaction: { + "memo": transaction.memo, + "account": transaction.account.label, + "owner_email": transaction.account.owner.email, } ) - .order_by("id") + .order_by(lambda transaction: transaction.id) + .limit(1) .all() ) - assert rows[0].model_dump() == { - "memo": "coffee", - "account_label": "a1", - "owner_email": "alice@example.com", - } + assert rows.model_dump() == [ + {"memo": "Salary", "account": "Checking", "owner_email": "ada@example.com"} + ] # --8<-- [end:aliases] + # --8<-- [start:collision] + try: + Transaction.select( + lambda transaction: (transaction.id, transaction.account.id) + ) + except ValueError as error: + assert "two fields named 'id'" in str(error) + assert "dict selector form" in str(error) + else: + raise AssertionError("two fields named 'id' must raise ValueError") + # --8<-- [end:collision] + # --8<-- [start:traversal-narrows] - # Traversal narrows exactly like a where() predicate on the same - # path: rows without the relation drop out (b1 has no owner)... - rows = await Transaction.select( - lambda t: {"memo": t.memo, "owner_email": t.account.owner.email} + owner_emails = await Transaction.select( + lambda transaction: { + "memo": transaction.memo, + "owner_email": transaction.account.owner.email, + } ).all() - assert {r.memo for r in rows} == {"coffee", "lunch"} + assert len(owner_emails) == 8 # Office coffee's account has no owner - # ... and left_join() is the opt-out: rows are kept, and their - # traversed fields decode to None — even from a non-nullable column. - rows = await ( - Transaction.select( - lambda t: {"memo": t.memo, "owner_email": t.account.owner.email} + with_ownerless = ( + await Transaction.select( + lambda transaction: { + "memo": transaction.memo, + "owner_email": transaction.account.owner.email, + } ) - .left_join(lambda t: t.account.owner) - .order_by("id") + .left_join(lambda transaction: transaction.account.owner) + .order_by(lambda transaction: transaction.id) .all() ) - assert rows.model_dump()[2] == {"memo": "dinner", "owner_email": None} + assert len(with_ownerless) == 9 + assert with_ownerless[7].model_dump() == { + "memo": "Office coffee", + "owner_email": None, + } # --8<-- [end:traversal-narrows] - # --8<-- [start:collision] - # Two selected fields resolving to the same output name is a - # build-time error naming the fix: the dict form. - try: - Transaction.select(lambda t: (t.id, t.account.id)) - except ValueError as exc: - assert "two fields named 'id'" in str(exc) - assert "dict selector form" in str(exc) - # --8<-- [end:collision] - # --8<-- [start:compose] - # Projection composes with everything a read query can do: traversal - # predicates, ordering by unselected columns, limit/offset, first(). - rows = await ( - Transaction.select(lambda t: (t.id, t.memo)) - .where(lambda t: t.account.label == "a1") - .order_by("amount", "desc") # amount is not selected — still sorts - .limit(1) + rows = ( + await Transaction.select( + lambda transaction: (transaction.id, transaction.memo) + ) + .where(lambda transaction: transaction.account.label == "Travel") + .order_by(lambda transaction: transaction.amount) # not selected .all() ) - assert rows.model_dump() == [{"id": 2, "memo": "lunch"}] - - row = await Transaction.select(lambda t: t.memo).order_by("id").first() - assert row is not None and row.memo == "coffee" + assert rows.model_dump() == [ + {"id": 6, "memo": "Train to Lyon"}, + {"id": 7, "memo": "Dinner in Lyon"}, + ] # --8<-- [end:compose] # --8<-- [start:projected-paging] - # Projected records page the same way instances do. position_of(Row) - # requires every order key in the projection; otherwise pass a tuple. - projected = ( - Transaction.select(lambda t: {"label": t.account.label, "id": t.id}) - .order_by(lambda t: t.account.label) - .order_by(lambda t: t.id) - ) - records = await projected.all() - next_records = ( - await projected.after(projected.position_of(records[1])).limit(2).all() + by_account = ( + Transaction.select( + lambda transaction: { + "account": transaction.account.label, + "id": transaction.id, + "memo": transaction.memo, + } + ) + .order_by(lambda transaction: transaction.account.label) + .order_by(lambda transaction: transaction.id) ) + first_page = await by_account.limit(5).all() + next_page = await by_account.after(first_page[-1]).limit(2).all() + assert [row.memo for row in next_page] == ["Office coffee", "From checking"] # --8<-- [end:projected-paging] - assert [r.id for r in records] == [1, 2, 3] - assert [r.id for r in next_records] == [3] # --8<-- [start:count] - # count()/exists() are unaffected by projection: they measure the - # same matching rows a full query would. - q = Transaction.select(lambda t: (t.id,)).where(lambda t: t.amount >= 20) - assert await q.count() == 2 - assert await q.exists() is True + uncleared = Transaction.select(lambda transaction: transaction.memo).where( + lambda transaction: transaction.cleared_on == None + ) + assert await uncleared.count() == 3 + assert await uncleared.exists() is True # --8<-- [end:count] + # --8<-- [start:typo] + try: + Transaction.select(lambda transaction: (transaction.id, transaction.amonut)) + except AttributeError as error: + assert str(error).startswith( + "Transaction has no queryable column 'amonut'. Did you mean 'amount'?" + ) + else: + raise AssertionError("a misspelled column must raise AttributeError") + # --8<-- [end:typo] + print("partial_selects example ran successfully") diff --git a/docs/examples/partial_selects_annotated.py b/docs/examples/partial_selects_annotated.py deleted file mode 100644 index 56fad5d6..00000000 --- a/docs/examples/partial_selects_annotated.py +++ /dev/null @@ -1,52 +0,0 @@ -"""Annotated-style companion to partial_selects.py (AGENTS.md I-7). - -Field options move into ``Annotated[...]``. Projection is independent of the -declaration style — the queries are identical in both — so only the schema and -one round-trip live here. -""" - -import asyncio -from typing import Annotated - -from ferro import BackRef, Field, ForeignKey, Model, Relation, connect, engines - - -# --8<-- [start:schema] -class Owner(Model): - id: Annotated[int | None, Field(default=None, primary_key=True)] - email: str - accounts: Relation[list["Account"]] = BackRef() - - -class Account(Model): - id: Annotated[int | None, Field(default=None, primary_key=True)] - label: str - owner: Annotated[Owner | None, ForeignKey(related_name="accounts")] = None - transactions: Relation[list["Transaction"]] = BackRef() - - -class Transaction(Model): - id: Annotated[int | None, Field(default=None, primary_key=True)] - amount: int - memo: str - account: Annotated[Account | None, ForeignKey(related_name="transactions")] = None - - -# --8<-- [end:schema] - - -async def main() -> None: - await connect("sqlite::memory:", auto_migrate=True) - - async with engines.session(): - a1 = await Account.create(id=1, label="a1") - await Transaction.create(id=1, amount=10, memo="coffee", account=a1) - - rows = await Transaction.select(lambda t: (t.id, t.amount)).all() - assert rows.model_dump() == [{"id": 1, "amount": 10}] - - print("partial_selects_annotated example ran successfully") - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/docs/examples/populated_relations.py b/docs/examples/populated_relations.py index a312fc1b..5d3d49f5 100644 --- a/docs/examples/populated_relations.py +++ b/docs/examples/populated_relations.py @@ -1,197 +1,181 @@ -"""Runnable companion to the "Populating Relations with include()" guide -section (docs/pages/guide/queries.md) and the populated-relations parts of -the relationships guide. - -Seeds a small ledger domain (Owner <- Account <- Transaction, both FKs -nullable) and runs every include() query the guides show, asserting the exact -populations that come back — the population contract, membership preservation, -multi-hop paths, identity/accumulation, the refresh rule, and the loud limits. - -Two session blocks on purpose: populations accumulate across a session's -queries, so the awaitable-contract and accumulation demos need instances no -earlier include has touched. +"""Runnable companion to "Populating relations with include()" in the Queries +guide (docs/pages/guide/queries.md); the relationships guide embeds ``basic``. + +Runs every include() query the guide shows against the guide's ledger +(declared in predicates.py, rows from _ledger_seed.py) and asserts the +populations that come back. + +Two sessions on purpose: populations accumulate across one session's queries, +so the accumulation example needs instances no earlier include has touched. """ import asyncio import inspect -from typing import Annotated - -from ferro import BackRef, Field, ForeignKey, Model, Relation, connect, engines, execute - - -# --8<-- [start:schema] -class Owner(Model): - id: int | None = Field(default=None, primary_key=True) - name: str - accounts: Relation[list["Account"]] = BackRef() - - -class Account(Model): - id: int | None = Field(default=None, primary_key=True) - label: str - owner: Annotated[Owner | None, ForeignKey(related_name="accounts")] = None - transactions: Relation[list["Transaction"]] = BackRef() - - -class Transaction(Model): - id: int | None = Field(default=None, primary_key=True) - amount: int - account: Annotated[Account | None, ForeignKey(related_name="transactions")] = None +import predicates +from _ledger_seed import seed +from predicates import Account, Transaction -# --8<-- [end:schema] - - -async def _seed() -> None: - """Owner o1 <- accounts a1/a2; account b1 has no owner; txn 5 has no - account. Transactions: a1 has 10/20, a2 has 30, b1 has 40, orphan 50.""" - o1 = Owner(id=1, name="o1") - await o1.save() - a1 = Account(id=1, label="a1", owner=o1) - a2 = Account(id=2, label="a2", owner=o1) - b1 = Account(id=3, label="b1", owner=None) - for row in (a1, a2, b1): - await row.save() - for txn in ( - Transaction(id=1, amount=10, account=a1), - Transaction(id=2, amount=20, account=a1), - Transaction(id=3, amount=30, account=a2), - Transaction(id=4, amount=40, account=b1), - Transaction(id=5, amount=50, account=None), - ): - await txn.save() +from ferro import connect, engines, execute async def main() -> None: await connect("sqlite::memory:", auto_migrate=True) async with engines.session(): - await _seed() + await seed(vars(predicates)) # --8<-- [start:awaitable] - # Without include(), a relation keeps the awaitable contract — access - # is a coroutine that runs its own query. - transaction = await Transaction.get(2) + transaction = await Transaction.get(6) - account = await transaction.account - assert account.label == "a1" + account = await transaction.account # runs its own query + assert account.label == "Travel" # --8<-- [end:awaitable] # --8<-- [start:basic] - # One SQL statement; every transaction's account arrives populated. - transactions = await Transaction.select().include(lambda t: t.account).all() + latest = ( + await Transaction.select() + .include(lambda transaction: transaction.account) + .order_by(lambda transaction: transaction.id, "desc") + .limit(3) + .all() + ) - for transaction in transactions[:3]: - # Plain attribute access — no await, no query. - print(transaction.amount, transaction.account.label) + for transaction in latest: + print(transaction.memo, "|", transaction.account.label) # no await # --8<-- [end:basic] - assert [t.account.label for t in transactions[:3]] == ["a1", "a1", "a2"] + assert [ + (transaction.memo, transaction.account.label) for transaction in latest + ] == [ + ("Weekly shop", "Checking"), + ("Office coffee", "Petty cash"), + ("Dinner in Lyon", "Travel"), + ] # --8<-- [start:membership] - # include() never changes which rows come back: the transaction whose - # account FK is NULL is still in the result, populated as None. - every = await Transaction.select().include(lambda t: t.account).all() + everything = ( + await Transaction.select() + .include(lambda transaction: transaction.category) + .all() + ) + assert len(everything) == 9 - orphan = next(t for t in every if t.id == 5) - assert len(every) == 5 - assert orphan.account is None + salary = next(row for row in everything if row.memo == "Salary") + assert salary.category is None # no category: populated as None # --8<-- [end:membership] + # --8<-- [start:interplay] + on_travel = ( + await Transaction.where( + lambda transaction: transaction.account.label == "Travel" + ) + .include(lambda transaction: transaction.account) + .all() + ) + assert {transaction.memo for transaction in on_travel} == { + "Train to Lyon", + "Dinner in Lyon", + } + # --8<-- [end:interplay] + + # --8<-- [start:count] + debits = Transaction.where(lambda transaction: transaction.amount < 0) + with_accounts = debits.include(lambda transaction: transaction.account) + + assert await with_accounts.count() == await debits.count() == 7 + # --8<-- [end:count] + # --8<-- [start:multi-hop] - # Including a path populates EVERY hop along it. - transactions = await ( - Transaction.select().include(lambda t: t.account.owner).order_by("id").all() + with_owners = ( + await Transaction.select() + .include(lambda transaction: transaction.account.owner) + .order_by(lambda transaction: transaction.id) + .all() ) - first = transactions[0] - assert first.account.label == "a1" # hop 1 populated - assert first.account.owner.name == "o1" # hop 2 populated + salary = with_owners[0] + assert salary.account.label == "Checking" # first hop + assert salary.account.owner.name == "Ada Lovelace" # second hop - # A NULL mid-chain ends the chain truthfully: b1 has no owner. - ownerless = transactions[3] - assert ownerless.account.label == "b1" - assert ownerless.account.owner is None + coffee = with_owners[7] + assert coffee.account.label == "Petty cash" + assert coffee.account.owner is None # the chain ends at the NULL # --8<-- [end:multi-hop] # --8<-- [start:identity] - # In a session, a populated instance IS the instance — the same object - # a direct fetch returns, deduped across result rows. - held = await Account.get(1) - transactions = await ( - Transaction.select() - .include(lambda t: t.account) - .where(lambda t: t.amount <= 20) + travel = await Account.get(3) + on_travel = ( + await Transaction.where( + lambda transaction: transaction.account_id == travel.id + ) + .include(lambda transaction: transaction.account) .all() ) - assert transactions[0].account is held - assert transactions[0].account is transactions[1].account + assert on_travel[0].account is travel + assert on_travel[1].account is on_travel[0].account # --8<-- [end:identity] async with engines.session(): # --8<-- [start:accumulate] - # Populations accumulate across a session's queries: a later query's - # include attaches onto the instances the session already holds. - plain = await Transaction.get(3) # no population yet - await Transaction.select().include(lambda t: t.account).all() - - assert plain.account.label == "a2" # now populated - # --8<-- [end:accumulate] + market_run = await Transaction.get(3) # nothing populated yet - # --8<-- [start:count] - # count()/exists() measure the same rows with or without the include. - base = Transaction.where(lambda t: t.amount >= 20) - assert await base.include(lambda t: t.account).count() == await base.count() - # --8<-- [end:count] - - # --8<-- [start:interplay] - # A predicate on the same path keeps its stage-1 INNER semantics — - # include attaches data, it never rewrites what a filter matches. - a1_rows = await ( - Transaction.where(lambda t: t.account.label == "a1") - .include(lambda t: t.account) - .all() + await ( + Transaction.select().include(lambda transaction: transaction.account).all() ) - assert [t.id for t in a1_rows] == [1, 2] - assert all(t.account.label == "a1" for t in a1_rows) - # --8<-- [end:interplay] + await ( + Transaction.select().include(lambda transaction: transaction.category).all() + ) + + assert market_run.account.label == "Checking" + assert market_run.category.name == "Groceries" + # --8<-- [end:accumulate] # --8<-- [start:refresh-rule] - # A refresh keeps a population only while it is still true. Change the - # row's FK underneath the ORM and re-fetch: the stale population is - # dropped, and access reverts to the awaitable. - transaction = await ( - Transaction.select() - .include(lambda t: t.account) - .where(lambda t: t.id == 1) + from_checking = await ( + Transaction.where(lambda transaction: transaction.id == 5) + .include(lambda transaction: transaction.account) .first() ) - assert transaction.account.label == "a1" + assert from_checking.account.label == "Savings" - await execute('UPDATE "transaction" SET account_id = ? WHERE id = ?', 2, 1) - await Transaction.get(1) # refreshes in place + # Move the row to another account behind the ORM's back, then fetch it. + # ($1 and $2 are placeholders SQLite and PostgreSQL both accept.) + await execute('UPDATE "transaction" SET account_id = $1 WHERE id = $2', 1, 5) + await Transaction.get(5) # refreshes the instance in place - pending = transaction.account # awaitable again - assert inspect.iscoroutine(pending) - assert (await pending).label == "a2" + pending = from_checking.account # awaitable again + assert inspect.isawaitable(pending) + assert (await pending).label == "Checking" # --8<-- [end:refresh-rule] # --8<-- [start:limits] - # The loud limits: reverse relations, projections, and mutations. try: - Account.select().include(lambda a: a.transactions) - except TypeError as exc: - assert "reverse (BackRef) or many-to-many" in str(exc) + Account.select().include(lambda account: account.transactions) + except TypeError as error: + assert "reverse (BackRef) or many-to-many" in str(error) + else: + raise AssertionError("including a reverse relation must raise") try: - Transaction.select().include(lambda t: t.account).select(lambda t: (t.id,)) - except ValueError as exc: - assert "traversed projection" in str(exc) + Transaction.select().include( + lambda transaction: transaction.account + ).select(lambda transaction: (transaction.id, transaction.memo)) + except ValueError as error: + assert "traversed projection" in str(error) + else: + raise AssertionError("include() with a projection must raise") try: - await Transaction.select().include(lambda t: t.account).delete() - except ValueError as exc: - assert "does not support include()" in str(exc) + await ( + Transaction.select() + .include(lambda transaction: transaction.account) + .delete() + ) + except ValueError as error: + assert "does not support include()" in str(error) + else: + raise AssertionError("delete() on an included query must raise") # --8<-- [end:limits] print("populated_relations example ran successfully") diff --git a/docs/examples/populated_relations_annotated.py b/docs/examples/populated_relations_annotated.py deleted file mode 100644 index bbfdd025..00000000 --- a/docs/examples/populated_relations_annotated.py +++ /dev/null @@ -1,59 +0,0 @@ -"""Annotated-style companion to populated_relations.py (AGENTS.md I-7). - -Field options move into ``Annotated[...]``. The relationship declarations are -identical in both styles: forward FKs are always ``Annotated[Target, -ForeignKey(...)]`` and ``BackRef()`` is always an assignment. The include() -queries themselves declare no fields, so they live only in -populated_relations.py. -""" - -import asyncio -from typing import Annotated - -from ferro import BackRef, Field, ForeignKey, Model, Relation, connect, engines - - -# --8<-- [start:schema] -class Owner(Model): - id: Annotated[int | None, Field(default=None, primary_key=True)] - name: str - accounts: Relation[list["Account"]] = BackRef() - - -class Account(Model): - id: Annotated[int | None, Field(default=None, primary_key=True)] - label: str - owner: Annotated[Owner | None, ForeignKey(related_name="accounts")] = None - transactions: Relation[list["Transaction"]] = BackRef() - - -class Transaction(Model): - id: Annotated[int | None, Field(default=None, primary_key=True)] - amount: int - account: Annotated[Account | None, ForeignKey(related_name="transactions")] = None - - -# --8<-- [end:schema] - - -async def main() -> None: - await connect("sqlite::memory:", auto_migrate=True) - - async with engines.session(): - owner = Owner(id=1, name="o1") - await owner.save() - account = Account(id=1, label="a1", owner=owner) - await account.save() - await Transaction(id=1, amount=10, account=account).save() - - transactions = ( - await Transaction.select().include(lambda t: t.account.owner).all() - ) - assert transactions[0].account.label == "a1" - assert transactions[0].account.owner.name == "o1" - - print("populated_relations_annotated example ran successfully") - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/docs/examples/predicates.py b/docs/examples/predicates.py index 20f3adec..d4b4a3ec 100644 --- a/docs/examples/predicates.py +++ b/docs/examples/predicates.py @@ -1,259 +1,369 @@ -"""Runnable companion to the Queries guide (docs/pages/guide/queries.md).""" +"""Runnable companion to the Queries guide (docs/pages/guide/queries.md). + +Declares the ledger every section of the guide queries, adds its rows +(``_ledger_seed.py``), then runs the filtering, ordering, paging and terminal +examples and asserts the rows that come back. The guide's other example +scripts import these models from here. +""" import asyncio -from datetime import UTC, datetime -# --8<-- [start:setup] -from ferro import Field, Model, connect, engines +# --8<-- [start:models] +from datetime import date +from decimal import Decimal +from typing import Annotated + +from ferro import BackRef, Field, ForeignKey, ManyToMany, Model, Relation -class User(Model): +class Owner(Model): id: int | None = Field(default=None, primary_key=True) name: str - age: int - role: str = "member" - archived: bool = False + email: str = Field(unique=True) + accounts: Relation[list["Account"]] = BackRef() + + +class Account(Model): + id: int | None = Field(default=None, primary_key=True) + label: str + owner: Annotated[Owner | None, ForeignKey(related_name="accounts")] = None + transactions: Relation[list["Transaction"]] = BackRef() -# --8<-- [end:setup] +class Category(Model): + id: int | None = Field(default=None, primary_key=True) + name: str = Field(unique=True) + parent: Annotated[ + "Category", ForeignKey(related_name="children", nullable=True) + ] = None + children: Relation[list["Category"]] = BackRef() + transactions: Relation[list["Transaction"]] = BackRef() + split_lines: Relation[list["SplitLine"]] = BackRef() -# --8<-- [start:nullable-model] -class Invoice(Model): +class Transaction(Model): + id: int | None = Field(default=None, primary_key=True) + memo: str + amount: Decimal + posted_on: date + cleared_on: date | None = None + account: Annotated[Account, ForeignKey(related_name="transactions")] + category: Annotated[Category | None, ForeignKey(related_name="transactions")] = None + lines: Relation[list["SplitLine"]] = BackRef() + transfer_out: "Transfer" = BackRef() + transfer_in: "Transfer" = BackRef() + tags: Relation[list["Tag"]] = BackRef() + + +class SplitLine(Model): id: int | None = Field(default=None, primary_key=True) - reference: str - amount: float | None = None # None until the invoice is issued + amount: Decimal + transaction: Annotated[Transaction, ForeignKey(related_name="lines")] + category: Annotated[Category, ForeignKey(related_name="split_lines")] -# --8<-- [end:nullable-model] +class Transfer(Model): + id: int | None = Field(default=None, primary_key=True) + outflow: Annotated[ + Transaction, ForeignKey(related_name="transfer_out", unique=True) + ] + inflow: Annotated[Transaction, ForeignKey(related_name="transfer_in", unique=True)] -# --8<-- [start:card-model] -class Card(Model): +class Tag(Model): id: int | None = Field(default=None, primary_key=True) - title: str - pinned_at: datetime | None = None - updated_at: datetime + name: str = Field(unique=True) + transactions: Relation[list[Transaction]] = ManyToMany(related_name="tags") -# --8<-- [end:card-model] +# --8<-- [end:models] async def main() -> None: - await connect("sqlite::memory:", auto_migrate=True) + from _ledger_seed import seed + + # --8<-- [start:first-query] + from ferro import connect, engines + await connect("sqlite::memory:", auto_migrate=True) async with engines.session(): - await User.bulk_create( - [ - User(name="alice", age=34, role="admin"), - User(name="bob", age=19), - User(name="carol", age=42, archived=True), - User(name="dave", age=17), - ] + await seed(globals()) # adds the rows this page queries + + large = ( + await Transaction.where(lambda transaction: transaction.amount <= -100) + .order_by(lambda transaction: transaction.amount) + .all() ) + for transaction in large: + print(transaction.memo, transaction.amount) + # --8<-- [end:first-query] + assert [transaction.memo for transaction in large] == ["Rent", "To savings"] # --8<-- [start:filtering] - adults = await User.where(lambda user: user.age >= 18).all() + uncleared = await Transaction.where( + lambda transaction: transaction.cleared_on == None + ).all() + assert {transaction.memo for transaction in uncleared} == { + "Train to Lyon", + "Dinner in Lyon", + "Weekly shop", + } # --8<-- [end:filtering] - assert len(adults) == 3 - # --8<-- [start:lambda-style] - admins = await User.where( - lambda user: (user.role == "admin") & (user.archived == False) # noqa: E712 - ).all() - # --8<-- [end:lambda-style] - assert len(admins) == 1 + # --8<-- [start:typo] + try: + Transaction.where(lambda transaction: transaction.amout < 0) + except AttributeError as error: + assert str(error) == ( + "Transaction has no queryable column 'amout'. Did you mean " + "'amount'? Valid columns: account_id, amount, category_id, " + "cleared_on, id, memo, posted_on. Valid relations: account, " + "category." + ) + else: + raise AssertionError("a misspelled column must raise AttributeError") + # --8<-- [end:typo] # --8<-- [start:operators] - teens = await User.where(lambda user: (user.age >= 13) & (user.age <= 19)).all() - a_names = await User.where(lambda user: user.name.like("a%")).all() - staff = await User.where( - lambda user: user.role.in_(["admin", "moderator"]) + coffee = await Transaction.where( + lambda transaction: transaction.memo.like("%coffee") + ).all() + assert {transaction.memo for transaction in coffee} == {"Office coffee"} + + lyon = await Transaction.where( + lambda transaction: transaction.posted_on.in_( + [date(2026, 3, 9), date(2026, 3, 10)] + ) ).all() + assert {transaction.memo for transaction in lyon} == { + "Train to Lyon", + "Dinner in Lyon", + } # --8<-- [end:operators] - assert len(teens) == 2 - assert len(a_names) == 1 - assert len(staff) == 1 # --8<-- [start:combining] - # & is AND, | is OR — parenthesize each side - flagged = await User.where( - lambda user: (user.age < 18) | (user.archived == True) # noqa: E712 + # & is AND, | is OR: parenthesize each side. + small_uncleared = await Transaction.where( + lambda transaction: ( + (transaction.cleared_on == None) & (transaction.amount > -90) + ) ).all() + assert {transaction.memo for transaction in small_uncleared} == { + "Train to Lyon", + "Dinner in Lyon", + } - # Chained .where() calls also AND together - young_members = ( - await User.where(lambda user: user.role == "member") - .where(lambda user: user.age < 21) + # Chained where() calls AND together. + march_debits = ( + await Transaction.where(lambda transaction: transaction.amount < 0) + .where(lambda transaction: transaction.posted_on >= date(2026, 3, 10)) .all() ) + assert {transaction.memo for transaction in march_debits} == { + "Dinner in Lyon", + "Office coffee", + "Weekly shop", + } # --8<-- [end:combining] - assert len(flagged) == 2 - assert len(young_members) == 2 # --8<-- [start:negation] - # ~ negates ANY predicate: a comparison, .in_(), .like(), or a whole - # &/| group. There are no per-operator negative forms to memorize. - not_staff = await User.where( - lambda user: ~user.role.in_(["admin", "moderator"]) - ).all() - not_a_names = await User.where(lambda user: ~user.name.like("a%")).all() - active_adults = await User.where( - lambda user: ~((user.age < 18) | (user.archived == True)) # noqa: E712 + not_lyon = await Transaction.where( + lambda transaction: ( + ~transaction.memo.like("%Lyon") & (transaction.amount < -50) + ) ).all() + assert {transaction.memo for transaction in not_lyon} == { + "Rent", + "Market run", + "To savings", + "Weekly shop", + } # --8<-- [end:negation] - assert len(not_staff) == 3 - assert len(not_a_names) == 3 - assert len(active_adults) == 2 - - await Invoice.bulk_create( - [ - Invoice(reference="inv-1", amount=250.0), - Invoice(reference="inv-2", amount=40.0), - Invoice(reference="inv-3", amount=None), - ] - ) # --8<-- [start:three-valued] - big = await Invoice.where(lambda invoice: invoice.amount > 100).all() - not_big = await Invoice.where(lambda invoice: ~(invoice.amount > 100)).all() - - # inv-3 (amount is NULL) appears in NEITHER list - assert {invoice.reference for invoice in big} == {"inv-1"} - assert {invoice.reference for invoice in not_big} == {"inv-2"} - - # keeping the NULL rows is an explicit extra condition - not_big_or_unissued = await Invoice.where( - lambda invoice: ~(invoice.amount > 100) | (invoice.amount == None) # noqa: E711 + before_march_5 = await Transaction.where( + lambda transaction: transaction.cleared_on < date(2026, 3, 5) + ).all() + not_before_march_5 = await Transaction.where( + lambda transaction: ~(transaction.cleared_on < date(2026, 3, 5)) ).all() - assert {invoice.reference for invoice in not_big_or_unissued} == { - "inv-2", - "inv-3", + assert {transaction.memo for transaction in before_march_5} == { + "Salary", + "Rent", + } + # The three uncleared rows are in neither list. + assert {transaction.memo for transaction in not_before_march_5} == { + "Market run", + "To savings", + "From checking", + "Office coffee", } # --8<-- [end:three-valued] - # --8<-- [start:ordering-slicing] - oldest_first = await User.select().order_by(lambda user: user.age, "desc").all() - second_page = ( - await User.select().order_by(lambda user: user.id).limit(2).offset(2).all() - ) - # --8<-- [end:ordering-slicing] - assert oldest_first[0].name == "carol" - assert len(second_page) == 2 + # --8<-- [start:keep-nulls] + not_before_or_uncleared = await Transaction.where( + lambda transaction: ( + ~(transaction.cleared_on < date(2026, 3, 5)) + | (transaction.cleared_on == None) + ) + ).all() + assert len(not_before_or_uncleared) == 7 + # --8<-- [end:keep-nulls] - # --8<-- [start:after-paging] - page = ( - await User.select() - .order_by(lambda user: user.age) - .order_by(lambda user: user.id) - .limit(2) - .all() - ) - next_page = ( - await User.select() - .order_by(lambda user: user.age) - .order_by(lambda user: user.id) - .after(page[-1]) - .limit(2) - .all() - ) - # --8<-- [end:after-paging] - assert [user.name for user in page] == ["dave", "bob"] - assert [user.name for user in next_page] == ["alice", "carol"] + # --8<-- [start:get] + from ferro import ModelDoesNotExist - # --8<-- [start:before-paging] - previous_page = ( - await User.select() - .order_by(lambda user: user.age) - .order_by(lambda user: user.id) - .before(next_page[0]) - .limit(2) + rent = await Transaction.get(2) + assert rent.memo == "Rent" + + try: + await Transaction.get(99) + except ModelDoesNotExist as error: + assert (error.model, error.pk) == (Transaction, 99) + else: + raise AssertionError("a missing row must raise ModelDoesNotExist") + + assert await Transaction.get_or_none(99) is None + # --8<-- [end:get] + + # --8<-- [start:ordering-slicing] + largest_debits = ( + await Transaction.select() + .order_by(lambda transaction: transaction.amount) + .limit(3) .all() ) - earlier = ( - await User.select() - .order_by(lambda user: user.age) - .order_by(lambda user: user.id) - .before(next_page[0]) + assert [transaction.memo for transaction in largest_debits] == [ + "Rent", + "To savings", + "Weekly shop", + ] + + next_three = ( + await Transaction.select() + .order_by(lambda transaction: transaction.amount) + .limit(3) + .offset(3) .all() ) - adjacent = ( - await User.select() - .order_by(lambda user: user.age) - .order_by(lambda user: user.id) - .before(next_page[0]) - .first() - ) - # --8<-- [end:before-paging] - assert [user.name for user in previous_page] == ["dave", "bob"] - assert [user.name for user in earlier] == ["dave", "bob"] - assert adjacent is not None and adjacent.name == "bob" - assert earlier[0].name == "dave" - assert adjacent.name != earlier[0].name - - t0 = datetime(2026, 1, 1, tzinfo=UTC) - t1 = datetime(2026, 2, 1, tzinfo=UTC) - t2 = datetime(2026, 3, 1, tzinfo=UTC) - await Card.bulk_create( - [ - Card(title="unpinned-old", pinned_at=None, updated_at=t0), - Card(title="pinned-early", pinned_at=t0, updated_at=t0), - Card(title="unpinned-new", pinned_at=None, updated_at=t2), - Card(title="pinned-late", pinned_at=t1, updated_at=t1), - ] - ) + assert [transaction.memo for transaction in next_three] == [ + "Market run", + "Train to Lyon", + "Dinner in Lyon", + ] + # --8<-- [end:ordering-slicing] # --8<-- [start:nulls-ordering] - cards = ( - await Card.select() - .order_by(lambda card: card.pinned_at, "desc", nulls="last") - .order_by(lambda card: card.updated_at, "desc") - .order_by(lambda card: card.id, "desc") + uncleared_first = ( + await Transaction.select() + .order_by(lambda transaction: transaction.cleared_on, "desc", nulls="first") + .order_by(lambda transaction: transaction.id) + .limit(4) .all() ) + assert [transaction.memo for transaction in uncleared_first] == [ + "Train to Lyon", + "Dinner in Lyon", + "Weekly shop", + "Office coffee", + ] # --8<-- [end:nulls-ordering] - assert [card.title for card in cards] == [ - "pinned-late", - "pinned-early", - "unpinned-new", - "unpinned-old", + + # --8<-- [start:after-paging] + by_date = ( + Transaction.select() + .order_by(lambda transaction: transaction.posted_on) + .order_by(lambda transaction: transaction.id) + ) + first_page = await by_date.limit(3).all() + second_page = await by_date.after(first_page[-1]).limit(3).all() + assert [transaction.memo for transaction in first_page] == [ + "Salary", + "Rent", + "Market run", ] + assert [transaction.memo for transaction in second_page] == [ + "To savings", + "From checking", + "Train to Lyon", + ] + # --8<-- [end:after-paging] + + # --8<-- [start:position-of] + position = by_date.position_of(first_page[-1]) + assert position == (date(2026, 3, 5), 3) + + same_page = await by_date.after(position).limit(3).all() + assert same_page == second_page + # --8<-- [end:position-of] + + # --8<-- [start:before-paging] + previous_page = await by_date.before(second_page[0]).limit(2).all() + assert [transaction.memo for transaction in previous_page] == [ + "Rent", + "Market run", + ] + + everything_earlier = await by_date.before(second_page[0]).all() + assert [transaction.memo for transaction in everything_earlier] == [ + "Salary", + "Rent", + "Market run", + ] + + previous_row = await by_date.before(second_page[0]).first() + assert previous_row is not None and previous_row.memo == "Market run" + # --8<-- [end:before-paging] # --8<-- [start:after-null-paging] - last_pinned = cards[1] - unpinned_page = ( - await Card.select() - .order_by(lambda card: card.pinned_at, "desc") - .order_by(lambda card: card.updated_at, "desc") - .order_by(lambda card: card.id, "desc") - .after(last_pinned) - .all() + by_clearing = ( + Transaction.select() + .order_by(lambda transaction: transaction.cleared_on, "desc") + .order_by(lambda transaction: transaction.id) ) - remaining_unpinned = ( - await Card.select() - .order_by(lambda card: card.pinned_at, "desc") - .order_by(lambda card: card.updated_at, "desc") - .order_by(lambda card: card.id, "desc") - .after((None, cards[2].updated_at, cards[2].id)) - .all() - ) - # --8<-- [end:after-null-paging] - assert [card.title for card in unpinned_page] == [ - "unpinned-new", - "unpinned-old", + rest_of_uncleared = await by_clearing.after((None, 6)).all() + assert [transaction.memo for transaction in rest_of_uncleared] == [ + "Dinner in Lyon", + "Weekly shop", ] - assert [card.title for card in remaining_unpinned] == ["unpinned-old"] + # --8<-- [end:after-null-paging] # --8<-- [start:terminals] - everyone = await User.all() - first_admin = await User.where(lambda user: user.role == "admin").first() - headcount = await User.select().count() - any_minors = await User.where(lambda user: user.age < 18).exists() + everything = await Transaction.all() + assert len(everything) == 9 + + newest = ( + await Transaction.select() + .order_by(lambda transaction: transaction.posted_on, "desc") + .first() + ) + assert newest is not None and newest.memo == "Weekly shop" + + uncleared_count = await Transaction.where( + lambda transaction: transaction.cleared_on == None + ).count() + assert uncleared_count == 3 + + any_refunds = await Transaction.where( + lambda transaction: transaction.memo.like("Refund%") + ).exists() + assert any_refunds is False # --8<-- [end:terminals] - assert len(everyone) == 4 - assert first_admin is not None - assert headcount == 4 - assert any_minors + + # --8<-- [start:immutability] + debits = Transaction.where(lambda transaction: transaction.amount < 0) + + debit_count = await debits.count() + biggest = await debits.order_by(lambda transaction: transaction.amount).first() + uncleared_debits = await debits.where( + lambda transaction: transaction.cleared_on == None + ).count() + + assert debit_count == 7 + assert biggest is not None and biggest.memo == "Rent" + assert uncleared_debits == 3 + assert await debits.count() == 7 # debits itself never changed + # --8<-- [end:immutability] print("predicates example ran successfully") diff --git a/docs/examples/predicates_annotated.py b/docs/examples/predicates_annotated.py index 39a3a29d..21bef9f6 100644 --- a/docs/examples/predicates_annotated.py +++ b/docs/examples/predicates_annotated.py @@ -1,94 +1,371 @@ -"""Annotated-style companion to predicates.py (AGENTS.md I-7).""" +"""Runnable companion to the Queries guide (docs/pages/guide/queries.md). + +Declares the ledger every section of the guide queries, adds its rows +(``_ledger_seed.py``), then runs the filtering, ordering, paging and terminal +examples and asserts the rows that come back. The guide's other example +scripts import these models from here. +""" import asyncio -from datetime import UTC, datetime -# --8<-- [start:setup] +# --8<-- [start:models] +from datetime import date +from decimal import Decimal from typing import Annotated -from ferro import Field, Model, connect, engines +from ferro import BackRef, Field, ForeignKey, ManyToMany, Model, Relation -class User(Model): - id: Annotated[int | None, Field(default=None, primary_key=True)] +class Owner(Model): + id: Annotated[int | None, Field(primary_key=True)] = None name: str - age: int - role: str = "member" - archived: bool = False + email: Annotated[str, Field(unique=True)] + accounts: Relation[list["Account"]] = BackRef() + + +class Account(Model): + id: Annotated[int | None, Field(primary_key=True)] = None + label: str + owner: Annotated[Owner | None, ForeignKey(related_name="accounts")] = None + transactions: Relation[list["Transaction"]] = BackRef() -# --8<-- [end:setup] +class Category(Model): + id: Annotated[int | None, Field(primary_key=True)] = None + name: Annotated[str, Field(unique=True)] + parent: Annotated[ + "Category", ForeignKey(related_name="children", nullable=True) + ] = None + children: Relation[list["Category"]] = BackRef() + transactions: Relation[list["Transaction"]] = BackRef() + split_lines: Relation[list["SplitLine"]] = BackRef() -# --8<-- [start:nullable-model] -class Invoice(Model): - id: Annotated[int | None, Field(default=None, primary_key=True)] - reference: str - amount: float | None = None # None until the invoice is issued +class Transaction(Model): + id: Annotated[int | None, Field(primary_key=True)] = None + memo: str + amount: Decimal + posted_on: date + cleared_on: date | None = None + account: Annotated[Account, ForeignKey(related_name="transactions")] + category: Annotated[Category | None, ForeignKey(related_name="transactions")] = None + lines: Relation[list["SplitLine"]] = BackRef() + transfer_out: "Transfer" = BackRef() + transfer_in: "Transfer" = BackRef() + tags: Relation[list["Tag"]] = BackRef() -# --8<-- [end:nullable-model] +class SplitLine(Model): + id: Annotated[int | None, Field(primary_key=True)] = None + amount: Decimal + transaction: Annotated[Transaction, ForeignKey(related_name="lines")] + category: Annotated[Category, ForeignKey(related_name="split_lines")] -# --8<-- [start:card-model] -class Card(Model): - id: Annotated[int | None, Field(default=None, primary_key=True)] - title: str - pinned_at: datetime | None = None - updated_at: datetime +class Transfer(Model): + id: Annotated[int | None, Field(primary_key=True)] = None + outflow: Annotated[ + Transaction, ForeignKey(related_name="transfer_out", unique=True) + ] + inflow: Annotated[Transaction, ForeignKey(related_name="transfer_in", unique=True)] -# --8<-- [end:card-model] +class Tag(Model): + id: Annotated[int | None, Field(primary_key=True)] = None + name: Annotated[str, Field(unique=True)] + transactions: Relation[list[Transaction]] = ManyToMany(related_name="tags") + + +# --8<-- [end:models] async def main() -> None: - await connect("sqlite::memory:", auto_migrate=True) + from _ledger_seed import seed + # --8<-- [start:first-query] + from ferro import connect, engines + + await connect("sqlite::memory:", auto_migrate=True) async with engines.session(): - await User.bulk_create( - [ - User(name="alice", age=34, role="admin"), - User(name="bob", age=19), - ] + await seed(globals()) # adds the rows this page queries + + large = ( + await Transaction.where(lambda transaction: transaction.amount <= -100) + .order_by(lambda transaction: transaction.amount) + .all() ) + for transaction in large: + print(transaction.memo, transaction.amount) + # --8<-- [end:first-query] + assert [transaction.memo for transaction in large] == ["Rent", "To savings"] + + # --8<-- [start:filtering] + uncleared = await Transaction.where( + lambda transaction: transaction.cleared_on == None + ).all() + assert {transaction.memo for transaction in uncleared} == { + "Train to Lyon", + "Dinner in Lyon", + "Weekly shop", + } + # --8<-- [end:filtering] - adults = await User.where(lambda user: user.age >= 18).all() - assert len(adults) == 2 + # --8<-- [start:typo] + try: + Transaction.where(lambda transaction: transaction.amout < 0) + except AttributeError as error: + assert str(error) == ( + "Transaction has no queryable column 'amout'. Did you mean " + "'amount'? Valid columns: account_id, amount, category_id, " + "cleared_on, id, memo, posted_on. Valid relations: account, " + "category." + ) + else: + raise AssertionError("a misspelled column must raise AttributeError") + # --8<-- [end:typo] - not_admins = await User.where(lambda user: ~(user.role == "admin")).all() - assert len(not_admins) == 1 + # --8<-- [start:operators] + coffee = await Transaction.where( + lambda transaction: transaction.memo.like("%coffee") + ).all() + assert {transaction.memo for transaction in coffee} == {"Office coffee"} - await Invoice.bulk_create( - [ - Invoice(reference="inv-1", amount=250.0), - Invoice(reference="inv-2", amount=None), - ] + lyon = await Transaction.where( + lambda transaction: transaction.posted_on.in_( + [date(2026, 3, 9), date(2026, 3, 10)] + ) + ).all() + assert {transaction.memo for transaction in lyon} == { + "Train to Lyon", + "Dinner in Lyon", + } + # --8<-- [end:operators] + + # --8<-- [start:combining] + # & is AND, | is OR: parenthesize each side. + small_uncleared = await Transaction.where( + lambda transaction: ( + (transaction.cleared_on == None) & (transaction.amount > -90) + ) + ).all() + assert {transaction.memo for transaction in small_uncleared} == { + "Train to Lyon", + "Dinner in Lyon", + } + + # Chained where() calls AND together. + march_debits = ( + await Transaction.where(lambda transaction: transaction.amount < 0) + .where(lambda transaction: transaction.posted_on >= date(2026, 3, 10)) + .all() ) + assert {transaction.memo for transaction in march_debits} == { + "Dinner in Lyon", + "Office coffee", + "Weekly shop", + } + # --8<-- [end:combining] + + # --8<-- [start:negation] + not_lyon = await Transaction.where( + lambda transaction: ( + ~transaction.memo.like("%Lyon") & (transaction.amount < -50) + ) + ).all() + assert {transaction.memo for transaction in not_lyon} == { + "Rent", + "Market run", + "To savings", + "Weekly shop", + } + # --8<-- [end:negation] + + # --8<-- [start:three-valued] + before_march_5 = await Transaction.where( + lambda transaction: transaction.cleared_on < date(2026, 3, 5) + ).all() + not_before_march_5 = await Transaction.where( + lambda transaction: ~(transaction.cleared_on < date(2026, 3, 5)) + ).all() + assert {transaction.memo for transaction in before_march_5} == { + "Salary", + "Rent", + } + # The three uncleared rows are in neither list. + assert {transaction.memo for transaction in not_before_march_5} == { + "Market run", + "To savings", + "From checking", + "Office coffee", + } + # --8<-- [end:three-valued] + + # --8<-- [start:keep-nulls] + not_before_or_uncleared = await Transaction.where( + lambda transaction: ( + ~(transaction.cleared_on < date(2026, 3, 5)) + | (transaction.cleared_on == None) + ) + ).all() + assert len(not_before_or_uncleared) == 7 + # --8<-- [end:keep-nulls] + + # --8<-- [start:get] + from ferro import ModelDoesNotExist + + rent = await Transaction.get(2) + assert rent.memo == "Rent" + + try: + await Transaction.get(99) + except ModelDoesNotExist as error: + assert (error.model, error.pk) == (Transaction, 99) + else: + raise AssertionError("a missing row must raise ModelDoesNotExist") + + assert await Transaction.get_or_none(99) is None + # --8<-- [end:get] - # three-valued logic: the NULL-amount row matches neither predicate - assert len(await Invoice.where(lambda invoice: invoice.amount > 100).all()) == 1 - assert ( - len(await Invoice.where(lambda invoice: ~(invoice.amount > 100)).all()) == 0 + # --8<-- [start:ordering-slicing] + largest_debits = ( + await Transaction.select() + .order_by(lambda transaction: transaction.amount) + .limit(3) + .all() ) + assert [transaction.memo for transaction in largest_debits] == [ + "Rent", + "To savings", + "Weekly shop", + ] - t0 = datetime(2026, 1, 1, tzinfo=UTC) - t1 = datetime(2026, 2, 1, tzinfo=UTC) - await Card.bulk_create( - [ - Card(title="unpinned", pinned_at=None, updated_at=t1), - Card(title="pinned", pinned_at=t0, updated_at=t0), - ] + next_three = ( + await Transaction.select() + .order_by(lambda transaction: transaction.amount) + .limit(3) + .offset(3) + .all() ) - cards = ( - await Card.select() - .order_by(lambda card: card.pinned_at, "desc", nulls="last") - .order_by(lambda card: card.updated_at, "desc") - .order_by(lambda card: card.id, "desc") + assert [transaction.memo for transaction in next_three] == [ + "Market run", + "Train to Lyon", + "Dinner in Lyon", + ] + # --8<-- [end:ordering-slicing] + + # --8<-- [start:nulls-ordering] + uncleared_first = ( + await Transaction.select() + .order_by(lambda transaction: transaction.cleared_on, "desc", nulls="first") + .order_by(lambda transaction: transaction.id) + .limit(4) .all() ) - assert [card.title for card in cards] == ["pinned", "unpinned"] + assert [transaction.memo for transaction in uncleared_first] == [ + "Train to Lyon", + "Dinner in Lyon", + "Weekly shop", + "Office coffee", + ] + # --8<-- [end:nulls-ordering] + + # --8<-- [start:after-paging] + by_date = ( + Transaction.select() + .order_by(lambda transaction: transaction.posted_on) + .order_by(lambda transaction: transaction.id) + ) + first_page = await by_date.limit(3).all() + second_page = await by_date.after(first_page[-1]).limit(3).all() + assert [transaction.memo for transaction in first_page] == [ + "Salary", + "Rent", + "Market run", + ] + assert [transaction.memo for transaction in second_page] == [ + "To savings", + "From checking", + "Train to Lyon", + ] + # --8<-- [end:after-paging] + + # --8<-- [start:position-of] + position = by_date.position_of(first_page[-1]) + assert position == (date(2026, 3, 5), 3) + + same_page = await by_date.after(position).limit(3).all() + assert same_page == second_page + # --8<-- [end:position-of] + + # --8<-- [start:before-paging] + previous_page = await by_date.before(second_page[0]).limit(2).all() + assert [transaction.memo for transaction in previous_page] == [ + "Rent", + "Market run", + ] + + everything_earlier = await by_date.before(second_page[0]).all() + assert [transaction.memo for transaction in everything_earlier] == [ + "Salary", + "Rent", + "Market run", + ] + + previous_row = await by_date.before(second_page[0]).first() + assert previous_row is not None and previous_row.memo == "Market run" + # --8<-- [end:before-paging] + + # --8<-- [start:after-null-paging] + by_clearing = ( + Transaction.select() + .order_by(lambda transaction: transaction.cleared_on, "desc") + .order_by(lambda transaction: transaction.id) + ) + rest_of_uncleared = await by_clearing.after((None, 6)).all() + assert [transaction.memo for transaction in rest_of_uncleared] == [ + "Dinner in Lyon", + "Weekly shop", + ] + # --8<-- [end:after-null-paging] + + # --8<-- [start:terminals] + everything = await Transaction.all() + assert len(everything) == 9 + + newest = ( + await Transaction.select() + .order_by(lambda transaction: transaction.posted_on, "desc") + .first() + ) + assert newest is not None and newest.memo == "Weekly shop" + + uncleared_count = await Transaction.where( + lambda transaction: transaction.cleared_on == None + ).count() + assert uncleared_count == 3 + + any_refunds = await Transaction.where( + lambda transaction: transaction.memo.like("Refund%") + ).exists() + assert any_refunds is False + # --8<-- [end:terminals] + + # --8<-- [start:immutability] + debits = Transaction.where(lambda transaction: transaction.amount < 0) + + debit_count = await debits.count() + biggest = await debits.order_by(lambda transaction: transaction.amount).first() + uncleared_debits = await debits.where( + lambda transaction: transaction.cleared_on == None + ).count() + + assert debit_count == 7 + assert biggest is not None and biggest.memo == "Rent" + assert uncleared_debits == 3 + assert await debits.count() == 7 # debits itself never changed + # --8<-- [end:immutability] - print("predicates_annotated example ran successfully") + print("predicates example ran successfully") if __name__ == "__main__": diff --git a/docs/examples/quickstart.py b/docs/examples/quickstart.py index b9159007..e62f6a59 100644 --- a/docs/examples/quickstart.py +++ b/docs/examples/quickstart.py @@ -1,21 +1,17 @@ """Runnable companion to the Quickstart tutorial (docs/pages/getting-started/quickstart.md).""" +# --8<-- [start:imports] import asyncio +from ferro import connect, engines, transaction + +# --8<-- [end:imports] + # --8<-- [start:models] -from datetime import datetime +from datetime import UTC, datetime from typing import Annotated -from ferro import ( - BackRef, - Field, - ForeignKey, - Model, - Relation, - connect, - engines, - transaction, -) +from ferro import BackRef, Field, ForeignKey, Model, Relation class Author(Model): @@ -30,7 +26,7 @@ class Post(Model): title: str body: str published: bool = False - created_at: datetime = Field(default_factory=datetime.now) + created_at: datetime = Field(default_factory=lambda: datetime.now(UTC)) author: Annotated[Author, ForeignKey(related_name="posts")] @@ -40,92 +36,95 @@ class Post(Model): async def main() -> None: # --8<-- [start:connect] await connect("sqlite::memory:", auto_migrate=True) - # --8<-- [end:connect] async with engines.session(): + assert await Author.select().count() == 0 + # --8<-- [end:connect] + # --8<-- [start:create] alice = await Author.create(name="Alice", email="alice@example.com") - post = await Post.create( - title="Why Ferro is Fast", - body="Ferro hands SQL generation and row hydration to a Rust engine...", + title="Why Ferro is fast", + body="Ferro hands SQL generation and row hydration to a Rust engine.", published=True, author=alice, ) + assert post.id == 1 # the database assigned it - # Insert many rows in a single statement await Post.bulk_create( [ Post( - title="Async Patterns", - body="...", + title="Async patterns", + body="One event loop, many queries.", published=True, - author_id=alice.id, + author=alice, + ), + Post( + title="Unfinished draft", + body="Still to write.", + author=alice, ), - Post(title="Unfinished Draft", body="...", author_id=alice.id), ] ) + assert await Post.select().count() == 3 # --8<-- [end:create] - assert post.id is not None - # --8<-- [start:query] - # Fetch by primary key + # --8<-- [start:get] same_post = await Post.get(post.id) + assert same_post is post # one row, one object + # --8<-- [end:get] - # Filter, order, and slice + # --8<-- [start:filter] published = ( - await Post.where(lambda post: post.published == True) # noqa: E712 - .order_by(lambda post: post.created_at, "desc") + await Post.where(lambda post: post.published == True) + .order_by(lambda post: post.title) .limit(10) .all() ) + assert [post.title for post in published] == [ + "Async patterns", + "Why Ferro is fast", + ] + # --8<-- [end:filter] - # Aggregate terminals + # --8<-- [start:aggregate] total = await Post.select().count() - has_drafts = await Post.where(lambda post: post.published == False).exists() # noqa: E712 - # --8<-- [end:query] - assert same_post is post # identity map: same Python object - assert len(published) == 2 - assert total == 3 - assert has_drafts + has_drafts = await Post.where(lambda post: post.published == False).exists() + assert (total, has_drafts) == (3, True) + # --8<-- [end:aggregate] # --8<-- [start:relationships] - # Forward access: awaiting the foreign key loads the related instance - author = await same_post.author + author = await post.author + assert author.name == "Alice" - # Reverse access: the BackRef is a chainable query alice_posts = await author.posts.where( - lambda post: post.published == True # noqa: E712 + lambda post: post.published == True ).all() - # --8<-- [end:relationships] - assert author.name == "Alice" assert len(alice_posts) == 2 + # --8<-- [end:relationships] # --8<-- [start:update-delete] - # Update one instance - post.title = "Why Ferro is *Really* Fast" + post.title = "Why Ferro is really fast" await post.save() + assert (await Post.get(post.id)).title == "Why Ferro is really fast" - # Update many rows at once - updated = await Post.where(lambda post: post.published == False).update( # noqa: E712 + updated = await Post.where(lambda post: post.published == False).update( published=True ) + assert updated == 1 - # Delete deleted = await Post.where( - lambda post: post.title == "Unfinished Draft" + lambda post: post.title == "Unfinished draft" ).delete() - # --8<-- [end:update-delete] - assert updated == 1 assert deleted == 1 + # --8<-- [end:update-delete] # --8<-- [start:transaction] async with transaction(): bob = await Author.create(name="Bob", email="bob@example.com") - await Post.create(title="Hello", body="...", author=bob) - # Commits on success, rolls back if the block raises + await Post.create(title="Hello", body="First post.", author=bob) + assert await Author.select().count() == 2 # both writes committed # --8<-- [end:transaction] - assert await Author.select().count() == 2 print("quickstart example ran successfully") diff --git a/docs/examples/quickstart_annotated.py b/docs/examples/quickstart_annotated.py index 5badb8c0..0d2b614d 100644 --- a/docs/examples/quickstart_annotated.py +++ b/docs/examples/quickstart_annotated.py @@ -1,32 +1,32 @@ -"""Annotated-style companion to quickstart.py (AGENTS.md I-7). - -Same models as docs/examples/quickstart.py, declared with ``Annotated`` -metadata instead of assignment. Field options move into ``Annotated[...]``; -relationship declarations are identical in both styles. -""" +"""Runnable companion to the Quickstart tutorial (docs/pages/getting-started/quickstart.md).""" +# --8<-- [start:imports] import asyncio +from ferro import connect, engines, transaction + +# --8<-- [end:imports] + # --8<-- [start:models] -from datetime import datetime +from datetime import UTC, datetime from typing import Annotated -from ferro import BackRef, Field, ForeignKey, Model, Relation, connect, engines +from ferro import BackRef, Field, ForeignKey, Model, Relation class Author(Model): - id: Annotated[int | None, Field(default=None, primary_key=True)] + id: Annotated[int | None, Field(primary_key=True)] = None name: str email: Annotated[str, Field(unique=True)] posts: Relation[list["Post"]] = BackRef() class Post(Model): - id: Annotated[int | None, Field(default=None, primary_key=True)] + id: Annotated[int | None, Field(primary_key=True)] = None title: str body: str published: bool = False - created_at: Annotated[datetime, Field(default_factory=datetime.now)] + created_at: datetime = Field(default_factory=lambda: datetime.now(UTC)) author: Annotated[Author, ForeignKey(related_name="posts")] @@ -34,21 +34,99 @@ class Post(Model): async def main() -> None: + # --8<-- [start:connect] await connect("sqlite::memory:", auto_migrate=True) async with engines.session(): + assert await Author.select().count() == 0 + # --8<-- [end:connect] + + # --8<-- [start:create] alice = await Author.create(name="Alice", email="alice@example.com") post = await Post.create( - title="Hello", body="...", published=True, author=alice + title="Why Ferro is fast", + body="Ferro hands SQL generation and row hydration to a Rust engine.", + published=True, + author=alice, ) - - assert post.id is not None - assert (await post.author).email == "alice@example.com" - assert ( - len(await alice.posts.where(lambda post: post.published == True).all()) == 1 # noqa: E712 + assert post.id == 1 # the database assigned it + + await Post.bulk_create( + [ + Post( + title="Async patterns", + body="One event loop, many queries.", + published=True, + author=alice, + ), + Post( + title="Unfinished draft", + body="Still to write.", + author=alice, + ), + ] ) - - print("quickstart_annotated example ran successfully") + assert await Post.select().count() == 3 + # --8<-- [end:create] + + # --8<-- [start:get] + same_post = await Post.get(post.id) + assert same_post is post # one row, one object + # --8<-- [end:get] + + # --8<-- [start:filter] + published = ( + await Post.where(lambda post: post.published == True) + .order_by(lambda post: post.title) + .limit(10) + .all() + ) + assert [post.title for post in published] == [ + "Async patterns", + "Why Ferro is fast", + ] + # --8<-- [end:filter] + + # --8<-- [start:aggregate] + total = await Post.select().count() + has_drafts = await Post.where(lambda post: post.published == False).exists() + assert (total, has_drafts) == (3, True) + # --8<-- [end:aggregate] + + # --8<-- [start:relationships] + author = await post.author + assert author.name == "Alice" + + alice_posts = await author.posts.where( + lambda post: post.published == True + ).all() + assert len(alice_posts) == 2 + # --8<-- [end:relationships] + + # --8<-- [start:update-delete] + post.title = "Why Ferro is really fast" + await post.save() + assert (await Post.get(post.id)).title == "Why Ferro is really fast" + + updated = await Post.where(lambda post: post.published == False).update( + published=True + ) + assert updated == 1 + + deleted = await Post.where( + lambda post: post.title == "Unfinished draft" + ).delete() + assert deleted == 1 + # --8<-- [end:update-delete] + + # --8<-- [start:transaction] + async with transaction(): + bob = await Author.create(name="Bob", email="bob@example.com") + await Post.create(title="Hello", body="First post.", author=bob) + assert await Author.select().count() == 2 # both writes committed + # --8<-- [end:transaction] + + print("quickstart example ran successfully") if __name__ == "__main__": diff --git a/docs/examples/raw_sql.py b/docs/examples/raw_sql.py index c35161b3..60aac5a2 100644 --- a/docs/examples/raw_sql.py +++ b/docs/examples/raw_sql.py @@ -1,57 +1,90 @@ -"""Runnable companion to the Raw SQL guide (docs/pages/guide/raw-sql.md).""" +"""Runnable companion to the Raw SQL guide (docs/pages/guide/raw-sql.md). + +Every block on the page uses the ``event`` table this script declares and +fills; the ``raw_sql`` prelude (``docs/examples/_preludes/raw_sql.py``) +imports both for the page's inline blocks. +""" import asyncio +from datetime import UTC, datetime -from ferro import ( - Field, - Model, - connect, - engines, - execute, - fetch_all, - fetch_one, - transaction, -) +from ferro import Field, Model +# --8<-- [start:models] class Event(Model): id: int | None = Field(default=None, primary_key=True) kind: str - payload: str = "" + occurred_at: datetime + + +# --8<-- [end:models] + + +async def add_events() -> None: + """Add the page's three events: two clicks and a signup.""" + await Event.bulk_create( + [ + Event(kind="click", occurred_at=datetime(2026, 3, 2, 9, 0, tzinfo=UTC)), + Event(kind="signup", occurred_at=datetime(2026, 3, 2, 9, 5, tzinfo=UTC)), + Event(kind="click", occurred_at=datetime(2026, 3, 2, 9, 30, tzinfo=UTC)), + ] + ) async def main() -> None: - await connect("sqlite::memory:", auto_migrate=True) + # --8<-- [start:first] + from ferro import connect, engines, fetch_all + await connect("sqlite::memory:", auto_migrate=True) async with engines.session(): - await Event.bulk_create( - [Event(kind="click"), Event(kind="click"), Event(kind="signup")] - ) + await add_events() # two clicks and a signup - # --8<-- [start:execute] - affected = await execute( - "UPDATE event SET payload = ? WHERE kind = ?", "{}", "click" + counts = await fetch_all( + "SELECT kind, COUNT(*) AS n FROM event GROUP BY kind ORDER BY n DESC" ) - # --8<-- [end:execute] - assert affected == 2 + print(counts) + # --8<-- [end:first] + assert counts == [{"kind": "click", "n": 2}, {"kind": "signup", "n": 1}] - # --8<-- [start:fetch] - rows = await fetch_all( - "SELECT kind, COUNT(*) AS n FROM event GROUP BY kind ORDER BY n DESC" + # --8<-- [start:fetch-one] + from ferro import fetch_one + + earliest = await fetch_one( + "SELECT kind, occurred_at FROM event ORDER BY occurred_at LIMIT 1" ) - top = await fetch_one( - "SELECT kind FROM event GROUP BY kind ORDER BY COUNT(*) DESC LIMIT 1" + assert earliest == {"kind": "click", "occurred_at": "2026-03-02T09:00:00Z"} + + nothing = await fetch_one("SELECT kind FROM event WHERE kind = ?", "purchase") + assert nothing is None + # --8<-- [end:fetch-one] + + # --8<-- [start:execute] + from ferro import execute + + renamed = await execute( + "UPDATE event SET kind = ? WHERE kind = ?", "tap", "click" ) - # --8<-- [end:fetch] - assert rows[0]["n"] == 2 - assert top is not None and top["kind"] == "click" + assert renamed == 2 + # --8<-- [end:execute] # --8<-- [start:in-transaction] + from ferro import transaction + async with transaction() as tx: - await tx.execute("DELETE FROM event WHERE kind = ?", "click") - remaining = await tx.fetch_all("SELECT * FROM event") + await tx.execute("DELETE FROM event WHERE kind = ?", "tap") + remaining = await tx.fetch_all("SELECT id, kind FROM event") + assert remaining == [{"id": 2, "kind": "signup"}] # --8<-- [end:in-transaction] - assert len(remaining) == 1 + + # --8<-- [start:no-session] + try: + await fetch_all("SELECT kind FROM event") + except RuntimeError as error: + assert str(error).startswith("No database route for this operation.") + else: + raise AssertionError("raw SQL outside a session must raise RuntimeError") + # --8<-- [end:no-session] print("raw_sql example ran successfully") diff --git a/docs/examples/raw_sql_annotated.py b/docs/examples/raw_sql_annotated.py new file mode 100644 index 00000000..77d3b751 --- /dev/null +++ b/docs/examples/raw_sql_annotated.py @@ -0,0 +1,94 @@ +"""Runnable companion to the Raw SQL guide (docs/pages/guide/raw-sql.md). + +Every block on the page uses the ``event`` table this script declares and +fills; the ``raw_sql`` prelude (``docs/examples/_preludes/raw_sql.py``) +imports both for the page's inline blocks. +""" + +import asyncio +from datetime import UTC, datetime +from typing import Annotated + +from ferro import Field, Model + + +# --8<-- [start:models] +class Event(Model): + id: Annotated[int | None, Field(primary_key=True)] = None + kind: str + occurred_at: datetime + + +# --8<-- [end:models] + + +async def add_events() -> None: + """Add the page's three events: two clicks and a signup.""" + await Event.bulk_create( + [ + Event(kind="click", occurred_at=datetime(2026, 3, 2, 9, 0, tzinfo=UTC)), + Event(kind="signup", occurred_at=datetime(2026, 3, 2, 9, 5, tzinfo=UTC)), + Event(kind="click", occurred_at=datetime(2026, 3, 2, 9, 30, tzinfo=UTC)), + ] + ) + + +async def main() -> None: + # --8<-- [start:first] + from ferro import connect, engines, fetch_all + + await connect("sqlite::memory:", auto_migrate=True) + async with engines.session(): + await add_events() # two clicks and a signup + + counts = await fetch_all( + "SELECT kind, COUNT(*) AS n FROM event GROUP BY kind ORDER BY n DESC" + ) + print(counts) + # --8<-- [end:first] + assert counts == [{"kind": "click", "n": 2}, {"kind": "signup", "n": 1}] + + # --8<-- [start:fetch-one] + from ferro import fetch_one + + earliest = await fetch_one( + "SELECT kind, occurred_at FROM event ORDER BY occurred_at LIMIT 1" + ) + assert earliest == {"kind": "click", "occurred_at": "2026-03-02T09:00:00Z"} + + nothing = await fetch_one("SELECT kind FROM event WHERE kind = ?", "purchase") + assert nothing is None + # --8<-- [end:fetch-one] + + # --8<-- [start:execute] + from ferro import execute + + renamed = await execute( + "UPDATE event SET kind = ? WHERE kind = ?", "tap", "click" + ) + assert renamed == 2 + # --8<-- [end:execute] + + # --8<-- [start:in-transaction] + from ferro import transaction + + async with transaction() as tx: + await tx.execute("DELETE FROM event WHERE kind = ?", "tap") + remaining = await tx.fetch_all("SELECT id, kind FROM event") + assert remaining == [{"id": 2, "kind": "signup"}] + # --8<-- [end:in-transaction] + + # --8<-- [start:no-session] + try: + await fetch_all("SELECT kind FROM event") + except RuntimeError as error: + assert str(error).startswith("No database route for this operation.") + else: + raise AssertionError("raw SQL outside a session must raise RuntimeError") + # --8<-- [end:no-session] + + print("raw_sql example ran successfully") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/docs/examples/readme_quickstart.py b/docs/examples/readme_quickstart.py new file mode 100644 index 00000000..05cebcbf --- /dev/null +++ b/docs/examples/readme_quickstart.py @@ -0,0 +1,36 @@ +"""The README quick start (README.md), runnable. + +tests/test_docs_examples.py checks that the README's Python block is this +file's code, marker comments aside, and runs it. +""" + +import asyncio + +# --8<-- [start:models] +from ferro import Field, Model, connect, engines + + +class User(Model): + id: int | None = Field(default=None, primary_key=True) + username: str + is_active: bool = True + + +# --8<-- [end:models] + + +async def main(): + # --8<-- [start:query] + await connect("sqlite::memory:", auto_migrate=True) + + async with engines.session(): + await User.create(username="alice") + await User.create(username="bob", is_active=False) + + active_users = await User.where(lambda user: user.is_active == True).all() + print(f"Found {len(active_users)} active users.") + # --8<-- [end:query] + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/docs/examples/readme_quickstart_annotated.py b/docs/examples/readme_quickstart_annotated.py new file mode 100644 index 00000000..5192a7ae --- /dev/null +++ b/docs/examples/readme_quickstart_annotated.py @@ -0,0 +1,37 @@ +"""The README quick start (README.md), runnable. + +tests/test_docs_examples.py checks that the README's Python block is this +file's code, marker comments aside, and runs it. +""" + +import asyncio +from typing import Annotated + +# --8<-- [start:models] +from ferro import Field, Model, connect, engines + + +class User(Model): + id: Annotated[int | None, Field(primary_key=True)] = None + username: str + is_active: bool = True + + +# --8<-- [end:models] + + +async def main(): + # --8<-- [start:query] + await connect("sqlite::memory:", auto_migrate=True) + + async with engines.session(): + await User.create(username="alice") + await User.create(username="bob", is_active=False) + + active_users = await User.where(lambda user: user.is_active == True).all() + print(f"Found {len(active_users)} active users.") + # --8<-- [end:query] + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/docs/examples/relationships.py b/docs/examples/relationships.py index 5f370691..44784725 100644 --- a/docs/examples/relationships.py +++ b/docs/examples/relationships.py @@ -1,149 +1,206 @@ -"""Runnable companion to the Relationships guide (docs/pages/guide/relationships.md).""" +"""Runnable companion to the Relationships guide (docs/pages/guide/relationships.md). + +One roster runs through the whole page: teams have players, a player may +have a profile and a mentor, players enter tournaments, and a tournament +may have a winning team. Every section's block is a range of ``main()``, +in page order, so later sections build on the rows earlier ones added. +""" import asyncio from typing import Annotated -from ferro import ( - BackRef, - Field, - ForeignKey, - ManyToMany, - Model, - Relation, - connect, - engines, -) +from ferro import BackRef, Field, ForeignKey, ManyToMany, Model, Relation -# --8<-- [start:one-to-many] +# --8<-- [start:models] class Team(Model): id: int | None = Field(default=None, primary_key=True) name: str - members: Relation[list["Player"]] = BackRef() + players: Relation[list["Player"]] = BackRef() + titles: Relation[list["Tournament"]] = BackRef() class Player(Model): id: int | None = Field(default=None, primary_key=True) name: str - team: Annotated[Team, ForeignKey(related_name="members")] - - -# --8<-- [end:one-to-many] - - -# --8<-- [start:one-to-one] -class User(Model): - id: int | None = Field(default=None, primary_key=True) - username: str + team: Annotated[Team, ForeignKey(related_name="players", index=True)] + mentor: Annotated[ + "Player", + ForeignKey(related_name="mentees", nullable=True, on_delete="SET NULL"), + ] = None + mentees: Relation[list["Player"]] = BackRef() profile: "Profile" = BackRef() + tournaments: Relation[list["Tournament"]] = ManyToMany(related_name="players") class Profile(Model): id: int | None = Field(default=None, primary_key=True) bio: str - user: Annotated[User, ForeignKey(related_name="profile", unique=True)] - + player: Annotated[Player, ForeignKey(related_name="profile", unique=True)] -# --8<-- [end:one-to-one] - -# --8<-- [start:many-to-many] -class Student(Model): +class Tournament(Model): id: int | None = Field(default=None, primary_key=True) name: str - courses: Relation[list["Course"]] = ManyToMany(related_name="students") - - -class Course(Model): - id: int | None = Field(default=None, primary_key=True) - title: str - students: Relation[list["Student"]] = BackRef() - - -# --8<-- [end:many-to-many] - - -# --8<-- [start:self-referential] -class Employee(Model): - id: int | None = Field(default=None, primary_key=True) - name: str - manager: Annotated[ - "Employee", ForeignKey(related_name="reports", nullable=True) + winner: Annotated[ + Team | None, ForeignKey(related_name="titles", on_delete="SET NULL") ] = None - reports: Relation[list["Employee"]] = BackRef() + players: Relation[list[Player]] = BackRef() -# --8<-- [end:self-referential] +# --8<-- [end:models] -# --8<-- [start:on-delete] -class Library(Model): - id: int | None = Field(default=None, primary_key=True) - name: str - documents: Relation[list["Document"]] = BackRef() - - -class Document(Model): - id: int | None = Field(default=None, primary_key=True) - title: str - library: Annotated[ - Library, ForeignKey(related_name="documents", on_delete="CASCADE") +def table_ddl(report, table: str) -> list[str]: + """The statements ``create_tables()`` ran for one table, in order.""" + return [ + statement.sql for statement in report.statements if statement.subject == table ] -# --8<-- [end:on-delete] - - async def main() -> None: - await connect("sqlite::memory:", auto_migrate=True) - - async with engines.session(): - # --8<-- [start:one-to-many-usage] - team = await Team.create(name="Rustaceans") - crab = await Player.create(name="Ferris", team=team) - - # Forward: awaiting the FK field loads the related instance - assert (await crab.team).name == "Rustaceans" + # --8<-- [start:first] + from ferro import connect, create_tables, engines - # The shadow column is available for direct reads and filters - assert crab.team_id == team.id + await connect("sqlite::memory:") + report = await create_tables() # creates the tables and lists what it ran - # Reverse: the BackRef is a chainable query - roster = await team.members.order_by(lambda player: player.name).all() - # --8<-- [end:one-to-many-usage] - assert len(roster) == 1 - - # --8<-- [start:one-to-one-usage] - user = await User.create(username="alice") - await Profile.create(bio="Pythonista", user=user) - - profile = await user.profile # single instance, not a list - # --8<-- [end:one-to-one-usage] - assert profile.bio == "Pythonista" - - # --8<-- [start:m2m-usage] - sam = await Student.create(name="Sam") - rust101 = await Course.create(title="Rust 101") - python201 = await Course.create(title="Python 201") - - await sam.courses.add(rust101, python201) - assert len(await sam.courses.all()) == 2 - - # The reverse side works the same way - assert len(await rust101.students.all()) == 1 - - await sam.courses.remove(python201) - await sam.courses.clear() - # --8<-- [end:m2m-usage] - assert len(await sam.courses.all()) == 0 - - # --8<-- [start:self-referential-usage] - boss = await Employee.create(name="Grace") - dev = await Employee.create(name="Linus", manager=boss) - - assert (await dev.manager).name == "Grace" - assert len(await boss.reports.all()) == 1 - # --8<-- [end:self-referential-usage] + async with engines.session(): + rustaceans = await Team.create(name="Rustaceans") + ferris = await Player.create(name="Ferris", team=rustaceans) + + team = await ferris.team # forward: runs one query + roster = await rustaceans.players.all() # reverse: a query you run + print(team.name, [player.name for player in roster]) + # --8<-- [end:first] + assert (team.name, [player.name for player in roster]) == ( + "Rustaceans", + ["Ferris"], + ) + + # --8<-- [start:one-to-many] + corro = await Player.create(name="Corro", team=rustaceans) + gophers = await Team.create(name="Gophers") + gordon = await Player.create(name="Gordon", team=gophers) + + roster = await rustaceans.players.order_by(lambda player: player.name).all() + assert [player.name for player in roster] == ["Corro", "Ferris"] + # --8<-- [end:one-to-many] + + # --8<-- [start:shadow-column] + assert ferris.team_id == rustaceans.id # read without a query + + on_rustaceans = await Player.where( + lambda player: player.team_id == rustaceans.id + ).count() + assert on_rustaceans == 2 + # --8<-- [end:shadow-column] + + # --8<-- [start:player-ddl] + for statement in report.statements: + if statement.subject == "player": + print(statement.sql) + # --8<-- [end:player-ddl] + # The page shows these statements whole, on both databases + # (tests/test_docs_sql.py); here, the names each relation gives them. + player_ddl = "\n".join(table_ddl(report, "player")) + assert 'CONSTRAINT "fk_player_mentor_id_player"' in player_ddl + assert 'CONSTRAINT "fk_player_team_id_team"' in player_ddl + assert '"idx_player_team_id" ON "player" ("team_id")' in player_ddl + + # --8<-- [start:populated] + players = ( + await Player.select() + .include(lambda player: player.team) + .order_by(lambda player: player.name) + .all() + ) + for player in players: + print(player.name, "|", player.team.name) # no await + # --8<-- [end:populated] + assert [(player.name, player.team.name) for player in players] == [ + ("Corro", "Rustaceans"), + ("Ferris", "Rustaceans"), + ("Gordon", "Gophers"), + ] + + # --8<-- [start:one-to-one] + await Profile.create(bio="Plays in the front row", player=ferris) + + profile = await ferris.profile # one instance, not a list + assert profile.bio == "Plays in the front row" + assert await corro.profile is None # Corro has no profile + # --8<-- [end:one-to-one] + # unique=True gives the shadow column a unique index. + assert ( + 'CREATE UNIQUE INDEX IF NOT EXISTS "uq_profile_player_id" ON "profile" ' + '("player_id")' + ) in table_ddl(report, "profile") + + # --8<-- [start:many-to-many] + spring = await Tournament.create(name="Spring Open") + autumn = await Tournament.create(name="Autumn Cup") + + await ferris.tournaments.add(spring, autumn) + await gordon.tournaments.add(spring) + + entered = await ferris.tournaments.order_by( + lambda tournament: tournament.name + ).all() + assert [tournament.name for tournament in entered] == [ + "Autumn Cup", + "Spring Open", + ] + + field = await spring.players.order_by(lambda player: player.name).all() + assert [player.name for player in field] == ["Ferris", "Gordon"] + # --8<-- [end:many-to-many] + + # --8<-- [start:m2m-remove] + await ferris.tournaments.remove(autumn) + entered = await ferris.tournaments.all() + assert [tournament.name for tournament in entered] == ["Spring Open"] + + await ferris.tournaments.clear() + assert await ferris.tournaments.count() == 0 + assert await spring.players.count() == 1 # Gordon is still entered + # --8<-- [end:m2m-remove] + + # --8<-- [start:join-table-ddl] + for statement in report.statements: + if statement.subject == "player_tournaments": + print(statement.sql) + # --8<-- [end:join-table-ddl] + link_ddl = "\n".join(table_ddl(report, "player_tournaments")) + assert 'CONSTRAINT "fk_player_tournaments_player_id_player"' in link_ddl + assert '"idx_player_tournaments_tournament_id_player_id"' in link_ddl + assert '"uq_player_tournaments_player_id_tournament_id"' in link_ddl + + # --8<-- [start:self-referential] + crabby = await Player.create(name="Crabby", team=rustaceans, mentor=ferris) + + assert (await crabby.mentor).name == "Ferris" + mentees = await ferris.mentees.all() + assert [player.name for player in mentees] == ["Crabby"] + assert ferris.mentor_id is None # Ferris has no mentor + # --8<-- [end:self-referential] + + # --8<-- [start:nullable] + assert autumn.winner_id is None # no winner yet + + spring.winner_id = rustaceans.id # set the shadow column, then save + await spring.save() + titles = await rustaceans.titles.all() + assert [tournament.name for tournament in titles] == ["Spring Open"] + # --8<-- [end:nullable] + + # --8<-- [start:on-delete] + await rustaceans.delete() + + assert [player.name for player in await Player.select().all()] == ["Gordon"] + assert await Profile.select().count() == 0 + assert (await Tournament.get(spring.id)).winner_id is None + # --8<-- [end:on-delete] print("relationships example ran successfully") diff --git a/docs/examples/relationships_annotated.py b/docs/examples/relationships_annotated.py index c7575db7..b49a2b49 100644 --- a/docs/examples/relationships_annotated.py +++ b/docs/examples/relationships_annotated.py @@ -1,131 +1,208 @@ -"""Annotated-style companion to relationships.py (AGENTS.md I-7). +"""Runnable companion to the Relationships guide (docs/pages/guide/relationships.md). -Field options move into ``Annotated[...]``. The relationship declarations -themselves are identical in both styles: forward FKs are always -``Annotated[Target, ForeignKey(...)]`` and ``BackRef()``/``ManyToMany()`` -are always assignments. +One roster runs through the whole page: teams have players, a player may +have a profile and a mentor, players enter tournaments, and a tournament +may have a winning team. Every section's block is a range of ``main()``, +in page order, so later sections build on the rows earlier ones added. """ import asyncio from typing import Annotated -from ferro import ( - BackRef, - Field, - ForeignKey, - ManyToMany, - Model, - Relation, - connect, - engines, -) +from ferro import BackRef, Field, ForeignKey, ManyToMany, Model, Relation -# --8<-- [start:one-to-many] +# --8<-- [start:models] class Team(Model): - id: Annotated[int | None, Field(default=None, primary_key=True)] + id: Annotated[int | None, Field(primary_key=True)] = None name: str - members: Relation[list["Player"]] = BackRef() + players: Relation[list["Player"]] = BackRef() + titles: Relation[list["Tournament"]] = BackRef() class Player(Model): - id: Annotated[int | None, Field(default=None, primary_key=True)] + id: Annotated[int | None, Field(primary_key=True)] = None name: str - team: Annotated[Team, ForeignKey(related_name="members")] - - -# --8<-- [end:one-to-many] - - -# --8<-- [start:one-to-one] -class User(Model): - id: Annotated[int | None, Field(default=None, primary_key=True)] - username: str + team: Annotated[Team, ForeignKey(related_name="players", index=True)] + mentor: Annotated[ + "Player", + ForeignKey(related_name="mentees", nullable=True, on_delete="SET NULL"), + ] = None + mentees: Relation[list["Player"]] = BackRef() profile: "Profile" = BackRef() + tournaments: Relation[list["Tournament"]] = ManyToMany(related_name="players") class Profile(Model): - id: Annotated[int | None, Field(default=None, primary_key=True)] + id: Annotated[int | None, Field(primary_key=True)] = None bio: str - user: Annotated[User, ForeignKey(related_name="profile", unique=True)] - - -# --8<-- [end:one-to-one] + player: Annotated[Player, ForeignKey(related_name="profile", unique=True)] -# --8<-- [start:many-to-many] -class Student(Model): - id: Annotated[int | None, Field(default=None, primary_key=True)] +class Tournament(Model): + id: Annotated[int | None, Field(primary_key=True)] = None name: str - courses: Relation[list["Course"]] = ManyToMany(related_name="students") - - -class Course(Model): - id: Annotated[int | None, Field(default=None, primary_key=True)] - title: str - students: Relation[list["Student"]] = BackRef() - - -# --8<-- [end:many-to-many] - - -# --8<-- [start:self-referential] -class Employee(Model): - id: Annotated[int | None, Field(default=None, primary_key=True)] - name: str - manager: Annotated[ - "Employee", ForeignKey(related_name="reports", nullable=True) + winner: Annotated[ + Team | None, ForeignKey(related_name="titles", on_delete="SET NULL") ] = None - reports: Relation[list["Employee"]] = BackRef() - - -# --8<-- [end:self-referential] + players: Relation[list[Player]] = BackRef() -# --8<-- [start:on-delete] -class Library(Model): - id: Annotated[int | None, Field(default=None, primary_key=True)] - name: str - documents: Relation[list["Document"]] = BackRef() +# --8<-- [end:models] -class Document(Model): - id: Annotated[int | None, Field(default=None, primary_key=True)] - title: str - library: Annotated[ - Library, ForeignKey(related_name="documents", on_delete="CASCADE") +def table_ddl(report, table: str) -> list[str]: + """The statements ``create_tables()`` ran for one table, in order.""" + return [ + statement.sql for statement in report.statements if statement.subject == table ] -# --8<-- [end:on-delete] - - async def main() -> None: - await connect("sqlite::memory:", auto_migrate=True) + # --8<-- [start:first] + from ferro import connect, create_tables, engines - async with engines.session(): - team = await Team.create(name="Rustaceans") - player = await Player.create(name="Ferris", team=team) - assert (await player.team).id == team.id - - user = await User.create(username="alice") - await Profile.create(bio="Pythonista", user=user) - assert (await user.profile).bio == "Pythonista" - - sam = await Student.create(name="Sam") - rust101 = await Course.create(title="Rust 101") - await sam.courses.add(rust101) - assert len(await sam.courses.all()) == 1 + await connect("sqlite::memory:") + report = await create_tables() # creates the tables and lists what it ran - boss = await Employee.create(name="Grace") - dev = await Employee.create(name="Linus", manager=boss) - assert (await dev.manager).name == "Grace" - - lib = await Library.create(name="Main") - await Document.create(title="Charter", library=lib) - assert len(await lib.documents.all()) == 1 - - print("relationships_annotated example ran successfully") + async with engines.session(): + rustaceans = await Team.create(name="Rustaceans") + ferris = await Player.create(name="Ferris", team=rustaceans) + + team = await ferris.team # forward: runs one query + roster = await rustaceans.players.all() # reverse: a query you run + print(team.name, [player.name for player in roster]) + # --8<-- [end:first] + assert (team.name, [player.name for player in roster]) == ( + "Rustaceans", + ["Ferris"], + ) + + # --8<-- [start:one-to-many] + corro = await Player.create(name="Corro", team=rustaceans) + gophers = await Team.create(name="Gophers") + gordon = await Player.create(name="Gordon", team=gophers) + + roster = await rustaceans.players.order_by(lambda player: player.name).all() + assert [player.name for player in roster] == ["Corro", "Ferris"] + # --8<-- [end:one-to-many] + + # --8<-- [start:shadow-column] + assert ferris.team_id == rustaceans.id # read without a query + + on_rustaceans = await Player.where( + lambda player: player.team_id == rustaceans.id + ).count() + assert on_rustaceans == 2 + # --8<-- [end:shadow-column] + + # --8<-- [start:player-ddl] + for statement in report.statements: + if statement.subject == "player": + print(statement.sql) + # --8<-- [end:player-ddl] + # The page shows these statements whole, on both databases + # (tests/test_docs_sql.py); here, the names each relation gives them. + player_ddl = "\n".join(table_ddl(report, "player")) + assert 'CONSTRAINT "fk_player_mentor_id_player"' in player_ddl + assert 'CONSTRAINT "fk_player_team_id_team"' in player_ddl + assert '"idx_player_team_id" ON "player" ("team_id")' in player_ddl + + # --8<-- [start:populated] + players = ( + await Player.select() + .include(lambda player: player.team) + .order_by(lambda player: player.name) + .all() + ) + for player in players: + print(player.name, "|", player.team.name) # no await + # --8<-- [end:populated] + assert [(player.name, player.team.name) for player in players] == [ + ("Corro", "Rustaceans"), + ("Ferris", "Rustaceans"), + ("Gordon", "Gophers"), + ] + + # --8<-- [start:one-to-one] + await Profile.create(bio="Plays in the front row", player=ferris) + + profile = await ferris.profile # one instance, not a list + assert profile.bio == "Plays in the front row" + assert await corro.profile is None # Corro has no profile + # --8<-- [end:one-to-one] + # unique=True gives the shadow column a unique index. + assert ( + 'CREATE UNIQUE INDEX IF NOT EXISTS "uq_profile_player_id" ON "profile" ' + '("player_id")' + ) in table_ddl(report, "profile") + + # --8<-- [start:many-to-many] + spring = await Tournament.create(name="Spring Open") + autumn = await Tournament.create(name="Autumn Cup") + + await ferris.tournaments.add(spring, autumn) + await gordon.tournaments.add(spring) + + entered = await ferris.tournaments.order_by( + lambda tournament: tournament.name + ).all() + assert [tournament.name for tournament in entered] == [ + "Autumn Cup", + "Spring Open", + ] + + field = await spring.players.order_by(lambda player: player.name).all() + assert [player.name for player in field] == ["Ferris", "Gordon"] + # --8<-- [end:many-to-many] + + # --8<-- [start:m2m-remove] + await ferris.tournaments.remove(autumn) + entered = await ferris.tournaments.all() + assert [tournament.name for tournament in entered] == ["Spring Open"] + + await ferris.tournaments.clear() + assert await ferris.tournaments.count() == 0 + assert await spring.players.count() == 1 # Gordon is still entered + # --8<-- [end:m2m-remove] + + # --8<-- [start:join-table-ddl] + for statement in report.statements: + if statement.subject == "player_tournaments": + print(statement.sql) + # --8<-- [end:join-table-ddl] + link_ddl = "\n".join(table_ddl(report, "player_tournaments")) + assert 'CONSTRAINT "fk_player_tournaments_player_id_player"' in link_ddl + assert '"idx_player_tournaments_tournament_id_player_id"' in link_ddl + assert '"uq_player_tournaments_player_id_tournament_id"' in link_ddl + + # --8<-- [start:self-referential] + crabby = await Player.create(name="Crabby", team=rustaceans, mentor=ferris) + + assert (await crabby.mentor).name == "Ferris" + mentees = await ferris.mentees.all() + assert [player.name for player in mentees] == ["Crabby"] + assert ferris.mentor_id is None # Ferris has no mentor + # --8<-- [end:self-referential] + + # --8<-- [start:nullable] + assert autumn.winner_id is None # no winner yet + + spring.winner_id = rustaceans.id # set the shadow column, then save + await spring.save() + titles = await rustaceans.titles.all() + assert [tournament.name for tournament in titles] == ["Spring Open"] + # --8<-- [end:nullable] + + # --8<-- [start:on-delete] + await rustaceans.delete() + + assert [player.name for player in await Player.select().all()] == ["Gordon"] + assert await Profile.select().count() == 0 + assert (await Tournament.get(spring.id)).winner_id is None + # --8<-- [end:on-delete] + + print("relationships example ran successfully") if __name__ == "__main__": diff --git a/docs/examples/row_level_security.py b/docs/examples/row_level_security.py index bbab97e1..0c81bfa6 100644 --- a/docs/examples/row_level_security.py +++ b/docs/examples/row_level_security.py @@ -1,24 +1,43 @@ -"""Runnable companion for row-level security (docs/pages/guide/row-level-security.md). +"""Runnable companion for the row-level security pages. -Row security is a PostgreSQL-only feature (docs/examples scripts run without a -live Postgres, so this one runs on SQLite): a model declaring ``__ferro_rls__`` -still registers and migrates there — with a loud warning — so the model works -everywhere, and the enforcement itself is exercised by -``tests/test_rls_end_to_end.py`` against a real PostgreSQL role. +Embedded by docs/pages/guide/row-level-security.md, +docs/pages/concepts/row-level-security.md and +docs/pages/reference/row-level-security.md. + +Row-level security is a Postgres feature, so the pages' demo needs Postgres: + + python docs/examples/row_level_security.py + +connects to ```` as the migrating role (a superuser, or a role +with BYPASSRLS), creates the tables, creates ```` as the +application's NOSUPERUSER login role, and checks what each session sees. +``tests/test_docs_examples.py`` runs it that way when a Postgres server is +configured. With no arguments it runs on SQLite, where Ferro creates the +tables without their policies and warns. """ import asyncio +import sys import uuid import warnings from typing import ClassVar +from urllib.parse import quote, urlparse, urlunparse +import ferro from ferro import Field, Model, RowPolicy, RowSecurity, connect, engines +from ferro.exceptions import OperationalError +from ferro.raw import execute + +ACME = uuid.UUID("a1a1a1a1-0000-4000-8000-000000000001") +GLOBEX = uuid.UUID("b2b2b2b2-0000-4000-8000-000000000002") +APP_PASSWORD = "app-password" # --8<-- [start:models] class Invoice(Model): id: int | None = Field(default=None, primary_key=True) tenant_id: uuid.UUID + customer: str total: int __ferro_rls__: ClassVar = RowSecurity( @@ -29,63 +48,204 @@ class Invoice(Model): # --8<-- [end:models] -# --8<-- [start:multi_policy] -class Doc(Model): +# --8<-- [start:composed] +class Document(Model): id: int | None = Field(default=None, primary_key=True) tenant_id: uuid.UUID owner: str title: str + published: bool = False __ferro_rls__: ClassVar = RowSecurity( - # RESTRICTIVE: AND-composes with everything below. No tenant scope, - # no rows — whoever you are. + # Restrictive: every other policy applies only inside the tenant. RowPolicy( name="tenant", column="tenant_id", setting="app.tenant_id", restrictive=True, ), - # Permissive, unscoped by command: the owner reads and writes. + # The owner reads and writes their own documents. RowPolicy( name="owner_all", - using="\"owner\" = NULLIF(current_setting('app.current_user', true), '')", - with_check="\"owner\" = NULLIF(current_setting('app.current_user', true), '')", - ), - # Permissive, SELECT-only: an invited member reads and nothing more. - RowPolicy( - name="invitee_read", - command="select", - using=( - '"id" IN (SELECT doc_id FROM membership WHERE member = ' - "NULLIF(current_setting('app.current_user', true), ''))" - ), + using="\"owner\" = NULLIF(current_setting('app.user', true), '')", + with_check="\"owner\" = NULLIF(current_setting('app.user', true), '')", ), + # Everyone in the tenant reads a published document. + RowPolicy(name="published_read", command="select", using='"published"'), ) -# --8<-- [end:multi_policy] +# --8<-- [end:composed] + + +def app_url_for(migrator_url: str, role: str) -> str: + """The same database as ``migrator_url``, logged in as ``role``.""" + parsed = urlparse(migrator_url) + netloc = f"{quote(role, safe='')}:{quote(APP_PASSWORD, safe='')}@{parsed.hostname}" + if parsed.port: + netloc += f":{parsed.port}" + return urlunparse(parsed._replace(netloc=netloc)) -async def main() -> None: +async def create_app_role(role: str) -> None: + """Create the application's login role, which row security applies to.""" + async with engines.session(): + schema = (await ferro.raw.fetch_one("SELECT current_schema() AS name"))["name"] + await execute( + f"CREATE ROLE \"{role}\" LOGIN NOSUPERUSER PASSWORD '{APP_PASSWORD}'" + ) + await execute(f'GRANT USAGE ON SCHEMA "{schema}" TO "{role}"') + for table in ("invoice", "document"): + await execute( + f'GRANT SELECT, INSERT, UPDATE, DELETE ON "{table}" TO "{role}"' + ) + await execute(f'GRANT USAGE ON ALL SEQUENCES IN SCHEMA "{schema}" TO "{role}"') + + +async def on_postgres(migrator_url: str, role: str) -> None: + # --8<-- [start:connect_migrator] + await connect(migrator_url, auto_migrate=True) + # --8<-- [end:connect_migrator] + await create_app_role(role) + app_url = app_url_for(migrator_url, role) + + # --8<-- [start:connect_app] + await connect(app_url, name="app") + # --8<-- [end:connect_app] + + # --8<-- [start:seed] + async with engines.session(): + await Invoice.create(tenant_id=ACME, customer="Initech", total=450) + await Invoice.create(tenant_id=ACME, customer="Hooli", total=120) + await Invoice.create(tenant_id=GLOBEX, customer="Vandelay", total=300) + await Invoice.create(tenant_id=GLOBEX, customer="Pied Piper", total=980) + # --8<-- [end:seed] + + # --8<-- [start:scoped] + async with engines.session("app", settings={"app.tenant_id": str(ACME)}): + invoices = await Invoice.select().order_by(lambda invoice: invoice.total).all() + assert [(invoice.customer, invoice.total) for invoice in invoices] == [ + ("Hooli", 120), + ("Initech", 450), + ] + # --8<-- [end:scoped] + + # --8<-- [start:unscoped] + async with engines.session("app"): + assert await Invoice.all() == [] + # --8<-- [end:unscoped] + + # --8<-- [start:migrator] + async with engines.session(): + assert await Invoice.select().count() == 4 + # --8<-- [end:migrator] + + # --8<-- [start:write_check] + async with engines.session("app", settings={"app.tenant_id": str(ACME)}): + try: + await Invoice.create(tenant_id=GLOBEX, customer="Initech", total=75) + except OperationalError as error: + assert error.sqlstate == "42501" + else: + raise AssertionError("a write into another tenant must be refused") + # --8<-- [end:write_check] + + # --8<-- [start:deferred] + async def scope_to_tenant(tenant_id: uuid.UUID) -> None: + session = ferro.current_session() + assert session is not None + await session.set_config("app.tenant_id", str(tenant_id)) + + async with engines.session("app"): + assert await Invoice.all() == [] + await scope_to_tenant(GLOBEX) + invoices = await Invoice.select().order_by(lambda invoice: invoice.total).all() + assert [(invoice.customer, invoice.total) for invoice in invoices] == [ + ("Vandelay", 300), + ("Pied Piper", 980), + ] + # --8<-- [end:deferred] + + # --8<-- [start:nested] + acme = str(ACME) + async with engines.session( + "app", settings={"app.tenant_id": acme, "app.user": "ada"} + ) as outer: + async with engines.session("app", settings={"app.user": "grace"}) as inner: + assert inner.effective_settings == { + "app.tenant_id": acme, + "app.user": "grace", + } + assert outer.effective_settings == {"app.tenant_id": acme, "app.user": "ada"} + # --8<-- [end:nested] + + async with engines.session(): + await Document.create(tenant_id=ACME, owner="ada", title="Q3 plan") + await Document.create( + tenant_id=ACME, owner="grace", title="Handbook", published=True + ) + await Document.create(tenant_id=ACME, owner="grace", title="Grace's draft") + await Document.create( + tenant_id=GLOBEX, owner="ada", title="Globex roadmap", published=True + ) + + # --8<-- [start:composed_query] + async with engines.session( + "app", settings={"app.tenant_id": str(ACME), "app.user": "ada"} + ): + documents = ( + await Document.select().order_by(lambda document: document.title).all() + ) + assert [document.title for document in documents] == ["Handbook", "Q3 plan"] + # --8<-- [end:composed_query] + + # --8<-- [start:forced_warning] with warnings.catch_warnings(record=True) as caught: warnings.simplefilter("always") - await connect("sqlite::memory:", auto_migrate=True) + await connect(app_url, name="app_migrate", migrate_updates=True) + for warning in caught: + print(warning.message) + # --8<-- [end:forced_warning] - # SQLite has no row-level security: both models register and their - # tables are created, unpoliced, each with a warning naming exactly that. - skip_warnings = [ - str(w.message) for w in caught if "PostgreSQL-only" in str(w.message) - ] - assert len(skip_warnings) == 2, skip_warnings - assert any("invoice" in message.lower() for message in skip_warnings) - assert any("doc" in message.lower() for message in skip_warnings) + +async def on_sqlite() -> None: + # --8<-- [start:sqlite] + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + await connect("sqlite::memory:", auto_migrate=True) + print(next(str(w.message) for w in caught if "'invoice'" in str(w.message))) + # ferro auto-migrate: Table 'invoice' declares __ferro_rls__, but row-level + # security is a PostgreSQL-only feature: the table is created without its + # policies and rows are NOT filtered on SQLite. Run against PostgreSQL for + # enforcement. + # --8<-- [end:sqlite] + skipped = [str(w.message) for w in caught if "PostgreSQL-only" in str(w.message)] + assert len(skipped) == 2, skipped + + # --8<-- [start:sqlite_settings] + try: + async with engines.session(settings={"app.tenant_id": str(ACME)}): + pass + except RuntimeError as error: + assert "Session settings require a Postgres connection" in str(error) + else: + raise AssertionError("settings= on SQLite must be refused") + # --8<-- [end:sqlite_settings] async with engines.session(): - await Invoice.create(tenant_id=uuid.uuid4(), total=100) - assert len(await Invoice.all()) == 1 + await Invoice.create(tenant_id=ACME, customer="Initech", total=450) + await Invoice.create(tenant_id=GLOBEX, customer="Vandelay", total=300) + assert await Invoice.select().count() == 2 + +async def main(argv: list[str]) -> None: + if argv: + migrator_url, role = argv + await on_postgres(migrator_url, role) + else: + await on_sqlite() print("row_level_security example ran successfully") if __name__ == "__main__": - asyncio.run(main()) + asyncio.run(main(sys.argv[1:])) diff --git a/docs/examples/row_level_security_annotated.py b/docs/examples/row_level_security_annotated.py index 533e77c5..badefbc4 100644 --- a/docs/examples/row_level_security_annotated.py +++ b/docs/examples/row_level_security_annotated.py @@ -1,17 +1,43 @@ -"""Annotated-style companion to row_level_security.py (AGENTS.md I-7).""" +"""Runnable companion for the row-level security pages. + +Embedded by docs/pages/guide/row-level-security.md, +docs/pages/concepts/row-level-security.md and +docs/pages/reference/row-level-security.md. + +Row-level security is a Postgres feature, so the pages' demo needs Postgres: + + python docs/examples/row_level_security.py + +connects to ```` as the migrating role (a superuser, or a role +with BYPASSRLS), creates the tables, creates ```` as the +application's NOSUPERUSER login role, and checks what each session sees. +``tests/test_docs_examples.py`` runs it that way when a Postgres server is +configured. With no arguments it runs on SQLite, where Ferro creates the +tables without their policies and warns. +""" import asyncio +import sys import uuid import warnings from typing import Annotated, ClassVar +from urllib.parse import quote, urlparse, urlunparse +import ferro from ferro import Field, Model, RowPolicy, RowSecurity, connect, engines +from ferro.exceptions import OperationalError +from ferro.raw import execute + +ACME = uuid.UUID("a1a1a1a1-0000-4000-8000-000000000001") +GLOBEX = uuid.UUID("b2b2b2b2-0000-4000-8000-000000000002") +APP_PASSWORD = "app-password" # --8<-- [start:models] class Invoice(Model): - id: Annotated[int | None, Field(default=None, primary_key=True)] + id: Annotated[int | None, Field(primary_key=True)] = None tenant_id: uuid.UUID + customer: str total: int __ferro_rls__: ClassVar = RowSecurity( @@ -22,61 +48,204 @@ class Invoice(Model): # --8<-- [end:models] -# --8<-- [start:multi_policy] -class Doc(Model): - id: Annotated[int | None, Field(default=None, primary_key=True)] +# --8<-- [start:composed] +class Document(Model): + id: Annotated[int | None, Field(primary_key=True)] = None tenant_id: uuid.UUID owner: str title: str + published: bool = False __ferro_rls__: ClassVar = RowSecurity( - # RESTRICTIVE: AND-composes with everything below. No tenant scope, - # no rows — whoever you are. + # Restrictive: every other policy applies only inside the tenant. RowPolicy( name="tenant", column="tenant_id", setting="app.tenant_id", restrictive=True, ), - # Permissive, unscoped by command: the owner reads and writes. + # The owner reads and writes their own documents. RowPolicy( name="owner_all", - using="\"owner\" = NULLIF(current_setting('app.current_user', true), '')", - with_check="\"owner\" = NULLIF(current_setting('app.current_user', true), '')", - ), - # Permissive, SELECT-only: an invited member reads and nothing more. - RowPolicy( - name="invitee_read", - command="select", - using=( - '"id" IN (SELECT doc_id FROM membership WHERE member = ' - "NULLIF(current_setting('app.current_user', true), ''))" - ), + using="\"owner\" = NULLIF(current_setting('app.user', true), '')", + with_check="\"owner\" = NULLIF(current_setting('app.user', true), '')", ), + # Everyone in the tenant reads a published document. + RowPolicy(name="published_read", command="select", using='"published"'), ) -# --8<-- [end:multi_policy] +# --8<-- [end:composed] + + +def app_url_for(migrator_url: str, role: str) -> str: + """The same database as ``migrator_url``, logged in as ``role``.""" + parsed = urlparse(migrator_url) + netloc = f"{quote(role, safe='')}:{quote(APP_PASSWORD, safe='')}@{parsed.hostname}" + if parsed.port: + netloc += f":{parsed.port}" + return urlunparse(parsed._replace(netloc=netloc)) + +async def create_app_role(role: str) -> None: + """Create the application's login role, which row security applies to.""" + async with engines.session(): + schema = (await ferro.raw.fetch_one("SELECT current_schema() AS name"))["name"] + await execute( + f"CREATE ROLE \"{role}\" LOGIN NOSUPERUSER PASSWORD '{APP_PASSWORD}'" + ) + await execute(f'GRANT USAGE ON SCHEMA "{schema}" TO "{role}"') + for table in ("invoice", "document"): + await execute( + f'GRANT SELECT, INSERT, UPDATE, DELETE ON "{table}" TO "{role}"' + ) + await execute(f'GRANT USAGE ON ALL SEQUENCES IN SCHEMA "{schema}" TO "{role}"') + + +async def on_postgres(migrator_url: str, role: str) -> None: + # --8<-- [start:connect_migrator] + await connect(migrator_url, auto_migrate=True) + # --8<-- [end:connect_migrator] + await create_app_role(role) + app_url = app_url_for(migrator_url, role) + + # --8<-- [start:connect_app] + await connect(app_url, name="app") + # --8<-- [end:connect_app] + + # --8<-- [start:seed] + async with engines.session(): + await Invoice.create(tenant_id=ACME, customer="Initech", total=450) + await Invoice.create(tenant_id=ACME, customer="Hooli", total=120) + await Invoice.create(tenant_id=GLOBEX, customer="Vandelay", total=300) + await Invoice.create(tenant_id=GLOBEX, customer="Pied Piper", total=980) + # --8<-- [end:seed] + + # --8<-- [start:scoped] + async with engines.session("app", settings={"app.tenant_id": str(ACME)}): + invoices = await Invoice.select().order_by(lambda invoice: invoice.total).all() + assert [(invoice.customer, invoice.total) for invoice in invoices] == [ + ("Hooli", 120), + ("Initech", 450), + ] + # --8<-- [end:scoped] + + # --8<-- [start:unscoped] + async with engines.session("app"): + assert await Invoice.all() == [] + # --8<-- [end:unscoped] + + # --8<-- [start:migrator] + async with engines.session(): + assert await Invoice.select().count() == 4 + # --8<-- [end:migrator] + + # --8<-- [start:write_check] + async with engines.session("app", settings={"app.tenant_id": str(ACME)}): + try: + await Invoice.create(tenant_id=GLOBEX, customer="Initech", total=75) + except OperationalError as error: + assert error.sqlstate == "42501" + else: + raise AssertionError("a write into another tenant must be refused") + # --8<-- [end:write_check] + + # --8<-- [start:deferred] + async def scope_to_tenant(tenant_id: uuid.UUID) -> None: + session = ferro.current_session() + assert session is not None + await session.set_config("app.tenant_id", str(tenant_id)) + + async with engines.session("app"): + assert await Invoice.all() == [] + await scope_to_tenant(GLOBEX) + invoices = await Invoice.select().order_by(lambda invoice: invoice.total).all() + assert [(invoice.customer, invoice.total) for invoice in invoices] == [ + ("Vandelay", 300), + ("Pied Piper", 980), + ] + # --8<-- [end:deferred] + + # --8<-- [start:nested] + acme = str(ACME) + async with engines.session( + "app", settings={"app.tenant_id": acme, "app.user": "ada"} + ) as outer: + async with engines.session("app", settings={"app.user": "grace"}) as inner: + assert inner.effective_settings == { + "app.tenant_id": acme, + "app.user": "grace", + } + assert outer.effective_settings == {"app.tenant_id": acme, "app.user": "ada"} + # --8<-- [end:nested] -async def main() -> None: + async with engines.session(): + await Document.create(tenant_id=ACME, owner="ada", title="Q3 plan") + await Document.create( + tenant_id=ACME, owner="grace", title="Handbook", published=True + ) + await Document.create(tenant_id=ACME, owner="grace", title="Grace's draft") + await Document.create( + tenant_id=GLOBEX, owner="ada", title="Globex roadmap", published=True + ) + + # --8<-- [start:composed_query] + async with engines.session( + "app", settings={"app.tenant_id": str(ACME), "app.user": "ada"} + ): + documents = ( + await Document.select().order_by(lambda document: document.title).all() + ) + assert [document.title for document in documents] == ["Handbook", "Q3 plan"] + # --8<-- [end:composed_query] + + # --8<-- [start:forced_warning] with warnings.catch_warnings(record=True) as caught: warnings.simplefilter("always") - await connect("sqlite::memory:", auto_migrate=True) + await connect(app_url, name="app_migrate", migrate_updates=True) + for warning in caught: + print(warning.message) + # --8<-- [end:forced_warning] - skip_warnings = [ - str(w.message) for w in caught if "PostgreSQL-only" in str(w.message) - ] - assert len(skip_warnings) == 2, skip_warnings - assert any("invoice" in message.lower() for message in skip_warnings) - assert any("doc" in message.lower() for message in skip_warnings) + +async def on_sqlite() -> None: + # --8<-- [start:sqlite] + with warnings.catch_warnings(record=True) as caught: + warnings.simplefilter("always") + await connect("sqlite::memory:", auto_migrate=True) + print(next(str(w.message) for w in caught if "'invoice'" in str(w.message))) + # ferro auto-migrate: Table 'invoice' declares __ferro_rls__, but row-level + # security is a PostgreSQL-only feature: the table is created without its + # policies and rows are NOT filtered on SQLite. Run against PostgreSQL for + # enforcement. + # --8<-- [end:sqlite] + skipped = [str(w.message) for w in caught if "PostgreSQL-only" in str(w.message)] + assert len(skipped) == 2, skipped + + # --8<-- [start:sqlite_settings] + try: + async with engines.session(settings={"app.tenant_id": str(ACME)}): + pass + except RuntimeError as error: + assert "Session settings require a Postgres connection" in str(error) + else: + raise AssertionError("settings= on SQLite must be refused") + # --8<-- [end:sqlite_settings] async with engines.session(): - await Invoice.create(tenant_id=uuid.uuid4(), total=100) - assert len(await Invoice.all()) == 1 + await Invoice.create(tenant_id=ACME, customer="Initech", total=450) + await Invoice.create(tenant_id=GLOBEX, customer="Vandelay", total=300) + assert await Invoice.select().count() == 2 + - print("row_level_security_annotated example ran successfully") +async def main(argv: list[str]) -> None: + if argv: + migrator_url, role = argv + await on_postgres(migrator_url, role) + else: + await on_sqlite() + print("row_level_security example ran successfully") if __name__ == "__main__": - asyncio.run(main()) + asyncio.run(main(sys.argv[1:])) diff --git a/docs/examples/soft_deletes.py b/docs/examples/soft_deletes.py index 14ff5497..afe42a4c 100644 --- a/docs/examples/soft_deletes.py +++ b/docs/examples/soft_deletes.py @@ -1,4 +1,4 @@ -"""Runnable companion to the Soft Deletes how-to (docs/pages/howto/soft-deletes.md).""" +"""Runnable companion to the soft-deletes how-to (docs/pages/howto/soft-deletes.md).""" import asyncio @@ -10,10 +10,10 @@ class SoftDeleteMixin: - """Soft-delete behavior as a plain mixin. + """Soft-delete behaviour as a plain class. - Declare ``is_deleted`` and ``deleted_at`` on each concrete model; - the mixin supplies the lifecycle methods. + Each model that uses it declares ``is_deleted`` and ``deleted_at`` + itself; the mixin supplies the methods. """ async def soft_delete(self) -> None: @@ -28,12 +28,12 @@ async def restore(self) -> None: @classmethod def active(cls) -> Query: - return cls.where(lambda invoice: invoice.is_deleted == False) # noqa: E712 + return cls.where(lambda row: row.is_deleted == False) class Invoice(SoftDeleteMixin, Model): id: int | None = Field(default=None, primary_key=True) - number: str + number: str = Field(unique=True) is_deleted: bool = False deleted_at: datetime | None = None @@ -42,21 +42,57 @@ class Invoice(SoftDeleteMixin, Model): async def main() -> None: + # --8<-- [start:session] await connect("sqlite::memory:", auto_migrate=True) async with engines.session(): - # --8<-- [start:usage] invoice = await Invoice.create(number="INV-001") await Invoice.create(number="INV-002") + await Invoice.create(number="INV-003") + # --8<-- [end:session] + # --8<-- [start:usage] await invoice.soft_delete() - assert await Invoice.active().count() == 1 - assert await Invoice.select().count() == 2 # row still exists + assert await Invoice.active().count() == 2 + assert await Invoice.select().count() == 3 await invoice.restore() - assert await Invoice.active().count() == 2 + assert await Invoice.active().count() == 3 # --8<-- [end:usage] + await invoice.soft_delete() + + # --8<-- [start:querying] + live = await ( + Invoice.active() + .where(lambda invoice: invoice.number >= "INV-002") + .order_by(lambda invoice: invoice.number) + .all() + ) + assert [invoice.number for invoice in live] == ["INV-002", "INV-003"] + + trashed = await Invoice.where(lambda invoice: invoice.is_deleted == True).all() + assert [invoice.number for invoice in trashed] == ["INV-001"] + # --8<-- [end:querying] + + # --8<-- [start:unique] + from ferro import UniqueViolationError + + try: + await Invoice.create(number="INV-001") + except UniqueViolationError: + print("INV-001 is still taken") + else: + raise AssertionError("expected UniqueViolationError") + # --8<-- [end:unique] + + # --8<-- [start:purge] + soft_deleted = Invoice.where(lambda invoice: invoice.is_deleted == True) + purged = await soft_deleted.delete() + assert purged == 1 + assert await Invoice.select().count() == 2 + # --8<-- [end:purge] + print("soft_deletes example ran successfully") diff --git a/docs/examples/soft_deletes_annotated.py b/docs/examples/soft_deletes_annotated.py index 1be5143e..3679240d 100644 --- a/docs/examples/soft_deletes_annotated.py +++ b/docs/examples/soft_deletes_annotated.py @@ -1,4 +1,4 @@ -"""Annotated-style companion to soft_deletes.py (AGENTS.md I-7).""" +"""Runnable companion to the soft-deletes how-to (docs/pages/howto/soft-deletes.md).""" import asyncio @@ -11,10 +11,10 @@ class SoftDeleteMixin: - """Soft-delete behavior as a plain mixin. + """Soft-delete behaviour as a plain class. - Declare ``is_deleted`` and ``deleted_at`` on each concrete model; - the mixin supplies the lifecycle methods. + Each model that uses it declares ``is_deleted`` and ``deleted_at`` + itself; the mixin supplies the methods. """ async def soft_delete(self) -> None: @@ -29,12 +29,12 @@ async def restore(self) -> None: @classmethod def active(cls) -> Query: - return cls.where(lambda invoice: invoice.is_deleted == False) # noqa: E712 + return cls.where(lambda row: row.is_deleted == False) class Invoice(SoftDeleteMixin, Model): - id: Annotated[int | None, Field(default=None, primary_key=True)] - number: str + id: Annotated[int | None, Field(primary_key=True)] = None + number: Annotated[str, Field(unique=True)] is_deleted: bool = False deleted_at: datetime | None = None @@ -43,18 +43,58 @@ class Invoice(SoftDeleteMixin, Model): async def main() -> None: + # --8<-- [start:session] await connect("sqlite::memory:", auto_migrate=True) async with engines.session(): invoice = await Invoice.create(number="INV-001") + await Invoice.create(number="INV-002") + await Invoice.create(number="INV-003") + # --8<-- [end:session] + + # --8<-- [start:usage] await invoice.soft_delete() - assert await Invoice.active().count() == 0 - assert await Invoice.select().count() == 1 + assert await Invoice.active().count() == 2 + assert await Invoice.select().count() == 3 await invoice.restore() - assert await Invoice.active().count() == 1 + assert await Invoice.active().count() == 3 + # --8<-- [end:usage] + + await invoice.soft_delete() - print("soft_deletes_annotated example ran successfully") + # --8<-- [start:querying] + live = await ( + Invoice.active() + .where(lambda invoice: invoice.number >= "INV-002") + .order_by(lambda invoice: invoice.number) + .all() + ) + assert [invoice.number for invoice in live] == ["INV-002", "INV-003"] + + trashed = await Invoice.where(lambda invoice: invoice.is_deleted == True).all() + assert [invoice.number for invoice in trashed] == ["INV-001"] + # --8<-- [end:querying] + + # --8<-- [start:unique] + from ferro import UniqueViolationError + + try: + await Invoice.create(number="INV-001") + except UniqueViolationError: + print("INV-001 is still taken") + else: + raise AssertionError("expected UniqueViolationError") + # --8<-- [end:unique] + + # --8<-- [start:purge] + soft_deleted = Invoice.where(lambda invoice: invoice.is_deleted == True) + purged = await soft_deleted.delete() + assert purged == 1 + assert await Invoice.select().count() == 2 + # --8<-- [end:purge] + + print("soft_deletes example ran successfully") if __name__ == "__main__": diff --git a/docs/examples/table_checks.py b/docs/examples/table_checks.py index 5dc014ed..e95394eb 100644 --- a/docs/examples/table_checks.py +++ b/docs/examples/table_checks.py @@ -1,43 +1,83 @@ -"""Runnable companion for table checks (docs/pages/guide/models-and-fields.md).""" +"""Runnable companion for table checks (docs/pages/guide/models-and-fields.md). + +A payment carries a card token or a bank account, never both. The SQL block +on the page is the ``DDL`` section below, checked against the statement +``create_tables()`` ran. +""" import asyncio +import re from typing import ClassVar -from ferro import Check, CheckViolationError, Field, Model, connect, engines +from ferro import ( + Check, + CheckViolationError, + Field, + Model, + connect, + create_tables, + engines, +) # --8<-- [start:models] -class Pair(Model): +class Payment(Model): __ferro_checks__: ClassVar[tuple[Check, ...]] = ( Check( - "at_most_one_side", - lambda pair: (pair.left == None) | (pair.right == None), # noqa: E711 + "at_most_one_method", + lambda payment: ( + (payment.card_token == None) | (payment.bank_account == None) + ), ), ) id: int | None = Field(default=None, primary_key=True) - left: str | None = None - right: str | None = None + card_token: str | None = None + bank_account: str | None = None # --8<-- [end:models] +DDL = """ +# --8<-- [start:ddl] +CREATE TABLE IF NOT EXISTS "payment" ( + "bank_account" varchar, + "card_token" varchar, + "id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, + CONSTRAINT "ck_payment_at_most_one_method" CHECK (("card_token" IS NULL) OR ("bank_account" IS NULL)) +); +# --8<-- [end:ddl] +""" + + +def _normalized(sql: str) -> str: + return re.sub(r"\s+", " ", sql).strip().rstrip(";").strip() + async def main() -> None: - await connect("sqlite::memory:", auto_migrate=True) + await connect("sqlite::memory:") + report = await create_tables() + [created] = [statement.sql for statement in report.statements] + expected = "\n".join(line for line in DDL.splitlines() if "--8<--" not in line) + assert _normalized(created) == _normalized(expected), created async with engines.session(): - await Pair.create(left="only-left") - await Pair.create(right="only-right") + await Payment.create(card_token="tok_visa_4242") + await Payment.create(bank_account="GB33BUKB20201555555555") # --8<-- [start:violation] try: - await Pair.create(left="both", right="set") - except CheckViolationError: - pass # ck_pair_at_most_one_side rejected the row + await Payment.create( + card_token="tok_visa_4242", + bank_account="GB33BUKB20201555555555", + ) + except CheckViolationError as exc: + assert "ck_payment_at_most_one_method" in str(exc) + # --8<-- [end:violation] else: raise AssertionError("expected CheckViolationError") - # --8<-- [end:violation] + + assert await Payment.select().count() == 2 print("table_checks example ran successfully") diff --git a/docs/examples/table_checks_annotated.py b/docs/examples/table_checks_annotated.py index db47481b..b2a00623 100644 --- a/docs/examples/table_checks_annotated.py +++ b/docs/examples/table_checks_annotated.py @@ -1,42 +1,85 @@ -"""Annotated-style companion to table_checks.py (AGENTS.md I-7).""" +"""Runnable companion for table checks (docs/pages/guide/models-and-fields.md). + +A payment carries a card token or a bank account, never both. The SQL block +on the page is the ``DDL`` section below, checked against the statement +``create_tables()`` ran. +""" import asyncio +import re from typing import Annotated, ClassVar -from ferro import Check, CheckViolationError, Field, Model, connect, engines +from ferro import ( + Check, + CheckViolationError, + Field, + Model, + connect, + create_tables, + engines, +) # --8<-- [start:models] -class Pair(Model): +class Payment(Model): __ferro_checks__: ClassVar[tuple[Check, ...]] = ( Check( - "at_most_one_side", - lambda pair: (pair.left == None) | (pair.right == None), # noqa: E711 + "at_most_one_method", + lambda payment: ( + (payment.card_token == None) | (payment.bank_account == None) + ), ), ) - id: Annotated[int | None, Field(default=None, primary_key=True)] - left: str | None = None - right: str | None = None + id: Annotated[int | None, Field(primary_key=True)] = None + card_token: str | None = None + bank_account: str | None = None # --8<-- [end:models] +DDL = """ +# --8<-- [start:ddl] +CREATE TABLE IF NOT EXISTS "payment" ( + "bank_account" varchar, + "card_token" varchar, + "id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, + CONSTRAINT "ck_payment_at_most_one_method" CHECK (("card_token" IS NULL) OR ("bank_account" IS NULL)) +); +# --8<-- [end:ddl] +""" + + +def _normalized(sql: str) -> str: + return re.sub(r"\s+", " ", sql).strip().rstrip(";").strip() + async def main() -> None: - await connect("sqlite::memory:", auto_migrate=True) + await connect("sqlite::memory:") + report = await create_tables() + [created] = [statement.sql for statement in report.statements] + expected = "\n".join(line for line in DDL.splitlines() if "--8<--" not in line) + assert _normalized(created) == _normalized(expected), created async with engines.session(): - await Pair.create(left="only-left") + await Payment.create(card_token="tok_visa_4242") + await Payment.create(bank_account="GB33BUKB20201555555555") + # --8<-- [start:violation] try: - await Pair.create(left="both", right="set") - except CheckViolationError: - pass + await Payment.create( + card_token="tok_visa_4242", + bank_account="GB33BUKB20201555555555", + ) + except CheckViolationError as exc: + assert "ck_payment_at_most_one_method" in str(exc) + # --8<-- [end:violation] else: raise AssertionError("expected CheckViolationError") - print("table_checks_annotated example ran successfully") + assert await Payment.select().count() == 2 + + print("table_checks example ran successfully") if __name__ == "__main__": diff --git a/docs/examples/testing_conftest.py b/docs/examples/testing_conftest.py index 62d26389..5e15bce7 100644 --- a/docs/examples/testing_conftest.py +++ b/docs/examples/testing_conftest.py @@ -1,6 +1,8 @@ -"""Example pytest fixtures for the Testing how-to (docs/pages/howto/testing.md). +"""The pytest fixtures of the testing how-to (docs/pages/howto/testing.md). -This file is snippeted into the docs; it is not meant to be executed directly. +Readers copy this file to ``tests/conftest.py``. +``tests/test_docs_testing_fixtures.py`` does the same in a scratch project +and runs the page's tests with it, so every fixture here runs under pytest. """ # --8<-- [start:fixtures] @@ -9,9 +11,13 @@ from ferro import connect, engines, reset_engine, transaction +class RollBack(Exception): + """Raised at the end of a test to roll its transaction back.""" + + @pytest.fixture async def db(): - """Fresh in-memory database per test.""" + """A fresh in-memory database, with a session open around the test.""" await connect("sqlite::memory:", auto_migrate=True) async with engines.session(): yield @@ -20,17 +26,13 @@ async def db(): @pytest.fixture async def db_transaction(db): - """Run a test inside a transaction sharing one connection.""" - async with transaction(): - yield + """The test runs in one transaction, which rolls back when it ends.""" + try: + async with transaction() as tx: + yield tx + raise RollBack + except RollBack: + pass # --8<-- [end:fixtures] - - -def main() -> None: # pragma: no cover - import smoke check only - """No-op: fixtures are exercised by pytest, not by running this file.""" - - -if __name__ == "__main__": - main() diff --git a/docs/examples/testing_conftest_postgres.py b/docs/examples/testing_conftest_postgres.py new file mode 100644 index 00000000..dbf34bf9 --- /dev/null +++ b/docs/examples/testing_conftest_postgres.py @@ -0,0 +1,44 @@ +"""The Postgres fixture of the testing how-to (docs/pages/howto/testing.md). + +``tests/test_docs_testing_fixtures.py`` runs it against a Postgres server +when one is available (``FERRO_POSTGRES_URL`` or local server binaries). +""" + +# --8<-- [start:postgres] +import os +import uuid +from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit + +import pytest + +from ferro import connect, engines, execute, reset_engine + +POSTGRES_URL = os.environ.get( + "TEST_POSTGRES_URL", "postgresql://localhost:5432/app_test" +) + + +def with_search_path(url: str, schema: str) -> str: + parts = urlsplit(url) + query = urlencode([*parse_qsl(parts.query), ("ferro_search_path", schema)]) + return urlunsplit(parts._replace(query=query)) + + +@pytest.fixture +async def pg_db(): + """A Postgres schema of its own for each test, dropped when it ends.""" + schema = f"test_{uuid.uuid4().hex[:8]}" + + await connect(POSTGRES_URL) + async with engines.session(): + await execute(f'CREATE SCHEMA "{schema}"') + reset_engine() + + await connect(with_search_path(POSTGRES_URL, schema), auto_migrate=True) + async with engines.session(): + yield + await execute(f'DROP SCHEMA "{schema}" CASCADE') + reset_engine() + + +# --8<-- [end:postgres] diff --git a/docs/examples/testing_models.py b/docs/examples/testing_models.py new file mode 100644 index 00000000..f32ca7ac --- /dev/null +++ b/docs/examples/testing_models.py @@ -0,0 +1,46 @@ +"""The models of the testing how-to (docs/pages/howto/testing.md). + +The page's readers keep these in ``myapp/models.py``; +``tests/test_docs_testing_fixtures.py`` copies this file there and runs the +page's tests against it. +""" + +import asyncio + +# --8<-- [start:models] +from typing import Annotated + +from ferro import BackRef, Field, ForeignKey, Model, Relation + + +class User(Model): + id: int | None = Field(default=None, primary_key=True) + username: str = Field(unique=True) + email: str = Field(unique=True) + posts: Relation[list["Post"]] = BackRef() + + +class Post(Model): + id: int | None = Field(default=None, primary_key=True) + title: str + content: str + author: Annotated[User, ForeignKey(related_name="posts")] + + +# --8<-- [end:models] + + +async def main() -> None: + from ferro import connect, engines + + await connect("sqlite::memory:", auto_migrate=True) + async with engines.session(): + ada = await User.create(username="ada", email="ada@example.com") + post = await Post.create(title="Notes", content="On the engine", author=ada) + assert (await post.author).username == "ada" + + print("testing_models example ran successfully") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/docs/examples/testing_models_annotated.py b/docs/examples/testing_models_annotated.py new file mode 100644 index 00000000..9aeba433 --- /dev/null +++ b/docs/examples/testing_models_annotated.py @@ -0,0 +1,46 @@ +"""The models of the testing how-to (docs/pages/howto/testing.md). + +The page's readers keep these in ``myapp/models.py``; +``tests/test_docs_testing_fixtures.py`` copies this file there and runs the +page's tests against it. +""" + +import asyncio + +# --8<-- [start:models] +from typing import Annotated + +from ferro import BackRef, Field, ForeignKey, Model, Relation + + +class User(Model): + id: Annotated[int | None, Field(primary_key=True)] = None + username: Annotated[str, Field(unique=True)] + email: Annotated[str, Field(unique=True)] + posts: Relation[list["Post"]] = BackRef() + + +class Post(Model): + id: Annotated[int | None, Field(primary_key=True)] = None + title: str + content: str + author: Annotated[User, ForeignKey(related_name="posts")] + + +# --8<-- [end:models] + + +async def main() -> None: + from ferro import connect, engines + + await connect("sqlite::memory:", auto_migrate=True) + async with engines.session(): + ada = await User.create(username="ada", email="ada@example.com") + post = await Post.create(title="Notes", content="On the engine", author=ada) + assert (await post.author).username == "ada" + + print("testing_models example ran successfully") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/docs/examples/timestamps.py b/docs/examples/timestamps.py index 83aa3239..ddf84e85 100644 --- a/docs/examples/timestamps.py +++ b/docs/examples/timestamps.py @@ -1,6 +1,15 @@ -"""Runnable companion to the Timestamps how-to (docs/pages/howto/timestamps.md).""" +"""Runnable companion to the timestamps how-to (docs/pages/howto/timestamps.md). + +Besides the recipe, it pins on SQLite that a batch ``update()`` that writes +``utcnow()`` stores the same text ``save()`` stores, so rows touched either +way sort together by ``updated_at`` and read back as aware datetimes +(AGENTS.md I-13). +""" import asyncio +import re + +from ferro import fetch_all # --8<-- [start:model] from datetime import UTC, datetime @@ -13,10 +22,10 @@ def utcnow() -> datetime: class TimestampMixin: - """Touch ``updated_at`` on every save. + """Set ``updated_at`` on every save. - A plain mixin (not a Model subclass): declare the timestamp fields on - each concrete model, and the mixin keeps them fresh. + A plain class, not a ``Model`` subclass: each model that uses it + declares ``created_at`` and ``updated_at`` itself. """ async def save(self, **kwargs) -> None: @@ -35,20 +44,56 @@ class Note(TimestampMixin, Model): async def main() -> None: + # --8<-- [start:session] await connect("sqlite::memory:", auto_migrate=True) async with engines.session(): + note = await Note.create(text="First draft") + assert note.created_at.tzinfo is UTC + # --8<-- [end:session] + # --8<-- [start:usage] - note = await Note.create(text="first draft") - original = note.updated_at + created = note.created_at + edited_before = note.updated_at - note.text = "second draft" + note.text = "Second draft" await note.save() - assert note.updated_at > original - assert note.created_at <= note.updated_at + assert note.created_at == created + assert note.updated_at > edited_before # --8<-- [end:usage] + # --8<-- [start:batch] + archived = await Note.create(text="Old idea") + await Note.create(text="Shopping list") + + touched = await Note.where(lambda note: note.text == "Old idea").update( + text="Old idea (archived)", updated_at=utcnow() + ) + assert touched == 1 + + latest = ( + await Note.select().order_by(lambda note: note.updated_at, "desc").first() + ) + assert latest.text == "Old idea (archived)" + assert latest.updated_at.tzinfo is not None + # --8<-- [end:batch] + + # SQLite stores datetimes as text. The batch update wrote the same + # text shape save() writes (I-13), so the rows sort by time, and + # every value reads back aware, comparable with the mixin's clock. + stored = await fetch_all('SELECT text, updated_at FROM "note"') + shapes = {re.sub(r"\d", "9", row["updated_at"]) for row in stored} + assert len(shapes) == 1, shapes + rows = await Note.select().order_by(lambda note: note.updated_at).all() + assert [row.text for row in rows] == [ + "Second draft", + "Shopping list", + "Old idea (archived)", + ] + assert archived.id == rows[-1].id + assert all(row.updated_at <= utcnow() for row in rows) + print("timestamps example ran successfully") diff --git a/docs/examples/timestamps_annotated.py b/docs/examples/timestamps_annotated.py index 440814e6..798fa9f6 100644 --- a/docs/examples/timestamps_annotated.py +++ b/docs/examples/timestamps_annotated.py @@ -1,6 +1,15 @@ -"""Annotated-style companion to timestamps.py (AGENTS.md I-7).""" +"""Runnable companion to the timestamps how-to (docs/pages/howto/timestamps.md). + +Besides the recipe, it pins on SQLite that a batch ``update()`` that writes +``utcnow()`` stores the same text ``save()`` stores, so rows touched either +way sort together by ``updated_at`` and read back as aware datetimes +(AGENTS.md I-13). +""" import asyncio +import re + +from ferro import fetch_all # --8<-- [start:model] from datetime import UTC, datetime @@ -14,10 +23,10 @@ def utcnow() -> datetime: class TimestampMixin: - """Touch ``updated_at`` on every save. + """Set ``updated_at`` on every save. - A plain mixin (not a Model subclass): declare the timestamp fields on - each concrete model, and the mixin keeps them fresh. + A plain class, not a ``Model`` subclass: each model that uses it + declares ``created_at`` and ``updated_at`` itself. """ async def save(self, **kwargs) -> None: @@ -26,27 +35,67 @@ async def save(self, **kwargs) -> None: class Note(TimestampMixin, Model): - id: Annotated[int | None, Field(default=None, primary_key=True)] + id: Annotated[int | None, Field(primary_key=True)] = None text: str - created_at: Annotated[datetime, Field(default_factory=utcnow)] - updated_at: Annotated[datetime, Field(default_factory=utcnow)] + created_at: datetime = Field(default_factory=utcnow) + updated_at: datetime = Field(default_factory=utcnow) # --8<-- [end:model] async def main() -> None: + # --8<-- [start:session] await connect("sqlite::memory:", auto_migrate=True) async with engines.session(): - note = await Note.create(text="first draft") - original = note.updated_at + note = await Note.create(text="First draft") + assert note.created_at.tzinfo is UTC + # --8<-- [end:session] + + # --8<-- [start:usage] + created = note.created_at + edited_before = note.updated_at - note.text = "second draft" + note.text = "Second draft" await note.save() - assert note.updated_at > original - print("timestamps_annotated example ran successfully") + assert note.created_at == created + assert note.updated_at > edited_before + # --8<-- [end:usage] + + # --8<-- [start:batch] + archived = await Note.create(text="Old idea") + await Note.create(text="Shopping list") + + touched = await Note.where(lambda note: note.text == "Old idea").update( + text="Old idea (archived)", updated_at=utcnow() + ) + assert touched == 1 + + latest = ( + await Note.select().order_by(lambda note: note.updated_at, "desc").first() + ) + assert latest.text == "Old idea (archived)" + assert latest.updated_at.tzinfo is not None + # --8<-- [end:batch] + + # SQLite stores datetimes as text. The batch update wrote the same + # text shape save() writes (I-13), so the rows sort by time, and + # every value reads back aware, comparable with the mixin's clock. + stored = await fetch_all('SELECT text, updated_at FROM "note"') + shapes = {re.sub(r"\d", "9", row["updated_at"]) for row in stored} + assert len(shapes) == 1, shapes + rows = await Note.select().order_by(lambda note: note.updated_at).all() + assert [row.text for row in rows] == [ + "Second draft", + "Shopping list", + "Old idea (archived)", + ] + assert archived.id == rows[-1].id + assert all(row.updated_at <= utcnow() for row in rows) + + print("timestamps example ran successfully") if __name__ == "__main__": diff --git a/docs/examples/transactions.py b/docs/examples/transactions.py index a1784f12..537c6b5f 100644 --- a/docs/examples/transactions.py +++ b/docs/examples/transactions.py @@ -2,33 +2,45 @@ import asyncio -from ferro import Field, Model, connect, engines, transaction +from ferro import IntegrityError, connect, engines, transaction + +# --8<-- [start:models] +from ferro import Field, Model class Account(Model): id: int | None = Field(default=None, primary_key=True) + number: str = Field(unique=True) owner: str balance: int = 0 +# --8<-- [end:models] + + +async def quote_interest(account: Account) -> int: + """Stand-in for a slow call made outside the transaction.""" + await asyncio.sleep(0) + return account.balance // 50 + + async def main() -> None: await connect("sqlite::memory:", auto_migrate=True) + # --8<-- [start:basic] async with engines.session(): - checking = await Account.create(owner="alice", balance=100) - savings = await Account.create(owner="alice", balance=0) + checking = await Account.create(number="CHK-001", owner="Alice", balance=100) + savings = await Account.create(number="SAV-001", owner="Alice", balance=0) - # --8<-- [start:basic] async with transaction(): checking.balance -= 25 await checking.save() savings.balance += 25 await savings.save() - # Both writes commit together when the block exits cleanly + + assert (checking.balance, savings.balance) == (75, 25) # --8<-- [end:basic] - await checking.refresh() - assert checking.balance == 75 # --8<-- [start:rollback] try: @@ -37,20 +49,77 @@ async def main() -> None: await checking.save() raise RuntimeError("insufficient funds") except RuntimeError: - pass + ... # the block raised, so everything in it rolled back + + assert checking.balance == -925 # the Python object kept the change await checking.refresh() - # The failed write was rolled back - assert checking.balance == 75 + assert checking.balance == 75 # the row did not # --8<-- [end:rollback] + # --8<-- [start:transfer] + async def transfer(src: Account, dst: Account, amount: int) -> bool: + try: + async with transaction(): + src.balance -= amount + await src.save() + if src.balance < 0: + raise ValueError("insufficient funds") + dst.balance += amount + await dst.save() + except ValueError: + await src.refresh() # undo the in-memory change too + return False + return True + + assert await transfer(checking, savings, 1000) is False + assert (checking.balance, savings.balance) == (75, 25) + + assert await transfer(checking, savings, 50) is True + assert (checking.balance, savings.balance) == (25, 75) + # --8<-- [end:transfer] + + # --8<-- [start:nested] + opened = [] + async with transaction(): + for number in ["CHK-002", "CHK-001", "SAV-002"]: + try: + async with transaction(): # a savepoint for each account + await Account.create(number=number, owner="Bob") + opened.append(number) + except IntegrityError: + ... # CHK-001 is taken: only this savepoint rolled back + + assert opened == ["CHK-002", "SAV-002"] + assert await Account.where(lambda account: account.owner == "Bob").count() == 2 + # --8<-- [end:nested] + # --8<-- [start:handle] async with transaction() as tx: - await Account.create(owner="bob") - # Raw SQL on the same connection, inside the same transaction + await Account.create(number="CHK-003", owner="Carol") rows = await tx.fetch_all("SELECT COUNT(*) AS n FROM account") + + assert rows[0]["n"] == 5 # the count saw Carol's uncommitted row # --8<-- [end:handle] - assert rows[0]["n"] == 3 + + # --8<-- [start:immediate] + async with transaction(immediate=True): + savings = await Account.get(savings.id) # read, then write + savings.balance += 10 + await savings.save() + + assert savings.balance == 85 + # --8<-- [end:immediate] + + # --8<-- [start:short] + interest = await quote_interest(savings) # slow work first + + async with transaction(): # then a short write + savings.balance += interest + await savings.save() + + assert savings.balance == 86 + # --8<-- [end:short] print("transactions example ran successfully") diff --git a/docs/examples/transactions_annotated.py b/docs/examples/transactions_annotated.py new file mode 100644 index 00000000..856fbfb7 --- /dev/null +++ b/docs/examples/transactions_annotated.py @@ -0,0 +1,129 @@ +"""Runnable companion to the Transactions guide (docs/pages/guide/transactions.md).""" + +import asyncio +from typing import Annotated + +from ferro import IntegrityError, connect, engines, transaction + +# --8<-- [start:models] +from ferro import Field, Model + + +class Account(Model): + id: Annotated[int | None, Field(primary_key=True)] = None + number: Annotated[str, Field(unique=True)] + owner: str + balance: int = 0 + + +# --8<-- [end:models] + + +async def quote_interest(account: Account) -> int: + """Stand-in for a slow call made outside the transaction.""" + await asyncio.sleep(0) + return account.balance // 50 + + +async def main() -> None: + await connect("sqlite::memory:", auto_migrate=True) + + # --8<-- [start:basic] + async with engines.session(): + checking = await Account.create(number="CHK-001", owner="Alice", balance=100) + savings = await Account.create(number="SAV-001", owner="Alice", balance=0) + + async with transaction(): + checking.balance -= 25 + await checking.save() + + savings.balance += 25 + await savings.save() + + assert (checking.balance, savings.balance) == (75, 25) + # --8<-- [end:basic] + + # --8<-- [start:rollback] + try: + async with transaction(): + checking.balance -= 1000 + await checking.save() + raise RuntimeError("insufficient funds") + except RuntimeError: + ... # the block raised, so everything in it rolled back + + assert checking.balance == -925 # the Python object kept the change + + await checking.refresh() + assert checking.balance == 75 # the row did not + # --8<-- [end:rollback] + + # --8<-- [start:transfer] + async def transfer(src: Account, dst: Account, amount: int) -> bool: + try: + async with transaction(): + src.balance -= amount + await src.save() + if src.balance < 0: + raise ValueError("insufficient funds") + dst.balance += amount + await dst.save() + except ValueError: + await src.refresh() # undo the in-memory change too + return False + return True + + assert await transfer(checking, savings, 1000) is False + assert (checking.balance, savings.balance) == (75, 25) + + assert await transfer(checking, savings, 50) is True + assert (checking.balance, savings.balance) == (25, 75) + # --8<-- [end:transfer] + + # --8<-- [start:nested] + opened = [] + async with transaction(): + for number in ["CHK-002", "CHK-001", "SAV-002"]: + try: + async with transaction(): # a savepoint for each account + await Account.create(number=number, owner="Bob") + opened.append(number) + except IntegrityError: + ... # CHK-001 is taken: only this savepoint rolled back + + assert opened == ["CHK-002", "SAV-002"] + assert await Account.where(lambda account: account.owner == "Bob").count() == 2 + # --8<-- [end:nested] + + # --8<-- [start:handle] + async with transaction() as tx: + await Account.create(number="CHK-003", owner="Carol") + rows = await tx.fetch_all("SELECT COUNT(*) AS n FROM account") + + assert rows[0]["n"] == 5 # the count saw Carol's uncommitted row + # --8<-- [end:handle] + + # --8<-- [start:immediate] + async with transaction(immediate=True): + savings = await Account.get(savings.id) # read, then write + savings.balance += 10 + await savings.save() + + assert savings.balance == 85 + # --8<-- [end:immediate] + + # --8<-- [start:short] + interest = await quote_interest(savings) # slow work first + + async with transaction(): # then a short write + savings.balance += interest + await savings.save() + + assert savings.balance == 86 + # --8<-- [end:short] + + print("transactions example ran successfully") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/docs/examples/traversal.py b/docs/examples/traversal.py index d79cfff2..ab70dd06 100644 --- a/docs/examples/traversal.py +++ b/docs/examples/traversal.py @@ -1,335 +1,324 @@ -"""Runnable companion to the "Querying Across Relationships" guide section +"""Runnable companion to "Querying across relationships" in the Queries guide (docs/pages/guide/queries.md). -Seeds a small ledger domain (Ledger <- Account -> Owner, with Transactions and -Notes hanging off Account) plus self-contained schemas for the two-FK, self-FK, -and many-to-many patterns, then runs every traversal query the guide shows and -asserts the exact rows that come back. +Runs every traversal query the guide shows against the guide's ledger +(declared in predicates.py, rows from _ledger_seed.py) and asserts the rows +that come back. """ import asyncio -from typing import Annotated +from datetime import date -from ferro import ( - BackRef, - Field, - ForeignKey, - ManyToMany, - Model, - Relation, - connect, - engines, -) +from _ledger_seed import seed +import predicates +from predicates import Account, Category, Owner, Tag, Transaction, Transfer - -# --8<-- [start:schema] -class Ledger(Model): - id: int | None = Field(default=None, primary_key=True) - name: str - accounts: Relation[list["Account"]] = BackRef() - - -class Owner(Model): - id: int | None = Field(default=None, primary_key=True) - email: str - accounts: Relation[list["Account"]] = BackRef() - - -class Account(Model): - id: int | None = Field(default=None, primary_key=True) - label: str - ledger: Annotated[Ledger, ForeignKey(related_name="accounts")] - owner: Annotated[Owner, ForeignKey(related_name="accounts")] - transactions: Relation[list["Transaction"]] = BackRef() - notes: Relation[list["Note"]] = BackRef() - - -class Transaction(Model): - id: int | None = Field(default=None, primary_key=True) - amount: int - account: Annotated[Account, ForeignKey(related_name="transactions")] - - -# --8<-- [end:schema] - - -# --8<-- [start:note-model] -class Note(Model): - id: int | None = Field(default=None, primary_key=True) - body: str - account: Annotated[Account | None, ForeignKey(related_name="notes")] = None - - -# --8<-- [end:note-model] - - -# --8<-- [start:two-fk-model] -class Airport(Model): - id: int | None = Field(default=None, primary_key=True) - code: str - departures: Relation[list["Flight"]] = BackRef() - arrivals: Relation[list["Flight"]] = BackRef() - - -class Flight(Model): - id: int | None = Field(default=None, primary_key=True) - origin: Annotated[Airport, ForeignKey(related_name="departures")] - destination: Annotated[Airport, ForeignKey(related_name="arrivals")] - - -# --8<-- [end:two-fk-model] - - -# --8<-- [start:self-fk-model] -class Employee(Model): - id: int | None = Field(default=None, primary_key=True) - name: str - manager: Annotated[ - "Employee", ForeignKey(related_name="reports", nullable=True) - ] = None - reports: Relation[list["Employee"]] = BackRef() - - -# --8<-- [end:self-fk-model] - - -# --8<-- [start:m2m-model] -class Author(Model): - id: int | None = Field(default=None, primary_key=True) - role: str - tags: Relation[list["Tag"]] = BackRef() - - -class Tag(Model): - id: int | None = Field(default=None, primary_key=True) - name: str - created_by: Annotated[Author, ForeignKey(related_name="tags")] - posts: Relation[list["Post"]] = BackRef() - - -class Post(Model): - id: int | None = Field(default=None, primary_key=True) - title: str - tags: Relation[list["Tag"]] = ManyToMany(related_name="posts") - - -# --8<-- [end:m2m-model] - - -async def _seed_ledger() -> None: - """Two ledgers, two owners, three accounts, six fan-in transactions. - - Ledger A holds a1 (owner o1) and a2 (owner o2); Ledger B holds b1 (owner o1). - Transactions: a1 has 10/20/30, a2 has 40, b1 has 50/60. - """ - la, lb = Ledger(id=1, name="ledger-a"), Ledger(id=2, name="ledger-b") - o1, o2 = Owner(id=1, email="o1@ferro.dev"), Owner(id=2, email="o2@ferro.dev") - for row in (la, lb, o1, o2): - await row.save() - - a1 = Account(id=1, label="a1", ledger=la, owner=o1) - a2 = Account(id=2, label="a2", ledger=la, owner=o2) - b1 = Account(id=3, label="b1", ledger=lb, owner=o1) - for row in (a1, a2, b1): - await row.save() - - for txn in ( - Transaction(id=1, amount=10, account=a1), - Transaction(id=2, amount=20, account=a1), - Transaction(id=3, amount=30, account=a1), - Transaction(id=4, amount=40, account=a2), - Transaction(id=5, amount=50, account=b1), - Transaction(id=6, amount=60, account=b1), - ): - await txn.save() - - # One note attached to a1, one orphan (nullable FK left NULL). - await Note(id=1, body="attached", account=a1).save() - await Note(id=2, body="orphan", account=None).save() +from ferro import connect, engines async def main() -> None: await connect("sqlite::memory:", auto_migrate=True) async with engines.session(): - await _seed_ledger() - - ledger_a = await Ledger.get(1) - owner_o2 = await Owner.get(2) - account_a1 = await Account.get(1) + await seed(vars(predicates)) # --8<-- [start:basic] - # Every transaction whose account belongs to ledger A. - rows = await Transaction.where( - lambda transaction: transaction.account.ledger_id == ledger_a.id + on_savings = await Transaction.where( + lambda transaction: transaction.account.label == "Savings" ).all() + assert [transaction.memo for transaction in on_savings] == ["From checking"] # --8<-- [end:basic] - assert {r.id for r in rows} == {1, 2, 3, 4} - - # --8<-- [start:composed] - top = await ( - Transaction.where( - lambda transaction: transaction.account.ledger_id == ledger_a.id - ) - .where(lambda transaction: transaction.amount >= 20) - .order_by(lambda transaction: transaction.amount, "desc") - .limit(2) - .all() - ) - # --8<-- [end:composed] - assert [r.amount for r in top] == [40, 30] # --8<-- [start:multi-hop] - rows = await Transaction.where( - lambda transaction: transaction.account.owner.email == "o2@ferro.dev" + graces = await Transaction.where( + lambda transaction: transaction.account.owner.email == "grace@example.com" ).all() + assert {transaction.memo for transaction in graces} == { + "Train to Lyon", + "Dinner in Lyon", + } # --8<-- [end:multi-hop] - assert {r.id for r in rows} == {4} + + # --8<-- [start:typo] + try: + Transaction.where( + lambda transaction: transaction.account.owner.emial == "ada@example.com" + ) + except AttributeError as error: + assert str(error) == ( + "Owner has no queryable column 'emial'. Did you mean 'email'? " + "Valid columns: email, id, name." + ) + else: + raise AssertionError("a misspelled hop must raise AttributeError") + # --8<-- [end:typo] # --8<-- [start:inner-narrows] - # Filtering through note.account drops the orphan note (NULL account FK). - ledger_a_notes = await Note.where( - lambda note: note.account.ledger_id == ledger_a.id + not_adas = await Transaction.where( + lambda transaction: transaction.account.owner.name != "Ada Lovelace" ).all() + # Office coffee is on Petty cash, which has no owner: it is not here. + assert {transaction.memo for transaction in not_adas} == { + "Train to Lyon", + "Dinner in Lyon", + } # --8<-- [end:inner-narrows] - assert {r.id for r in ledger_a_notes} == {1} + + # --8<-- [start:inner-or] + graces_or_coffee = await Transaction.where( + lambda transaction: ( + (transaction.account.owner.name == "Grace Hopper") + | (transaction.memo == "Office coffee") + ) + ).all() + # The join drops Office coffee before the OR is evaluated. + assert {transaction.memo for transaction in graces_or_coffee} == { + "Train to Lyon", + "Dinner in Lyon", + } + # --8<-- [end:inner-or] # --8<-- [start:count] - ledger_a_count = await Transaction.where( - lambda transaction: transaction.account.ledger_id == ledger_a.id + adas_count = await Transaction.where( + lambda transaction: transaction.account.owner.name == "Ada Lovelace" ).count() + assert adas_count == 6 # --8<-- [end:count] - assert ledger_a_count == 4 # --8<-- [start:order-by] - ordered = await ( - Transaction.select() - .order_by(lambda transaction: transaction.account.label) + by_account = ( + await Transaction.select() + .order_by(lambda transaction: transaction.account.label, "desc") .order_by(lambda transaction: transaction.id) + .limit(4) .all() ) + assert [transaction.memo for transaction in by_account] == [ + "Train to Lyon", # Travel + "Dinner in Lyon", # Travel + "From checking", # Savings + "Office coffee", # Petty cash + ] # --8<-- [end:order-by] - assert [r.id for r in ordered] == [1, 2, 3, 4, 5, 6] + + # --8<-- [start:order-by-narrows] + by_owner = ( + await Transaction.select() + .order_by(lambda transaction: transaction.account.owner.name) + .order_by(lambda transaction: transaction.id) + .all() + ) + assert len(by_owner) == 8 # Office coffee has no owner to sort by + # --8<-- [end:order-by-narrows] # --8<-- [start:traversed-paging] - # after/before take a decoded tuple — include() is not required to page. - next_page = await ( + by_label = ( Transaction.select() .order_by(lambda transaction: transaction.account.label) .order_by(lambda transaction: transaction.id) - .after(("a1", 2)) + ) + next_two = await by_label.after(("Petty cash", 8)).limit(2).all() + assert [transaction.memo for transaction in next_two] == [ + "From checking", + "Train to Lyon", + ] + # --8<-- [end:traversed-paging] + + # --8<-- [start:composed] + adas_largest_debits = ( + await Transaction.where( + lambda transaction: transaction.account.owner.name == "Ada Lovelace" + ) + .where(lambda transaction: transaction.amount < 0) + .order_by(lambda transaction: transaction.amount) .limit(2) .all() ) - # --8<-- [end:traversed-paging] - assert [r.id for r in next_page] == [3, 4] + assert [transaction.memo for transaction in adas_largest_debits] == [ + "Rent", + "To savings", + ] + # --8<-- [end:composed] - # --8<-- [start:instance-eq] - # `== instance` filters by the shadow FK column, with no join. - on_a1 = await Transaction.where( - lambda transaction: transaction.account == account_a1 + # --8<-- [start:shadow-column] + savings = await Account.get(2) + on_savings = await Transaction.where( + lambda transaction: transaction.account_id == savings.id ).all() - not_on_a1 = await Transaction.where( - lambda transaction: transaction.account != account_a1 + assert [transaction.memo for transaction in on_savings] == ["From checking"] + # --8<-- [end:shadow-column] + + # --8<-- [start:instance-eq] + on_savings = await Transaction.where( + lambda transaction: transaction.account == savings ).all() + elsewhere = await Transaction.where( + lambda transaction: transaction.account != savings + ).count() + assert [transaction.memo for transaction in on_savings] == ["From checking"] + assert elsewhere == 8 # --8<-- [end:instance-eq] - assert {r.id for r in on_a1} == {1, 2, 3} - assert {r.id for r in not_on_a1} == {4, 5, 6} # --8<-- [start:deep-instance-eq] - owned_by_o2 = await Transaction.where( - lambda transaction: transaction.account.owner == owner_o2 + grace = await Owner.get(2) + graces = await Transaction.where( + lambda transaction: transaction.account.owner == grace ).all() + assert {transaction.memo for transaction in graces} == { + "Train to Lyon", + "Dinner in Lyon", + } # --8<-- [end:deep-instance-eq] - assert {r.id for r in owned_by_o2} == {4} - # --8<-- [start:is-null] - # Join-free IS NULL / IS NOT NULL on the shadow FK. - orphans = await Note.where(lambda note: note.account == None).all() # noqa: E711 - attached = await Note.where(lambda note: note.account != None).all() # noqa: E711 - # --8<-- [end:is-null] - assert {r.id for r in orphans} == {2} - assert {r.id for r in attached} == {1} + # --8<-- [start:unpersisted] + try: + Transaction.where( + lambda transaction: transaction.account == Account(label="Brokerage") + ) + except ValueError as error: + assert str(error).startswith( + "cannot compare relation 'account' to an unpersisted Account instance" + ) + else: + raise AssertionError("an unpersisted instance must be refused") + # --8<-- [end:unpersisted] # --8<-- [start:existence] - # Bare .join() keeps only rows whose nullable relation exists. - with_account = await Note.select().join(lambda note: note.account).all() + owned = await Account.select().join(lambda account: account.owner).all() + assert {account.label for account in owned} == {"Checking", "Savings", "Travel"} # --8<-- [end:existence] - assert {r.id for r in with_account} == {1} + + # --8<-- [start:is-null] + ownerless = await Account.where(lambda account: account.owner == None).all() + owned_count = await Account.where(lambda account: account.owner != None).count() + assert [account.label for account in ownerless] == ["Petty cash"] + assert owned_count == 3 + # --8<-- [end:is-null] # --8<-- [start:left-join] - # .left_join() keeps the orphan row that traversal would drop. - all_notes = await Note.select().left_join(lambda note: note.account).all() + by_owner_name = ( + await Account.select() + .left_join(lambda account: account.owner) + .order_by(lambda account: account.owner.name) + .order_by(lambda account: account.id) + .all() + ) + assert [account.label for account in by_owner_name] == [ + "Checking", # Ada Lovelace + "Savings", # Ada Lovelace + "Travel", # Grace Hopper + "Petty cash", # no owner: NULL sorts last + ] # --8<-- [end:left-join] - assert {r.id for r in all_notes} == {1, 2} + + # --8<-- [start:two-fk-query] + checking_to_savings = await Transfer.where( + lambda transfer: ( + (transfer.outflow.account.label == "Checking") + & (transfer.inflow.account.label == "Savings") + ) + ).all() + assert [ + (transfer.outflow_id, transfer.inflow_id) + for transfer in checking_to_savings + ] == [(4, 5)] + # --8<-- [end:two-fk-query] + + # --8<-- [start:self-fk-query] + food_categories = await Category.where( + lambda category: category.parent.name == "Food" + ).all() + assert {category.name for category in food_categories} == { + "Groceries", + "Dining", + } + # --8<-- [end:self-fk-query] + + # --8<-- [start:m2m-query] + reimbursable = await Tag.where(lambda tag: tag.name == "reimbursable").first() + assert reimbursable is not None + + reimbursable_meals = await reimbursable.transactions.where( + lambda transaction: transaction.category.name == "Dining" + ).all() + assert [transaction.memo for transaction in reimbursable_meals] == [ + "Dinner in Lyon" + ] + # --8<-- [end:m2m-query] + + # --8<-- [start:reverse-filter] + checking = await Account.get(1) + + largest_debits = ( + await checking.transactions.where( + lambda transaction: transaction.amount < 0 + ) + .order_by(lambda transaction: transaction.amount) + .limit(2) + .all() + ) + checking_count = await checking.transactions.count() + + assert [transaction.memo for transaction in largest_debits] == [ + "Rent", + "To savings", + ] + assert checking_count == 5 + # --8<-- [end:reverse-filter] + + # --8<-- [start:mutate-error] + try: + await Transaction.where( + lambda transaction: transaction.account.label == "Travel" + ).update(cleared_on=date(2026, 3, 14)) + except ValueError as error: + assert str(error).startswith( + "update() does not support relation traversal: portable SQL has " + "no UPDATE ... JOIN." + ) + else: + raise AssertionError("a traversed update() must be refused") + # --8<-- [end:mutate-error] # --8<-- [start:mutate-limitation] - # Traversed predicates cannot mutate — fetch primary keys first... - ids = [ - note.id - for note in await Note.where( - lambda note: note.account.ledger_id == ledger_a.id + # Fetch the primary keys through the join... + travel_ids = [ + transaction.id + for transaction in await Transaction.where( + lambda transaction: transaction.account.label == "Travel" ).all() ] - # ...then delete by that key set with a join-free predicate. - await Note.where(lambda note: note.id.in_(ids)).delete() + # ...then update by that key set, with no join. + updated = await Transaction.where( + lambda transaction: transaction.id.in_(travel_ids) + ).update(cleared_on=date(2026, 3, 14)) + assert updated == 2 # --8<-- [end:mutate-limitation] - assert await Note.where(lambda note: note.id.in_(ids)).count() == 0 - # Restore the note for later assertions in this script. - await Note(id=1, body="attached", account=account_a1).save() - await _run_two_fk() - await _run_self_fk() - await _run_m2m() - - print("traversal example ran successfully") + # --8<-- [start:guardrails] + try: + Transaction.where(lambda transaction: transaction.account) + except TypeError: + pass + else: + raise AssertionError("a bare relation is not a predicate") + try: + Transaction.where( + lambda transaction: ( + (transaction.amount < 0) and (transaction.cleared_on == None) + ) + ) + except TypeError as error: + assert str(error).startswith( + "QueryNode cannot be used in a boolean context; use & / | to " + "combine predicates and ~ to negate them" + ) + else: + raise AssertionError("`and` between predicates must raise TypeError") + # --8<-- [end:guardrails] -async def _run_two_fk() -> None: - jfk = await Airport.create(id=1, code="JFK") - lax = await Airport.create(id=2, code="LAX") - await Flight.create(id=1, origin=jfk, destination=lax) - await Flight.create(id=2, origin=lax, destination=jfk) - - # --8<-- [start:two-fk-query] - # Each FK is its own relation path, so each traversal is its own join — - # no aliases to name. - departing_jfk = await Flight.where(lambda flight: flight.origin.code == "JFK").all() - arriving_jfk = await Flight.where( - lambda flight: flight.destination.code == "JFK" - ).all() - # --8<-- [end:two-fk-query] - assert {f.id for f in departing_jfk} == {1} - assert {f.id for f in arriving_jfk} == {2} - - -async def _run_self_fk() -> None: - boss = await Employee.create(id=1, name="boss", manager=None) - await Employee.create(id=2, name="alice", manager=boss) - await Employee.create(id=3, name="bob", manager=boss) - - # --8<-- [start:self-fk-query] - boss_reports = await Employee.where( - lambda employee: employee.manager.name == "boss" - ).all() - # --8<-- [end:self-fk-query] - assert {e.name for e in boss_reports} == {"alice", "bob"} - - -async def _run_m2m() -> None: - admin = await Author.create(id=1, role="admin") - member = await Author.create(id=2, role="member") - urgent = await Tag.create(id=1, name="urgent", created_by=admin) - chill = await Tag.create(id=2, name="chill", created_by=member) - post = await Post.create(id=1, title="launch") - await post.tags.add(urgent, chill) - - # --8<-- [start:m2m-query] - # The association context (post.tags) and forward-FK traversal on the tag - # compose in one statement. - admin_tags = await post.tags.where(lambda tag: tag.created_by.role == "admin").all() - # --8<-- [end:m2m-query] - assert {t.id for t in admin_tags} == {1} + print("traversal example ran successfully") if __name__ == "__main__": diff --git a/docs/examples/traversal_annotated.py b/docs/examples/traversal_annotated.py deleted file mode 100644 index 55ee8402..00000000 --- a/docs/examples/traversal_annotated.py +++ /dev/null @@ -1,136 +0,0 @@ -"""Annotated-style companion to traversal.py (AGENTS.md I-7). - -Field options move into ``Annotated[...]``. The relationship declarations are -identical in both styles: forward FKs are always ``Annotated[Target, -ForeignKey(...)]`` and ``BackRef()``/``ManyToMany()`` are always assignments. -The queries themselves declare no fields, so they live only in traversal.py. -""" - -import asyncio -from typing import Annotated - -from ferro import ( - BackRef, - Field, - ForeignKey, - ManyToMany, - Model, - Relation, - connect, - engines, -) - - -# --8<-- [start:schema] -class Ledger(Model): - id: Annotated[int | None, Field(default=None, primary_key=True)] - name: str - accounts: Relation[list["Account"]] = BackRef() - - -class Owner(Model): - id: Annotated[int | None, Field(default=None, primary_key=True)] - email: str - accounts: Relation[list["Account"]] = BackRef() - - -class Account(Model): - id: Annotated[int | None, Field(default=None, primary_key=True)] - label: str - ledger: Annotated[Ledger, ForeignKey(related_name="accounts")] - owner: Annotated[Owner, ForeignKey(related_name="accounts")] - transactions: Relation[list["Transaction"]] = BackRef() - notes: Relation[list["Note"]] = BackRef() - - -class Transaction(Model): - id: Annotated[int | None, Field(default=None, primary_key=True)] - amount: int - account: Annotated[Account, ForeignKey(related_name="transactions")] - - -# --8<-- [end:schema] - - -# --8<-- [start:note-model] -class Note(Model): - id: Annotated[int | None, Field(default=None, primary_key=True)] - body: str - account: Annotated[Account | None, ForeignKey(related_name="notes")] = None - - -# --8<-- [end:note-model] - - -# --8<-- [start:two-fk-model] -class Airport(Model): - id: Annotated[int | None, Field(default=None, primary_key=True)] - code: str - departures: Relation[list["Flight"]] = BackRef() - arrivals: Relation[list["Flight"]] = BackRef() - - -class Flight(Model): - id: Annotated[int | None, Field(default=None, primary_key=True)] - origin: Annotated[Airport, ForeignKey(related_name="departures")] - destination: Annotated[Airport, ForeignKey(related_name="arrivals")] - - -# --8<-- [end:two-fk-model] - - -# --8<-- [start:self-fk-model] -class Employee(Model): - id: Annotated[int | None, Field(default=None, primary_key=True)] - name: str - manager: Annotated[ - "Employee", ForeignKey(related_name="reports", nullable=True) - ] = None - reports: Relation[list["Employee"]] = BackRef() - - -# --8<-- [end:self-fk-model] - - -# --8<-- [start:m2m-model] -class Author(Model): - id: Annotated[int | None, Field(default=None, primary_key=True)] - role: str - tags: Relation[list["Tag"]] = BackRef() - - -class Tag(Model): - id: Annotated[int | None, Field(default=None, primary_key=True)] - name: str - created_by: Annotated[Author, ForeignKey(related_name="tags")] - posts: Relation[list["Post"]] = BackRef() - - -class Post(Model): - id: Annotated[int | None, Field(default=None, primary_key=True)] - title: str - tags: Relation[list["Tag"]] = ManyToMany(related_name="posts") - - -# --8<-- [end:m2m-model] - - -async def main() -> None: - await connect("sqlite::memory:", auto_migrate=True) - - async with engines.session(): - la = await Ledger.create(id=1, name="ledger-a") - o1 = await Owner.create(id=1, email="o1@ferro.dev") - a1 = await Account.create(id=1, label="a1", ledger=la, owner=o1) - await Transaction.create(id=1, amount=10, account=a1) - - rows = await Transaction.where( - lambda transaction: transaction.account.ledger_id == la.id - ).all() - assert {r.id for r in rows} == {1} - - print("traversal_annotated example ran successfully") - - -if __name__ == "__main__": - asyncio.run(main()) diff --git a/docs/examples/upgrade_account.py b/docs/examples/upgrade_account.py new file mode 100644 index 00000000..b01fe1c7 --- /dev/null +++ b/docs/examples/upgrade_account.py @@ -0,0 +1,55 @@ +"""The running example of the Upgrade guide (docs/pages/upgrading.md). + +One ``Account`` model, declared in the assignment style, and the session a +current release runs its queries in. The Annotated twin is +``upgrade_account_annotated.py``. +""" + +import asyncio +from datetime import UTC, datetime +from enum import StrEnum + +from ferro import Field, Model, connect, engines + + +# --8<-- [start:models] +class Role(StrEnum): + ADMIN = "admin" + MEMBER = "member" + + +class Account(Model): + # --8<-- [start:primary-key] + id: int | None = Field(default=None, primary_key=True) + email: str = Field(unique=True) + # --8<-- [end:primary-key] + name: str + role: Role = Role.MEMBER + age: int + active: bool = True + seen_at: datetime | None = None + prefs: dict = Field(default_factory=dict) + + +# --8<-- [end:models] + + +async def main() -> None: + await connect("sqlite::memory:", auto_migrate=True) + # --8<-- [start:session] + async with engines.session(): + await Account.create( + name="Ada Lovelace", + email="ada@example.com", + role=Role.ADMIN, + age=36, + seen_at=datetime(2026, 3, 1, 15, 0, tzinfo=UTC), + ) + await Account.create(name="Grace Hopper", email="grace@example.com", age=45) + adults = await Account.where(lambda account: account.age >= 40).all() + assert [account.name for account in adults] == ["Grace Hopper"] + # --8<-- [end:session] + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/docs/examples/upgrade_account_annotated.py b/docs/examples/upgrade_account_annotated.py new file mode 100644 index 00000000..caa87c59 --- /dev/null +++ b/docs/examples/upgrade_account_annotated.py @@ -0,0 +1,56 @@ +"""The running example of the Upgrade guide (docs/pages/upgrading.md). + +One ``Account`` model, declared in the assignment style, and the session a +current release runs its queries in. The Annotated twin is +``upgrade_account_annotated.py``. +""" + +import asyncio +from datetime import UTC, datetime +from enum import StrEnum +from typing import Annotated + +from ferro import Field, Model, connect, engines + + +# --8<-- [start:models] +class Role(StrEnum): + ADMIN = "admin" + MEMBER = "member" + + +class Account(Model): + # --8<-- [start:primary-key] + id: Annotated[int | None, Field(primary_key=True)] = None + email: Annotated[str, Field(unique=True)] + # --8<-- [end:primary-key] + name: str + role: Role = Role.MEMBER + age: int + active: bool = True + seen_at: datetime | None = None + prefs: dict = Field(default_factory=dict) + + +# --8<-- [end:models] + + +async def main() -> None: + await connect("sqlite::memory:", auto_migrate=True) + # --8<-- [start:session] + async with engines.session(): + await Account.create( + name="Ada Lovelace", + email="ada@example.com", + role=Role.ADMIN, + age=36, + seen_at=datetime(2026, 3, 1, 15, 0, tzinfo=UTC), + ) + await Account.create(name="Grace Hopper", email="grace@example.com", age=45) + adults = await Account.where(lambda account: account.age >= 40).all() + assert [account.name for account in adults] == ["Grace Hopper"] + # --8<-- [end:session] + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/docs/examples/upgrade_foreign_default.py b/docs/examples/upgrade_foreign_default.py new file mode 100644 index 00000000..fe182f2c --- /dev/null +++ b/docs/examples/upgrade_foreign_default.py @@ -0,0 +1,54 @@ +"""Runnable companion for the Upgrade guide's foreign-default recipe +(docs/pages/upgrading.md). + +A database where someone set a server default by hand on ``seen_at``, a column +the ``Account`` model declares without one. The pass reports it on every start. +""" + +import asyncio +import os +import sqlite3 +import tempfile +import warnings +from pathlib import Path + +from upgrade_account import Account # noqa: F401 - registers the model + +import ferro + +BEFORE = """ +CREATE TABLE "account" ( + "id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, + "name" varchar NOT NULL, + "email" varchar NOT NULL, + "role" varchar NOT NULL, + "age" integer NOT NULL, + "active" boolean NOT NULL, + "seen_at" DATETIME DEFAULT CURRENT_TIMESTAMP, + "prefs" JSON NOT NULL +); +CREATE UNIQUE INDEX "uq_account_email" ON "account" ("email"); +""" + + +async def main() -> None: + cwd = os.getcwd() + with tempfile.TemporaryDirectory() as scratch: + connection = sqlite3.connect(Path(scratch) / "dev.db") + connection.executescript(BEFORE) + connection.close() + os.chdir(scratch) + # --8<-- [start:warning] + await ferro.connect("sqlite:dev.db", migrate_updates=True) + report = await ferro.migrate() + [warning] = report.warnings + assert "account.seen_at has default CURRENT_TIMESTAMP" in str(warning) + # --8<-- [end:warning] + print(warning) + ferro.reset_engine() + os.chdir(cwd) + + +if __name__ == "__main__": + warnings.simplefilter("ignore") + asyncio.run(main()) diff --git a/docs/examples/upgrade_keep.py b/docs/examples/upgrade_keep.py new file mode 100644 index 00000000..81c6237c --- /dev/null +++ b/docs/examples/upgrade_keep.py @@ -0,0 +1,51 @@ +"""Runnable companion for the Upgrade guide's keep recipes +(docs/pages/upgrading.md). + +``Account`` with the three fields that opt out of a derived storage type that +changed: a naive timestamp, an enum stored as text with a check, and plain-JSON +storage. The Annotated twin is ``upgrade_keep_annotated.py``. +""" + +import asyncio +from datetime import datetime +from enum import StrEnum + +from ferro import Field, Model, connect, create_tables, engines + + +class Role(StrEnum): + ADMIN = "admin" + MEMBER = "member" + + +class Account(Model): + id: int | None = Field(default=None, primary_key=True) + name: str + email: str = Field(unique=True) + # --8<-- [start:keep-timestamp] + seen_at: datetime | None = Field(default=None, db_type="timestamp") + # --8<-- [end:keep-timestamp] + # --8<-- [start:keep-varchar] + role: Role = Field(default=Role.MEMBER, db_type="text", db_check=True) + # --8<-- [end:keep-varchar] + # --8<-- [start:keep-json] + prefs: dict = Field(default_factory=dict, db_type="json") + # --8<-- [end:keep-json] + + +async def main() -> None: + await connect("sqlite::memory:") + report = await create_tables() + [statement] = [ + s.sql + for s in report.statements + if s.sql.startswith('CREATE TABLE IF NOT EXISTS "account"') + ] + assert '"role" text NOT NULL CONSTRAINT "ck_account_role" CHECK' in statement + async with engines.session(): + ada = await Account.create(name="Ada", email="ada@example.com") + assert ada.role is Role.MEMBER + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/docs/examples/upgrade_keep_annotated.py b/docs/examples/upgrade_keep_annotated.py new file mode 100644 index 00000000..e65146aa --- /dev/null +++ b/docs/examples/upgrade_keep_annotated.py @@ -0,0 +1,52 @@ +"""Runnable companion for the Upgrade guide's keep recipes +(docs/pages/upgrading.md). + +``Account`` with the three fields that opt out of a derived storage type that +changed: a naive timestamp, an enum stored as text with a check, and plain-JSON +storage. The Annotated twin is ``upgrade_keep_annotated.py``. +""" + +import asyncio +from datetime import datetime +from enum import StrEnum +from typing import Annotated + +from ferro import Field, Model, connect, create_tables, engines + + +class Role(StrEnum): + ADMIN = "admin" + MEMBER = "member" + + +class Account(Model): + id: Annotated[int | None, Field(primary_key=True)] = None + name: str + email: Annotated[str, Field(unique=True)] + # --8<-- [start:keep-timestamp] + seen_at: Annotated[datetime | None, Field(db_type="timestamp")] = None + # --8<-- [end:keep-timestamp] + # --8<-- [start:keep-varchar] + role: Annotated[Role, Field(db_type="text", db_check=True)] = Role.MEMBER + # --8<-- [end:keep-varchar] + # --8<-- [start:keep-json] + prefs: Annotated[dict, Field(db_type="json")] = Field(default_factory=dict) + # --8<-- [end:keep-json] + + +async def main() -> None: + await connect("sqlite::memory:") + report = await create_tables() + [statement] = [ + s.sql + for s in report.statements + if s.sql.startswith('CREATE TABLE IF NOT EXISTS "account"') + ] + assert '"role" text NOT NULL CONSTRAINT "ck_account_role" CHECK' in statement + async with engines.session(): + ada = await Account.create(name="Ada", email="ada@example.com") + assert ada.role is Role.MEMBER + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/docs/examples/upgrade_primary_key.py b/docs/examples/upgrade_primary_key.py new file mode 100644 index 00000000..81aa6da6 --- /dev/null +++ b/docs/examples/upgrade_primary_key.py @@ -0,0 +1,21 @@ +"""Runnable companion for the Upgrade guide's primary-key recipe +(docs/pages/upgrading.md). + +A model with two ``primary_key=True`` fields is refused at class definition. +The Annotated twin is ``upgrade_primary_key_annotated.py``. +""" + +from ferro import Field, Model + +# --8<-- [start:refused] +try: + + class Account(Model): + id: int | None = Field(default=None, primary_key=True) + email: str = Field(primary_key=True) # a second primary key + +except TypeError as error: + assert "declares multiple primary-key columns" in str(error) +else: + raise AssertionError("a model with two primary keys was accepted") +# --8<-- [end:refused] diff --git a/docs/examples/upgrade_primary_key_annotated.py b/docs/examples/upgrade_primary_key_annotated.py new file mode 100644 index 00000000..6c26aee5 --- /dev/null +++ b/docs/examples/upgrade_primary_key_annotated.py @@ -0,0 +1,23 @@ +"""Runnable companion for the Upgrade guide's primary-key recipe +(docs/pages/upgrading.md). + +A model with two ``primary_key=True`` fields is refused at class definition. +The Annotated twin is ``upgrade_primary_key_annotated.py``. +""" + +from typing import Annotated + +from ferro import Field, Model + +# --8<-- [start:refused] +try: + + class Account(Model): + id: Annotated[int | None, Field(primary_key=True)] = None + email: Annotated[str, Field(primary_key=True)] # a second primary key + +except TypeError as error: + assert "declares multiple primary-key columns" in str(error) +else: + raise AssertionError("a model with two primary keys was accepted") +# --8<-- [end:refused] diff --git a/docs/includes/abbreviations.md b/docs/includes/abbreviations.md new file mode 100644 index 00000000..2f2acf22 --- /dev/null +++ b/docs/includes/abbreviations.md @@ -0,0 +1,19 @@ +*[ADR]: Architecture Decision Record: a short note that records one design decision and why it was made +*[ASGI]: Asynchronous Server Gateway Interface: the async Python web server standard +*[CRUD]: Create, Read, Update, Delete +*[DBAPI]: Python Database API (PEP 249) +*[DDL]: Data Definition Language: the CREATE, ALTER and DROP statements that change a schema +*[DML]: Data Manipulation Language: the INSERT, UPDATE and DELETE statements that change rows +*[FFI]: Foreign Function Interface: the boundary between Ferro's Python layer and its Rust core +*[FK]: Foreign key +*[FKs]: Foreign keys +*[GIL]: Global Interpreter Lock +*[GUC]: Grand Unified Configuration: a key/value Postgres runtime setting, such as the ones a Session applies for Row-Level Security +*[GUCs]: Grand Unified Configuration settings: key/value Postgres runtime settings, such as the ones a Session applies for Row-Level Security +*[IR]: Intermediate representation: the description of models or a query that Ferro's Python layer hands to its Rust core +*[JSONB]: Postgres binary JSON type +*[ORM]: Object-Relational Mapper +*[PK]: Primary key +*[RLS]: Row-Level Security: Postgres policies that decide which rows each connection may see or change +*[TTY]: Terminal +*[UUID]: Universally Unique Identifier diff --git a/docs/includes/field-types.md b/docs/includes/field-types.md new file mode 100644 index 00000000..2322ec7e --- /dev/null +++ b/docs/includes/field-types.md @@ -0,0 +1,40 @@ + + + +| Annotation | SQLite | PostgreSQL | +| :--- | :--- | :--- | +| `int` primary key | `integer PRIMARY KEY AUTOINCREMENT` | `serial PRIMARY KEY` | +| `int` | `integer` | `integer` | +| `str` | `varchar` | `varchar` | +| `bool` | `integer` | `bool` | +| `float` | `double` | `double precision` | +| `decimal.Decimal` | `NUMERIC` | `decimal` | +| `bytes` | `blob` | `bytea` | +| `datetime.datetime` | `DATETIME` | `timestamp with time zone` | +| `datetime.date` | `DATE` | `date` | +| `datetime.time` | `TIME` | `time` | +| `uuid.UUID` | `CHAR(32)` | `uuid` | +| `Status`, a `StrEnum` (`draft`, `active`, `retired`) | `varchar(7)` | native enum type `status` | +| `Priority`, an `IntEnum` (`1`, `2`, `3`) | `varchar(1)` | native enum type `priority` | +| `Literal["S", "M", "L"]` | `varchar(1)` | native enum type `size` | +| `dict[str, str]` | `JSON` | `jsonb` | +| `list[str]` | `JSON` | `jsonb` | +| `Dimensions`, a Pydantic `BaseModel` | `JSON` | `jsonb` | + + + +| `db_type` | SQLite | PostgreSQL | +| :--- | :--- | :--- | +| `"text"` | `text` | `text` | +| `varchar(255)` | `varchar(255)` | `varchar(255)` | +| `"smallint"` | `smallint` | `smallint` | +| `"int"` | `integer` | `integer` | +| `"bigint"` | `bigint` | `bigint` | +| `"uuid"` | `char(32)` | `uuid` | +| `"timestamp"` | `DATETIME` | `timestamp` | +| `"timestamptz"` | `DATETIME` | `timestamp with time zone` | +| `"date"` | `DATE` | `date` | +| `"time"` | `TIME` | `time` | +| `"json"` | `JSON` | `json` | +| `"jsonb"` | `JSON` | `jsonb` | + diff --git a/docs/pages/api/connection.md b/docs/pages/api/connection.md index bd7d6703..361b3088 100644 --- a/docs/pages/api/connection.md +++ b/docs/pages/api/connection.md @@ -1,12 +1,12 @@ -# Connection & Registry +--- +description: "API reference for connect(), sessions and the model registry." +--- -Functions for managing database connections and the global model registry. `connect()` registers a (optionally named) connection pool. `reset_engine()` tears everything down. The registry helpers control schema creation and the identity map. Sessionized routing is exposed via `ferro.engines.session(name)` / `ferro.Session`. See the [Connections & Databases guide](../guide/connections.md). +# connect, Session and the registry -::: ferro.Session +`connect()` registers a connection pool, optionally under a name, and `engines.session()` opens a session on one; every model call runs inside a session. `reset_engine()` closes every connection. See the [Connections and databases guide](../guide/connections.md). -::: ferro.engines - -::: ferro.current_session +## Connecting ::: ferro.connect @@ -16,6 +16,20 @@ Functions for managing database connections and the global model registry. `conn ::: ferro.reset_engine +::: ferro.version + +## Sessions + +::: ferro.engines + +::: ferro.session.EngineManager + +::: ferro.Session + +::: ferro.current_session + +## Registry + ::: ferro.create_tables ::: ferro.migrate @@ -26,13 +40,11 @@ Functions for managing database connections and the global model registry. `conn ::: ferro.evict_instance -::: ferro.version - -## What an auto-migrate pass did +## Auto-migrate reports -`ferro.migrate()` and `ferro.create_tables()` return a `PassReport`, built from the statements the database actually ran. A pass that fails partway raises its usual error with `.report` set to what committed before the failure. +`ferro.migrate()` and `ferro.create_tables()` return a `PassReport`, built from the statements the database actually ran. A pass that fails partway raises its usual error with `.report` set to what committed before the failure: -```python +```python {test=compile reason="needs a database one column behind its models"} report = await ferro.migrate(updates=True) [(s.subject, s.sql) for s in report.statements if s.role == "schema"] # [('author', 'ALTER TABLE "author" ADD COLUMN "slug" TEXT')] @@ -40,6 +52,8 @@ report = await ferro.migrate(updates=True) # [] ``` +Each `ExecutedStatement` names its table (`subject`) and its `role`; each warning is a typed `Report`. + ::: ferro.PassReport ::: ferro.ExecutedStatement diff --git a/docs/pages/api/exceptions.md b/docs/pages/api/exceptions.md index f2f1615d..d02d10b4 100644 --- a/docs/pages/api/exceptions.md +++ b/docs/pages/api/exceptions.md @@ -1,4 +1,8 @@ -# Exceptions +--- +description: "API reference for FerroError and the subclasses Ferro raises, each catchable by type." +--- + +# FerroError and its subclasses Every database failure Ferro raises is catchable by type. The tree is DBAPI-shaped, rooted at `FerroError`: @@ -18,20 +22,21 @@ FerroError └── MigrationRefused a migration call refused; .report says how far it got ├── PendingMigrationsError ├── DatabaseAheadError - ├── AlreadyTrackedError - └── AppliedAboveBaselineError + └── RunRefused a migration run, or connect(auto_migrate=True) on a tracked database, refused + ├── AlreadyTrackedError + └── AppliedAboveBaselineError ``` `ForeignKeyViolationError` covers every foreign-key rejection: a dangling insert, and deleting a row still referenced by `ForeignKey(on_delete="RESTRICT")`. Catch the type, not a SQLSTATE. On PostgreSQL 17 that RESTRICT delete reports `23503`; PostgreSQL 18 reports `23001` (`restrict_violation`). -Both are this class. `exc.sqlstate` is the raw driver code — it is not -rewritten. +Both are this class. `exc.sqlstate` is the raw driver code; Ferro does not +rewrite it. Catch a duplicate insert without matching driver text: -```python +```python {test=compile reason="uses an application's User model with a unique email"} from ferro import UniqueViolationError try: @@ -42,51 +47,33 @@ except UniqueViolationError as exc: print(exc.driver_message) # original driver text, for logs ``` -The same pattern applies to table checks declared in `__ferro_checks__` — a row that violates the predicate raises `CheckViolationError`: +The same pattern applies to table checks declared in `__ferro_checks__`. A row that violates the predicate raises `CheckViolationError`: -```python +```python {test=compile reason="uses the Pair model of the table checks guide"} from ferro import CheckViolationError try: await Pair.create(left="both", right="set") except CheckViolationError as exc: - print(exc.constraint) # e.g. "ck_pair_at_most_one_side" on Postgres + print(exc.constraint) # "ck_pair_at_most_one_side" on Postgres ``` -`ModelDoesNotExist` is raised by primary-key lookups like `Model.get(pk)` when -no row matches — use `Model.get_or_none(pk)` if you prefer `None` over an -exception — and by `save()` on a persisted instance whose row no longer exists +`ModelDoesNotExist` is raised by primary-key lookups such as `Model.get(pk)` +when no row matches; use `Model.get_or_none(pk)` to get `None` instead. It is +also raised by `save()` on a persisted instance whose row no longer exists (see [Saving: INSERT or UPDATE](../guide/mutations.md#saving-insert-or-update)). -It remains a `LookupError`, so pre-existing `except LookupError` handlers keep -working. +It is a `LookupError` too, so an `except LookupError` handler still catches it. `SettingsError` is raised by `FerroSettings` and every call that reads the project configuration; its message names the key to add or move, the line to write, or the variable to set. A refused migration call raises -[`MigrationRefused`](migrations.md#ferro.migrations.MigrationRefused), or one -of its subclasses for the states an application branches on: -[`PendingMigrationsError`](migrations.md#ferro.migrations.PendingMigrationsError) -when `require_applied()` finds the database behind its migrations, -[`DatabaseAheadError`](migrations.md#ferro.migrations.DatabaseAheadError) when -the database has applied migrations this checkout does not have (`.ahead` -names them, `.contract_steps` each of their steps that is not `expand`, as -`NNNN_:NN`: what `allow_ahead="expand-only"` refuses on), -[`AlreadyTrackedError`](migrations.md#ferro.migrations.AlreadyTrackedError) -when `baseline()` finds the database already tracked (another pre-deploy -adopted it first; `.applied` names the recorded migrations, `.head` the -newest), and -[`AppliedAboveBaselineError`](migrations.md#ferro.migrations.AppliedAboveBaselineError) -when `remove_baseline()` finds a run applied migrations above the baseline -(`.above` names them). Its `.report` is the report of the call that refused, -when it has one: the run's report (what `up()` applied before it stopped, the -refusal itself as `report.refused`), the `check()` report whose -`raise_for_problems()` raised it, or the status report of the database a -baseline refusal is about. A refusal is raised, never returned: an application -that called `up()` must not go on serving a database the run did not reach. - -```python +[`MigrationRefused`](migrations.md#ferro.migrations.MigrationRefused) or one of +its subclasses; see [Errors](migrations.md#errors) in the migrations reference. +Its `.report` says how far the call got: + +```python {test=compile reason="needs a project with a migrations directory"} import ferro.migrations from ferro.migrations import MigrationRefused diff --git a/docs/pages/api/fields.md b/docs/pages/api/fields.md index ae043be4..b498f5a0 100644 --- a/docs/pages/api/fields.md +++ b/docs/pages/api/fields.md @@ -1,8 +1,15 @@ -# Fields & Types +--- +description: "API reference for Field, FerroField, DbType and the other field metadata." +--- -Fields are declared as type annotations on a [`Model`](model.md), with `Field(...)` supplying database options (primary key, unique, index, defaults). The lower-level types on this page — `FerroField`, `DbType`, and friends — power `Annotated[...]` declarations and explicit column-type overrides. +# Field, FerroField and DbType + +Fields are declared as type annotations on a [`Model`](model.md), with `Field(...)` supplying database options (primary key, unique, index, defaults). The other entries on this page (`FerroField`, `FerroNullable`, `varchar`, `DbType` and `DbTypeToken`) describe the field metadata and the explicit column types `db_type=` takes. ::: ferro.Field + options: + show_overloads: false + parameter_headings: true ::: ferro.FerroField diff --git a/docs/pages/api/migrations.md b/docs/pages/api/migrations.md index c732a3ba..efb95134 100644 --- a/docs/pages/api/migrations.md +++ b/docs/pages/api/migrations.md @@ -1,8 +1,12 @@ -# Migrations +--- +description: "API reference for ferro.migrations, which applications, data steps, tests and an Alembic env.py import." +--- + +# ferro.migrations `ferro.migrations` is what an application, a data step, a test and an Alembic `env.py` import. The guides show each in use: [Deploying migrations](../guide/schema/deploying.md) for `up` and `require_applied`, [Data steps and backfills](../guide/schema/data-steps.md) for the declarations, [Testing migrations](../guide/schema/testing.md) for the harness, [Alembic](../guide/schema/alembic.md) for the bridge. The commands are in the [CLI reference](../reference/cli.md). -```python +```python {test=compile reason="needs a project with a migrations directory and a database URL"} import ferro import ferro.migrations @@ -13,7 +17,16 @@ await ferro.migrations.require_applied() # a server that migrates in its deplo ## Running migrations -Each call reads the project configuration (`settings=None` is `FerroSettings()`) and picks its database (`database=None` is the one configured). The calls that read a database (`up`, `require_applied`, `status`, `drift`, `baseline` and `remove_baseline`) take `using=`, the open connection to work on, or `url=`, to open a private connection for the call and close it after; with neither they work on the default connection, and with both they refuse. `check` reads only the models and the migrations directory and takes no connection. A refusal is raised, never returned: a refused `up` raises `MigrationRefused` whose `.report` is the run's report, saying what it applied before it stopped. +Each call takes these parameters to say which project, database and connection it works on: + +| Parameter | Meaning | Taken by | +| :--- | :--- | :--- | +| `settings=` | `None` is `FerroSettings()`, the project's config file | every call | +| `database=` | `None` is the one database configured | every call | +| `using=` | the open connection of that name; `None` is the default connection, unless `url=` is given | `up`, `require_applied`, `status`, `drift`, `baseline`, `remove_baseline` | +| `url=` | a private connection opened for the call and closed after it | the same six | + +Passing both `using=` and `url=` raises `MigrationRefused`. `check` reads only the models and the migrations directory and takes no connection. A refusal is raised, never returned: a refused `up` raises `MigrationRefused` whose `.report` is the run's report, saying what it applied before it stopped. ::: ferro.migrations.up @@ -23,6 +36,16 @@ Each call reads the project configuration (`settings=None` is `FerroSettings()`) ::: ferro.migrations.check +### Reports + +`up()` returns a `RunReport`, `status()` a `StatusReport` and `check()` a `CheckReport`. None of the three is exported: you read them, you never build one. + +::: ferro.migrations.runner.RunReport + +::: ferro.migrations.report.StatusReport + +::: ferro.migrations.generate.CheckReport + ## Baseline and drift ::: ferro.migrations.baseline @@ -39,13 +62,13 @@ Each call reads the project configuration (`settings=None` is `FerroSettings()`) ## Errors -A database ahead of the checkout raises `DatabaseAheadError`: `.ahead` names the migrations it has applied that the directory lacks, and `.contract_steps` each of their steps whose class is not `expand` (`["0008_drop_bio:01"]`), the steps `allow_ahead="expand-only"` refuses on. The [rolling-deploy recipe](../guide/schema/deploying.md#rolling-deploys-and-the-two-release-contract) shows it in use. +A database ahead of the checkout raises `DatabaseAheadError`: `.ahead` names the migrations it has applied that the directory lacks, and `.contract_steps` each of their steps whose class is not `expand` (`["0003_drop_bio:01"]`), the steps `allow_ahead="expand-only"` refuses on. The [rolling-deploy recipe](../guide/schema/deploying.md#rolling-deploys-and-the-two-release-contract) shows it in use. -```python +```python {test=compile reason="needs a database ahead of the checkout"} try: await ferro.migrations.require_applied(allow_ahead="expand-only") except ferro.migrations.DatabaseAheadError as ahead: - print(ahead.contract_steps) # ['0008_drop_bio:01'] + print(ahead.contract_steps) # ['0003_drop_bio:01'] ``` ::: ferro.migrations.MigrationRefused @@ -54,6 +77,8 @@ except ferro.migrations.DatabaseAheadError as ahead: ::: ferro.migrations.DatabaseAheadError +::: ferro.migrations.RunRefused + ::: ferro.migrations.AlreadyTrackedError ::: ferro.migrations.AppliedAboveBaselineError @@ -62,7 +87,7 @@ except ferro.migrations.DatabaseAheadError as ahead: ::: ferro.migrations.atomic -::: ferro.migrations.steps.chunked +::: ferro.migrations.chunked ::: ferro.migrations.irreversible @@ -82,10 +107,10 @@ except ferro.migrations.DatabaseAheadError as ahead: ## The Alembic bridge -Requires the `ferro-orm[alembic]` extra. These three load on first use, so the calls above never import Alembic. +Requires the `ferro-orm[alembic]` extra. These three functions load on first use, so the calls above never import Alembic. -::: ferro.migrations.alembic.get_metadata +::: ferro.migrations.get_metadata -::: ferro.migrations.alembic.ferro_options +::: ferro.migrations.ferro_options -::: ferro.migrations.alembic.render_item +::: ferro.migrations.render_item diff --git a/docs/pages/api/model.md b/docs/pages/api/model.md index df9b99f8..d991903c 100644 --- a/docs/pages/api/model.md +++ b/docs/pages/api/model.md @@ -1,9 +1,19 @@ +--- +description: "API reference for Model, the base class every Ferro model inherits from." +--- + # Model `Model` is the base class every Ferro model inherits from. The lifecycle is: define a subclass with annotated fields (which registers its table schema), [`connect()`](connection.md) to a database, then perform CRUD through classmethods (`create`, `get`, `where`, ...) and instance methods (`save`, `delete`, `refresh`). Because `Model` is a Pydantic model, instances validate on construction and serialize like any other Pydantic object. ::: ferro.Model +## Routing one call + +[`Model.using(name)`][ferro.Model.using] binds the model's class methods to one named connection and returns a `ModelConnection`. + +::: ferro.models.ModelConnection + ## Table-level declarations A model declares its table checks in `__ferro_checks__` and its row security in `__ferro_rls__`; see [Table checks](../guide/models-and-fields.md#table-checks) and [Using with Postgres RLS](../guide/row-level-security.md). diff --git a/docs/pages/api/queries.md b/docs/pages/api/queries.md index 265673cd..47396696 100644 --- a/docs/pages/api/queries.md +++ b/docs/pages/api/queries.md @@ -1,56 +1,88 @@ -# Queries +--- +description: "API reference for Query, the immutable builder that Model.where() and Model.select() return." +--- -`Model.where(...)` and `Model.select()` return a `Query` — an immutable, chainable builder that executes when awaited via `all()`, `first()`, `count()`, `exists()`, `update()`, or `delete()`. Predicates are lambda-only (`User.where(lambda user: user.age >= 18)`) — a fresh `QueryProxy` validates column names against the model at build time. +# Query -Prefix `~` negates **any** predicate — leaf comparison or `&`/`|` compound — rendering as SQL `NOT (...)` over the condition it wraps (ADR-0008). It is the universal negation rule: there are no per-operator negative forms (`~t.role.in_([...])` is NOT IN, `~t.email.like(p)` is NOT LIKE), and double negation nests. Like SQL `NOT` and the `!=` operator, a negated comparison excludes rows where the compared column is `NULL` — see [Negation and NULL values](../guide/queries.md#negation-and-null-values). +`Model.where(...)` and `Model.select()` return a `Query`: an immutable, chainable builder that runs when you await `all()`, `first()`, `count()`, `exists()`, `update()` or `delete()`. Predicates are lambdas over a `QueryProxy`, which checks every column name against the model when the query is built: -`where()` and `order_by()` lambdas may **traverse** a forward-FK relation (`lambda t: t.account.ledger_id == 1`): each hop renders one INNER join, deduplicated by relation path (ADR-0006). `join()` forces a join on a relation path (a bare `join()` is an existence filter on a nullable relation), and `left_join()` marks the whole path LEFT to keep relation-less rows. Omitted `nulls=` on `order_by` means `NULLS LAST` on every backend; pass `nulls="first"`, `nulls="last"`, or `nulls="native"` (dialect default) to override — see [Ordering, Limit & Offset](../guide/queries.md#ordering-limit-offset). See the [Querying Across Relationships](../guide/queries.md#querying-across-relationships) guide for worked examples. +```python {test=run prelude=reference} +popular = await Post.where(lambda post: post.views > 1000).order_by("title").all() +assert [post.title for post in popular] == ["Notes on the Analytical Engine"] +``` -A reverse (`BackRef`) or many-to-many relation in a predicate supports exactly one verb — the **existence test** `t.rel.exists(inner_lambda=None)` (ADR-0007). It renders as a correlated `EXISTS` at every cardinality (never a join, so the result stays root-shaped and each matching root returns once), negates with `~`, and the optional inner lambda is a full ferro predicate over the related model (operators, `&`/`|`/`~`, forward traversal rendered inside the subquery, nested tests). Everything else on a reverse edge — column access, comparisons (including `!= None`), `in_` (including a query RHS), `join()`/`left_join()` — raises at build time naming `.exists()`; an inner lambda referencing any scope but its own parameter is likewise rejected ([#309](https://github.com/syn54x/ferro-orm/issues/309)). See [Existence Tests](../guide/queries.md#existence-tests-on-reverse-many-to-many-relations) for worked examples. +The query ran once, at `all()`, and returned the one post with more than 1,000 views. -## `include()` and populated relations +The [Queries guide](../guide/queries.md) covers each feature with worked examples: -`include(lambda t: t.account)` delivers each result with the relation **populated** (ADR-0008): access becomes a plain attribute holding the complete related instance — no await, no query — while unpopulated relations keep the awaitable contract. Include is the third orthogonal query axis (joins decide membership, projection decides shape, include decides attached data): it never changes which rows come back, `.all()` still returns `list[Model]`, and `count()`/`exists()` are unaffected. Paths populate whole (`include(lambda t: t.account.owner)` populates both hops); includes are cumulative, order-free, and idempotent; populated instances run the full session identity-map protocol, and a refresh keeps a population only while the row's FK still points at it. - -Loud limits, at build time: forward-FK lambda paths only (a `BackRef`/M2M selector, a string, or a column selector raises `TypeError`); combining include with a projection raises `ValueError` in either chain order (one materialization plan per query; record results are flat — reach across a relation with traversed projection instead); `update()`/`delete()` on an included query raise `ValueError`. See [Populating Relations with include()](../guide/queries.md#populating-relations-with-include) for worked examples. +- `~` negates any predicate, rendering SQL `NOT (...)`: [Negation and NULL values](../guide/queries.md#negation-and-null-values). +- A lambda that follows a foreign key adds one `INNER JOIN` per relation path; `left_join()` keeps rows without the relation: [Querying across relationships](../guide/queries.md#querying-across-relationships). +- A `BackRef` or many-to-many relation in a predicate takes one verb, `.exists()`, rendered as a correlated `EXISTS`: [Existence tests](../guide/queries.md#existence-tests-on-reverse-many-to-many-relations). +- `include()` returns each result with a forward relation loaded, so reading it needs no `await`: [Populating relations with `include()`](../guide/queries.md#populating-relations-with-include). +- `nulls=` on `order_by()` defaults to `NULLS LAST` on every backend: [Ordering, limit and offset](../guide/queries.md#ordering-limit-offset). ## `select()` overloads -`select()` has four forms, resolved at build time: +`select()` has four forms, resolved when the query is built: | Call | Returns | Result of `.all()` | | :--- | :--- | :--- | -| `Model.select()` | `Query[Model]` | `list[Model]` — the full query, unchanged. | -| `Model.select(lambda t: (t.id, t.account.name))` | `ProjectedQuery[Model]` | `Rows[Row]` — projected records; fields may traverse forward-FK paths at any depth and take the bare leaf column name (single-field form: `select(lambda t: t.amount)`). | -| `Model.select(lambda t: {"owner_email": t.account.owner.email})` | `ProjectedQuery[Model]` | `Rows[Row]` — dict keys name the output fields (output aliases); values are field references or aggregate expressions. | -| `Model.select("id", "amount")` | `ProjectedQuery[Model]` | `Rows[Row]` — `order_by()`'s string contract: root columns only, never traversal, never mixed with a lambda. | +| `Post.select()` | `Query[Post]` | `list[Post]`: the full query, unchanged. | +| `Post.select(lambda post: (post.id, post.author.name))` | `ProjectedQuery[Post]` | `Rows[Row]`: one record per row; a field may follow forward foreign keys at any depth and takes the leaf column's name (one field: `select(lambda post: post.views)`). | +| `Post.select(lambda post: {"author_email": post.author.email})` | `ProjectedQuery[Post]` | `Rows[Row]`: the dict keys name the output fields; the values are field references or aggregates. | +| `Post.select("id", "views")` | `ProjectedQuery[Post]` | `Rows[Row]`: root columns only, never a traversal, never mixed with a lambda. | -A `ProjectedQuery` composes like any `Query` (`where()` with traversal, `order_by()` by unselected columns, `limit()`/`offset()`, `first()`, `count()`, `exists()`); `update()`/`delete()`, a second `select()`, output-name collisions, and nested selector shapes raise at build time. Projection traversal narrows per ADR-0006 (INNER, one join per path, shared with `where()`/`order_by()`); `left_join()` keeps relation-less rows with their traversed fields decoded to `None`. See [Selecting a Column Subset](../guide/queries.md#selecting-a-column-subset) for worked examples and the complete-instance invariant behind the `Row` result shape. +A `ProjectedQuery` chains like a `Query` (`where()`, `order_by()`, `limit()`, `offset()`, `first()`, `count()`, `exists()`). `update()`, `delete()`, a second `select()`, two fields with one output name and a nested selector raise when the query is built. See [Selecting a column subset](../guide/queries.md#selecting-a-column-subset). ## Aggregates and grouped queries -Five methods on column references build aggregate fields for the dict selector: `t.amount.count()` / `.sum()` / `.avg()` / `.min()` / `.max()` (traversal included: `t.account.balance.avg()`). Source families validate at build time — `sum`/`avg` take numeric columns, `min`/`max` orderable ones (numeric, text, date/time), `count` any column; enum/uuid/json/bool are rejected. Result types are a pinned cross-backend contract derived from the source column (`count → int`, `min`/`max` → source type, `sum` → source numeric type, `avg` → `float` for int/float and `Decimal` for Decimal); empty input passes SQL through verbatim (`None`, `count → 0`). +`count()`, `sum()`, `avg()`, `min()` and `max()` on a field build aggregates for the dict selector, also across a relation (`post.author.id.count()`). An aggregate-only projection returns one record; adding plain fields groups by them. Result types are the same on every backend: + +| Aggregate | Takes | Returns | On no rows | +| :--- | :--- | :--- | :--- | +| `count()` | any column | `int` | `0` | +| `sum()` | a numeric column | the column's type | `None` | +| `avg()` | a numeric column | `float` for `int` and `float`, `Decimal` for `Decimal` | `None` | +| `min()`, `max()` | a numeric, text or date/time column | the column's type | `None` | -An aggregate-only projection collapses to one record (read with `first()`). Mixing plain fields in makes the query **grouped**: every plain field is a group key — `GROUP BY` is derived from the projection, never declared. On a projected query `order_by()` strings resolve output field names first, then root columns; the lambda form spells source expressions including aggregates (`order_by(lambda t: t.amount.sum(), "desc")`); on an aggregate projection every sort key must be a group key or an aggregate — anything else raises at build time, as do `count()`/`exists()` (ambiguous between rows and groups) and aggregate predicates in `where()` (post-aggregation filtering is `having()`, #291). See [Aggregations & Grouped Queries](../guide/aggregations.md) for worked examples. +See [Aggregations and grouped queries](../guide/aggregations.md). -::: ferro.query.builder.Query +## Builders -::: ferro.query.builder.ProjectedQuery +::: ferro.query.Query + options: + # add, remove and clear work only on a many-to-many relation; they are + # documented on ferro.Relation. + filters: ["!^_", "!^(add|remove|clear)$"] ::: ferro.RenderedSql ::: ferro.REDACTED -::: ferro.query.rendered.Terminal +::: ferro.query.Terminal -::: ferro.query.Row +::: ferro.query.ProjectedQuery -::: ferro.query.Rows +## Predicates ::: ferro.query.QueryProxy ::: ferro.query.Predicate +::: ferro.query.FieldProxy + +::: ferro.query.RelationProxy + +::: ferro.query.QueryNode + +::: ferro.query.ValueExpr + +## Projections + +::: ferro.Row + +::: ferro.Rows + ::: ferro.query.RowSelector ::: ferro.query.AggregateExpr diff --git a/docs/pages/api/raw-sql.md b/docs/pages/api/raw-sql.md index 7192438c..48e6cf72 100644 --- a/docs/pages/api/raw-sql.md +++ b/docs/pages/api/raw-sql.md @@ -1,6 +1,10 @@ -# Raw SQL +--- +description: "API reference for execute(), fetch_all() and fetch_one()." +--- -Escape hatches for SQL the query builder doesn't cover. All three functions take a SQL string plus positional bind parameters — placeholders are `?` on SQLite and `$1`, `$2`, ... on Postgres — and honor an active `transaction()` block. See the [Raw SQL guide](../guide/raw-sql.md). +# execute, fetch_all and fetch_one + +Escape hatches for SQL the query builder does not cover. All three functions take an SQL string and positional bind parameters (placeholders are `?` on SQLite and `$1`, `$2`, ... on Postgres) and honor an active `transaction()` block. See the [Raw SQL guide](../guide/raw-sql.md). ::: ferro.execute diff --git a/docs/pages/api/relationships.md b/docs/pages/api/relationships.md index b2a49185..ef3cc627 100644 --- a/docs/pages/api/relationships.md +++ b/docs/pages/api/relationships.md @@ -1,6 +1,10 @@ -# Relationships +--- +description: "API reference for ForeignKey, BackRef, ManyToMany and Relation." +--- -Relationships are declared with annotations: `Annotated[Other, ForeignKey(...)]` for the owning side, `BackRef()` for the reverse side, and `ManyToMany(...)` for join-table relations. At runtime, related data is accessed through `Relation` — an awaitable, chainable query bound to the instance. See the [Relationships guide](../guide/relationships.md) for usage patterns. +# ForeignKey, BackRef, ManyToMany and Relation + +Relationships are declared with annotations: `Annotated[Other, ForeignKey(...)]` for the owning side, `BackRef()` for the reverse side, and `ManyToMany(...)` for join-table relations. At runtime you reach related rows through `Relation`, an awaitable, chainable query bound to the instance. See the [Relationships guide](../guide/relationships.md) for usage patterns. ::: ferro.ForeignKey @@ -9,3 +13,5 @@ Relationships are declared with annotations: `Annotated[Other, ForeignKey(...)]` ::: ferro.ManyToMany ::: ferro.Relation + options: + inherited_members: [add, remove, clear] diff --git a/docs/pages/api/transactions.md b/docs/pages/api/transactions.md index 0ef49784..2d37c158 100644 --- a/docs/pages/api/transactions.md +++ b/docs/pages/api/transactions.md @@ -1,6 +1,10 @@ -# Transactions +--- +description: "API reference for transaction() and the Transaction handle it yields." +--- -`transaction()` is an async context manager that runs everything inside the block on one connection, committing on exit and rolling back on exception. It yields a `Transaction` handle for raw SQL that must run on the transaction's connection. It can route by `using=` or by an explicit/ambient session (`ferro.engines.session(...)`). See the [Transactions guide](../guide/transactions.md) for semantics and patterns. +# transaction and Transaction + +`transaction()` is an async context manager that runs everything inside the block on one connection, committing on exit and rolling back on exception. It yields a `Transaction` handle for raw SQL that must run on the transaction's connection. Route it with `using=`, an explicit `session=`, or the ambient session (`ferro.engines.session(...)`). See the [Transactions guide](../guide/transactions.md) for semantics and patterns. ::: ferro.transaction diff --git a/docs/pages/changelog.md b/docs/pages/changelog.md index 786b75d5..66e81436 100644 --- a/docs/pages/changelog.md +++ b/docs/pages/changelog.md @@ -1 +1,7 @@ ---8<-- "CHANGELOG.md" +--- +description: "Every change in every Ferro release." +--- + +# Changelog + +--8<-- "CHANGELOG.md:2:" diff --git a/docs/pages/concepts/architecture.md b/docs/pages/concepts/architecture.md index 7e771c1b..446bedcd 100644 --- a/docs/pages/concepts/architecture.md +++ b/docs/pages/concepts/architecture.md @@ -1,79 +1,91 @@ +--- +description: "How one model and one query travel through Ferro's Python layer, its Rust core and the database." +--- + # Architecture -Ferro is a Python ORM with a Rust core. You write Pydantic models and async Python; SQL generation, database I/O, and row hydration happen in compiled Rust. +This page follows one model and one query through Ferro's layers: what runs in Python, what crosses into Rust, and when the database is involved. -## Overview +The running example is a `User` model: -Ferro has two main pipelines: **schema compilation** (happens at import/class-creation) and **query execution** (happens per `await`). They share IR contracts but run at different times. +=== "Assignment" -### Schema path (compile-time) + ```python + --8<-- "docs/examples/concepts.py:models" + ``` -Runs when a `Model` subclass is defined. `ModelMetaclass` calls `build_model_schema()` once per model; that enriched schema fans out to SchemaIR compilation and Rust registry registration. +=== "Annotated" + + ```python + --8<-- "docs/examples/concepts_annotated.py:models" + ``` + +With four users saved, this query reads the active ones, oldest first: + +```python +--8<-- "docs/examples/concepts.py:query" +``` + +Edsger is inactive, so three users come back. Python built the query, Rust wrote the SQL, ran it and turned the rows into `User` instances, and the session decided which connection ran it. The rest of this page takes those steps one layer at a time. + +## Two pipelines + +Ferro does its work at two different times: + +- **When a model class is defined**, Ferro compiles its schema. This happens once, at import, and touches no database. +- **When you await a query or a write**, Ferro runs it. This happens on every `await`, inside a session. + +### Schema path + +Defining `User` runs the model's metaclass, which compiles the class into a **SchemaIR** envelope: a versioned, JSON-shaped description of the table, its columns, keys, indexes and constraints. The envelope is kept in Python. When you call `connect()`, Ferro installs the whole set of model envelopes into the Rust engine in one step, and every consumer of the schema reads that one description: ```mermaid graph LR - subgraph python [Python] - Models[Pydantic Models] + subgraph python [Python, at class definition] + Models[Model classes] Metaclass[ModelMetaclass] - Builder[build_model_schema] - Compiler[SchemaIR Compiler] - end - - subgraph artifacts [Artifacts] - SchemaIR[SchemaIR Envelope] - EnrichedJSON[Enriched Schema JSON] + SchemaIR[SchemaIR envelopes] + Models --> Metaclass --> SchemaIR end subgraph consumers [Consumers] + Registry[Rust model registry] + Planner[Schema planner] Alembic[Alembic get_metadata] - Registry[Rust Model Registry] - Migrate[Runtime DDL / ferro-migrate] end - Models --> Metaclass - Metaclass --> Builder - Builder --> EnrichedJSON - Builder --> Compiler - Compiler --> SchemaIR + SchemaIR -->|connect| Registry + SchemaIR -->|connect| Planner SchemaIR --> Alembic - EnrichedJSON --> Registry - EnrichedJSON --> Migrate ``` -> **Implementation status (2026-06-26):** only `SchemaIR --> Alembic` is a fully -> realized SchemaIR consumer. The runtime DDL / `ferro-migrate` path currently -> re-derives its own SchemaIR in Rust from the enriched JSON (it does not consume -> the Python-compiled SchemaIR envelope), and the runtime CREATE emitter -> (`src/schema.rs`) uses its own canonical type system rather than the shared -> `ferro-ddl-lowering` crate. Collapsing these onto one SchemaIR producer and one -> lowering library is tracked as **Phase 8.5** — see the -> [IR-first lowering consolidation audit](https://github.com/syn54x/ferro-orm/blob/main/docs/solutions/architecture-patterns/ir-first-lowering-consolidation-audit.md). +The registry gives the engine each column's type for binding values and hydrating rows. The planner compares the models with a database and writes the DDL; auto-migrate, `ferro migrate new` and the Alembic bridge all use it, which is covered in [Schema changes](#schema-changes) below. -### Runtime path (per operation) +### Runtime path -Runs inside an active session when you await a terminal query method or execute a mutation. QueryIR crosses the FFI boundary; Rust owns SQL, I/O, and hydration. +Awaiting a query runs it inside the active session. The query crosses into Rust as a **QueryIR** envelope, and Rust owns the SQL, the database I/O and the hydration of rows into instances: ```mermaid graph TB subgraph python [Python] direction TB Session[engines.session] - Builder[Query Builder] + Builder[Query builder] Session --> Builder end - Builder -->|await| QueryIR[QueryIR Envelope] + Builder -->|await| QueryIR[QueryIR envelope] QueryIR --> Planner - subgraph rust [Rust Engine] + subgraph rust [Rust engine] direction TB - Planner[Query Planner] - Codec[Codec Registry] + Planner[Query planner] + Codec[Codec registry] SeaQuery[Sea-Query] SQLx[SQLx] - Hydration[Hydration ABI] - IdentityMap[Session Identity Map] + Hydration[Hydration] + IdentityMap[Session identity map] Planner --> Codec --> SeaQuery --> SQLx Hydration --> IdentityMap end @@ -84,133 +96,100 @@ graph TB Session -.->|scopes| IdentityMap ``` -In short: +Each layer owns one part of the work: -```text -Python owns the model authoring surface. -SchemaIR and QueryIR own the cross-language contracts. -Rust owns execution, codecs, and hydration. -Sea-Query owns SQL shape. -SQLx owns typed database I/O. -Sessions scope routing, transactions, and the identity map. -The backend kind decides which database-specific path is legal. -``` +| Layer | Owns | +| :--- | :--- | +| Python | Models, validation, building queries | +| SchemaIR and QueryIR | The contracts that cross from Python to Rust | +| Rust engine | Planning, value conversion, hydration | +| Sea-Query | The shape of the SQL for each dialect | +| SQLx | Connection pools and database I/O | +| Sessions | Which connection runs the work, transactions, the identity map | + +The backend decides which database-specific path is legal: a SQLite pool and a PostgreSQL pool are separate types, not one abstraction pretending to be both. -## The Layers +## Layers ### Python -Ferro models are real Pydantic V2 `BaseModel` subclasses. The Python layer owns: +Ferro models are Pydantic v2 `BaseModel` subclasses, and the Python layer owns everything you write: + +- **Model definition.** Annotated fields become columns; Pydantic handles validation, defaults, serialization and JSON schema generation. See [Type safety](type-safety.md). +- **Schema compilation.** The metaclass reads each model at class definition (primary key, uniques, indexes, foreign keys, nullability, composite constraints) and compiles its SchemaIR envelope. +- **Query building.** Each call in `User.where(...).order_by(...).limit(10)` returns a new, immutable query and touches no database. Nothing runs until you await a terminal method such as `.all()`, `.first()`, `.count()`, `.exists()`, `.update()` or `.delete()`. The lambda's parameter is a [`QueryProxy`][ferro.query.QueryProxy] that checks each attribute name as the lambda runs; [Typed query predicates](query-typing.md) explains it. +- **Session routing.** ORM and raw SQL calls run inside an active `engines.session()` block, or take an explicit `session=` handle. -- **Model definition.** Annotated fields become columns; Pydantic handles validation, defaults, serialization, and JSON schema generation. -- **Metaclass registration.** `ModelMetaclass` inspects each model at class-creation time, builds an enriched JSON schema (primary keys, uniques, indexes, foreign keys, nullability, composite constraints), registers it with the Rust engine, and compiles a versioned **SchemaIR** artifact. -- **Query building.** Chains like `User.where(lambda t: t.active).order_by(lambda t: t.age, "desc").limit(10)` are pure Python — each call returns a new, immutable `Query` that accumulates an in-memory query definition. Nothing touches the database until you await a terminal method (`.all()`, `.first()`, `.count()`, `.exists()`, `.update()`, `.delete()`). The lambda's parameter is a `QueryProxy` — a validating attribute proxy built fresh per predicate, not a class-level replacement of the model's fields — see [Typed Query Predicates](query-typing.md). -- **Session routing.** ORM and raw operations run inside an active `engines.session()` context (or with an explicit `session=` handle). Sessions scope connection routing, transactions, and the identity map under concurrency. +### Bridge -### The Bridge +The boundary between Python and Rust is built on [PyO3](https://pyo3.rs), with `pyo3-async-runtimes` connecting Python's asyncio event loop to Rust's tokio runtime. Three things cross it: -The FFI boundary is built on [PyO3](https://pyo3.rs) with `pyo3-async-runtimes` bridging Python's asyncio event loop to Rust's tokio runtime. Versioned IR envelopes cross the boundary: +- **SchemaIR** (`ir_kind: "schema"`): one envelope per model, compiled at class definition and installed into the engine by `connect()`. The Alembic bridge's `get_metadata()` reads the same envelopes. +- **QueryIR** (`ir_kind: "query"`): one envelope per operation, `{ir_kind, ir_version, payload}`. Rust plans SQL from it and binds every value through the codec registry. +- **Rows**, on the way back, which Rust hydrates into Python instances. -- **SchemaIR** (`ir_kind: "schema"`) — compiled at class-creation time from the enriched model schema. Cached in Python (`ferro.state`) and consumed by Alembic `get_metadata()` and parity tests. The Rust registry receives enriched JSON schema for runtime query/bind metadata, and the runtime migration planner currently re-derives its own SchemaIR in Rust from that JSON rather than consuming this envelope (consolidation tracked in [Phase 8.5](https://github.com/syn54x/ferro-orm/blob/main/docs/plans/2026-06-19-001-ir-first-roadmap.md)). -- **QueryIR** (`ir_kind: "query"`) — emitted per operation from the query builder as `{ir_kind, ir_version, payload}`. Rust deserializes the envelope, plans SQL, and binds parameters through the shared codec registry. -- **Rows** travel back as typed values that Rust hydrates into Python objects via the hydration ABI (direct `__dict__` population with required Pydantic slots initialized). +While Rust waits on the database, it holds no Python lock: the await runs on tokio, so other coroutines and threads keep running. [Async from end to end](#async-from-end-to-end) covers what that means for concurrent work. -The GIL is released while Rust waits on the database. An awaited Ferro query does not block other Python coroutines or threads. +### Rust engine -### The Rust Engine +The engine owns everything between the QueryIR envelope and the database: -The engine owns everything between the IR envelope and the database: +- **Query planning.** The QueryIR payload becomes Sea-Query conditions, with each bound value typed from the model's schema. +- **Codec registry.** One set of conversions for inserts, updates, filters, many-to-many links and hydration: typed `NULL`s, UUIDs, decimals, dates and times. +- **SQL generation** through [Sea-Query](https://github.com/SeaQL/sea-query), with the SQLite or PostgreSQL builder and bound parameters. Values are bound, never interpolated into the SQL. +- **Connection pooling and execution** through [SQLx](https://github.com/launchbadge/sqlx) typed pools. +- **Hydration.** Rust decodes each row and fills the instance's fields directly, without calling the model's `__init__`, and sets the Pydantic attributes `__init__` would set (`__pydantic_fields_set__`, `__pydantic_extra__`, `__pydantic_private__`). A hydrated `User` behaves like one you constructed. +- **Session identity map**, so one row is one Python instance within a session. See [Identity map](identity-map.md). -- **Query planning** — QueryIR payloads lower to Sea-Query conditions with schema-aware bind typing. -- **Codec registry** — unified bind/fetch lowering for inserts, updates, filters, M2M links, and hydration (typed NULLs, UUIDs, decimals, temporals). -- **SQL generation** via [Sea-Query](https://github.com/SeaQL/sea-query), which lowers each operation through the dialect-specific builder (SQLite or PostgreSQL) with safely bound parameters. -- **Connection pooling and execution** via [SQLx](https://github.com/launchbadge/sqlx) typed pools — a real SQLite pool or a real PostgreSQL pool, not a generic abstraction pretending to be both. -- **Hydration ABI** — decoding database values into Python-compatible shapes and constructing instances without calling Pydantic `__init__`, while initializing `__pydantic_fields_set__`, `__pydantic_extra__`, and `__pydantic_private__`. -- **Session-scoped identity map** — a per-session cache so one row maps to one Python instance within the active session. See [Identity Map](identity-map.md). +## Life of a query -## Life of a Query +The query from the top of the page goes through these steps: ```mermaid sequenceDiagram - participant App as Your Code + participant App as Your code participant Session as engines.session - participant QB as Query Builder (Python) - participant Rust as Rust Engine + participant QB as Query builder (Python) + participant Rust as Rust engine participant DB as Database - App->>Session: async with engines.session("app") - App->>QB: User.where(lambda t: t.age > 18).all() - QB->>QB: Build QueryNode tree (no I/O) - QB->>Rust: await QueryIR envelope via FFI - Note over Rust: GIL released - Rust->>Rust: Plan query, codec bind, Sea-Query SQL - Rust->>DB: SELECT ... WHERE age > $1 + App->>Session: async with engines.session() + App->>QB: User.where(lambda user: user.active == True)...all() + QB->>QB: Build the query tree (no I/O) + QB->>Rust: await the QueryIR envelope + Rust->>Rust: Plan, bind values, render SQL + Rust->>DB: SELECT ... WHERE ... ORDER BY ... LIMIT ... DB-->>Rust: Rows - Rust->>Rust: Hydrate via hydration ABI - Rust->>Rust: Reconcile with session identity map - Rust-->>QB: Python objects - QB-->>App: list[User] (validated Pydantic instances) + Rust->>Rust: Hydrate, reconcile with the identity map + Rust-->>QB: User instances + QB-->>App: list[User] ``` Step by step: -1. **Session** — `async with ferro.engines.session("app")` establishes ambient routing for ORM/raw operations (or pass `session=` explicitly). -2. **Construction** — `User.where(lambda t: t.age > 18)` builds a `QueryNode` tree in Python. No database interaction. -3. **Execution trigger** — awaiting `.all()` wraps the query in a **QueryIR envelope** and calls into Rust. -4. **SQL generation** — the planner and codec registry lower the IR into dialect-correct, parameterized SQL. Values are bound, never interpolated. -5. **Execution** — SQLx runs the statement on a pooled connection (or on the pinned connection if a [transaction](../guide/transactions.md) is active). -6. **Hydration** — Rust decodes each row through the hydration ABI, consulting the session identity map so a primary key you've already loaded resolves to the existing object. -7. **Return** — your code receives a plain `list[User]` of fully validated Pydantic instances. +1. **Session.** `async with engines.session():` sets the connection that ORM and raw SQL calls inside it use. You can pass `session=` explicitly instead. +2. **Construction.** `User.where(lambda user: user.active == True)` calls the lambda once with a proxy and builds a query tree in Python. No database is involved. +3. **Execution.** Awaiting `.all()` wraps the query in a QueryIR envelope and calls into Rust. +4. **SQL generation.** The planner and the codec registry turn the envelope into parameterized SQL for the connection's dialect. +5. **Execution.** SQLx runs the statement on a pooled connection, or on the pinned connection of an open [transaction](../guide/transactions.md). +6. **Hydration.** Rust decodes each row and checks the session's identity map, so a row you already hold comes back as the instance you hold. +7. **Return.** Your code receives a `list[User]`. -## Model Registration +## Schema changes -Defining a model is itself a registration step: - -=== "Assignment" - - ```python - from ferro import Field, Model - - - class User(Model): - id: int | None = Field(default=None, primary_key=True) - username: str = Field(unique=True) - email: str - ``` - -=== "Annotated" - - ```python - from typing import Annotated - - from ferro import Field, Model - - - class User(Model): - id: Annotated[int | None, Field(default=None, primary_key=True)] - username: Annotated[str, Field(unique=True)] - email: str - ``` - -At class-creation time the metaclass calls `build_model_schema()`, registers the enriched JSON with the Rust engine's model registry, and compiles **SchemaIR** artifacts (per-model and model-set fingerprints) from the same schema dict. Importing your models is therefore enough for Ferro to know your full schema — but it does **not** connect to a database. - -Schema DDL happens later, when you connect: +Defining `User` compiles its schema, but connects to nothing. The schema reaches a database when you connect with auto-migrate: ```python -from ferro import connect - -await connect("sqlite::memory:", auto_migrate=True) +--8<-- "docs/examples/concepts.py:connect" ``` -- `auto_migrate=True` creates missing tables for every registered model. -- `migrate_updates=True` (0.11.0) additionally adds missing columns to existing tables, and on PostgreSQL reconciles type drift, nullability drift, and foreign-key definition drift (`on_delete`, target) for ferro-owned constraints. Runtime updates are planned from **SchemaIR** diffing (`ferro-migrate`) and executed as backend-specific DDL. -- `migrate_destructive=True` (0.11.0) additionally drops live columns no longer on the model (never whole tables). +That call creates the `user` table, because no table of that name existed. `auto_migrate=True` creates missing tables and leaves existing ones alone; `migrate_updates=True` and `migrate_destructive=True` go further, and [Auto-migrate](../guide/schema/auto-migrate.md) lists what each one changes. -Reviewed changes go through [migrations](../guide/schema/migrations.md): `ferro migrate new` diffs the models' SchemaIR against the previous migration's schema snapshot with the same planner and renders the same DDL per target dialect; the [Alembic bridge](../guide/schema/alembic.md) writes that planner's decisions as an Alembic revision. **SchemaIR** is the canonical contract for cross-emitter DDL parity — runtime CREATE, auto-migrate, and Alembic `get_metadata()` all derive from the Python-compiled SchemaIR modelset and shared `ferro-ddl-lowering` tokens. Parity is enforced structurally (`test_cross_emitter_parity.py`, `test_db_type_cross_emitter_parity.py`, `test_migrate_plan.py`). The enriched JSON schema in the Rust registry remains the runtime source for query bind typing and hydration metadata. +Auto-migrate, [migrations](../guide/schema/migrations.md) (`ferro migrate new`) and the [Alembic bridge](../guide/schema/alembic.md) are three doors to the same planner. Each compares the models' SchemaIR with a database or a saved snapshot, and each renders the change through the same functions. Because they share one planner, they write the same statement for the same change, byte for byte, so switching from one to another never shows a phantom difference. [Schema management](../guide/schema/overview.md) compares the three. -## Async Architecture +## Async from end to end -Ferro is async end to end. Python's `await` hands off to Rust's tokio runtime; there are no thread pools wrapping a synchronous driver, and no sync API with async bolted on. +Ferro has no synchronous API and no thread pool wrapping a synchronous driver. Python's `await` hands off to Rust's tokio runtime, and SQLx talks to the database asynchronously: ```mermaid graph LR @@ -218,26 +197,26 @@ graph LR B -->|SQLx async I/O| C[(Database)] ``` -Consequences: +Three consequences follow: -- Database I/O never holds the GIL, so other coroutines keep running while queries are in flight. -- Concurrent queries from multiple tasks share the connection pool naturally when each task uses its own session (or explicit `session=` routing). -- [Sessions](../guide/connections.md) and [transactions](../guide/transactions.md) pin enclosed work to a scoped connection and identity map via context variables. +- Database I/O never holds Python's interpreter lock, so other coroutines keep running while queries are in flight. +- Concurrent tasks share the connection pool when each task opens its own session, or routes with an explicit `session=`. +- [Sessions](../guide/connections.md#sessions) and [transactions](../guide/transactions.md) pin the work inside them to a connection and an identity map through context variables. ## Trade-offs -What this design costs: +The design has costs: -- **Compiled wheel dependency.** Ferro ships a compiled extension module. Prebuilt wheels cover common platforms; anything else means building from source with a Rust toolchain. -- **Debugging crosses a language boundary.** A stack trace stops at the FFI. Ferro works to surface clear Python exceptions, but stepping a debugger *into* SQL generation or hydration is not possible the way it is with a pure-Python ORM. +- **A compiled wheel.** Ferro ships a compiled extension module. Prebuilt wheels cover the common platforms; anywhere else, installing builds from source with a Rust toolchain. +- **Debugging crosses a language boundary.** A Python traceback stops where the call enters Rust. Ferro raises Python exceptions with clear messages, but you cannot step a Python debugger into SQL generation or hydration. - **Limited runtime introspection.** You cannot monkeypatch the engine's internals or hook arbitrary points of the execution pipeline from Python. -- **Young ecosystem.** Ferro is pre-1.0 with a small community, fewer integrations, and fewer Stack Overflow answers than SQLAlchemy or Django ORM. +- **A young ecosystem.** Ferro is pre-1.0, with a small community, fewer integrations and fewer answered questions online than SQLAlchemy or the Django ORM. -## See Also +## See also -- [Identity Map](identity-map.md) — session-scoped instance caching -- [Query Typing](query-typing.md) — lambda predicates and the query DSL -- [Type Safety](type-safety.md) — the Pydantic integration in depth -- [Backends](backends.md) — SQLite and PostgreSQL specifics -- [Performance](performance.md) — where the Rust core pays off -- [Schema Management](../guide/schema/overview.md) — auto-migrate, migrations and the Alembic bridge +- [Identity map](identity-map.md): one row, one instance, within a session +- [Typed query predicates](query-typing.md): lambda predicates and how they are checked +- [Type safety](type-safety.md): the Pydantic integration +- [Database backends](backends.md): SQLite and PostgreSQL differences +- [Performance](performance.md): where the Rust core helps +- [Schema management](../guide/schema/overview.md): auto-migrate, migrations and the Alembic bridge diff --git a/docs/pages/concepts/backends.md b/docs/pages/concepts/backends.md index 16c8284f..db14fd7a 100644 --- a/docs/pages/concepts/backends.md +++ b/docs/pages/concepts/backends.md @@ -1,107 +1,105 @@ -# Backends +--- +description: "Where SQLite and PostgreSQL still differ under Ferro: column types, raw SQL and schema changes." +--- -Ferro supports SQLite and PostgreSQL through one Python API. Your models, queries, and transactions look identical on both; the Rust core selects the right SQLx driver, SQL dialect, and value conversion rules based on the connection URL. +# Database backends -## Supported Backends +Ferro runs the same models, queries and transactions on SQLite and PostgreSQL; this page explains where the two backends still differ: column types, raw SQL and schema changes. -The URL scheme passed to `connect()` decides the backend: +The examples on this page use this `User` model: -```python -from ferro import connect +=== "Assignment" -# SQLite — file-backed (rwc = read/write/create) or in-memory -await connect("sqlite:app.db?mode=rwc") -await connect("sqlite::memory:") - -# PostgreSQL — both scheme spellings work -await connect("postgresql://user:password@localhost:5432/app") -await connect("postgres://user:password@localhost:5432/app") -``` + ```python + --8<-- "docs/examples/concepts.py:models" + ``` -Hosted PostgreSQL providers such as **Supabase** work with a standard `postgresql://` connection string — Ferro's test matrix runs against Supabase-managed databases as well as local PostgreSQL. +=== "Annotated" -Unsupported schemes (e.g. `mysql://`) fail at `connect()` time with an error naming the supported schemes. Backend detection happens once, during connection; after that the engine carries its backend kind and typed connection pool, so no operation rediscovers the database from URL strings. + ```python + --8<-- "docs/examples/concepts_annotated.py:models" + ``` -!!! tip "Schema isolation for PostgreSQL tests" - Ferro supports a private `ferro_search_path` URL parameter (stripped before SQLx connects) that runs `SET search_path TO ` on every pooled connection. Combined with `auto_migrate=True`, this lets many test runs share one PostgreSQL database while each sees only its own schema. Names must be ASCII alphanumeric or `_`. +The URL passed to `connect()` picks the backend, and nothing after it changes: -## Type Mapping +```python +--8<-- "docs/examples/concepts.py:backends" +``` -Ferro maps Python field types to backend column types from your model annotations. SQLite uses type *affinity* (declared types are advisory), so the practical difference is mostly on the PostgreSQL side, where columns are strictly typed. +That block runs as written against an empty database, with `url` set to any of these: -| Python type | SQLite | PostgreSQL | Notes | -|---|---|---|---| -| `int` | `INTEGER` | `INTEGER`/`BIGINT` | Autoincrement PKs come from `last_insert_rowid()` on SQLite; PostgreSQL uses `RETURNING`. Values always materialize as Python `int`. | -| `str` | text affinity | `VARCHAR` | Use `varchar(n)` / `db_type` tokens to pin an explicit length or `TEXT`. | -| `float` | numeric affinity | double precision | | -| `bool` | stored as integer 0/1 | `BOOLEAN` | Hydrates back to Python `bool` on both. | -| `uuid.UUID` | stored as text | native `uuid` | PostgreSQL expressions get explicit `::uuid` casts where needed; values round-trip as `uuid.UUID`. | -| `Decimal` | flexible (text/numeric affinity) | `NUMERIC` | Read as text on the wire so Python reconstructs an exact `Decimal` — no float precision loss. | -| `dict` / `list` | stored as JSON text | `JSON` | PostgreSQL writes cast JSON strings to `json`; reads parse back into Python values. | -| `datetime` / `date` | stored as ISO text | `TIMESTAMPTZ` / `DATE` | Temporal values cross the bridge as ISO strings and reconstruct into `datetime`/`date`. `db_type` tokens select `timestamp` vs `timestamptz`. | -| `Enum` | stored as text | text, or native enum type | Schema metadata carries the enum type name; PostgreSQL applies enum casts where the column uses a native enum type. | -| `bytes` | `BLOB` | `BYTEA` | | +| Backend | URL | +| :--- | :--- | +| SQLite, file-backed (`rwc` creates the file if missing) | `sqlite:app.db?mode=rwc` | +| SQLite, in memory | `sqlite::memory:` | +| PostgreSQL | `postgresql://user:password@localhost:5432/app` | +| PostgreSQL, short scheme | `postgres://user:password@localhost:5432/app` | -The guiding rule: **the Python model is the contract**. Backends may store a value differently, but it must hydrate back into the annotated Python type exactly. +Hosted PostgreSQL services take the same `postgresql://` URL; [Connections › Supabase](../guide/connections.md#supabase) shows how to require TLS on one. -## Backend Differences +Ferro reads the backend from the URL once, in `connect()`. From then on the engine holds a SQLite pool or a PostgreSQL pool, two separate types, and no operation looks at the URL again. Any other scheme fails in `connect()`, naming the ones it accepts: -### Placeholders: `?` vs `$1` +```python +--8<-- "docs/examples/concepts.py:unsupported-scheme" +``` -The ORM handles parameter binding for you, but [raw SQL](../guide/queries.md) is passed through to the driver verbatim — placeholders are native to the backend, with no translation layer: +It prints: -| Backend | Placeholder syntax | Example | -|---|---|---| -| PostgreSQL | `$1, $2, ...` | `select * from users where id = $1` | -| SQLite | `?` (positional) | `select * from users where id = ?` | + +```text +InterfaceError: DB Connection failed for mysql://***: Unsupported database URL scheme 'mysql'. Supported schemes: sqlite, postgres, postgresql +``` -A mismatch surfaces as the database driver's own error. +The failed call registers no connection, so you can fix the URL and call `connect()` again. -### Type fidelity in raw SQL +## Type mapping -Raw SQL has no schema map, so Ferro does not auto-cast bind values. Rich Python types are marshalled to wire-close primitives (`UUID`, `datetime`, `Decimal` → text; `dict`/`list` → JSON text), and rows come back as plain dicts of primitives. On PostgreSQL you write casts yourself where the column type demands them: +Ferro derives each column's type from the field's annotation, and [Models and fields › Field types](../guide/models-and-fields.md#field-types) lists the type every annotation gets on each backend, generated from the code that decides it. The differences that matter come from how each database treats a declared type: -```python -from ferro import fetch_all +- **PostgreSQL types are strict.** A `uuid.UUID` field is a native `uuid` column, a `bool` is `bool`, a `datetime` is `timestamp with time zone`, and `dict`, `list` and nested-model fields are `jsonb` (`db_type="json"` gives plain `json`). A `StrEnum` becomes a native enum type. +- **SQLite types are advisory.** SQLite stores any value in any column, whatever its declared type, so a `bool` is stored as an integer, a `UUID` and a `datetime` as text, and a JSON field as JSON text. -rows = await fetch_all( - "select id, name from orders where id = $1::uuid and total > $2::numeric", - order_id, - minimum, -) -``` +Either way, a value comes back as the field's annotated Python type: a `bool` from SQLite's `0` or `1`, an exact `Decimal` with no float rounding, a `UUID`, a `datetime`, `date` or `time`, an enum member, or the `dict` or `list` you saved. The model is the contract; each backend stores the value its own way, and hydration turns it back into the annotated type. -SQLite's flexible affinity usually needs no casts. Either way: if you want typed rows (`UUID`, `datetime`, `Decimal` objects), use the ORM — raw SQL is an escape hatch. +An `int` field is `integer` on both backends. For a 64-bit column, declare `db_type="bigint"`; [Column type overrides with `db_type`](../guide/models-and-fields.md#column-type-overrides-with-db_type) lists every token. -### Migration capabilities +## Raw SQL -`connect(auto_migrate=True)` creates missing tables identically on both backends, and `migrate_updates=True` adds missing columns on both. Beyond that, capabilities diverge with what each database can do in place: +The ORM binds every value for you, with the casts the column needs. [Raw SQL](../guide/raw-sql.md) goes to the driver as you wrote it, so two differences show through: -- **PostgreSQL** supports in-place column **type changes** (`ALTER COLUMN ... TYPE ... USING` cast) and **nullability changes** (`SET`/`DROP NOT NULL`) when the live column disagrees with the model. -- **SQLite** cannot alter column types, nullability or constraints in place. Auto-migrate emits a `UserWarning` naming the drifted column and changes nothing; [migrations](../guide/schema/migrations.md#sqlite-table-rebuilds) generate the change as a reviewed table rebuild (the [Alembic bridge](../guide/schema/alembic.md#what-autogenerate-refuses) refuses it, naming `ferro migrate new`). In practice SQLite's type affinity makes declared-type drift mostly cosmetic. +- **Placeholders are the backend's own:** `?` on SQLite, `$1`, `$2`, ... on PostgreSQL. Ferro does not translate them, and a mismatch fails with the database's own error. See [Raw SQL › Placeholders](../guide/raw-sql.md#placeholders). +- **Bound values are not cast.** Raw SQL has no model to read types from, so a `UUID`, `datetime` or `Decimal` is sent as text, and rows come back as plain dicts. PostgreSQL rejects text compared with a `uuid` or `numeric` column until you write the cast (`$1::uuid`, `$2::numeric`); SQLite accepts it, so a raw query that works on SQLite can fail on PostgreSQL. See [Raw SQL › Bind types and casts](../guide/raw-sql.md#bind-types-and-casts). -`migrate_destructive=True` drops model-removed columns on both backends (dependency-aware: covering indexes are dropped first; primary-key or constraint-enforced columns fail with a clear error instead). After any schema change the pool is refreshed so no cached statement observes the pre-migration schema. +If you want typed values in and out, use the ORM. -## Troubleshooting +## Schema changes -### `Engine not initialized` +`connect(auto_migrate=True)` creates missing tables the same way on both backends, and `migrate_updates=True` adds missing columns and indexes on both. Beyond that, each backend changes what it can change in place: -You called a model or query method before `await connect(...)`. Importing models registers their schemas, but it does not connect to a database — call `connect()` during application startup. +- **PostgreSQL** changes a column's type (`ALTER COLUMN ... TYPE ... USING`), its nullability (`SET NOT NULL`, `DROP NOT NULL`) and Ferro's constraints in place. +- **SQLite** cannot change a column's type, nullability or constraints on an existing table; only a rebuild of the table can. Auto-migrate never rebuilds a table, so it raises a `UserWarning` naming the column and changes nothing. [`ferro migrate new`](../guide/schema/migrations.md#sqlite-table-rebuilds) writes the change as a reviewed table rebuild; the Alembic bridge refuses it and names `ferro migrate new`. -### Unsupported URL scheme +Because SQLite stores any value in any column, a SQLite column whose declared type differs from the model's still accepts every write; the warning is about the declared schema, not your data. [Auto-migrate › Applying column changes](../guide/schema/auto-migrate.md#applying-column-changes-with-migrate_updates) lists every change per backend. -Only `sqlite:`, `postgres://`, and `postgresql://` are accepted. Other databases (e.g. MySQL) are not currently supported. +## PostgreSQL schema isolation -### UUID or Decimal comparisons fail only on PostgreSQL +Add `ferro_search_path` to a PostgreSQL URL and every pooled connection runs `SET search_path` to that schema: -SQLite's affinity hides type mismatches that PostgreSQL enforces. In raw SQL, add explicit casts (`$1::uuid`, `$1::numeric`). Through the ORM this is handled for you — if you hit a case where it isn't, that's a bug worth [reporting](https://github.com/syn54x/ferro-orm/issues). +```python {test=run prelude=connections backend=postgres} +from ferro import connect -### Type or nullability drift warning on SQLite +await connect( + "postgresql://user:password@localhost:5432/app?ferro_search_path=test_42", + auto_migrate=True, +) +``` -`migrate_updates=True` detected that a live column's declared type or nullability disagrees with your model, and SQLite can't change it in place. Generate the change with `ferro migrate new`, which writes it as a [table rebuild](../guide/schema/migrations.md#sqlite-table-rebuilds), or ignore it if the drift is cosmetic. The Alembic bridge refuses SQLite table rebuilds. +Unqualified table names now resolve in `test_42`, so many test runs can share one database, each with its own tables. [Connections › Postgres schema isolation](../guide/connections.md#postgres-schema-isolation) covers the rules for the name. -## See Also +## See also -- [Architecture](architecture.md) — how the backend layer fits into the engine -- [Schema Management](../guide/schema/overview.md) — auto-migrate, migrations and the Alembic bridge on each backend -- [Queries](../guide/queries.md) — the query builder and raw SQL escape hatch +- [Connections](../guide/connections.md): URLs, sessions, pools and named connections +- [Models and fields › Field types](../guide/models-and-fields.md#field-types): the column type of every annotation +- [Raw SQL](../guide/raw-sql.md): placeholders and casts on each backend +- [Schema management](../guide/schema/overview.md): auto-migrate, migrations and the Alembic bridge on each backend +- [Architecture](architecture.md): where the backend layer sits in the engine diff --git a/docs/pages/concepts/identity-map.md b/docs/pages/concepts/identity-map.md index cbceb95b..c9e16922 100644 --- a/docs/pages/concepts/identity-map.md +++ b/docs/pages/concepts/identity-map.md @@ -1,212 +1,151 @@ -# Identity Map +--- +description: "Why a row fetched twice in one session is the same Python object, refreshed to the database's current values." +--- -Ferro keeps an identity map: within a single session, the same database row resolves to the same Python object. The map holds weak references only, so it never keeps an instance alive on its own, and it never serves a stale row — a re-fetch always reflects the database's current values. +# Identity map -## What It Is +Within a session, Ferro's identity map makes one database row one Python object: fetching a row you already hold returns that same instance, refreshed to the database's current values. -The identity map is a session-scoped table in the Rust engine, keyed by `(connection, model name, primary key)`. Each entry is a **weak reference** to a Python instance, not the instance itself. - -```mermaid -graph LR - Q1["User.get(1)"] --> IM[Identity Map] - Q2["User.where(...).first()"] --> IM - IM --> Same[Same Python instance] -``` - -Two consequences follow directly from "weak reference": - -- **The map never keeps an instance alive.** When your code drops the last reference to an instance, it is released like any other Python object — the map's entry becomes a tombstone that is cleaned up (either on the next lookup that hits it, or by a periodic sweep) rather than a leak. The next fetch for that row hydrates a brand-new instance. -- **Dedup only holds while you hold a reference.** `a is b` across two fetches is guaranteed only as long as something in your code still references the first instance. - -Each active session has its own map, so instances from one session are never returned by a fetch in another. There is no process-global map: **`Model.get()` and every other fetch always execute a query against the database** — the identity map decides whether that query's result is returned as a new instance or as the instance you already hold; it is a deduplication table, not a query cache. - -## The Guarantee: Refresh-on-Load - -Within a session, fetching a row you already hold returns the same object, updated in place to the database's current values. Unsaved local mutations are overwritten by a re-fetch — the database wins. +The examples on this page use this `User` model, with four users saved (ada, grace, alan and edsger): === "Assignment" ```python - from ferro import Field, Model, connect, engines, execute - - - class User(Model): - id: int | None = Field(default=None, primary_key=True) - username: str - email: str - - - await connect("sqlite::memory:", auto_migrate=True) - - async with engines.session(): - user = await User.create(username="alice", email="alice@example.com") - - # Something else — another request, a script, a raw SQL statement — - # changes this row without going through `user`. - await execute("UPDATE user SET email = ? WHERE id = ?", "new@example.com", user.id) - - refetched = await User.get(user.id) - assert refetched is user # same object - assert user.email == "new@example.com" # refreshed in place - - # An unsaved local edit does not survive a re-fetch: the database wins. - user.email = "unsaved edit" - again = await User.get(user.id) - assert again is user - assert user.email == "new@example.com" + --8<-- "docs/examples/concepts.py:models" ``` === "Annotated" ```python - from typing import Annotated + --8<-- "docs/examples/concepts_annotated.py:models" + ``` - from ferro import Field, Model, connect, engines, execute +Fetching ada twice in one session, with a change to her row in between, gives back one object with the new value: +```python +--8<-- "docs/examples/concepts.py:refresh-on-load" +``` - class User(Model): - id: Annotated[int | None, Field(default=None, primary_key=True)] - username: str - email: str +The second `User.get(1)` ran a query, found the instance the session already held and updated it in place. The unsaved edit was overwritten the same way: on a re-fetch, the database wins. +## Refresh on load - await connect("sqlite::memory:", auto_migrate=True) +Every fetch runs a query; the identity map decides only whether its row comes back as a new instance or as the one you hold. It is a table for removing duplicates, not a query cache, so a held instance always shows what the last fetch read. - async with engines.session(): - user = await User.create(username="alice", email="alice@example.com") +Ferro does not merge old and new values, because a merge policy is how stale-read bugs survive. If you have local changes you care about, `save()` them before the next fetch of that row, or do not hold a changed, unsaved instance across an `await` that could fetch it again. - # Something else — another request, a script, a raw SQL statement — - # changes this row without going through `user`. - await execute("UPDATE user SET email = ? WHERE id = ?", "new@example.com", user.id) +## One row, one instance - refetched = await User.get(user.id) - assert refetched is user # same object - assert user.email == "new@example.com" # refreshed in place +Without an identity map, two queries that return the same row would give you two separate copies, and a change to one would not show through the other. With it, every way of fetching the row returns the same object. This block runs inside a [session](../guide/connections.md#sessions): - # An unsaved local edit does not survive a re-fetch: the database wins. - user.email = "unsaved edit" - again = await User.get(user.id) - assert again is user - assert user.email == "new@example.com" - ``` +```python {test=run prelude=concepts} +by_key = await User.get(2) +by_filter = await User.where(lambda user: user.username == "grace").first() +assert by_filter is by_key -Merging old and new field values was considered and rejected: a merge policy is exactly how stale-read bugs survive. If you have local changes you care about, `save()` them before the next fetch touches that row — or don't hold onto a mutated-but-unsaved instance across an `await` that could re-fetch it. +by_key.email = "grace@navy.mil" +assert by_filter.email == "grace@navy.mil" +``` -## Why It Matters +The change made through `by_key` shows through `by_filter`, because they are one object. The guarantee holds within one session on one connection; it does not reach across processes or across sessions in the same process, since each session has its own map. -Without an identity map, two queries that return the same row give you two disconnected copies; an update to one is invisible through the other: +## Weak-reference map -```python -async with engines.session(): - created = await User.create(username="alice", email="alice@example.com") - fetched = await User.get(created.id) - filtered = await User.where(lambda t: t.username == "alice").first() - - # One row, one instance. - assert fetched is created - assert filtered is created - - # A change made through any reference is visible through all of them. - fetched.email = "new@example.com" - assert created.email == "new@example.com" -``` +The identity map lives in the Rust engine, one per session, keyed by connection, model name and primary key. Each entry is a weak reference to a Python instance, not the instance itself, and two things follow from that: + +- **The map never keeps an instance alive.** When your code drops the last reference to an instance, Python releases it like any other object. Its entry is cleaned up on the next lookup that finds it, or by a periodic sweep, and the next fetch of that row builds a new instance. +- **`a is b` across two fetches holds only while you hold the first instance.** Once it is released, there is nothing for the second fetch to return. + +Each session's map is its own, so a fetch in one session never returns an instance from another. -This holds within one session on one connection. It does not synchronize across processes or across sessions in the same process — each session has its own map, by design (see below). +## Sessionless operations -## No Session, No Identity +The session bounds the map's lifetime, so identity is a session feature. Ferro resolves the connection for every operation once, with three outcomes: -Identity is a session feature, because the session is what bounds the map's lifetime. Ferro resolves the connection for every operation exactly once, and that resolution has three outcomes: +- **A session is active**, from `async with engines.session():` or an explicit `session=`: the operation uses that session's identity map. +- **`using="name"` with no session**: the operation runs against that connection with no identity map. Every load returns a new instance, and nothing is kept, so nothing can go stale. +- **No session and no `using=`**: Ferro raises `RuntimeError` instead of guessing a connection. -- **A session is active** (ambient `async with engines.session():`, or an explicit `session=`) — full identity semantics: dedup, weak-value map, refresh-on-load, all scoped to that session. -- **`using="name"` alone, no session** — the operation runs normally against that connection, but with **no identity map at all**: every load returns a fresh instance, nothing is cached, and therefore nothing can go stale. -- **No session and no `using=`** — Ferro raises `RuntimeError("No database route for this operation...")` rather than guessing a connection. +These calls run with no session open: ```python -await connect("sqlite::memory:", auto_migrate=True) - -# `using=` alone: no session, so no identity map. Every load is a fresh -# instance — correct data, no `a is b` guarantee. -created = await User.using("default").create(username="bob", email="bob@example.com") -first = await User.using("default").get(created.id) -second = await User.using("default").get(created.id) -assert first is not second -assert first.email == second.email # same row, different objects - -# No session, no `using=`: nothing to route through. Raises RuntimeError. -await User.get(created.id) +--8<-- "docs/examples/concepts.py:sessionless" ``` -An explicit `using=` that names a *different* connection than the session you're ambiently inside of is also an error (`ValueError`) rather than a silent sessionless fallback — Ferro never routes an operation around the session you opened without telling you. +Both `using=` loads read grace's row, as two separate objects. The last call has no route and raises `RuntimeError: No database route for this operation. Open a session (...)`. -## What Gets Cached +A `using=` that names a different connection from the session you are inside raises `ValueError`, so an operation never runs around the session you opened without telling you. -Instances enter the session's identity map whenever Ferro hydrates or persists a full model, inside a session: +## What enters the map + +Inside a session, an instance enters the map whenever Ferro hydrates or saves a full model: - `Model.get(pk)` and `Model.get_or_none(pk)` -- `.first()` and `.all()` query results +- query results from `.first()` and `.all()` - `Model.create(...)` - `instance.save()` and `instance.refresh()` -What does **not** populate the map: +Three kinds of operation leave the map alone: -- Any of the above run outside a session via `using=` alone — sessionless operations never touch a map, by design (see above). -- `Model.bulk_create([...])` — bulk inserts return a row count, not instances, and deliberately skip the map for memory efficiency. Re-query if you need tracked instances afterward. -- Raw SQL (`fetch_all` / `fetch_one`) — raw rows are plain dicts and never touch the map. +- Any of the above with `using=` and no session, as [Sessionless operations](#sessionless-operations) describes. +- `Model.bulk_create([...])`, which returns a row count, not instances. Query the rows again if you need instances in the map. +- Raw SQL (`fetch_all`, `fetch_one`), whose rows are plain dicts. -## Eviction and Invalidation +## Eviction -Because refresh-on-load already keeps a held instance current, you rarely need to force anything by hand. Two situations still call for it: forcing a *different* instance while you hold the old one, and letting Ferro invalidate on your behalf. +Because a re-fetch already refreshes a held instance, you rarely need to manage the map by hand. Two operations let you do so. These blocks run inside a [session](../guide/connections.md#sessions). -**Refresh** reloads an instance you hold, in place — this is the same mechanism as an ordinary re-fetch, exposed as a method: +`refresh()` reloads an instance you hold, in place. It is the same mechanism as a re-fetch, as a method: -```python -async with engines.session(): - user = await User.create(username="bob", email="bob@example.com") - # ... something external updates the row ... - await user.refresh() +```python {test=run prelude=concepts} +from ferro import execute + +alan = await User.get(3) +await execute('UPDATE "user" SET age = 18 WHERE id = 3') +await alan.refresh() +assert alan.age == 18 ``` -**Evict** removes an entry from the session's map so the *next* fetch hydrates a fresh instance instead of the one you're currently holding — useful when you want to detach from an instance you still have a reference to, rather than wait for it to go out of scope: +The `UPDATE` went around `alan`, and `refresh()` read the new age into the same object. -```python -from ferro import evict_instance +`evict_instance()` removes one entry from the session's map, so the next fetch builds a new instance instead of returning the one you hold. Use it when you want to detach from an instance you still reference: -async with engines.session(): - user = await User.create(username="bob", email="bob@example.com") +```python {test=run prelude=concepts} +from ferro import evict_instance - evict_instance("User", str(user.id)) - fresh = await User.get(user.id) # a new instance, not `user` - assert fresh is not user +ada = await User.get(1) +evict_instance(User, str(ada.id)) +fresh = await User.get(1) +assert fresh is not ada ``` -`evict_instance` resolves the same session/`using=` scope as any other operation (pass `using=` or `session=` explicitly if you're not inside the ambient session you want to target). +`fresh` is a new object for the same row, and `ada` is now detached from the map. The `pk` argument is the primary key as a string. `evict_instance` resolves its route the way any operation does; pass `using=` or `session=` to target a map other than the current session's. -Ferro also evicts automatically, scoped to the session, so a batch job's cache never grows past what a memory-bounded weak-value map already keeps in check: +Ferro also evicts on its own, scoped to the session: -- **Rolling back a transaction** evicts that session's entire map — a rollback means every instance created or modified inside it may no longer reflect the database, so Ferro clears the slate rather than leave you to guess which ones. -- **Bulk `update()` / `delete()`** evict only that model's entries within the session — an unrelated model's cached instances are untouched. +- **A rolled-back transaction** empties the session's whole map. After a rollback, any instance created or changed inside it may no longer match the database, so Ferro starts over rather than guess which ones. +- **A bulk `update()` or `delete()`** evicts that model's entries in the session. Other models' instances stay. -Because eviction is scoped to `(connection, session)` or `(connection, model)`, one session's rollback or bulk write never disturbs another session's cache. +One session's rollback or bulk write never touches another session's map. -One case is intentionally *not* covered: running `ferro.migrate()` while a session is open does not evict that session's identity map. Schema changes and live sessions are your own responsibility to sequence — this is unchanged from before weak-valued maps and refresh-on-load existed. +!!! note "`ferro.migrate()` does not evict" + Running `ferro.migrate()` while a session is open leaves that session's identity map as it is. Sequence schema changes and open sessions yourself. -## Opting Out +## Turning the map off -The identity map is on by default and can be disabled per connection: +The identity map is on by default. Pass `identity_map=False` to `connect()` to turn it off for every session on that connection: ```python -from ferro import connect - -await connect("sqlite::memory:", auto_migrate=True, identity_map=False) +--8<-- "docs/examples/concepts.py:opt-out" ``` -With `identity_map=False`, sessions opened on this connection never keep an identity map: every load returns a fresh instance, and there is no `a is b` guarantee even inside a session. Since the map is already weak-valued and memory-bounded, this option is about giving up dedup semantics you don't need — not about memory pressure — and suits read-heavy, ETL-style workloads where instance identity carries no meaning. +Inside the session, two loads of the same row returned two objects. The map is weak-valued and session-scoped, so turning it off saves the deduplication work, not memory; it suits read-heavy, ETL-style jobs where instance identity means nothing. -## Identity Map in Tests +## Identity map in tests -Because a session's map lives for the session's lifetime, tests that share a process should still reset connection state between cases so one test's connections and sessions never leak into the next. `reset_engine()` tears down all connections and any sessions still open on them: +A session's map lives as long as the session, so tests that share a process reset connection state between cases. `reset_engine()` closes every connection and any sessions still open on them: -```python +```python {test=compile reason="a pytest fixture; pytest runs it, not the docs harness"} import pytest from ferro import connect, engines, reset_engine @@ -220,8 +159,11 @@ async def db(): reset_engine() ``` -## See Also +Each test that takes `db` starts with an empty database and a session of its own. [Testing](../howto/testing.md) covers fixtures in full. + +## See also -- [Architecture](architecture.md) — where the identity map lives in the engine -- [Performance](performance.md) — memory implications of caching -- [Queries](../guide/queries.md) — query behavior in depth +- [Architecture](architecture.md): where the identity map sits in the engine +- [Performance](performance.md): what the map costs and saves +- [Connections › Sessions](../guide/connections.md#sessions): opening and nesting sessions +- [`evict_instance`][ferro.evict_instance]: the reference entry diff --git a/docs/pages/concepts/performance.md b/docs/pages/concepts/performance.md index 5d62befe..79c278a7 100644 --- a/docs/pages/concepts/performance.md +++ b/docs/pages/concepts/performance.md @@ -1,108 +1,162 @@ +--- +description: "Which workloads Ferro's Rust core speeds up, which it does not, and how to get the most from it." +--- + # Performance -Ferro moves SQL generation, parameter binding, and row hydration out of Python and into compiled Rust, with the GIL released during database I/O. That helps some workloads and not others. +Ferro builds SQL, binds values and turns rows into instances in compiled Rust, and waits on the database without holding Python's interpreter lock; this page explains which workloads that speeds up, which it does not, and how to get the most from it. -## Where the Rust Core Pays Off +The examples on this page use this `User` model, with four users saved (ada, grace, alan and edsger): -The Rust core helps most where a traditional ORM spends significant CPU time in Python: +=== "Assignment" -**Bulk inserts.** `bulk_create` serializes and binds an entire batch in Rust and writes it in as few statements as the backend's bind-parameter limit allows. The per-row Python overhead — building parameter lists, driver round-trips, object bookkeeping — largely disappears: + ```python + --8<-- "docs/examples/concepts.py:models" + ``` -```python -users = [ - User(username=f"user_{i}", email=f"user{i}@example.com") - for i in range(10_000) -] -await User.bulk_create(users) +=== "Annotated" + + ```python + --8<-- "docs/examples/concepts_annotated.py:models" + ``` + +A batch insert is where the Rust core does the most work for you: + +```python {test=run prelude=concepts} +from ferro import engines + +async with engines.session(): + inserted = await User.bulk_create( + [ + User(username=f"user_{i}", email=f"user{i}@example.com", age=20 + i % 50) + for i in range(10_000) + ] + ) + assert inserted == 10_000 ``` -**Hydrating large result sets.** Converting thousands of database rows into model instances is CPU work. Ferro decodes rows and assembles instances in Rust, so queries returning many rows spend far less time in pure-Python parsing loops than a traditional ORM does. +Ten thousand rows went in with one call. Python dumped each instance to plain values once, and Rust bound the whole batch and wrote it as multi-row `INSERT` statements, as few as the backend's limit on bound parameters allows, all or nothing. + +## Where Rust helps + +The Rust core helps where a pure-Python ORM spends its CPU time in Python: -**Concurrent workloads.** Because the GIL is released while queries are in flight, other coroutines keep running. Many in-flight queries don't serialize behind Python's interpreter lock. +- **Bulk inserts.** `bulk_create` builds the statements and binds every value in Rust, so the per-row Python work is building, validating and dumping each instance. +- **Large result sets.** Decoding rows and filling instances happens in Rust, not in a Python loop over the driver's rows. +- **Concurrent work.** The database wait runs on Rust's tokio runtime without Python's interpreter lock, so other coroutines keep running while queries are in flight. [Architecture › Async from end to end](architecture.md#async-from-end-to-end) explains how. -## Where It Doesn't +## Where Rust does not help -Be skeptical of any ORM benchmark — including ours — that ignores these: +Be skeptical of any ORM benchmark, ours included, that ignores these: -**Network and disk dominate small operations.** A single-row `get()` spends most of its wall-clock time waiting for the database. Rust can't make the network faster; for point lookups, Ferro performs in the same neighborhood as any async ORM. +- **The network and the disk.** A single-row `get()` spends most of its time waiting on the database, and Rust cannot make that wait shorter. +- **Slow queries.** A missing index or a full table scan costs the same whichever library sends the SQL. Ferro does not change query plans. +- **Application logic.** If an endpoint spends its time in business logic, template rendering or calls to other services, a faster ORM does not change it. Profile first. -**Slow queries stay slow.** A missing index or a full-table scan costs the same regardless of which library sends the SQL. Ferro doesn't fix query plans. +## Tuning checklist -**Application logic bottlenecks.** If your endpoint spends its time in business logic, template rendering, or external API calls, swapping the ORM moves nothing. Profile first. +Each block in this section runs inside a [session](../guide/connections.md#sessions). -## Getting the Most Out of Ferro +### `bulk_create` instead of a `create` loop -**Use `bulk_create` instead of create loops.** One statement beats N statements: +A loop of `create()` calls sends one `INSERT` per row, each its own round trip; `bulk_create` sends one multi-row `INSERT` per chunk: + +```python {test=run prelude=concepts} +new_users = [ + User(username=f"intern_{i}", email=f"intern{i}@example.com", age=19) + for i in range(1_000) +] -```python -# Slow: N round-trips -for i in range(1000): - await User.create(username=f"user_{i}") +# 1,000 statements: +# for user in new_users: +# await User.create(**user.model_dump(exclude={"id"})) -# Fast: one bulk statement -await User.bulk_create([User(username=f"user_{i}") for i in range(1000)]) +inserted = await User.bulk_create(new_users) +assert inserted == 1_000 ``` -Batch size is unbounded — Ferro splits large batches under the backend's bind-parameter limit automatically and keeps the whole call all-or-nothing — so size batches by what your application can hold in memory, not by statement limits. Note that `bulk_create` skips the [identity map](identity-map.md) by design. +The 1,000 interns went in through one call. Batch size has no limit of its own: Ferro splits a batch under the backend's parameter limit and keeps the whole call all or nothing, so size batches by what your application can hold in memory. `bulk_create` returns a count, not instances, and skips the [identity map](identity-map.md#what-enters-the-map). -**Use batch update/delete instead of instance loops.** Push the work into one SQL statement: +### Bulk `update()` and `delete()` instead of instance loops -```python -# Slow: fetch everything, save row by row -users = await User.where(lambda t: t.active == False).all() -for user in users: - user.status = "archived" - await user.save() +Fetching rows to change and save them one by one sends a query plus one `UPDATE` per row. A query's `update()` sends one statement: -# Fast: one UPDATE -await User.where(lambda t: t.active == False).update(status="archived") +```python {test=run prelude=concepts} +changed = await User.where(lambda user: user.active == False).update( + active=True +) +assert changed == 1 +assert await User.where(lambda user: user.active == False).count() == 0 ``` -**Wrap write bursts in a transaction.** Each standalone write commits individually; a transaction amortizes the commit cost across the batch and pins all work to one connection: +One `UPDATE` reactivated edsger, the one inactive user, and returned the number of rows it changed. [Mutations](../guide/mutations.md) covers bulk updates and deletes. -```python +### A transaction around a burst of writes + +Outside a transaction, each write commits on its own. A transaction pays for one commit across the batch and pins the work to one connection: + +```python {test=run prelude=concepts} from ferro import transaction async with transaction(): - order = await Order.create(total=total) - await OrderLine.bulk_create(lines) + ada = await User.get(1) + ada.age += 1 + await ada.save() + await User.create(username="barbara", email="barbara@example.com", age=33) + +assert (await User.get(1)).age == 37 ``` -Keep transactions short — don't hold one open across external API calls. +Both writes committed together. Keep transactions short, and do not hold one open across a call to another service. -**Prefer `.exists()` over `.count()` for presence checks**, and add `index=True` to fields you filter on frequently: +### Indexes for frequent filters -```python -if await User.where(lambda t: t.email == email).exists(): - raise ValueError("Email taken") +A filter on a column with no index reads the whole table. Add `index=True` to a field you filter on often; `unique=True`, as on `username`, creates a unique index too: + +```python {test=run prelude=concepts} +taken = await User.where(lambda user: user.username == "grace").exists() +assert taken ``` -**Know your identity map effects.** Repeated fetches of the same row return the cached instance rather than re-hydrating, which is a win for hot rows. The flip side: every hydrated instance stays cached for the connection's lifetime, so long-running jobs sweeping huge tables should paginate and evict as they go, or connect with `identity_map=False`. See [Identity Map](identity-map.md). +SQLite and PostgreSQL can answer the lookup on `username` from the `uq_user_username` index. `exists()` runs a `COUNT`, so it costs what `count()` costs; use whichever reads better. [Models and fields › Uniques and indexes](../guide/models-and-fields.md#uniques-and-indexes) covers both options. + +### The identity map -**Watch for N+1 relationship access.** Awaiting `post.author` in a loop issues one query per post. Eager loading (`prefetch_related`-style) is [on the roadmap](../roadmap.md); until then, restructure hot paths to filter on the parent side or batch by IDs with `in_()`. +The [identity map](identity-map.md) holds weak references and lasts as long as the session, so it does not grow a long job's memory; once your code drops an instance, its entry goes too. It saves object identity, not queries: every fetch still runs its query. For ETL-style reads where instance identity means nothing, `connect(..., identity_map=False)` skips the deduplication work. -## Benchmark It Yourself +### Relations in one query with `include()` -We deliberately don't publish "Nx faster" tables here: results vary enormously with database engine, hardware, network latency, batch size, and row shape. If performance matters to your decision, measure your own workload: +Reading `post.author` on each post of a list runs one query per post. `Post.where(...).include(lambda post: post.author)` loads every author in the same statement as the posts; [Queries › Populating relations](../guide/queries.md#populating-relations-with-include) shows it. `BackRef` and many-to-many collections are not populated by `include()` yet ([roadmap](../roadmap.md#reverse-and-many-to-many-population)); batch them by id with `in_()`. -```python +## Benchmarking + +Ferro publishes no "N times faster" table: results depend on the database, hardware, network, batch size and row shape. If performance decides your choice, time your own workload. This block runs inside a [session](../guide/connections.md#sessions): + +```python {test=run prelude=concepts} import time -rows = [User(username=f"user_{i}") for i in range(5000)] +rows = [ + User(username=f"bench_{i}", email=f"bench{i}@example.com", age=30) + for i in range(5_000) +] start = time.perf_counter() await User.bulk_create(rows) print(f"bulk_create: {time.perf_counter() - start:.3f}s") start = time.perf_counter() -fetched = await User.all() +users = await User.all() print(f"all() with hydration: {time.perf_counter() - start:.3f}s") +assert len(users) == 5_004 ``` -For a fair comparison against another ORM: use the same database, the same schema, warmed connections, realistic batch sizes, and run each measurement several times. Compare the operations your application actually performs — not just the ones that flatter either library. +It prints two lines, `bulk_create: s` and `all() with hydration: s`, for the 5,000 new rows and the 5,004 rows read back. The numbers are your machine's; run the same block against your own database and with your own models before you compare. + +For a fair comparison with another ORM, use the same database and schema, warm the connections, use realistic batch sizes and run each measurement several times. Compare the operations your application performs, not the ones that flatter either library. -## See Also +## See also -- [Architecture](architecture.md) — why the engine is shaped this way -- [Identity Map](identity-map.md) — caching behavior and memory -- [Queries](../guide/queries.md) — batch update and delete +- [Architecture](architecture.md): why the engine is shaped this way +- [Identity map](identity-map.md): what the map keeps and for how long +- [Mutations](../guide/mutations.md): `bulk_create`, bulk updates and deletes +- [Queries › Populating relations](../guide/queries.md#populating-relations-with-include): `include()` diff --git a/docs/pages/concepts/query-typing.md b/docs/pages/concepts/query-typing.md index fe46379e..2b3d5832 100644 --- a/docs/pages/concepts/query-typing.md +++ b/docs/pages/concepts/query-typing.md @@ -1,153 +1,152 @@ -# Typed Query Predicates +--- +description: "How Ferro's lambda predicates are checked, by your type checker and when you build a query, and where checking stops." +--- -`Model.where`, `Query.where`, and `Relation.where` accept a single predicate -shape: a lambda that receives a `QueryProxy` and returns a `QueryNode`. +# Typed query predicates -```python -rows = await User.where(lambda user: user.archived == False).all() -rows = await User.where( - lambda user: (user.role == "admin") & (user.active == True) +This page explains how Ferro's lambda predicates are checked: what your type checker sees, what Ferro checks when you build a query, and where checking stops. + +`Model.where()`, `Query.where()` and a relation's `where()` take one kind of argument, a lambda whose parameter stands for one row of the model: + +```python {test=run prelude=queries} +uncleared_debits = await Transaction.where( + lambda transaction: (transaction.amount < 0) + & (transaction.cleared_on == None) ).all() +assert {transaction.memo for transaction in uncleared_debits} == { + "Train to Lyon", + "Dinner in Lyon", + "Weekly shop", +} ``` -## How It Works +Each comparison in the lambda returns a condition, and `where()` turns the conditions into the query's `WHERE` clause. The examples on this page use the ledger from the [Queries guide](../guide/queries.md#the-ledger) and run inside a [session](../guide/connections.md#sessions). -The lambda receives a fresh `QueryProxy` for the model being queried. -Attribute access on the proxy is validated against the model's declared -columns (plus shadow `{fk}_id` columns) and returns a `FieldProxy`, so -`user.archived == False` builds a `QueryNode`. A misspelled column — -`user.archievd` — raises `AttributeError` naming the closest valid match and -listing every queryable column, at build time, before any query reaches the -database. +## Proxy validation -Name the lambda parameter after the model in lowercase singular (`user` for -`User`, `post` for `Post`) so predicates read like English. The full operator -surface is available: `==`, `!=`, `<`, `<=`, `>`, `>=`, `.like()`, `.in_()`, -`&`, `|`, `== None`, and shadow FK columns (`user.author_id`). +The lambda never sees a `Transaction`. Ferro calls it with a [`QueryProxy`][ferro.query.QueryProxy] for the model, whose attributes are the model's columns, including the `_id` column each foreign key adds. Reading an attribute returns a `FieldProxy`, and comparing a `FieldProxy` returns a condition, a `QueryNode`. -That gives you two concrete guarantees today: +The proxy checks every name when the lambda runs, which is when you build the query. A misspelled column raises `AttributeError` with the closest match and the full list, before anything reaches the database: + +```python {test=run prelude=queries} +try: + Transaction.where(lambda transaction: transaction.cleard_on == None) +except AttributeError as error: + assert "Did you mean 'cleared_on'?" in str(error) +else: + raise AssertionError("a misspelled column must raise AttributeError") +``` -- **A valid predicate type-checks as `QueryNode`.** `lambda user: user.age >= 18` passes `ty check` / Pyright because `>=` on a `FieldProxy` is typed to return `QueryNode`, which is exactly what `where()` expects. -- **A junk predicate fails the checker.** `lambda user: True` — a callable that doesn't return a `QueryNode` at all — is a type error, not a silent no-op query, because `where()`'s parameter type is `Predicate = Callable[[QueryProxy[TModel]], QueryNode]`. +The query is never built, so no SQL runs. -What isn't checked yet is the *right-hand side* of a comparison: the proxy attribute type is `FieldProxy[Any]`, so `user.age >= "eighteen"` type-checks even though it would fail at runtime. Closing that gap needs per-field static types on the proxy — TypeScript-style mapped types, proposed for Python in [PEP 827](https://peps.python.org/pep-0827/) ("Type Manipulation", draft status, targeting Python 3.16). When type checkers support it, `QueryProxy` attribute typing upgrades from `FieldProxy[Any]` to each field's real declared type — with zero runtime change — and `user.age >= "eighteen"` starts failing the checker too. +## What the type checker sees -`Relation.where` (used on `BackRef` collections) accepts the same shape: +`where()` is typed to take a [`Predicate`][ferro.query.Predicate]: `Callable[[QueryProxy[TModel]], QueryNode]`. That gives two guarantees: -```python -published = await author.posts.where(lambda post: post.published == True).all() +- **A valid predicate type-checks.** `lambda transaction: transaction.amount < 0` passes `ty check` and Pyright, because `<` on a `FieldProxy` is typed to return `QueryNode`. +- **A predicate that returns something else fails the checker.** `lambda transaction: True` returns a `bool`, not a `QueryNode`, so it is a type error rather than a query that matches every row. + +A relation's `where()`, such as `account.transactions.where(...)`, takes the same `Predicate`: + +```python {test=run prelude=queries} +checking = await Account.get(1) +uncleared = await checking.transactions.where( + lambda transaction: transaction.cleared_on == None +).all() +assert [transaction.memo for transaction in uncleared] == ["Weekly shop"] ``` -## Traversing Relations +Weekly shop is the one uncleared transaction on Checking. -Reaching through a foreign key inside a predicate — `lambda transaction: transaction.account.ledger_id == 1` — type-checks as a predicate for the same reason a plain comparison does: attribute access on the proxy keeps returning a chainable proxy, and the terminal comparison returns `QueryNode`. The checker sees a valid `Predicate[Transaction]`; the [Queries guide](../guide/queries.md#querying-across-relationships) covers what the traversal *means* (one INNER join per relation path). +## Traversing relations -The static gate checks the **shape** of the predicate, not the **names** in it: +A predicate that follows a foreign key, such as `transaction.account.owner.name == "Ada Lovelace"`, type-checks for the same reason: each attribute returns another proxy, and the final comparison returns `QueryNode`. The [Queries guide](../guide/queries.md#querying-across-relationships) covers what a traversal does to the query. -- **A traversal that ends in a comparison type-checks.** `transaction.account.ledger_id == 1` is a `QueryNode`, so it satisfies `where()`. -- **A bare relation as a predicate fails the checker.** `lambda transaction: transaction.account` returns a proxy, not a `QueryNode`, so it is a type error — the same failure mode as `lambda user: True`. -- **Name typos are caught at build time, not by the checker.** A misspelled hop (`transaction.accont`, `account.emial`) is `FieldProxy[Any]` to the type checker, so it type-checks; at build time the proxy validates every hop against the real model and raises `AttributeError` with a did-you-mean naming that hop's model. The static gate guarantees shape; the runtime proxy guarantees names. +The checker checks the shape of the predicate, and the proxy checks the names in it: -## Projected Queries +- **A traversal that ends in a comparison type-checks**, because it returns a `QueryNode`. +- **A bare relation fails the checker.** `lambda transaction: transaction.account` returns a proxy, not a `QueryNode`, the same failure as `lambda transaction: True`. +- **A misspelled step type-checks, and fails when you build the query.** To the checker, `transaction.accont` is a `FieldProxy` like any other attribute; at build time the proxy checks each step against its own model and raises `AttributeError` naming that model. -Partial selects extend the same shape-not-names promise through projection. -`select(...)` with a selector flips the query's static type from -`Query[Model]` to `ProjectedQuery[Model]`, so the terminals change shape: -`.all()` checks as `Rows[Row]` and `.first()` as `Row | None` — never -`list[Model]`: +## Projected queries -```python -rows = await Transaction.select(lambda t: (t.id, t.amount)).all() # Rows[Row] -row = await Transaction.select(lambda t: t.amount).first() # Row | None -full = await Transaction.select().all() # list[Transaction] — bare form unchanged +`select()` with a selector changes the query's type from `Query[Transaction]` to `ProjectedQuery[Transaction]`, so the terminals change type with it: `all()` checks as [`Rows[Row]`][ferro.query.Rows] and `first()` as [`Row`][ferro.query.Row]` | None`. A bare `select()` keeps `Query[Transaction]`: + +```python {test=run prelude=queries} +from ferro import Row, Rows + +records = await Transaction.select( + lambda transaction: (transaction.id, transaction.amount) +).all() # Rows[Row] +first = await Transaction.select(lambda transaction: transaction.memo).first() # Row | None +everything = await Transaction.select().all() # list[Transaction] + +assert isinstance(records, Rows) +assert isinstance(first, Row) +assert isinstance(everything[0], Transaction) ``` -Every selector form keeps that shape: the tuple, the single field, traversal -at any depth (`t.account.owner.email` chains as `FieldProxy[Any]`), and the -dict form, whose values may also be **aggregate expressions**: +Each result has the type its annotation claims. + +A selected query returns records because a Ferro model instance always holds a whole row. If `select()` returned a `Transaction` with two of its seven columns loaded, every `save()`, every refresh and every function that takes a `Transaction` would have to know which query produced its argument, and reading an unloaded field would fail far from that query. A `Row` says what it is: the columns you asked for and nothing else. The query carries its result shape with it, and records are built on the same no-validation path as model instances, so selecting fewer columns is never slower per row than selecting all of them. -```python -row = await Transaction.select( - lambda t: {"acct": t.account_id, "total": t.amount.sum()} -).first() # Row | None +Every selector form keeps that type: a tuple, a single field, a traversal at any depth, and a dict, whose values may also be aggregates: + +```python {test=run prelude=queries} +totals = await Transaction.select( + lambda transaction: { + "account_id": transaction.account_id, + "total": transaction.amount.sum(), + } +).all() # Rows[Row] +assert len(totals) == 4 ``` -An aggregate expression types **opaquely**: `t.amount.sum()` is an -`AggregateExpr` — deliberately not a value, not a `QueryNode`, not a column. -That is what makes the misuse gates work: an aggregate compared inside -`where()` raises at build time (post-aggregation filtering is `having()`), -and an `AggregateExpr` is only ever valid as a dict-selector value or an -`order_by()` lambda result. - -The gate divides the work exactly like predicates do: - -- **Shape is checked statically.** Passing a `Row` where a model instance is - expected fails the checker (`list[Transaction] = await - Transaction.select(lambda t: (t.id,)).all()` is an `invalid-assignment`), - and mutating a projected query (`.update()` / `.delete()`) is a static - error at the call site as well as a build-time `ValueError`. -- **Names are checked at build time.** A selected column is `FieldProxy[Any]` - to the checker; a misspelled column raises `AttributeError` with a - did-you-mean when the query is built, before any round-trip — the same - runtime validation as `where()` and `order_by()`. - -The promise is **shape, not names**: the checker knows a projected query -yields `Rows[Row]`, but not that this particular `Row` has a `.total` of type -`int | None` — record fields type as `Any` on access. Inferring per-field -record types from the selector (so `row.total` checks as what the aggregate -contract says it is) is the record-field inference lane, tracked in -[#290](https://github.com/syn54x/ferro-orm/issues/290) — another place -typing can sharpen later with zero runtime change. - -## Included Queries - -`include()` deliberately does **not** flip the query's type the way `select()` -does. An included query stays `Query[Model]`-shaped: `.all()` checks as -`list[Transaction]`, `.first()` as `Transaction | None`, and populated access -(`txns[0].account.label`) type-checks through the field's ordinary declared -annotation. There is no `Loaded[Transaction]`. - -That is a decision, not a gap, for two reasons: - -- **A per-query "loaded" type is unsound under the identity map.** In a - session, the instance an included query returns is the *same object* a - plain query returns — populations attach to shared instances and accumulate - across queries. A static brand that says "this object's `account` is - populated" asserts something the runtime cannot pin to a type: the next - refresh may drop the population (see the - [refresh rule](../guide/queries.md#refreshes-drop-populations-that-stopped-being-true)), - and a plain query can hand you the already-populated object. The honest - static type of every instance is the model itself. -- **The declared annotation already claims the instance.** The field says - `account: Account`; a distinct loaded type would need to *transform* - per-field types on the unloaded side instead (`account: Awaitable[Account]` - → `Account`), which requires TypeScript-style mapped types — - [PEP 827](https://peps.python.org/pep-0827/) (draft, targeting - Python 3.16). Population is what makes the declared annotation *true* at - runtime, exactly where the user opted in. - -When PEP 827 lands, typing can sharpen with zero runtime change — precisely -because the runtime type stayed single. Meanwhile the statically catchable -misuses do fail the gate: `include()` on a projected query is an error at the -call site (the `self: Never` pin), and a string selector fails the callable -parameter type. - -## What This Doesn't Change - -- Your model annotations. `archived: bool = False` stays exactly as it is. -- Pydantic schema generation, JSON schema output, or model validation. -- The Rust FFI bridge architecture (predicates serialize through QueryIR envelopes). +The four accounts each get one record with their total. + +An aggregate such as `transaction.amount.sum()` is an [`AggregateExpr`][ferro.query.AggregateExpr]: not a value, not a `QueryNode` and not a column. That is what lets Ferro refuse its misuse: comparing an aggregate inside `where()` raises when you build the query, and an `AggregateExpr` is valid only as a dict value in a selector or as the result of an `order_by()` lambda. + +The work divides as it does for predicates: + +- **The checker checks the shape.** Assigning a `Rows[Row]` to a variable typed `list[Transaction]` is an `invalid-assignment` error, and calling `update()` or `delete()` on a selected query is a type error at the call as well as a `ValueError` when it runs. +- **The proxy checks the names.** A selected column is a `FieldProxy` to the checker; a misspelled one raises `AttributeError` when you build the query. + +The checker knows a selected query returns `Rows[Row]`, but not which fields a given `Row` has: `row.total` types as `Any`. Typing each record's fields from its selector needs the type features described under [Where checking stops](#where-checking-stops). + +## Included queries + +`include()` does not change the query's type. An included query is still a `Query[Transaction]`: `all()` checks as `list[Transaction]`, `first()` as `Transaction | None`, and a populated relation (`transactions[0].account.label`) type-checks through the field's own annotation. There is no `Loaded[Transaction]` type. + +`include()` keeps the model type for two reasons: + +- **A "loaded" type would be wrong under the identity map.** In a session, an included query returns the same objects a plain query returns: populations attach to shared instances and accumulate across queries. A type that says "this object's `account` is populated" claims something the next refresh may undo (see [Population after a refresh](../guide/queries.md#population-after-a-refresh)), and a plain query can hand you an object an earlier query populated. The one type that is always true of an instance is its model. +- **The annotation already names the instance.** The field says `account: Account`. A separate loaded type would have to rewrite each field's type on the unloaded side (`account: Awaitable[Account]` to `Account`), which needs type features Python does not have yet; see [Where checking stops](#where-checking-stops). Populating a relation is what makes the declared annotation true, where you asked for it. + +The misuses a checker can catch do fail it: `include()` on a selected query is a type error at the call, and a string selector fails the lambda parameter type. + +## Where checking stops + +The checker does not check the right-hand side of a comparison. A proxy attribute is a `FieldProxy[Any]`, so `transaction.amount < "a lot"` type-checks and fails only when it runs. The same gap leaves record fields typed `Any` and rules out a "loaded" model type. + +Closing it needs per-field types on the proxy: TypeScript-style mapped types, which [PEP 827](https://peps.python.org/pep-0827/) ("Type Manipulation", a draft) proposes for Python. Once type checkers support them, `QueryProxy` attributes can carry each field's declared type, record fields can carry their selected types, and `transaction.amount < "a lot"` will fail the checker. None of that needs a change at runtime, because the runtime already has one type per result. + +## Unchanged behavior + +Lambda predicates change nothing outside the query: + +- Your model annotations stay as they are: `cleared_on: date | None = None` is still a plain Pydantic field. +- Pydantic's schema generation, JSON schema output and validation are unchanged. +- Predicates reach the Rust core through the same query envelopes as every other query. ## Reference -- `ferro.query.QueryProxy` — validating attribute proxy passed to lambda predicates. -- `ferro.query.Predicate` — `Callable[[QueryProxy[TModel]], QueryNode]`, the type of any lambda predicate. -- `ferro.query.FieldProxy` — generic over the column's Python type (`FieldProxy[T]`); the mechanism a `QueryProxy` attribute access returns. -- `ferro.query.RowSelector` — the type of a `select()` lambda selector: a callable returning one field, a tuple of fields, or a dict of output names to fields/aggregates. -- `ferro.query.AggregateExpr` — the opaque type of `t.amount.sum()` and friends; valid only as a dict-selector value or an `order_by()` lambda result. -- `ferro.query.Row` / `ferro.query.Rows` — projected records and their list-like container, the result shape of a projected query. +- [`QueryProxy`][ferro.query.QueryProxy]: the object a predicate's parameter receives +- [`Predicate`][ferro.query.Predicate]: the type of a lambda predicate +- [`RowSelector`][ferro.query.RowSelector]: the type of a `select()` lambda +- [`AggregateExpr`][ferro.query.AggregateExpr]: the type of `transaction.amount.sum()` and the other aggregates +- [`Row`][ferro.query.Row] and [`Rows`][ferro.query.Rows]: the records a selected query returns -## See Also +## See also -- [Queries Guide](../guide/queries.md) -- [Type Safety](type-safety.md) +- [Queries guide](../guide/queries.md) +- [Type safety](type-safety.md) diff --git a/docs/pages/concepts/row-level-security.md b/docs/pages/concepts/row-level-security.md new file mode 100644 index 00000000..e30d1ad5 --- /dev/null +++ b/docs/pages/concepts/row-level-security.md @@ -0,0 +1,139 @@ +--- +description: "How Postgres row-level security policies read session settings, and how Ferro delivers those settings on each connection." +status: new +--- + +# Row-level security and session settings + +Postgres row-level security filters rows inside the database, using policies that read a setting on the connection; Ferro declares the policies on the model and sends the setting on every connection a session uses. + +A session opened with a tenant's id reads only that tenant's invoices: + +```python +--8<-- "docs/examples/row_level_security.py:scoped" +``` + +Two things made that happen. The `invoice` table carries a policy that compares each row's `tenant_id` with the connection's `app.tenant_id` setting, and Ferro put `app.tenant_id` on the connection before the query ran. Outside a `transaction()` block, the query reached Postgres as: + +```sql +BEGIN; +SELECT set_config($1, $2, true); -- 'app.tenant_id', 'a1a1a1a1-0000-4000-8000-000000000001' +-- the query itself +COMMIT; +``` + +The value lives until `COMMIT`, so the connection goes back to the pool with no tenant on it. This page explains each half: what the policy does with the setting, and how and when Ferro delivers it. [Scope rows by tenant](../guide/row-level-security.md) is the step-by-step setup, and the [reference](../reference/row-level-security.md) lists every option. + +## Policy expressions + +The `RowPolicy(column="tenant_id", setting="app.tenant_id")` shorthand renders this expression for both `USING` (rows a query sees) and `WITH CHECK` (rows a write may leave behind): + +```sql +"tenant_id" = NULLIF(current_setting('app.tenant_id', true), '')::uuid +``` + +`current_setting(key, true)` returns the setting's value, or `NULL` when it was never set, instead of raising. The `NULLIF` covers a second state. A custom setting that was set and later reset on a connection reads back as an empty string, not `NULL`, and `''::uuid` is an error. Closing a session under [connection delivery](#connection-delivery) resets the keys it set, so this state is common on a reused connection. With the `NULLIF`, both states fail closed: + +| The setting on the connection | `current_setting(key, true)` | `NULLIF(..., '')::uuid` | Rows | +| :--- | :--- | :--- | :--- | +| never set | `NULL` | `NULL` | none | +| set, then reset | `''` | `NULL` | none | +| `'a1a1…'` | `'a1a1…'` | `'a1a1…'::uuid` | that tenant's | + +Without the `NULLIF`, the second row raises `invalid input syntax for type uuid: ""` on every query against the table. Write the same wrapper into a policy you write in SQL. + +## `FORCE` and `BYPASSRLS` + +Postgres exempts a table's owner from its policies unless the table has `FORCE ROW LEVEL SECURITY`. An application that connects as the owner of its tables, the common single-role setup, would otherwise declare policies that never apply. `RowSecurity(force=True)`, the default, emits `FORCE` as well as `ENABLE`. + +Superusers and roles with `BYPASSRLS` skip every policy, `FORCE` included. A connection that must see every row, such as the migrating role or an admin job, gets `BYPASSRLS` as a role property, which the database records and audits. Leaving a setting out never grants access; it shows no rows. + +## Session settings + +A **session setting** is a Postgres setting (a GUC) that a session sends with each operation it runs. You declare settings when you open the session, or later with `Session.set_config`. + +### Round trips per operation + +How the settings reach the database depends on whether the operation runs inside a `transaction()` block. Under the default `transaction` delivery: + +- **Inside a `transaction()` block**, the settings follow its `BEGIN` as one `SELECT set_config(...)` statement: one extra round trip for the whole block. Savepoints inside it inherit them. +- **Outside one**, each operation wraps itself in a transaction to carry them: `BEGIN`, the `set_config` statement, the operation's own statements, then `COMMIT`. That is three extra round trips per operation. An operation is one call into Ferro (`all()`, `create()`, `save()`, a raw query), so a chunked `bulk_create` is one operation that pays once. +- **A session without settings** sends nothing extra. + +A flow of several operations is cheaper inside one `transaction()` block, which pays once. `ferro.raw.execute(..., autocommit=True)` opts one statement out of the wrap, and out of the scope with it, for the few statements Postgres refuses to run inside a transaction. + +### Visibility of a new value + +`set_config` changes the session's settings and returns once the change is recorded: + +- An operation or `transaction()` block that **starts after** `set_config` returns runs with the new value, whichever task starts it. +- A `transaction()` block of this session that is **already open**, in any task sharing the session, gets the new value applied to its own connection before `set_config` returns. Its next statement sees it. +- An operation **already running** outside a `transaction()` block (a chunked `bulk_create` in another task, say) finishes with the value it started with. + +`set_config` also empties the session's identity map, so an instance loaded under the old value is never handed back under the new one. Setting a key to the value it already holds changes nothing and sends nothing. + +If delivering the new value to an already-open `transaction()` block fails, `set_config` raises `RuntimeError`, but the session's settings and identity map have already changed. The failed statement leaves that transaction aborted, so it cannot go on to read under the old value; roll it back and carry on with the new one. + +### Nested sessions + +A session opened inside another starts from the outer session's settings and overrides them key by key. The merge is a snapshot taken when the inner session opens, and nothing flows back to the outer one: + +```python +--8<-- "docs/examples/row_level_security.py:nested" +``` + +The inner session kept the outer `app.tenant_id` and replaced `app.user`; the outer session's settings are unchanged. + +Settings follow the session, not the connection. A session nested on another named connection inherits the outer settings there too, and that is the way to reach a second Postgres connection with the same scope. `transaction(using=...)` cannot do it alone: inside a session, `using=` that names a connection other than the session's own raises `ValueError`. + +Settings a session declares itself must be honoured, so opening a session with `settings=` on a non-Postgres connection raises `RuntimeError`. Settings a session only inherits are different: a nested session on SQLite opens and runs without them, since SQLite has no row security to scope. + +## The pooler constraint + +A setting made with plain `SET` stays on the server connection. Behind a pooler in transaction mode (PgBouncer's default, and the front door of most managed Postgres services), that server connection passes to another client between transactions, so two requests can share it seconds apart: + + +```text +tenant A → pooler → server connection 7: SET app.tenant_id = 'A' +tenant B → pooler → server connection 7: SELECT … FROM invoice + -- runs with tenant A's scope +``` + +Plain `SET` behind a transaction-mode pooler leaks one tenant's scope to the next request. A client cannot see the pooler's mode (`SHOW pool_mode` needs the pooler's admin database), so Ferro cannot detect the case and refuse. Instead its default, `transaction` delivery, uses `set_config(key, value, true)`: the `true` is Postgres's `is_local` flag, so the value ends at `COMMIT` and never reaches the connection's next client. That is safe on a direct connection, PgBouncer in transaction mode, RDS Proxy and the Supabase pooler alike. + +## Connection delivery + +`PoolConfig(settings_delivery="connection")` trades that safety for fewer round trips. A settings-bearing session takes one pool connection at its first operation and keeps it until it closes. It sends `set_config(key, value, false)` once, so the value lasts for the whole database session, and then sends its statements with no wrap. Use it only when the pool connects to Postgres directly, with no transaction-mode pooler in between: behind one, it is the leak described above. + +What changes under connection delivery: + +- **Cost.** One `set_config` round trip per session, instead of three extra round trips per operation outside a `transaction()` block. +- **Concurrency.** Each settings-bearing session holds a connection from its first operation to its close, so no more of them run at once than the pool has connections. Size `max_connections` for the peak number of concurrent scoped sessions. +- **One operation at a time.** Sibling tasks sharing one session run their operations one after another, and a sibling's operation waits for an open `transaction()` block instead of running inside it. +- **A check on release.** While any session on the pool holds a connection, every connection released to that pool runs one `SELECT current_setting(...)`, including releases from sessions without settings. While none does, the check costs no query. +- **`autocommit=True` stays scoped.** There is no wrap to opt out of, since the setting lives on the connection. +- **A late first setting.** A session that opens without settings and gets its first one from `set_config` holds no connection yet, so it takes one at its next operation and sends the values then. +- **`set_config` reaches running work.** The new value lands on the held connection at once, so a sibling task's operation that is already running sees it from its next statement. Under `transaction` delivery that operation would finish with its old value. +- **Closing resets.** A closing session resets exactly the keys it set, to the value the connection started with. For a custom setting that is the empty string, which the `NULLIF` handles. A value configured with `ALTER ROLE ... SET` or `ALTER DATABASE ... SET` comes back instead, so never give a tenancy key a role or database default. Nothing else is reset, so the `search_path` the pool sets on every connection stays. +- **Migrations wait.** `connect(migrate_updates=...)` and `migrate()` refresh the pool afterwards, and the refresh waits for every held connection to come back. Ferro warns with the number of connections it is waiting on. Run schema changes before opening tenant sessions. + +Sessions without settings never hold a connection, so they behave as they do under `transaction` delivery. + +## Drift and ownership + +Ferro manages the policies it names `rls__` and leaves every other policy alone. + +When `migrate_updates` finds a shorthand policy whose live body differs from the declaration, it drops and recreates it. Ferro rendered that body itself, so it knows what the catalog should hold, and the rebuild reads no rows. + +A policy written in SQL is different, because Postgres rewrites SQL when it stores it (`x BETWEEN a AND b` comes back as `(x >= a) AND (x <= b)`). A textual difference does not prove a real change, and rebuilding on every rewrite would lock the table on every connect. Ferro reports the declared and the stored text and leaves the policy in place. To clear the report, express the policy as a shorthand, or drop the policy so the next `migrate_updates` recreates it. A migration compares two declarations, never the live catalog, so `ferro migrate new` rebuilds a changed SQL policy as written. + +The table flags only switch on. `migrate_updates` turns `ENABLE` and `FORCE` on, never off. A model that drops its `__ferro_rls__` or its `force=` keeps the live flags and gets a warning on every connect until the declaration returns or `migrate_destructive` removes them. An orphaned `rls_*` policy, declared once and no longer, warns under `migrate_updates` and is dropped under `migrate_destructive`. + +All of that applies only to a table with at least one live `rls_*` policy. A table where a database administrator turned row security on by hand, with only their own policies, gets no warning and no change, even from a destructive run. + +Alembic autogenerate writes what `migrate_destructive=True` would do, with no destructive gate, since a revision is reviewed before it runs. It does not carry the two cases the runtime pass only warns about: a drifted policy written in SQL, and a policy Ferro does not own. Watch the `migrate_updates` warnings for those. + +## See also + +- [Scope rows by tenant with Postgres row-level security](../guide/row-level-security.md): the setup, step by step. +- [Row-level security reference](../reference/row-level-security.md): `RowSecurity`, `RowPolicy`, session settings and `PoolConfig(settings_delivery=...)`. diff --git a/docs/pages/concepts/type-safety.md b/docs/pages/concepts/type-safety.md index f4e09ad5..016f0efb 100644 --- a/docs/pages/concepts/type-safety.md +++ b/docs/pages/concepts/type-safety.md @@ -1,110 +1,111 @@ -# Type Safety +--- +description: "How Pydantic validates Ferro models at runtime and how a type checker knows what each query returns." +--- -Ferro is built on Pydantic V2 and Python's type system. Models validate at runtime, queries return typed results, and the Rust boundary ships with type stubs, so both your IDE and your type checker understand Ferro code. +# Type safety -## Pydantic at the Core +Ferro models are Pydantic models and Ferro's API is fully annotated, so building an instance validates its fields at runtime and a type checker knows what each query returns. -Ferro models *are* Pydantic models, not Pydantic-flavored lookalikes: +The examples on this page use this `User` model, with four users saved (ada, grace, alan and edsger): -```python -from pydantic import BaseModel +=== "Assignment" -from ferro import Model + ```python + --8<-- "docs/examples/concepts.py:models" + ``` +=== "Annotated" -class User(Model): - username: str - age: int + ```python + --8<-- "docs/examples/concepts_annotated.py:models" + ``` +A query returns `User` instances, and each one is a Pydantic model: -assert issubclass(User, BaseModel) +```python {test=run prelude=concepts} +from ferro import engines -user = User(username="alice", age=30) -print(user.model_dump()) # {'username': 'alice', 'age': 30} -print(user.model_dump_json()) # '{"username":"alice","age":30}' +async with engines.session(): + ada = await User.where(lambda user: user.username == "ada").first() + assert ada is not None + assert ada.model_dump() == { + "id": 1, + "username": "ada", + "email": "ada@example.com", + "age": 36, + "active": True, + } ``` -Everything Pydantic gives you — `model_dump`, `model_validate`, JSON schema generation, serialization config — works unchanged. And validation runs on the write paths: `Model(...)` construction, `Model.create(...)`, and `instance.save()` all go through Pydantic before any data reaches the database. +`first()` returns `None` when no row matches, so the first `assert` rules that out before `ada` is used. `model_dump()` is Pydantic's own method, and it returns ada's row as a dict. -```python +## Pydantic models + +`Model` is a subclass of Pydantic's `BaseModel`, so `model_dump`, `model_validate`, JSON schema generation and serialization settings all work unchanged. Validation runs whenever an instance is built, by constructing a model or by `Model.create(...)`, before any data reaches the database. Constructing a user with an age that is not a number fails: + +```python {test=run prelude=concepts} from pydantic import ValidationError try: - User(username="alice", age="not a number") -except ValidationError as e: - print(e) # age: Input should be a valid integer + User(username="linus", email="linus@example.com", age="twenty") +except ValidationError as error: + print(error) +else: + raise AssertionError("an age that is not a number must fail validation") ``` -## Static Typing - -Ferro's public API carries full type hints, so static checkers (mypy, Pyright, Pylance) and your IDE know exactly what each call returns: +It prints: -```python -# get() returns the model — and raises ModelDoesNotExist if missing -user: User = await User.get(1) - -# get_or_none() makes absence explicit -maybe_user: User | None = await User.get_or_none(999) - -# Query terminals are typed -users: list[User] = await User.all() -first: User | None = await User.where(lambda t: t.age >= 18).first() -n: int = await User.where(lambda t: t.age >= 18).count() -present: bool = await User.where(lambda t: t.username == "alice").exists() + +```text +1 validation error for User +age + Input should be a valid integer, unable to parse string as an integer [type=int_parsing, input_value='twenty', input_type=str] + For further information visit https://errors.pydantic.dev/2.12/v/int_parsing ``` -Because results are real model instances with annotated fields, downstream code is checked too: `user.username` is a `str`, `user.age` is an `int`, and `user.nonexistent` is a type error. - -## IDE Support +The error names the field and the rule it broke, and nothing was written. An assignment to a field of an existing instance is not validated unless the model sets `validate_assignment=True` in its `model_config`, and `save()` writes the values the instance holds. Pydantic's coercion rules apply too: in its default, non-strict mode, `age="36"` becomes the integer `36`. Field validators, model validators and `model_config` settings behave as they do in Pydantic; [Models and fields › Validation](../guide/models-and-fields.md#validation) shows one. -The same annotations drive autocomplete: +Rich field types validate from end to end: `datetime`, `date`, `time`, `Decimal`, `UUID`, enums, and `dict` and `list` fields stored as JSON all come back from the database as the annotated Python type. [Database backends](backends.md#type-mapping) shows the column type each one gets. -- Model instances complete their fields and methods (`save`, `delete`, `refresh`, ...). -- `Model.where(...)`, `.order_by(...)`, `.limit(...)` chains preserve the model type, so `await User.where(...).first()` completes `User` attributes on the result. -- Field names complete when you type `User.` inside a query expression. +## Typed results -No plugins are required — Ferro deliberately stays plugin-free and relies on standard typing constructs (`Self`, generics, overloads). +Ferro's public API carries type hints, so a type checker (ty, mypy, Pyright) and your editor know what each call returns. This block runs inside a [session](../guide/connections.md#sessions): -## The Rust Boundary +```python {test=run prelude=concepts} +from typing import assert_type -The compiled extension module can't be introspected by type checkers, so Ferro ships a stub file (`src/ferro/_core.pyi`) describing every FFI function — `connect`, `create_tables`, `migrate`, the fetch/save primitives, transaction control, and raw SQL entry points. The package is also marked with `py.typed`, so type checkers pick all of this up automatically when you depend on `ferro-orm`. - -In practice you rarely touch `ferro._core` directly; the typed Python layer (`Model`, `Query`, `connect`, `transaction`) is the public API, and the stub exists so that even the boundary itself is checkable. - -## Validators and Coercion - -Because models are Pydantic models, the full validator toolbox applies: +assert_type(await User.get(1), User) +assert_type(await User.get_or_none(99), User | None) +assert_type(await User.all(), list[User]) +assert_type(await User.where(lambda user: user.age >= 18).count(), int) +assert_type(await User.where(lambda user: user.username == "ada").exists(), bool) +``` -```python -from pydantic import field_validator +Each `assert_type` passes `ty check`, and at runtime it does nothing. `first()` is annotated the same way, returning the model or `None`. `get()` returns `User` because it raises `ModelDoesNotExist` when no row has the key; `get_or_none()` returns `User | None` instead. A result is a model instance, so code that uses it is checked too: `ada.age` is an `int`, and `ada.nickname` is a type error. -from ferro import Model +## Editor completion +The same annotations drive completion in your editor: -class Account(Model): - username: str - email: str +- A model instance completes its fields and its methods (`save`, `delete`, `refresh`, ...). +- `get()`, `get_or_none()` and `all()` return `User`, so their results complete `User` attributes. +- Inside a lambda predicate, attribute names are not completed: the lambda's parameter is a query proxy whose attributes are typed `FieldProxy[Any]`. A misspelled name raises `AttributeError` when the query is built, before anything reaches the database; see [Typed query predicates › Proxy validation](query-typing.md#proxy-validation). - @field_validator("email") - @classmethod - def normalize_email(cls, v: str) -> str: - if "@" not in v: - raise ValueError("Invalid email") - return v.lower() -``` +Ferro needs no type checker plugin. It uses standard typing constructs (`Self`, generics, overloads) that every checker reads. -Validators run on construction and on `create`/`save`, so invalid data is rejected before it is written. Pydantic's coercion rules also apply — `Account(username="a", email="A@B.COM")` stores `"a@b.com"`, and numeric strings coerce to `int`/`float` fields under Pydantic's standard (non-strict) mode. +## Rust boundary -Rich field types validate end to end: `datetime`, `date`, `Decimal`, `UUID`, enums, and JSON-shaped `dict`/`list` fields all round-trip through the database back into their proper Python types. See [Backends](backends.md) for how each maps to column types. +A type checker cannot read a compiled extension module, so Ferro ships a stub file, `ferro/_core.pyi`, that describes every function the Rust core exposes. The package is marked with `py.typed`, so checkers use the stub and Ferro's own annotations when you depend on `ferro-orm`. -## Limits +You rarely call `ferro._core` directly: `Model`, `Query`, `connect` and `transaction` are the public API. The stub exists so the boundary itself is checked too. -**Query predicates** are one place static typing needs a small assist: the lambda parameter passed to `where()` is typed as a `QueryProxy`, whose attributes resolve to `FieldProxy[Any]` rather than each field's declared type. That means the checker catches a predicate that returns the wrong *shape* — `lambda user: True` fails `ty check` / Pyright because it doesn't return a `QueryNode` — but not one that compares a field to the wrong *type*, like `user.age >= "eighteen"`. Wiring per-field types through the proxy needs mapped types, proposed for Python in [PEP 827](https://peps.python.org/pep-0827/) (draft, targeting Python 3.16); Ferro adopts it with zero runtime change once checkers support it. +## Where checking stops -This is a deep enough topic to get its own page: see [Typed Query Predicates](query-typing.md). +A type checker catches a predicate that returns the wrong kind of value, but not a comparison with the wrong type: `lambda user: user.age >= "eighteen"` type-checks, because `user.age` is `FieldProxy[Any]`. [Typed query predicates › Where checking stops](query-typing.md#where-checking-stops) explains why, and what would close the gap. -## See Also +## See also -- [Typed Query Predicates](query-typing.md) — how lambda predicates type-check -- [Models & Fields](../guide/models-and-fields.md) -- [Architecture](architecture.md) +- [Typed query predicates](query-typing.md): how lambda predicates are checked +- [Models and fields](../guide/models-and-fields.md): declaring fields, types and validation +- [Architecture](architecture.md): where Python ends and Rust begins diff --git a/docs/pages/contributing.md b/docs/pages/contributing.md index c54b169a..88694a2b 100644 --- a/docs/pages/contributing.md +++ b/docs/pages/contributing.md @@ -1,5 +1,7 @@ -# Contributing +--- +description: "Set up a development environment, run the tests and linters, and open a pull request against Ferro." +--- -Ferro welcomes contributions — bug reports, docs fixes, and code across both the Python and Rust layers. The guide below is the canonical contributor reference from the repository. +# Contributing --8<-- "CONTRIBUTING.md:contributing" diff --git a/docs/pages/faq.md b/docs/pages/faq.md index 103544ea..03fd2f74 100644 --- a/docs/pages/faq.md +++ b/docs/pages/faq.md @@ -1,105 +1,161 @@ -# Frequently Asked Questions +--- +description: "Short answers about what Ferro is, what it supports and how to fix its most common errors." +--- -## General +# FAQ -### What is Ferro? +Short answers about what Ferro is, what it supports and how to fix its most common errors. The answers share one pair of models, `Author` and `Post`, declared here once: -Ferro is an async Python ORM with a Rust core. Models are Pydantic V2 `BaseModel` subclasses; SQL generation, connection pooling, query execution, and row hydration happen in compiled Rust (PyO3 + SQLx + Sea-Query). You write ordinary async Python — the Rust engine is invisible at the API level. See [Architecture](concepts/architecture.md). +=== "Assignment" -### Is Ferro production-ready? + ```python + --8<-- "docs/examples/faq.py:models" + ``` -Ferro is pre-1.0. The core feature set — models, queries, relationships, transactions, named multi-database connections, raw SQL, auto-migration, and the Alembic bridge — is implemented and tested against both SQLite and PostgreSQL on every change. But pre-1.0 means what it says: APIs may still shift between minor versions, the community is small, and you'll find fewer battle scars documented than for SQLAlchemy or Django ORM. +=== "Annotated" -Well suited to new projects and services where you control the upgrade cadence and have good test coverage. Pin your version, read the [changelog](changelog.md) before upgrading, and report what you hit. + ```python + --8<-- "docs/examples/faq_annotated.py:models" + ``` -### What license is Ferro under? +An `Author` has many `Post` rows, and each `Post` points back at its `Author`. Every block below runs inside `async with engines.session():`; see [Connections › Sessions](guide/connections.md#sessions) for what a session is. -Apache-2.0. +## What Ferro is -### Why does Ferro require Python 3.13+? +Ferro is an async Python ORM with a Rust core. Models are Pydantic V2 `BaseModel` subclasses; SQL generation, connection pooling, query execution and row hydration run in compiled Rust (PyO3, SQLx and Sea-Query). You write ordinary async Python and the engine stays out of sight. See [Architecture](concepts/architecture.md). -Ferro is a young project and deliberately targets current Python rather than carrying compatibility shims for older interpreters. That keeps the codebase on modern typing features and current Pydantic and PyO3 releases. The floor may widen as the project matures, but supporting older Pythons is not a near-term goal. +To see it in use, ask for the authors named Ada Lovelace: -### Do I need to know Rust? +```python {test=run prelude=reference} +from ferro import engines -No. Ferro's API is 100% Python and ships as a prebuilt wheel. Rust only enters the picture if you build from source or contribute to the engine itself. +async with engines.session(): + authors = await Author.where(lambda author: author.name == "Ada Lovelace").all() + assert [author.email for author in authors] == ["ada@example.com"] +``` -## Features +The query returns `Author` instances, and the assert shows the one it found. -### Does Ferro support synchronous code? +## Production readiness -No — Ferro is async-only. Every database operation is a coroutine (`await User.all()`). There is no sync facade and none is planned; if your application is synchronous, you'd be running an event loop per call (`asyncio.run(...)`), which works for scripts but defeats the purpose in a server. Ferro is designed for async frameworks like FastAPI, Litestar, and aiohttp. +Ferro is pre-1.0. The core feature set (models, queries, relationships, transactions, named connections, raw SQL, auto-migration and the Alembic bridge) is implemented and tested against SQLite and PostgreSQL on every change. Pre-1.0 means APIs may still shift between minor versions, the community is small, and fewer problems are documented than for SQLAlchemy or Django ORM. -### Which databases are supported? +Ferro suits new projects and services where you control the upgrade cadence and have good test coverage. Pin your version, read the [changelog](changelog.md) before you upgrade, and report what you hit. -SQLite and PostgreSQL (including hosted providers such as Supabase). MySQL and other databases are not currently supported — unsupported URL schemes fail at `connect()` time. See [Backends](concepts/backends.md). +## License -### What's the migrations story? +Ferro is licensed under Apache-2.0. The terms are in the repository's [`LICENSE`](https://github.com/syn54x/ferro-orm/blob/main/LICENSE) file. -Two doors, one per database: +## Python version -- **Auto-migrate at connect time** — `connect(url, auto_migrate=True)` creates missing tables; `migrate_updates=True` also alters existing ones as far as an in-place statement can, and `migrate_destructive=True` drops model-removed columns. Great for tests and development. -- **Migrations** — `ferro migrate new` writes each change as a numbered, reviewable directory of SQL (one rendering per target dialect) and Python data steps, and `ferro migrate up` applies it. Renames, backfills, SQLite table rebuilds and a way back down live here. Install with `pip install "ferro-orm[cli]"`. +Ferro requires Python 3.13 or later. It targets current Python on purpose, so the codebase uses modern typing features and current Pydantic and PyO3 releases instead of compatibility shims. The floor may widen as the project matures; supporting older interpreters is not a near-term goal. -A project already on Alembic can keep it: the [Alembic bridge](guide/schema/alembic.md) writes revisions from the same planner. See [Schema Management](guide/schema/overview.md). +## Rust -### Does Ferro support multiple databases? +You do not need to know Rust. Ferro's API is Python, and it ships as a prebuilt wheel. Rust matters only if you build from source or contribute to the engine; see [Contributing](contributing.md). -Yes — register each pool under a name and route explicitly: +## Synchronous code -```python -import ferro +Ferro is async-only. Every database operation is a coroutine (`await Author.all()`), and there is no sync facade. A synchronous script can call `asyncio.run(...)` once per entry point, but a synchronous server would pay for an event loop per call, which defeats the purpose. Ferro is designed for async frameworks such as FastAPI, Litestar and aiohttp. + +## Supported databases + +Ferro supports SQLite and PostgreSQL, including hosted providers such as Supabase. MySQL and other databases are not supported, and an unsupported URL scheme fails at `connect()`. See [Backends](concepts/backends.md). + +## Migrations + +Ferro has two ways to change a schema, plus a bridge to Alembic: -await ferro.connect(APP_DATABASE_URL, name="app", default=True) -await ferro.connect(SERVICE_DATABASE_URL, name="service") +- **Auto-migrate at connect time.** `connect(url, auto_migrate=True)` creates missing tables. `migrate_updates=True` also alters existing ones as far as an in-place statement can, and `migrate_destructive=True` drops columns the model no longer declares. Use it in tests and development. +- **Migrations.** `ferro migrate new` writes each change as a numbered, reviewable directory of SQL (one rendering per target dialect) and Python data steps, and `ferro migrate up` applies it. Renames, backfills, SQLite table rebuilds and a way back down live here. Install the commands with `pip install "ferro-orm[cli]"`. +- **The Alembic bridge.** A project already on Alembic can keep it: the [Alembic bridge](guide/schema/alembic.md) writes revisions from the same planner. -async with ferro.engines.session(): - users = await User.all() # default connection +See [Schema management](guide/schema/overview.md) for choosing between them. -jobs = await Job.using("service").all() # explicit routing +## Multiple databases + +Ferro registers each pool under a name, and you route to it explicitly: + +```python +--8<-- "docs/examples/faq.py:databases" ``` -Automatic read/write splitting, cross-database joins, and distributed transactions are out of scope. +The default connection answers inside a session, and `using("analytics")` is its own explicit route to the second database. Automatic read/write splitting, cross-database joins and distributed transactions are out of scope. See [Multiple databases](howto/multiple-databases.md) for the full recipe. -### Does Ferro support raw SQL? +## Raw SQL -Yes. `execute`, `fetch_all`, and `fetch_one` (top-level or on the handle yielded by `transaction()`) run parameterized raw SQL. Rows come back as plain dicts of primitives — it's an escape hatch, not a typed query path. See [Queries](guide/queries.md). +Ferro runs parameterized raw SQL with `execute`, `fetch_all` and `fetch_one`, at the top level or on the handle that `transaction()` yields: -### Does Ferro have eager loading (`prefetch_related` / `select_related`)? +```python {test=run prelude=reference} +from ferro import fetch_all -Not yet — it's on the [roadmap](roadmap.md). Today, awaiting a relationship attribute issues a query, so be deliberate about relationship access inside loops (the classic N+1 pattern). +rows = await fetch_all("SELECT title FROM post WHERE views > ?", 1000) +assert rows == [{"title": "Notes on the Analytical Engine"}] +``` -## Performance +Rows come back as plain dicts of primitives. Raw SQL is an escape hatch, not a typed query path. See [Raw SQL](guide/raw-sql.md). -### Is Ferro faster than other Python ORMs? +## Eager loading -For CPU-bound ORM work — bulk inserts, hydrating large result sets, heavy concurrent query loads — yes, meaningfully: that work runs in Rust with the GIL released. For network- or disk-bound operations (single-row fetches, slow queries, remote databases), wait time dominates and any async ORM performs similarly. We intentionally don't publish benchmark multipliers; measure your own workload. See [Performance](concepts/performance.md) for what to optimize and how to benchmark fairly. +Ferro has no `prefetch_related` or `select_related`. A forward foreign key loads in the same statement with `include()`: + +```python {test=run prelude=reference} +posts = await ( + Post.where(lambda post: post.views > 100) + .order_by(lambda post: post.views, "desc") + .include(lambda post: post.author) + .all() +) +assert [(post.title, post.author.name) for post in posts] == [ + ("Notes on the Analytical Engine", "Ada Lovelace"), + ("The Education of a Computer", "Grace Hopper"), +] +``` + +Each `post.author` arrives populated, with no second query. Reverse and many-to-many collections are not populated yet, so awaiting `author.posts` in a loop issues one query per author (the N+1 pattern); see the [roadmap](roadmap.md#query-features). See [Queries](guide/queries.md#populating-relations-with-include) for the full contract. + +## Speed + +For CPU-bound ORM work (bulk inserts, hydrating large result sets, heavy concurrent query loads), Ferro is meaningfully faster than pure-Python ORMs, because that work runs in Rust with the GIL released. For network- or disk-bound work (single-row fetches, slow queries, remote databases), wait time dominates and any async ORM performs similarly. Ferro publishes no benchmark multipliers; measure your own workload. See [Performance](concepts/performance.md). ## Troubleshooting -### I get "Engine not initialized" +### `Engine not initialized` + +You ran a schema operation such as `ferro.create_tables()` before connecting. Importing models registers their schemas but does not open a database connection, so call `await ferro.connect(...)` during application startup, before the first call. -You ran a query before connecting. Importing models registers their schemas but does not open a database connection — call `await ferro.connect(...)` during application startup, before the first query. +### `No database route for this operation` -### I get "Relationship resolution failed" or an error about `related_name` +You ran a query outside a session, with no `using=` or `session=`. Open a session around the work, as the first block on this page does, or route with `using=`. See [Connections › Sessions](guide/connections.md#sessions). -Relationships are declared on both sides and cross-validated when you connect. Two common causes: +### `defines a relationship ... but ... does not have that field defined as BackRef()` -- The model named in a string/forward reference (`"Author"`) was never imported, so it isn't registered — import all your model modules before `connect()`. -- The target model is missing the reverse field: a `ForeignKey(related_name="posts")` requires the target to declare `posts: Relation[list["Post"]] = BackRef()`. The error message names the model and field to add. +Relationships are declared on both sides and cross-validated when you connect. The full message reads `Model 'P' defines a relationship to 'A' with related_name='posts', but 'A' does not have that field defined as BackRef()/Field(back_ref=True)`. It has two common causes: + +- The target model has no reverse field. A `ForeignKey(related_name="posts")` needs the target to declare `posts: Relation[list["Post"]] = BackRef()`, as `Author` does above. +- The model named in a forward reference (`"Author"`) was never imported, so it is not registered. Import every model module before `connect()`. See [Relationships](guide/relationships.md). -### I get `ModelDoesNotExist` +### `ModelDoesNotExist` -`Model.get(pk)` raises `ferro.ModelDoesNotExist` (a `LookupError` subclass) when no row matches the primary key. If absence is an expected case, use `Model.get_or_none(pk)`, which returns `None` instead: +`Model.get(pk)` raises `ferro.ModelDoesNotExist`, a `LookupError` subclass, when no row has that primary key. If a missing row is an expected case, use `Model.get_or_none(pk)`, which returns `None`: -```python -user = await User.get_or_none(user_id) -if user is None: - ... # handle missing row +```python {test=run prelude=reference} +from ferro import ModelDoesNotExist + +try: + await Post.get(99) +except ModelDoesNotExist as error: + assert str(error) == "No Post record found for primary key 99" +else: + raise AssertionError("expected ModelDoesNotExist") + +assert await Post.get_or_none(99) is None ``` -### Where do I report bugs or ask questions? +`get()` raises with the model and key in the message, and `get_or_none()` returns `None` for the same key. + +## Reporting bugs and asking questions -Bugs and feature requests: [GitHub Issues](https://github.com/syn54x/ferro-orm/issues). Questions and discussion: [GitHub Discussions](https://github.com/syn54x/ferro-orm/discussions). +Report bugs and request features on [GitHub Issues](https://github.com/syn54x/ferro-orm/issues). diff --git a/docs/pages/getting-started/installation.md b/docs/pages/getting-started/installation.md index de47899a..9938789e 100644 --- a/docs/pages/getting-started/installation.md +++ b/docs/pages/getting-started/installation.md @@ -1,11 +1,10 @@ -# Installation - -## Requirements +--- +description: "Install Ferro with pip or uv, add the optional extras and check the installed version." +--- -- **Python 3.13 or newer** -- macOS, Linux, or Windows — pre-compiled wheels are published for all three, so no Rust toolchain is needed for a normal install +# Installation -## Install +Install Ferro with `pip` or `uv`, then check the installed version. Ferro ships pre-compiled wheels, so a normal install needs no Rust toolchain: === "uv" @@ -19,60 +18,70 @@ pip install ferro-orm ``` -### Migration support +The install adds Ferro and its two runtime dependencies, Pydantic and pydantic-settings. + +## Requirements + +- Python 3.13 or newer. Ferro is tested on 3.13 and 3.14. +- macOS, Linux or Windows. Wheels are published for all three. + +## Extras -Reviewed schema migrations (`ferro migrate init`, `new`, `up`) need the `cli` extra, which installs the `ferro` command line: +Ferro has two optional extras. The `cli` extra installs the `ferro` command, which runs reviewed schema migrations (`ferro migrate init`, `new`, `up`). The `alembic` extra adds Alembic and SQLAlchemy so a project that already runs Alembic can generate revisions from Ferro models; Ferro never uses SQLAlchemy at runtime. + +Add the extra you need: === "uv" ```bash uv add "ferro-orm[cli]" + uv add "ferro-orm[alembic]" ``` === "pip" ```bash pip install "ferro-orm[cli]" + pip install "ferro-orm[alembic]" ``` -See [Schema Management](../guide/schema/overview.md) for how migrations and auto-migrate fit together, and [Migrations](../guide/schema/migrations.md) for the workflow. - -A project that already runs Alembic can keep it with the `alembic` extra instead (`pip install "ferro-orm[alembic]"`), which adds Alembic and SQLAlchemy for generating revisions only; Ferro never uses SQLAlchemy at runtime. See [Alembic](../guide/schema/alembic.md). +[Schema management](../guide/schema/overview.md) explains how migrations and auto-migrate fit together, [Migrations](../guide/schema/migrations.md) covers the workflow, and [Alembic](../guide/schema/alembic.md) covers the bridge. -## Database Drivers +## Database drivers -You don't need any. Ferro's Rust engine bundles SQLite and PostgreSQL support via SQLx, so there are no driver packages to install or configure — `pip install ferro-orm` is enough for both backends. +Ferro's Rust engine bundles SQLite and PostgreSQL support through SQLx, so there are no driver packages to install or configure. `ferro-orm` alone covers both backends. -## Building from Source +## Build from source -!!! note - Most users never need this. Pre-compiled wheels cover all common platforms. - -Building from source (for example, on an unsupported platform) requires a recent Rust toolchain and [maturin](https://www.maturin.rs/): +Most users never need this, because wheels cover the common platforms. To build on an unsupported platform, install a Rust toolchain and [uv](https://docs.astral.sh/uv/), then build into a project environment: ```bash -# Install Rust curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh - -# Clone and build git clone https://github.com/syn54x/ferro-orm.git cd ferro-orm -pip install maturin -maturin develop --release +uv sync +uv run maturin develop --release ``` -Expect the first compile to take a few minutes. +The first compile takes a few minutes, and the build installs Ferro into the project's `.venv`. -## Verify Your Installation +## Installed version -```python +Print the version to confirm the install: + +```python {test=compile reason="prints the version, which changes every release"} import ferro print(ferro.version()) ``` -If this prints a version number, you're ready to go. +The number is the installed Ferro version: + + +```text +0.25.0 +``` -## Next Steps +## Next steps -Build your first app with the [Quickstart Tutorial](quickstart.md). +Build a small app in the [Quickstart](quickstart.md). diff --git a/docs/pages/getting-started/next-steps.md b/docs/pages/getting-started/next-steps.md deleted file mode 100644 index 72a02d6b..00000000 --- a/docs/pages/getting-started/next-steps.md +++ /dev/null @@ -1,56 +0,0 @@ -# Next Steps - -You've finished the [Quickstart](quickstart.md) and have a working Ferro app. Pick a path from what you're building. - -## Learn by Use Case - -### Building an API - -For REST APIs with FastAPI, Starlette, or similar: - -1. **[Models & Fields](../guide/models-and-fields.md)** — field types, constraints, and defaults -2. **[Relationships](../guide/relationships.md)** — foreign keys, back-references, many-to-many -3. **[Queries](../guide/queries.md)** — filtering with lambda predicates, ordering, and pagination -4. **[How-To: Pagination](../howto/pagination.md)** — efficient offset and cursor pagination -5. **[How-To: Testing](../howto/testing.md)** — fast, isolated tests with in-memory SQLite - -### Data Processing - -For ETL pipelines and bulk workloads: - -1. **[Mutations](../guide/mutations.md)** — `bulk_create` and bulk update/delete for throughput -2. **[Transactions](../guide/transactions.md)** — make multi-step writes atomic -3. **[Queries](../guide/queries.md)** — filter in the database, not in Python - -### Production Deployment - -When you're ready to ship: - -1. **[Connections & Databases](../guide/connections.md)** — connection URLs and pool configuration -2. **[Schema Management](../guide/schema/overview.md)** — migrations for evolving a production schema, and [deploying them](../guide/schema/deploying.md) -3. **[How-To: Multiple Databases](../howto/multiple-databases.md)** — working with more than one database -4. **[How-To: Testing](../howto/testing.md)** — a test suite you can trust before deploying - -## Common Patterns - -Recipes for things most applications need: - -- **[Testing](../howto/testing.md)** — per-test database isolation and fixtures -- **[Pagination](../howto/pagination.md)** — offset- and cursor-based pagination -- **[Timestamps](../howto/timestamps.md)** — `created_at` / `updated_at` on every model -- **[Soft Deletes](../howto/soft-deletes.md)** — flag rows as deleted instead of removing them -- **[Multiple Databases](../howto/multiple-databases.md)** — connecting to more than one database -- **[Migrating from SQLAlchemy](../howto/migrate-from-sqlalchemy.md)** — a side-by-side translation guide - -## Reference Material - -- **API Reference** — complete documentation for every public class and method: [Model](../api/model.md), [Queries](../api/queries.md), [Fields & Types](../api/fields.md), [Relationships](../api/relationships.md) -- **Concepts** — how Ferro works under the hood: [Architecture](../concepts/architecture.md), [Identity Map](../concepts/identity-map.md), [Type Safety](../concepts/type-safety.md) -- **[Roadmap](../roadmap.md)** — what's implemented today and what's planned - -## Get Help - -- **[GitHub Issues](https://github.com/syn54x/ferro-orm/issues)** — report bugs or request features -- **[GitHub](https://github.com/syn54x/ferro-orm)** — star the repo to follow releases - -Build something real and reach for the guides above as questions come up. diff --git a/docs/pages/getting-started/quickstart.md b/docs/pages/getting-started/quickstart.md index 4aa24636..7ceda8f2 100644 --- a/docs/pages/getting-started/quickstart.md +++ b/docs/pages/getting-started/quickstart.md @@ -1,12 +1,22 @@ +--- +description: "Build a small blog backend with two related models and run every core Ferro operation on SQLite." +--- + # Quickstart -In about 10 minutes you'll build a small blog backend: two related models (`Author` and `Post`), an in-memory SQLite database, and every core Ferro operation — create, query, traverse relationships, update, delete, and transactions. +This tutorial builds a small blog backend on an in-memory SQLite database: two related models, `Author` and `Post`, and every core Ferro operation: create, query, traverse a relationship, update, delete and run a transaction. + +Every code block comes from one runnable script, shown in full under [Complete script](#complete-script). To follow along in a file of your own, start with these imports: -Every code block on this page comes from one runnable script, shown in full at the [bottom of the page](#complete-script). Follow along in a file of your own, or just run the script. +```python +--8<-- "docs/examples/quickstart.py:imports" +``` -## Define Your Models +The model block comes next at module level. Every block after it uses `await`, so it goes inside one `async def main()` that you run with `asyncio.run(main())`, in the order the page shows them; the script has that shape. You can also run the script as it is. -Ferro supports two equivalent field-declaration styles — options on the assignment side, or inside `typing.Annotated`. Every model example in these docs shows both; pick one and stay consistent in your project. +## Models + +Ferro supports two equivalent field-declaration styles: options on the assignment side, or inside `typing.Annotated`. Every model example in these docs shows both, so pick one and stay consistent in your project. Declare the two models: === "Assignment" @@ -20,84 +30,124 @@ Ferro supports two equivalent field-declaration styles — options on the assign --8<-- "docs/examples/quickstart_annotated.py:models" ``` -A Ferro model is a Pydantic model — annotated fields become columns, and defaults work exactly as in Pydantic: +A Ferro model is a Pydantic model: annotated fields become columns, and defaults work as they do in Pydantic. -- `Field(default=None, primary_key=True)` marks `id` as the primary key. It's `int | None` because the database assigns it on insert. -- `Field(unique=True)` adds a unique constraint; `default_factory=datetime.now` gives each post a creation timestamp. -- `Annotated[Author, ForeignKey(related_name="posts")]` declares the many-to-one side: each `Post` stores an `author_id` column pointing at an `Author`. -- `Relation[list["Post"]] = BackRef()` is the reverse side: `author.posts` becomes a chainable query for that author's posts. `related_name="posts"` is what links the two. +- `Field(default=None, primary_key=True)` marks `id` as the primary key. The type is `int | None` because the database assigns the value on insert. +- `Field(unique=True)` adds a unique constraint, and `default_factory` gives each post a creation timestamp in UTC. +- `Annotated[Author, ForeignKey(related_name="posts")]` declares the many-to-one side: each `Post` stores an `author_id` column that points at an `Author`. +- `Relation[list["Post"]] = BackRef()` declares the reverse side, so `author.posts` becomes a query for that author's posts. `related_name="posts"` links the two. ## Connect +Connect to a database, then open a session. Every ORM call runs inside `async with engines.session():`, and raises `No database route` outside it: + ```python --8<-- "docs/examples/quickstart.py:connect" ``` -`connect()` takes a database URL. `sqlite::memory:` is a throwaway in-memory database, useful for this tutorial and for tests. For a file-backed database use `sqlite:app.db?mode=rwc` (`rwc` = read/write/create), or a `postgres://...` URL for PostgreSQL. +The new database has no authors yet, so the count is `0`. `connect()` takes a database URL: `sqlite::memory:` is a throwaway in-memory database, `sqlite:app.db?mode=rwc` is a file that is created if missing (`rwc` means read, write, create), and a `postgres://...` URL connects to PostgreSQL. `auto_migrate=True` creates a table for every registered model on connect. Use it in development and tests; for a database whose data you would mind losing, use [migrations](../guide/schema/migrations.md) (`ferro migrate new`, then `ferro migrate up`), which [Schema management](../guide/schema/overview.md) compares with auto-migrate. + +The blocks that follow run inside this session. See [Sessions](../guide/connections.md#sessions) for named connections and request-scoped sessions. -`auto_migrate=True` creates tables for every registered model on connect. Use it in development and tests; for a database whose data you would mind losing, use [migrations](../guide/schema/migrations.md) (`ferro migrate new`, then `ferro migrate up`). [Schema Management](../guide/schema/overview.md) compares the two. +## Create data -## Create Data +Create an author and a first post, then insert two more posts in one statement with `bulk_create()`: ```python --8<-- "docs/examples/quickstart.py:create" ``` -- `Model.create(...)` validates the data, inserts one row, and returns the instance with its database-assigned `id` populated. Notice you can pass a model instance (`author=alice`) for the foreign key. It never overwrites an existing row — a duplicate raises [`UniqueViolationError`](../api/exceptions.md). -- `Model.bulk_create([...])` inserts many rows in a single statement — use it whenever you're loading more than a handful of rows. Here we set `author_id` directly instead of passing the instance. +`create()` validates the data, inserts one row and returns the instance with its database-assigned `id`. Passing the instance (`author=alice`) sets the foreign key. A duplicate unique value raises [`UniqueViolationError`](../api/exceptions.md); `create()` never overwrites a row. `bulk_create()` writes the whole batch at once, so use it whenever you load more than a handful of rows. The table now holds three posts. ## Query +Fetch the first post by primary key: + ```python ---8<-- "docs/examples/quickstart.py:query" +--8<-- "docs/examples/quickstart.py:get" ``` -- `Post.get(pk)` fetches one row by primary key. -- `where(...)` filters, `order_by(...)` sorts, `limit(...)` slices — and nothing touches the database until a terminal like `.all()`, `.first()`, `.count()`, or `.exists()` runs the query. -- `lambda post: post.published == True` is a lambda predicate — the only predicate style `where()` accepts. Name the parameter after the row type in lowercase singular (`user` for `User`, `post` for `Post`). See [Queries](../guide/queries.md#predicate-style) for the full operator surface and how column names are validated at build time. +`get()` returned the same Python object as the `post` you created, because within a session one database row is one object. See [Identity map](../concepts/identity-map.md). + +Filter, order and slice the posts: -!!! note "What happened" - Thanks to Ferro's identity map, `Post.get(post.id)` returns the *same Python object* as the `post` you created earlier — not a duplicate copy. One row, one instance. +```python +--8<-- "docs/examples/quickstart.py:filter" +``` -## Work with Relationships +`where()` takes a lambda predicate, the only predicate style it accepts; name the parameter after the model in lowercase singular. `order_by()` and `limit()` shape the result, and nothing touches the database until a terminal such as `.all()`, `.first()`, `.count()` or `.exists()` runs. See [Queries](../guide/queries.md#predicate-style) for the operators and how column names are checked when you build the query. + +Count the posts and check for drafts: + +```python +--8<-- "docs/examples/quickstart.py:aggregate" +``` + +There are three posts, and at least one is unpublished. + +## Relationships + +Load a post's author, then the author's published posts: ```python --8<-- "docs/examples/quickstart.py:relationships" ``` -Two directions, two idioms: +Awaiting the foreign key (`post.author`) loads the related `Author`. The reverse side (`author.posts`) is a query, so you can chain `.where()` and `.order_by()` before awaiting it. -- **Forward** (`post.author`): awaiting the foreign key attribute loads the related `Author`. -- **Reverse** (`author.posts`): the `BackRef` is a query, so you can chain `.where()`, `.order_by()`, and friends before awaiting it. +## Update and delete -## Update & Delete +Change one post with `save()`, publish every draft with `update()` and remove a post with `delete()`: ```python --8<-- "docs/examples/quickstart.py:update-delete" ``` -- For a single instance: mutate attributes, then `await post.save()`. `save()` UPDATEs an instance that came from the database and INSERTs one that never did — see [Mutations](../guide/mutations.md#saving-insert-or-update). -- For many rows: chain `.update(field=value)` or `.delete()` onto a `where()` query. Both return the number of affected rows. +`save()` updates an instance that came from the database and inserts one that did not; see [Mutations](../guide/mutations.md#saving-insert-or-update). Chained `.update(field=value)` and `.delete()` act on every matching row and return the number of rows they changed, here `1` each. -## Wrap It in a Transaction +## Transactions + +Create an author and their post together: ```python --8<-- "docs/examples/quickstart.py:transaction" ``` -Everything inside `async with transaction():` commits together when the block exits cleanly — and rolls back entirely if it raises. Use it whenever multiple writes must succeed or fail as one. More in [Transactions](../guide/transactions.md). +Everything inside `async with transaction():` commits when the block exits cleanly and rolls back if it raises. Use it whenever several writes must succeed or fail together. See [Transactions](../guide/transactions.md). -## Complete Script +## Complete script -The whole tutorial as one runnable file — it lives in the repo at `docs/examples/quickstart.py`: +The whole tutorial as one runnable file, also at `docs/examples/quickstart.py` in the repository: ```python --8<-- "docs/examples/quickstart.py" ``` -## What's Next +Running it prints `quickstart example ran successfully`. + +## Next steps + +Pick a path by what you are building. + +For an API with FastAPI, Starlette or similar: + +1. [Models and fields](../guide/models-and-fields.md) covers field types, constraints and defaults. +2. [Relationships](../guide/relationships.md) covers foreign keys, back-references and many-to-many. +3. [Queries](../guide/queries.md) covers filtering, ordering and slicing. +4. [Pagination](../howto/pagination.md) shows offset and cursor pagination. +5. [Testing](../howto/testing.md) shows fast, isolated tests on in-memory SQLite. + +For data processing and bulk loads: + +1. [Mutations](../guide/mutations.md) covers `bulk_create` and bulk update and delete. +2. [Transactions](../guide/transactions.md) makes multi-step writes atomic. +3. [Queries](../guide/queries.md) shows how to filter in the database instead of in Python. + +For shipping to production: + +1. [Connections](../guide/connections.md) covers connection URLs and pool settings. +2. [Schema management](../guide/schema/overview.md) and [Deploying migrations](../guide/schema/deploying.md) cover evolving a production schema. +3. [CLI reference](../reference/cli.md) and [Configuration](../reference/configuration.md) list the `ferro` commands and settings. +4. [Multiple databases](../howto/multiple-databases.md) covers more than one database. -- [Next Steps](next-steps.md) — pick a path based on what you're building -- [Models & Fields](../guide/models-and-fields.md) — every field type and constraint -- [Queries](../guide/queries.md) — the lambda predicate style in depth, ordering, slicing, terminals -- [Relationships](../guide/relationships.md) — foreign keys, back-references, many-to-many +More recipes are under [How-to guides](../howto/testing.md), every public class and method is in the [API reference](../api/model.md), and [Releases on GitHub](https://github.com/syn54x/ferro-orm/releases) lists new versions. diff --git a/docs/pages/guide/aggregations.md b/docs/pages/guide/aggregations.md index dbf29b42..299a1baa 100644 --- a/docs/pages/guide/aggregations.md +++ b/docs/pages/guide/aggregations.md @@ -1,8 +1,34 @@ -# Aggregations & Grouped Queries +--- +description: "Count, total and average rows with count(), sum(), avg(), min() and max(), and group them by plain fields." +--- -Ferro aggregates with five methods on the columns themselves — `count()`, `sum()`, `avg()`, `min()`, `max()` — inside the same `select()` lambda that projects columns. There is no `group_by()` chainer: **the record shape is the grouping**. Bare fields are the keys; aggregate fields are the measures. +# Aggregations and grouped queries -The examples on this page use this schema: +This guide shows how to total, count and average rows with the five aggregate methods (`count()`, `sum()`, `avg()`, `min()` and `max()`) inside a `select()` lambda, and how a record that mixes plain fields with aggregates becomes a grouped query. The full signatures are in the [aggregates reference](../api/queries.md#aggregates-and-grouped-queries). + +Here is a program that connects, opens a session, adds the four transactions of [the ledger](#the-ledger) and asks for their count, total, average, smallest and largest amount in one query: + +```python +--8<-- "docs/examples/aggregations.py:global" +``` + +The single record holds five numbers over the four rows: a count of 4, a total of 115, an average of 28.75, a smallest amount of 10 and a largest of 45. Every query on this page runs inside a session like this one; see [Sessions](connections.md#sessions) for why and how to open one. + +Every example on this page is followed by the SQL it sends, captured from Ferro when the example runs. `first()` sends this statement; on PostgreSQL only the placeholders differ, `$1` where SQLite has `?`: + +```sql +-- aggregations.py:global (sqlite) +SELECT COUNT("transaction"."id") AS "n", SUM("transaction"."amount") AS "total", AVG("transaction"."amount") AS "average", MIN("transaction"."amount") AS "smallest", MAX("transaction"."amount") AS "largest" +FROM "transaction" +WHERE TRUE +LIMIT ?; +``` + +Each dict key names an aggregate with `AS`, and `first()` adds a `LIMIT` of 1. + +## The ledger + +Every example on this page queries one small ledger. An account holds transactions, and a transaction may have no account: === "Assignment" @@ -16,147 +42,355 @@ The examples on this page use this schema: --8<-- "docs/examples/aggregations_annotated.py:schema" ``` -## Global Aggregates +The ledger holds these four transactions: -An aggregate is a method on a column reference: `t.amount.sum()`. Aggregate fields are **user-named** — they live in the dict selector form, where the key names the output field. A projection containing *only* aggregates collapses the whole result to exactly one record, read idiomatically with `first()`: +| `id` | `memo` | `amount` | `price` | account | +| ---: | :--- | ---: | ---: | :--- | +| 1 | coffee | 10 | 1.25 | Checking | +| 2 | lunch | 20 | 2.75 | Checking | +| 3 | dinner | 45 | 5.00 | Savings | +| 4 | orphan | 40 | | | -```python ---8<-- "docs/examples/aggregations.py:global" -``` +Checking totals 30, Savings totals 45, and the orphan has no account and no price. + +## Global aggregates + +An aggregate is a method on a column reference, such as `transaction.amount.sum()`. Aggregate fields are named by you, so they live in the dict form of `select()`, where each key names an output field. A projection of only aggregates collapses the result to exactly one record, which you read with `first()`, as the first example did. -Aggregates measure whatever `where()` leaves in — relation traversal included: +Aggregates measure whatever `where()` leaves in, relation traversal included: ```python --8<-- "docs/examples/aggregations.py:global-where" ``` -The source column may itself traverse (`t.account.balance.avg()`): the traversal narrows to rows where the relation exists, exactly like a `where()` predicate on the same path, and shares that path's join with any other clause that references it. +Only the two Checking transactions are measured, so the total is 30. The source column can traverse as well (`transaction.account.label.min()`): the traversal keeps rows where the relation exists, the way a `where()` on the same path does, and shares that path's join with any other clause that uses it. + +The query sends this; on PostgreSQL only the placeholders differ: + +```sql +-- aggregations.py:global-where (sqlite) +SELECT SUM("transaction"."amount") AS "total" +FROM "transaction" +INNER JOIN "account" AS "j1_account" ON "transaction"."account_id" = "j1_account"."id" +WHERE "j1_account"."label" = ? +LIMIT ?; +``` -### What each aggregate returns +The traversal is an `INNER JOIN` on `account`, so `SUM` measures only the joined rows. -Result types are a **pinned cross-backend contract**, derived from the source column's Python type — the same query returns the same Python types on SQLite and Postgres, even where the databases themselves disagree (Postgres widens `SUM(bigint)` to `numeric`; SQLite averages everything as a float): +### Result types + +The result type comes from the source column's Python type, so a query returns the same Python types on SQLite and Postgres, even where the databases differ (Postgres widens `SUM(bigint)` to `numeric`; SQLite averages everything as a float): | Aggregate | Source column | Result type | | :--- | :--- | :--- | -| `count()` | any column | `int` — never `None` | +| `count()` | any column | `int`, never `None` | | `sum()` | `int` / `float` / `Decimal` | the source numeric type | | `avg()` | `int` / `float` | `float` | -| `avg()` | `Decimal` | `Decimal` — never a silently lossy `float` | -| `min()` / `max()` | numeric, `str`, `datetime` / `date` / `time` | the source type, via the source codec | +| `avg()` | `Decimal` | `Decimal`, never a lossy `float` | +| `min()` / `max()` | numeric, `str`, `datetime` / `date` / `time` | the source type | + +For example, the average of the three prices stays a `Decimal`: ```python --8<-- "docs/examples/aggregations.py:decimal-contract" ``` -Every aggregate result except `count` is `T | None`, because of what comes next. +The prices total 9.00 over three rows, so the average is `Decimal("3")`. Every aggregate result except `count()` can be `None`, as the next section shows. + +The query sends this; on PostgreSQL only the placeholders differ: + +```sql +-- aggregations.py:decimal-contract (sqlite) +SELECT AVG("transaction"."price") AS "avg_price" +FROM "transaction" +WHERE "transaction"."price" IS NOT NULL +LIMIT ?; +``` + +SQL's `AVG` returns a float on SQLite and a `numeric` on PostgreSQL; Ferro reads either back as a `Decimal`, because the source column is one. -### Empty input: `None` and `0`, no COALESCE +### Empty input -SQL answers an aggregate over zero rows with `NULL` (`COUNT` with `0`), and Ferro passes that through verbatim — "sum of no rows" and "sum of rows totaling zero" are different facts, and zero would be the wrong identity for `min`/`max` anyway: +SQL answers an aggregate over zero rows with `NULL` (`COUNT` answers `0`), and Ferro passes that through. "Sum of no rows" and "sum of rows that total zero" are different facts, and zero would be the wrong answer for `min()` and `max()` anyway: ```python --8<-- "docs/examples/aggregations.py:empty" ``` -### `count()` counts non-NULL values +No transaction is over 10,000, so the query still returns one record: `n` is `0` and `total` is `None`. -`t.price.count()` is SQL's `COUNT(price)`: it counts rows where *that column* is non-NULL. Count rows regardless of any column with the primary key (`t.id.count()`): +The query sends this; on PostgreSQL only the placeholders differ: + +```sql +-- aggregations.py:empty (sqlite) +SELECT COUNT("transaction"."id") AS "n", SUM("transaction"."amount") AS "total" +FROM "transaction" +WHERE "transaction"."amount" > ? +LIMIT ?; +``` + +With no `GROUP BY`, SQL answers an aggregate-only `SELECT` with exactly one row, even over zero rows. + +### Counting a column + +`transaction.price.count()` is SQL's `COUNT(price)`: it counts the rows where that column is not `NULL`. To count rows whatever else they hold, count the primary key (`transaction.id.count()`): ```python --8<-- "docs/examples/aggregations.py:count-nulls" ``` -## Grouped Queries: Bare Fields Are the Keys +All four transactions are counted by `id`, and three by `price`, because the orphan has no price. + +The query sends this; on PostgreSQL only the placeholders differ: -Mix a plain field into an aggregate projection and the query becomes **grouped** — every non-aggregate field is a group key, and each group collapses to exactly one record: +```sql +-- aggregations.py:count-nulls (sqlite) +SELECT COUNT("transaction"."id") AS "rows", COUNT("transaction"."price") AS "priced" +FROM "transaction" +WHERE TRUE +LIMIT ?; +``` + +`COUNT("transaction"."price")` skips the `NULL` price; `COUNT("transaction"."id")` counts every row. + +## Grouped queries + +Mix a plain field into an aggregate projection and the query is grouped. Every plain field is a group key, and each group collapses to one record: ```python --8<-- "docs/examples/aggregations.py:grouped" ``` -This is SQL's own rule, made unwritable to violate: SQL already requires every bare selected column to be a group key, so declaring the grouping separately could only ever restate the projection or contradict it (Postgres rejects the contradiction at runtime; SQLite silently answers with an arbitrary row's value). Deriving `GROUP BY` from the record shape deletes that entire error class — the dict literal is the whole story. +Account 1 totals 30 and account 2 totals 45, one record each. There is no `group_by()`: the record shape is the grouping. SQL already requires every selected bare column to be a group key, so a separate `group_by()` could only repeat the projection or contradict it. Postgres rejects the contradiction at run time, and SQLite answers with an arbitrary row's value. Deriving the grouping from the projection removes that whole class of error. + +The query sends this; on PostgreSQL only the placeholders differ: + +```sql +-- aggregations.py:grouped (sqlite) +SELECT "transaction"."account_id" AS "acct", SUM("transaction"."amount") AS "total" +FROM "transaction" +WHERE "transaction"."account_id" IS NOT NULL +GROUP BY "transaction"."account_id" +ORDER BY "acct" ASC NULLS LAST; +``` -Group keys may traverse, and traversal narrows exactly like a predicate: +The plain field became the `GROUP BY` key, and `order_by("acct")` sorts by its output name. + +Group keys can traverse, and traversal narrows rows the way a predicate does: ```python --8<-- "docs/examples/aggregations.py:grouped-traversed" ``` -### "Has no relation" is a visible bucket +The orphan has no account, so it drops out and the two labelled groups remain. + +The query sends this; on PostgreSQL only the placeholders differ: + +```sql +-- aggregations.py:grouped-traversed (sqlite) +SELECT "j1_account"."label" AS "account_label", SUM("transaction"."amount") AS "total" +FROM "transaction" +INNER JOIN "account" AS "j1_account" ON "transaction"."account_id" = "j1_account"."id" +WHERE TRUE +GROUP BY "j1_account"."label" +ORDER BY "account_label" ASC NULLS LAST; +``` + +The `INNER JOIN` drops the orphan before grouping, and the joined `label` is the group key. + +### The `None` group for missing relations -Add `left_join()` and relation-less rows stay in — grouped under a `None` key instead of silently dropped: +Add `left_join()` and rows without a relation stay in, grouped under a `None` key instead of being dropped: ```python --8<-- "docs/examples/aggregations.py:none-bucket" ``` -### Zero rows, zero groups +The orphan's 40 now appears as its own group. -A grouped query over zero matching rows returns zero records — unlike a global aggregate, there is no group to report on: +The query sends this; on PostgreSQL only the placeholders differ: + +```sql +-- aggregations.py:none-bucket (sqlite) +SELECT "j1_account"."label" AS "account_label", SUM("transaction"."amount") AS "total" +FROM "transaction" +LEFT JOIN "account" AS "j1_account" ON "transaction"."account_id" = "j1_account"."id" +WHERE TRUE +GROUP BY "j1_account"."label"; +``` + +The `LEFT JOIN` keeps the orphan, whose `label` is `NULL`, so it forms the `None` group. + +### Grouped queries over zero rows + +A grouped query over zero matching rows returns zero records. Unlike a global aggregate, there is no group to report on: ```python --8<-- "docs/examples/aggregations.py:zero-groups" ``` -### Grouping is not partitioning +No transaction is over 10,000, so `rows` is empty. The query sends this; on PostgreSQL only the placeholders differ: + +```sql +-- aggregations.py:zero-groups (sqlite) +SELECT "transaction"."account_id" AS "acct", SUM("transaction"."amount") AS "total" +FROM "transaction" +WHERE "transaction"."amount" > ? +GROUP BY "transaction"."account_id"; +``` + +With a `GROUP BY` and no rows, SQL returns no groups at all. -Grouping **collapses** rows: each group becomes one projected record of keys and measures, and the individual rows are gone. If you want complete model instances *bucketed* by a key — every transaction, arranged per account — that is a different, client-side operation over a full query: +### Grouping and partitioning + +Grouping collapses rows: each group becomes one record of keys and measures, and the individual rows are gone. If you want whole model instances bucketed by a key, every transaction arranged per account, do that in Python over a full query: ```python -from collections import defaultdict +--8<-- "docs/examples/aggregations.py:partition" +``` -by_account: dict[int | None, list[Transaction]] = defaultdict(list) -for txn in await Transaction.select().all(): - by_account[txn.account_id].append(txn) +Account 1 holds two transactions, account 2 holds one, and the orphan sits under `None`. The query sends this; on PostgreSQL only the placeholders differ: + +```sql +-- aggregations.py:partition (sqlite) +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."id", "transaction"."memo", "transaction"."price" +FROM "transaction" +WHERE TRUE; ``` -Reach for aggregation when you want *facts about groups*; partition in Python when you want *the rows themselves, organized*. +It is a plain `SELECT` of whole rows, with no `GROUP BY`: the bucketing happens in Python. Reach for aggregation when you want facts about groups, and partition in Python when you want the rows themselves, organized. + +## Ordering grouped results -## Ordering Grouped Results +`order_by()` on a projected query follows SQL's own ORDER BY scoping. Three rules apply. -`order_by` on a projected query follows SQL's own ORDER BY scoping, pinned as three rules. +### Sorting by an output name -**Strings resolve output field names first, then root columns.** `order_by("total")` sorts by the aggregate you named `total`, even if the model happens to have a column with the same name: +A string resolves output field names first, then root columns. `order_by("total")` sorts by the aggregate you named `total`, even if the model has a column of the same name. Combined with `limit()`, this is the top-N idiom: keys plus aggregates, ordered by the measure's output name, with `limit()` applied to the groups: ```python --8<-- "docs/examples/aggregations.py:top-n" ``` -That is the **top-N idiom** — keys plus aggregates, ordered by the measure's output name, `limit()` applied to the *groups*. `limit()`/`offset()` always act on groups in a grouped query, so pagination pages through the summary, not the underlying rows. +Savings has the largest total, so it is the one group returned. `limit()` and `offset()` always act on groups in a grouped query, so pagination pages through the summary, not the underlying rows. -**The lambda form spells source expressions — aggregates included.** Where strings name outputs, lambdas write the expression itself: +The query sends this; on PostgreSQL only the placeholders differ: + +```sql +-- aggregations.py:top-n (sqlite) +SELECT "j1_account"."label" AS "account_label", SUM("transaction"."amount") AS "total" +FROM "transaction" +INNER JOIN "account" AS "j1_account" ON "transaction"."account_id" = "j1_account"."id" +WHERE TRUE +GROUP BY "j1_account"."label" +ORDER BY "total" DESC NULLS LAST +LIMIT ?; +``` + +`ORDER BY "total"` names the output field, and `LIMIT` applies to the groups. + +### Sorting by an aggregate expression + +Where strings name outputs, a lambda writes the source expression itself, aggregates included: ```python --8<-- "docs/examples/aggregations.py:order-by-lambda" ``` -An aggregate sort key must match a projected aggregate field (same function, same column, same path); an expression you did not project is a build-time error telling you to name it in the projection. +The groups come back with the largest total first. An aggregate sort key must match a projected aggregate (same function, same column, same path). An expression you did not project raises when you build the query and tells you to name it in the projection. + +The query sends this; on PostgreSQL only the placeholders differ: + +```sql +-- aggregations.py:order-by-lambda (sqlite) +SELECT "transaction"."account_id" AS "acct", SUM("transaction"."amount") AS "total" +FROM "transaction" +WHERE "transaction"."account_id" IS NOT NULL +GROUP BY "transaction"."account_id" +ORDER BY "total" DESC NULLS LAST; +``` + +The lambda's `transaction.amount.sum()` matched the projected aggregate, so the sort reads its output name, `"total"`. + +### Allowed sort keys + +On an aggregate projection, every sort key must be a group key or an aggregate. Sorting groups by any other column is the arbitrary-row trap again, so Ferro rejects it when you build the query, whether the key is a string, a lambda, or was chained before the `select()`. On a plain, non-aggregate projection nothing changes: unselected root columns still sort. -**On an aggregate projection, every sort key must be a group key or an aggregate.** Sorting groups by a column that is neither is the arbitrary-row trap again — SQLite would happily pick *some* row's value per group — so Ferro rejects it when you build the query, whether the key is a string, a lambda, or was chained before the `select()`: +## Unsupported aggregate forms +Each form below fails when you build the query, before any SQL is sent, with an error that names the fix. Each example runs inside a session and raises if the expected error does not appear. + +`sum()` and `avg()` take numeric columns (`int`, `float`, `Decimal`), `min()` and `max()` take orderable ones (numeric, text, date and time), and `count()` takes anything. Enum, UUID, JSON and `bool` columns are rejected, because `min()` over a UUID does not exist on Postgres and `max()` over a native enum differs between backends (definition order against lexical order): + +```python +--8<-- "docs/examples/aggregations.py:error-family" +``` + + ```text ->>> Transaction.select(lambda t: {"acct": t.account_id, "total": t.amount.sum()}).order_by("memo") -ValueError: order_by('memo') on an aggregate projection must name a group key or an aggregate: ... +TypeError: t.memo.sum() is not supported: Transaction.memo is string-typed, and sum() takes a numeric column (int, float, or Decimal). No portable cross-backend meaning exists for this aggregate over a string column. ``` -On a plain (non-aggregate) projection nothing changes: unselected root columns still sort, as they always have. +`where()` filters rows before aggregation, so it cannot hold an aggregate. The error points at `having()`, the post-aggregation filter, which does not exist yet. Until it does, filter rows with `where()` and compare the aggregated results in Python: -## The Loud Limits +```python +--8<-- "docs/examples/aggregations.py:error-where" +``` -Everything below fails **when you build the query** — before any SQL, with an error that names the fix: + +```text +TypeError: t.amount.sum() > ... is not a where() predicate: WHERE filters rows before aggregation, so an aggregate cannot appear in it. Post-aggregation filtering is having() (#291); until it lands, filter rows with where() and compare the aggregated result in Python. +``` -- **Source families with no portable meaning.** `sum()`/`avg()` take numeric columns (`int`, `float`, `Decimal`); `min()`/`max()` take orderable ones (numeric, text, date/time); `count()` takes anything. Enum, UUID, JSON, and bool columns are rejected — `min()` over a UUID does not exist on Postgres, and `max()` over a native enum silently diverges between backends (definition order vs. lexical order). Where no portable meaning exists, build time is the only honest place to fail. -- **No aggregates in `where()`.** `WHERE` filters rows *before* aggregation, so `where(lambda t: t.amount.sum() > 100)` raises pointing at `having()` — the post-aggregation filter, tracked in [#291](https://github.com/syn54x/ferro-orm/issues/291). Until it lands, filter rows with `where()` and compare aggregated results in Python. -- **The builtin-`sum` trap.** `sum(t.amount)` (Python's builtin over a column reference) raises `did you mean t.amount.sum()?` instead of failing obscurely. -- **Aggregates are user-named.** An aggregate outside the dict form (`select(lambda t: t.amount.sum())`) raises: give it an output name. -- **`count()`/`exists()` on an aggregate projection.** "Count" is ambiguous between rows and groups, so both raise with both spellings: count matching rows with an unprojected query (`Transaction.where(...).count()`), count groups with `len(await q.all())`. +Python's builtin `sum()` over a column reference raises instead of failing obscurely. The error text writes the lambda parameter as `t`, whatever you named it: ```python ---8<-- "docs/examples/aggregations.py:errors" +--8<-- "docs/examples/aggregations.py:error-builtin-sum" ``` -## See Also + +```text +TypeError: t.amount is a column reference and cannot be iterated; did you mean t.amount.sum()? Aggregates are methods on the column (.count()/.sum()/.avg()/.min()/.max()). +``` + +An aggregate outside the dict form has no output name, so it raises: + +```python +--8<-- "docs/examples/aggregations.py:error-unnamed" +``` + + +```text +TypeError: aggregate fields are user-named (#294): give t.amount.sum() an output name with the dict selector form (e.g. select(lambda t: {"total": t.amount.sum()})). +``` + +Sorting groups by a column that is not a group key raises, as the ordering rules above say: + +```python +--8<-- "docs/examples/aggregations.py:error-order-key" +``` + + +```text +ValueError: order_by('memo') on an aggregate projection must name a group key or an aggregate: it is neither an output field nor a group key's source column, so each group would answer with an arbitrary row's value. Sort by an output field name, or project the column as a group key. +``` + +`count()` and `exists()` on an aggregate projection are ambiguous between rows and groups, so both raise and name both spellings. Count matching rows with an unprojected query (`Transaction.where(...).count()`), and count groups with `len(await query.all())`: + +```python +--8<-- "docs/examples/aggregations.py:error-count" +``` + + +```text +ValueError: count() on an aggregate projection is ambiguous: count the matching rows with an unprojected query (Model.where(...).count()), or count the groups with len(await q.all()). +``` + +For the full list of aggregate forms and their result types, see [Aggregates and grouped queries](../api/queries.md#aggregates-and-grouped-queries) in the reference. + +## See also -- [Selecting a Column Subset](queries.md#selecting-a-column-subset) — projections, traversed fields, and output aliases, which aggregations build on -- [Querying Across Relationships](queries.md#querying-across-relationships) — traversal and join semantics (ADR-0006) -- [Typed Query Predicates](../concepts/query-typing.md#projected-queries) — how aggregate expressions type +- [Selecting a column subset](queries.md#selecting-a-column-subset): projections, traversed fields and output aliases, which aggregations build on. +- [Querying across relationships](queries.md#querying-across-relationships): traversal and join semantics. +- [Typed query predicates](../concepts/query-typing.md#projected-queries): how aggregate expressions are typed. diff --git a/docs/pages/guide/connections.md b/docs/pages/guide/connections.md index 8ae2bebe..f3844651 100644 --- a/docs/pages/guide/connections.md +++ b/docs/pages/guide/connections.md @@ -1,226 +1,326 @@ -# Connections & Databases +--- +description: "Connect Ferro to SQLite or PostgreSQL, configure the connection pool and run every ORM call inside a session." +--- -Ferro talks to SQLite and PostgreSQL through connection pools managed by the Rust core (SQLx). Connect once at application startup with `await ferro.connect(url, ...)` before performing any database operation. +# Connections + +Ferro reaches SQLite and PostgreSQL through connection pools: you connect once at startup, then run every ORM call inside a session. + +Connect to an in-memory SQLite database, open a session and run a query: + +```python {test=run prelude=connections} +from ferro import connect, engines + +await connect("sqlite::memory:", auto_migrate=True) + +async with engines.session(): + await Author.create(name="Ada Lovelace", email="ada@example.com") + print(await Author.select().count()) +``` + +It prints `1`. `auto_migrate=True` created the `author` table as it connected, which suits development and tests. + +The examples on this page use the quickstart's models: + +=== "Assignment" + + ```python + --8<-- "docs/examples/quickstart.py:models" + ``` + +=== "Annotated" + + ```python + --8<-- "docs/examples/quickstart_annotated.py:models" + ``` ## Connecting +`connect(url)` opens a pool for the database the URL names and registers it as the default connection. + ### SQLite -```python -import ferro +Pass a file path, or `:memory:` for a database that lives as long as the process: -# File database — `mode=rwc` creates the file if it doesn't exist (recommended) -await ferro.connect("sqlite:app.db?mode=rwc") +```python {test=compile reason="creates app.db in the working directory"} +from ferro import connect -# In-memory database (great for tests) -await ferro.connect("sqlite::memory:") +await connect("sqlite:app.db?mode=rwc") ``` -Modes: `rwc` (read/write/create), `rw` (read/write, file must exist), `ro` (read-only). +`mode=rwc` reads, writes and creates the file if it does not exist. `mode=rw` needs the file to exist, and `mode=ro` opens it read-only. `sqlite::memory:` gives an in-memory database, for tests. ### PostgreSQL -```python -import ferro +Pass a `postgresql://` URL; add `sslmode=require` to require TLS: -await ferro.connect("postgresql://user:password@localhost:5432/dbname") +```python {test=compile reason="needs a TLS-enabled server"} +from ferro import connect -# Require TLS -await ferro.connect("postgresql://user:password@localhost:5432/dbname?sslmode=require") +await connect("postgresql://user:password@localhost:5432/app?sslmode=require") ``` -Load credentials from the environment rather than hard-coding them, and percent-encode reserved characters in passwords (`%24` for `$`, etc.) if you assemble URLs yourself. +The pool connects over TLS. Load credentials from the environment rather than writing them into code, and percent-encode reserved characters in a password (`%24` for `$`) if you build the URL yourself. ### Supabase -Supabase hosts PostgreSQL behind TLS; Ferro's driver stack ships with Rustls and the webpki CA bundle, so published wheels connect out of the box. Copy the exact URI from **Project Settings → Database** in the Supabase dashboard (pooler hostnames look like `*.pooler.supabase.com`, and the username may include the project ref), then ensure TLS is requested: +Supabase hosts PostgreSQL behind TLS, and Ferro's published wheels ship with Rustls and the webpki CA bundle, so they connect without extra setup. Copy the URI from **Project Settings → Database** in the Supabase dashboard (pooler hostnames look like `*.pooler.supabase.com`, and the username may include the project ref), then make sure it requests TLS: -```python +```python {test=compile reason="needs a Supabase DATABASE_URL"} import os -import ferro +from ferro import connect url = os.environ["DATABASE_URL"] if "sslmode=" not in url: - sep = "&" if "?" in url else "?" - url = f"{url}{sep}sslmode=require" + separator = "&" if "?" in url else "?" + url = f"{url}{separator}sslmode=require" -await ferro.connect(url) +await connect(url) ``` -Keep service-role credentials server-side, and never make an elevated connection the default in user-facing runtimes. - -## Connection Options - -`connect()` accepts: +The connection now refuses to fall back to plain text. Keep service-role credentials on the server, and never make an elevated connection the default in a user-facing runtime. -| Parameter | Default | Description | -| :--- | :--- | :--- | -| `url` | required | Database connection string. | -| `auto_migrate` | `False` | Create tables for all registered models; existing tables are left untouched. | -| `name` | `None` | Connection name. Omitted connections register as `"default"`. | -| `default` | `False` | Make this named connection the default for unqualified operations. | -| `pool` | `None` | A [`PoolConfig`](#connection-pooling); defaults apply when omitted. | -| `identity_map` | `True` | Keep a per-connection [identity map](../concepts/identity-map.md) so one primary key maps to one Python instance. `False` trades the `a is b` guarantee for lower memory use. | -| `migrate_updates` | `False` | Also `ALTER` existing tables to match the models (implies `auto_migrate`). See [Auto-migrate](schema/auto-migrate.md#applying-column-changes-with-migrate_updates). *Added in 0.11.0.* | -| `migrate_destructive` | `False` | Also drop live columns removed from the models (implies `migrate_updates`). See [Auto-migrate](schema/auto-migrate.md#destructive-drops-with-migrate_destructive). *Added in 0.11.0.* | +## Sessions -## Connection Pooling +Every ORM and raw SQL call runs in a session. A session is one unit of work: it picks the connection the calls use and keeps an [identity map](../concepts/identity-map.md), so loading the same row twice gives you the same Python object. Open one with `async with engines.session():`: -Size each connection's pool with `PoolConfig`: +```python {test=run prelude=connections} +from ferro import connect, engines -```python -import ferro +await connect("sqlite::memory:", auto_migrate=True) -await ferro.connect( - "postgresql://localhost/app", - pool=ferro.PoolConfig(max_connections=10, min_connections=1), -) +async with engines.session(): + ada = await Author.create(name="Ada Lovelace", email="ada@example.com") + same = await Author.where(lambda author: author.email == "ada@example.com").first() + print(same is ada) ``` -`max_connections` defaults to 5, `min_connections` to 0, and `min_connections` may not exceed `max_connections`. For web applications, connect once at startup and let the long-lived pool serve all requests. +It prints `True`: inside the session, one row maps to one instance. With no name, `engines.session()` opens a session on the [default connection](#the-default-connection). -## Named Connections +A call outside any session has no connection to run on, and raises: -Ferro can hold multiple live pools in one process — separate databases, roles, or pool sizes: +```python {test=run prelude=connections} +from ferro import connect -```python ---8<-- "docs/examples/multiple_databases.py:connect" +await connect("sqlite::memory:", auto_migrate=True) + +try: + await Author.select().count() +except RuntimeError as error: + print(f"{type(error).__name__}: {error}") +else: + raise AssertionError("expected RuntimeError") ``` -Route individual operations with `Model.using("name")`, which exposes the full ORM surface (`create`, `get`, `where`, `bulk_create`, `get_or_create`, ...) bound to that connection: +It prints: -```python ---8<-- "docs/examples/multiple_databases.py:routing" + +```text +RuntimeError: No database route for this operation. Open a session (`async with ferro.engines.session(...)`) or pass `using=`/`session=` explicitly. Implicit default-connection routing was removed in v0.14 (deprecated since v0.12); see the upgrade guide. ``` -Raw SQL routes the same way via `using=`: `await ferro.execute("...", using="analytics")`. Transactions opened with `transaction(using="analytics")` pin everything inside to that connection — see [Transactions](transactions.md#using-named-connections). +The only other route is an explicit one: `Author.using("name")`, or `using=` / `session=` on a call. Explicit routes are covered under [Named connections](#named-connections). -!!! warning "Connection names are code, not input" - Choose `using` values from constants or trusted server-side logic. Never bind connection names from request parameters, headers, or other untrusted input. - -Ferro does not provide automatic router policies, read/write splitting, distributed transactions, or cross-connection joins — route each operation explicitly. See [Multiple Databases](../howto/multiple-databases.md) for fuller patterns. +### A session per request -## Sessions (recommended) +For a short unit of work, such as an HTTP handler, a script or a test, open a session for its length: -Session-scoped routing is now the preferred runtime model: +```python {test=run prelude=connections} +from ferro import connect, engines -```python -import ferro +await connect("sqlite::memory:", auto_migrate=True) -async with ferro.engines.session("analytics") as s: - rows = await s.query(User).where(lambda t: t.active == True).all() # noqa: E712 -``` -Inside an active session context, convenience APIs (`User.all()`, `User.where(...)`, `ferro.execute(...)`) automatically bind to that session's connection. +async def list_authors() -> list[str]: + async with engines.session(): + authors = await Author.select().order_by(lambda author: author.name).all() + return [author.name for author in authors] -### Request-scoped sessions -For short units of work (HTTP handlers, scripts, tests), use `async with`: +async with engines.session(): + await Author.create(name="Grace Hopper", email="grace@example.com") + await Author.create(name="Ada Lovelace", email="ada@example.com") -```python -async with ferro.engines.session("analytics"): - rows = await User.where(lambda t: t.active == True).all() # noqa: E712 +print(await list_authors()) ``` -### Long-lived sessions (GUI, app lifecycle) +It prints `['Ada Lovelace', 'Grace Hopper']`. Each request gets its own session, and with it its own identity map. + +### A session for the app's lifetime + +When startup and shutdown run in different asyncio tasks, as with Textual's `on_mount` and `on_unmount`, open a session handle at startup and close it at shutdown: + +```python {test=run prelude=connections} +from ferro import connect, engines -When startup and shutdown run in different asyncio tasks (for example Textual `on_mount` / `on_unmount`), open a session handle at startup and close it explicitly at shutdown: -```python class App: async def on_mount(self) -> None: - self.session = ferro.engines.session("default") + await connect("sqlite::memory:", auto_migrate=True) + self.session = engines.session() await self.session.__aenter__() async def on_unmount(self) -> None: await self.session.close() + + +app = App() +await app.on_mount() +await Author.create(name="Ada Lovelace", email="ada@example.com") +print(await Author.select().count()) +await app.on_unmount() ``` -`Session.close()` is safe across asyncio contexts and idempotent. Prefer explicit `session=` routing when ambient context may be unavailable in the closing task. +It prints `1`: calls between `on_mount` and `on_unmount` run in the app's session. `Session.close()` works from any asyncio task, and closing twice does nothing. When the closing task may not see the session, pass it explicitly with `session=`. -Close raises `RuntimeError` if session-scoped transactions are still open — exit all `transaction()` blocks before closing the session. +Rules for closing a session: -After a cross-context close, the enter task may still hold a stale ambient session object until a new session is opened; ambient operations then raise `RuntimeError` with a session-closed message (including when `using=` names another connection). If nested sessions are open, close inner sessions from the same task or via cross-context `close()` before closing the outer handle in the original task. +- `close()` raises `RuntimeError` while a `transaction()` in the session is still open. Leave every `transaction()` block first. +- A call on a closed session, ambient or passed as `session=`, raises `RuntimeError`. It never falls back to another route. +- If you close a session from another task, the task that opened it may still hold it as its ambient session until it opens a new one. Calls there raise `RuntimeError` saying the session is closed. +- With nested sessions, close the inner ones before the outer one. -Operations on a closed session handle (ambient or explicit `session=`) raise `RuntimeError` rather than falling back to legacy default routing. +`reset_engine()` tears down every connection. It exists for test isolation, not for closing one session. -`ferro.reset_engine()` tears down **all** connections and is intended for test isolation — not per-session app lifecycle teardown. +The [`Session`](../api/connection.md#ferro.Session) and [`engines`](../api/connection.md#ferro.engines) reference entries list every argument, including `settings=` for [row-level security](../concepts/row-level-security.md#session-settings). -Legacy implicit default-connection routing (calling unqualified operations outside a session) was removed in `0.13`; such calls now raise `RuntimeError`. -See [Upgrade Guide: Connection routing requires a session](../howto/upgrade-guide.md#connection-routing-requires-a-session) to remove these legacy call sites. +!!! note "Changed in 0.14" + Calls outside a session used to run on the default connection; they now raise `RuntimeError`. The [upgrade guide](../upgrading.md#connection-routing-requires-a-session) shows how to move old call sites into sessions. -## The Default Connection +## Connection options -Unqualified operations (`User.all()`, top-level `execute(...)`) use the default connection. It is established three ways: +Beyond the URL, `connect()` takes the options you will reach for most: -1. `connect(url)` without a `name` registers as `"default"`. -2. `connect(url, name="app", default=True)` makes a named connection the default. -3. `ferro.set_default_connection("app")` switches the default at runtime. +- `auto_migrate=True` creates the missing tables for every registered model; existing tables are left alone. `migrate_updates` and `migrate_destructive` also change existing tables; see [Auto-migrate](schema/auto-migrate.md). The three flags run under the same run lock as `ferro migrate up`, are refused on a database that migrations govern, and wait for locks under the project's `lock_timeout` and `ddl_lock_timeout` ([Concurrent passes and the run lock](schema/auto-migrate.md#concurrent-passes-and-the-run-lock)). +- `name="analytics"` registers a [named connection](#named-connections). Without a name, the connection registers as `"default"`. +- `identity_map=False` turns off the identity map for sessions on this connection, trading the `a is b` guarantee for lower memory use. +- `pool=PoolConfig(...)` sizes the pool, below. + +The [`connect`](../api/connection.md#ferro.connect) reference lists every option and its default. + +### Pool size -```python -import ferro +Size a connection's pool with `PoolConfig`: -await ferro.connect("sqlite:app.db?mode=rwc", name="app", default=True) -await ferro.connect("sqlite:analytics.db?mode=rwc", name="analytics") +```python {test=run prelude=connections backend=postgres} +from ferro import PoolConfig, connect -ferro.set_default_connection("analytics") # unqualified ops now hit analytics +await connect( + "postgresql://localhost/app", + pool=PoolConfig(max_connections=10, min_connections=1), +) ``` -## Creating Tables Manually +The pool holds up to ten connections and keeps one open. `max_connections` defaults to 5 and `min_connections` to 0, and `min_connections` may not exceed `max_connections`. In a web application, connect once at startup and let the pool serve every request. `PoolConfig(settings_delivery=...)` chooses how sessions send their [row-level security settings](../concepts/row-level-security.md#connection-delivery); the [`PoolConfig`](../api/connection.md#ferro.PoolConfig) reference covers every field. -To control table creation yourself instead of `auto_migrate=True`, make sure your models are imported (importing registers them), then call `create_tables()`: +## Named connections -```python -import ferro +One process can hold several pools at once: separate databases, roles or pool sizes. Name each one with `name=`, and route to it with a named session or `Model.using(name)`: +```python {test=run prelude=connections} +from ferro import connect, engines -async def init() -> None: - await ferro.connect("sqlite::memory:") +await connect("sqlite::memory:", name="app", default=True, auto_migrate=True) +await connect("sqlite::memory:", name="archive", auto_migrate=True) - from myapp.models import Post, User # noqa: F401 — importing registers models +async with engines.session(): # the default connection, "app" + await Author.create(name="Ada Lovelace", email="ada@example.com") - await ferro.create_tables() # default connection - await ferro.create_tables(using="analytics") # a named connection +async with engines.session("archive"): + await Author.create(name="Grace Hopper", email="grace@example.com") + +print([author.name for author in await Author.using("archive").all()]) ``` -`create_tables()` creates missing tables (including many-to-many join tables) and never modifies existing ones. +It prints `['Grace Hopper']`: each connection is its own database. `Author.using("archive")` routes one call and needs no session. + +Raw SQL takes `using=` too (`await execute("...", using="archive")`), and `transaction(using="archive")` pins a whole transaction to one connection. [How to use several databases](../howto/multiple-databases.md) walks through routing, transactions and table creation across two databases. -## Postgres Schema Isolation +!!! warning "Connection names are code, not input" + Choose connection names from constants or trusted server-side logic. Never take one from a request parameter, a header or other untrusted input. -Append the Ferro-specific `ferro_search_path` URL parameter to pin a connection to a PostgreSQL schema. Ferro strips the parameter from the URL before handing it to the driver and applies `SET search_path` on every pooled connection: +Ferro has no automatic routing: no read/write splitting, distributed transactions or cross-connection joins. Each call goes where you route it. -```python -import ferro +### The default connection -await ferro.connect( - "postgresql://user:password@localhost:5432/app?ferro_search_path=tenant_a" -) +`engines.session()` with no name opens a session on the default connection. The default is set three ways: + +1. `connect(url)` without a `name` registers it as `"default"`. +2. `connect(url, name="app", default=True)` makes a named connection the default. +3. `set_default_connection("archive")` switches the default at runtime. + +```python {test=run prelude=connections} +from ferro import connect, engines, set_default_connection + +await connect("sqlite::memory:", name="app", default=True, auto_migrate=True) +await connect("sqlite::memory:", name="archive", auto_migrate=True) + +set_default_connection("archive") + +async with engines.session(): + await Author.create(name="Ada Lovelace", email="ada@example.com") + +print(len(await Author.using("archive").all()), len(await Author.using("app").all())) ``` -The value must be a single identifier of ASCII letters, digits, and underscores — anything else raises `ValueError`. This is handy for schema-per-tenant setups and for isolating test runs against one physical database. +It prints `1 0`: after the switch, a session with no name opens on `archive`. + +## Creating tables manually -## Resetting the Engine +To create tables yourself instead of with `auto_migrate=True`, import your models (importing registers them), then call `create_tables()`: -`ferro.reset_engine()` tears down all pools and connection state — primarily for test suites that reconnect with a fresh database per test: +```python {test=run prelude=connections} +from ferro import connect, create_tables -```python -import ferro +await connect("sqlite::memory:") -ferro.reset_engine() -await ferro.connect("sqlite::memory:", auto_migrate=True) +report = await create_tables() +for statement in report.statements: + print(statement.sql) ``` -See [Testing](../howto/testing.md) for a ready-made pytest fixture. +On SQLite it prints: + + +```text +CREATE TABLE IF NOT EXISTS "author" ( "email" varchar NOT NULL, "id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, "name" varchar NOT NULL ) +CREATE UNIQUE INDEX IF NOT EXISTS "uq_author_email" ON "author" ("email") +CREATE TABLE IF NOT EXISTS "post" ( "author_id" integer NOT NULL, "body" varchar NOT NULL, "created_at" DATETIME NOT NULL, "id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, "published" integer NOT NULL, "title" varchar NOT NULL, CONSTRAINT "fk_post_author_id_author" FOREIGN KEY ("author_id") REFERENCES "author" ("id") ON DELETE CASCADE ) +``` + +`create_tables()` returns a [`PassReport`](../api/connection.md#ferro.PassReport) of the statements it ran, here two tables and the unique index on `email`. It creates missing tables, many-to-many join tables included, and never changes existing ones. `create_tables(using="archive")` targets a named connection. + +## Postgres schema isolation + +Add the `ferro_search_path` parameter to a PostgreSQL URL to pin the connection to one schema: + +```python {test=run prelude=connections backend=postgres} +from ferro import connect + +await connect("postgresql://user:password@localhost:5432/app?ferro_search_path=tenant_a") +``` + +Ferro removes the parameter before passing the URL to the driver and runs `SET search_path` on every pooled connection, so unqualified table names resolve in `tenant_a`. The value must be one identifier of ASCII letters, digits and underscores; anything else raises `ValueError`. Use it for schema-per-tenant setups, or to isolate test runs that share one database. + +## Resetting the engine + +`reset_engine()` tears down every pool and all connection state, so a test suite can reconnect to a fresh database for each test: + +```python {test=run prelude=connections} +from ferro import connect, reset_engine + +reset_engine() +await connect("sqlite::memory:", auto_migrate=True) +``` -!!! note "No `disconnect()` yet" - An explicit `disconnect()` is **not yet implemented** — pools are cleaned up on process exit. See the [Roadmap](../roadmap.md). +The new connection registers as the default again. Pools also close when the process exits. [Testing](../howto/testing.md) has a ready-made pytest fixture. ## Logging The `ferro.sql` logger is the statement log: one `DEBUG` record for each statement Ferro sends, written as the statement finishes. Turn it on with your usual logging setup: -```python +```python {test=run prelude=connections} import logging logging.basicConfig(format="%(name)s %(levelname)s %(message)s") @@ -229,6 +329,7 @@ logging.getLogger("ferro.sql").setLevel(logging.DEBUG) A schema change now prints each statement it runs, with its parameters and how long it took: + ```text ferro.sql DEBUG CREATE TABLE IF NOT EXISTS "post" (...) params=() 1.2ms ferro.sql DEBUG ALTER TABLE "author" ADD COLUMN "bio" varchar params=() 0.8ms @@ -266,10 +367,10 @@ Statements a Postgres pool sends on its own, outside any operation, are not in t The logger is the only switch: while `ferro.sql` is not enabled for `DEBUG`, Ferro collects nothing. Ferro installs only a `NullHandler` on its `ferro` logger, which propagates, so your logging configuration (and pytest's `caplog`) decides where the records go. The `ferro` command line installs its own handler, so a data step's `ctx.log.info(...)` lines still print there. -## See Also +## See also -- [Schema Management](schema/overview.md) — auto-migrate, migrations and Alembic -- [Transactions](transactions.md) — connection affinity -- [Multiple Databases](../howto/multiple-databases.md) — multi-connection patterns -- [Testing](../howto/testing.md) — test database setup -- [Database Backends](../concepts/backends.md) — SQLite vs PostgreSQL specifics +- [Transactions](transactions.md): commit and roll back on one connection +- [How to use several databases](../howto/multiple-databases.md): routing across named connections +- [Schema management](schema/overview.md): auto-migrate, migrations and Alembic +- [Database backends](../concepts/backends.md): SQLite and PostgreSQL differences +- [Connection reference](../api/connection.md): `connect`, `PoolConfig`, `engines` and `Session` diff --git a/docs/pages/guide/migrations.md b/docs/pages/guide/migrations.md deleted file mode 100644 index ee0d8a06..00000000 --- a/docs/pages/guide/migrations.md +++ /dev/null @@ -1,40 +0,0 @@ -# Schema Migrations has moved - -This page became the **Schema Management** group. Start at the [overview](schema/overview.md); each old section now lives here: - -| Old section | Now | -| :--- | :--- | -| Three Ways to Manage Schema | [Overview › The ladder](schema/overview.md#the-ladder) | -| Auto-Migration | [Auto-migrate](schema/auto-migrate.md) | -| Applying column changes with `migrate_updates` | [Auto-migrate › Applying column changes](schema/auto-migrate.md#applying-column-changes-with-migrate_updates) | -| Evolving enums: label addition | [Auto-migrate › Evolving enums](schema/auto-migrate.md#evolving-enums-label-addition) | -| Destructive drops with `migrate_destructive` | [Auto-migrate › Destructive drops](schema/auto-migrate.md#destructive-drops-with-migrate_destructive) | -| On-demand `migrate()` | [Auto-migrate › On-demand `migrate()`](schema/auto-migrate.md#on-demand-migrate) | -| Safety guidance | [Auto-migrate › Safety guidance](schema/auto-migrate.md#safety-guidance) | -| Alembic for Production | [Alembic](schema/alembic.md) | -| Choosing a Workflow | [Overview › Choosing](schema/overview.md#choosing) | - - diff --git a/docs/pages/guide/models-and-fields.md b/docs/pages/guide/models-and-fields.md index d8a8ef9e..253797e6 100644 --- a/docs/pages/guide/models-and-fields.md +++ b/docs/pages/guide/models-and-fields.md @@ -1,416 +1,271 @@ -# Models & Fields +--- +description: "Declare a table as a Ferro model and add keys, constraints, indexes, defaults and column types with Field()." +--- -Models define your schema in plain Python type hints, validate data with Pydantic, and map to database tables in the Rust engine. +# Models and fields -## Defining a Model - -Inherit from `ferro.Model` and declare fields with standard type annotations: +A model is a Python class that declares one database table: each annotated attribute is a column, and [`Field()`](../api/fields.md#ferro.Field) adds the database options. This page builds one product catalog and adds a feature in each section. Here is the whole `Product` model, created in an in-memory SQLite database and saved once: === "Assignment" ```python - --8<-- "docs/examples/quickstart.py:models" + --8<-- "docs/examples/models_and_fields.py:program" ``` === "Annotated" ```python - --8<-- "docs/examples/quickstart_annotated.py:models" + --8<-- "docs/examples/models_and_fields_annotated.py:program" ``` -Every Ferro model is a full [Pydantic](https://docs.pydantic.dev/latest/) `BaseModel`, so validation, serialization (`model_dump()`, `model_dump_json()`), and `model_config` all work. - -## Declaration Styles - -Ferro's `Field()` merges database options (`primary_key`, `unique`, `index`, ...) with Pydantic's validation options in a single call. You can attach it in two equivalent ways: +Run it with `asyncio.run(main())`. The database assigns `id`, and `status` takes its default. `create_tables()` returns the statements it ran; this is the table it created: -=== "Assignment with Field()" - - Put defaults and options on the assignment side — the classic Pydantic style: - - ```python - from decimal import Decimal - - from ferro import Field, Model - - - class Product(Model): - id: int | None = Field(default=None, primary_key=True) - slug: str = Field(unique=True, index=True) - name: str = Field(max_length=200, description="Display name") - price: Decimal = Field(ge=0, decimal_places=2) - ``` +```sql +--8<-- "docs/examples/models_and_fields.py:product-table" +``` -=== "Annotated metadata" +Columns are written in name order, and every annotation that does not allow `None` is `NOT NULL`. The rest of this page takes the model apart one feature at a time. Each later example runs inside the session the first one opens; see [Sessions](connections.md#sessions). - Keep the plain type on the left and pass `Field(...)` inside `typing.Annotated`: +Every Ferro model is also a [Pydantic](https://docs.pydantic.dev/latest/) `BaseModel`, so `model_dump()`, `model_dump_json()`, validators and `model_config` work as they do in Pydantic. - ```python - from decimal import Decimal - from typing import Annotated +## Declaration styles - from ferro import Field, Model +`Field()` takes the database options (`primary_key`, `unique`, `index`, `db_type`) and Pydantic's validation options (`max_length`, `ge`, `pattern`) in one call. The two tabs above attach it in the two ways Ferro accepts, and they create the same schema: +- **Assignment** puts `Field(...)` on the right of `=`, the classic Pydantic style. +- **Annotated** puts it inside `typing.Annotated[...]` and keeps the default as the assigned value, so a type checker sees the default. - class Product(Model): - id: Annotated[int | None, Field(default=None, primary_key=True)] - slug: Annotated[str, Field(unique=True, index=True)] - name: Annotated[str, Field(max_length=200, description="Display name")] - price: Annotated[Decimal, Field(ge=0, decimal_places=2)] - ``` +Pick one style per project. Every example in these docs shows both, in tabs. A forward foreign key is always written `Annotated[Author, ForeignKey(...)]`, in either style; see [Relationships](relationships.md). -Both produce identical schemas — every example in these docs shows both styles in tabs; pick one and stay consistent within a project. The `Annotated` style keeps the bare type visible at a glance and is also how forward relationships are declared (`Annotated[Author, ForeignKey(...)]` — see [Relationships](relationships.md)). +## Field types -!!! note "Advanced: `FerroField`" - For a type-first style that carries only database flags (no Pydantic validation kwargs), you can attach `ferro.FerroField(...)` as `Annotated` metadata instead: `id: Annotated[int, FerroField(primary_key=True)]`. It accepts the same database options as `Field()`. +Ferro chooses each column's type from the annotation. The table lists what the create pass writes for each annotation, on each database: -## Field Types +--8<-- "docs/includes/field-types.md:annotations" -Ferro maps Python annotations to backend-appropriate column types: +The table is generated from the code that chooses the type, so it matches the schema Ferro creates. A few rows need a sentence more: -| Python type | SQLite | PostgreSQL | Notes | -| :--- | :--- | :--- | :--- | -| `int` | `INTEGER` | `INTEGER` / `BIGINT` | | -| `str` | `TEXT` | `TEXT` | Override with `db_type=varchar(n)` | -| `bool` | `BOOLEAN` (0/1) | `BOOLEAN` | | -| `float` | `DOUBLE` | `DOUBLE PRECISION` | | -| `datetime.datetime` | `TEXT` (ISO 8601) | `TIMESTAMP` / `TIMESTAMPTZ` | | -| `datetime.date` | `TEXT` (ISO 8601) | `DATE` | | -| `datetime.time` | `TEXT` (ISO 8601) | `TIME` | | -| `uuid.UUID` | `TEXT` (36 chars) | `UUID` | | -| `decimal.Decimal` | `NUMERIC` | `NUMERIC` | For money and other exact values | -| `bytes` | `BLOB` | `BYTEA` | | -| `enum.Enum` | `TEXT` | native `ENUM` | See note below | -| `dict` / `list` | `TEXT` (JSON) | `JSONB` (default) | `db_type="json"` opts out | +- An annotation that allows `None`, such as `int | None`, drops `NOT NULL` and keeps the type. +- `datetime` is `timestamp with time zone` on PostgreSQL. For a column without a time zone, declare `db_type="timestamp"`. +- A `UUID` column is declared `CHAR(32)` on SQLite, and the value Ferro stores in it is the 36-character hyphenated string. SQLite does not enforce a declared length. +- A JSON-family field, `dict`, `list`, `tuple`, `set` or a nested Pydantic model, is `jsonb` on PostgreSQL and `JSON` on SQLite. See [JSON storage](#json-storage-json-and-jsonb). -### Overriding the column type with `db_type` +### Column type overrides with `db_type` -When the inferred type isn't what you want — for example `varchar(255)` instead of unbounded `TEXT` — pass a `db_type` token: +When you want a different column type for a field, for example a length limit of `varchar(200)` instead of unbounded `varchar`, pass a `db_type` token: === "Assignment" ```python - from ferro import Field, Model, varchar - - - class Document(Model): - id: int | None = Field(default=None, primary_key=True) - title: str = Field(db_type=varchar(255)) - body: str = Field(db_type="text") + --8<-- "docs/examples/models_and_fields.py:db-type" ``` === "Annotated" ```python - from typing import Annotated - - from ferro import Field, Model, varchar - - - class Document(Model): - id: Annotated[int | None, Field(default=None, primary_key=True)] - title: Annotated[str, Field(db_type=varchar(255))] - body: Annotated[str, Field(db_type="text")] + --8<-- "docs/examples/models_and_fields_annotated.py:db-type" ``` -Valid values are the `DbTypeToken` literals — `"text"`, `"smallint"`, `"int"`, `"bigint"`, `"uuid"`, `"timestamp"`, `"timestamptz"`, `"date"`, `"time"`, `"json"`, `"jsonb"` — plus `varchar(n)` built with `ferro.varchar`. Prefer `varchar(255)` over the raw string `"varchar(255)"` so type checkers see a deliberate vocabulary choice. The override is validated against the Python annotation at class-definition time — on both declaration styles and on raw `json_schema_extra` declarations alike — so an incompatible combination fails immediately. - -!!! note "Enum storage" - `enum.Enum` fields are stored as text on SQLite and as named native `ENUM` types on PostgreSQL (created by every schema door: auto-migrate, [migrations](schema/migrations.md) and the [Alembic bridge](schema/alembic.md)). For closed-domain string columns with a DB-side `CHECK` constraint, combine `db_type` with `db_check=True`. - -### JSON storage: `json` and `jsonb` - -Json-family fields — `dict`, `list` (any element type, including `list[SomeNestedModel]`), and nested Pydantic models — store as JSON documents. On PostgreSQL the default is **JSONB** (binary storage, GIN-indexable, containment operators — the type PostgreSQL itself recommends); declare `db_type="json"` per column to opt out into plain `json` when you need representation fidelity: +The column becomes: -=== "Assignment" +```sql +--8<-- "docs/examples/models_and_fields.py:name-column" +``` - ```python - from typing import Any +`max_length=200` is the Pydantic check that rejects a longer name before it reaches the database; `db_type` sets only the column. Build `varchar(n)` with [`ferro.varchar`](../api/fields.md#ferro.varchar) rather than the string `"varchar(200)"`, so a type checker sees a token it knows. These are the tokens and what each writes: - from pydantic import BaseModel +--8<-- "docs/includes/field-types.md:db-type-tokens" - from ferro import Field, Model +Ferro checks the token against the annotation when the class is defined, so `x: int = Field(db_type="uuid")` raises `TypeError` when the class is defined, not at the first query. +### Enums - class Payment(BaseModel): - currency: str - amount: int +An `enum.Enum` field stores the enum's values. `Product.status` is a `StrEnum` with a default: +```python +--8<-- "docs/examples/models_and_fields.py:enum" +``` - class Transaction(Model): - id: int | None = Field(default=None, primary_key=True) - raw_payload: dict[str, Any] = Field() # JSONB (the default) - line_items: list[Payment] = Field(db_type="jsonb") # explicit, same as default - audit_note: dict[str, str] = Field(db_type="json") # plain-json opt-out - ``` +On SQLite the column is `varchar(n)`, sized to the longest value (`retired`, 7 characters): -=== "Annotated" +```sql +--8<-- "docs/examples/models_and_fields.py:status-column" +``` - ```python - from typing import Annotated, Any +On PostgreSQL the column uses a native enum type named after the class, `status`, which every schema door creates before the table: [auto-migrate](schema/auto-migrate.md), [migrations](schema/migrations.md) and the [Alembic bridge](schema/alembic.md). Either way the database holds the value, not the member name: - from pydantic import BaseModel +```python +--8<-- "docs/examples/models_and_fields.py:enum-value" +``` - from ferro import Field, Model +The row reads back `"active"`, the value of `Status.ACTIVE`. To keep a closed set of values in a plain column with a `CHECK` constraint instead, declare `Field(db_type="text", db_check=True)`; the constraint is named `ck_
_`. +### JSON storage: `json` and `jsonb` - class Payment(BaseModel): - currency: str - amount: int +A `dict`, a `list` or a nested Pydantic model is stored as one JSON document. `Product` has two: +=== "Assignment" - class Transaction(Model): - id: Annotated[int | None, Field(default=None, primary_key=True)] - raw_payload: Annotated[dict[str, Any], Field()] # JSONB (the default) - line_items: Annotated[list[Payment], Field(db_type="jsonb")] # explicit, same as default - audit_note: Annotated[dict[str, str], Field(db_type="json")] # plain-json opt-out + ```python + --8<-- "docs/examples/models_and_fields.py:json" ``` -Declaring `db_type="jsonb"` on anything outside the json family (an `int`, a `set`, ...) fails at class-definition time. - -**SQLite lowering.** On SQLite both tokens store as plain JSON text — one model definition runs on both backends, and switching between `json` and `jsonb` never changes SQLite behavior. - -**Round-trip contract is value equality, not byte fidelity.** PostgreSQL's `jsonb` does not preserve dict key order (keys come back sorted), duplicate keys, or whitespace — that's inherent to the storage type, not Ferro. Values compare equal with Python `==` (dict equality ignores order), but if you iterate keys and need insertion order — or need the stored text byte-for-byte — opt out with `db_type="json"`. +=== "Annotated" -**Migrating between `json` and `jsonb` is a declaration edit.** With [`migrate_updates`](schema/auto-migrate.md) enabled, a storage change emits exactly one `ALTER TABLE ... TYPE ... USING ...` on PostgreSQL in either direction (existing values survive the cast). A [migration](schema/migrations.md) and the [Alembic bridge](schema/alembic.md) render the same DDL. On SQLite the same edit is a no-op. + ```python + --8<-- "docs/examples/models_and_fields_annotated.py:json" + ``` -!!! warning "Upgrading from a pre-JSONB-default release" - Databases created before the JSONB default carry plain `json` columns for derived `dict`/`list` fields. On your first connect with `migrate_updates=True`, each such column upgrades in place with one `ALTER ... TYPE jsonb USING ...` — values survive, but dict key order stops being preserved. To keep a column as plain `json`, add `db_type="json"` to its declaration **before** upgrading; the diff is then zero operations. +Both columns are `jsonb` on PostgreSQL and `JSON` on SQLite, and a value reads back equal to the one saved: -!!! note "Querying inside JSONB" - Containment and path operators (`@>`, `->`, `#>>`) and GIN index declarations are not in the query builder yet — they're planned follow-ups. Today JSONB gives you honest storage and round-tripping; query the column as a whole. +```python +--8<-- "docs/examples/models_and_fields.py:json-round-trip" +``` -## Field Options +`jsonb` is binary storage that PostgreSQL can index and compare, and it is the type PostgreSQL recommends. It does not keep the text you wrote: keys come back sorted, a duplicate key keeps its last value, and whitespace is dropped. Python's `==` ignores dict key order, so the assertions above hold. If you iterate the keys and need their insertion order, or need the stored text byte for byte, declare `db_type="json"` on that field. `db_type="jsonb"` on a field outside the JSON family, such as an `int`, raises `TypeError` when the class is defined. -Database options accepted by `Field()` (and `FerroField()`): +On SQLite both tokens store plain JSON text, so switching between them changes nothing there. On PostgreSQL the switch is a declaration edit: with [`migrate_updates`](schema/auto-migrate.md) on, the pass runs one `ALTER TABLE ... TYPE ... USING ...` and existing values survive the cast. A [migration](schema/migrations.md) and the [Alembic bridge](schema/alembic.md) write the same statement. -| Option | Type | Default | Description | -| :--- | :--- | :--- | :--- | -| `primary_key` | `bool` | `False` | Mark this column as the table's primary key. | -| `autoincrement` | `bool \| None` | `None` | Database-generated values. Inferred `True` for integer primary keys; pass `False` for manual integer keys. | -| `unique` | `bool` | `False` | Single-column uniqueness constraint. For multi-column uniqueness see [Composite Constraints](#composite-constraints). | -| `index` | `bool` | `False` | Create a non-unique index on this column. | -| `nullable` | `"infer" \| bool` | `"infer"` | Column nullability. `"infer"` follows whether the annotation allows `None`; `True`/`False` force it (useful when the Python type diverges from the column on purpose). | -| `default` | any | — | Pydantic default value. Also used to backfill when [`migrate_updates`](schema/auto-migrate.md) adds a NOT NULL column — including a JSON object or array on a json-family field. | -| `default_factory` | callable | — | Pydantic default factory, e.g. `default_factory=datetime.now`. On json-family fields (`dict` / `list` / nested model), `default_factory=dict` / `list` is snapshotted once as a backfill literal; `uuid4` and `datetime.now` are not. | -| `db_type` | `DbType \| None` | `None` | Column-type override (see above). | -| `db_check` | `bool` | `False` | Emit a DB-side `CHECK` constraint for closed-domain types; only valid with `db_type`. | -| `renamed_from` | `str \| None` | `None` | The column's previous name: a rename hint. `ferro migrate new` writes `RENAME COLUMN` instead of a drop and an add, so every row keeps its value; `ForeignKey(..., renamed_from=...)` does the same for a relation's `*_id` column. Inert once its migration is generated. See [Renames](schema/migrations.md#renames). | -| `retired` | `bool` | `False` | Retire the column: the field leaves the model (no attribute, no keyword, no statement the model issues names it) and the column stays in the schema, nullable, until the declaration is deleted in a later release. The annotation must allow `None`; `ForeignKey(retired=True)` does the same for a relation's `*_id` column. See [Retiring a column](schema/migrations.md#retiring-a-column). | +A filter compares a JSON column as a whole value. The query builder has no containment or path operators (`@>`, `->`, `#>>`) and declares no GIN index. To change some keys of a stored object on PostgreSQL, use `.merge()` in a [recipe update](mutations.md#expression-updates). -On top of these, every Pydantic validation option works in the same call: `min_length`, `max_length`, `pattern`, `gt`, `ge`, `lt`, `le`, `multiple_of`, `decimal_places`, `description`, and the rest — see [Pydantic's Field docs](https://docs.pydantic.dev/latest/api/fields/#pydantic.fields.Field). +!!! note "Changed in 0.15" + Before 0.15, a `dict` or `list` field without `db_type` created a plain `json` column on PostgreSQL. Read [Upgrading to 0.15](../upgrading.md#upgrading-to-015) before you connect such a database with `migrate_updates`. -## Primary Keys +## Primary keys ### Auto-increment -Integer primary keys auto-increment by default. Declare them as `int | None` with `default=None` so unsaved instances can exist before the database assigns an ID: +An integer primary key is generated by the database. Declare it `int | None` with a default of `None`, so an instance can exist before it is saved: === "Assignment" ```python - from ferro import Field, Model - - - class User(Model): - id: int | None = Field(default=None, primary_key=True) - name: str + --8<-- "docs/examples/models_and_fields.py:primary-key" ``` === "Annotated" ```python - from typing import Annotated - - from ferro import Field, Model + --8<-- "docs/examples/models_and_fields_annotated.py:primary-key" + ``` +It is `integer PRIMARY KEY AUTOINCREMENT` on SQLite and `serial PRIMARY KEY` on PostgreSQL. `save()` writes the generated value back onto the instance: - class User(Model): - id: Annotated[int | None, Field(default=None, primary_key=True)] - name: str - ``` +```python +--8<-- "docs/examples/models_and_fields.py:save" +``` -After `await user.save()` (or `User.create(...)`), the generated ID is written back onto the instance. +The second product gets `id` 2. For an integer key that you assign yourself, pass `Field(primary_key=True, autoincrement=False)`. -### Manual +### Natural keys -For natural keys, or integer keys you assign yourself, disable auto-increment: +A column that already identifies the row, such as a category code, can be the primary key. `Category` uses its `code`: === "Assignment" ```python - from ferro import Field, Model - - - class Country(Model): - code: str = Field(primary_key=True) # natural key, e.g. "US" - name: str - - - class LegacyRecord(Model): - id: int = Field(primary_key=True, autoincrement=False) - payload: str + --8<-- "docs/examples/models_and_fields.py:category" ``` === "Annotated" ```python - from typing import Annotated - - from ferro import Field, Model - - - class Country(Model): - code: Annotated[str, Field(primary_key=True)] # natural key, e.g. "US" - name: str - - - class LegacyRecord(Model): - id: Annotated[int, Field(primary_key=True, autoincrement=False)] - payload: str + --8<-- "docs/examples/models_and_fields_annotated.py:category" ``` +The key is a plain `varchar` primary key, and you pass it on create: `Category.create(code="lighting", title="Lighting")`. `__ferro_table__` names the table; see [Custom table names](#custom-table-names). + ### UUID primary keys -Generate UUIDs client-side with `default_factory`: +A `UUID` key is generated in Python, by a `default_factory`, before the row is written. `Order` uses `uuid.uuid4`: === "Assignment" ```python - import uuid - - from ferro import Field, Model - - - class Order(Model): - id: uuid.UUID = Field(default_factory=uuid.uuid4, primary_key=True) - total_cents: int = 0 + --8<-- "docs/examples/models_and_fields.py:order" ``` === "Annotated" ```python - import uuid - from typing import Annotated + --8<-- "docs/examples/models_and_fields_annotated.py:order" + ``` - from ferro import Field, Model +On SQLite the column is declared `CHAR(32)`: +```sql +--8<-- "docs/examples/models_and_fields.py:order-ddl" +``` - class Order(Model): - id: Annotated[uuid.UUID, Field(default_factory=uuid.uuid4, primary_key=True)] - total_cents: int = 0 - ``` +The value in it is the 36-character hyphenated string, as a raw query shows: -On PostgreSQL this is a native `UUID` column; on SQLite it is stored as a 36-character string. +```python +--8<-- "docs/examples/models_and_fields.py:uuid-value" +``` -## Composite Constraints +On PostgreSQL the column is a native `uuid`. -### Composite uniques +## Uniques and indexes -When a row must be unique across several columns *together* (e.g. one membership per `(user_id, org_id)` pair), per-column `unique=True` is not what you want. Declare the `ClassVar` `__ferro_composite_uniques__` instead: +### Single-column uniques -=== "Assignment" +`unique=True` on a field adds a unique index named `uq_
_`. `Product.sku` has one: - ```python - import uuid - from typing import ClassVar +```sql +--8<-- "docs/examples/models_and_fields.py:sku-unique" +``` - from ferro import Field, Model +`index=True` adds a plain index the same way, named `idx_
_`. +### Composite uniques - class OrgMembership(Model): - __ferro_composite_uniques__: ClassVar[tuple[tuple[str, ...], ...]] = ( - ("user_id", "org_id"), - ) +When a row must be unique across several columns together, declare the `ClassVar` `__ferro_composite_uniques__`. A product name must be unique within its category: - id: int | None = Field(default=None, primary_key=True) - user_id: uuid.UUID - org_id: uuid.UUID - ``` +```python +--8<-- "docs/examples/models_and_fields.py:composite-uniques" +``` -=== "Annotated" +The index is named after the table and its columns, in order: - ```python - import uuid - from typing import Annotated, ClassVar - - from ferro import Field, Model +```sql +--8<-- "docs/examples/models_and_fields.py:composite-unique-ddl" +``` +A second desk lamp in the lighting category fails on that index: - class OrgMembership(Model): - __ferro_composite_uniques__: ClassVar[tuple[tuple[str, ...], ...]] = ( - ("user_id", "org_id"), - ) +```python +--8<-- "docs/examples/models_and_fields.py:composite-unique-violation" +``` - id: Annotated[int | None, Field(default=None, primary_key=True)] - user_id: uuid.UUID - org_id: uuid.UUID - ``` +The error is a `UniqueViolationError`; SQLite's message names the two columns. -- Each inner tuple lists **database column names** (for a `ForeignKey` field named `user`, use the shadow column `user_id`). +- Each inner tuple lists column names. For a `ForeignKey` field named `author`, use its column, `author_id`. - Declare several tuples for several independent composite uniques. -- Unknown column names raise at model registration time. +- An unknown column name raises when the class is defined. -!!! note "NULL semantics" - SQL `UNIQUE` treats `NULL` values as distinct from each other, so nullable members of a composite unique can admit multiple rows that differ only by `NULL`. Prefer `NOT NULL` columns in composite uniques when you need strict "at most one row per pair" semantics. +SQL `UNIQUE` treats `NULL` values as distinct, so a nullable column in a composite unique lets through rows that differ only by `NULL`. Declare the columns without `None` when you need at most one row per combination. ### Composite indexes -For non-unique multi-column indexes — read-path optimization on common filter combinations like `(user_id, created_at)` — declare `__ferro_composite_indexes__` with the same shape: - -=== "Assignment" - - ```python - from datetime import datetime - from typing import ClassVar - - from ferro import Field, Model - - - class Comment(Model): - __ferro_composite_indexes__: ClassVar[tuple[tuple[str, ...], ...]] = ( - ("user_id", "created_at"), - ) - - id: int | None = Field(default=None, primary_key=True) - user_id: int - created_at: datetime - body: str - ``` - -=== "Annotated" - - ```python - from datetime import datetime - from typing import Annotated, ClassVar - - from ferro import Field, Model +For a non-unique index over several columns, such as a common filter on `status` sorted by `created_at`, declare `__ferro_composite_indexes__` in the same shape: +```python +--8<-- "docs/examples/models_and_fields.py:composite-indexes" +``` - class Comment(Model): - __ferro_composite_indexes__: ClassVar[tuple[tuple[str, ...], ...]] = ( - ("user_id", "created_at"), - ) +It creates: - id: Annotated[int | None, Field(default=None, primary_key=True)] - user_id: int - created_at: datetime - body: str - ``` +```sql +--8<-- "docs/examples/models_and_fields.py:composite-index-ddl" +``` -Validation mirrors composite uniques: at least two columns per tuple, columns must exist, and **order is preserved** (it matters for leftmost-prefix optimization). For single-column indexes use `Field(index=True)`. Declaring the same ordered tuple in both `__ferro_composite_uniques__` and `__ferro_composite_indexes__` emits a `UserWarning` and drops the redundant index. +Each tuple names at least two columns, in the order the index uses them, which matters because a query can use the index for any leading run of its columns. For one column, use `Field(index=True)`. If the same ordered tuple is also a composite unique, Ferro emits a `UserWarning` and drops the index, since the unique index already covers it. -Both ClassVars flow through to [migrations](schema/migrations.md) and to [Alembic autogenerate](schema/alembic.md) as matching `UniqueConstraint` / `Index` objects. +Both `ClassVar`s reach [migrations](schema/migrations.md) and [Alembic autogenerate](schema/alembic.md) as the same `UniqueConstraint` and `Index` names. ### Table checks -When a rule spans several columns on the same row — "at most one of these two sides may be set" — a per-column option is not enough. Declare a **table check**: a named predicate over this model's own columns in the `ClassVar` `__ferro_checks__`. - -Each entry is `Check(suffix, predicate)`. You supply a short suffix (for example `at_most_one_side`); ferro derives the live constraint name `ck_
_` (here, `ck_pair_at_most_one_side`). The predicate is a ferro lambda with a **lowercase-singular** parameter named after the model (`pair` for `Pair`), using the same style as [`where()`](queries.md) filters — not a SQL string. - -The predicate is the root-column `where()` dialect: comparisons, `== None` / `!= None` (including on a forward foreign key or its shadow `*_id` column), `&` / `|` / `~`, `.in_()`, `.like()`, and column-to-column (`pair.left != pair.right`). Values are literals inlined into the CHECK — not bind parameters. Relation traversal, `.exists()`, aggregates, and closed-over variables fail at class definition. - -For a single-column closed domain (for example a `StrEnum` stored as text with `IN ('admin', 'user')`), use `Field(db_check=True)` instead — that emits `ck_
_` and is a different artifact from table checks. +When a rule spans several columns of one row, declare a table check: a named predicate in the `ClassVar` `__ferro_checks__`. A payment carries a card token or a bank account, never both: === "Assignment" @@ -424,213 +279,125 @@ For a single-column closed domain (for example a `StrEnum` stored as text with ` --8<-- "docs/examples/table_checks_annotated.py:models" ``` -- Each `Check` suffix must be a lowercase identifier (`[a-z][a-z0-9_]*`), unique per model, and must not collide with a column-check name (`ck_
_`). -- Unknown columns, duplicate suffixes, and unsupported predicate forms raise at model registration time. -- Table checks are emitted **inline in `CREATE TABLE`** on both PostgreSQL and SQLite. See [Auto-migrate](schema/auto-migrate.md) for how they reconcile on existing tables, and [Migrations](schema/migrations.md#sqlite-table-rebuilds) for the SQLite table rebuild. - -A violating insert raises `CheckViolationError` (PostgreSQL also sets `exc.constraint` to the live `ck_*` name): +Each entry is `Check(suffix, predicate)`. Ferro names the constraint `ck_
_` and writes it inside `CREATE TABLE`, on both databases: -```python ---8<-- "docs/examples/table_checks.py:violation" +```sql +--8<-- "docs/examples/table_checks.py:ddl" ``` -## Custom table names - -By default a model's table is its class name lowercased: `class User` → -table `user`. Set `__ferro_table__` in the class body to choose the table -name yourself: - -=== "Assignment" - - ```python - from typing import ClassVar - - from ferro import Field, Model +The predicate is a lambda written like a [`where()`](queries.md) filter, with its parameter named after the model in lowercase (`payment` for `Payment`). It accepts comparisons, `== None` and `!= None` (on a forward foreign key or its `*_id` column too), `&`, `|`, `~`, `.in_()`, `.like()`, and column-to-column comparisons. Values are written into the `CHECK` as literals, not bound as parameters. Relation traversal, `.exists()`, aggregates and variables from the enclosing scope raise when the class is defined. +A row that breaks the check raises `CheckViolationError`, whose message names the constraint: - class User(Model): - __ferro_table__: ClassVar[str] = "app_users" - - id: int | None = Field(default=None, primary_key=True) - email: str = Field(unique=True) - ``` +```python +--8<-- "docs/examples/table_checks.py:violation" +``` -=== "Annotated" +On PostgreSQL the error also sets `exc.constraint` to `ck_payment_at_most_one_method`. - ```python - from typing import Annotated, ClassVar +- A suffix is a lowercase identifier (`[a-z][a-z0-9_]*`), unique within the model, and must not collide with a column check's name (`ck_
_`). +- An unknown column, a repeated suffix or an unsupported predicate raises when the class is defined. +- [Auto-migrate](schema/auto-migrate.md) explains how checks reconcile on a table that already exists, and [SQLite table rebuilds](schema/migrations.md#sqlite-table-rebuilds) how a migration changes one on SQLite. - from ferro import Field, Model +For a closed set of values in a single column, use `Field(db_check=True)` instead; see [Enums](#enums). +## Custom table names - class User(Model): - __ferro_table__: ClassVar[str] = "app_users" +A model's table is its class name in lowercase: `class Product` is `product`. Set `__ferro_table__` in the class body to choose the name yourself, as [`Category`](#natural-keys) does: - id: Annotated[int | None, Field(default=None, primary_key=True)] - email: Annotated[str, Field(unique=True)] - ``` +```python +--8<-- "docs/examples/models_and_fields.py:table-name" +``` -Both declare a `User` model stored in `app_users` — every generated -statement follows it: +Every statement Ferro writes for the model uses the name you set: ```sql -CREATE TABLE IF NOT EXISTS "app_users" ( - "email" varchar NOT NULL, - "id" integer NOT NULL PRIMARY KEY AUTOINCREMENT -); -CREATE UNIQUE INDEX IF NOT EXISTS "uq_app_users_email" ON "app_users" ("email"); +--8<-- "docs/examples/models_and_fields.py:category-ddl" ``` -Relationships follow it too: a `ForeignKey` to `User` renders -`REFERENCES "app_users" ("id")`, and default many-to-many join tables are -named from the source model's table name and the relationship's field name — -a `tags` relation on a model stored in `wiki_pages` gets the join table -`wiki_pages_tags` — with join columns (`wiki_pages_id`, `wiki_tags_id`) -following both participants' configured tables. +A `ForeignKey` to `Category` renders `REFERENCES "categories" ("code")`. A default many-to-many join table is named from the source model's table and the relation's field: a `tags` relation on a model stored in `wiki_pages` gets the join table `wiki_pages_tags`. -`__ferro_table__` applies only to the class that declares it — a subclass -gets its own default (`classname.lower()`) unless it declares its own. -The name must be a 1–63 character identifier (`[A-Za-z_][A-Za-z0-9_]*`). +- `__ferro_table__` applies only to the class that declares it; a subclass gets the default name unless it sets its own. +- The name is an identifier of 1 to 63 characters matching `[A-Za-z_][A-Za-z0-9_]*`. +- Two models cannot share one table. Defining a second model that resolves to a table name already in use raises at class definition and names both models. Redefining the same class, in a REPL or a re-imported module, is allowed. -Two distinct models cannot share one table: defining a second model that -resolves to an already-claimed table name raises immediately at class -definition, naming both models. Redefining the *same* class (in a REPL or -re-imported module) is fine. +## Validation -## Pydantic Validation - -Ferro models *are* Pydantic models, so validation runs whenever an instance is constructed — including inside `Model.create(...)` and before `bulk_create(...)` hits the database: +A model validates its fields whenever an instance is built, including inside `create()` and before `bulk_create()` sends anything. `Product.price` must not be negative: === "Assignment" ```python - from ferro import Field, Model - - - class Account(Model): - id: int | None = Field(default=None, primary_key=True) - username: str = Field(unique=True, min_length=3, max_length=50) - email: str = Field(unique=True, pattern=r"^[\w\.-]+@[\w\.-]+\.\w+$") - age: int = Field(ge=0, le=150) + --8<-- "docs/examples/models_and_fields.py:validation-fields" ``` === "Annotated" ```python - from typing import Annotated - - from ferro import Field, Model - - - class Account(Model): - id: Annotated[int | None, Field(default=None, primary_key=True)] - username: Annotated[str, Field(unique=True, min_length=3, max_length=50)] - email: Annotated[str, Field(unique=True, pattern=r"^[\w\.-]+@[\w\.-]+\.\w+$")] - age: Annotated[int, Field(ge=0, le=150)] + --8<-- "docs/examples/models_and_fields_annotated.py:validation-fields" ``` -```python -from pydantic import ValidationError +A negative price raises Pydantic's `ValidationError` before Ferro writes the row: -try: - await Account.create(username="ab", email="not-an-email", age=-1) -except ValidationError as exc: - print(exc.error_count(), "validation errors — nothing was written") +```python +--8<-- "docs/examples/models_and_fields.py:validation" ``` -Custom `@field_validator` / `@model_validator` methods and `model_config` settings (e.g. `validate_assignment=True`, `str_strip_whitespace=True`) work too, since they are plain Pydantic features. - -## Reusing Fields Across Models +The error names the field and the rule it broke. `@field_validator`, `@model_validator` and `model_config` settings such as `validate_assignment=True` work as they do in Pydantic. -!!! warning "Model subclasses cannot inherit fields" - You **cannot** declare fields on a `Model` base class and inherit them in subclasses. `Model`'s metaclass registers a table schema per concrete model class at class-creation time, so a "base model with shared columns" silently produces broken subclass defaults. Don't do this: +## Fields on a model base class - ```python - class TimestampedModel(Model): # WRONG — do not inherit fields from a Model - created_at: datetime = Field(default_factory=datetime.now) - - - class Article(TimestampedModel): # broken defaults - title: str - ``` - -The supported pattern is a **plain mixin** (not a `Model` subclass) for shared *behavior*, with the fields declared on each concrete model: +A class that subclasses `Model` is a model, with a table of its own, even when you meant it only as a base for others. Here `Timestamped` declares `created_at` for `Review` to inherit: === "Assignment" ```python - from datetime import UTC, datetime - - from ferro import Field, Model - - - def utcnow() -> datetime: - return datetime.now(UTC) - - - class TimestampMixin: - """Touch ``updated_at`` on every save. A plain class, not a Model.""" - - async def save(self, **kwargs) -> None: - self.updated_at = utcnow() - await super().save(**kwargs) - - - class Note(TimestampMixin, Model): - id: int | None = Field(default=None, primary_key=True) - text: str - created_at: datetime = Field(default_factory=utcnow) - updated_at: datetime = Field(default_factory=utcnow) - - - class Task(TimestampMixin, Model): - id: int | None = Field(default=None, primary_key=True) - title: str - created_at: datetime = Field(default_factory=utcnow) - updated_at: datetime = Field(default_factory=utcnow) + --8<-- "docs/examples/models_and_fields.py:base-class" ``` === "Annotated" ```python - from datetime import UTC, datetime - from typing import Annotated - - from ferro import Field, Model - + --8<-- "docs/examples/models_and_fields_annotated.py:base-class" + ``` - def utcnow() -> datetime: - return datetime.now(UTC) +`create_tables()` creates two tables. `review` has the inherited column, and `timestamped` is a second table with only `created_at`: +```sql +--8<-- "docs/examples/models_and_fields.py:base-class-ddl" +``` - class TimestampMixin: - """Touch ``updated_at`` on every save. A plain class, not a Model.""" +The subclass works: its rows go to `review`, and the inherited default fills `created_at`. Nothing is written to `timestamped`: - async def save(self, **kwargs) -> None: - self.updated_at = utcnow() - await super().save(**kwargs) +```python +--8<-- "docs/examples/models_and_fields.py:base-class-rows" +``` +The `timestamped` table is the cost. Every schema door treats `Timestamped` as a model, so auto-migrate, a migration and the Alembic bridge all create its table, though no code stores a row there. To share fields without that table, declare the fields on each model and share behaviour through a plain class that does not subclass `Model`; [Timestamps](../howto/timestamps.md) and [Soft deletes](../howto/soft-deletes.md) show the pattern. - class Note(TimestampMixin, Model): - id: Annotated[int | None, Field(default=None, primary_key=True)] - text: str - created_at: Annotated[datetime, Field(default_factory=utcnow)] - updated_at: Annotated[datetime, Field(default_factory=utcnow)] +## Field options +`Field()` takes these database options, beside every Pydantic option (`min_length`, `max_length`, `pattern`, `gt`, `ge`, `lt`, `le`, `decimal_places`, `description` and the rest): - class Task(TimestampMixin, Model): - id: Annotated[int | None, Field(default=None, primary_key=True)] - title: str - created_at: Annotated[datetime, Field(default_factory=utcnow)] - updated_at: Annotated[datetime, Field(default_factory=utcnow)] - ``` +| Option | Default | What it does | +| :--- | :--- | :--- | +| `primary_key` | `False` | Makes the column the primary key; see [Primary keys](#primary-keys). | +| `autoincrement` | `None` | `None` turns it on for an integer primary key; `False` turns it off. | +| `unique` | `False` | Adds a single-column unique index; see [Uniques and indexes](#uniques-and-indexes). | +| `index` | `False` | Adds a single-column index. | +| `nullable` | `"infer"` | `"infer"` follows whether the annotation allows `None`; `True` or `False` forces it. | +| `default`, `default_factory` | none | Pydantic's default. [Auto-migrate](schema/auto-migrate.md) also uses a literal default to fill existing rows when it adds a `NOT NULL` column. | +| `db_type` | `None` | Overrides the column type; see [Column type overrides](#column-type-overrides-with-db_type). | +| `db_check` | `False` | Adds a `CHECK` for a closed set of values; needs `db_type`. | +| `renamed_from` | `None` | The column's previous name; see [Rename hints](schema/renames-and-retirements.md#rename-hints). | +| `retired` | `False` | Keeps the column in the schema after the field leaves the model; see [Retiring a column](schema/renames-and-retirements.md#retiring-a-column). | -The few repeated field lines are deliberate: each concrete model owns its full schema, and the mixin contributes behavior only. For complete worked examples, see [Timestamps](../howto/timestamps.md) and [Soft Deletes](../howto/soft-deletes.md). +The full signature is in the reference: [`Field`](../api/fields.md#ferro.Field), [`FerroField`](../api/fields.md#ferro.FerroField) and [`DbType`](../api/fields.md#ferro.DbType). -## See Also +## See also -- [Relationships](relationships.md) — foreign keys, one-to-many, many-to-many -- [Queries](queries.md) — fetching and filtering data -- [Mutations](mutations.md) — creating, updating, and deleting records -- [Schema Management](schema/overview.md) — how fields become DDL, and how a change reaches the database -- [Identity Map](../concepts/identity-map.md) — instance caching semantics +- [Relationships](relationships.md): foreign keys, one-to-many and many-to-many. +- [Queries](queries.md): fetching and filtering rows. +- [Mutations](mutations.md): creating, updating and deleting rows. +- [Schema management](schema/overview.md): how fields become DDL, and how a change reaches the database. +- [`Model`](../api/model.md) in the API reference. diff --git a/docs/pages/guide/mutations.md b/docs/pages/guide/mutations.md index e359b5b4..e9077487 100644 --- a/docs/pages/guide/mutations.md +++ b/docs/pages/guide/mutations.md @@ -1,269 +1,768 @@ +--- +description: "Insert, update and delete rows with create(), save(), batch update() and delete(), and the insert-or-update helpers." +--- + # Mutations -Creating, updating, and deleting records. All mutations are executed by the Rust engine, and all of them participate in an active [transaction](transactions.md) automatically. +This guide shows how to insert, update and delete rows: `create()`, `save()`, batch `update()` and `delete()`, and the insert-or-update helpers. -Ferro's write surface is three verbs with three distinct intents — none of them ever silently does more work than you asked for: +Every call runs inside a session. Connect, open one, and create a customer: -| Verb | Intent | On a primary-key / unique conflict | -| --- | --- | --- | -| `Model.create(**fields)` | Insert a new row | Raises [`UniqueViolationError`](../api/exceptions.md) | -| `instance.save()` | Persist *this* instance | INSERT if never persisted, otherwise UPDATE by primary key | -| `Model.upsert(**fields)` | Insert **or** replace by primary key | Updates the existing row | +```python +--8<-- "docs/examples/mutations.py:session" +``` -## Creating Records +`create()` returns the saved instance, with the primary key the database assigned and `plan` filled from its default. The blocks below run inside this session; see [Sessions](connections.md#sessions). -### create +Every example on this page is followed by the SQL it sends, captured from Ferro when the example runs. `create()` sends one `INSERT`. PostgreSQL also casts the `datetime` parameter to `timestamptz` and asks for the new `id` back with `RETURNING`: -`Model.create(**fields)` validates, inserts, and returns the persisted instance in one call. For inserting many rows, `Model.bulk_create(instances)` writes the whole batch in bulk and returns the inserted count: +=== "SQLite" -```python ---8<-- "docs/examples/quickstart.py:create" -``` + ```sql + -- mutations.py:session (sqlite) + INSERT INTO "customer" ("email", "name", "plan", "views", "updated_at") + VALUES (?, ?, ?, ?, ?); + ``` + +=== "PostgreSQL" + + ```sql + -- mutations.py:session (postgres) + INSERT INTO "customer" ("email", "name", "plan", "views", "updated_at") + VALUES ($1, $2, $3, $4, CAST($5 AS timestamptz)) + RETURNING "id"; + ``` + +On SQLite the driver reports the new row's `id` without a `RETURNING` clause. + +The examples use one model, a customer with a unique email: + +=== "Assignment" + + ```python + --8<-- "docs/examples/mutations.py:models" + ``` + +=== "Annotated" + + ```python + --8<-- "docs/examples/mutations_annotated.py:models" + ``` + +Because `email` is unique, a second customer with the same email is refused. + +Ferro has four ways to write a row, and each does only what its name says: + +| Call | Intent | Behaviour | +| --- | --- | --- | +| `Customer.create(**fields)` | Insert a new row | Raises [`UniqueViolationError`](../api/exceptions.md#ferro.UniqueViolationError) on a duplicate primary key or unique value | +| `Customer.bulk_create(instances)` | Insert many new rows | Inserts the whole batch or none of it | +| `customer.save()` | Persist this instance | Inserts it if it was never saved, otherwise updates its row by primary key | +| `Customer.upsert(**fields)` | Insert or replace by primary key | Updates the existing row on a primary-key conflict | -Batch size is unbounded. Both backends cap how many parameters one statement may bind (SQLite 32,766; PostgreSQL 65,535), so Ferro splits a large batch across as many INSERT statements as the active backend requires — an implementation detail you never see. Atomicity is preserved either way: inside a [`transaction()`](transactions.md) block that transaction is the boundary, and a bare `bulk_create` call is all-or-nothing — if any row fails (say, a unique violation late in the batch), no rows from the batch are inserted. +## Creating rows -`create()` is a plain INSERT — it never updates an existing row. A duplicate primary key or unique value raises [`UniqueViolationError`](../api/exceptions.md), and the existing row is left untouched: +`create()` validates the fields, inserts the row and returns the saved instance. `bulk_create()` inserts a list of instances and returns how many it inserted: ```python ---8<-- "docs/examples/mutations.py:create-strict" +--8<-- "docs/examples/mutations.py:create" ``` -Reach for [`upsert()`](#upsert) when you want insert-or-update semantics. +The table now holds four customers. Fields you leave out take their defaults, so Carol's `plan` is `"free"`. -Pass related instances directly (`author=alice`) or set the shadow foreign-key column (`author_id=alice.id`) — see [Relationships](relationships.md). Pydantic validation runs when each instance is constructed, so invalid data raises *before* the database is touched. +`create()`, `bulk_create()` and `count()` send these statements. PostgreSQL also casts the `datetime` parameter to `timestamptz`, and its single-row insert returns the new `id`: -### Defaults +=== "SQLite" -Fields with `default` or `default_factory` fill themselves in: + ```sql + -- mutations.py:create (sqlite) + INSERT INTO "customer" ("email", "name", "plan", "views", "updated_at") + VALUES (?, ?, ?, ?, ?); -=== "Assignment" + INSERT INTO "customer" ("email", "name", "plan", "views", "updated_at") + VALUES (?, ?, ?, ?, ?), (?, ?, ?, ?, ?); - ```python - from datetime import datetime + SELECT COUNT(*) + FROM "customer" + WHERE TRUE; + ``` - from ferro import Field, Model +=== "PostgreSQL" + ```sql + -- mutations.py:create (postgres) + INSERT INTO "customer" ("email", "name", "plan", "views", "updated_at") + VALUES ($1, $2, $3, $4, CAST($5 AS timestamptz)) + RETURNING "id"; - class Article(Model): - id: int | None = Field(default=None, primary_key=True) - title: str - draft: bool = True - created_at: datetime = Field(default_factory=datetime.now) + INSERT INTO "customer" ("email", "name", "plan", "views", "updated_at") + VALUES ($1, $2, $3, $4, CAST($5 AS timestamptz)), ($6, $7, $8, $9, CAST($10 AS timestamptz)); + + SELECT COUNT(*) + FROM "customer" + WHERE TRUE; ``` -=== "Annotated" +`bulk_create()` sends one multi-row `INSERT` with no `RETURNING`, which is why it writes no IDs back. + +To create a row with a foreign key, see [Relationships](relationships.md). Pydantic validates each instance when it is built, so invalid data raises before Ferro sends anything to the database. + +A batch can be any size. SQLite binds at most 32,766 parameters in one statement and PostgreSQL 65,535, so Ferro splits a large batch across as many `INSERT` statements as the database needs. The batch stays atomic: inside a [`transaction()`](transactions.md) block the transaction is the boundary, and outside one, a failing row (a unique violation late in the batch, say) leaves none of the batch inserted. + +!!! note "`bulk_create()` skips the identity map" + For speed, `bulk_create()` sends the instances straight to the database. It does not register them in the session's [identity map](../concepts/identity-map.md) and does not write generated IDs back onto them. Query the rows again when you need tracked instances: ```python - from datetime import datetime - from typing import Annotated + --8<-- "docs/examples/mutations.py:bulk-identity" + ``` + + The instance passed to `bulk_create()` still has `id = None`; the one loaded back has the database's ID, `5`. + +The batch and the lookup in that note send these statements. PostgreSQL also casts the `datetime` parameter to `timestamptz`: + +=== "SQLite" - from ferro import Field, Model + ```sql + -- mutations.py:bulk-identity (sqlite) + INSERT INTO "customer" ("email", "name", "plan", "views", "updated_at") + VALUES (?, ?, ?, ?, ?); + SELECT "customer"."email", "customer"."id", "customer"."name", "customer"."plan", "customer"."updated_at", "customer"."views" + FROM "customer" + WHERE "customer"."email" = ? + LIMIT ?; + ``` + +=== "PostgreSQL" - class Article(Model): - id: Annotated[int | None, Field(default=None, primary_key=True)] - title: str - draft: bool = True - created_at: Annotated[datetime, Field(default_factory=datetime.now)] + ```sql + -- mutations.py:bulk-identity (postgres) + INSERT INTO "customer" ("email", "name", "plan", "views", "updated_at") + VALUES ($1, $2, $3, $4, CAST($5 AS timestamptz)); + + SELECT "customer"."email", "customer"."id", "customer"."name", "customer"."plan", "customer"."updated_at", "customer"."views" + FROM "customer" + WHERE "customer"."email" = $1 + LIMIT $2; ``` +The batch's `INSERT` returns no `id` on either database. + +`create()` always inserts. A duplicate primary key or unique value raises `UniqueViolationError` and leaves the existing row as it was: + ```python -article = await Article.create(title="Hello") -# article.draft is True, article.created_at is set +--8<-- "docs/examples/mutations.py:create-strict" ``` -## Saving: INSERT or UPDATE +Alice's row still holds the name `"Alice"`. Use [`upsert()`](#upsert) when you want insert-or-update instead. + +The refused insert and the lookup send these statements. PostgreSQL also casts the `datetime` parameter to `timestamptz` and adds `RETURNING "id"`: -Every instance is either **transient** (constructed with `Model(...)` and never persisted) or **persisted** (fetched from the database, or successfully saved). `save()` uses that state to decide what to do: +=== "SQLite" -- a transient instance is **INSERTed** — a duplicate primary key or unique value raises `UniqueViolationError`, exactly like `create()`; -- a persisted instance is **UPDATEd** by primary key (`UPDATE ... WHERE pk = ?`). + ```sql + -- mutations.py:create-strict (sqlite) + INSERT INTO "customer" ("email", "name", "plan", "views", "updated_at") + VALUES (?, ?, ?, ?, ?); + + SELECT "customer"."email", "customer"."id", "customer"."name", "customer"."plan", "customer"."updated_at", "customer"."views" + FROM "customer" + WHERE "customer"."id" = ? + LIMIT ?; + ``` + +=== "PostgreSQL" + + ```sql + -- mutations.py:create-strict (postgres) + INSERT INTO "customer" ("email", "name", "plan", "views", "updated_at") + VALUES ($1, $2, $3, $4, CAST($5 AS timestamptz)) + RETURNING "id"; + + SELECT "customer"."email", "customer"."id", "customer"."name", "customer"."plan", "customer"."updated_at", "customer"."views" + FROM "customer" + WHERE "customer"."id" = $1 + LIMIT $2; + ``` + +The database refused the `INSERT`; `UniqueViolationError` carries its report. + +## Saving: insert or update + +An instance is either **transient** (built with `Customer(...)` and never saved) or **persisted** (loaded from the database, or saved). `save()` reads that state: it inserts a transient instance and updates a persisted one by primary key: ```python --8<-- "docs/examples/mutations.py:save-insert-update" ``` -`instance.delete()` returns the instance to transient, so a subsequent `save()` INSERTs a new row. `refresh()` keeps it persisted. +The first `save()` inserted Frank and set his `id`; the second updated the same row, so there is still one Frank. A transient instance with a duplicate primary key or unique value raises `UniqueViolationError`, as `create()` does. + +The two saves and the count send these statements. PostgreSQL also casts the `datetime` parameter to `timestamptz` and returns the new `id`: + +=== "SQLite" + + ```sql + -- mutations.py:save-insert-update (sqlite) + INSERT INTO "customer" ("email", "name", "plan", "views", "updated_at") + VALUES (?, ?, ?, ?, ?); + + UPDATE "customer" + SET "email" = ?, "name" = ?, "plan" = ?, "views" = ?, "updated_at" = ? + WHERE "id" = ?; + + SELECT COUNT(*) + FROM "customer" + WHERE "customer"."name" = ?; + ``` + +=== "PostgreSQL" + + ```sql + -- mutations.py:save-insert-update (postgres) + INSERT INTO "customer" ("email", "name", "plan", "views", "updated_at") + VALUES ($1, $2, $3, $4, CAST($5 AS timestamptz)) + RETURNING "id"; + + UPDATE "customer" + SET "email" = $1, "name" = $2, "plan" = $3, "views" = $4, "updated_at" = CAST($5 AS timestamptz) + WHERE "id" = $6; + + SELECT COUNT(*) + FROM "customer" + WHERE "customer"."name" = $1; + ``` -### Partial UPDATE with `only=` / `exclude=` +The second `save()` wrote every column, by primary key. -A loaded instance can persist an explicit write-set. `only=` is the allowlist — other columns on the row are left alone: +If the row behind a persisted instance is gone (another writer deleted it, or you changed the primary key before saving), the update matches nothing. `save()` then raises `ModelDoesNotExist` instead of inserting the row again: + +```python +--8<-- "docs/examples/mutations.py:save-missing" +``` + +Frank stays deleted. `instance.delete()` makes an instance transient again, so a later `save()` on it inserts a new row. + +The batch delete and the save send these statements. PostgreSQL also casts the `datetime` parameter to `timestamptz`: + +=== "SQLite" + + ```sql + -- mutations.py:save-missing (sqlite) + DELETE + FROM "customer" + WHERE "customer"."id" = ?; + + UPDATE "customer" + SET "email" = ?, "name" = ?, "plan" = ?, "views" = ?, "updated_at" = ? + WHERE "id" = ?; + ``` + +=== "PostgreSQL" + + ```sql + -- mutations.py:save-missing (postgres) + DELETE + FROM "customer" + WHERE "customer"."id" = $1; + + UPDATE "customer" + SET "email" = $1, "name" = $2, "plan" = $3, "views" = $4, "updated_at" = CAST($5 AS timestamptz) + WHERE "id" = $6; + ``` + +The `UPDATE` matched no row, so `save()` raised instead of sending an `INSERT`. + +!!! note "Copies share persistence state" + `model_copy()` copies the persistence state too, so saving a copy of a persisted instance updates the same row. To clone a row, build a new instance. + +### Partial updates with `only=` and `exclude=` + +`save(only=...)` writes only the columns you name and leaves the rest of the row alone: ```python --8<-- "docs/examples/mutations.py:save-only" ``` -`exclude=` is the denylist — every persisted column except those names is written. `exclude=set()` is a full write: +The row kept `plan = "pro"`, while the instance still held `"enterprise"` until `refresh()` reloaded it. Ferro runs no `SELECT` after the update, which is why the instance and the row can differ. + +The fetch, the partial save and the refresh send these statements. PostgreSQL also casts the `datetime` parameter to `timestamptz`: + +=== "SQLite" + + ```sql + -- mutations.py:save-only (sqlite) + SELECT "customer"."email", "customer"."id", "customer"."name", "customer"."plan", "customer"."updated_at", "customer"."views" + FROM "customer" + WHERE "customer"."id" = ? + LIMIT ?; + + UPDATE "customer" + SET "name" = ?, "updated_at" = ? + WHERE "id" = ?; + + SELECT "customer"."email", "customer"."id", "customer"."name", "customer"."plan", "customer"."updated_at", "customer"."views" + FROM "customer" + WHERE "customer"."id" = ? + LIMIT ?; + ``` + +=== "PostgreSQL" + + ```sql + -- mutations.py:save-only (postgres) + SELECT "customer"."email", "customer"."id", "customer"."name", "customer"."plan", "customer"."updated_at", "customer"."views" + FROM "customer" + WHERE "customer"."id" = $1 + LIMIT $2; + + UPDATE "customer" + SET "name" = $1, "updated_at" = CAST($2 AS timestamptz) + WHERE "id" = $3; + + SELECT "customer"."email", "customer"."id", "customer"."name", "customer"."plan", "customer"."updated_at", "customer"."views" + FROM "customer" + WHERE "customer"."id" = $1 + LIMIT $2; + ``` + +The `UPDATE` sets only `name` and `updated_at`. + +`save(exclude=...)` writes every column except the ones you name: ```python --8<-- "docs/examples/mutations.py:save-exclude" ``` -Rules: +The name changed and the plan did not. -- `only=` / `exclude=` are a persisted UPDATE. Transient `save(only=…)` / `save(exclude=…)` and the same with `on_conflict="update"` raise `ValueError` — persist the row first, or use [`Query.update()`](#recipe-updates) for set-oriented / expression writes (`n+1`, `now`, `.merge()`). -- Pass one write-set, not both: `save(only=…, exclude=…)` raises `ValueError`. -- Names are persisted column names (`messages`, shadow `user_id`). Relation / BackRef / M2M names are rejected (the error names the shadow `*_id` when one exists). Unknown names are rejected with the legal column list. -- The primary key may appear in either set and is never SET. `only=set()`, `only={pk}`, and `exclude=` of every non-PK column raise — the write-set would be empty. A PK-only model's **bare** `save()` is still today's existence check. -- Named columns write current memory, including `None` → `NULL`. Mixins that assign `updated_at` then call `super().save(**kwargs)` forward `only=` / `exclude=` without expanding the write-set. -- There is no post-UPDATE `SELECT`. The instance and identity-map entry stay as they are, so an omitted in-memory mutation can diverge from the database until you `refresh()`. +The save and the refresh send these statements. PostgreSQL also casts the `datetime` parameter to `timestamptz`: -If the row behind a persisted instance no longer exists — deleted by another writer, or you mutated the primary-key field before saving — the UPDATE matches nothing and `save()` raises `ModelDoesNotExist` rather than silently resurrecting the row. +=== "SQLite" -!!! note "Copies share persistence state" - `model_copy()` copies persistence state: saving a copy of a persisted instance UPDATEs the **same row**. To clone a row, construct a fresh instance instead. See [Identity Map](../concepts/identity-map.md) for how instances are tracked per connection. + ```sql + -- mutations.py:save-exclude (sqlite) + UPDATE "customer" + SET "email" = ?, "name" = ?, "views" = ?, "updated_at" = ? + WHERE "id" = ?; + + SELECT "customer"."email", "customer"."id", "customer"."name", "customer"."plan", "customer"."updated_at", "customer"."views" + FROM "customer" + WHERE "customer"."id" = ? + LIMIT ?; + ``` -## Upsert +=== "PostgreSQL" + + ```sql + -- mutations.py:save-exclude (postgres) + UPDATE "customer" + SET "email" = $1, "name" = $2, "views" = $3, "updated_at" = CAST($4 AS timestamptz) + WHERE "id" = $5; + + SELECT "customer"."email", "customer"."id", "customer"."name", "customer"."plan", "customer"."updated_at", "customer"."views" + FROM "customer" + WHERE "customer"."id" = $1 + LIMIT $2; + ``` -`Model.upsert(**fields)` inserts the row, or updates the existing row when the **primary key** conflicts: +Every column but `plan` and the primary key is in the `SET`. + +Two rules matter most. A partial save is an update, so it needs a persisted instance: on a transient one, `only=` and `exclude=` raise `ValueError`. And a call takes one write-set, not both. The [`Model.save`](../api/model.md#ferro.Model.save) reference lists the rest: which names count as columns, and which write-sets are empty and refused. + +## Batch updates and deletes + +`update(**values)` and `delete()` on a query change every matching row in one statement, without loading instances. Each returns the number of rows it changed: ```python ---8<-- "docs/examples/mutations.py:upsert" +--8<-- "docs/examples/mutations.py:update-delete" ``` -Things to know: +Three free customers moved to the `"trial"` plan, and Erin's row is gone. -- The conflict target is the primary key only. A conflict on some *other* unique column still raises `UniqueViolationError`. -- The whole row is written on update — fields you leave unset are written with their defaults, they do not preserve the stored values (see the `name` field above). -- With an autoincrement primary key left unset there is nothing to conflict on, so the call degrades to a plain INSERT. -- `upsert()` is sugar for the primitive `instance.save(on_conflict="update")`, which applies insert-or-update semantics to an instance you already hold, regardless of its persistence state. +The update and the delete send these statements; on PostgreSQL only the placeholders differ: + +```sql +-- mutations.py:update-delete (sqlite) +UPDATE "customer" +SET "plan" = ? +WHERE "customer"."plan" = ?; + +DELETE +FROM "customer" +WHERE "customer"."name" = ?; +``` -## Get-or-Create +Each is one statement over every matching row, with the query's filter as its `WHERE`. -`get_or_create(defaults={...}, **filters)` looks up a row by exact-match filters and creates it when missing. It returns an `(instance, created)` tuple; `defaults` are applied **only** on the create path: +!!! warning "Batch updates do not touch instances you hold" + `where(...).update(...)` writes to the database only. Instances already in memory, including those in the identity map, keep their old values until you call `refresh()` on them. + +### Expression updates + +`update()` with keyword arguments sets literal values. To compute a value in the database, pass a lambda that returns a dict of column expressions: ```python ---8<-- "docs/examples/mutations.py:get-or-create" +--8<-- "docs/examples/mutations.py:recipe-update" ``` -## Update-or-Create +The database added 1 to Alice's `views` and stamped `updated_at` with its own clock. -`update_or_create(defaults={...}, **filters)` has the same shape, but when a match exists it applies `defaults` to the instance and saves it: +The update and the refresh send these statements; on PostgreSQL only the placeholders differ: -```python ---8<-- "docs/examples/mutations.py:update-or-create" +```sql +-- mutations.py:recipe-update (sqlite) +UPDATE "customer" +SET "views" = "views" + ?, "updated_at" = CURRENT_TIMESTAMP +WHERE "customer"."id" = ?; + +SELECT "customer"."email", "customer"."id", "customer"."name", "customer"."plan", "customer"."updated_at", "customer"."views" +FROM "customer" +WHERE "customer"."id" = ? +LIMIT ?; ``` -!!! note "Concurrency" - Both helpers are a read followed by a write, not a single atomic upsert. Under concurrent writers, two processes can race past the lookup; with a unique constraint on the filter columns the loser's INSERT raises [`UniqueViolationError`](../api/exceptions.md) — catch it and retry the get. For an atomic insert-or-update keyed on the primary key, use [`upsert()`](#upsert). +`customer.views + 1` became `"views" + ?`, and `now` became `CURRENT_TIMESTAMP`. + +`+` and `-` follow SQL, so `NULL + 1` is `NULL`. `now` is a value you import (`from ferro import now`), not a function you call, and it can only be assigned to a `datetime` column. -## Batch Updates +On PostgreSQL, `.merge()` merges a dict into a JSON object column, with the patch winning where keys overlap. It replaces top-level keys only. A `NULL` column counts as `{}`. The examples use a conversation whose `turns` column holds a JSON object: -Update many rows in one statement — no instances are loaded — and delete the same way. `update(**values)` and `delete()` are query terminals that return the affected row count: +=== "Assignment" + + ```python + --8<-- "docs/examples/mutations.py:conversation" + ``` + +=== "Annotated" + + ```python + --8<-- "docs/examples/mutations_annotated.py:conversation" + ``` + +This block needs a PostgreSQL database. It creates a conversation and merges one turn into it: ```python ---8<-- "docs/examples/quickstart.py:update-delete" +--8<-- "docs/examples/mutations.py:merge" ``` -!!! warning "Batch updates bypass in-memory instances" - A `where(...).update(...)` writes directly to the database. Instances you already hold (including identity-mapped ones) are **not** mutated — call `refresh()` on them if you need the new values. +After the update, `turns` holds `{"r1": {"role": "user"}}`. The create, the update and the refresh send these statements: -### No limit/offset on mutations +```sql +-- mutations.py:merge (postgres) +INSERT INTO "conversation" ("turns") +VALUES (CAST($1 AS json)) +RETURNING "id"; -Portable SQL has no `DELETE ... LIMIT` or `UPDATE ... LIMIT`, so calling `update()` or `delete()` on a query with `limit()` or `offset()` set raises `ValueError` instead of silently mutating every matching row. To mutate a bounded subset, fetch primary keys first and mutate by primary-key set: +UPDATE "conversation" +SET "turns" = (COALESCE("turns"::jsonb, '{}'::jsonb)) || CAST($1 AS jsonb) +WHERE "conversation"."id" = $2; + +SELECT "conversation"."id", "conversation"."turns" +FROM "conversation" +WHERE "conversation"."id" = $1 +LIMIT $2; +``` + +The `||` operator merges the patch into the column read as `jsonb`, and `COALESCE` stands in `{}` for a `NULL` column. `.merge()` raises when the query is built if the connection is SQLite, the column holds a list, or the patch is a list. + +### No `limit()` on batch writes + +SQL has no portable `UPDATE ... LIMIT` or `DELETE ... LIMIT`. Rather than change every matching row, `update()` and `delete()` raise `ValueError` on a query with `limit()`, `offset()`, `after()` or `before()`: ```python --8<-- "docs/examples/mutations.py:mutation-guard" ``` -## Refreshing from the Database +The error prints: -`refresh()` reloads an instance from its primary key, discarding local state: + +```text +ValueError: delete() does not support limit/offset/after/before: portable SQL has no DELETE ... LIMIT. Remove the .limit()/.offset()/.after()/.before() call, or fetch primary keys first and delete by primary-key set. +``` + +To change a bounded set of rows, fetch their primary keys first, then filter on them: ```python ---8<-- "docs/examples/mutations.py:refresh" +--8<-- "docs/examples/mutations.py:mutation-by-key" ``` -It raises `RuntimeError` if the instance has no primary key or the row no longer exists. +Only Alice, the oldest trial customer, was deleted. -## Deleting +The fetch and the delete send these statements; on PostgreSQL only the placeholders differ: -Delete a single instance, or batch-delete via a query: +```sql +-- mutations.py:mutation-by-key (sqlite) +SELECT "customer"."email", "customer"."id", "customer"."name", "customer"."plan", "customer"."updated_at", "customer"."views" +FROM "customer" +WHERE "customer"."plan" = ? +ORDER BY "customer"."id" ASC NULLS LAST +LIMIT ?; + +DELETE +FROM "customer" +WHERE "customer"."id" IN (?); +``` + +The `DELETE` filters on the fetched keys with `IN`. + +## Deleting one row + +`instance.delete()` deletes the row behind an instance: ```python -user = await User.get_or_none(42) -if user is not None: - await user.delete() +--8<-- "docs/examples/mutations.py:delete-one" +``` -removed = await User.where(lambda user: user.archived == True).delete() # noqa: E712 +Carol's row is gone. Deleting a parent row applies the `on_delete` rule of every foreign key that points at it, `CASCADE` by default; read [Delete behavior](relationships.md#delete-behavior) before deleting rows that have children. + +The lookup, the delete and the check send these statements; on PostgreSQL only the placeholders differ: + +```sql +-- mutations.py:delete-one (sqlite) +SELECT "customer"."email", "customer"."id", "customer"."name", "customer"."plan", "customer"."updated_at", "customer"."views" +FROM "customer" +WHERE "customer"."email" = ? +LIMIT ?; + +DELETE +FROM "customer" +WHERE "customer"."id" = ?; + +SELECT "customer"."email", "customer"."id", "customer"."name", "customer"."plan", "customer"."updated_at", "customer"."views" +FROM "customer" +WHERE "customer"."id" = ? +LIMIT ?; ``` -Deleting a parent row triggers the `on_delete` behavior of any foreign keys pointing at it — `CASCADE` by default. See [Delete Behavior](relationships.md#delete-behavior) before deleting rows with children. +`carol.delete()` deletes one row by primary key. -## Handling Database Errors +## `upsert()` -Every database failure raises a typed exception from the DBAPI-shaped tree rooted at `FerroError` — you never need to match on driver message text. Constraint violations are subclasses of `IntegrityError`, and the original driver detail is preserved as attributes: +`upsert(**fields)` inserts the row, or updates the existing row when the primary key already exists: ```python ---8<-- "docs/examples/mutations.py:handling-errors" +--8<-- "docs/examples/mutations.py:upsert" ``` -See the [Exceptions API reference](../api/exceptions.md) for the full hierarchy. +The second call updated row 100 in place. It wrote the whole row, so `name` went back to its default `""`: pass every field you want to keep. + +The two upserts, the count and the lookup send these statements. PostgreSQL also casts the `datetime` parameter to `timestamptz`: + +=== "SQLite" + + ```sql + -- mutations.py:upsert (sqlite) + INSERT INTO "customer" ("id", "email", "name", "plan", "views", "updated_at") + VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT ("id") DO UPDATE + SET "email" = "excluded"."email", "name" = "excluded"."name", "plan" = "excluded"."plan", "views" = "excluded"."views", "updated_at" = "excluded"."updated_at"; + + INSERT INTO "customer" ("id", "email", "name", "plan", "views", "updated_at") + VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT ("id") DO UPDATE + SET "email" = "excluded"."email", "name" = "excluded"."name", "plan" = "excluded"."plan", "views" = "excluded"."views", "updated_at" = "excluded"."updated_at"; + + SELECT COUNT(*) + FROM "customer" + WHERE "customer"."id" = ?; + + SELECT "customer"."email", "customer"."id", "customer"."name", "customer"."plan", "customer"."updated_at", "customer"."views" + FROM "customer" + WHERE "customer"."id" = ? + LIMIT ?; + ``` + +=== "PostgreSQL" + + ```sql + -- mutations.py:upsert (postgres) + INSERT INTO "customer" ("id", "email", "name", "plan", "views", "updated_at") + VALUES ($1, $2, $3, $4, $5, CAST($6 AS timestamptz)) + ON CONFLICT ("id") DO UPDATE + SET "email" = "excluded"."email", "name" = "excluded"."name", "plan" = "excluded"."plan", "views" = "excluded"."views", "updated_at" = "excluded"."updated_at"; + + INSERT INTO "customer" ("id", "email", "name", "plan", "views", "updated_at") + VALUES ($1, $2, $3, $4, $5, CAST($6 AS timestamptz)) + ON CONFLICT ("id") DO UPDATE + SET "email" = "excluded"."email", "name" = "excluded"."name", "plan" = "excluded"."plan", "views" = "excluded"."views", "updated_at" = "excluded"."updated_at"; + + SELECT COUNT(*) + FROM "customer" + WHERE "customer"."id" = $1; + + SELECT "customer"."email", "customer"."id", "customer"."name", "customer"."plan", "customer"."updated_at", "customer"."views" + FROM "customer" + WHERE "customer"."id" = $1 + LIMIT $2; + ``` -## Bulk Operations and the Identity Map +Each upsert is one `INSERT ... ON CONFLICT ("id") DO UPDATE` that sets every column from the row it tried to insert, which is why `name` was reset. -By default Ferro keeps a per-connection [identity map](../concepts/identity-map.md): loading the same primary key twice yields the same Python object, and `create()`/`save()` register instances in it. +The conflict target is the primary key only, so a conflict on another unique column still raises `UniqueViolationError`. Without a primary key there is nothing to conflict on, and the call inserts a new row. `upsert()` is shorthand for `instance.save(on_conflict="update")`, which does the same for an instance you already hold. -`bulk_create()` is the deliberate exception — it serializes the given instances straight to the engine and **skips the identity map** for throughput. The instances you passed in are not registered (and auto-generated IDs are not written back onto them); re-query the rows when you need tracked instances. +## `get_or_create()` and `update_or_create()` + +`get_or_create(defaults={...}, **filters)` looks up a row by exact-match filters and creates it if none matches. It returns `(instance, created)`, and applies `defaults` only when it creates: ```python -inserted = await User.bulk_create([User(name="a", age=1), User(name="b", age=2)]) -fresh = await User.where(lambda user: user.name.in_(["a", "b"])).all() +--8<-- "docs/examples/mutations.py:get-or-create" ``` -## Recipe updates +The first call created Heidi; the second found the same row. + +The two calls send these statements. PostgreSQL also casts the `datetime` parameter to `timestamptz` and returns the new `id`: + +=== "SQLite" + + ```sql + -- mutations.py:get-or-create (sqlite) + SELECT "customer"."email", "customer"."id", "customer"."name", "customer"."plan", "customer"."updated_at", "customer"."views" + FROM "customer" + WHERE "customer"."email" = ? + LIMIT ?; + + INSERT INTO "customer" ("email", "name", "plan", "views", "updated_at") + VALUES (?, ?, ?, ?, ?); -Keyword `update(name="x")` stays literals-only. A recipe callable assigns expressions that run in the database — increment a counter, copy a column, stamp `updated_at` with the database clock, or shallow-merge a JSON object: + SELECT "customer"."email", "customer"."id", "customer"."name", "customer"."plan", "customer"."updated_at", "customer"."views" + FROM "customer" + WHERE "customer"."email" = ? + LIMIT ?; + ``` + +=== "PostgreSQL" + + ```sql + -- mutations.py:get-or-create (postgres) + SELECT "customer"."email", "customer"."id", "customer"."name", "customer"."plan", "customer"."updated_at", "customer"."views" + FROM "customer" + WHERE "customer"."email" = $1 + LIMIT $2; + + INSERT INTO "customer" ("email", "name", "plan", "views", "updated_at") + VALUES ($1, $2, $3, $4, CAST($5 AS timestamptz)) + RETURNING "id"; + + SELECT "customer"."email", "customer"."id", "customer"."name", "customer"."plan", "customer"."updated_at", "customer"."views" + FROM "customer" + WHERE "customer"."email" = $1 + LIMIT $2; + ``` + +The first call looked the row up, found nothing and inserted it; the second stopped after the lookup. + +`update_or_create(defaults={...}, **filters)` takes the same arguments, but when a row matches it applies `defaults` to it and saves it: ```python ---8<-- "docs/examples/mutations.py:recipe-update" +--8<-- "docs/examples/mutations.py:update-or-create" ``` -`+` and `-` are honest SQL: `NULL + 1` is NULL. They do not fill empties. `.merge()` does — a NULL object column is treated as `{}` (`COALESCE`). `now` is a singleton import (`from ferro import now`), not `now()`, and only assigns to a `datetime` column. +Heidi's row now has the `"pro"` plan. -`.merge()` is Postgres-only shallow object merge (right-hand patch wins on key overlap). It is not deep merge, not `jsonb_set`, and not key-remove. After `connect`, SQLite, a list-typed column, or a list patch fail at build time (list concat is `.concat()`, not shipped yet). The dialect check uses the same connection the UPDATE will run on (ambient or explicit session, not only the default engine). +The call sends these statements. PostgreSQL also casts the `datetime` parameter to `timestamptz`: -=== "Assignment" +=== "SQLite" - ```python - from ferro import Field, Model + ```sql + -- mutations.py:update-or-create (sqlite) + SELECT "customer"."email", "customer"."id", "customer"."name", "customer"."plan", "customer"."updated_at", "customer"."views" + FROM "customer" + WHERE "customer"."email" = ? + LIMIT ?; + UPDATE "customer" + SET "email" = ?, "name" = ?, "plan" = ?, "views" = ?, "updated_at" = ? + WHERE "id" = ?; + ``` - class Conversation(Model): - id: int | None = Field(default=None, primary_key=True) - turns: dict = Field(default_factory=dict) +=== "PostgreSQL" + ```sql + -- mutations.py:update-or-create (postgres) + SELECT "customer"."email", "customer"."id", "customer"."name", "customer"."plan", "customer"."updated_at", "customer"."views" + FROM "customer" + WHERE "customer"."email" = $1 + LIMIT $2; - await Conversation.where(lambda conversation: conversation.id == cid).update( - lambda conversation: {"turns": conversation.turns.merge({run_id: entry})} - ) + UPDATE "customer" + SET "email" = $1, "name" = $2, "plan" = $3, "views" = $4, "updated_at" = CAST($5 AS timestamptz) + WHERE "id" = $6; ``` -=== "Annotated" +It looked the row up, then saved it with an `UPDATE` by primary key. - ```python - from typing import Annotated +!!! note "Not atomic" + Both helpers read, then write. Two writers can both miss the row and both try to insert it; with a unique constraint on the filter columns, the second insert raises [`UniqueViolationError`](../api/exceptions.md#ferro.UniqueViolationError), so catch it and look the row up again. For an atomic insert-or-update on the primary key, use [`upsert()`](#upsert). + +## `refresh()` + +`refresh()` reloads an instance from its row and discards its in-memory values: + +```python +--8<-- "docs/examples/mutations.py:refresh" +``` + +The batch update changed the row but not the instance; `refresh()` brought the new name in. It raises `RuntimeError` if the instance has no primary key or its row no longer exists. + +The update and the refresh send these statements; on PostgreSQL only the placeholders differ: + +```sql +-- mutations.py:refresh (sqlite) +UPDATE "customer" +SET "name" = ? +WHERE "customer"."id" = ?; + +SELECT "customer"."email", "customer"."id", "customer"."name", "customer"."plan", "customer"."updated_at", "customer"."views" +FROM "customer" +WHERE "customer"."id" = ? +LIMIT ?; +``` - from ferro import Field, Model +`refresh()` is a `SELECT` by primary key. +## Database errors - class Conversation(Model): - id: Annotated[int | None, Field(default=None, primary_key=True)] - turns: Annotated[dict, Field(default_factory=dict)] +Every database failure raises a typed exception under `FerroError`, so you never match on driver message text. Constraint violations are subclasses of `IntegrityError` and keep the driver's details as attributes: + +```python +--8<-- "docs/examples/mutations.py:handling-errors" +``` + +On SQLite it prints: + + +```text +2067 +None +UNIQUE constraint failed: customer.email +``` + +`sqlstate` is the database's error code (`"23505"` on PostgreSQL, `"2067"` on SQLite), `constraint` names the violated constraint on PostgreSQL and is `None` on SQLite, and `driver_message` is the driver's own text, for logs. The [exceptions reference](../api/exceptions.md) lists the whole hierarchy. + +The refused statement is this `INSERT`. PostgreSQL also casts the `datetime` parameter to `timestamptz` and adds `RETURNING "id"`: + +=== "SQLite" + + ```sql + -- mutations.py:handling-errors (sqlite) + INSERT INTO "customer" ("email", "name", "plan", "views", "updated_at") + VALUES (?, ?, ?, ?, ?); + ``` +=== "PostgreSQL" - await Conversation.where(lambda conversation: conversation.id == cid).update( - lambda conversation: {"turns": conversation.turns.merge({run_id: entry})} - ) + ```sql + -- mutations.py:handling-errors (postgres) + INSERT INTO "customer" ("email", "name", "plan", "views", "updated_at") + VALUES ($1, $2, $3, $4, CAST($5 AS timestamptz)) + RETURNING "id"; ``` -Multiplication and string concat are not in this door yet. +The database rejected it, and the exception carries the database's own report. -## See Also +## See also -- [Queries](queries.md) — fetching and filtering data -- [Exceptions](../api/exceptions.md) — the typed error hierarchy -- [Transactions](transactions.md) — grouping mutations atomically -- [Relationships](relationships.md) — creating related records, cascade rules -- [Identity Map](../concepts/identity-map.md) — instance caching semantics -- [Testing](../howto/testing.md) — testing code that mutates data +- [Transactions](transactions.md): group several writes so they commit or roll back together +- [Queries](queries.md): find the rows to change +- [Identity map](../concepts/identity-map.md): how a session tracks instances +- [`Model` reference](../api/model.md): every write method and its options diff --git a/docs/pages/guide/queries.md b/docs/pages/guide/queries.md index bc2c1c0d..057a505e 100644 --- a/docs/pages/guide/queries.md +++ b/docs/pages/guide/queries.md @@ -1,876 +1,1859 @@ -# Queries - -Ferro builds queries in Python and executes them on the Rust engine. All values are parameterized — user input is never concatenated into SQL. +--- +description: "Filter rows with lambda predicates, order and page them, reach across relations and select only the columns you need." +--- -The examples on this page use this model: +# Queries -=== "Assignment" +This guide shows how to read rows with Ferro: filter them with a lambda, order and page them, reach across relations, populate related rows and select only the columns you need. - ```python - --8<-- "docs/examples/predicates.py:setup" - ``` +Here is a whole program. It connects, opens a session, and asks for every transaction of 100 or more going out, largest first: -=== "Annotated" +```python +--8<-- "docs/examples/predicates.py:first-query" +``` - ```python - --8<-- "docs/examples/predicates_annotated.py:setup" - ``` +It prints: -## Queries Are Immutable + +```text +Rent -1450 +To savings -500 +``` -Every chain call — `.where()`, `.order_by()`, `.limit()`, `.offset()` — returns a **new** `Query`. None of them mutate the query they were called on, so a partially-built query is safe to keep around and reuse as the base for several different follow-ups: +`where()` takes a lambda whose parameter stands for one row, and `order_by()` takes a lambda naming the column to sort by. Nothing reaches the database until you await `all()`. The values in a query are always sent as parameters, never pasted into the SQL. Every query on this page runs inside a session like this one; see [Sessions](connections.md#sessions) for why and how to open one. -```python -base = User.where(lambda user: user.role == "admin") +Every example on this page is followed by the SQL it sends, captured from Ferro when the example runs. Awaiting `all()` sends this statement; on PostgreSQL only the placeholders differ, `$1` where SQLite has `?`: -page1 = base.limit(10) # first 10 admins -page2 = base.limit(10).offset(10) # next 10 admins +```sql +-- predicates.py:first-query (sqlite) +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" +FROM "transaction" +WHERE "transaction"."amount" <= ? +ORDER BY "transaction"."amount" ASC NULLS LAST; ``` -`base` still matches every admin with no `limit()` applied — building `page1` and `page2` from it doesn't change it, and `page1` and `page2` don't affect each other either. This is what makes patterns like "build a filtered base query, then branch into a count and a page of results" safe: +The query names every column of `transaction`, the filter compares `amount` with a placeholder, and `NULLS LAST` fixes where `NULL` values sort on both databases. -```python -active = User.where(lambda user: user.archived == False) # noqa: E712 +## The ledger -total = await active.count() -first_page = await active.order_by(lambda user: user.id).limit(20).all() -``` +Every example on this page queries one small ledger. An owner holds accounts, an account holds transactions, a transaction may have a category and split lines, a transfer links the two halves of a move between accounts, and tags attach to transactions many-to-many: -`active` is never consumed or altered by either await — you can keep branching off it as many times as you like. +=== "Assignment" -## Fetching by Primary Key + ```python + --8<-- "docs/examples/predicates.py:models" + ``` -`Model.get(pk)` loads exactly one row and returns your model type — not `YourModel | None`. If no row exists it raises `ModelDoesNotExist`, a `LookupError` subclass carrying `.model` and `.pk` (handy for HTTP 404s and structured logging). When a missing row is a normal outcome, use `Model.get_or_none(pk)` instead: +=== "Annotated" -```python -from ferro import ModelDoesNotExist + ```python + --8<-- "docs/examples/predicates_annotated.py:models" + ``` -user = await User.get(42) # User — raises if missing +`Category.parent` points back at `Category`. A class cannot name itself in its own annotation, so the self-reference is a string (`"Category"`) and `nullable=True` makes the column optional. -try: - user = await User.get(client_supplied_id) -except ModelDoesNotExist: - ... # e.g. return 404 from your HTTP layer +The ledger holds these nine transactions: -maybe = await User.get_or_none(999) # User | None — never raises for "not found" -``` +| `id` | `memo` | `amount` | `posted_on` | `cleared_on` | account | category | +| ---: | :--- | ---: | :--- | :--- | :--- | :--- | +| 1 | Salary | 3200.00 | 2026-03-01 | 2026-03-02 | Checking | | +| 2 | Rent | -1450.00 | 2026-03-02 | 2026-03-03 | Checking | Housing | +| 3 | Market run | -86.40 | 2026-03-05 | 2026-03-06 | Checking | Groceries | +| 4 | To savings | -500.00 | 2026-03-07 | 2026-03-08 | Checking | | +| 5 | From checking | 500.00 | 2026-03-07 | 2026-03-08 | Savings | | +| 6 | Train to Lyon | -64.00 | 2026-03-09 | | Travel | | +| 7 | Dinner in Lyon | -48.50 | 2026-03-10 | | Travel | Dining | +| 8 | Office coffee | -12.75 | 2026-03-11 | 2026-03-12 | Petty cash | Dining | +| 9 | Weekly shop | -95.00 | 2026-03-12 | | Checking | | -Both methods also exist on `Model.using("name")` for [named connections](connections.md#named-connections). +Ada Lovelace owns Checking and Savings, Grace Hopper owns Travel, and Petty cash has no owner. Groceries and Dining sit under Food. Weekly shop is split into a Groceries line of -72.00 and a Dining line of -23.00. One transfer links To savings (its outflow) with From checking (its inflow). Rent carries the tag `tax`; Train to Lyon and Dinner in Lyon carry `reimbursable`. -## Filtering with where() +## Filtering with `where()` -`Model.where(...)` starts a chainable query; terminals like `.all()` execute it. Predicates are written as lambdas — the parameter (`t` by convention) is a query proxy whose attributes stand in for your model's columns: +[`where()`][ferro.query.builder.Query.where] takes a lambda. Its parameter, named after the model in lowercase (`transaction` for `Transaction`), stands for one row; each attribute on it is a column, and comparing a column builds a condition: ```python --8<-- "docs/examples/predicates.py:filtering" ``` -`Model.select()` starts an unfiltered query — useful when you only want ordering, slicing, or a count. +The three transactions with no `cleared_on` come back. `Model.where(...)` starts a query and the `await ... .all()` runs it; `Model.select()` starts a query with no filter, for when you only want to order, page or count. + +The query sends this; on PostgreSQL only the placeholders differ: -## Predicate Style +```sql +-- predicates.py:filtering (sqlite) +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" +FROM "transaction" +WHERE "transaction"."cleared_on" IS NULL; +``` -`where()` accepts a lambda predicate — a callable that receives a query proxy and returns a comparison. The proxy's attributes are validated against your model's columns at build time (a misspelled column raises `AttributeError` naming the closest match, before any query reaches the database): +`== None` became `IS NULL`: in SQL, `= NULL` matches no row. + +Ferro checks every column name when you build the query, before anything reaches the database. A misspelled column raises `AttributeError` with the closest match and every column and relation you can use: ```python ---8<-- "docs/examples/predicates.py:lambda-style" +--8<-- "docs/examples/predicates.py:typo" ``` -Typo a column name and you find out immediately, not after the query round-trips to the database: +The list includes the `account_id` and `category_id` columns that each foreign key adds, so you can filter on them directly. -```text ->>> await User.where(lambda user: user.naem == "alice").all() -AttributeError: User has no queryable column 'naem'. Did you mean 'name'? Valid columns: age, archived, id, name, role. -``` +## Predicate style -The valid-columns list includes shadow `{fk}_id` foreign-key columns (see [Querying Across Relationships](#querying-across-relationships)), so the error is always a complete picture of what you can filter on. +A predicate is the lambda you pass to `where()`. It is the only form `where()` accepts: anything that is not callable raises `TypeError` naming the lambda form. Name the parameter after the model in lowercase singular, so the predicate reads as a sentence: `lambda transaction: transaction.amount < 0`. -Lambda predicates keep the call site fully type-checked: the proxy's attributes are real `FieldProxy` objects in the type checker's eyes, not your Pydantic annotations. See [Typed Query Predicates](../concepts/query-typing.md) for the full reasoning. +The lambda's return value is type-checked as well: `where()` expects a callable that returns a condition, so `lambda transaction: True` fails your type checker. [Typed query predicates](../concepts/query-typing.md) explains what the checker sees and where checking stops. + +!!! note "Changed in 0.14" + The operator style (`Transaction.where(Transaction.amount < 0)`) and the `col()` helper were removed in 0.14. [Upgrading to 0.14](../upgrading.md#predicates-operator-style-and-col-are-gone) shows how to rewrite them as lambdas. ## Operators +Each comparison maps to one SQL operator: + | Python | SQL | Example | | :--- | :--- | :--- | -| `==` | `=` | `lambda user: user.role == "admin"` | -| `!=` | `!=` | `lambda user: user.role != "admin"` | -| `>` | `>` | `lambda user: user.age > 18` | -| `>=` | `>=` | `lambda user: user.age >= 21` | -| `<` | `<` | `lambda user: user.age < 100` | -| `<=` | `<=` | `lambda user: user.age <= 65` | -| `.like(pattern)` | `LIKE` | `lambda user: user.name.like("a%")` | -| `.in_(values)` | `IN` | `lambda user: user.role.in_(["admin", "moderator"])` | -| `== None` | `IS NULL` | `lambda user: user.deleted_at == None` | -| `!= None` | `IS NOT NULL` | `lambda user: user.deleted_at != None` | -| `~` | `NOT` | `lambda user: ~user.role.in_(["admin", "moderator"])` | -| `.exists(...)` | `EXISTS (SELECT 1 …)` | `lambda user: user.posts.exists(lambda post: post.published == True)` — reverse/M2M relations only; see [Existence Tests](#existence-tests-on-reverse-many-to-many-relations) | +| `==` | `=` | `lambda transaction: transaction.memo == "Rent"` | +| `!=` | `!=` | `lambda transaction: transaction.memo != "Rent"` | +| `>` `>=` `<` `<=` | `>` `>=` `<` `<=` | `lambda transaction: transaction.amount < 0` | +| `.like(pattern)` | `LIKE` | `lambda transaction: transaction.memo.like("%coffee")` | +| `.in_(values)` | `IN` | `lambda transaction: transaction.account_id.in_([1, 2])` | +| `== None` | `IS NULL` | `lambda transaction: transaction.cleared_on == None` | +| `!= None` | `IS NOT NULL` | `lambda transaction: transaction.cleared_on != None` | +| `~` | `NOT` | `lambda transaction: ~transaction.memo.like("%Lyon")` | +| `&` | `AND` | `lambda transaction: (transaction.amount < 0) & (transaction.cleared_on == None)` | +| `\|` | `OR` | `lambda transaction: (transaction.amount < -1000) \| (transaction.amount > 1000)` | +| `.exists(...)` | `EXISTS (...)` | `lambda transaction: transaction.lines.exists()`; see [existence tests](#existence-tests-on-reverse-many-to-many-relations) | + +`.like()` and `.in_()` work on any column, a date included: ```python --8<-- "docs/examples/predicates.py:operators" ``` -## Combining Conditions +`%` in a `LIKE` pattern matches any run of characters, so `"%coffee"` matches Office coffee, and the two dates match the two Lyon rows. + +The two queries send these statements. PostgreSQL also casts each date parameter to `date`: + +=== "SQLite" + + ```sql + -- predicates.py:operators (sqlite) + SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" + FROM "transaction" + WHERE "transaction"."memo" LIKE ?; + + SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" + FROM "transaction" + WHERE "transaction"."posted_on" IN (?, ?); + ``` + +=== "PostgreSQL" -Combine predicates with `&` (AND) and `|` (OR), or chain multiple `.where()` calls (which AND together): + ```sql + -- predicates.py:operators (postgres) + SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" + FROM "transaction" + WHERE "transaction"."memo" LIKE $1; + + SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" + FROM "transaction" + WHERE "transaction"."posted_on" IN (CAST($1 AS date), CAST($2 AS date)); + ``` + +`.like()` became `LIKE`, and `.in_()` became `IN` with one placeholder per value. + +## Combining conditions + +Join conditions with `&` (and) and `|` (or), or chain several `where()` calls, which are joined with and: ```python --8<-- "docs/examples/predicates.py:combining" ``` -!!! warning "Always parenthesize `&` and `|` operands" - Python's `&` and `|` bind tighter than comparison operators, so `user.age < 18 | user.archived == True` parses as `user.age < (18 | user.archived) == True` — not what you meant. Wrap each condition in parentheses: `(user.age < 18) | (user.archived == True)`. +Only the two Lyon rows are uncleared and above -90; Weekly shop is uncleared but spent -95.00. The chained query keeps the debits posted on or after 10 March. + +The two queries send these statements. PostgreSQL also casts the date parameter to `date`: + +=== "SQLite" -## Negating Conditions + ```sql + -- predicates.py:combining (sqlite) + SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" + FROM "transaction" + WHERE "transaction"."cleared_on" IS NULL AND "transaction"."amount" > ?; -Prefix `~` negates **any** predicate — a comparison, `.in_()`, `.like()`, or a whole `&`/`|` group. It is the one negation rule in Ferro: there are no per-operator negative forms (no `not_in()`, no `not_like()`), because `~` already covers every predicate the same way: + SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" + FROM "transaction" + WHERE "transaction"."amount" < ? AND "transaction"."posted_on" >= ?; + ``` + +=== "PostgreSQL" + + ```sql + -- predicates.py:combining (postgres) + SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" + FROM "transaction" + WHERE "transaction"."cleared_on" IS NULL AND "transaction"."amount" > $1; + + SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" + FROM "transaction" + WHERE "transaction"."amount" < $1 AND "transaction"."posted_on" >= CAST($2 AS date); + ``` + +`&` and the chained `where()` both became `AND`. + +!!! warning "Parenthesize each side of `&` and `|`" + In Python, `&` and `|` bind tighter than comparisons, so `transaction.amount < 0 & transaction.cleared_on == None` parses as `transaction.amount < (0 & transaction.cleared_on) == None`. Wrap each condition in parentheses: `(transaction.amount < 0) & (transaction.cleared_on == None)`. + +Python's own `and`, `or` and `not` cannot combine conditions: Python asks each side whether it is true, and a condition has no answer until the database runs it. Ferro raises `TypeError` and names `&`, `|` and `~` instead. + +## Negating conditions + +Put `~` in front of any condition to negate it: a comparison, `.like()`, `.in_()` or a whole group in parentheses. There is no `not_like()` or `not_in()`; `~` covers them all: ```python --8<-- "docs/examples/predicates.py:negation" ``` -Each `~` renders as a faithful SQL `NOT (...)` over the condition it wraps. The last example above ships to the database as: +The four debits below -50 come back, minus Train to Lyon, whose memo matches `"%Lyon"`. Each `~` becomes SQL `NOT (...)` around the condition it wraps, so negated conditions combine with `&` and `|` like any other, and `~~` cancels out. + +The query sends this; on PostgreSQL only the placeholders differ: ```sql -SELECT ... FROM user WHERE NOT (user.age < 18 OR user.archived = TRUE) +-- predicates.py:negation (sqlite) +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" +FROM "transaction" +WHERE (NOT "transaction"."memo" LIKE ?) AND "transaction"."amount" < ?; ``` -Negated conditions compose exactly like un-negated ones — mix them into `&`/`|` trees at any depth, chain `.order_by()`/`.limit()` after them, and double negation (`~~p`) means what it says. Adding a `~` never restructures your query. +`~` wrapped the `LIKE` in `NOT`, kept in parentheses beside the `AND`. ### Negation and NULL values -SQL comparisons follow **three-valued logic**: a comparison against `NULL` is neither true nor false but *unknown*, and a `WHERE` clause only keeps rows whose condition is **true**. `NOT` maps unknown to unknown — so a negated comparison still excludes rows where the column is `NULL`. `~` is SQL's `NOT`, not Python's set complement. +SQL compares `NULL` to nothing: `cleared_on < '2026-03-05'` is neither true nor false for a row whose `cleared_on` is `NULL`, and `WHERE` keeps only the rows where the condition is true. `NOT` of an unknown is still unknown, so a negated comparison drops the `NULL` rows too: -Concretely, with a nullable column: +```python +--8<-- "docs/examples/predicates.py:three-valued" +``` -=== "Assignment" +Two transactions cleared before 5 March and four cleared later, which leaves out the three uncleared ones from both lists. `!=` behaves the same way: `transaction.cleared_on != date(2026, 3, 2)` also leaves out the uncleared rows. - ```python - --8<-- "docs/examples/predicates.py:nullable-model" - ``` +The two queries send these statements. PostgreSQL also casts the date parameter to `date`: -=== "Annotated" +=== "SQLite" - ```python - --8<-- "docs/examples/predicates_annotated.py:nullable-model" + ```sql + -- predicates.py:three-valued (sqlite) + SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" + FROM "transaction" + WHERE "transaction"."cleared_on" < ?; + + SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" + FROM "transaction" + WHERE NOT "transaction"."cleared_on" < ?; ``` -`~(invoice.amount > 100)` renders as: +=== "PostgreSQL" -```sql -SELECT ... FROM invoice WHERE NOT (invoice.amount > 100) -``` + ```sql + -- predicates.py:three-valued (postgres) + SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" + FROM "transaction" + WHERE "transaction"."cleared_on" < CAST($1 AS date); + + SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" + FROM "transaction" + WHERE NOT "transaction"."cleared_on" < CAST($1 AS date); + ``` + +Neither `WHERE` mentions `NULL`, so neither keeps the uncleared rows. -For a row where `amount` is `NULL`, `amount > 100` is unknown, `NOT unknown` is still unknown, and the row is excluded — from the negated query *and* from the original one. This matches the `!=` operator you already use (`invoice.amount != 100` excludes `NULL` rows too); `~` just makes the rule visible on every predicate. When you want the `NULL` rows kept, say so explicitly with an `IS NULL` branch: +To keep the `NULL` rows, ask for them with an `== None` branch: ```python ---8<-- "docs/examples/predicates.py:three-valued" +--8<-- "docs/examples/predicates.py:keep-nulls" ``` -## Ordering, Limit & Offset - -Sort with `.order_by(field, direction, *, nulls=...)` and slice with `.limit()` / `.offset()`. Direction defaults to ascending; pass `"desc"` to reverse. +The four later rows and the three uncleared rows come back. -Omitted `nulls=` means `NULL`s sort last on every backend. Pass `nulls="first"` to lead with `NULL`s, or `nulls="native"` for each dialect's default placement. +The query sends this. PostgreSQL also casts the date parameter to `date`: -`field` is a lambda naming the column (`order_by(lambda u: u.created_at, "desc")`, matching the `where()` predicate style) or a column-name string (`order_by("created_at", "desc")`). Both forms are validated against the model's queryable columns at build time. +=== "SQLite" -A pinned-first list is the usual reason to care — put unpinned cards (`pinned_at IS NULL`) after pinned ones, then break ties by recency: + ```sql + -- predicates.py:keep-nulls (sqlite) + SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" + FROM "transaction" + WHERE (NOT "transaction"."cleared_on" < ?) OR "transaction"."cleared_on" IS NULL; + ``` -=== "Assignment" +=== "PostgreSQL" - ```python - --8<-- "docs/examples/predicates.py:card-model" + ```sql + -- predicates.py:keep-nulls (postgres) + SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" + FROM "transaction" + WHERE (NOT "transaction"."cleared_on" < CAST($1 AS date)) OR "transaction"."cleared_on" IS NULL; ``` -=== "Annotated" +The `IS NULL` branch is what brings the uncleared rows back. - ```python - --8<-- "docs/examples/predicates_annotated.py:card-model" - ``` +## Fetching by primary key + +[`Model.get(pk)`][ferro.models.Model.get] loads one row by its primary key and returns the model, never `None`. A missing row raises `ModelDoesNotExist`, a `LookupError` that carries `.model` and `.pk`, so an HTTP layer can turn it into a 404. When a missing row is normal, use [`get_or_none()`][ferro.models.Model.get_or_none]: ```python ---8<-- "docs/examples/predicates.py:nulls-ordering" +--8<-- "docs/examples/predicates.py:get" ``` +`get(2)` returns Rent, `get(99)` raises, and `get_or_none(99)` returns `None`. Both methods also exist on `Model.using("name")` for [named connections](connections.md#named-connections). + +The three calls send one statement each, the same statement; on PostgreSQL only the placeholders differ: + ```sql -ORDER BY pinned_at DESC NULLS LAST, updated_at DESC, id DESC +-- predicates.py:get (sqlite) +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" +FROM "transaction" +WHERE "transaction"."id" = ? +LIMIT ?; + +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" +FROM "transaction" +WHERE "transaction"."id" = ? +LIMIT ?; + +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" +FROM "transaction" +WHERE "transaction"."id" = ? +LIMIT ?; ``` -Omitting `nulls=` on a nullable sort key means `NULLS LAST` on every backend. Pass `nulls="first"` to lead with `NULL`s, or `nulls="native"` when you want each dialect's default (PostgreSQL and SQLite disagree on `DESC`). +Each is a primary-key filter with a `LIMIT` of 1. `get(99)` sent its statement too; the error comes from the empty result. + +## Ordering, limit & offset + +[`order_by()`][ferro.query.builder.Query.order_by] takes a lambda naming the column, and an optional direction (`"asc"`, the default, or `"desc"`). `limit()` and `offset()` cut a slice out of the ordered rows: ```python --8<-- "docs/examples/predicates.py:ordering-slicing" ``` -Chain `.order_by()` multiple times for multi-column sorts. +The first three are the largest debits; `offset(3)` skips them and returns the next three. Chain `order_by()` again to break ties on a second column. + +The two queries send these statements; on PostgreSQL only the placeholders differ: + +```sql +-- predicates.py:ordering-slicing (sqlite) +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" +FROM "transaction" +WHERE TRUE +ORDER BY "transaction"."amount" ASC NULLS LAST +LIMIT ?; + +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" +FROM "transaction" +WHERE TRUE +ORDER BY "transaction"."amount" ASC NULLS LAST +LIMIT ? +OFFSET ?; +``` -To page forward from a known row, pass that row's place in the declared order to `.after()`. The bound is exclusive and the order keys must include the primary key. `None` is legal in every non-PK slot — that is how a pinned-first list continues through unpinned rows: +`WHERE TRUE` stands in for the missing filter, and `limit()` and `offset()` became `LIMIT` and `OFFSET`. + +`order_by()` also accepts a column name as a string (`order_by("amount", "desc")`). The lambda is the documented form because only a lambda can [order by a related column](#ordering-by-a-related-column). + +`NULL` values sort last on every database unless you say otherwise. Pass `nulls="first"` to put them first, or `nulls="native"` for each database's own default, which differs between PostgreSQL and SQLite for `"desc"`: + +```python +--8<-- "docs/examples/predicates.py:nulls-ordering" +``` + +The three uncleared transactions lead, then the latest cleared one, Office coffee. + +The query sends this; on PostgreSQL only the placeholders differ: + +```sql +-- predicates.py:nulls-ordering (sqlite) +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" +FROM "transaction" +WHERE TRUE +ORDER BY "transaction"."cleared_on" DESC NULLS FIRST, "transaction"."id" ASC NULLS LAST +LIMIT ?; +``` + +`nulls="first"` became `NULLS FIRST`; the tie-breaker on `id` keeps the default, `NULLS LAST`. + +## Paging with `after()` and `before()` + +[`after()`][ferro.query.builder.Query.after] starts a page right after a row you already have, in the order the query declares. Pass it the last row of the page you showed: ```python --8<-- "docs/examples/predicates.py:after-paging" ``` -`after(row)` is the same as `after(position_of(row))`. `after()` cannot be combined with `offset()` — a query has one start. For the pinned-first shape (`order_by(pinned_at, "desc")`, omitted `nulls=` → last), `after((None, id))` continues through the remaining unpinned rows: +The second page starts at To savings, the row after Market run. Unlike `offset()`, it does not skip or repeat rows when rows are added or removed between the two requests. + +The two pages send these statements. PostgreSQL also casts the date parameters to `date`: + +=== "SQLite" + + ```sql + -- predicates.py:after-paging (sqlite) + SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" + FROM "transaction" + WHERE TRUE + ORDER BY "transaction"."posted_on" ASC NULLS LAST, "transaction"."id" ASC NULLS LAST + LIMIT ?; + + SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" + FROM "transaction" + WHERE "transaction"."posted_on" > ? OR "transaction"."posted_on" IS NULL OR ("transaction"."posted_on" = ? AND ("transaction"."id" > ? OR "transaction"."id" IS NULL)) + ORDER BY "transaction"."posted_on" ASC NULLS LAST, "transaction"."id" ASC NULLS LAST + LIMIT ?; + ``` + +=== "PostgreSQL" + + ```sql + -- predicates.py:after-paging (postgres) + SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" + FROM "transaction" + WHERE TRUE + ORDER BY "transaction"."posted_on" ASC NULLS LAST, "transaction"."id" ASC NULLS LAST + LIMIT $1; + + SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" + FROM "transaction" + WHERE "transaction"."posted_on" > CAST($1 AS date) OR "transaction"."posted_on" IS NULL OR ("transaction"."posted_on" = CAST($2 AS date) AND ("transaction"."id" > $3 OR "transaction"."id" IS NULL)) + ORDER BY "transaction"."posted_on" ASC NULLS LAST, "transaction"."id" ASC NULLS LAST + LIMIT $4; + ``` + +The second page starts with a condition on the order keys instead of an `OFFSET`: a later `posted_on`, or Market run's `posted_on` with a larger `id`. The `IS NULL` branches match the `NULLS LAST` order. + +`after()` follows four rules: + +- The order keys must include the primary key, so every row has exactly one place in the order. Ferro does not add it for you: `after()` raises `ValueError` if `id` is missing. Here, To savings and From checking share a `posted_on`, and `id` settles which comes first. +- The bound is exclusive: the row you pass is not on the next page. +- A query has one start. `after()` cannot be combined with `offset()` or with `before()`. +- Any order key except the primary key may be `None`. + +`after(row)` reads the row's order-key values for you. [`position_of()`][ferro.query.builder.Query.position_of] returns those values as a tuple, which you can put in a cursor and pass back later: ```python ---8<-- "docs/examples/predicates.py:after-null-paging" +--8<-- "docs/examples/predicates.py:position-of" ``` -To page backward, `.before(position)` is the other start. With a limit it is the **adjacent previous page**, still yielded in the declared order. Without a limit it is every earlier row in declared order — a prefix, not a page. The bound is exclusive. `after` and `before` cannot share a query. +Market run's position is its `posted_on` and its `id`, and paging after that tuple returns the same page as paging after the row. `query.after(row)` is `query.after(query.position_of(row))`. + +Paging after the tuple sends the same statement as paging after the row. PostgreSQL also casts the date parameters to `date`: + +=== "SQLite" + + ```sql + -- predicates.py:position-of (sqlite) + SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" + FROM "transaction" + WHERE "transaction"."posted_on" > ? OR "transaction"."posted_on" IS NULL OR ("transaction"."posted_on" = ? AND ("transaction"."id" > ? OR "transaction"."id" IS NULL)) + ORDER BY "transaction"."posted_on" ASC NULLS LAST, "transaction"."id" ASC NULLS LAST + LIMIT ?; + ``` + +=== "PostgreSQL" + + ```sql + -- predicates.py:position-of (postgres) + SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" + FROM "transaction" + WHERE "transaction"."posted_on" > CAST($1 AS date) OR "transaction"."posted_on" IS NULL OR ("transaction"."posted_on" = CAST($2 AS date) AND ("transaction"."id" > $3 OR "transaction"."id" IS NULL)) + ORDER BY "transaction"."posted_on" ASC NULLS LAST, "transaction"."id" ASC NULLS LAST + LIMIT $4; + ``` + +`position_of()` itself sends nothing; it reads the values off the row. + +[`before()`][ferro.query.builder.Query.before] pages backwards. With a limit it returns the page immediately before the bound, still in the declared order; without one it returns every earlier row: ```python --8<-- "docs/examples/predicates.py:before-paging" ``` -On unbounded `before()`, `first()` and `all()[0]` disagree: `first()` is `limit(1)` (the adjacent previous row) and `all()[0]` is the head of the prefix (the earliest earlier row). That is accepted. +`before(...).limit(2)` returns the two rows before To savings, and `before(...)` alone returns all three earlier rows. `first()` returns the row immediately before the bound, while `all()[0]` returns the earliest row, so use `first()` for "the previous row". + +The three queries send these statements. PostgreSQL also casts the date parameters to `date`: + +=== "SQLite" + + ```sql + -- predicates.py:before-paging (sqlite) + SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" + FROM "transaction" + WHERE "transaction"."posted_on" < ? OR ("transaction"."posted_on" = ? AND "transaction"."id" < ?) + ORDER BY "transaction"."posted_on" DESC NULLS FIRST, "transaction"."id" DESC NULLS FIRST + LIMIT ?; -The same chainers work when an order key is a related column (`order_by(lambda t: t.account.label)`) or the query is a projected record: pass a tuple of the order-key values. `position_of` on a model instance requires those relations populated; `position_of` on a `Row` requires every order key to be in the projection. + SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" + FROM "transaction" + WHERE "transaction"."posted_on" < ? OR ("transaction"."posted_on" = ? AND "transaction"."id" < ?) + ORDER BY "transaction"."posted_on" DESC NULLS FIRST, "transaction"."id" DESC NULLS FIRST; -For pagination patterns, see [Pagination](../howto/pagination.md). + SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" + FROM "transaction" + WHERE "transaction"."posted_on" < ? OR ("transaction"."posted_on" = ? AND "transaction"."id" < ?) + ORDER BY "transaction"."posted_on" DESC NULLS FIRST, "transaction"."id" DESC NULLS FIRST + LIMIT ?; + ``` + +=== "PostgreSQL" + + ```sql + -- predicates.py:before-paging (postgres) + SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" + FROM "transaction" + WHERE "transaction"."posted_on" < CAST($1 AS date) OR ("transaction"."posted_on" = CAST($2 AS date) AND "transaction"."id" < $3) + ORDER BY "transaction"."posted_on" DESC NULLS FIRST, "transaction"."id" DESC NULLS FIRST + LIMIT $4; + + SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" + FROM "transaction" + WHERE "transaction"."posted_on" < CAST($1 AS date) OR ("transaction"."posted_on" = CAST($2 AS date) AND "transaction"."id" < $3) + ORDER BY "transaction"."posted_on" DESC NULLS FIRST, "transaction"."id" DESC NULLS FIRST; + + SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" + FROM "transaction" + WHERE "transaction"."posted_on" < CAST($1 AS date) OR ("transaction"."posted_on" = CAST($2 AS date) AND "transaction"."id" < $3) + ORDER BY "transaction"."posted_on" DESC NULLS FIRST, "transaction"."id" DESC NULLS FIRST + LIMIT $4; + ``` -## Executing Queries +`before()` flips each comparison and reverses each sort key with its `NULLS` placement, so the `LIMIT` keeps the rows nearest the bound; Ferro puts them back in the declared order. -Queries are lazy — nothing hits the database until you await a terminal: +A `None` order-key value is how paging continues through `NULL`s. Ordered by `cleared_on` descending, the uncleared rows sort last; to continue after Train to Lyon, pass `None` for its `cleared_on`: + +```python +--8<-- "docs/examples/predicates.py:after-null-paging" +``` + +The two uncleared rows after Train to Lyon come back. The same methods page a query ordered by a [related column](#ordering-by-a-related-column) or a [projected query](#composing-projections). [Pagination](../howto/pagination.md) shows how to wire this into an API. + +The query sends this; on PostgreSQL only the placeholders differ: + +```sql +-- predicates.py:after-null-paging (sqlite) +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" +FROM "transaction" +WHERE "transaction"."cleared_on" IS NULL AND ("transaction"."id" > ? OR "transaction"."id" IS NULL) +ORDER BY "transaction"."cleared_on" DESC NULLS LAST, "transaction"."id" ASC NULLS LAST; +``` + +With `None` for `cleared_on`, the condition stays among the `NULL` rows and continues by `id`. + +## Executing queries + +A query runs when you await one of its terminals: [`all()`][ferro.query.builder.Query.all] returns a list of instances, [`first()`][ferro.query.builder.Query.first] returns the first row or `None`, [`count()`][ferro.query.builder.Query.count] returns how many rows match, and [`exists()`][ferro.query.builder.Query.exists] returns whether any does: ```python --8<-- "docs/examples/predicates.py:terminals" ``` -| Terminal | Returns | Semantics | -| :--- | :--- | :--- | -| `.all()` | `list[Model]` | All matching rows, hydrated to instances. | -| `.first()` | `Model \| None` | First matching row, or `None` if there are no matches. | -| `.count()` | `int` | `COUNT(*)` of matching rows — no instances hydrated. | -| `.exists()` | `bool` | `True` if at least one row matches; stops at the first match. | +`count()` and `exists()` hydrate no instances. `Model.all()` is short for `Model.select().all()`. -!!! tip "Prefer `.exists()` over `.count() > 0`" - `.exists()` lets the database stop at the first match instead of counting every row. +The four calls send these statements; on PostgreSQL only the placeholders differ: -`Model.all()` is shorthand for `Model.select().all()`. +```sql +-- predicates.py:terminals (sqlite) +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" +FROM "transaction"; + +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" +FROM "transaction" +WHERE TRUE +ORDER BY "transaction"."posted_on" DESC NULLS LAST +LIMIT ?; -On a **projected** query (`select(lambda t: (t.id, t.amount))`), `.all()` returns `Rows[Row]` and `.first()` returns `Row | None` — records, not model instances; `count()` and `exists()` are unchanged on a plain projection (on an aggregate projection they raise with guidance). See [Selecting a Column Subset](#selecting-a-column-subset) and [Aggregations & Grouped Queries](aggregations.md). +SELECT COUNT(*) +FROM "transaction" +WHERE "transaction"."cleared_on" IS NULL; + +SELECT COUNT(*) +FROM "transaction" +WHERE "transaction"."memo" LIKE ?; +``` -## Querying Across Relationships +`count()` and `exists()` both send `SELECT COUNT(*)`; `exists()` compares the count with zero. -A `where()` or `order_by()` lambda can reach *through* a `ForeignKey` to a column on the related model. Write the relation field, then keep going: +## Query immutability + +`where()`, `order_by()`, `limit()` and the other chain methods each return a new query and leave the one you called them on unchanged. You can keep a base query and branch from it: ```python -ledger_a = await Ledger.get(1) +--8<-- "docs/examples/predicates.py:immutability" +``` + +The three queries built from `debits` are independent, and `debits` still counts all seven debits afterwards. -rows = await Transaction.where( - lambda transaction: transaction.account.ledger_id == ledger_a.id -).all() +The four calls send these statements; on PostgreSQL only the placeholders differ: + +```sql +-- predicates.py:immutability (sqlite) +SELECT COUNT(*) +FROM "transaction" +WHERE "transaction"."amount" < ?; + +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" +FROM "transaction" +WHERE "transaction"."amount" < ? +ORDER BY "transaction"."amount" ASC NULLS LAST +LIMIT ?; + +SELECT COUNT(*) +FROM "transaction" +WHERE "transaction"."amount" < ? AND "transaction"."cleared_on" IS NULL; + +SELECT COUNT(*) +FROM "transaction" +WHERE "transaction"."amount" < ?; ``` -That is one SQL statement — a `SELECT` over `transactions` with a join to `accounts` — and it returns the transactions whose account belongs to ledger A. You never write the join; the relation you traverse (`transaction.account`) *is* the join. +Every statement carries the `amount < ?` filter of `debits`; only the branches add to it. -The examples below use this schema — a `Transaction` points at an `Account`, and an `Account` points at both a `Ledger` and an `Owner`: +## Querying across relationships -=== "Assignment" +A `where()` or `order_by()` lambda can follow a foreign key to the related model's columns. Write the relation, then the column: - ```python - --8<-- "docs/examples/traversal.py:schema" - ``` +```python +--8<-- "docs/examples/traversal.py:basic" +``` -=== "Annotated" +`transaction.account.label` reads the label of each transaction's account, and From checking is the one transaction on Savings. You do not write the join: following `transaction.account` is the join, and the query still returns `Transaction` instances. These examples run inside the session from [the first block](#queries). The query sends this; on PostgreSQL only the placeholders differ: - ```python - --8<-- "docs/examples/traversal_annotated.py:schema" - ``` +```sql +-- traversal.py:basic (sqlite) +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" +FROM "transaction" +INNER JOIN "account" AS "j1_account" ON "transaction"."account_id" = "j1_account"."id" +WHERE "j1_account"."label" = ?; +``` + +Following `transaction.account` became one `INNER JOIN` on `account`, under the alias `j1_account`. ### Traversing at any depth -Each hop resolves against the related model, so you can chain as many as the schema allows. `transaction.account.owner.email` walks `Transaction → Account → Owner` in a single statement with two joins: +Each step resolves against the next model, so you can chain as far as the foreign keys go. `transaction.account.owner.email` walks from `Transaction` to `Account` to `Owner` in one statement: ```python --8<-- "docs/examples/traversal.py:multi-hop" ``` -Every hop is validated when you build the query — before anything reaches the database. A misspelled column or relation raises `AttributeError` naming the model at that hop and the closest match, and the suggestion pool spans both columns *and* relations: +Grace Hopper owns Travel, so both Travel transactions come back. The query sends this; on PostgreSQL only the placeholders differ: -```text ->>> Transaction.where(lambda transaction: transaction.accont.ledger_id == 1) -AttributeError: Transaction has no queryable column 'accont'. Did you mean 'account'? Valid columns: account_id, amount, id. Valid relations: account. +```sql +-- traversal.py:multi-hop (sqlite) +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" +FROM "transaction" +INNER JOIN "account" AS "j1_account" ON "transaction"."account_id" = "j1_account"."id" +INNER JOIN "owner" AS "j2_owner" ON "j1_account"."owner_id" = "j2_owner"."id" +WHERE "j2_owner"."email" = ?; +``` ->>> Transaction.where(lambda transaction: transaction.account.owner.emial == "x") -AttributeError: Owner has no queryable column 'emial'. Did you mean 'email'? Valid columns: email, id. +Each step is one more join: `j2_owner` joins on the account's `owner_id`. Ferro checks each step against its own model when you build the query, and a misspelled step names that model and the closest match: + +```python +--8<-- "docs/examples/traversal.py:typo" ``` -### Every traversed hop is an INNER join +The error comes from `Owner`, the model at the misspelled step. -Traversal always renders an **INNER** join, at every hop, no matter whether the foreign key is nullable (ADR-0006). To see what that means for nullable relations, the narrowing examples in this and the following sections add a `Note` model whose `account` relation is **optional** — a note may or may not be attached to an account: +### Join type for traversed relations -=== "Assignment" +Every step is an inner join, whether or not the foreign key is nullable. A row whose relation is missing therefore drops out of a query that follows that relation: - ```python - --8<-- "docs/examples/traversal.py:note-model" - ``` +```python +--8<-- "docs/examples/traversal.py:inner-narrows" +``` -=== "Annotated" +Only Grace's two transactions come back. Office coffee is on Petty cash, which has no owner; it has no owner name to compare, so the join drops it. The join type never depends on nullability, so making a foreign key nullable later never changes what an existing query returns. To keep the rows without a relation, use [`left_join()`](#keeping-rows-that-have-no-relation). - ```python - --8<-- "docs/examples/traversal_annotated.py:note-model" - ``` +The query sends this; on PostgreSQL only the placeholders differ: -The practical consequence of INNER-everywhere: **traversing a relation narrows the result to rows where that relation exists**. A `Note` whose `account` FK is `NULL` simply does not appear in a query that filters through `note.account`: +```sql +-- traversal.py:inner-narrows (sqlite) +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" +FROM "transaction" +INNER JOIN "account" AS "j1_account" ON "transaction"."account_id" = "j1_account"."id" +INNER JOIN "owner" AS "j2_owner" ON "j1_account"."owner_id" = "j2_owner"."id" +WHERE "j2_owner"."name" <> ?; +``` + +The `INNER JOIN` on `owner` drops Office coffee: Petty cash has no `owner_id` to join on. + +The join applies to the whole query, not to one side of an `|`: ```python ---8<-- "docs/examples/traversal.py:inner-narrows" +--8<-- "docs/examples/traversal.py:inner-or" ``` -This is deliberate and stable. A hop's nullability never changes the join type, so making a foreign key nullable later never silently rewrites the meaning of an existing query, and an N-hop path is trivial to reason about — no hop poisons the ones after it. Keeping the relation-less rows is an explicit opt-in (`left_join`, below). +Office coffee matches the second branch, but the join on `account.owner` has already dropped it. + +The query sends this; on PostgreSQL only the placeholders differ: + +```sql +-- traversal.py:inner-or (sqlite) +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" +FROM "transaction" +INNER JOIN "account" AS "j1_account" ON "transaction"."account_id" = "j1_account"."id" +INNER JOIN "owner" AS "j2_owner" ON "j1_account"."owner_id" = "j2_owner"."id" +WHERE "j2_owner"."name" = ? OR "transaction"."memo" = ?; +``` -The narrowing is query-wide, not per-clause: the join is rendered once for the whole statement, so a traversal branch inside an `|` still narrows the entire result. `(note.account.ledger_id == 1) | (note.body == "orphan")` drops every relation-less note — the INNER `account` join removes it before the `OR` is ever evaluated, so the `body == "orphan"` branch can never rescue it. Reach for `left_join` when a traversal branch of an `|` must keep relation-less rows. +The `OR` sits in the `WHERE`, after both joins. -`count()` and the other terminals see exactly this narrowed set — a many-to-one join never multiplies root rows, so `.count()` equals the number of matching transactions, not the number of joined pairs: +`count()` counts the same rows. Following a foreign key reaches one related row per row, so the join never multiplies results: ```python --8<-- "docs/examples/traversal.py:count" ``` +Ada's two accounts hold six transactions. + +`count()` sends this; on PostgreSQL only the placeholders differ: + +```sql +-- traversal.py:count (sqlite) +SELECT COUNT(*) +FROM "transaction" +INNER JOIN "account" AS "j1_account" ON "transaction"."account_id" = "j1_account"."id" +INNER JOIN "owner" AS "j2_owner" ON "j1_account"."owner_id" = "j2_owner"."id" +WHERE "j2_owner"."name" = ?; +``` + +`count()` counts over the same two joins. + ### Ordering by a related column -`order_by()` traverses the same way, to any depth, and shares its joins with `where()`: +`order_by()` follows relations the same way: ```python --8<-- "docs/examples/traversal.py:order-by" ``` -Because the sort join is INNER too, ordering by a related column drops relation-less rows — the same narrowing as `where()`. (Use `left_join` to keep them; see below.) +Accounts sort by label, descending: Travel, Savings, Petty cash, then Checking after the limit. The query sends this; on PostgreSQL only the placeholders differ: -`after()` / `before()` take a decoded **tuple** of those order-key values — `include()` is not required to page. `position_of` on a model instance does require the relation populated. +```sql +-- traversal.py:order-by (sqlite) +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" +FROM "transaction" +INNER JOIN "account" AS "j1_account" ON "transaction"."account_id" = "j1_account"."id" +WHERE TRUE +ORDER BY "j1_account"."label" DESC NULLS LAST, "transaction"."id" ASC NULLS LAST +LIMIT ?; +``` + +The sort reads the joined `label`, then `id` breaks ties. Sorting is an inner join too, so ordering by a column that some rows cannot reach drops those rows: + +```python +--8<-- "docs/examples/traversal.py:order-by-narrows" +``` + +Eight of the nine transactions come back. The query sends this; on PostgreSQL only the placeholders differ: + +```sql +-- traversal.py:order-by-narrows (sqlite) +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" +FROM "transaction" +INNER JOIN "account" AS "j1_account" ON "transaction"."account_id" = "j1_account"."id" +INNER JOIN "owner" AS "j2_owner" ON "j1_account"."owner_id" = "j2_owner"."id" +WHERE TRUE +ORDER BY "j2_owner"."name" ASC NULLS LAST, "transaction"."id" ASC NULLS LAST; +``` + +Office coffee drops out at the `owner` join. `after()` and `before()` take a tuple of the related values, so you can page without loading the relation: ```python --8<-- "docs/examples/traversal.py:traversed-paging" ``` -### One join per relation path +The page starts after Office coffee, the last Petty cash row. `position_of()` and `after(row)` read the related value off the instance, so they need the relation [populated with `include()`](#populating-relations-with-include); a tuple does not. -The relation path is the join's identity. Reference the same path in two `where()` calls, in an `&`/`|` tree, or across `where()` and `order_by()`, and it renders as **one** join. Distinct paths — even to the same table — render as distinct joins. There is no alias to name and none to manage; the path does that job. +The query sends this; on PostgreSQL only the placeholders differ: -That is what lets a traversal filter compose cleanly with plain root-column clauses — the motivating query pairs one `account` join with a root-column filter, ordering, and a limit, all in a single statement: +```sql +-- traversal.py:traversed-paging (sqlite) +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" +FROM "transaction" +INNER JOIN "account" AS "j1_account" ON "transaction"."account_id" = "j1_account"."id" +WHERE "j1_account"."label" > ? OR "j1_account"."label" IS NULL OR ("j1_account"."label" = ? AND ("transaction"."id" > ? OR "transaction"."id" IS NULL)) +ORDER BY "j1_account"."label" ASC NULLS LAST, "transaction"."id" ASC NULLS LAST +LIMIT ?; +``` + +The position compares the joined `label` and then `id`, as a position on the model's own columns would. + +### One join per relation path + +The path decides the join. Use `transaction.account` in two `where()` calls, in both sides of an `|`, or in `where()` and `order_by()`, and the query still joins `account` once. You never name or manage an alias: ```python --8<-- "docs/examples/traversal.py:composed" ``` +The traversal filter, a plain column filter, an ordering and a limit run as one statement, and Ada's two largest debits come back. + +The query sends this; on PostgreSQL only the placeholders differ: + +```sql +-- traversal.py:composed (sqlite) +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" +FROM "transaction" +INNER JOIN "account" AS "j1_account" ON "transaction"."account_id" = "j1_account"."id" +INNER JOIN "owner" AS "j2_owner" ON "j1_account"."owner_id" = "j2_owner"."id" +WHERE "j2_owner"."name" = ? AND "transaction"."amount" < ? +ORDER BY "transaction"."amount" ASC NULLS LAST +LIMIT ?; +``` + +`account` and `owner` are joined once each; the plain filter and the ordering read `transaction` itself. + ### Comparing to a related instance -Every `ForeignKey` also exposes a shadow `{fk}_id` column, so you can filter by a related row's primary key directly — no join at all: +Each foreign key adds a `_id` column, and filtering on it needs no join: ```python -posts = await Post.where(lambda post: post.author_id == user.id).all() +--8<-- "docs/examples/traversal.py:shadow-column" ``` -Comparing the relation itself to a **persisted** instance is sugar for exactly that shadow-column check — still join-free: +From checking is the one transaction whose `account_id` is Savings' `id`. Fetching Savings and the query send these statements; on PostgreSQL only the placeholders differ: + +```sql +-- traversal.py:shadow-column (sqlite) +SELECT "account"."id", "account"."label", "account"."owner_id" +FROM "account" +WHERE "account"."id" = ? +LIMIT ?; + +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" +FROM "transaction" +WHERE "transaction"."account_id" = ?; +``` + +The filter reads `account_id` on `transaction`, with no join. Comparing the relation to a saved instance means the same thing, with the same join-free query: ```python --8<-- "docs/examples/traversal.py:instance-eq" ``` -Deep instance equality compares the shadow column of the *last* hop under the prefix join: `transaction.account.owner == some_owner` filters `owner_id` on the joined `accounts` row. +`transaction.account == savings` is `transaction.account_id == savings.id`; `!=` returns the other eight. The two queries send these statements; on PostgreSQL only the placeholders differ: + +```sql +-- traversal.py:instance-eq (sqlite) +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" +FROM "transaction" +WHERE "transaction"."account_id" = ?; + +SELECT COUNT(*) +FROM "transaction" +WHERE "transaction"."account_id" <> ?; +``` + +Both compare `account_id`, the second with `<>`. At the end of a longer path, the comparison uses the last step's foreign key column: ```python --8<-- "docs/examples/traversal.py:deep-instance-eq" ``` -Comparing an **unpersisted** instance is a build-time error naming the model (`cannot compare relation 'account' to an unpersisted Account instance …`) — there is no primary key to match on yet. +`transaction.account.owner == grace` joins `account` and compares its `owner_id`. Fetching Grace and the query send these statements; on PostgreSQL only the placeholders differ: + +```sql +-- traversal.py:deep-instance-eq (sqlite) +SELECT "owner"."email", "owner"."id", "owner"."name" +FROM "owner" +WHERE "owner"."id" = ? +LIMIT ?; -### Testing for existence or absence +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" +FROM "transaction" +INNER JOIN "account" AS "j1_account" ON "transaction"."account_id" = "j1_account"."id" +WHERE "j1_account"."owner_id" = ?; +``` -A bare `.join()` with no predicate is a meaningful **existence filter** on a nullable relation — it narrows to rows where the relation is present (the same INNER narrowing traversal gives you, expressed directly): +The first statement fetched Grace; the query joins `account` and never `owner`. An instance that has not been saved has no primary key to compare, so Ferro refuses it when you build the query: + +```python +--8<-- "docs/examples/traversal.py:unpersisted" +``` + +Save the account first, then compare. + +### Testing for a relation + +A bare `join()` with no condition keeps the rows whose relation exists: ```python --8<-- "docs/examples/traversal.py:existence" ``` -For the opposite question — "has *no* related row" — compare the relation to `None`. `relation == None` / `!= None` lower to `IS NULL` / `IS NOT NULL` on the shadow column, join-free, so "has no account" needs no `left_join`: +Petty cash has no owner and is left out. The query sends this; on PostgreSQL only the placeholders differ: + +```sql +-- traversal.py:existence (sqlite) +SELECT "account"."id", "account"."label", "account"."owner_id" +FROM "account" +INNER JOIN "owner" AS "j1_owner" ON "account"."owner_id" = "j1_owner"."id" +WHERE TRUE; +``` + +The bare `join()` is an `INNER JOIN` with nothing added to the `WHERE`. To ask the opposite, compare the relation to `None`: `== None` and `!= None` become `IS NULL` and `IS NOT NULL` on the foreign key column, with no join: ```python --8<-- "docs/examples/traversal.py:is-null" ``` -Both spellings here are the **forward** direction — the FK column lives on the queried table. Asking the same question in the *reverse* direction ("has at least one / no related child row") is an [existence test](#existence-tests-on-reverse-many-to-many-relations): `t.lines.exists()` / `~t.lines.exists()`. +Both forms ask about a foreign key on the model you query. To ask whether a row has related rows on another table, such as "has at least one split line", use an [existence test](#existence-tests-on-reverse-many-to-many-relations). + +The two queries send these statements; on PostgreSQL only the placeholders differ: + +```sql +-- traversal.py:is-null (sqlite) +SELECT "account"."id", "account"."label", "account"."owner_id" +FROM "account" +WHERE "account"."owner_id" IS NULL; + +SELECT COUNT(*) +FROM "account" +WHERE "account"."owner_id" IS NOT NULL; +``` + +Neither statement joins `owner`. ### Keeping rows that have no relation -When you want the relation-less rows *kept* rather than filtered out, opt into a `left_join`. It marks **every edge of its path** LEFT (the whole-path rule), so a left-marked two-hop path retains rows missing the relation at either hop: +[`left_join()`][ferro.query.builder.Query.left_join] keeps the rows whose relation is missing. It makes every step of its path a left join, and a `where()` or `order_by()` on the same path uses it too: ```python --8<-- "docs/examples/traversal.py:left-join" ``` -A bare `left_join` on a path also traversed by `where()` lifts the shared edge to LEFT — an explicit LEFT always beats an implicit INNER on the same edge (declaring `join` **and** `left_join` on one edge is a build-time `ValueError`). - -!!! note "NULL ordering defaults to last unless you override it" - Relation-less rows under `left_join` land as `NULL` sort keys on a related column — omitted `nulls=` still means last. Pass `nulls="first"` to lead with `NULL`s, or `nulls="native"` for dialect-default placement — see [Ordering, Limit & Offset](#ordering-limit-offset). +All four accounts come back, Petty cash last because its owner name is `NULL`. Without `left_join()`, ordering by `account.owner.name` would drop it. Declaring `join()` and `left_join()` on the same path raises `ValueError`. -### Two foreign keys to the same table - -Distinct relation paths are distinct joins even when they point at the same table, so two FKs to one model just work — no alias ceremony: +The query sends this; on PostgreSQL only the placeholders differ: -=== "Assignment" +```sql +-- traversal.py:left-join (sqlite) +SELECT "account"."id", "account"."label", "account"."owner_id" +FROM "account" +LEFT JOIN "owner" AS "j1_owner" ON "account"."owner_id" = "j1_owner"."id" +WHERE TRUE +ORDER BY "j1_owner"."name" ASC NULLS LAST, "account"."id" ASC NULLS LAST; +``` - ```python - --8<-- "docs/examples/traversal.py:two-fk-model" - ``` +The `LEFT JOIN` keeps Petty cash with `NULL` in every `j1_owner` column, and `NULLS LAST` sorts it to the end. -=== "Annotated" +### Two foreign keys to the same table - ```python - --8<-- "docs/examples/traversal_annotated.py:two-fk-model" - ``` +Two foreign keys to one model are two paths, so they are two joins: ```python --8<-- "docs/examples/traversal.py:two-fk-query" ``` -### Self-referential traversal +`transfer.outflow` and `transfer.inflow` both point at `Transaction`, and each reaches its own account. The one transfer runs from transaction 4 on Checking to transaction 5 on Savings. -A self-referencing FK traverses like any other — the join target is the same table, reached by a distinct path: +The query sends this; on PostgreSQL only the placeholders differ: -=== "Assignment" +```sql +-- traversal.py:two-fk-query (sqlite) +SELECT "transfer"."id", "transfer"."inflow_id", "transfer"."outflow_id" +FROM "transfer" +INNER JOIN "transaction" AS "j1_outflow" ON "transfer"."outflow_id" = "j1_outflow"."id" +INNER JOIN "account" AS "j2_account" ON "j1_outflow"."account_id" = "j2_account"."id" +INNER JOIN "transaction" AS "j3_inflow" ON "transfer"."inflow_id" = "j3_inflow"."id" +INNER JOIN "account" AS "j4_account" ON "j3_inflow"."account_id" = "j4_account"."id" +WHERE "j2_account"."label" = ? AND "j4_account"."label" = ?; +``` - ```python - --8<-- "docs/examples/traversal.py:self-fk-model" - ``` +Each path has its own aliases, `j1_outflow` and `j3_inflow`, each with its own join on `account`. -=== "Annotated" +### Self-referential traversal - ```python - --8<-- "docs/examples/traversal_annotated.py:self-fk-model" - ``` +A foreign key to the same model traverses like any other: ```python --8<-- "docs/examples/traversal.py:self-fk-query" ``` -### Traversing from a many-to-many +Groceries and Dining have Food as their parent. -An association query (`post.tags`) composes with forward-FK traversal on the target model: the association join and the traversal join coexist in one statement. +The query sends this; on PostgreSQL only the placeholders differ: -=== "Assignment" +```sql +-- traversal.py:self-fk-query (sqlite) +SELECT "category"."id", "category"."name", "category"."parent_id" +FROM "category" +INNER JOIN "category" AS "j1_parent" ON "category"."parent_id" = "j1_parent"."id" +WHERE "j1_parent"."name" = ?; +``` - ```python - --8<-- "docs/examples/traversal.py:m2m-model" - ``` +The `category` table joins itself under the alias `j1_parent`. -=== "Annotated" +### Traversing from a many-to-many - ```python - --8<-- "docs/examples/traversal_annotated.py:m2m-model" - ``` +A many-to-many relation on an instance is a query, and its lambda can follow foreign keys on the other model: ```python --8<-- "docs/examples/traversal.py:m2m-query" ``` +Of the two reimbursable transactions, only Dinner in Lyon is in Dining. + +Fetching the tag and the query send these statements; on PostgreSQL only the placeholders differ: + +```sql +-- traversal.py:m2m-query (sqlite) +SELECT "tag"."id", "tag"."name" +FROM "tag" +WHERE "tag"."name" = ? +LIMIT ?; + +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" +FROM "transaction" +INNER JOIN "tag_transactions" ON "transaction"."id" = "tag_transactions"."transaction_id" +INNER JOIN "category" AS "j1_category" ON "transaction"."category_id" = "j1_category"."id" +WHERE "tag_transactions"."tag_id" = ? AND "j1_category"."name" = ?; +``` + +The many-to-many query joins the link table, `tag_transactions`, and `transaction.category` adds a join on `category`. + ### Filtering reverse relations -Reverse relations (`BackRef`) are chainable queries in their own right — filter, order, and slice them before executing, and their predicates can traverse too: +A reverse relation on an instance (`account.transactions`) is a query too. Filter, order and count it before you await it: ```python -published = await author.posts.where(lambda post: post.published == True).all() # noqa: E712 -latest = await author.posts.order_by(lambda post: post.created_at, "desc").limit(5).all() -n = await author.posts.count() +--8<-- "docs/examples/traversal.py:reverse-filter" ``` -That is a query *from* an instance. To filter the **root query** on membership in a reverse relation — "authors who have at least one published post" — use an [existence test](#existence-tests-on-reverse-many-to-many-relations): `Author.where(lambda a: a.posts.exists(lambda post: post.published == True))`. +Checking holds five transactions, and its two largest debits are Rent and To savings. That query starts from one account. To filter accounts by what their transactions hold, use an [existence test](#existence-tests-on-reverse-many-to-many-relations). -### Results are plain root instances +Fetching Checking and the two queries send these statements; on PostgreSQL only the placeholders differ: -A traversed query is **shape-preserving**: filtering `Transaction` through `transaction.account.ledger_id` still returns `Transaction` instances, no matter how deep the predicate reaches. Traversal does *not* pre-load the related rows onto the results — `await transaction.account` still issues its own query, exactly as it does without any traversal. Attaching related data is a separate, explicit request: [`include()`](#populating-relations-with-include). +```sql +-- traversal.py:reverse-filter (sqlite) +SELECT "account"."id", "account"."label", "account"."owner_id" +FROM "account" +WHERE "account"."id" = ? +LIMIT ?; + +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" +FROM "transaction" +WHERE "transaction"."account_id" = ? AND "transaction"."amount" < ? +ORDER BY "transaction"."amount" ASC NULLS LAST +LIMIT ?; -### `update()` and `delete()` cannot traverse +SELECT COUNT(*) +FROM "transaction" +WHERE "transaction"."account_id" = ?; +``` -Portable SQL has no `UPDATE … JOIN` / `DELETE … JOIN`, so a traversed predicate on `update()`/`delete()` is rejected **before any SQL runs**: +The reverse relation became a filter on `account_id`. -```text ->>> await Transaction.where(lambda transaction: transaction.account.label == "a1").delete() -ValueError: delete() does not support relation traversal: portable SQL has no DELETE ... JOIN. Fetch primary keys via the joined query first, then delete by primary-key set. (A join-free relation filter like `t.account == instance` is allowed.) +### Result type of traversed queries + +A traversed query returns the model you queried: filtering `Transaction` through `transaction.account.owner` still returns `Transaction` instances, and it does not load the accounts. `await transaction.account` still runs its own query. To load related rows with the results, use [`include()`](#populating-relations-with-include). + +### Traversal in `update()` and `delete()` + +Portable SQL has no `UPDATE ... JOIN` or `DELETE ... JOIN`, so `update()` and `delete()` refuse a traversed predicate before any SQL runs: + +```python +--8<-- "docs/examples/traversal.py:mutate-error" ``` -Do the two-step: fetch the primary keys with the joined query, then mutate by that key set (a join-free `relation == instance` or `== None` filter *is* allowed on mutations): +The message names the fix. Fetch the primary keys through the join first, then update or delete by that key set: ```python --8<-- "docs/examples/traversal.py:mutate-limitation" ``` -### Guardrails +Both Travel transactions are marked cleared. A join-free relation filter, such as `transaction.account == travel` or `transaction.category == None`, is allowed on `update()` and `delete()` as it is. + +The fetch and the update send these statements. PostgreSQL also casts the date parameter to `date`: + +=== "SQLite" + + ```sql + -- traversal.py:mutate-limitation (sqlite) + SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" + FROM "transaction" + INNER JOIN "account" AS "j1_account" ON "transaction"."account_id" = "j1_account"."id" + WHERE "j1_account"."label" = ?; + + UPDATE "transaction" + SET "cleared_on" = ? + WHERE "transaction"."id" IN (?, ?); + ``` + +=== "PostgreSQL" -- A predicate lambda that returns a **bare relation** (`lambda transaction: transaction.account`) is meaningless as a filter and raises `TypeError` pointing you at `== None`, `== an instance`, or a column comparison. The same bare relation in `order_by()` is likewise rejected. -- Combining predicates with Python's `and`/`or`/`not` (instead of `&`/`|`/`~`) coerces a node to `bool` and raises `TypeError: QueryNode cannot be used in a boolean context; use & / | to combine predicates and ~ to negate them`. Always parenthesize and use the bitwise operators. + ```sql + -- traversal.py:mutate-limitation (postgres) + SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" + FROM "transaction" + INNER JOIN "account" AS "j1_account" ON "transaction"."account_id" = "j1_account"."id" + WHERE "j1_account"."label" = $1; -See [Relationships](relationships.md) for the schema-declaration side of foreign keys and reverse relations. + UPDATE "transaction" + SET "cleared_on" = CAST($1 AS date) + WHERE "transaction"."id" IN ($2, $3); + ``` + +The join stays in the `SELECT`; the `UPDATE` filters on the fetched keys alone. -## Existence Tests on Reverse & Many-to-Many Relations +### Unsupported predicate forms -Traversal reaches *forward* along a foreign key. The reverse question — "which transactions have at least one split line?", "which transactions appear in any transfer?" — is a different shape: the related rows live on the **other** table, keyed back at you. In a predicate, a reverse (`BackRef`) or many-to-many relation supports exactly one verb for that question, the **existence test**: +A lambda that returns a bare relation (`lambda transaction: transaction.account`) filters nothing, so it raises `TypeError` and names the forms that do: `== None`, `== instance` or a column comparison. Python's `and` and `or` raise `TypeError` too: ```python -matches = await Transaction.where(lambda t: t.lines.exists()).all() +--8<-- "docs/examples/traversal.py:guardrails" ``` -`.exists()` renders as a correlated `EXISTS` subquery — never a join — so the result stays **root-shaped**: each matching row comes back exactly once (a transaction with three lines is one result, no `DISTINCT` bookkeeping), rows are never multiplied, and the test composes with every other predicate, ordering, and paging. One verb covers every cardinality: a one-to-one `BackRef`, a to-many `BackRef`, and an M2M edge all spell the same, so a schema cardinality change never breaks a call site. +Both refusals happen when you build the query. [Relationships](relationships.md) shows how to declare foreign keys and reverse relations. The [`Query` reference](../api/queries.md) lists `join()` and `left_join()` with their parameters. -The examples below use this schema — a transfer links two transactions through unique FKs (one-to-one BackRefs), split lines hang off a transaction (to-many), and a category is referenced by both layers: +## Existence tests on reverse & many-to-many relations -=== "Assignment" +A foreign key points from the row you query to one related row. A reverse relation (`BackRef`) or a many-to-many relation goes the other way, to any number of rows on another table. In a predicate, those relations answer one question, "is there at least one?", with `.exists()`: - ```python - --8<-- "docs/examples/existence_tests.py:schema" - ``` +```python +--8<-- "docs/examples/existence_tests.py:first" +``` -=== "Annotated" +Weekly shop is the only transaction with split lines. `.exists()` becomes a correlated `EXISTS` subquery, not a join, so each matching row comes back once however many related rows it has. It works the same for a one-to-one reverse relation, a one-to-many one and a many-to-many one, so changing a relation's cardinality never breaks a query. These examples run inside the session from [the first block](#queries). - ```python - --8<-- "docs/examples/existence_tests_annotated.py:schema" - ``` +The query sends this; on PostgreSQL only the placeholders differ: + +```sql +-- existence_tests.py:first (sqlite) +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" +FROM "transaction" +WHERE EXISTS(SELECT ? FROM "splitline" AS "x1_lines" WHERE "x1_lines"."transaction_id" = "transaction"."id"); +``` + +The `EXISTS` subquery over `splitline` refers back to the outer `transaction` row. What it selects does not matter to `EXISTS`, so it selects a bound constant. ### Bare tests and negation -A bare `.exists()` asks "is any related row there?". Negation is the uniform `~` — NOT EXISTS is not a separate spelling: +A bare `.exists()` asks whether any related row is there, and `~` asks whether none is: ```python --8<-- "docs/examples/existence_tests.py:bare" ``` -The first query ships to the database as: +`transfer_out` and `transfer_in` are the one-to-one reverse relations of `Transfer.outflow` and `Transfer.inflow`. The two halves of the transfer come back, and the other seven rows are in no transfer. + +The two queries send these statements; on PostgreSQL only the placeholders differ: ```sql -SELECT ... FROM transaction t -WHERE EXISTS (SELECT 1 FROM transfer WHERE transfer.outflow_transaction_id = t.id) - OR EXISTS (SELECT 1 FROM transfer WHERE transfer.inflow_transaction_id = t.id) +-- existence_tests.py:bare (sqlite) +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" +FROM "transaction" +WHERE EXISTS(SELECT ? FROM "transfer" AS "x1_transfer_out" WHERE "x1_transfer_out"."outflow_id" = "transaction"."id") OR EXISTS(SELECT ? FROM "transfer" AS "x2_transfer_in" WHERE "x2_transfer_in"."inflow_id" = "transaction"."id"); + +SELECT COUNT(*) +FROM "transaction" +WHERE (NOT EXISTS(SELECT ? FROM "transfer" AS "x1_transfer_out" WHERE "x1_transfer_out"."outflow_id" = "transaction"."id")) AND (NOT EXISTS(SELECT ? FROM "transfer" AS "x2_transfer_in" WHERE "x2_transfer_in"."inflow_id" = "transaction"."id")); ``` +`~` became `NOT EXISTS`, and each relation has its own subquery. + ### Scoping with an inner predicate -Pass a lambda to filter *which* related rows count. The inner lambda is a **full ferro predicate over the related model** — every operator, `&`/`|`/`~`, forward traversal, even nested existence tests — not a sub-language: +Pass `.exists()` a lambda to choose which related rows count. The inner lambda is a full predicate over the related model, with every operator, `&`, `|`, `~`, traversal and further existence tests: ```python --8<-- "docs/examples/existence_tests.py:scoped" ``` -Rendered SQL — the root branch keeps line-less transactions, the `EXISTS` branch finds categories on any line: +Market run has Groceries itself, and Weekly shop has it on a split line. The query sends this; on PostgreSQL only the placeholders differ: ```sql -SELECT ... FROM transaction t -WHERE t.category_id IN (...) - OR EXISTS (SELECT 1 FROM split_line l - WHERE l.txn_id = t.id AND l.category_id IN (...)) +-- existence_tests.py:scoped (sqlite) +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" +FROM "transaction" +WHERE "transaction"."category_id" = ? OR EXISTS(SELECT ? FROM "splitline" AS "x1_lines" WHERE "x1_lines"."transaction_id" = "transaction"."id" AND "x1_lines"."category_id" = ?); ``` -Because the result is root-shaped, keyset ordering and paging compose unchanged: +The inner lambda's condition joined the subquery's `WHERE`. The result has one row per transaction, so ordering and paging work as on any query: ```python --8<-- "docs/examples/existence_tests.py:composes" ``` -### Grouping is explicit +Weekly shop, at -95.00, is the larger of the two. + +The query sends this; on PostgreSQL only the placeholders differ: + +```sql +-- existence_tests.py:composes (sqlite) +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" +FROM "transaction" +WHERE "transaction"."category_id" = ? OR EXISTS(SELECT ? FROM "splitline" AS "x1_lines" WHERE "x1_lines"."transaction_id" = "transaction"."id" AND "x1_lines"."category_id" = ?) +ORDER BY "transaction"."amount" ASC NULLS LAST +LIMIT ?; +``` + +`ORDER BY` and `LIMIT` apply to the outer query. -With conditions on the *same* relation, there are two different questions: does **one** related row match all conditions, or does **some** related row match each? The lambda scope makes the choice visible — one test with a compound inner predicate, or two tests combined outside: +### Grouping conditions + +Two conditions on the same relation can ask two different questions: is there one related row that matches both, or a row for each? Where you put the conditions decides which: ```python --8<-- "docs/examples/existence_tests.py:grouping" ``` -This is why reverse relations have an explicit combinator rather than implicit path traversal (`t.lines.category_id == x` raises): with implicit traversal, nothing on the page says which of those two questions `(t.lines.a == 1) & (t.lines.b == 2)` asks (ADR-0007). +No single split line is both Dining and below -50, but Weekly shop has a Dining line and a different line below -50. Writing `transaction.lines.category_id` directly raises an error, because nothing on the page would say which of the two questions it asks. + +The two queries send these statements; on PostgreSQL only the placeholders differ: + +```sql +-- existence_tests.py:grouping (sqlite) +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" +FROM "transaction" +WHERE EXISTS(SELECT ? FROM "splitline" AS "x1_lines" WHERE "x1_lines"."transaction_id" = "transaction"."id" AND ("x1_lines"."category_id" = ? AND "x1_lines"."amount" < ?)); + +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" +FROM "transaction" +WHERE EXISTS(SELECT ? FROM "splitline" AS "x1_lines" WHERE "x1_lines"."transaction_id" = "transaction"."id" AND "x1_lines"."category_id" = ?) AND EXISTS(SELECT ? FROM "splitline" AS "x2_lines" WHERE "x2_lines"."transaction_id" = "transaction"."id" AND "x2_lines"."amount" < ?); +``` + +One subquery holding both conditions asks for one line that matches both; two subqueries ask for a line for each. ### Traversal inside the test, and nesting -Forward-FK traversal works inside the inner lambda — its joins render *inside* the `EXISTS` subquery with the same [INNER semantics](#every-traversed-hop-is-an-inner-join) as everywhere else — and existence tests nest to any depth: +The inner lambda can follow foreign keys on the related model. Those joins run inside the `EXISTS` subquery, with the same [inner-join rule](#join-type-for-traversed-relations), so they filter the related rows and never the outer ones: ```python --8<-- "docs/examples/existence_tests.py:traversal-inside" ``` -### Many-to-many +Weekly shop's lines are in Groceries and Dining, both under Food. The query sends this; on PostgreSQL only the placeholders differ: -An M2M relation spells identically, from either side — the test correlates through the association table and the inner lambda scopes over the target model: +```sql +-- existence_tests.py:traversal-inside (sqlite) +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" +FROM "transaction" +WHERE EXISTS(SELECT ? FROM "splitline" AS "x1_lines" INNER JOIN "category" AS "x2_category" ON "x1_lines"."category_id" = "x2_category"."id" INNER JOIN "category" AS "x3_parent" ON "x2_category"."parent_id" = "x3_parent"."id" WHERE "x1_lines"."transaction_id" = "transaction"."id" AND "x3_parent"."name" = ?); +``` -=== "Assignment" +Both `category` joins sit inside the subquery. Existence tests nest to any depth: - ```python - --8<-- "docs/examples/existence_tests.py:m2m-schema" - ``` +```python +--8<-- "docs/examples/existence_tests.py:nested" +``` -=== "Annotated" +Only Ada has an account holding a split transaction. - ```python - --8<-- "docs/examples/existence_tests_annotated.py:m2m-schema" - ``` +The query sends this; on PostgreSQL only the placeholders differ: + +```sql +-- existence_tests.py:nested (sqlite) +SELECT "owner"."email", "owner"."id", "owner"."name" +FROM "owner" +WHERE EXISTS(SELECT ? FROM "account" AS "x1_accounts" WHERE "x1_accounts"."owner_id" = "owner"."id" AND EXISTS(SELECT ? FROM "transaction" AS "x2_transactions" WHERE "x2_transactions"."account_id" = "x1_accounts"."id" AND EXISTS(SELECT ? FROM "splitline" AS "x3_lines" WHERE "x3_lines"."transaction_id" = "x2_transactions"."id"))); +``` + +Each level is an `EXISTS` inside the one before, tied to its parent's row. + +### Many-to-many + +A many-to-many relation reads the same from either side. The test goes through the link table, and the inner lambda scopes over the other model: ```python --8<-- "docs/examples/existence_tests.py:m2m" ``` -### One verb, loud dead ends +Rent carries `tax`, `reimbursable` is on the uncleared Lyon rows, and six transactions have no tag. + +The three queries send these statements; on PostgreSQL only the placeholders differ: + +```sql +-- existence_tests.py:m2m (sqlite) +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" +FROM "transaction" +WHERE EXISTS(SELECT ? FROM "tag_transactions" AS "x1_tags" INNER JOIN "tag" AS "x2_tags" ON "x1_tags"."tag_id" = "x2_tags"."id" WHERE "x1_tags"."transaction_id" = "transaction"."id" AND "x2_tags"."name" = ?); -Reverse relations are **tested, never traversed** (ADR-0007). Every other way of naming a reverse or M2M relation in a query fails at build time with the supported spelling in the message: +SELECT "tag"."id", "tag"."name" +FROM "tag" +WHERE EXISTS(SELECT ? FROM "tag_transactions" AS "x1_transactions" INNER JOIN "transaction" AS "x2_transactions" ON "x1_transactions"."transaction_id" = "x2_transactions"."id" WHERE "x1_transactions"."tag_id" = "tag"."id" AND "x2_transactions"."cleared_on" IS NULL); -- **Column access** (`t.lines.category_id`) raises `AttributeError` — scope the columns with the inner lambda instead: `t.lines.exists(lambda line: line.category_id == ...)`. -- **Comparisons**, including the tempting `t.transfer_out != None`, raise `TypeError` — a reverse relation has no root-side column to be `NULL`; the spelling is `t.transfer_out.exists()` (and `~t.transfer_out.exists()` for absence). -- **`in_()` with a query** (`t.id.in_(subquery)`) raises `TypeError` — the workloads an `IN (subquery)` serves are existence-test workloads, without hand-correlating on id columns. -- **`join()` / `left_join()` on a reverse edge** raise `TypeError` — a join there would multiply root rows; membership is the existence test's job. -- **The inner lambda sees only its own parameter.** Referencing the outer lambda's parameter (or comparing column-to-column) is a build-time error — cross-scope correlation is a tracked future capability ([#309](https://github.com/syn54x/ferro-orm/issues/309)), never a silent misrender. +SELECT COUNT(*) +FROM "transaction" +WHERE NOT EXISTS(SELECT ? FROM "tag_transactions" AS "x1_tags" INNER JOIN "tag" AS "x2_tags" ON "x1_tags"."tag_id" = "x2_tags"."id" WHERE "x1_tags"."transaction_id" = "transaction"."id"); +``` -Existence tests answer **membership** only. *Populating* a reverse collection onto results (the data axis) is a separate future mechanism — see [Not Yet Supported](#not-yet-supported); until it lands, fetch collections through the relation itself (`await txn.lines.all()`). +Each test reaches the other model through the link table, `tag_transactions`, from either side. -!!! note "Two `exists`, two levels" - `t.lines.exists(...)` inside a predicate is the existence *test* on a relation. `await query.exists()` is the query *terminal* asking whether the whole query matches any row. Same word, deliberately — both ask "is there at least one?" — at different levels. +### Unsupported reverse-relation forms -## Populating Relations with include() +A reverse or many-to-many relation is tested, never traversed. Every other use in a query fails when you build it, and the message names the `.exists()` form to use: -Every relation access is a query. A list view that renders 100 transactions with their account labels awaits `transaction.account` 100 times — 101 statements for one screen. `include()` cures that N+1: ask the query to bring the related rows along, and each result's relation arrives **populated**: +- Reading a column (`transaction.lines.category_id`) raises `AttributeError`. Put the condition in the inner lambda: `transaction.lines.exists(lambda split_line: split_line.category_id == ...)`. +- Comparing the relation, including `transaction.transfer_out != None`, raises `TypeError`. The relation has no column on this table to be `NULL`; write `transaction.transfer_out.exists()`, or `~transaction.transfer_out.exists()` for none. +- `in_()` with a query (`transaction.id.in_(query)`) raises `TypeError`. An existence test answers the same question without matching ids by hand. +- `join()` and `left_join()` on a reverse relation raise `TypeError`, because the join would repeat each row once per related row. +- The inner lambda sees only its own parameter. Referring to the outer lambda's parameter raises an error; see [Not yet supported](#not-yet-supported). + +```python +--8<-- "docs/examples/existence_tests.py:dead-ends" +``` + +Each of the three refusals names the existence test to write instead. + +!!! note "Two meanings of `exists`" + `transaction.lines.exists(...)` inside a predicate tests a relation. `await query.exists()` is the terminal that asks whether the whole query matches any row. Both ask "is there at least one?", at different levels. + +The [`Query` reference](../api/queries.md) describes `.exists()` with its parameters. + +## Populating relations with `include()` + +Reading a relation runs a query. A list of 100 transactions that shows each account's label awaits `transaction.account` 100 times: 101 statements for one screen. [`include()`][ferro.query.builder.Query.include] asks the query to load the related rows in the same statement: ```python --8<-- "docs/examples/populated_relations.py:basic" ``` -One SQL statement. The query still returns the same `list[Transaction]` it always did — and `transaction.account` is now a plain attribute holding the complete `Account` instance, exactly as the field's annotation (`account: Account`) always claimed. No await, no query. +It prints: -The examples in this section use this schema (both foreign keys nullable, so you can see what happens when a relation is absent): + +```text +Weekly shop | Checking +Office coffee | Petty cash +Dinner in Lyon | Travel +``` -=== "Assignment" +The query still returns `list[Transaction]`, and `transaction.account` is now the `Account` instance itself, as the field's annotation (`account: Account`) says. Reading it needs no `await` and runs no query. These examples run inside the session from [the first block](#queries). - ```python - --8<-- "docs/examples/populated_relations.py:schema" - ``` +The query sends this; on PostgreSQL only the placeholders differ: -=== "Annotated" +```sql +-- populated_relations.py:basic (sqlite) +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on", "j1_account"."id" AS "j1_account__id", "j1_account"."label" AS "j1_account__label", "j1_account"."owner_id" AS "j1_account__owner_id" +FROM "transaction" +LEFT JOIN "account" AS "j1_account" ON "transaction"."account_id" = "j1_account"."id" +WHERE TRUE +ORDER BY "transaction"."id" DESC NULLS LAST +LIMIT ?; +``` - ```python - --8<-- "docs/examples/populated_relations_annotated.py:schema" - ``` +The account's columns come back in the same rows, through a `LEFT JOIN`, under `j1_account__` names. ### The population contract -A relation is in exactly one of two states on any given instance: +A relation on an instance is either populated or not: -- **Populated** (the query included it): access is a **plain attribute** returning the complete related instance. A nullable FK with no target populates as `None` — truthful to the declared `Account | None`. -- **Unpopulated** (everything else): access keeps today's **awaitable** contract, unchanged — a coroutine that runs its own query: +- **Populated**, because a query included it: the attribute is the related instance. A nullable foreign key with no row populates as `None`. +- **Not populated**: the attribute is awaitable and runs its own query: ```python --8<-- "docs/examples/populated_relations.py:awaitable" ``` -!!! warning "Awaiting a populated relation is a hard break" - `await transaction.account` on a *populated* instance raises `TypeError: 'Account' object can't be awaited` — the attribute is the instance itself, not a coroutine. Code that must handle both states cheaply can check `inspect.isawaitable(...)`, but the better pattern is to decide at the query: if you are going to read the relation, include it. +`await transaction.account` fetched the Travel account with a second query. -There is no separate "loaded" model type: `.all()` on an included query still returns `list[Transaction]`, and a populated instance is an ordinary instance of the target model. (Why no `Loaded[Transaction]`? See [Typed Query Predicates](../concepts/query-typing.md#included-queries).) +The fetch and the await send these statements; on PostgreSQL only the placeholders differ: -### Include never changes membership +```sql +-- populated_relations.py:awaitable (sqlite) +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" +FROM "transaction" +WHERE "transaction"."id" = ? +LIMIT ?; + +SELECT "account"."id", "account"."label", "account"."owner_id" +FROM "account" +WHERE "account"."id" = ? +LIMIT ?; +``` + +The second statement is the one `await transaction.account` ran. -Joins decide **membership**, projection decides **shape**, include decides **attached data** — three orthogonal axes. Adding `.include(...)` to any query returns exactly the rows that query returned without it: +!!! warning "Awaiting a populated relation raises" + On a populated instance, `await transaction.account` raises `TypeError: 'Account' object can't be awaited`, because the attribute is the instance, not a coroutine. Decide at the query: if you will read the relation, include it. + +There is no separate "loaded" model type: an included query returns `list[Transaction]`, and a populated relation is an ordinary instance. [Typed query predicates](../concepts/query-typing.md#included-queries) explains why. + +### Row membership under `include()` + +`include()` never changes which rows a query returns. A row whose relation is missing stays, with the relation populated as `None`: ```python --8<-- "docs/examples/populated_relations.py:membership" ``` -Under the hood, an edge only the include touches renders a LEFT join, and an edge any other clause references keeps that clause's join type. So a `where()` traversal on the same path keeps its INNER narrowing — include attaches data to the surviving rows, it never rewrites what a filter matches: +All nine transactions come back, and Salary's `category` is `None`. The query sends this; on PostgreSQL only the placeholders differ: + +```sql +-- populated_relations.py:membership (sqlite) +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on", "j1_category"."id" AS "j1_category__id", "j1_category"."name" AS "j1_category__name", "j1_category"."parent_id" AS "j1_category__parent_id" +FROM "transaction" +LEFT JOIN "category" AS "j1_category" ON "transaction"."category_id" = "j1_category"."id" +WHERE TRUE; +``` + +A relation only `include()` uses becomes a `LEFT JOIN`, which keeps Salary. A relation a `where()` or `order_by()` also follows keeps that clause's inner join, so the filter still decides membership: ```python --8<-- "docs/examples/populated_relations.py:interplay" ``` -`count()` and `exists()` are likewise unaffected — they measure the same rows with or without the include: +The `where()` keeps the two Travel rows, and `include()` attaches their account. The query sends this; on PostgreSQL only the placeholders differ: + +```sql +-- populated_relations.py:interplay (sqlite) +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on", "j1_account"."id" AS "j1_account__id", "j1_account"."label" AS "j1_account__label", "j1_account"."owner_id" AS "j1_account__owner_id" +FROM "transaction" +INNER JOIN "account" AS "j1_account" ON "transaction"."account_id" = "j1_account"."id" +WHERE "j1_account"."label" = ?; +``` + +The filter and the population share one `INNER JOIN`. `count()` and `exists()` measure the same rows with or without `include()`: ```python --8<-- "docs/examples/populated_relations.py:count" ``` -### Multi-hop paths populate every hop +Both count the seven debits. -`include(lambda t: t.account.owner)` populates the whole path — a populated graph is never missing its intermediate nodes. A `NULL` somewhere along the chain ends the chain as a populated `None`, with the root row retained: +The two calls send the same statement; on PostgreSQL only the placeholders differ: + +```sql +-- populated_relations.py:count (sqlite) +SELECT COUNT(*) +FROM "transaction" +WHERE "transaction"."amount" < ?; + +SELECT COUNT(*) +FROM "transaction" +WHERE "transaction"."amount" < ?; +``` + +`include()` adds nothing to a count. + +### Multi-hop paths + +Including a path populates every step of it. A `NULL` along the way ends the chain with `None`, and the row stays: ```python --8<-- "docs/examples/populated_relations.py:multi-hop" ``` -Includes are cumulative, order-free, and idempotent: `.include(lambda t: t.account)` plus `.include(lambda t: t.account.owner)` — in either order — is the same query as `.include(lambda t: t.account.owner)` alone. +Salary's account and its owner are both loaded. Office coffee's account is Petty cash, whose owner is `None`. -### Populated instances are session instances +The query sends this; on PostgreSQL only the placeholders differ: -In a session, population runs through the identity map like every other fetch: same row, same object. A populated `Account` **is** the `Account` a direct fetch returns — deduped across result rows, and attached onto instances the session already holds: +```sql +-- populated_relations.py:multi-hop (sqlite) +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on", "j1_account"."id" AS "j1_account__id", "j1_account"."label" AS "j1_account__label", "j1_account"."owner_id" AS "j1_account__owner_id", "j2_owner"."email" AS "j2_owner__email", "j2_owner"."id" AS "j2_owner__id", "j2_owner"."name" AS "j2_owner__name" +FROM "transaction" +LEFT JOIN "account" AS "j1_account" ON "transaction"."account_id" = "j1_account"."id" +LEFT JOIN "owner" AS "j2_owner" ON "j1_account"."owner_id" = "j2_owner"."id" +WHERE TRUE +ORDER BY "transaction"."id" ASC NULLS LAST; +``` + +Each step is one more `LEFT JOIN`, and each related row's columns come back under its alias. + +Includes add up, in any order, and repeating one changes nothing: `.include(lambda transaction: transaction.account)` followed by `.include(lambda transaction: transaction.account.owner)` is the same query as the second one alone. + +### Identity of populated instances + +In a session, each row is one object. A populated account is the same object a direct fetch of that account returns, shared by every row that points at it: ```python --8<-- "docs/examples/populated_relations.py:identity" ``` -Populations **accumulate** across a session's queries — an `account` include and a later `attachment` include both stick, so shared objects get richer, never forked: +Both Travel transactions point at the `travel` object fetched first. Fetching Travel and the query send these statements; on PostgreSQL only the placeholders differ: + +```sql +-- populated_relations.py:identity (sqlite) +SELECT "account"."id", "account"."label", "account"."owner_id" +FROM "account" +WHERE "account"."id" = ? +LIMIT ?; + +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on", "j1_account"."id" AS "j1_account__id", "j1_account"."label" AS "j1_account__label", "j1_account"."owner_id" AS "j1_account__owner_id" +FROM "transaction" +LEFT JOIN "account" AS "j1_account" ON "transaction"."account_id" = "j1_account"."id" +WHERE "transaction"."account_id" = ?; +``` + +The account's columns come back with each row, and the session hands back the object it already holds. Populations also accumulate on the objects a session holds: a later query's `include()` attaches to instances an earlier query returned: ```python --8<-- "docs/examples/populated_relations.py:accumulate" ``` -Outside a session there is no identity map, so an included query returns fresh instances per row — including a fresh related instance per row, with no query-local dedup. Sessionless Ferro keeps exactly one identity story: the session. +`market_run` was fetched with nothing populated, and the two later queries populated its account and its category. Outside a session there is no identity map, so each row of an included query gets its own related instance. [Identity map](../concepts/identity-map.md) covers sessions and shared instances. + +The three queries send these statements; on PostgreSQL only the placeholders differ: + +```sql +-- populated_relations.py:accumulate (sqlite) +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" +FROM "transaction" +WHERE "transaction"."id" = ? +LIMIT ?; + +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on", "j1_account"."id" AS "j1_account__id", "j1_account"."label" AS "j1_account__label", "j1_account"."owner_id" AS "j1_account__owner_id" +FROM "transaction" +LEFT JOIN "account" AS "j1_account" ON "transaction"."account_id" = "j1_account"."id" +WHERE TRUE; + +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on", "j1_category"."id" AS "j1_category__id", "j1_category"."name" AS "j1_category__name", "j1_category"."parent_id" AS "j1_category__parent_id" +FROM "transaction" +LEFT JOIN "category" AS "j1_category" ON "transaction"."category_id" = "j1_category"."id" +WHERE TRUE; +``` + +Neither later query names `market_run`; the session matched its row by primary key. -### Refreshes drop populations that stopped being true +### Population after a refresh -Every fetch refreshes instances the session already holds. A refresh keeps each population **iff the row's foreign key still points at it**; if the FK changed (or went `NULL`) underneath you, the now-lying population is dropped and access reverts to the awaitable — a populated relation never points at the wrong row: +Every fetch refreshes the instances a session already holds. A refresh keeps a population only while the row's foreign key still points at it; if the foreign key changed, the population is dropped and the relation is awaitable again: ```python --8<-- "docs/examples/populated_relations.py:refresh-rule" ``` -### The loud limits +After the row moved to Checking, `from_checking.account` no longer claims Savings; awaiting it fetches Checking. + +The four statements are the same on PostgreSQL, apart from the placeholders Ferro writes: + +```sql +-- populated_relations.py:refresh-rule (sqlite) +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on", "j1_account"."id" AS "j1_account__id", "j1_account"."label" AS "j1_account__label", "j1_account"."owner_id" AS "j1_account__owner_id" +FROM "transaction" +LEFT JOIN "account" AS "j1_account" ON "transaction"."account_id" = "j1_account"."id" +WHERE "transaction"."id" = ? +LIMIT ?; + +UPDATE "transaction" +SET account_id = $1 +WHERE id = $2; -Misuse raises at build time, before any SQL: +SELECT "transaction"."account_id", "transaction"."amount", "transaction"."category_id", "transaction"."cleared_on", "transaction"."id", "transaction"."memo", "transaction"."posted_on" +FROM "transaction" +WHERE "transaction"."id" = ? +LIMIT ?; + +SELECT "account"."id", "account"."label", "account"."owner_id" +FROM "account" +WHERE "account"."id" = ? +LIMIT ?; +``` -- **Forward foreign keys only.** Including a `BackRef` or `ManyToMany` relation raises `TypeError`: reverse and M2M population will be a separate mechanism (a batched second query stitched onto the results), not `include()`. Until it lands, fetch collections through the relation itself (`await author.posts.all()`). *Filtering* on reverse membership is a different axis and already works — the [existence test](#existence-tests-on-reverse-many-to-many-relations). -- **Lambda selectors only.** `include("account")` raises pointing at the lambda form — strings never traverse. Selecting a *column* (`include(lambda t: t.account.label)`) raises too: every populated hop is a complete row, so there is nothing to select per column. -- **No include × projection.** A query carries exactly one materialization plan — populated instances or projected records, never both — and record results are flat, permanently. Either order raises `ValueError` pointing at [traversed projection](#reaching-across-a-relation), the record-shaped way across a relation. -- **No mutations.** `update()`/`delete()` on an included query raise — a mutation returns no instances to populate. +Ferro sends the raw `UPDATE` as written. The last `SELECT` is the await, fetching Checking. + +### Unsupported `include()` forms + +Ferro refuses these when you build the query, before any SQL: + +| Form | Error | Use instead | +| :--- | :--- | :--- | +| A `BackRef` or `ManyToMany` relation | `TypeError` | `await account.transactions.all()`; to filter on the relation, an [existence test](#existence-tests-on-reverse-many-to-many-relations) | +| A string (`include("account")`) or a column (`include(lambda transaction: transaction.account.label)`) | `TypeError` | A lambda that names a relation; each populated step is a whole row | +| `include()` with `select(...)`, in either order | `ValueError` | [A projection that reaches across the relation](#reaching-across-a-relation) | +| `update()` or `delete()` on an included query | `ValueError` | The same query without `include()` | ```python --8<-- "docs/examples/populated_relations.py:limits" ``` -Include composes with the M2M association context (`post.tags.include(lambda tag: tag.created_by)` populates each tag's forward FK) and with everything else a query does: `where()` (traversal included), `order_by()`, `limit()`/`offset()`, `first()`, and query branching. +Each refusal names the form to use. `include()` also works on a many-to-many query (`tag.transactions.include(lambda transaction: transaction.account)`), and with `where()`, `order_by()`, `limit()`, `offset()` and `first()`. The [`Query` reference](../api/queries.md#ferro.query.Query.include) lists its parameters. -## Selecting a Column Subset +## Selecting a column subset -Every query so far loads complete rows into complete model instances. When a list view only reads two columns of a wide table, ask for less: pass `select()` a lambda naming the columns, and the query becomes a **projection**: +A query loads whole rows into whole model instances. When a view only reads two columns of a wide table, pass [`select()`][ferro.query.builder.Query.select] a lambda naming the columns, and the query returns records with only those columns: ```python --8<-- "docs/examples/partial_selects.py:basic" ``` -The examples in this section use this schema: +Each record has a `memo` and an `amount`, read as attributes like a model's. These examples run inside the session from [the first block](#queries). -=== "Assignment" +The query sends this; on PostgreSQL only the placeholders differ: - ```python - --8<-- "docs/examples/partial_selects.py:schema" - ``` - -=== "Annotated" - - ```python - --8<-- "docs/examples/partial_selects_annotated.py:schema" - ``` +```sql +-- partial_selects.py:basic (sqlite) +SELECT "transaction"."memo" AS "memo", "transaction"."amount" AS "amount" +FROM "transaction" +WHERE TRUE +ORDER BY "transaction"."id" ASC NULLS LAST +LIMIT ?; +``` -### What you get back — and why it isn't a `Transaction` +Only the two selected columns are in the `SELECT`, each named with `AS`. -A projected query does **not** return `Transaction` instances. In Ferro, **a model instance always carries a complete row** — there is no such thing as a partial or deferred-field model instance, anywhere. If projection handed you a two-column `Transaction`, every `save()`, every refresh, and every helper that takes a model would have to wonder which query produced its argument, and touching an unselected field would blow up far from the query that caused it. +### The `Row` result type -So anything narrower than a full row comes back honestly typed as what it is: a **projected record** — a `Row`, delivered in the list-like `Rows` container: +A selected query does not return `Transaction` instances. It returns [`Row`][ferro.query.Row] records in a [`Rows`][ferro.query.Rows] list: ```python --8<-- "docs/examples/partial_selects.py:not-a-model" ``` -A `Row` is read-only in the persistence sense: it has no `save()`, no refresh, and never enters the identity map — a record can never masquerade as a row in the database. It carries exactly the columns you selected, in selection order, decoded by the same machinery as full hydration — a projected `datetime`, `UUID`, enum, or `Decimal` column has the same Python type and value it would have on the model, on both backends. Under the hood the query declares this result shape explicitly (a *materialization plan* travels with the query), which is also why asking for less is never slower per row than asking for everything: records are built on the same zero-validation path as models. +A `Row` holds the selected columns, in the order you selected them, with the same Python types the model would give them. It has no `save()`, is not refreshed, and never enters the identity map. A model instance always holds a whole row; [Typed query predicates](../concepts/query-typing.md#projected-queries) explains why a selected query returns records instead. -Selecting a single column needs no tuple ceremony: +One column needs no tuple: ```python --8<-- "docs/examples/partial_selects.py:single" ``` -### `Rows` is a list you can ship +Each record holds only the `memo` of an uncleared transaction. -`Rows` behaves like a list — index, slice, iterate, `len()` — and both `Rows` and `Row` are pydantic-shaped: `model_dump()` yields `list[dict]`, and `Rows[Row]` drops straight into a FastAPI `response_model`: +The query sends this; on PostgreSQL only the placeholders differ: + +```sql +-- partial_selects.py:single (sqlite) +SELECT "transaction"."memo" AS "memo" +FROM "transaction" +WHERE "transaction"."cleared_on" IS NULL +ORDER BY "transaction"."id" ASC NULLS LAST; +``` + +The statement selects `memo` alone. + +### The `Rows` container + +`Rows` behaves like a list: index it, slice it, iterate it and take its `len()`. `model_dump()` returns a list of dicts, and `Rows[Row]` works as a FastAPI `response_model`: ```python --8<-- "docs/examples/partial_selects.py:container" ``` +A slice of a `Rows` is a `Rows` too, with the two newest records. + +The query sends this; on PostgreSQL only the placeholders differ: + +```sql +-- partial_selects.py:container (sqlite) +SELECT "transaction"."id" AS "id", "transaction"."memo" AS "memo" +FROM "transaction" +WHERE TRUE +ORDER BY "id" DESC NULLS LAST; +``` + +Indexing, slicing and `model_dump()` run in Python, over the records this one statement returned. + ### Column-name strings -Quick scripts can pass column names as strings — the same string contract as `order_by()`: root columns only (shadow `{fk}_id` columns included), validated at build time. The lambda form remains the documented style: +`select()` also takes column names as strings, with the same rules as `order_by()`: columns on the queried model only, the `_id` columns included, checked when you build the query: ```python --8<-- "docs/examples/partial_selects.py:strings" ``` +The lambda form is the documented one because only a lambda can reach across a relation. + +The query sends this; on PostgreSQL only the placeholders differ: + +```sql +-- partial_selects.py:strings (sqlite) +SELECT "transaction"."id" AS "id", "transaction"."memo" AS "memo" +FROM "transaction" +WHERE TRUE +ORDER BY "id" ASC NULLS LAST +LIMIT ?; +``` + +Each string became a column of `transaction`, named after itself. + ### Reaching across a relation -A selected field may traverse a forward-FK relation, at any depth — the same attribute chaining as a `where()` predicate. Unaliased, a traversed field takes the **bare leaf column name**: +A selected column can follow a foreign key, at any depth, as in a `where()` lambda. A column reached this way is named after its last step: ```python --8<-- "docs/examples/partial_selects.py:traversed" ``` -Strings never traverse (`select("account.label")` is rejected permanently) — traversed projection is lambda-only. +`transaction.account.label` becomes the field `label`. Strings never follow relations: `select("account.label")` raises. + +The query sends this; on PostgreSQL only the placeholders differ: + +```sql +-- partial_selects.py:traversed (sqlite) +SELECT "transaction"."memo" AS "memo", "j1_account"."label" AS "label" +FROM "transaction" +INNER JOIN "account" AS "j1_account" ON "transaction"."account_id" = "j1_account"."id" +WHERE TRUE +ORDER BY "transaction"."id" ASC NULLS LAST +LIMIT ?; +``` + +The traversal joins `account`, and `AS "label"` names the field after the last step. ### Naming output fields -Return a **dict** from the selector and the keys name the record's fields — an *output alias*. Aliases name output fields only, never joins or tables, and the dict's insertion order is the record's field order: +Return a dict from the lambda to name the record's fields. The keys name the fields, in their order: ```python --8<-- "docs/examples/partial_selects.py:aliases" ``` -The dict form is also how you resolve a name collision: `t.id` and `t.account.id` both want to be called `id`, so selecting them unaliased is a build-time error naming this fix: +The record has the three names the dict gave it. The query sends this; on PostgreSQL only the placeholders differ: + +```sql +-- partial_selects.py:aliases (sqlite) +SELECT "transaction"."memo" AS "memo", "j1_account"."label" AS "account", "j2_owner"."email" AS "owner_email" +FROM "transaction" +INNER JOIN "account" AS "j1_account" ON "transaction"."account_id" = "j1_account"."id" +INNER JOIN "owner" AS "j2_owner" ON "j1_account"."owner_id" = "j2_owner"."id" +WHERE TRUE +ORDER BY "transaction"."id" ASC NULLS LAST +LIMIT ?; +``` + +Each dict key became an `AS` name. The dict form also settles a name clash: `transaction.id` and `transaction.account.id` would both be `id`, and selecting both without names raises `ValueError` that suggests the dict: ```python --8<-- "docs/examples/partial_selects.py:collision" ``` -### Traversal narrows; `left_join()` opts out +The message names the dict form as the fix. -Projection traversal is ordinary traversal (ADR-0006): it renders an INNER join per relation path — **shared** with any `where()`/`order_by()` traversal of the same path — so rows without the relation drop out. `left_join()` keeps them, and their traversed fields decode to `None`, *even when the source column is non-nullable* — a projected record describes the row you got, not the related model: +### Traversal and `left_join()` in projections + +A selected column that follows a relation uses the same [inner join](#join-type-for-traversed-relations) as a `where()` on that path, and shares it. Rows without the relation drop out; `left_join()` keeps them, with `None` in the fields they cannot reach: ```python --8<-- "docs/examples/partial_selects.py:traversal-narrows" ``` -### Projections compose like any other query +Office coffee drops out of the first query and comes back in the second with `owner_email` set to `None`. A field filled from a missing relation is `None` even when its column is not nullable: the record describes the row you got. + +The two queries send these statements; on PostgreSQL only the placeholders differ: -`where()` (relation traversal included), `order_by()` (even by columns the projection does not select), `limit()`/`offset()`, `after()`/`before()`, and `first()` all work unchanged. +```sql +-- partial_selects.py:traversal-narrows (sqlite) +SELECT "transaction"."memo" AS "memo", "j2_owner"."email" AS "owner_email" +FROM "transaction" +INNER JOIN "account" AS "j1_account" ON "transaction"."account_id" = "j1_account"."id" +INNER JOIN "owner" AS "j2_owner" ON "j1_account"."owner_id" = "j2_owner"."id" +WHERE TRUE; -On a plain projection, `count()` and `exists()` are unaffected: they measure the same matching rows a full query would. On an *aggregate* projection they raise with guidance instead (see [Aggregations & Grouped Queries](aggregations.md#the-loud-limits)). +SELECT "transaction"."memo" AS "memo", "j2_owner"."email" AS "owner_email" +FROM "transaction" +LEFT JOIN "account" AS "j1_account" ON "transaction"."account_id" = "j1_account"."id" +LEFT JOIN "owner" AS "j2_owner" ON "j1_account"."owner_id" = "j2_owner"."id" +WHERE TRUE +ORDER BY "transaction"."id" ASC NULLS LAST; +``` -`position_of` on a `Row` requires every order key in the projection. Otherwise pass a tuple. +`left_join()` turned both joins into `LEFT JOIN`s. + +### Composing projections + +A selected query takes `where()` (traversal included), `order_by()` (even on a column you did not select), `limit()`, `offset()` and `first()`: ```python --8<-- "docs/examples/partial_selects.py:compose" ``` +The Travel rows come back ordered by `amount`, which is not in the records. The query sends this; on PostgreSQL only the placeholders differ: + +```sql +-- partial_selects.py:compose (sqlite) +SELECT "transaction"."id" AS "id", "transaction"."memo" AS "memo" +FROM "transaction" +INNER JOIN "account" AS "j1_account" ON "transaction"."account_id" = "j1_account"."id" +WHERE "j1_account"."label" = ? +ORDER BY "transaction"."amount" ASC NULLS LAST; +``` + +`ORDER BY` reads `transaction.amount`, a column the `SELECT` does not list. `after()` and `before()` page a selected query too. `after(row)` reads the order keys from the record, so every order key must be selected; otherwise pass a tuple: + ```python --8<-- "docs/examples/partial_selects.py:projected-paging" ``` +The first page holds the five Checking rows, so the next starts at Petty cash. The two pages send these statements; on PostgreSQL only the placeholders differ: + +```sql +-- partial_selects.py:projected-paging (sqlite) +SELECT "j1_account"."label" AS "account", "transaction"."id" AS "id", "transaction"."memo" AS "memo" +FROM "transaction" +INNER JOIN "account" AS "j1_account" ON "transaction"."account_id" = "j1_account"."id" +WHERE TRUE +ORDER BY "j1_account"."label" ASC NULLS LAST, "id" ASC NULLS LAST +LIMIT ?; + +SELECT "j1_account"."label" AS "account", "transaction"."id" AS "id", "transaction"."memo" AS "memo" +FROM "transaction" +INNER JOIN "account" AS "j1_account" ON "transaction"."account_id" = "j1_account"."id" +WHERE "j1_account"."label" > ? OR "j1_account"."label" IS NULL OR ("j1_account"."label" = ? AND ("transaction"."id" > ? OR "transaction"."id" IS NULL)) +ORDER BY "j1_account"."label" ASC NULLS LAST, "id" ASC NULLS LAST +LIMIT ?; +``` + +The second page's position compares the selected `account` and `id`. `count()` and `exists()` count the same rows as the query without `select()`: + ```python --8<-- "docs/examples/partial_selects.py:count" ``` -### Build-time validation and the loud limits +Both count the three uncleared transactions. The two calls send the same statement; on PostgreSQL only the placeholders differ: -A misspelled column fails when you build the query, with a did-you-mean — exactly like `where()` and `order_by()`, and a traversed field validates every hop against that hop's model: +```sql +-- partial_selects.py:count (sqlite) +SELECT COUNT(*) +FROM "transaction" +WHERE "transaction"."cleared_on" IS NULL; -```text ->>> Transaction.select(lambda t: (t.id, t.amonut)) -AttributeError: Transaction has no queryable column 'amonut'. Did you mean 'amount'? Valid columns: account_id, amount, id, memo. +SELECT COUNT(*) +FROM "transaction" +WHERE "transaction"."cleared_on" IS NULL; ``` -Misuse is loud, never silently ignored: +A projection changes no count: both calls send `COUNT(*)` with the query's filter. -- **No output-name collisions.** Two selected fields resolving to the same name raise `ValueError` naming the dict form (see above). -- **One selector, one shape.** A dict nested in a tuple, a tuple (or dict) as a dict value, and non-string dict keys all raise `TypeError` — a projected record is flat. -- **No mutations through a projection.** `update()` / `delete()` on a projected query raise `ValueError` at the call, before any SQL — a projection is a read shape, and silently ignoring it would make `select(...)` a no-op on mutations. Mutate through an unprojected query instead. -- **No double `select()`.** Replacing a projection mid-chain would change the result type; name every field in one call, or start a new query from the model. -- **No mixing forms.** Strings and a lambda in one `select()` call raise `TypeError`. -- **No `include()` with a projection**, in either chain order: a query carries exactly one materialization plan, and record results are flat — permanently. Reach across the relation with traversed projection, or populate instances on an unprojected query. +### Unsupported projection forms + +A misspelled column fails when you build the query, as in `where()`; a column that follows a relation is checked at each step: + +```python +--8<-- "docs/examples/partial_selects.py:typo" +``` + +The message lists the valid columns, as in `where()`. Ferro also refuses these, before any SQL: + +| Form | Error | Use instead | +| :--- | :--- | :--- | +| Two fields with the same name | `ValueError` | The dict form, to name them | +| A dict inside a tuple, a tuple or dict as a dict value, or a key that is not a string | `TypeError` | One flat tuple or one flat dict | +| `update()` or `delete()` on a selected query | `ValueError` | The same query without `select(...)` | +| A second `select(...)` on a selected query | `ValueError` | One `select()` naming every field | +| Strings and a lambda in one `select()` | `TypeError` | One form | +| `include()` with `select(...)`, in either order | `ValueError` | A selected column that reaches across the relation | -Static typing follows the same shape promise as predicates: `select(...)` flips the query's type so `.all()` checks as `Rows[Row]` and `.first()` as `Row | None` — passing a `Row` where a model instance is expected fails the type checker. See [Typed Query Predicates](../concepts/query-typing.md#projected-queries). +Your type checker sees the change too: after `select(...)`, `all()` is `Rows[Row]` and `first()` is `Row | None`, so passing a record where a `Transaction` is expected fails the check. -Aggregation builds directly on this machinery — `select(lambda t: {"total": t.amount.sum()})` — and mixing aggregate and plain fields turns the projection into a **grouped query**. That is its own chapter: [Aggregations & Grouped Queries](aggregations.md). +Aggregates build on projection: `select(lambda transaction: {"total": transaction.amount.sum()})` sums a column, and mixing aggregates with plain columns groups the rows. [Aggregations & grouped queries](aggregations.md) covers them. The [`select()` reference](../api/queries.md#select-overloads) lists every selector form. -## Not Yet Supported +## Not yet supported -!!! note "On the roadmap" - The following query features are **not yet implemented** — see the [Roadmap](../roadmap.md): +These query features are not available yet; the [Roadmap](../roadmap.md) tracks them: - - `having()` — post-aggregation filtering; `where()` rejects aggregate predicates pointing at it ([#291](https://github.com/syn54x/ferro-orm/issues/291)) - - Reverse (`BackRef`) and many-to-many **population** — [`include()`](#populating-relations-with-include) covers forward FKs; collection population is a separate future mechanism. (*Filtering* on reverse/M2M membership is supported — that's the [existence test](#existence-tests-on-reverse-many-to-many-relations).) - - Cross-scope correlation inside an existence test — comparing an inner-lambda column to an outer column ([#309](https://github.com/syn54x/ferro-orm/issues/309)); rejected loudly at build time today - - Case-insensitive `ilike()` +- `having()`, to filter on an aggregate. `where()` refuses an aggregate and names `having()`. +- Populating a reverse or many-to-many relation with its rows. `include()` covers foreign keys; until then, await the relation (`await account.transactions.all()`). Filtering on one is the [existence test](#existence-tests-on-reverse-many-to-many-relations). +- Referring to the outer row inside an existence test's inner lambda, for example comparing a split line's amount to its transaction's amount. Ferro refuses it when you build the query. +- Case-insensitive `ilike()`. -## See Also +## See also -- [Aggregations & Grouped Queries](aggregations.md) — `count`/`sum`/`avg`/`min`/`max` and derived grouping -- [Mutations](mutations.md) — creating, updating, and deleting records -- [Relationships](relationships.md) — forward and reverse relations -- [Typed Query Predicates](../concepts/query-typing.md) — why three predicate styles exist -- [Raw SQL](raw-sql.md) — the escape hatch for queries the ORM can't express -- [Pagination](../howto/pagination.md) — efficient pagination patterns +- [Query reference](../api/queries.md): `Query`, `ProjectedQuery`, `Row`, `Rows` and every method on this page +- [Aggregations & grouped queries](aggregations.md): `count`, `sum`, `avg`, `min`, `max` and grouping +- [Mutations](mutations.md): creating, updating and deleting rows +- [Relationships](relationships.md): declaring foreign keys and reverse relations +- [Typed query predicates](../concepts/query-typing.md): how lambda predicates are type-checked +- [Raw SQL](raw-sql.md): for queries the query builder cannot express +- [Pagination](../howto/pagination.md): offset and keyset paging in an API diff --git a/docs/pages/guide/raw-sql.md b/docs/pages/guide/raw-sql.md index 992fea5d..bd4441b3 100644 --- a/docs/pages/guide/raw-sql.md +++ b/docs/pages/guide/raw-sql.md @@ -1,63 +1,114 @@ +--- +description: "Run SQL you write yourself with execute(), fetch_all() and fetch_one(), for statements a model query cannot express." +--- + # Raw SQL -Ferro exposes a raw SQL escape hatch — `execute`, `fetch_all`, `fetch_one` — for statements that don't fit a `Model`. +[`execute()`][ferro.execute], [`fetch_all()`][ferro.fetch_all] and [`fetch_one()`][ferro.fetch_one] run SQL you write yourself, for the statements a model query cannot express. -## When to Reach for Raw SQL +Every example on this page uses one table of events. The `Event` model creates it, and Ferro names a table after its model in lowercase, so the SQL says `event`: -Reach for raw SQL when the ORM can't express what you need: complex reports (window functions, CTEs, `HAVING`), Postgres GUCs (`set_config`, `SET LOCAL`), advisory locks, `LISTEN/NOTIFY`, database-side functions, or one-off maintenance statements. For everyday CRUD — and for plain [aggregations and grouped queries](aggregations.md), which are ORM territory now — prefer the ORM: it returns typed, validated instances; raw SQL returns plain dicts of primitives. +=== "Assignment" -!!! warning "Raw SQL is an escape hatch" - Bind values cross the FFI as wire-close primitives, and rows come back as `dict[str, str | int | float | bool | bytes | None]`. UUID, datetime, and JSON columns are returned as **strings**. If you want typed rows, use the ORM. + ```python + --8<-- "docs/examples/raw_sql.py:models" + ``` -## Executing Statements +=== "Annotated" + + ```python + --8<-- "docs/examples/raw_sql_annotated.py:models" + ``` -`execute(sql, *args)` runs a statement and returns the number of affected rows: +Here is a whole program. It connects, opens a session, adds two clicks and a signup, then counts the events by kind: ```python ---8<-- "docs/examples/raw_sql.py:execute" +--8<-- "docs/examples/raw_sql.py:first" +``` + +It prints: + + +```text +[{'kind': 'click', 'n': 2}, {'kind': 'signup', 'n': 1}] ``` -One statement per call — multi-statement strings are not supported. **Never** f-string user input into the `sql` argument; pass values as positional args so they are bound as parameters. +Each row is a plain `dict` keyed by column name. Every example on this page runs inside a session like this one; see [Sessions](connections.md#sessions) for why and how to open one. -## Fetching Rows +## Use cases -`fetch_all(sql, *args)` returns a list of dicts; `fetch_one(sql, *args)` returns the first row or `None` (add `LIMIT 1` when more rows could match): +Raw SQL fits what the ORM cannot express: reports with window functions, CTEs or `HAVING`, Postgres settings (`set_config`, `SET LOCAL`), advisory locks, `LISTEN`/`NOTIFY`, database functions and one-off maintenance statements. For everyday reads and writes, and for [aggregations and grouped queries](aggregations.md), which the ORM handles, use the ORM: it returns typed, validated model instances, where raw SQL returns dicts of primitives. + +!!! note "Rows hold primitives" + A raw row is a `dict[str, str | int | float | bool | bytes | None]`. UUID, datetime and JSON columns come back as strings. Load through the ORM when you want typed values. + +## Fetching rows + +`fetch_all(sql, *args)` returns every row as a list of dicts, as the first program showed. `fetch_one(sql, *args)` returns the first row, or `None` when no row matches: ```python ---8<-- "docs/examples/raw_sql.py:fetch" +--8<-- "docs/examples/raw_sql.py:fetch-one" ``` -All three functions accept `using="name"` to route to a [named connection](connections.md#named-connections). +The earliest event is the 09:00 click, and its `occurred_at` comes back as the string `"2026-03-02T09:00:00Z"`, not as a `datetime`. No event has the kind `purchase`, so the second call returns `None`. `fetch_one()` does not add `LIMIT 1` for you; write it when more than one row could match. + +## Running a statement with `execute()` + +`execute(sql, *args)` runs a statement and returns the number of rows it changed: + +```python +--8<-- "docs/examples/raw_sql.py:execute" +``` + +Both clicks matched, so `execute()` returns `2`. Pass one statement per call; a string holding several statements is not supported. Pass every value as an argument after the SQL, never formatted into the string: arguments are sent as bound parameters, so a value cannot change the statement. + +### Statements that refuse a transaction + +A few Postgres statements refuse to run inside a transaction block: `CREATE INDEX CONCURRENTLY`, `VACUUM`, and `ALTER TYPE ... ADD VALUE` before Postgres 12. In a session that carries settings, Ferro runs every operation inside a transaction, so these statements fail with error `25001`. Pass `autocommit=True` to run one outside it: + +```python {test=run prelude=raw_sql backend=postgres} +async with engines.session(settings={"myapp.tenant_id": "acme"}): + await execute( + "CREATE INDEX CONCURRENTLY idx_event_kind ON event (kind)", + autocommit=True, + ) +``` + +The index builds outside any transaction. An `autocommit` statement carries none of the session's settings, so a row-security policy that reads `current_setting(...)` sees nothing while it runs. Use it for maintenance statements, never for statements that read or write rows a policy protects. Inside an explicit `transaction()` block, `autocommit=True` changes nothing and Postgres rejects the statement. On a connection whose pool sets `settings_delivery="connection"`, the settings live on the connection itself, so an `autocommit` statement still sees them; see [Connection delivery](../concepts/row-level-security.md#connection-delivery). ## Placeholders -Placeholders are **native to the backend** — there is no translation layer. What you write is what the driver runs, and mismatches surface as the database's own error. +Ferro hands your SQL to the driver unchanged, so you write each backend's own placeholders, and a mismatch surfaces as the database's own error: === "SQLite" Positional `?` placeholders: - ```python - from ferro import fetch_all - - rows = await fetch_all("SELECT * FROM users WHERE role = ? AND age >= ?", "admin", 18) + ```python {test=run prelude=raw_sql} + rows = await fetch_all( + "SELECT id, kind FROM event WHERE kind = ? AND id >= ?", "click", 2 + ) + assert rows == [{"id": 3, "kind": "click"}] ``` === "PostgreSQL" - Numbered `$1, $2, ...` placeholders: - - ```python - from ferro import fetch_all + Numbered `$1`, `$2`, ... placeholders: - rows = await fetch_all("SELECT * FROM users WHERE role = $1 AND age >= $2", "admin", 18) + ```python {test=run prelude=raw_sql backend=postgres} + rows = await fetch_all( + "SELECT id, kind FROM event WHERE kind = $1 AND id >= $2", "click", 2 + ) + assert rows == [{"id": 3, "kind": "click"}] ``` -## Type Caveats +Both return the 09:30 click, the only click with an `id` of 2 or more. + +## Bind types and casts -Raw SQL has no schema map, so Ferro does not auto-cast bind values (matching asyncpg / psycopg behavior). Python values are marshalled to wire-close primitives: +Raw SQL has no schema map, so Ferro does not cast bind values for you, as asyncpg and psycopg do not. Each Python value is sent as the closest wire primitive: -| Python type | Sent as | Postgres cast you must write | +| Python type | Sent as | Postgres cast you write | | :--- | :--- | :--- | | `None` | `NULL` | — | | `bool` | bool | — | @@ -70,40 +121,50 @@ Raw SQL has no schema map, so Ferro does not auto-cast bind values (matching asy | `datetime.date` | ISO 8601 text | `$N::date` | | `datetime.time` | ISO 8601 text | `$N::time` | | `decimal.Decimal` | text | `$N::numeric` | -| `enum.Enum` | recursive on `.value` | depends on `.value` type | +| `enum.Enum` | its `.value`, by the same rules | depends on `.value` | | `dict` / `list` | `json.dumps(...)` text | `$N::jsonb` | | anything else | raises `TypeError` | — | -On PostgreSQL, write the casts in the SQL when the column type is stricter than text: +On PostgreSQL, write the cast in the SQL wherever the column type is stricter than text: -```python -from ferro import execute +```python {test=run prelude=raw_sql backend=postgres} +from datetime import UTC, datetime -sql = ( - "UPDATE events SET payload = $1::jsonb, occurred_at = $2::timestamptz " - "WHERE id = $3::uuid" +since = datetime(2026, 3, 2, 9, 5, tzinfo=UTC) +rows = await fetch_all( + "SELECT id, kind FROM event WHERE occurred_at >= $1::timestamptz ORDER BY id", + since, ) -await execute(sql, payload_dict, occurred_at, event_id) +assert rows == [{"id": 2, "kind": "signup"}, {"id": 3, "kind": "click"}] ``` -The same caveat applies on the way out: UUID, datetime, and JSON **result columns** come back as strings — parse them yourself, or load through the ORM for typed values. +`since` travels as ISO 8601 text and `::timestamptz` turns it back into a timestamp, so the signup and the later click come back. Without the cast, Postgres refuses the comparison with `operator does not exist: timestamp with time zone >= text`. Result columns work the same way in reverse: UUID, datetime and JSON values come back as strings, as `fetch_one()` showed. -## Raw SQL in Transactions +## Raw SQL in transactions -Inside an `async with transaction()` block, top-level `execute` / `fetch_all` / `fetch_one` automatically run on the transaction's connection. The yielded `Transaction` handle (`as tx`) offers the same three methods bound explicitly, which is the hard-to-misuse path: +Inside an `async with transaction()` block, the top-level `execute()`, `fetch_all()` and `fetch_one()` run on the transaction's connection. The handle the block yields (`as tx`) has the same three methods bound to that connection, which is harder to misuse: ```python --8<-- "docs/examples/raw_sql.py:in-transaction" ``` -Passing `using=...` for a *different* connection inside an active transaction raises — a transaction is pinned to one connection. +The delete and the read share one connection, so the read already sees the two `tap` rows gone and only the signup left. Passing `using=` for a different connection inside an active transaction raises `ValueError`, because a transaction is pinned to one connection. + +Outside a transaction, two consecutive calls may run on different pooled connections. Wrap a sequence that needs one connection, such as `SET LOCAL`, an advisory lock or `LISTEN`/`NOTIFY`, in `transaction()`. [Transactions](transactions.md) covers the handle and nesting. + +## Routing + +All three functions take `using="name"` to run on a [named connection](connections.md#named-connections), and `session=` to run in a session you hold instead of the current one. With neither, they run in the current session, and outside one they raise `RuntimeError`: + +```python +--8<-- "docs/examples/raw_sql.py:no-session" +``` -!!! tip "Connection affinity" - Outside a transaction, consecutive top-level calls may use **different pooled connections**. Wrap connection-affinity-sensitive sequences — `SET LOCAL`, advisory locks, `LISTEN/NOTIFY` — in `transaction()` so they share one connection. +The error names the fix: open a session with `async with engines.session():`, or pass `using=` or `session=`. The [Raw SQL reference](../api/raw-sql.md) lists every parameter of `execute()`, `fetch_all()` and `fetch_one()`. -## See Also +## See also -- [Transactions](transactions.md) — the `Transaction` handle and nesting behavior -- [Connections & Databases](connections.md) — named connections and routing -- [Queries](queries.md) — what the ORM can express without raw SQL -- [Raw SQL API reference](../api/raw-sql.md) — full signatures +- [Transactions](transactions.md): the `Transaction` handle and nesting +- [Connections and databases](connections.md): named connections and sessions +- [Queries](queries.md): what the ORM expresses without raw SQL +- [Raw SQL reference](../api/raw-sql.md): full signatures diff --git a/docs/pages/guide/relationships.md b/docs/pages/guide/relationships.md index cc533866..83ec617a 100644 --- a/docs/pages/guide/relationships.md +++ b/docs/pages/guide/relationships.md @@ -1,254 +1,594 @@ +--- +description: "Link models with foreign keys, back-references and many-to-many join tables, and load related rows." +--- + # Relationships -Ferro connects models with foreign keys, reverse lookups, and join tables the ORM creates. +This guide shows how to link models with foreign keys: a player belongs to a team, a team lists its players, and players enter tournaments through a join table Ferro creates. + +Here is a whole program. It creates the tables, adds a team and a player, then reads the relation in both directions: + +```python +--8<-- "docs/examples/relationships.py:first" +``` + +It prints: + + +```text +Rustaceans ['Ferris'] +``` + +`await ferris.team` runs one query and returns the `Team`. `rustaceans.players` is a query that runs when you call a terminal such as `.all()`. Every example on this page runs inside the session this program opens; see [Sessions](connections.md#sessions) for why and how to open one. -## Overview +Every example on this page is followed by the SQL it sends, captured from Ferro when the example runs. The two creates and the two reads send these statements. PostgreSQL also asks each single-row `INSERT` for the new `id` with `RETURNING`: -Relationships are **lazy** — nothing is fetched until you ask for it: +=== "SQLite" -- **Forward relations** (a `ForeignKey` field): `await post.author` performs one query and returns the related instance. A query can also deliver the relation already **populated** — `Post.select().include(lambda p: p.author)` makes `post.author` a plain attribute, no await, no extra query (see [Populated relations](#populated-relations)). -- **Reverse relations** (a `BackRef` field): `author.posts` is a chainable query — filter, order, and slice it before awaiting a terminal. + ```sql + -- relationships.py:first (sqlite) + INSERT INTO "team" ("name") + VALUES (?); -A forward `ForeignKey(related_name="x")` always pairs with a reverse field named `x` on the target model. The pairing is **required and checked at `connect()`** — a `ForeignKey` whose `related_name` has no matching `BackRef()` on the target raises at connect time. + INSERT INTO "player" ("name", "team_id", "mentor_id") + VALUES (?, ?, ?); + + SELECT "team"."id", "team"."name" + FROM "team" + WHERE "team"."id" = ? + LIMIT ?; + + SELECT "player"."id", "player"."mentor_id", "player"."name", "player"."team_id" + FROM "player" + WHERE "player"."team_id" = ?; + ``` + +=== "PostgreSQL" + + ```sql + -- relationships.py:first (postgres) + INSERT INTO "team" ("name") + VALUES ($1) + RETURNING "id"; + + INSERT INTO "player" ("name", "team_id", "mentor_id") + VALUES ($1, $2, $3) + RETURNING "id"; + + SELECT "team"."id", "team"."name" + FROM "team" + WHERE "team"."id" = $1 + LIMIT $2; + + SELECT "player"."id", "player"."mentor_id", "player"."name", "player"."team_id" + FROM "player" + WHERE "player"."team_id" = $1; + ``` -## One-to-Many +`await ferris.team` is a `SELECT` on `team` by primary key; `rustaceans.players.all()` is a `SELECT` on `player` filtered by `team_id`. -The most common shape: a `ForeignKey` on the "child" model, declared as `Annotated` metadata, plus a `Relation[list[...]] = BackRef()` on the "parent": +## The roster + +Every example on this page uses one roster. A team has players, a player may have a profile and a mentor, players enter tournaments, and a tournament may have a winning team: === "Assignment" ```python - --8<-- "docs/examples/relationships.py:one-to-many" + --8<-- "docs/examples/relationships.py:models" ``` === "Annotated" ```python - --8<-- "docs/examples/relationships_annotated.py:one-to-many" + --8<-- "docs/examples/relationships_annotated.py:models" ``` +Each relation has two sides. The forward side is a [`ForeignKey`][ferro.ForeignKey] field on the model whose table holds the key column. The reverse side is a [`BackRef()`][ferro.BackRef] on the other model, named by the forward side's `related_name`. Ferro checks the pairing at `connect()`: a `related_name` with no matching `BackRef()` raises `RuntimeError` naming both models. + +`create_tables()` returns a report of every statement it ran. The sections below print parts of it, so you can see the tables each relation creates. + +## One-to-many + +`Player.team` is a `ForeignKey` to `Team`, and `Team.players` is its reverse side, a `Relation[list["Player"]]`. Add two more players, then read one team's roster in name order: + ```python ---8<-- "docs/examples/relationships.py:one-to-many-usage" +--8<-- "docs/examples/relationships.py:one-to-many" ``` -### Shadow FK columns +Gordon plays for the Gophers, so the Rustaceans' roster is Corro and Ferris. `rustaceans.players` is a query, so you can filter, order and page it before you run it. -For every `ForeignKey` field (e.g. `team`), Ferro creates a shadow scalar column and matching Pydantic field named `{field}_id` (e.g. `team_id`) holding the related row's primary key. Its Python type follows the target model's primary-key annotation. Read it or filter on it like any other column — `Player.where(lambda t: t.team_id == team.id)` — with no extra query. +The three creates and the roster query send these statements. PostgreSQL also asks each single-row `INSERT` for the new `id` with `RETURNING`: -To filter or order by a column on the *related* model, a query lambda can traverse the relation itself (`Player.where(lambda player: player.team.name == "Rustaceans")`) — see [Querying Across Relationships](queries.md#querying-across-relationships). +=== "SQLite" -### Populated relations + ```sql + -- relationships.py:one-to-many (sqlite) + INSERT INTO "player" ("name", "team_id", "mentor_id") + VALUES (?, ?, ?); + + INSERT INTO "team" ("name") + VALUES (?); + + INSERT INTO "player" ("name", "team_id", "mentor_id") + VALUES (?, ?, ?); + + SELECT "player"."id", "player"."mentor_id", "player"."name", "player"."team_id" + FROM "player" + WHERE "player"."team_id" = ? + ORDER BY "player"."name" ASC NULLS LAST; + ``` + +=== "PostgreSQL" + + ```sql + -- relationships.py:one-to-many (postgres) + INSERT INTO "player" ("name", "team_id", "mentor_id") + VALUES ($1, $2, $3) + RETURNING "id"; + + INSERT INTO "team" ("name") + VALUES ($1) + RETURNING "id"; + + INSERT INTO "player" ("name", "team_id", "mentor_id") + VALUES ($1, $2, $3) + RETURNING "id"; + + SELECT "player"."id", "player"."mentor_id", "player"."name", "player"."team_id" + FROM "player" + WHERE "player"."team_id" = $1 + ORDER BY "player"."name" ASC NULLS LAST; + ``` + +The reverse relation is a filter on the shadow column, `team_id`. -Awaiting a forward relation in a loop is the classic N+1 pattern — each access is a query. When a query knows it will read a relation, ask for it up front with `include()` and the relation arrives **populated**: a plain attribute holding the complete related instance, fetched in the same single statement as the roots: +### Shadow foreign-key columns + +For each `ForeignKey` field, Ferro adds a column and a model field named after it with `_id` (`team_id` for `team`), holding the related row's primary key. Its Python type follows the target model's primary key. Read it or filter on it without loading the team: ```python ---8<-- "docs/examples/populated_relations.py:basic" +--8<-- "docs/examples/relationships.py:shadow-column" ``` -A relation is populated only on instances an including query delivered (or refreshed); everywhere else the awaitable form above keeps working, unchanged. Nullable FKs populate truthfully — `transaction.account` is `None` (not a coroutine) when the row has no account. The full semantics — membership preservation, multi-hop paths, identity-map behavior, and the refresh rule — live in [Populating Relations with include()](queries.md#populating-relations-with-include). +`ferris.team_id` is already on the instance, and the count runs one query with no join. It sends this; on PostgreSQL only the placeholders differ: -`include()` covers **forward** foreign keys only. Populating a `BackRef` collection or an M2M set is a separate future mechanism; until it lands, reverse relations stay chainable queries (`await author.posts.all()`). +```sql +-- relationships.py:shadow-column (sqlite) +SELECT COUNT(*) +FROM "player" +WHERE "player"."team_id" = ?; +``` -## One-to-One +The `player` table shows the shadow columns and their constraints: -Add `unique=True` to the `ForeignKey` to enforce at most one child per parent. The reverse side is a plain (non-list) `BackRef` that resolves to a single instance: +```python +--8<-- "docs/examples/relationships.py:player-ddl" +``` -=== "Assignment" +It prints these statements, laid out one column per line. PostgreSQL declares `id` as `serial` and runs the statements under a lock timeout: - ```python - --8<-- "docs/examples/relationships.py:one-to-one" +=== "SQLite" + + ```sql + -- relationships.py:player-ddl (sqlite) + CREATE TABLE IF NOT EXISTS "player" ( + "id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, + "mentor_id" integer, + "name" varchar NOT NULL, + "team_id" integer NOT NULL, + CONSTRAINT "fk_player_mentor_id_player" FOREIGN KEY ("mentor_id") REFERENCES "player" ("id") ON DELETE SET NULL, + CONSTRAINT "fk_player_team_id_team" FOREIGN KEY ("team_id") REFERENCES "team" ("id") ON DELETE CASCADE + ); + + CREATE INDEX IF NOT EXISTS "idx_player_team_id" ON "player" ("team_id"); ``` -=== "Annotated" +=== "PostgreSQL" - ```python - --8<-- "docs/examples/relationships_annotated.py:one-to-one" + ```sql + -- relationships.py:player-ddl (postgres) + SET LOCAL lock_timeout = '5000ms'; + + CREATE TABLE IF NOT EXISTS "player" ( + "id" serial PRIMARY KEY NOT NULL, + "mentor_id" integer, + "name" varchar NOT NULL, + "team_id" integer NOT NULL, + CONSTRAINT "fk_player_mentor_id_player" FOREIGN KEY ("mentor_id") REFERENCES "player" ("id") ON DELETE SET NULL, + CONSTRAINT "fk_player_team_id_team" FOREIGN KEY ("team_id") REFERENCES "team" ("id") ON DELETE CASCADE + ); + + CREATE INDEX IF NOT EXISTS "idx_player_team_id" ON "player" ("team_id"); ``` +Each foreign key becomes a constraint named `fk_
__`. `idx_player_team_id` comes from `index=True`; see [Indexing foreign keys](#indexing-foreign-keys). + +To filter or order by a column of the related model, a lambda can follow the relation, as in `Player.where(lambda player: player.team.name == "Rustaceans")`. [Querying across relationships](queries.md#querying-across-relationships) covers it. + +### Populated relations + +Awaiting a forward relation in a loop runs one query per row. When a query will read a relation, ask for it with `include()`, and the related rows arrive in the same statement: + ```python ---8<-- "docs/examples/relationships.py:one-to-one-usage" +--8<-- "docs/examples/relationships.py:populated" ``` -`unique=True` implies an index on the shadow column (combining it with `index=True` is redundant and emits a `UserWarning`). +It prints: -## Many-to-Many + +```text +Corro | Rustaceans +Ferris | Rustaceans +Gordon | Gophers +``` -Declare `ManyToMany(related_name=...)` on one side and a `BackRef()` collection on the other. Ferro synthesizes the join table automatically: +`player.team` is now the `Team` instance itself: reading it needs no `await` and runs no query. `include()` covers forward foreign keys only; load a `BackRef` or many-to-many collection with `await rustaceans.players.all()`. [Populating relations with `include()`](queries.md#populating-relations-with-include) covers nullable relations, multi-hop paths and how long a relation stays populated. -=== "Assignment" +The query sends this; on PostgreSQL only the placeholders differ: - ```python - --8<-- "docs/examples/relationships.py:many-to-many" +```sql +-- relationships.py:populated (sqlite) +SELECT "player"."id", "player"."mentor_id", "player"."name", "player"."team_id", "j1_team"."id" AS "j1_team__id", "j1_team"."name" AS "j1_team__name" +FROM "player" +LEFT JOIN "team" AS "j1_team" ON "player"."team_id" = "j1_team"."id" +WHERE TRUE +ORDER BY "player"."name" ASC NULLS LAST; +``` + +`include()` added a `LEFT JOIN` on `team`, whose columns come back under `j1_team__` names. + +The [`ForeignKey`][ferro.ForeignKey], [`BackRef`][ferro.BackRef] and [`Relation`][ferro.Relation] reference lists every parameter. + +## One-to-one + +Add `unique=True` to a `ForeignKey` to allow at most one child per parent. `Profile.player` is unique, so its reverse side `Player.profile` is a `BackRef()` annotated with the single model, `"Profile"`, and awaiting it returns one instance or `None`: + +```python +--8<-- "docs/examples/relationships.py:one-to-one" +``` + +Ferris's profile comes back as one `Profile`. Corro has none, so `await corro.profile` returns `None`. Ferro backs `unique=True` with a unique index on the shadow column, `uq_profile_player_id`, so a second profile for Ferris would fail. That index also serves lookups, so `index=True` next to `unique=True` is redundant, and Ferro warns with a `UserWarning` and ignores it. + +The create and the two reads send these statements. PostgreSQL also asks each single-row `INSERT` for the new `id` with `RETURNING`: + +=== "SQLite" + + ```sql + -- relationships.py:one-to-one (sqlite) + INSERT INTO "profile" ("bio", "player_id") + VALUES (?, ?); + + SELECT "profile"."bio", "profile"."id", "profile"."player_id" + FROM "profile" + WHERE "profile"."player_id" = ? + LIMIT ?; + + SELECT "profile"."bio", "profile"."id", "profile"."player_id" + FROM "profile" + WHERE "profile"."player_id" = ? + LIMIT ?; ``` -=== "Annotated" +=== "PostgreSQL" - ```python - --8<-- "docs/examples/relationships_annotated.py:many-to-many" + ```sql + -- relationships.py:one-to-one (postgres) + INSERT INTO "profile" ("bio", "player_id") + VALUES ($1, $2) + RETURNING "id"; + + SELECT "profile"."bio", "profile"."id", "profile"."player_id" + FROM "profile" + WHERE "profile"."player_id" = $1 + LIMIT $2; + + SELECT "profile"."bio", "profile"."id", "profile"."player_id" + FROM "profile" + WHERE "profile"."player_id" = $1 + LIMIT $2; ``` +Each read of the one-to-one side is a `SELECT` on `profile` by `player_id` with a `LIMIT` of 1. + +## Many-to-many + +Declare [`ManyToMany(related_name=...)`][ferro.ManyToMany] on one side and a `BackRef()` collection on the other. `Player.tournaments` and `Tournament.players` are the two sides. Enter players in tournaments, then read both directions: + ```python ---8<-- "docs/examples/relationships.py:m2m-usage" +--8<-- "docs/examples/relationships.py:many-to-many" ``` -Both sides expose the chainable query API plus the link mutators `add(*instances)`, `remove(*instances)`, and `clear()`. +Ferris entered both tournaments, and the Spring Open has Ferris and Gordon. The creates, the links and the two reads send these statements. PostgreSQL also asks each single-row `INSERT` for the new `id` with `RETURNING`: -`ManyToMany` accepts: +=== "SQLite" -| Parameter | Default | Description | -| :--- | :--- | :--- | -| `related_name` | required | Name of the reverse field on the related model. | -| `through` | `None` | Explicit join-table name; auto-generated when omitted. | -| `reverse_index` | `True` | Add a non-unique composite index on `(target_col, source_col)` in the join table so reverse queries also hit an index. Pass `False` to opt out on write-heavy join tables. | + ```sql + -- relationships.py:many-to-many (sqlite) + INSERT INTO "tournament" ("name", "winner_id") + VALUES (?, ?); -The default join table gets a **composite unique** on its two foreign-key columns, so the same link can never be stored twice. `reverse_index` lives on the forward `ManyToMany(...)` declaration — passing it to `BackRef()` raises `TypeError`. + INSERT INTO "tournament" ("name", "winner_id") + VALUES (?, ?); -## Self-Referential + INSERT INTO "player_tournaments" ("player_id", "tournament_id") + VALUES (?, ?), (?, ?); -A model can reference itself — org charts, threaded comments, category trees. The forward reference must be the **quoted class name**, and because a string annotation cannot carry `| None`, nullability must be declared explicitly with `nullable=True`: + INSERT INTO "player_tournaments" ("player_id", "tournament_id") + VALUES (?, ?); -=== "Assignment" + SELECT "tournament"."id", "tournament"."name", "tournament"."winner_id" + FROM "tournament" + INNER JOIN "player_tournaments" ON "tournament"."id" = "player_tournaments"."tournament_id" + WHERE "player_tournaments"."player_id" = ? + ORDER BY "tournament"."name" ASC NULLS LAST; - ```python - --8<-- "docs/examples/relationships.py:self-referential" + SELECT "player"."id", "player"."mentor_id", "player"."name", "player"."team_id" + FROM "player" + INNER JOIN "player_tournaments" ON "player"."id" = "player_tournaments"."player_id" + WHERE "player_tournaments"."tournament_id" = ? + ORDER BY "player"."name" ASC NULLS LAST; ``` -=== "Annotated" +=== "PostgreSQL" - ```python - --8<-- "docs/examples/relationships_annotated.py:self-referential" + ```sql + -- relationships.py:many-to-many (postgres) + INSERT INTO "tournament" ("name", "winner_id") + VALUES ($1, $2) + RETURNING "id"; + + INSERT INTO "tournament" ("name", "winner_id") + VALUES ($1, $2) + RETURNING "id"; + + INSERT INTO "player_tournaments" ("player_id", "tournament_id") + VALUES ($1, $2), ($3, $4); + + INSERT INTO "player_tournaments" ("player_id", "tournament_id") + VALUES ($1, $2); + + SELECT "tournament"."id", "tournament"."name", "tournament"."winner_id" + FROM "tournament" + INNER JOIN "player_tournaments" ON "tournament"."id" = "player_tournaments"."tournament_id" + WHERE "player_tournaments"."player_id" = $1 + ORDER BY "tournament"."name" ASC NULLS LAST; + + SELECT "player"."id", "player"."mentor_id", "player"."name", "player"."team_id" + FROM "player" + INNER JOIN "player_tournaments" ON "player"."id" = "player_tournaments"."player_id" + WHERE "player_tournaments"."tournament_id" = $1 + ORDER BY "player"."name" ASC NULLS LAST; ``` +Each `add()` is one `INSERT` into the join table, and each side reads through it with an `INNER JOIN`. Both sides are queries with three link methods, `add(*instances)`, `remove(*instances)` and `clear()`: + ```python ---8<-- "docs/examples/relationships.py:self-referential-usage" +--8<-- "docs/examples/relationships.py:m2m-remove" ``` -Without `nullable=True` the root of the tree (an `Employee` with no manager) could never be stored. +`remove()` drops one of Ferris's links and `clear()` drops the rest. The tournaments themselves stay, and so does Gordon's entry. `add()` and `remove()` read each instance's `id` attribute, so they work only when the related model's primary key is named `id`. -## Nullable Relationships +The calls send these statements; on PostgreSQL only the placeholders differ: -For an optional relation to *another* model, put the union **inside** `Annotated` and default the field to `None`: +```sql +-- relationships.py:m2m-remove (sqlite) +DELETE +FROM "player_tournaments" +WHERE "player_id" = ? AND "tournament_id" IN (?); -=== "Assignment" +SELECT "tournament"."id", "tournament"."name", "tournament"."winner_id" +FROM "tournament" +INNER JOIN "player_tournaments" ON "tournament"."id" = "player_tournaments"."tournament_id" +WHERE "player_tournaments"."player_id" = ?; - ```python - from typing import Annotated +DELETE +FROM "player_tournaments" +WHERE "player_id" = ?; - from ferro import BackRef, Field, ForeignKey, Model, Relation +SELECT COUNT(*) +FROM "tournament" +INNER JOIN "player_tournaments" ON "tournament"."id" = "player_tournaments"."tournament_id" +WHERE "player_tournaments"."player_id" = ?; +SELECT COUNT(*) +FROM "player" +INNER JOIN "player_tournaments" ON "player"."id" = "player_tournaments"."player_id" +WHERE "player_tournaments"."tournament_id" = ?; +``` - class Category(Model): - id: int | None = Field(default=None, primary_key=True) - name: str - products: Relation[list["Product"]] = BackRef() +`remove()` and `clear()` delete rows of the join table only. +### The join table - class Product(Model): - id: int | None = Field(default=None, primary_key=True) - name: str - category: Annotated[Category | None, ForeignKey(related_name="products")] = None - ``` +Ferro creates the join table itself. It names the table after the side that declares `ManyToMany`, as `
_`, here `player_tournaments`, with one `
_id` column for each side. Print its statements from the `create_tables()` report: -=== "Annotated" +```python +--8<-- "docs/examples/relationships.py:join-table-ddl" +``` - ```python - from typing import Annotated +It prints these statements, laid out one column per line. PostgreSQL runs them under a lock timeout: - from ferro import BackRef, Field, ForeignKey, Model, Relation +=== "SQLite" + ```sql + -- relationships.py:join-table-ddl (sqlite) + CREATE TABLE IF NOT EXISTS "player_tournaments" ( + "player_id" integer NOT NULL, + "tournament_id" integer NOT NULL, + CONSTRAINT "fk_player_tournaments_player_id_player" FOREIGN KEY ("player_id") REFERENCES "player" ("id") ON DELETE CASCADE, + CONSTRAINT "fk_player_tournaments_tournament_id_tournament" FOREIGN KEY ("tournament_id") REFERENCES "tournament" ("id") ON DELETE CASCADE + ); - class Category(Model): - id: Annotated[int | None, Field(default=None, primary_key=True)] - name: str - products: Relation[list["Product"]] = BackRef() + CREATE INDEX IF NOT EXISTS "idx_player_tournaments_tournament_id_player_id" ON "player_tournaments" ("tournament_id", "player_id"); + CREATE UNIQUE INDEX IF NOT EXISTS "uq_player_tournaments_player_id_tournament_id" ON "player_tournaments" ("player_id", "tournament_id"); + ``` - class Product(Model): - id: Annotated[int | None, Field(default=None, primary_key=True)] - name: str - category: Annotated[Category | None, ForeignKey(related_name="products")] = None +=== "PostgreSQL" + + ```sql + -- relationships.py:join-table-ddl (postgres) + SET LOCAL lock_timeout = '5000ms'; + + CREATE TABLE IF NOT EXISTS "player_tournaments" ( + "player_id" integer NOT NULL, + "tournament_id" integer NOT NULL, + CONSTRAINT "fk_player_tournaments_player_id_player" FOREIGN KEY ("player_id") REFERENCES "player" ("id") ON DELETE CASCADE, + CONSTRAINT "fk_player_tournaments_tournament_id_tournament" FOREIGN KEY ("tournament_id") REFERENCES "tournament" ("id") ON DELETE CASCADE + ); + + CREATE INDEX IF NOT EXISTS "idx_player_tournaments_tournament_id_player_id" ON "player_tournaments" ("tournament_id", "player_id"); + + CREATE UNIQUE INDEX IF NOT EXISTS "uq_player_tournaments_player_id_tournament_id" ON "player_tournaments" ("player_id", "tournament_id"); ``` -!!! warning "Union placement matters" - Write `Annotated[Category | None, ForeignKey(...)] = None` — the union goes **inside** `Annotated`. The form `Annotated[Category, ForeignKey(...)] | None` is not supported and will not produce a nullable foreign key. +Both link columns cascade, so deleting a player or a tournament deletes its links. The unique index stores each link once, and the index on `(tournament_id, player_id)` serves lookups from the tournament side. -With the default `nullable="infer"`, Ferro derives column nullability from whether the relation annotation allows `None`. `on_delete="SET NULL"` also implies a nullable column (and explicitly combining it with `nullable=False` raises). +`ManyToMany` takes three parameters: -## Delete Behavior +| Parameter | Default | Description | +| :--- | :--- | :--- | +| `related_name` | required | Name of the `BackRef()` field on the related model. | +| `through` | `None` | Join-table name; `
_` when omitted. | +| `reverse_index` | `True` | Adds the index on `(target_col, source_col)`, so lookups from the other side use an index. Pass `False` to skip it on a write-heavy join table. | -`ForeignKey(on_delete=...)` controls what happens to child rows when their parent is deleted: +`reverse_index` belongs on `ManyToMany(...)`; passing it to `BackRef()` raises `TypeError`. The [`ManyToMany` reference][ferro.ManyToMany] lists every parameter. -| `on_delete` | Effect when the parent row is deleted | -| :--- | :--- | -| `"CASCADE"` (default) | Child rows are deleted too. | -| `"RESTRICT"` | Deletion fails while child rows exist. | -| `"SET NULL"` | The shadow FK column is set to `NULL` (requires a nullable relation). | -| `"SET DEFAULT"` | The shadow FK column is reset to its column default. | -| `"NO ACTION"` | The constraint is not enforced at delete time (backend semantics apply). | +## Self-referential relations -=== "Assignment" +A model can point at itself, for mentors, org charts, threads or category trees. `Player.mentor` is a `ForeignKey` to `Player`. A class cannot name itself in its own annotations, so the target is the string `"Player"`, and because that string cannot carry `| None`, `nullable=True` makes the column optional: - ```python - --8<-- "docs/examples/relationships.py:on-delete" +```python +--8<-- "docs/examples/relationships.py:self-referential" +``` + +Crabby's mentor is Ferris, and Ferris's mentees are Crabby. Ferris has no mentor, which the `mentor_id` column allows only because of `nullable=True`. Writing `"Player | None"` instead fails at `connect()` with `RuntimeError: Relationship resolution failed: 'Player | None' not found`. + +The create and the two reads send these statements. PostgreSQL also asks each single-row `INSERT` for the new `id` with `RETURNING`: + +=== "SQLite" + + ```sql + -- relationships.py:self-referential (sqlite) + INSERT INTO "player" ("name", "team_id", "mentor_id") + VALUES (?, ?, ?); + + SELECT "player"."id", "player"."mentor_id", "player"."name", "player"."team_id" + FROM "player" + WHERE "player"."id" = ? + LIMIT ?; + + SELECT "player"."id", "player"."mentor_id", "player"."name", "player"."team_id" + FROM "player" + WHERE "player"."mentor_id" = ?; ``` -=== "Annotated" +=== "PostgreSQL" - ```python - --8<-- "docs/examples/relationships_annotated.py:on-delete" + ```sql + -- relationships.py:self-referential (postgres) + INSERT INTO "player" ("name", "team_id", "mentor_id") + VALUES ($1, $2, $3) + RETURNING "id"; + + SELECT "player"."id", "player"."mentor_id", "player"."name", "player"."team_id" + FROM "player" + WHERE "player"."id" = $1 + LIMIT $2; + + SELECT "player"."id", "player"."mentor_id", "player"."name", "player"."team_id" + FROM "player" + WHERE "player"."mentor_id" = $1; ``` -Because `CASCADE` is the default, deleting a parent silently removes its children unless you choose otherwise — pick `RESTRICT` when orphan deletion would be a bug you want surfaced. +`await crabby.mentor` reads `player` by primary key, and `ferris.mentees` filters `player` on `mentor_id`. -## Indexing Foreign Keys +## Nullable relations -PostgreSQL does not automatically index foreign-key columns. For FKs that appear in hot query paths (tenant IDs on every list endpoint, for instance), request a non-unique index on the shadow column with `index=True`: +For an optional relation to another model, put `| None` inside `Annotated` and default the field to `None`. `Tournament.winner` is `Annotated[Team | None, ForeignKey(...)] = None`, so a tournament can exist before anyone wins it: -=== "Assignment" +```python +--8<-- "docs/examples/relationships.py:nullable" +``` - ```python - from typing import Annotated +The Autumn Cup has no winner, so its `winner_id` is `None`. Setting `winner_id` and saving records the Rustaceans' Spring Open win, and `rustaceans.titles` finds it. A relation field cannot be assigned on an instance (`spring.winner = rustaceans` raises `AttributeError`), so set the shadow column. - from ferro import BackRef, Field, ForeignKey, Model, Relation +The save and the query send these statements; on PostgreSQL only the placeholders differ: +```sql +-- relationships.py:nullable (sqlite) +UPDATE "tournament" +SET "name" = ?, "winner_id" = ? +WHERE "id" = ?; - class Org(Model): - id: int | None = Field(default=None, primary_key=True) - name: str - projects: Relation[list["Project"]] = BackRef() +SELECT "tournament"."id", "tournament"."name", "tournament"."winner_id" +FROM "tournament" +WHERE "tournament"."winner_id" = ?; +``` +`save()` wrote `winner_id` with the rest of the row, and `rustaceans.titles` filters on it. - class Project(Model): - id: int | None = Field(default=None, primary_key=True) - name: str - org: Annotated[Org, ForeignKey(related_name="projects", index=True)] - ``` +With the default `nullable="infer"`, the column's nullability follows the annotation. `on_delete="SET NULL"` needs a nullable column, so combining it with `nullable=False` raises `ValueError`. -=== "Annotated" +!!! note "Put `| None` inside `Annotated`" + Write `Annotated[Team | None, ForeignKey(...)] = None`. In the form `Annotated[Team, ForeignKey(...)] | None`, Ferro does not see the `ForeignKey`: it adds no `winner_id` column and no constraint, and raises no error. - ```python - from typing import Annotated +## Delete behavior - from ferro import BackRef, Field, ForeignKey, Model, Relation +`ForeignKey(on_delete=...)` decides what happens to child rows when their parent row is deleted: +| `on_delete` | When the parent row is deleted | +| :--- | :--- | +| `"CASCADE"` (default) | The child rows are deleted too. | +| `"RESTRICT"` | The delete fails with `ForeignKeyViolationError` while child rows exist. | +| `"SET NULL"` | The shadow column is set to `NULL`; the relation must be nullable. | +| `"SET DEFAULT"` | The shadow column is reset to its column default. | +| `"NO ACTION"` | Like `RESTRICT`, the delete fails while child rows exist; the database checks at the end of the statement rather than at once. | - class Org(Model): - id: Annotated[int | None, Field(default=None, primary_key=True)] - name: str - projects: Relation[list["Project"]] = BackRef() +Delete the Rustaceans, and each rule in the roster runs: +```python +--8<-- "docs/examples/relationships.py:on-delete" +``` - class Project(Model): - id: Annotated[int | None, Field(default=None, primary_key=True)] - name: str - org: Annotated[Org, ForeignKey(related_name="projects", index=True)] - ``` +Corro, Ferris and Crabby go with their team (`CASCADE` on `Player.team`), and Ferris's profile goes with Ferris (`CASCADE` on `Profile.player`). The Spring Open stays, with no winner (`SET NULL` on `Tournament.winner`). Gordon plays for the Gophers and stays. Had only Ferris been deleted, `SET NULL` on `Player.mentor` would have kept Crabby, with no mentor. + +The delete and the checks send these statements; on PostgreSQL only the placeholders differ: + +```sql +-- relationships.py:on-delete (sqlite) +DELETE +FROM "team" +WHERE "team"."id" = ?; + +SELECT "player"."id", "player"."mentor_id", "player"."name", "player"."team_id" +FROM "player" +WHERE TRUE; + +SELECT COUNT(*) +FROM "profile" +WHERE TRUE; + +SELECT "tournament"."id", "tournament"."name", "tournament"."winner_id" +FROM "tournament" +WHERE "tournament"."id" = ? +LIMIT ?; +``` + +Ferro sends one `DELETE`; the database applies each `on_delete` rule itself. + +Because `CASCADE` is the default, deleting a parent removes its children unless you choose otherwise. Pick `RESTRICT` when deleting a parent that still has children would be a bug you want to surface. + +## Indexing foreign keys + +PostgreSQL does not index foreign-key columns on its own. For a foreign key on a busy query path, such as a tenant id that every list filters on, pass `index=True`. `Player.team` does, as `ForeignKey(related_name="players", index=True)`, and that is where `idx_player_team_id` in the [`player` table](#shadow-foreign-key-columns) comes from. -One-to-one relations (`unique=True`) already get an index; for multi-column indexes that start with the FK column, use [composite indexes](models-and-fields.md#composite-indexes). +A one-to-one relation already has its unique index. For an index over several columns that starts with the foreign key, use [composite indexes](models-and-fields.md#composite-indexes). -## See Also +## See also -- [Models & Fields](models-and-fields.md) — field declaration styles and constraints -- [Queries](queries.md) — filtering on shadow FK columns, traversing relations, and reverse relations -- [Mutations](mutations.md) — creating related records, cascade implications -- [Schema Management](schema/overview.md) — how foreign keys and join tables reach the database, through auto-migrate, [migrations](schema/migrations.md) or the [Alembic bridge](schema/alembic.md) +- [Relationships reference](../api/relationships.md): `ForeignKey`, `BackRef`, `ManyToMany` and `Relation` +- [Models and fields](models-and-fields.md): field declaration styles and constraints +- [Queries](queries.md): filtering on shadow columns, traversing relations and `include()` +- [Mutations](mutations.md): creating related rows and what a delete cascades to +- [Schema management](schema/overview.md): how foreign keys and join tables reach the database, through auto-migrate, [migrations](schema/migrations.md) or the [Alembic bridge](schema/alembic.md) diff --git a/docs/pages/guide/row-level-security.md b/docs/pages/guide/row-level-security.md index 31de5e69..948cadd8 100644 --- a/docs/pages/guide/row-level-security.md +++ b/docs/pages/guide/row-level-security.md @@ -1,112 +1,24 @@ -# Using with Postgres RLS +--- +description: "Set up Postgres row-level security so each request reads and writes only its own tenant's rows." +--- -PostgreSQL's row-level security (RLS) lets the database itself refuse to -return or write rows a connection isn't scoped to see — the boundary that -would otherwise live entirely in application discipline (a `where` clause -nobody forgets, forever). Ferro ships two features as one pair to make that -boundary real: +# Scope rows by tenant with Postgres row-level security -- **Session settings** — a `Session` carries Postgres settings (GUCs) and - delivers them to every connection its operations touch. -- **Row security declarations** — a model declares its policies once - (`__ferro_rls__`), and Ferro creates and reconciles them like any other - schema artifact. +This page sets up Postgres row-level security so that each request reads and writes only its own tenant's rows, with no `where` filter in your code. -Read this whole page before you turn `FORCE ROW LEVEL SECURITY` on in -production — the first two sections cover the two ways this feature bites -teams that skip them. +Once it is in place, a session that carries a tenant id sees that tenant's invoices and no others: -## The pooler constraint +```python +--8<-- "docs/examples/row_level_security.py:scoped" +``` -A GUC set with plain `SET` lives on the **server-side** Postgres connection. -Behind a connection pooler running in *transaction* mode — PgBouncer's default -`pool_mode`, and what most managed Postgres front doors use — that server -connection is handed to a different client between transactions. Two of your -requests can share one server backend seconds apart: +`ACME` and `GLOBEX` are the two tenants' ids, and each tenant has two invoices. The query has no `where` clause: Postgres applied the table's policy to every row, and Globex's invoices never left the database. -``` -tenant A → pgbouncer → server conn 7: SET app.tenant_id = 'A' -tenant B → pgbouncer → server conn 7: SELECT ... FROM invoice - -- sees tenant A's policy scope -``` +The steps below build that example: declare the policy on the model, give the application a role the policy applies to, open each session with the tenant's id, and deploy the two halves in the right order. The Python examples, apart from the web framework recipes, run against Postgres in the docs test suite. Row-level security is a Postgres feature, and on SQLite Ferro creates the tables without policies and warns (see [SQLite](../reference/row-level-security.md#sqlite)). -`SET` there isn't a correctness gap — it's a cross-tenant data leak, and -because the pooler is invisible to its clients (`SHOW pool_mode` needs -superuser access to the admin database), Ferro cannot detect it and refuse. -So Ferro doesn't guess: - -**Default delivery is `transaction`**: every setting is applied with -parameter-bound `set_config($1, $2, true)` — the `true` is Postgres' `is_local` -flag, so the value dies at `COMMIT` and can never survive onto a connection's -next user. It rides `BEGIN` for every explicit `transaction()` block, and a -plain operation outside one (`Model.where(...).all()`, `create()`, a raw -query) wraps itself in an implicit transaction to carry it — see -[Session settings](#session-settings) below. This is safe on direct Postgres, -PgBouncer transaction mode, RDS Proxy, and the Supabase pooler alike. - -**`connection` delivery is opt-in and never automatic** — -`connect(url, pool=PoolConfig(settings_delivery="connection"))`. It pins one -pool connection per settings-bearing session and applies values once, for the -whole session, with the same parameter-bound `set_config($1, $2, false)` — -never a plain `SET`, never interpolated SQL, just `is_local=false` instead of -`true` so the value survives past the transaction that set it. That is **an -explicit operator promise that this pool talks to Postgres directly.** -Reaching for it against a transaction-mode pooler is the exact leak the -paragraph above describes — see -[Connection delivery mode](#connection-delivery-mode) for the full trade-off -before opting in. - -## Deploy ordering - -Ship the pair in this order: - -1. **Settings delivery first.** Get your app opening a settings-bearing - session around every request (see [Framework integration](#framework-integration)) - and prove it in a staging environment *before* any table enforces - anything. -2. **Row security declarations second.** Add `__ferro_rls__` to your models - and run `migrate_updates` (or an Alembic revision) once delivery is live. - -Reversed, the failure is silent and total: the moment `FORCE ROW LEVEL -SECURITY` is live on a table and no connection anywhere is delivering the -setting the policy reads, every query on that table returns zero rows — a -blank dashboard, not an error, not a warning your monitoring will catch by -itself. - -There is deliberately no adoption-gate flag (no `migrate_rls=False` to hold -the line back). Declaring `RowSecurity` and running `migrate_updates` enforces -it immediately, the same as every other schema artifact Ferro manages. - -One automatic guardrail exists, and it's narrower than a general reversed-order -detector — know its exact shape before you rely on it. On a -`connect(migrate_updates=True)` (or `migrate_destructive=True`, which implies -it) connect, for every **existing** table whose model declares -`RowSecurity(force=True)`, Ferro checks whether the connecting role is a -superuser or `BYPASSRLS` — with no check of whether that table's live `FORCE` -flag was already set before this connect — and warns if it is not: - -> The connected role is neither a superuser nor BYPASSRLS, and this migration -> touches table(s) 'invoice' with FORCE ROW LEVEL SECURITY. Row policies apply -> to the migrating role too, so a backfill or data step can silently see and -> update zero rows. Migrate as a role with BYPASSRLS if this pass moves data. - -This fires only on the `migrate_updates` connect path, and only for tables -that already existed before that connect — a plain `auto_migrate=True` create -pass (or a bare `create_tables()` call) never reaches this check, because a -freshly created table has no existing rows for a migration step to silently -miss. It still catches the same-shaped mistake for your migrator on every -later `migrate_updates` deploy: if a role this unprivileged would be filtered -by its own migration, ordinary application traffic on an unscoped connection -is in exactly as much trouble. Give your migrator `BYPASSRLS` (see -[BYPASSRLS](#bypassrls) below), and don't rely on this warning alone to catch -a reversed rollout on its very first deploy — prove settings delivery in -staging first, as step 1 above says. - -## Declaring row security - -A model declares its own row security once, as a `RowSecurity` container on -the `__ferro_rls__` `ClassVar` — the single owner of that table's policies and -flags: +## The tenant policy + +Add a `RowSecurity` to the model, on the `__ferro_rls__` class variable. `RowPolicy(column=..., setting=...)` compares a column with a session setting: === "Assignment" @@ -120,299 +32,100 @@ flags: --8<-- "docs/examples/row_level_security_annotated.py:models" ``` -`auto_migrate` (or `migrate_updates` on an existing table) creates the table -with row-level security switched on and the policy in place: - -```sql -ALTER TABLE "invoice" ENABLE ROW LEVEL SECURITY -ALTER TABLE "invoice" FORCE ROW LEVEL SECURITY -CREATE POLICY "rls_invoice_tenant_id" ON "invoice" FOR ALL - USING ("tenant_id" = NULLIF(current_setting('app.tenant_id', true), '')::uuid) - WITH CHECK ("tenant_id" = NULLIF(current_setting('app.tenant_id', true), '')::uuid) -``` - -From here the database decides which rows a query can see — a connection -whose `app.tenant_id` setting is unset sees no rows, and one that carries a -tenant id sees only that tenant's rows. A forgotten `where` filter is no -longer a data leak. - -### The shorthand - -`RowPolicy(column=..., setting=...)` compares one column to one session -setting and renders the `NULLIF(current_setting(...), '')::` expression -above for both `USING` and `WITH CHECK`. The cast comes from the column's -own storage type. `uuid`, `text`/`varchar`, and the integer families are -supported. Anything else (`timestamptz`, `jsonb`, ...) is a -class-definition-time error that names the raw form as the way out. - -The policy's live name defaults to the column name -(`rls_invoice_tenant_id` above). Pass `name=` to choose your own. +The policy takes its name from the column, `rls_invoice_tenant_id`, and its cast from the column's type, `uuid`. -### Multiple policies: composing permissive and restrictive +`auto_migrate`, `ferro migrate new` and the Alembic bridge all create the table with row-level security switched on and the policy in place: -`RowSecurity(*policies, force=True)` takes any number of `RowPolicy` values, -the same way Postgres composes them: **permissive** policies (the default) -OR-compose with each other, and **restrictive** (`restrictive=True`) policies -AND-compose with everything. That is what lets a model express Postgres' -full "owner has full access, an invited member can read" shape declaratively — -own scope-fence plus shared-read: - -=== "Assignment" +```sql +ALTER TABLE "invoice" ENABLE ROW LEVEL SECURITY; - ```python - --8<-- "docs/examples/row_level_security.py:multi_policy" - ``` +ALTER TABLE "invoice" FORCE ROW LEVEL SECURITY; -=== "Annotated" +CREATE POLICY "rls_invoice_tenant_id" ON "invoice" FOR ALL USING ("tenant_id" = NULLIF(current_setting('app.tenant_id', true), '')::uuid) WITH CHECK ("tenant_id" = NULLIF(current_setting('app.tenant_id', true), '')::uuid); +``` - ```python - --8<-- "docs/examples/row_level_security_annotated.py:multi_policy" - ``` +`USING` decides which rows a query sees and `WITH CHECK` decides which rows it may write; both compare `tenant_id` with the connection's `app.tenant_id` setting. [Policy expressions](../concepts/row-level-security.md#policy-expressions) explains the `NULLIF`, and the [reference](../reference/row-level-security.md#rowpolicy) lists every `RowPolicy` option, including policies you write in SQL. -Every policy needs a unique `name` per model (duplicates are a -class-definition-time error); `command=` scopes a policy to `"all"` (the -default), `"select"`, `"insert"`, `"update"`, or `"delete"`. `name=` is a -**suffix**, not the live name: `RowPolicy(name="tenant", ...)` on `Doc` -becomes the catalog policy `rls_doc_tenant`, matching the `rls_
_` -naming every other policy on this page follows — it must itself match -`[a-z][a-z0-9_]*` and may not already start with `rls_`. +## The application's role -### The raw escape hatch +Postgres never applies a policy to a superuser or to a role with `BYPASSRLS`, so the application must connect as a role that has neither. The migrating role is the opposite: it needs to see every row, so it keeps `BYPASSRLS` (or is the superuser). Create the application's role once, as an administrator: -When the shorthand's single column/setting comparison isn't enough — a -membership subquery, a function call, boolean composition — pass `using=` -and/or `with_check=` directly, as `owner_all` and `invitee_read` do above. -The raw form requires `name=` (there's no column to derive it from), and -Ferro validates the same command/clause rules Postgres itself enforces — -eagerly, at class-definition time, before any connection exists: `using=` is -rejected on a command that only writes (Postgres never consults it there), -`with_check=` is rejected on a command that only reads, a `command="insert"` -policy requires `with_check=` (there's no existing row for `using=` to read), -and every other command requires `using=`. +```sql +CREATE ROLE invoice_app LOGIN NOSUPERUSER PASSWORD 'change-me'; +GRANT USAGE ON SCHEMA public TO invoice_app; +GRANT SELECT, INSERT, UPDATE, DELETE ON invoice TO invoice_app; +GRANT USAGE ON ALL SEQUENCES IN SCHEMA public TO invoice_app; +ALTER ROLE migrator BYPASSRLS; +``` -## Session settings +`invoice_app` can read and write `invoice`, and every statement it sends passes through the policy. The docs example runs the first four statements with its own role and schema names; it migrates as a superuser, which needs no `BYPASSRLS`. -### Opening a session with settings= +Then connect twice. The migrating role creates the tables on the default connection, and the application's role gets a [named connection](connections.md#named-connections), `"app"`: ```python -async with ferro.engines.session(settings={"app.tenant_id": "acme"}): - open_invoices = await Invoice.where( - lambda invoice: invoice.total > 0 - ).all() +--8<-- "docs/examples/row_level_security.py:connect_migrator" +--8<-- "docs/examples/row_level_security.py:connect_app" ``` -Every operation in the session is scoped, whether or not you opened a -`transaction()` — a plain `where().all()`, `create()`, `save()`, or raw query -outside one wraps itself in an implicit transaction to carry the value: a -`BEGIN`, the `set_config` batch, and a `COMMIT` around the operation's own -statement(s), three extra round-trips instead of zero. Put a multi-operation -flow in `transaction()` and it pays that cost once, for the whole block, -instead of once per operation. - -Settings are validated eagerly, before any connection is touched: values must -be `str` (Postgres settings are text), keys must contain a dot -(`"app.tenant_id"`, never `"timezone"` or another built-in — this is a -tenancy API, not a general connection-mutation one), and values are always -bound parameters, never interpolated into SQL. +Every session opened on `"app"` is filtered by the policy, and sessions on the default connection are not. `migrator_url` and `app_url` are the two roles' database URLs. -### Deferred resolution: `current_session().set_config` +## Tenant-scoped sessions -The tenant isn't always known when the session opens — an auth chain that -resolves it from a token partway through the request is the common shape. -Open the session bare and supply the scope once you have it: +Pass the tenant's id to `engines.session(settings=...)`, as the first example on this page does. The examples below run on the four invoices the migrating role added, two for each tenant: ```python -async with ferro.engines.session() as session: # tenant not known yet - await handle_request(request) # ...called deep inside... - - -async def handle_request(request) -> None: - tenant = await resolve_tenant_from_auth_header(request) - await ferro.current_session().set_config("app.tenant_id", tenant) - # every query for the rest of this request is scoped to `tenant` +--8<-- "docs/examples/row_level_security.py:seed" ``` -`ferro.current_session()` returns the session ambient in the current asyncio -task (or `None` outside every session), so deeply nested helper code never -needs the session threaded through its signature. +The migrating role bypasses the policy, so it can write rows for any tenant. -### Nesting and inheritance - -A nested session's effective settings are the parent's, shallow-merged with -its own (the child wins per key), snapshotted the moment it opens — there is -no live propagation back to the parent, and a settings-less nested session -inherits everything: +A session on `"app"` that carries no setting sees nothing: ```python -async with ferro.engines.session(settings={"app.tenant_id": "acme", "app.role": "owner"}): - async with ferro.engines.session(settings={"app.role": "auditor"}): - ... # sees tenant_id=acme (inherited), role=auditor (overridden) +--8<-- "docs/examples/row_level_security.py:unscoped" ``` -Settings follow the **session**, not the connection route: nest a session on -a different named connection and it inherits the outer scope there too — - -```python -async with ferro.engines.session("primary", settings={"app.tenant_id": "acme"}): - async with ferro.engines.session("secondary"): - async with ferro.transaction(using="secondary") as tx: - ... # sees app.tenant_id = 'acme' here too -``` +The policy compares `tenant_id` with an unset setting, which matches no row. A request that forgets to scope its session gets an empty result, never another tenant's rows. -— which is the only way to reach a second Postgres connection with the outer -scope. `transaction(using=...)` cannot do it on its own while a session is -already ambient: an explicit `using=` that names a connection **other than** -the ambient session's own is a routing error (`ValueError`), not a way to -cross connections — open the nested session first, as above, and `using=` -naming that session's own connection then matches it with nothing to -conflict. - -Only settings **declared** at a session's own open (`settings=` on that call) -have to be honourable on that connection: opening a settings-bearing session -against a non-Postgres connection raises immediately, rather than silently -scoping nothing. **Inherited** settings are treated differently — a nested -session (or an operation an inherited-settings session runs) against a -non-Postgres connection never raises. It cannot apply what it inherited -there: opening the session and running its operations both work, unwrapped -and unscoped, because there is nothing on that backend to `set_config` in the -first place. - -### Scope stability mid-operation - -`set_config` takes effect for anything **started** after it returns — from -any task, since the change commits before `set_config`'s `await` returns. What -"already running" means splits three ways, and the split matters if a session -is ever shared across tasks: - -- **A self-wrapped, multi-statement operation** (the implicit transaction a - chunked `bulk_create` opens, say) that is already in flight keeps running - under the scope it started with — the pre-existing operation-atomicity - guarantee at work, not a race `set_config` has to resolve. -- **An explicit `transaction()` block already open when `set_config` runs** — - in the calling task or any other task sharing the session — gets the new - value re-applied (a fresh `SET LOCAL` batch on that transaction's own - connection) and its very next statement sees it, regardless of which task - opened the block. This is what makes the same-task case from the example - above work; it is not special-cased, it falls out of re-applying to every - transaction the session currently has open. -- **Under `connection` delivery**, the new value also lands on the session's - pinned connection immediately, at database-session scope. That reaches - further than the transaction case: a *sibling* task's self-wrapped - multi-statement operation already in flight can observe the change from its - next statement onward too — the one place `connection` delivery's scope - stability is looser than `transaction` delivery's. Reaching this case at all - requires sharing one session across tasks and changing its scope underneath - them, which is an unusual thing to do on purpose. - -## The NULLIF contract - -The shorthand's `NULLIF(current_setting(key, true), '')::` exists for -exactly one reason: a bare `current_setting(...)::uuid` throws on almost every -policed query that should instead see zero rows. Two states, both fail -closed, neither ever an error: - -| GUC state on the connection | `current_setting(key, true)` | With `NULLIF(..., '')::uuid` | -| :--- | :--- | :--- | -| never set | `NULL` | `NULL` → policy matches nothing → zero rows | -| set, then `RESET` | `''` | `NULL` (via `NULLIF`) → zero rows, not `''::uuid` erroring | - -The second row is the one worth internalizing: a custom GUC that was ever -`SET` and later `RESET` on that connection — which is exactly what closing a -`connection`-delivery session does to its touched keys — reads back as an -**empty string**, not `NULL`. Without `NULLIF`, that state is -`invalid input syntax for type uuid: ""` on every query against the table, -turning a fail-closed design into a hard outage the moment a connection is -reused. Write raw policies (the [escape hatch](#the-raw-escape-hatch) above) -with the same wrapper if you hand-write the comparison yourself. - -## BYPASSRLS - -`force=True` (the default on `RowSecurity`) binds the table's **owner** too — -without it, a deployment that connects as the table owner (the common -single-role setup) gets policies that are never consulted, because -Postgres exempts owners from `RLS` unless `FORCE` is set. - -Once `FORCE` is on, any connection that legitimately needs to see every row — -your migrator, a bypass connection for admin jobs — needs the database-level -escape hatch, which is a **role property**, never a settings omission: +The migrating role's session, on the default connection, sees all four: -```sql -ALTER ROLE migrator_role BYPASSRLS; +```python +--8<-- "docs/examples/row_level_security.py:migrator" ``` -Superusers already bypass RLS unconditionally (`FORCE` included), which is -also why every enforcement test in this feature runs as a created -`NOSUPERUSER` role — a green test connected as a superuser proves nothing. +Its role bypasses the policy, so the count covers both tenants. -For an application-side admin job that must see every tenant's rows, open a -named connection whose role carries `BYPASSRLS` and route to it explicitly -(`transaction(using="admin")`, or a session on that named connection) — -`BYPASSRLS` is what makes the bypass auditable in the database itself, rather -than an implicit consequence of "forgot to set a GUC". +The same policy refuses a write into another tenant: -## The JSON-claims recipe - -The Supabase pattern: instead of one GUC per claim, deliver a single JSON blob -as one session setting and unpack it in SQL. Ferro delivers the setting; it -does not manage the unpacking function's DDL (`CREATE FUNCTION` is not a -schema object Ferro owns or reconciles) — create it yourself, once, in a -migration: - -```sql -CREATE OR REPLACE FUNCTION auth_uid() RETURNS uuid -LANGUAGE sql STABLE -AS $$ - SELECT NULLIF(current_setting('request.jwt.claims', true), '')::json ->> 'sub' -$$; +```python +--8<-- "docs/examples/row_level_security.py:write_check" ``` -Deliver the claims blob as one setting, the same way as any other: +Postgres rejects the insert with SQLSTATE `42501`, which Ferro raises as `OperationalError`, and the session keeps working afterwards. -```python -import json +Setting keys are dotted names such as `app.tenant_id`, and values are strings, so convert a UUID with `str()`. Ferro sends them as bound parameters, never as SQL text. [Session settings](../concepts/row-level-security.md#session-settings) explains when a setting reaches the database and what each operation costs. -claims = {"sub": str(user.id), "role": user.role} -async with ferro.engines.session(settings={"request.jwt.claims": json.dumps(claims)}): - ... -``` +## A tenant known after the session opens -and reference the function from a raw policy, wrapped in a scalar subquery: +When the tenant is known only partway through a request (an auth step that reads a token, say), open the session without settings and set the value once you have it, through `ferro.current_session()`: ```python -RowPolicy( - name="owner", - using="owner_id = (SELECT auth_uid())", - with_check="owner_id = (SELECT auth_uid())", -) +--8<-- "docs/examples/row_level_security.py:deferred" ``` -The `(SELECT auth_uid())` wrapper matters for performance, not just style: -combined with `STABLE`, Postgres evaluates it once per statement rather than -once per row, because a `STABLE` function called through a scalar subquery is -eligible to be hoisted into an initPlan. Calling `auth_uid()` directly in the -expression loses that hoist on some query shapes and re-evaluates the claims -parse per row. +The first query sees no rows, and every query after `set_config` sees Globex's two invoices. `current_session()` returns the session open in the current task, so helper code deep in a call stack does not need the session passed in. -## Framework integration +## Web framework middleware -Every recipe below shares one shape: open (or scope) a settings-bearing -session **before** any route handler runs, so tenancy scoping is automatic at -the app level and an unscoped route is the explicit exception, never the -default. +Open the settings-bearing session in middleware, before any route handler runs, so every route is scoped and an unscoped route is the exception you opt into. Each recipe below assumes an authentication middleware that runs first and puts the user on `scope["user"]` (Litestar, generic ASGI) or `request.state.user` (FastAPI), with the tenant's id on `user.tenant_id`. Change those two lines to wherever your authentication puts the user. ### Litestar -Litestar middleware, placed **after** authentication in the stack, reads the -tenant from the auth-populated `scope["user"]` and wraps the downstream call -in a settings-bearing session. Litestar 2.15 introduced `ASGIMiddleware`; -`AbstractMiddleware` remains valid on every 2.x release. Register -`ASGIMiddleware` as an instance and `AbstractMiddleware` as the class. -Path and `exclude_opt_key` opt-out, and the `Provide` warning below, apply -to both. +Place the middleware after authentication. `ASGIMiddleware` needs Litestar 2.15 or later, and `AbstractMiddleware` works on every 2.x release: === "ASGIMiddleware" - ```python + ```python {test=compile reason="needs Litestar and a Postgres connection"} from litestar import Litestar, get from litestar.enums import ScopeType from litestar.middleware import ASGIMiddleware @@ -423,31 +136,31 @@ to both. class TenantScopingMiddleware(ASGIMiddleware): scopes = (ScopeType.HTTP,) - # Per-route opt-out for the rare unscoped endpoint (health checks, ...). + # Routes that run unscoped: health checks and the like. exclude_path_pattern = ("/health", "/metrics") exclude_opt_key = "skip_tenant_scope" async def handle( self, scope: Scope, receive: Receive, send: Send, next_app: ASGIApp ) -> None: - user = scope["user"] # populated by an earlier auth middleware/guard + user = scope["user"] # set by your authentication middleware async with ferro.engines.session( - settings={"app.tenant_id": str(user.tenant_id)} + "app", settings={"app.tenant_id": str(user.tenant_id)} ): await next_app(scope, receive, send) - app = Litestar(route_handlers=[...], middleware=[TenantScopingMiddleware()]) - - @get("/health", opt={"skip_tenant_scope": True}) - async def health() -> dict: + async def health() -> dict[str, str]: return {"status": "ok"} + + + app = Litestar(route_handlers=[health], middleware=[TenantScopingMiddleware()]) ``` === "AbstractMiddleware" - ```python + ```python {test=compile reason="needs Litestar and a Postgres connection"} from litestar import Litestar, get from litestar.middleware import AbstractMiddleware from litestar.types import Receive, Scope, Send @@ -457,83 +170,57 @@ to both. class TenantScopingMiddleware(AbstractMiddleware): scopes = {"http"} - # Per-route opt-out for the rare unscoped endpoint (health checks, ...). + # Routes that run unscoped: health checks and the like. exclude = ["/health", "/metrics"] exclude_opt_key = "skip_tenant_scope" async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: - user = scope["user"] # populated by an earlier auth middleware/guard + user = scope["user"] # set by your authentication middleware async with ferro.engines.session( - settings={"app.tenant_id": str(user.tenant_id)} + "app", settings={"app.tenant_id": str(user.tenant_id)} ): await self.app(scope, receive, send) - app = Litestar(route_handlers=[...], middleware=[TenantScopingMiddleware]) - - @get("/health", opt={"skip_tenant_scope": True}) - async def health() -> dict: + async def health() -> dict[str, str]: return {"status": "ok"} - ``` - -!!! warning "`Provide` dependency injection cannot do run-always scoping" - Litestar's `Provide` only evaluates a dependency when a handler's own - signature requests it by name. A route that forgets to declare the - dependency parameter runs completely unscoped — silently, since nothing - about the request looks wrong. Middleware runs for every matched route - regardless of its signature, which is the only way to make "every request - is scoped by default" actually true. Use `Provide` for values a handler - wants to *read*, never for something that must apply unconditionally. - -If the tenant isn't resolvable until deeper in the request (a guard or a -dependency that itself needs to run first), open the session bare in the -middleware and resolve mid-request with the deferred pattern: - -=== "ASGIMiddleware" - ```python - class TenantScopingMiddleware(ASGIMiddleware): - scopes = (ScopeType.HTTP,) - async def handle( - self, scope: Scope, receive: Receive, send: Send, next_app: ASGIApp - ) -> None: - async with ferro.engines.session(): - await next_app(scope, receive, send) + app = Litestar(route_handlers=[health], middleware=[TenantScopingMiddleware]) ``` -=== "AbstractMiddleware" +Both register the middleware for every route, and the path pattern and `skip_tenant_scope` opt let individual routes out. - ```python - class TenantScopingMiddleware(AbstractMiddleware): - scopes = {"http"} +!!! note "Litestar's `Provide` cannot scope every request" + `Provide` runs a dependency only when the handler's signature asks for it by name. A route that forgets the parameter runs unscoped, and nothing in the request shows it. Middleware runs for every matched route, so use middleware for the session and `Provide` for values a handler reads. - async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: - async with ferro.engines.session(): - await self.app(scope, receive, send) - ``` +When the tenant is known only after a guard or dependency has run, open the session without settings in the middleware and set it later, as in [A tenant known after the session opens](#a-tenant-known-after-the-session-opens). The middleware's `handle` (or `__call__`) then opens a bare session: -```python -# in a guard or dependency further down the stack: -async def resolve_tenant(request) -> None: - tenant = await resolve_tenant_from_auth_header(request) - await ferro.current_session().set_config("app.tenant_id", tenant) +```python {test=compile reason="a fragment of the ASGIMiddleware class above"} +async def handle( + self, scope: Scope, receive: Receive, send: Send, next_app: ASGIApp +) -> None: + async with ferro.engines.session("app"): + await next_app(scope, receive, send) ``` +The guard or dependency then calls `ferro.current_session().set_config("app.tenant_id", ...)` once it knows the tenant. + ### FastAPI -An app-wide `yield` dependency, applied globally rather than per-route: +Add a `yield` dependency to the application itself, so it runs for every route: -```python +```python {test=compile reason="needs FastAPI and a Postgres connection"} from fastapi import Depends, FastAPI, Request import ferro async def tenant_session(request: Request): + user = request.state.user # set by your authentication middleware async with ferro.engines.session( - settings={"app.tenant_id": str(request.state.user.tenant_id)} + "app", settings={"app.tenant_id": str(user.tenant_id)} ): yield @@ -541,20 +228,16 @@ async def tenant_session(request: Request): app = FastAPI(dependencies=[Depends(tenant_session)]) ``` -`dependencies=` at the `FastAPI(...)` level runs for every route the app -serves, with no per-handler signature to remember — the same "run always" -property the Litestar middleware recipe relies on. It also applies to routers -attached with `app.include_router(...)`, since those routes still belong to -this same `app`. The only way to opt a set of routes out is to serve them from -a genuinely separate ASGI application `app.mount(...)`s alongside this one — -a mounted sub-app has its own dependency graph and never sees `tenant_session`. +`dependencies=` on `FastAPI(...)` applies to every route the application serves, routers added with `app.include_router(...)` included. FastAPI runs a `yield` dependency's code before the handler in the same context, so the session is open while the handler runs. To serve some routes unscoped, put them in a separate ASGI application that you `app.mount(...)` beside this one; a mounted application has its own dependencies. ### Generic ASGI -Any other ASGI framework: the middleware shape underneath both recipes above, -with no framework-specific pieces: +Any other ASGI framework takes the same shape as plain ASGI middleware: + +```python {test=compile reason="needs an ASGI server and a Postgres connection"} +import ferro + -```python class TenantScopingASGIMiddleware: def __init__(self, app): self.app = app @@ -563,174 +246,103 @@ class TenantScopingASGIMiddleware: if scope["type"] != "http": return await self.app(scope, receive, send) - user = scope.get("user") + user = scope.get("user") # set by your authentication middleware settings = {"app.tenant_id": str(user.tenant_id)} if user else {} - async with ferro.engines.session(settings=settings): + async with ferro.engines.session("app", settings=settings): await self.app(scope, receive, send) ``` -### Outside the request context - -Session scope propagates through `contextvars`, which covers a request's -entire transitive call graph — including synchronous handlers a framework -runs in a threadpool. It does **not** cover work that starts outside that -graph: a background task scheduled after the response is sent, a job handed -to a task queue (Celery, arq, ...), a cron job. That code runs in its own -context with no ambient session, so it needs to open its own settings-bearing -session — and until it does, every operation it runs against a policed table -fails closed (zero rows, never an error), exactly as an unscoped request -would. - -## Connection delivery mode - -`PoolConfig(settings_delivery="connection")` is the opt-in alternative to the -default `transaction` delivery: a settings-bearing session pins one pool -connection at its first operation, applies its settings once with a -parameter-bound `set_config($1, $2, false)` batch (`is_local=false`, so the -value lives for the whole session rather than one transaction), and sends -every later statement bare — no per-operation wrap. - -**When to use it.** Only when you know the pool is a **direct** connection to -Postgres — no PgBouncer transaction mode, no transaction-mode proxy in front -of it. It trades the three extra round-trips (`BEGIN`, `set_config`, `COMMIT`) -`transaction` delivery spends on every non-transactional operation for one -`set_config` round-trip spent once, for the session's entire life — which -matters for request-heavy, short-transaction workloads on a direct pool. - -**The concurrency cap.** A settings-bearing session holds its pinned -connection from first use until it closes, so no more settings-bearing -sessions can run at once than the pool has connections — size -`max_connections` for peak concurrent scoped sessions, not average. - -**Operation serialization.** One session, one connection: everything the -session does happens one thing at a time. Two sibling tasks sharing the -session run sequentially, and a sibling operation waits out an open -`transaction()` block rather than running inside it. Operations inside that -`transaction()` block itself are unaffected — they already own the -connection. - -**The release-probe cost.** This mode installs a release hook that a -`transaction`-delivery pool never registers at all. While no session on this -pool is pinned, the hook costs no query — one atomic check and it returns. -While *any* session on this pool is pinned, every connection release on that -pool pays one `SELECT current_setting(...)` — including releases from -settings-less sessions and sessionless operations sharing the same pool. - -**`autocommit=True` is tenant-scoped here — the reverse of `transaction` -delivery.** Under the default mode, `autocommit=True` opts a statement out of -the implicit-transaction wrap that carries the setting, so that statement runs -unscoped (documented, and useful for maintenance DDL that must skip the -scope). Under `connection` delivery there's no wrap to opt out of — the -setting already lives on the connection itself — so an `autocommit=True` -statement on a pinned session still sees it. - -**The `ALTER ROLE`/`ALTER DATABASE ... SET` caveat.** Closing a session resets -exactly the keys it touched to whatever value the connection *started* with. -For an ordinary custom setting with no server-side default, that's the empty -string — which is what the [NULLIF contract](#the-nullif-contract) relies on. -If an operator has configured a startup value with `ALTER ROLE ... SET -app.tenant_id = ...` or `ALTER DATABASE ... SET ...`, closing the session -restores *that* value instead of clearing it. Never configure a tenancy key as -a role or database default. - -**Schema changes wait for pinned sessions.** `connect(migrate_updates=...)` -and `migrate()` refresh the connection pool afterward, and a refresh cannot -complete until every connection — pinned ones included — comes back. Running -a migration while long-lived tenant sessions are open blocks until they -close; Ferro warns, naming how many connections it's waiting on. Run schema -changes before opening tenant-scoped sessions, not concurrently with them. - -Sessions **without** settings never pin under this mode: same statements, -same connections, no wrap, no serialization — the mode only changes anything -for the sessions actually using it. - -## Raw policies and drift - -Reconciliation on a live table (`migrate_updates`) follows the same pattern -Ferro already uses for table checks: a **shorthand** policy that drifts from -its declaration is rebuilt (`DROP POLICY` + `CREATE POLICY`, metadata-only — -no row is read or validated), because Ferro rendered the body and knows -exactly what the catalog should store for it. - -A **raw** (`using=`/`with_check=`) policy that drifts is different: Postgres -rewrites author SQL when it stores it (`BETWEEN a AND b` comes back as -`(x >= a) AND (x <= b)`), so a textual difference can't be trusted to mean -"this needs rebuilding" — rebuilding on every cosmetic rewrite would take an -exclusive table lock on every single connect, forever. Instead, Ferro reports -the difference — both the declared text and the live catalog's — and leaves -the policy alone. Two ways out: express the policy so it fits the -[shorthand](#the-shorthand) (which Ferro can always compare exactly), or drop -the policy so the next `migrate_updates` recreates it from the declaration. - -**The flags are one-way.** `migrate_updates` can only turn `ENABLE`/`FORCE` -**on**. A model that drops its `__ferro_rls__` declaration, or drops -`force=`, keeps its live flags and warns on *every* connect — not just the -first — until the declaration is restored or `migrate_destructive` tears it -down explicitly. A policy Ferro doesn't recognize (any name that isn't -`rls_*`) is reported and never altered, on any flag; an orphaned `rls_*` -policy (declared once, no longer) warns on `migrate_updates` and drops only -under `migrate_destructive`. - -**Teardown only touches tables Ferro has evidence it manages.** Every one of -the mechanisms above — the removed-declaration warning, the orphan drop, the -`migrate_destructive` teardown itself — is gated on that table carrying at -least one live `rls_*` policy. `relrowsecurity` alone carries no authorship, -so a table a DBA enabled RLS on by hand, with only their own policies, is left -completely alone by all of it — no warning accusing anyone of removing a -declaration nobody wrote, and no flag a destructive run for an unrelated -column could take with it (ADR-0019). - -## Migrations and Alembic - -Row security is schema like any other, so every door writes the same -statements the auto-migrate pass runs (`ENABLE` / `FORCE ROW LEVEL SECURITY`, -`CREATE POLICY`), byte for byte. - -**[Migrations](schema/migrations.md).** `ferro migrate new` writes a declaration -added to an existing table as one schema step, and its down tears it down: +A request without a user opens a session with no settings, which sees no rows of a policed table. + +### Background jobs + +The session reaches everything a request calls, including synchronous handlers a framework runs in a thread pool. It does not reach work that starts outside the request: a background task that runs after the response, a job on a task queue (Celery, arq), a cron job. Open a session with the tenant's settings at the start of each job; until it does, every query it runs on a policed table returns no rows. + +## Admin jobs across every tenant + +A job that must see every tenant's rows runs on a connection whose role has `BYPASSRLS`, as the migrating role's default connection does in this example. Route the job there by name (`engines.session("admin")`, or `transaction(using="admin")`) rather than opening an `"app"` session without settings, which sees nothing. The `BYPASSRLS` role property records in the database itself which role may see every tenant. + +## Deploy order + +!!! warning "Deploy settings delivery before policies" + Once `FORCE ROW LEVEL SECURITY` is live on a table, a connection that does not send the setting sees no rows of it, and nothing raises an error. If the policies ship before the middleware, every page that reads the table goes blank. + +Ship the two halves in this order: + +1. **Settings delivery.** Deploy the middleware that opens a settings-bearing session around every request, and check in staging that requests carry the tenant's id. +2. **The policies.** Add `__ferro_rls__` to the models, then run `ferro migrate new` (or `connect(migrate_updates=True)`, or an Alembic revision). + +For a table that already exists, `ferro migrate new` writes the declaration as one schema step, and its down step removes it: + +```sql +-- 0002_invoice_rls/01_schema.up.postgres.sql +ALTER TABLE "invoice" ENABLE ROW LEVEL SECURITY; + +ALTER TABLE "invoice" FORCE ROW LEVEL SECURITY; + +CREATE POLICY "rls_invoice_tenant_id" ON "invoice" FOR ALL USING ("tenant_id" = NULLIF(current_setting('app.tenant_id', true), '')::uuid) WITH CHECK ("tenant_id" = NULLIF(current_setting('app.tenant_id', true), '')::uuid); + +-- 0002_invoice_rls/01_schema.down.postgres.sql +DROP POLICY "rls_invoice_tenant_id" ON "invoice"; +ALTER TABLE "invoice" NO FORCE ROW LEVEL SECURITY; + +ALTER TABLE "invoice" DISABLE ROW LEVEL SECURITY; +``` + +The SQLite step files say `-- ferro: not-applicable`, and a project that targets only SQLite gets no step at all. See [Migrations](schema/migrations.md) for the rest of the workflow. + +Ferro has no switch to hold the policies back: declaring `RowSecurity` and migrating enforces it at once, as with every other schema change Ferro manages. Migrate as a role with `BYPASSRLS`, so a data step in the same migration sees every row; `connect(migrate_updates=True)` warns when the connecting role has neither superuser nor `BYPASSRLS` and an existing table it migrates has `FORCE` declared (the text is in the [reference](../reference/row-level-security.md#warnings)). + +## JSON claims + +Some platforms (Supabase among them) pass the whole set of authentication claims as one JSON setting and read it in SQL. Ferro sends the setting; the function that unpacks it is yours to create, once, in a migration: + +```sql +CREATE OR REPLACE FUNCTION auth_uid() RETURNS uuid +LANGUAGE sql STABLE +AS $$ + SELECT (NULLIF(current_setting('request.jwt.claims', true), '')::json ->> 'sub')::uuid +$$; +``` + +`auth_uid()` returns the token's `sub` claim as a `uuid`, or `NULL` when the session sent no claims. + +Send the claims as one setting: + +```python {test=run prelude=jwt_claims backend=postgres} +import json + +claims = {"sub": "5f0c1a4e-2b7d-4c1e-9a3f-6d8e2b1c0a94", "role": "editor"} +async with ferro.engines.session( + "app", settings={"request.jwt.claims": json.dumps(claims)} +): + row = await ferro.fetch_one("SELECT auth_uid() AS uid") +print(row["uid"]) +``` + +Every query in that session can read the claims through `auth_uid()`: + + ```text -0002_order_rls/ - 01_schema.up.postgres.sql ALTER TABLE "rlsorder" ENABLE ROW LEVEL SECURITY; - ALTER TABLE "rlsorder" FORCE ROW LEVEL SECURITY; - CREATE POLICY "rls_rlsorder_tenant_id" ON "rlsorder" … USING (…); - 01_schema.down.postgres.sql DROP POLICY "rls_rlsorder_tenant_id" ON "rlsorder"; - ALTER TABLE "rlsorder" NO FORCE ROW LEVEL SECURITY; - ALTER TABLE "rlsorder" DISABLE ROW LEVEL SECURITY; - 01_schema.up.sqlite.sql -- ferro: not-applicable +5f0c1a4e-2b7d-4c1e-9a3f-6d8e2b1c0a94 ``` -A changed body (shorthand or raw) is dropped and recreated, and its down puts -the previous migration's body back; a removed declaration is dropped and torn -down, and its down recreates it. A migration diffs two declarations, never a -live database, so a raw body is compared as you wrote it and the -[unverifiable-body](#raw-policies-and-drift) case does not arise. A project -targeting only SQLite gets nothing to write. - -**[Alembic](schema/alembic.md).** A generated revision holds what the planner -decides against the live database, so it proposes exactly what -`migrate_destructive=True` would do: missing policies, rebuilt ones, the flags, -and the drops for removed declarations and orphaned `rls_*` policies (with no -destructive gate, since a revision is reviewed before it runs). Its -`downgrade()` is the planner run back to the database as it was, under the -same ownership gate: row security that predates Ferro's declaration (a DBA's -own fence) is never disabled by a downgrade. - -**Autogenerate is silent exactly where the runtime pass only warns.** Raw -(`using=`/`with_check=`) policies whose body ferro cannot verify, and -policies ferro does not own at all, are things the runtime reconciliation -pass reports with a `UserWarning` on every connect (see -[Raw policies and drift](#raw-policies-and-drift)); autogenerate carries -neither into the revision. Reading the revision's diff is not a substitute -for watching your `migrate_updates` warnings for those two cases. - -## See Also - -- [Session settings](#session-settings) — `engines.session(settings=...)`, - `current_session()` -- [Schema Management](schema/overview.md) — auto-migrate, migrations and the - Alembic bridge -- [Connections & Databases](connections.md) — `PoolConfig` and named - connections -- [Transactions](transactions.md) — connection affinity and `using=` routing -- [Queries](queries.md) — the lambda predicate style used throughout this page +Then call the function from a policy written in SQL, inside a scalar subquery: + +```python {test=compile reason="a RowPolicy argument, not a statement"} +RowPolicy( + name="owner", + using="owner_id = (SELECT auth_uid())", + with_check="owner_id = (SELECT auth_uid())", +) +``` + +Wrapped in `(SELECT ...)`, a `STABLE` function runs once per statement instead of once per row, because Postgres can evaluate the subquery once and reuse its result. + +## See also + +- [Row-level security and session settings](../concepts/row-level-security.md): how settings reach the database, what each operation costs, connection delivery, and drift. +- [Row-level security reference](../reference/row-level-security.md): `RowSecurity`, `RowPolicy`, the command table, the shorthand's casts, and composing several policies. +- [Connections & Databases](connections.md): named connections and `PoolConfig`. +- [Transactions](transactions.md): one transaction for several operations. diff --git a/docs/pages/howto/adopting-migrations.md b/docs/pages/guide/schema/adopting-migrations.md similarity index 64% rename from docs/pages/howto/adopting-migrations.md rename to docs/pages/guide/schema/adopting-migrations.md index ae017a2e..690b3e76 100644 --- a/docs/pages/howto/adopting-migrations.md +++ b/docs/pages/guide/schema/adopting-migrations.md @@ -1,6 +1,10 @@ -# Adopting Migrations on an Existing Database +--- +description: "Move a database that auto-migrate or Alembic built onto Ferro's migrations without running any DDL on it." +--- -Your database already exists: `connect(url, migrate_updates=True)` built it, or an Alembic chain did. You want [Migrations](../guide/schema/migrations.md) from now on, without running any DDL on it and without losing a row. A **baseline** records migrations as applied on a database that already has their schema; it runs nothing, and it records only after checking. +# Adopting migrations on an existing database + +This how-to moves a database that `connect(url, migrate_updates=True)` or an Alembic chain built onto [Migrations](migrations.md), without running any DDL on it and without losing a row. The tool is a **baseline**: it records migrations as applied on a database that already has their schema. It runs nothing, and it records only after checking. ## The four steps @@ -9,7 +13,7 @@ Your database already exists: `connect(url, migrate_updates=True)` built it, or ```bash ferro migrate init - ferro migrate new initial + ferro migrate new create_author ``` `new` prints the files it wrote: @@ -17,7 +21,7 @@ Your database already exists: `connect(url, migrate_updates=True)` built it, or ```text default · migrations - ok Wrote migrations/0001_initial/ + ok Wrote migrations/0001_create_author/ 01_schema.up.postgres.sql schema step, up (postgres) 01_schema.down.postgres.sql schema step, down (postgres) 01_schema.up.sqlite.sql schema step, up (sqlite) @@ -30,14 +34,18 @@ Your database already exists: `connect(url, migrate_updates=True)` built it, or Commit it. Run `ferro migrate up` on a fresh database and you get the same schema the old door built. -3. **Baseline each existing database.** +3. **Baseline each existing database.** Point `DATABASE_URL` at it and record the migrations: - `ferro migrate baseline` records it: + ```bash + ferro migrate baseline + ``` + + `baseline` checks the database against the migrations, then records them: ```text default · sqlite:///./app.db - recorded 0001_initial as baseline (1 step) + recorded 0001_create_author as baseline (1 step) baseline checked what `ferro migrate drift` checks: objects ferro does not own (functions, triggers) were not compared ``` @@ -53,9 +61,9 @@ default · sqlite:///./app.db This database has no ferro migration records, but table "author" from 0001 already exists. If it was built by auto-migrate or Alembic, run "ferro migrate baseline 0001". up will not run 0001 over it. ``` -## When baseline says no +## Baseline refusals on drift {#when-baseline-says-no} -Baseline compares the database with the target migration's schema snapshot exactly as `ferro migrate drift` does. Any difference and it records nothing, prints each one and exits 4: +Baseline compares the database with the target migration's schema snapshot exactly as `ferro migrate drift` does. On any difference, `ferro migrate baseline 0001` records nothing, prints each one and exits 4: ```text @@ -65,7 +73,7 @@ drift against 0001_create_author: nothing was recorded ``` -Each line is a piece of work, and there is no flag that records past them (ADR-0031): either the database is not where you think it is, or the migration does not describe it. Fix whichever is wrong and baseline again: +Each line is a piece of work, and no flag records past them: a baseline recorded over a difference would leave the next migration to run against a schema it was not written for. Either the database is not where you think it is, or the migration does not describe it. Fix whichever is wrong and baseline again: - **The database is ahead of the target** (here, it already has the `slug` a later migration adds): baseline a later migration instead, `ferro migrate baseline` with no target or `ferro migrate baseline 0002`. - **The database is behind the models**: finish step 1 (one more `migrate_updates` start or `alembic upgrade head`). @@ -82,25 +90,29 @@ Each line is a piece of work, and there is no flag that records past them (ADR-0 nothing was recorded ``` - Drop it, then baseline again. Either give the old door one last start that removes what the models never declared, `connect(url, migrate_destructive=True)`, or drop it by hand: + Drop it, then baseline again. Either give the old door one last start that removes what the models never declared, `connect(url, migrate_destructive=True)`, or drop it by hand on Postgres: ```sql ALTER TABLE "author" ALTER COLUMN "slug" DROP DEFAULT; ``` - On SQLite neither works: SQLite drops a default only by rebuilding the table, so rebuild it without the `DEFAULT` by hand (create a copy, copy the rows across, drop the table, rename the copy). + The column then has no default, and the next baseline records. On SQLite neither works: SQLite drops a default only by rebuilding the table, so rebuild it without the `DEFAULT` by hand (create a copy, copy the rows across, drop the table, rename the copy). + +Three things are never a difference: -Tables the snapshot does not declare (`alembic_version`, another application's tables) are never a difference. Neither are the two server defaults Ferro leaves itself: a `serial` key's sequence on Postgres (recognised by the sequence the column owns, so a renamed table's still counts), and on SQLite a literal default on a `NOT NULL` column, which `migrate_updates` keeps when it adds a required column. Objects Ferro does not own (functions, triggers) are not compared, and the report says so. +- Tables the snapshot does not declare: `alembic_version`, another application's tables. +- The two server defaults Ferro leaves itself: a `serial` key's sequence on Postgres (recognised by the sequence the column owns, so a renamed table's still counts), and on SQLite a literal default on a `NOT NULL` column, which `migrate_updates` keeps when it adds a required column. +- Objects Ferro does not own, such as functions and triggers. They are not compared, and the report says so. ## Databases at different points Every database is baselined at the migration it actually matches. A production database at the head takes `ferro migrate baseline`; a staging copy restored from an older dump takes `ferro migrate baseline 0001`, then `ferro migrate up` applies the rest: -`ferro migrate baseline 0001 --url "$STAGING_URL"` records the one migration the staging copy matches: +With `STAGING_URL` naming the staging copy (`sqlite:staging.db` here), `ferro migrate baseline 0001 --url "$STAGING_URL"` records the one migration it matches: ```text -default · sqlite:///./app.db +default · sqlite:///./staging.db recorded 0001_create_author as baseline (1 step) baseline checked what `ferro migrate drift` checks: objects ferro does not own (functions, triggers) were not compared ``` @@ -109,13 +121,11 @@ baseline checked what `ferro migrate drift` checks: objects ferro does not own ( ```text -default · sqlite:///./app.db +default · sqlite:///./staging.db 0002_author_slug 01_schema applied (0 ms) ``` -`status` shows what each database was baselined at: - -A production database baselined at the head shows both migrations as `applied (baseline)`: +`ferro migrate status` on a production database baselined at the head shows both migrations as `applied (baseline)`: ```text @@ -125,7 +135,7 @@ ok 2 applied main._ferro_migrations ok 0002_author_slug applied (baseline) 2026-10-15 04:30Z ``` -Baseline is refused on a database that already has migration records; `status` says where it stands. +The staging copy shows `0001_create_author` as `applied (baseline)` and `0002_author_slug` as `applied`. Baseline is refused on a database that already has migration records; `status` says where it stands. ## Coming from Alembic @@ -133,7 +143,7 @@ The steps are the same, with Alembic as the old door: - Step 1 is `alembic upgrade head`. - In step 4, remove `get_metadata()` from `env.py`'s `target_metadata` once every database is baselined. Ferro's migrations ignore `alembic_version`; drop it when no database needs the Alembic chain any more. -- A project whose Alembic chain also manages its own SQLAlchemy tables keeps that chain for them: drop `get_metadata()` from `target_metadata` and keep `**ferro_options()`, which keeps Alembic off Ferro's tables and tracking tables. See [Alembic](../guide/schema/alembic.md). +- A project whose Alembic chain also manages its own SQLAlchemy tables keeps that chain for them: drop `get_metadata()` from `target_metadata` and keep `**ferro_options()`, which keeps Alembic off Ferro's tables and tracking tables. See [Alembic](alembic.md). ## An app that migrates at start-up @@ -143,11 +153,13 @@ A desktop or local-first app cannot baseline its users' files by hand. The relea --8<-- "docs/examples/migrations_adopting.py:recipe" ``` +A file the previous release built matches `0001` and is recorded. A brand-new, empty file matches nothing, so the baseline records nothing and `up()` creates the tables from `0001`. + `baseline()` itself decides whether the file still needs adopting. On a tracked file (an earlier start recorded it, or `up()` has run) it records nothing and raises `AlreadyTrackedError`, a `MigrationRefused` whose `applied` names the recorded migrations, `head` the newest of them and `report` the file's status. Nothing is wrong, so the app carries on to `up()`. This also covers a race. Two instances that start together (two windows of the app, or two services' pre-deploys against one shared database) both find no records and both call `baseline()`. The run lock lets one of them record, and the other raises `AlreadyTrackedError`. Do not check `status()` first to decide whether to call `baseline()`: the answer can change before the baseline takes the lock. Every other refusal (a target the directory lacks, a lock wait that runs out) is still raised as `MigrationRefused`. -The baseline still checks the file. A file that matches neither (an even older build, or one edited by hand) is not recorded, and `up()` then refuses to run `0001` over its tables: the app fails loudly at start-up instead of serving a half-migrated file. +The baseline still checks the file. A file that matches neither (an even older build, or one edited by hand) is not recorded: on drift, `baseline()` raises nothing and returns a [`BaselineReport`](../../api/migrations.md#ferro.migrations.BaselineReport) whose `drift` lists the differences. `up()` then refuses to run `0001` over the file's tables, so the app fails loudly at start-up instead of serving a half-migrated file. To fail at the baseline instead, keep the report and call `report.raise_for_problems()`. ## Undoing a baseline @@ -169,8 +181,8 @@ ferro migrate: 0002 was recorded by `baseline` and created nothing here; `down` In code, `ferro.migrations.baseline()` and `ferro.migrations.remove_baseline()` do the same as the two commands. -## See Also +## See also -- [Schema Management overview](../guide/schema/overview.md) — one door per database -- [Deploying migrations](../guide/schema/deploying.md) — what comes after the baseline -- [CLI reference](../reference/cli.md#ferro-migrate-baseline-target) +- [Schema management overview](overview.md): one door per database +- [Deploying migrations](deploying.md): what comes after the baseline +- [CLI reference](../../reference/cli.md#ferro-migrate-baseline-target) and [Migrations API reference](../../api/migrations.md#ferro.migrations.baseline): every flag and parameter of `baseline` diff --git a/docs/pages/guide/schema/alembic.md b/docs/pages/guide/schema/alembic.md index 4136bb7a..df080589 100644 --- a/docs/pages/guide/schema/alembic.md +++ b/docs/pages/guide/schema/alembic.md @@ -1,30 +1,66 @@ -# Alembic +--- +description: "Let alembic revision --autogenerate write revisions for your Ferro models through the Alembic bridge." +--- -Ferro bridges your models into the SQLAlchemy metadata [Alembic](https://alembic.sqlalchemy.org/) uses, so `alembic revision --autogenerate` writes revisions for them. It is a supported alternative to [Migrations](migrations.md) for a project that already runs Alembic, or keeps its own SQLAlchemy tables beside its Ferro models. Otherwise, Ferro recommends Migrations: the bridge writes no backfills and no SQLite table rebuilds. +# Alembic bridge -One database is managed by one door: Alembic's revisions, or Ferro's migrations, or auto-migrate ([one door per database](overview.md#one-door-per-database)). +Ferro bridges your models into the SQLAlchemy metadata [Alembic](https://alembic.sqlalchemy.org/) reads, so `alembic revision --autogenerate` writes revisions for them. Add a `nickname` to the overview's `Author`: -## Install +=== "Assignment" + + ```python + --8<-- "docs/examples/migrations_quickstart.py:models" + ``` + +=== "Annotated" + + ```python + --8<-- "docs/examples/migrations_quickstart_annotated.py:models" + ``` + +Against a SQLite database that holds `author` without the field, `alembic revision --autogenerate -m "add nickname"` writes this revision: + + +```python {test=compile reason="an Alembic revision; the docs tests regenerate it and compare"} +def upgrade() -> None: + op.add_column('author', sa.Column('nickname', sa.String(), nullable=True)) + op.execute(sa.DDL('CREATE INDEX IF NOT EXISTS "idx_author_nickname" ON "author" ("nickname")')) + + +def downgrade() -> None: + op.drop_index('idx_author_nickname', table_name='author') + op.drop_column('author', 'nickname') +``` + +The column is Alembic's own `op.add_column`, and its index is the statement auto-migrate runs for the same change, byte for byte. + +The bridge is a supported alternative to [Migrations](migrations.md) for a project that already runs Alembic, or keeps its own SQLAlchemy tables beside its Ferro models. Otherwise, use Migrations: the bridge writes no backfills and no SQLite table rebuilds. One database is managed by one door: Alembic's revisions, Ferro's migrations or auto-migrate ([one door per database](overview.md#one-door-per-database)). + +## The `alembic` extra {#install} + +Install the extra: ```bash pip install "ferro-orm[alembic]" ``` -This adds Alembic and SQLAlchemy, used only to generate and run revisions, never by Ferro at runtime. +It adds Alembic and SQLAlchemy, which generate and run revisions; Ferro never imports them at runtime. + +## The Alembic environment {#initialize} -## Initialize +Scaffold an Alembic environment in a directory other than `migrations/`: ```bash alembic init alembic ``` -This scaffolds `alembic.ini` and an `alembic/` directory with `env.py` and `versions/`. Avoid `alembic init migrations`: `migrations/` is where `ferro migrate init` puts Ferro's own migrations, and it refuses a directory that holds an Alembic environment. +It writes `alembic.ini` and an `alembic/` directory holding `env.py` and `versions/`. Keep it out of `migrations/`: that is where `ferro migrate init` puts Ferro's own migrations, and it refuses a directory that holds an Alembic environment. -## Configure env.py +## `env.py` {#configure-envpy} -`env.py` passes Ferro's metadata and Ferro's options to `context.configure(...)`: +Pass Ferro's metadata and Ferro's options to `context.configure(...)` in `env.py`: -```python +```python {test=compile reason="an Alembic env.py: Alembic runs it, not the docs harness"} # alembic/env.py from alembic import context @@ -45,13 +81,15 @@ def run_migrations_online() -> None: # The rest of env.py stays as generated. ``` -- **`get_metadata()`** imports the models the [project configuration](../../reference/configuration.md) names (`ferro.toml` or `[tool.ferro]`) and returns their tables; with several configured databases, name one: `get_metadata("billing")`. Without a configuration it renders every registered model, so import your models module in `env.py` first. -- **`ferro_options()`** keeps Alembic's own comparator off Ferro's tables and both tracking tables, so every operation on a Ferro table comes from Ferro's planner, and renders the enum columns Alembic cannot. A project with its own `include_object` or `render_item` hooks passes them through it: `**ferro_options(include_object=mine, render_item=mine)`. Ferro's filter asks yours about every object it does not hide; Ferro's renderer falls through to yours. +Autogenerate now compares Ferro's tables through Ferro's planner, and every other table through Alembic's own comparator: + +- **`get_metadata()`** imports the models the [project configuration](../../reference/configuration.md) names (`ferro.toml` or `[tool.ferro]`) and returns their tables. With several configured databases, name one: `get_metadata("billing")`. Without a configuration it renders every registered model, so import your models module in `env.py` first. +- **`ferro_options()`** keeps Alembic's own comparator off Ferro's tables and both tracking tables, and renders the enum columns Alembic cannot. A project with its own `include_object` or `render_item` hooks passes them through it: `**ferro_options(include_object=mine, render_item=mine)`. Ferro's filter asks yours about every object it does not hide, and Ferro's renderer falls through to yours. - Autogenerate over `get_metadata()` without `ferro_options()` is refused, naming the line to add. -An async `env.py` works the same way: call `context.configure(...)` inside the function you hand to `connection.run_sync(...)`: +An async `env.py` calls `context.configure(...)` inside the function it hands to `connection.run_sync(...)`: -```python +```python {test=compile reason="an Alembic env.py: Alembic runs it, not the docs harness"} from alembic import context from ferro.migrations import ferro_options, get_metadata @@ -73,44 +111,53 @@ async def run_async_migrations() -> None: await connection.run_sync(do_run_migrations) ``` -## One decider +The options are the same; only where `configure` runs changes. + +## How the bridge plans a revision {#one-decider} -The bridge does not compare schemas itself. Autogenerate reads the live database the way the auto-migrate pass reads it, plans the difference with the same planner (destructive changes included, since a revision is reviewed before it runs), and writes the planner's operations as Alembic ops, in the planner's order. An empty revision means no drift; a non-empty one holds exactly what `connect(url, migrate_destructive=True)` would have done. `downgrade()` is the same planner run back from the models to the database as it was. +The bridge does not compare schemas itself. Autogenerate reads the live database the way the auto-migrate pass reads it, plans the difference with the same planner, destructive changes included (a revision is reviewed before it runs), and writes the planner's operations as Alembic ops in the planner's order. An empty revision means no drift. A non-empty one holds what `connect(url, migrate_destructive=True)` would have done, plus one thing the pass never does: deleting a model drops its live table (`op.drop_table`, marked destructive), followed by `DROP TYPE` for any native enum type only that table used. `downgrade()` is the same planner run back from the models to the database as it was. -Where Alembic has an op of its own (a table, a column, its type and nullability, an index, a foreign key, a rename), the revision uses it. Everything else is the statement the auto-migrate pass would run, byte for byte, as `op.execute(sa.DDL(...))`. +Where Alembic has an op of its own (a table, a column, its type and nullability, an index, a foreign key, a rename), the revision uses it. Everything else is the statement the auto-migrate pass would run, byte for byte, written as `op.execute(sa.DDL(...))`. -One change goes beyond the pass: deleting a model drops its live table (`op.drop_table`, marked destructive), the same table Alembic would drop, followed by `DROP TYPE` for any native enum type only that table used. Alembic's version table, ferro's tracking tables and any table your `include_object` or `include_name` filters exclude are never dropped, and the `downgrade()` puts the type and the table back as far as ferro can read them (columns, types, nullability, indexes, foreign keys, checks and policies; not server defaults or comments). +Alembic's version table, Ferro's tracking tables and any table your `include_object` or `include_name` filters exclude are never dropped. A dropped table's `downgrade()` puts the type and the table back as far as Ferro can read them: columns, types, nullability, indexes, foreign keys, checks and policies, but not server defaults or comments. + +Generate and apply a revision: ```bash alembic revision --autogenerate -m "add nickname" alembic upgrade head ``` -Always review the revision before applying it. +The first command writes the revision shown at the top of this page into `alembic/versions/`; read it before the second applies it. ### Marked operations -An operation that drops data, or that fails on existing rows, carries a comment saying so: +An operation that drops data, or that fails on existing rows, carries a comment saying so. Say `Author` also had `bio: str | None = None` and you delete that line; the revision marks the drop: + + +```python {test=compile reason="an Alembic revision; the docs tests regenerate it and compare"} +def upgrade() -> None: + # ferro: destructive (drops author.bio and the data it holds) + op.drop_column('author', 'bio') -```python -def upgrade(): - # ferro: data-dependent (fails while card has rows; ... `ferro migrate new`) - op.add_column('card', sa.Column('size', sa.Integer(), nullable=False)) - # ferro: destructive (drops card.legacy and the data it holds) - op.drop_column('card', 'legacy') + +def downgrade() -> None: + op.add_column('author', sa.Column('bio', sa.String(), nullable=True)) ``` -A **data-dependent** op is a change existing rows need a value for (here, a required column). The bridge writes it plain: the backfill is yours to write as `op.execute(...)` statements ahead of it, or generate the change as a [migration](data-steps.md), which writes the backfill for you. On SQLite such an add is refused instead (below). +The `downgrade()` puts the column back empty: the values it held are gone. + +A required column added to a table that has rows is a **data-dependent** op, a change existing rows need a value for. On Postgres the bridge writes it plain, under `# ferro: data-dependent (fails while author has rows; ferro migrations generate the backfill: \`ferro migrate new\`)`. The backfill is yours to write as `op.execute(...)` statements ahead of it, or generate the change as a [migration](data-steps.md), which writes the backfill for you. On SQLite the same add is refused (below). ### Enum types -Every native Postgres enum type a revision needs is created by the planner's own guarded `CREATE TYPE` statement, ahead of the table operations, and every column of it is written `postgresql.ENUM(..., create_type=False)` so SQLAlchemy never creates it a second time (the `render_item` that `ferro_options()` carries writes that flag; SQLAlchemy's own rendering drops it). A `downgrade()` drops the types its upgrade created, after the tables that used them. +Every native Postgres enum type a revision needs is created by the planner's own guarded `CREATE TYPE` statement, ahead of the table operations. Every column of that type is written `postgresql.ENUM(..., create_type=False)`, so SQLAlchemy never creates the type a second time (the `render_item` that `ferro_options()` carries writes that flag; SQLAlchemy's own rendering drops it). A `downgrade()` drops the types its upgrade created, after the tables that used them. Enum label changes follow the planner too: -- A label added to a `StrEnum` is `ALTER TYPE ... ADD VALUE IF NOT EXISTS` inside an `autocommit_block()`. Its `downgrade()` raises: enum labels are append-only on this door (ADR-0011), since rows may hold the label. +- A label added to a `StrEnum` is `ALTER TYPE ... ADD VALUE IF NOT EXISTS` inside an `autocommit_block()`. Its `downgrade()` raises: rows may hold the label, so labels are append-only on this door. - A label renamed with `__ferro_renamed_labels__` is `ALTER TYPE ... RENAME VALUE`. -- A label removed is a change existing rows need a value for: [Migrations](data-steps.md#removing-an-enum-label) generate its backfill and the swap to a type without it. +- A label removed is not written: the revision leaves that change out, and the type keeps the label. Existing rows need a new value first, so [Migrations](data-steps.md#removing-an-enum-label) generate its backfill and the swap to a type without it. ## What autogenerate refuses @@ -121,15 +168,15 @@ The bridge refuses, before writing a revision, what an Alembic revision cannot w | `env.py` passes `get_metadata()` without `**ferro_options()` | Alembic would compare Ferro's tables a second way | Add `**ferro_options()` to `context.configure(...)` | | The database carries Ferro's `_ferro_migrations` tracking table | The database is managed by Ferro's migrations | `ferro migrate new`. A project whose Alembic chain still manages its own SQLAlchemy tables drops `get_metadata()` from `target_metadata` and keeps `**ferro_options()` | | A primary key changes | No door changes a key in place | [Changing a primary key](migrations.md#changing-a-primary-key) | -| A change SQLite can only make by rebuilding the table (a type, a nullability, a constraint on an existing table), or a `NOT NULL` column added to a SQLite table with rows and no default | Alembic's batch mode does not handle foreign-key pragmas, so its rebuild cascades into `ON DELETE CASCADE` children | `ferro migrate new`, which writes the [table rebuild](migrations.md#sqlite-table-rebuilds) (and the backfill) | +| A change SQLite can only make by rebuilding the table (a type, a nullability, a constraint on an existing table), or a `NOT NULL` column with no default added to an existing SQLite table | Alembic's batch mode does not handle foreign-key pragmas, so its rebuild cascades into `ON DELETE CASCADE` children | `ferro migrate new`, which writes the [table rebuild](migrations.md#sqlite-table-rebuilds) (and the backfill) | -A refused rename hint (a hint whose old name is still declared) is refused with the generator's reason. +A rename hint whose old name is still declared is refused with the generator's reason. ## Moving from Alembic to Migrations -A database Alembic manages can move to Ferro's migrations without running any DDL: bring it to `alembic upgrade head`, generate the first migration, and `ferro migrate baseline` it. See [Adopting migrations on an existing database](../../howto/adopting-migrations.md#coming-from-alembic). +A database Alembic manages can move to Ferro's migrations without running any DDL: bring it to `alembic upgrade head`, generate the first migration, and `ferro migrate baseline` it. [Adopting migrations on an existing database](adopting-migrations.md#coming-from-alembic) walks through it. -## See Also +## See also -- [Schema Management overview](overview.md) — which door covers which change -- [Migrations API reference](../../api/migrations.md) — `get_metadata()`, `ferro_options()`, `render_item()` +- [Schema management overview](overview.md): which door covers which change +- [Migrations API reference](../../api/migrations.md): `get_metadata()`, `ferro_options()`, `render_item()` diff --git a/docs/pages/guide/schema/auto-migrate.md b/docs/pages/guide/schema/auto-migrate.md index 198dadfa..4959321d 100644 --- a/docs/pages/guide/schema/auto-migrate.md +++ b/docs/pages/guide/schema/auto-migrate.md @@ -1,170 +1,165 @@ +--- +description: "Make the database match your models every time you connect, with no files to write or review." +--- + # Auto-migrate -Auto-migrate makes the database match the models when you connect. It is rungs 1 and 2 of [the ladder](overview.md#the-ladder): no files and nothing to review, which is what you want in tests, scripts and early development, and what you do not want for a database whose data matters (use [Migrations](migrations.md) there). +Auto-migrate makes the database match the models when you connect, with no files and nothing to review. Take the overview's `Author`, which has just gained a `nickname`: -A database is managed by auto-migrate or by migrations, never both: on a database that has run a migration, every flag on this page is refused before any DDL ([one door per database](overview.md#one-door-per-database)). +=== "Assignment" -## Creating tables with `auto_migrate=True` + ```python + --8<-- "docs/examples/migrations_quickstart.py:models" + ``` + +=== "Annotated" + + ```python + --8<-- "docs/examples/migrations_quickstart_annotated.py:models" + ``` -```python +The development database `dev.db` was built before that change: its `author` table has no `nickname` column, still has a `bio` column the model has since dropped, and holds one row, Ada Lovelace's. Connect with `migrate_updates=True`, and the connect adds the column before your code runs: + +```python {test=run prelude=auto_migrate} import ferro -await ferro.connect("sqlite:dev.db?mode=rwc", auto_migrate=True) +await ferro.connect("sqlite:dev.db?mode=rwc", migrate_updates=True) +async with engines.session(): + ada = await Author.where(lambda author: author.email == "ada@example.com").first() + assert ada.nickname is None ``` -Creates tables for every registered model (including many-to-many join tables) and leaves existing tables untouched, whatever their shape. +The connect ran `ALTER TABLE "author" ADD COLUMN "nickname" varchar` and created `idx_author_nickname`; Ada's row is still there, with no nickname yet. The ORM calls run inside a [session](../connections.md#sessions), as every block on this page does. -## Applying column changes with `migrate_updates` +Auto-migrate is rungs 1 and 2 of [the ladder](overview.md#the-ladder). It suits tests, scripts and early development, and does not suit a database whose data matters: use [Migrations](migrations.md) there. A database is managed by auto-migrate or by migrations, never both: on a database that has run a migration, every flag on this page is refused before any DDL ([one door per database](overview.md#one-door-per-database)). + +## Creating tables with `auto_migrate=True` -*Added in 0.11.0.* When models gain or change fields between runs, `migrate_updates=True` reconciles existing tables at connect time: +Pass `auto_migrate=True` to `connect()` and every missing table is created: -```python +```python {test=run prelude=auto_migrate} import ferro -await ferro.connect("sqlite:dev.db?mode=rwc", migrate_updates=True) +await ferro.connect("sqlite:test.db?mode=rwc", auto_migrate=True) +async with engines.session(): + grace = await Author.create( + name="Grace Hopper", email="grace@example.com", nickname="amazing grace" + ) + assert grace.id == 1 ``` -What it covers depends on what each backend can do in place: - -| Change | SQLite | PostgreSQL | -| :--- | :--- | :--- | -| Add missing column | ✅ `ADD COLUMN` | ✅ `ADD COLUMN` | -| Rename a column declared `renamed_from` | ✅ `RENAME COLUMN` | ✅ `RENAME COLUMN` | -| Add the column's index (`index=True`) | ✅ `CREATE INDEX` | ✅ `CREATE INDEX` | -| Add composite index (`__ferro_composite_indexes__`) to existing columns | ✅ `CREATE INDEX` | ✅ `CREATE INDEX` | -| Add table check (`__ferro_checks__`) on CREATE TABLE | ✅ inline CHECK | ✅ inline CHECK | -| Add table check to existing table | ⚠️ `UserWarning`, no DDL; [migrations](migrations.md#sqlite-table-rebuilds) generate the table rebuild | ✅ `ADD CONSTRAINT` | -| Rebuild table check on body drift | ⚠️ `UserWarning`, no DDL; migrations generate the table rebuild | ✅ rebuild: `DROP CONSTRAINT` + `ADD CONSTRAINT` | -| Add column `db_check=True` on existing column | ⚠️ `UserWarning`, no DDL; migrations generate the table rebuild | ✅ `ADD CONSTRAINT` | -| Leftover ferro check (`ck_*` live, removed from model) under `migrate_updates` | ⚠️ `UserWarning`, constraint stays | ⚠️ `UserWarning`, constraint stays | -| Drop orphaned ferro check (`ck_*`) | ⚠️ `UserWarning`, no DDL; migrations generate the table rebuild | ✅ with `migrate_destructive=True` | -| Add unique column (`unique=True`) | ✅ via explicit unique index + warning | ✅ inline `UNIQUE` | -| Add foreign-key column | ✅ column only, no FK constraint + warning | ✅ column + FK constraint | -| Add missing FK constraint to an existing column | ⚠️ `UserWarning`, no DDL; migrations generate the table rebuild | ✅ `ADD CONSTRAINT` | -| Drop a ferro foreign key (`fk_*`) from a column the model keeps (`team: Annotated[Team, ForeignKey(...)]` became `team_id: int`) | ⚠️ `UserWarning`, constraint stays; migrations generate the table rebuild | ✅ with `migrate_destructive=True`: `DROP CONSTRAINT` | -| Change a foreign key's `on_delete` (or target) | ⚠️ `UserWarning`, no DDL; migrations generate the table rebuild | ✅ rebuild: `DROP CONSTRAINT` + `ADD CONSTRAINT` | -| Change column type | ⚠️ `UserWarning`, no DDL (SQLite type affinity makes drift mostly cosmetic); migrations generate the table rebuild | ✅ `ALTER COLUMN ... TYPE ... USING` cast | -| Change nullability | ⚠️ `UserWarning`, no DDL; migrations generate the table rebuild | ✅ `SET NOT NULL` / `DROP NOT NULL`. `SET NOT NULL` backfills nothing, whatever the default: it fails the connect if any row holds `NULL`. A [migration](data-steps.md) writes the backfill first | -| A server default no model declares (a *foreign default*, [below](#destructive-drops-with-migrate_destructive)) | ⚠️ `UserWarning`, default stays; only a hand rebuild drops it (an expression, or any default on a nullable column; a literal on a `NOT NULL` column is ferro's own) | ⚠️ `UserWarning` under `migrate_updates`; ✅ with `migrate_destructive=True`: `DROP DEFAULT` | -| Drop orphaned Ferro-named index (`idx_*` / `uq_*`) | ✅ with `migrate_destructive=True` | ✅ with `migrate_destructive=True` | -| Redefine an index that keeps its name (a long name cut to 63 characters, two column groups that join to one name, or a live `idx_*` / `uq_*` index written another way) | ✅ `DROP INDEX` + `CREATE INDEX` under `migrate_updates`. A unique one over duplicate values fails the connect, counting them | ✅ same, in the table's transaction | -| Add a missing enum label (a `StrEnum` grew a member) | ✅ nothing to do — enums store as text | ✅ `ALTER TYPE ... ADD VALUE` *0.18.0+* | -| Rename an enum label declared with `__ferro_renamed_labels__` | ✅ nothing to do | ✅ `ALTER TYPE ... RENAME VALUE` | -| Remove an enum label | ✅ nothing to do | ⚠️ `UserWarning`, no DDL. [Migrations](migrations.md) generate it with its [backfill](data-steps.md#removing-an-enum-label) | -| Inline single-column `UNIQUE` on an existing column, index option changes | ❌ never here; [migrations](migrations.md) generate it | ❌ never here; migrations generate it | -| Rename a table | ✅ `RENAME TABLE` under `migrate_updates` when the database holds the old name from `__ferro_renamed_from__` and not the new one; derived index and check names follow. Without `migrate_updates` a `UserWarning` names the table, the hint and both doors, and nothing is created. [Migrations](migrations.md#renames) are the reviewed path | ✅ same | -| Drop a table | ❌ never here; [migrations](migrations.md#reading-a-migration) generate it, marked destructive | ❌ same | -| Change a primary key | ❌ never; no door generates it. Migrations refuse it with [the recipe](migrations.md#changing-a-primary-key) | ❌ same | +The empty `test.db` gets an `author` table with every column and index the model declares. `auto_migrate=True` creates the tables of every registered model, many-to-many join tables included, and leaves every table that already exists untouched, whatever its shape: on `dev.db` it would have added no `nickname`. + +## Applying column changes with `migrate_updates` + +`migrate_updates=True`, the flag in the first example, also brings existing tables to the models' shape, as far as one in-place statement can: + +- A new column, a new index, a column declared `renamed_from` and a new enum label are added or renamed on both backends. +- A change of type or nullability, and a new or changed check or foreign key, run on PostgreSQL. SQLite can only make them by rebuilding the table, so there the connect raises a `UserWarning` and runs nothing ([below](#sqlite-changes-that-need-a-table-rebuild)). +- Nothing is dropped unless you also pass `migrate_destructive=True`, and a table never is. + +The [per-backend table](../../reference/migrations.md#auto-migrate-changes-per-backend) lists every change the pass makes or refuses, checks, foreign keys, defaults, index redefinitions and renames included. Rules worth knowing: -- **NOT NULL additions need a literal default.** Existing rows must get a value, so a new required field without a literal default fails the connect: +- **A new required column needs a literal default.** Existing rows must get a value, so a new required field without a literal default fails the connect: + ```text - Cannot add NOT NULL column 'author.bio' to an existing table: it has no literal default to backfill existing rows. Make the field nullable, give it a literal default, or generate a reviewed migration with `ferro migrate new`. + ValueError: Cannot add NOT NULL column 'author.bio' to an existing table: it has no literal default to backfill existing rows. Make the field nullable, give it a literal default, or generate a reviewed migration with `ferro migrate new`. ``` - A migration splits that change into an expand step, a [backfill](data-steps.md) you write, and a contract step. Json-family fields (`dict` / `list` / nested model) may use a JSON object or array — `Field(default={})` or `default_factory=dict` / `list` — as that literal. The factory is called once when the column spec is compiled, not per row: `lambda: {"id": str(uuid4())}` freezes one UUID onto every existing row, the same as writing that dict in `default=`. Postgres drops the backfill `DEFAULT` after the add. SQLite has no `DROP DEFAULT`, so on SQLite the column keeps the literal as its `DEFAULT` (same as `default="draft"`): the column is `NOT NULL` as the model says, and a raw `INSERT` that leaves it out gets the model's value. It is the one server default ferro leaves behind; a migration from `ferro migrate new` rebuilds the table without it. + Nothing was added. A migration splits that change into an expand step, a [backfill](data-steps.md) you write, and a contract step. + + - Json-family fields (`dict`, `list` or a nested model) may use a JSON object or array as that literal: `Field(default={})`, or `default_factory=dict` / `list`. + - Postgres drops the backfill `DEFAULT` after the add. + - SQLite has no `DROP DEFAULT`, so on SQLite the column keeps the literal as its `DEFAULT`, the same as `default="draft"`: the column is `NOT NULL` as the model says, and a raw `INSERT` that leaves it out gets the model's value. It is the one server default Ferro leaves behind; a migration from `ferro migrate new` rebuilds the table without it. + + !!! warning "A `default_factory` backfills one value into every row" + The factory is called once, when the column's DDL is compiled, not once per row. `default_factory=lambda: {"id": str(uuid4())}` writes the same UUID onto every existing row, as if you had written that dict in `default=`. + - **Added columns reuse the exact `CREATE TABLE` DDL**, so a database brought forward by `migrate_updates` matches one created fresh, and `alembic revision --autogenerate` stays clean afterwards. -- **Only ferro-owned constraints are rebuilt.** FK reconciliation matches the `fk_
__` names ferro emits (just as index reconciliation only touches `idx_*`/`uq_*`). A drifting constraint with any other name is left untouched and reported with a `UserWarning` — user-created schema survives auto-migrate. Rebuilding is metadata-only: rows are never touched, and the new `ADD CONSTRAINT` validates existing rows, failing loudly (and rolling back the table's plan on Postgres) if they violate it. -- **Table checks and column `db_check` share the `ck_*` prefix.** Every ferro-owned `ck_*` — table checks from `__ferro_checks__` and column checks from `Field(db_check=True)` — participates in the same reconciliation pass on PostgreSQL: missing checks are added on `migrate_updates`, same-name body drift triggers a rebuild, and orphaned ferro-owned checks drop only on `migrate_destructive`. A leftover `ck_*` that the model no longer declares stays live under `migrate_updates` and emits a `UserWarning` (silence would leave the database rejecting rows the model now allows). -- **Postgres type changes take an exclusive lock** and fail the connect if existing data does not cast cleanly — fine for a development flag, but worth knowing. Every statement waits for its table lock under the [`ddl_lock_timeout`](../../reference/configuration.md#ddl_lock_timeout) of the project configuration (default `5s`), retried up to ten times before the connect fails. +- **Only Ferro-owned constraints are rebuilt.** Foreign-key reconciliation matches the `fk_
__` names Ferro emits, as index reconciliation only touches `idx_*` and `uq_*`. A drifting constraint with any other name is left untouched and reported with a `UserWarning`, so schema you created by hand survives auto-migrate. Rebuilding is metadata-only: rows are never touched, and the new `ADD CONSTRAINT` validates existing rows, failing loudly (and rolling back the table's plan on Postgres) if they violate it. +- **Table checks and column `db_check` share the `ck_*` prefix.** Every Ferro-owned `ck_*` (table checks from `__ferro_checks__`, column checks from `Field(db_check=True)`) is reconciled the same way on PostgreSQL: a missing check is added under `migrate_updates`, a check whose body drifted is rebuilt, and an orphaned one drops only under `migrate_destructive`. A leftover `ck_*` the model no longer declares stays live under `migrate_updates` and raises a `UserWarning`, because the database would otherwise go on rejecting rows the model now allows without saying so. +- **Postgres type changes take an exclusive lock** and fail the connect if existing data does not cast cleanly. Every statement waits for its table lock under the [`ddl_lock_timeout`](../../reference/configuration.md#ddl_lock_timeout) of the project configuration (default `5s`), retried up to ten times before the connect fails. - **The pool refreshes after any schema change**, so no cached statement or stale identity-mapped instance can observe the pre-migration schema. -### SQLite: warn and skip +### SQLite changes that need a table rebuild -SQLite cannot add, change or drop a constraint, a type or a nullability on a table that exists; the only way is to create the table again in its new shape and copy the rows across (a *table rebuild*). Auto-migrate never rebuilds a table: it raises a `UserWarning` naming the object and the door that can, and changes nothing (ADR-0014). For example: +SQLite cannot add, change or drop a constraint, a type or a nullability on a table that exists; the only way is to create the table again in its new shape and copy the rows across (a *table rebuild*). Auto-migrate never rebuilds a table, because a rebuild drops whatever the copy does not carry. It raises a `UserWarning` naming the object and the door that can, and changes nothing. Declare `db_check=True` on an existing column `author.kind`, for example, and the connect warns: + ```text -Check constraint 'ck_author_kind' on column 'author.kind' is declared but missing from the live table, and SQLite cannot add a constraint to an existing column (it requires a full table rebuild). The invariant is not database-enforced; generate a reviewed migration with `ferro migrate new` to apply it. +ferro auto-migrate: Check constraint 'ck_author_kind' on column 'author.kind' is declared but missing from the live table, and SQLite cannot add a constraint to an existing column (it requires a full table rebuild). The invariant is not database-enforced; generate a reviewed migration with `ferro migrate new` to apply it. ``` -[Migrations](migrations.md#sqlite-table-rebuilds) generate the rebuild as a reviewed step. A *foreign default* (a server default on a column that no model declares, below) is the one exception: no migration plans it, since the migrations' snapshots never hold it, so the warning names the hand rebuild that drops it, or a hand-written SQL step that does (`ferro migrate new --sql-step drop_default`). Row-level security is Postgres-only on every door: SQLite gets one warning per table and no DDL. +The connect goes on, and the column stays unchecked. [Migrations](migrations.md#sqlite-table-rebuilds) generate the rebuild as a reviewed step. A *foreign default* (a server default on a column that no model declares, [below](#destructive-drops-with-migrate_destructive)) is the one exception: no migration plans it, since the migrations' snapshots never hold it, so the warning names the hand rebuild that drops it, or a hand-written SQL step that does (`ferro migrate new --sql-step drop_default`). Row-level security is Postgres-only on every door: SQLite gets one warning per table and no DDL. ## Evolving enums: label addition -*Added in 0.18.0.* On PostgreSQL, `StrEnum` fields create a **native enum type**, and a type that already exists in the database does not learn new members on its own. When a `StrEnum` grows, `migrate_updates=True` performs **label addition**: it compares the model's members against the live type and appends what's missing with `ALTER TYPE ... ADD VALUE IF NOT EXISTS`. - -!!! danger "This gap is invisible to your tests" - Under plain `auto_migrate=True` (without `migrate_updates`), an existing enum type is **never** updated — like every existing object, it belongs to the update pass. The failure mode is nasty: every test suite that creates its schema fresh gets the complete enum and stays green, while every *existing* database rejects the new member at runtime with `invalid input value for enum`. No app-side test against a throwaway schema can catch this. If your models' enums evolve, run with `migrate_updates=True`, or generate the change as a [migration](migrations.md) (the [Alembic bridge](alembic.md) sees the same drift). +On PostgreSQL, a `StrEnum` field creates a **native enum type**, and a type that already exists in the database does not learn new members on its own. When a `StrEnum` grows, `migrate_updates=True` performs **label addition**: it compares the model's members with the live type and appends what is missing. Say `Author` gains a `plan`, and `Plan` later gains `PRO`: === "Assignment" ```python - from enum import StrEnum - - import ferro - from ferro import Model - - - class Provider(StrEnum): - PLAID = "plaid" - MX = "mx" # new member — the live type only has 'plaid' - - - class Feed(Model): - id: int | None = ferro.Field(primary_key=True, default=None) - provider: Provider - - - await ferro.connect("postgres://...", migrate_updates=True) - # → ALTER TYPE "provider" ADD VALUE IF NOT EXISTS 'mx' - - feed = await Feed.create(provider=Provider.MX) - recent = await Feed.where(lambda feed: feed.provider == Provider.MX).all() + --8<-- "docs/examples/enum_labels.py:models" ``` === "Annotated" ```python - from enum import StrEnum - from typing import Annotated - - import ferro - from ferro import FerroField, Model + --8<-- "docs/examples/enum_labels_annotated.py:models" + ``` +Connect to the Postgres database with `migrate_updates=True`: - class Provider(StrEnum): - PLAID = "plaid" - MX = "mx" # new member — the live type only has 'plaid' +```python {test=run prelude=label_addition backend=postgres} +await ferro.connect("postgresql://localhost/app", migrate_updates=True) +async with engines.session(): + labels = await fetch_all("SELECT unnest(enum_range(NULL::plan))::text AS label") +print([row["label"] for row in labels]) +``` - class Feed(Model): - id: Annotated[int | None, FerroField(primary_key=True)] = None - provider: Provider +The connect runs `ALTER TYPE "plan" ADD VALUE IF NOT EXISTS 'pro'`, so the type's labels are now: + +```text +['free', 'pro'] +``` - await ferro.connect("postgres://...", migrate_updates=True) - # → ALTER TYPE "provider" ADD VALUE IF NOT EXISTS 'mx' +From then on an author can be saved with `plan=Plan.PRO`. - feed = await Feed.create(provider=Provider.MX) - recent = await Feed.where(lambda feed: feed.provider == Provider.MX).all() - ``` +!!! warning "Existing enum types without `migrate_updates`" + Under plain `auto_migrate=True`, an existing enum type is never updated: like every existing object, it belongs to the update pass. Every test suite that creates its schema fresh gets the complete enum and stays green, while every existing database rejects the new member at runtime with `invalid input value for enum`. No test against a throwaway schema can catch this. If your models' enums evolve, run with `migrate_updates=True`, or generate the change as a [migration](migrations.md) (the [Alembic bridge](alembic.md) sees the same drift). The contract, precisely: - **Append-only, metadata-only.** Label addition adds labels and does nothing else; rows are never touched. A shared `StrEnum` used by several models is one type and reconciles once. -- **Removals are never automatic here.** A live label the model no longer declares raises a `UserWarning` naming the type and labels — rows may still hold that label, and older code may still be running against the schema mid-deploy — and the label stays. A label you declare renamed with `__ferro_renamed_labels__` ([Renames](migrations.md#renames)) is renamed in place; a migration removes one with a generated [backfill and contract](data-steps.md#removing-an-enum-label). +- **Removals are never automatic here.** A live label the model no longer declares raises a `UserWarning` naming the type and labels, and the label stays: rows may still hold it, and older code may still be running against the schema mid-deploy. A label you declare renamed with `__ferro_renamed_labels__` ([Renames](renames-and-retirements.md#rename-hints)) is renamed in place; a migration removes one with a generated [backfill and contract](data-steps.md#removing-an-enum-label). - **Labels commit before table changes.** Additions run as their own autocommit statements ahead of the per-table plans, so a new column whose literal default is a brand-new member works in a single deploy, on every supported PostgreSQL version. - **Appended labels sort last.** `ADD VALUE` appends: a member inserted mid-enum in Python lands at the end of the database ordering, and `ORDER BY` on an enum column follows *database* order, not declaration order. - **SQLite is unaffected.** Enums store as text there; a new member needs no DDL. ## Destructive drops with `migrate_destructive` -*Added in 0.11.0.* Also **drop** live columns that no longer exist on the model (never whole tables): +`migrate_destructive=True` also drops live columns that no longer exist on the model, never whole tables. On `dev.db`, it drops the `bio` column the model no longer declares: -```python +```python {test=run prelude=auto_migrate} import ferro await ferro.connect("sqlite:dev.db?mode=rwc", migrate_destructive=True) +async with engines.session(): + columns = await ferro.fetch_all("PRAGMA table_info(author)") + assert [column["name"] for column in columns] == ["email", "id", "name", "nickname"] ``` -Dropping is dependency-aware and fails loudly rather than skipping silently: +Ada's `bio` is gone with its column; `nickname` was added in the same connect. Dropping is dependency-aware and fails loudly rather than skipping: -- Explicit indexes covering a dropped column are dropped first (they would be orphaned anyway). +- Explicit indexes covering a dropped column are dropped first, since they would be orphaned anyway. - Columns that are **primary keys**, enforced by table constraints, or referenced by other tables' **foreign keys** abort with an error naming the constraint and pointing at `ferro migrate new`, which writes the drop as a reviewed [migration](migrations.md). -It also drops a **foreign default**: a server default on a column that ferro did not leave there. Ferro persists no server default of its own, so a default someone set by hand (`ALTER TABLE "author" ALTER COLUMN "slug" SET DEFAULT ''`) is removed, and the `PassReport` lists the statement: +It also drops a **foreign default**: a server default on a column that Ferro did not leave there. Ferro persists no server default of its own, so a default someone set by hand (`ALTER TABLE "author" ALTER COLUMN "slug" SET DEFAULT ''`) is removed, and the `PassReport` lists the statement: ```sql ALTER TABLE "author" ALTER COLUMN "slug" DROP DEFAULT @@ -172,115 +167,76 @@ ALTER TABLE "author" ALTER COLUMN "slug" DROP DEFAULT Under `migrate_updates` alone the default stays, and the pass warns on every start: + ```text -author.slug has default '' in the database, none declared; it is left in place: connect(..., migrate_destructive=True) drops it, or run ALTER TABLE "author" ALTER COLUMN "slug" DROP DEFAULT. +ferro auto-migrate: author.slug has default '' in the database, none declared; it is left in place: connect(..., migrate_destructive=True) drops it, or run ALTER TABLE "author" ALTER COLUMN "slug" DROP DEFAULT. ``` -Ferro's own two defaults are never foreign: a `serial` key's sequence on Postgres (recognised by the sequence the column owns, so a renamed table's still counts), and on SQLite a literal default on a `NOT NULL` column (what `migrate_updates` keeps when it adds a required column, above). On SQLite any other default (an expression such as `CURRENT_TIMESTAMP`, or any default on a nullable column) is foreign, and the pass warns and leaves it (above: only a rebuild drops a default there). `ferro migrate drift` and `ferro migrate baseline` report the same defaults. +Ferro's own two defaults are never foreign: a `serial` key's sequence on Postgres (recognised by the sequence the column owns, so a renamed table's still counts), and on SQLite a literal default on a `NOT NULL` column (what `migrate_updates` keeps when it adds a required column, above). On SQLite any other default (an expression such as `CURRENT_TIMESTAMP`, or any default on a nullable column) is foreign, and the pass warns and leaves it, since only a rebuild drops a default there. `ferro migrate drift` and `ferro migrate baseline` report the same defaults. ## On-demand `migrate()` -Run the same pass explicitly on a live connection instead of at connect time: +`ferro.migrate()` runs the same pass on a connection that is already open, and returns a `PassReport`: every statement the pass sent to the database, in order, and every warning it raised. Run it on `dev.db` and read what it did: -```python +```python {test=run prelude=auto_migrate} import ferro -await ferro.migrate() # create missing tables + apply updates (default) -await ferro.migrate(destructive=True) # also drop removed columns -await ferro.migrate(using="service") # against a named connection -``` - -`ferro.create_tables()` runs only the create pass. Both are refused on a database that has run a migration, as the flags are. - -## What the pass did: `PassReport` - -`ferro.migrate()` and `ferro.create_tables()` return a `PassReport`: every statement the pass sent to the database, in order, and every warning it raised. Say `Author` gains a `slug` field: - -=== "Assignment" - - ```python - import ferro - from ferro import Model - - - class Author(Model): - id: int | None = ferro.Field(primary_key=True, default=None) - name: str - slug: str | None = None # new - ``` - -=== "Annotated" - - ```python - from typing import Annotated - - from ferro import FerroField, Model - - - class Author(Model): - id: Annotated[int | None, FerroField(primary_key=True)] = None - name: str - slug: str | None = None # new - ``` - -```python +await ferro.connect("sqlite:dev.db?mode=rwc") report = await ferro.migrate() - -[(s.subject, s.sql) for s in report.statements if s.role == "schema"] -# [('author', 'ALTER TABLE "author" ADD COLUMN "slug" varchar')] - -[(w.kind, str(w)) for w in report.warnings] -# [] +assert [(s.subject, s.sql) for s in report.statements if s.role == "schema"] == [ + ("author", 'ALTER TABLE "author" ADD COLUMN "nickname" varchar'), + ("author", 'CREATE INDEX IF NOT EXISTS "idx_author_nickname" ON "author" ("nickname")'), +] +assert report.warnings == () ``` -The report is built from what the pass actually executed, never from its plan, so it never lists a statement that did not run. +The pass created the column and its index, raised no warning, and left `bio` in place. The report is built from what the pass executed, never from its plan, so it never lists a statement that did not run. `ferro.migrate()` runs the create pass and the updates by default: -- **`statements`** is a tuple of `ExecutedStatement(subject, sql, role)`. `subject` is the table or enum type the statement belongs to. `role` is one of: - - `"schema"`: the create pass, the enum type statements and the reconciliation; - - `"lock_timeout"`: the `SET LOCAL lock_timeout` (or `SET` / `RESET`) the pass wraps each Postgres unit in, so a statement never queues behind a long lock; - - `"probe"`: the row read SQLite costs for a [label rename](#evolving-enums-label-addition) on a column with no check. -- **`warnings`** is a tuple of `Report(kind, subject, text, recurs)`; `str(warning)` is its sentence. Every warning is still raised as a `UserWarning` too. `kind` names it, so code can match on it rather than on the text: - - the planner's and renderer's kinds: `LeftoverChecks`, `ExtraEnumLabels`, `ForeignFkDrift`, `ForeignDefault`, `HintRefused`, the row-security kinds (`DroppedRowSecurity`, `ForeignPolicies`, `UnverifiablePolicy`, `PolicyBodyReplaced`, `RowSecurityTeardown`, `ExtraPolicies`), `RefusedConversion`, `SqliteInPlace`, `PrimaryKeyKept`, `EnumTypeMove` (a column moving to or from a native enum type, reported with the recipe a migration follows), `RowSecuritySkipped`; - - the pass's own: `PendingTableRename`, `StrandedLabelRename`, `RowSecurityUnderMigrator`, `RunLockWait` (it waited for the run lock) and `DdlLockRetry` (a statement timed out waiting for a table lock and its unit is retried). +- `ferro.migrate(destructive=True)` also drops removed columns, as `migrate_destructive` does (on `dev.db`, `ALTER TABLE "author" DROP COLUMN "bio"`). +- `ferro.migrate(updates=False)` runs only the create pass, as `ferro.create_tables()` does. +- `ferro.migrate(using="service")` runs against a named connection. - `recurs` is `True` for a warning raised on every pass until someone acts. +Each statement is an `ExecutedStatement` whose `role` says what it was for (`"schema"` for the DDL itself), and each warning a `Report` whose `kind` names it, so code can match on the kind rather than the sentence. Every warning is still raised as a `UserWarning` too. The [API reference](../../api/connection.md#ferro.PassReport) lists every field, role and kind. Both `migrate()` and `create_tables()` are refused on a database that has run a migration, as the flags are. A pass that fails partway raises its usual error with `.report` set to what committed before the failure, the failing statement left out. On Postgres each table is its own transaction, so earlier tables stay changed, and the report says which: -```python +```python {test=compile reason="needs a pass that fails partway on Postgres"} try: await ferro.migrate() except ferro.OperationalError as error: changed = {s.subject for s in error.report.statements if s.role == "schema"} ``` -`connect(..., auto_migrate=True)` runs the same pass and returns nothing; its failure carries the same `.report`. Each statement is also a record on the `ferro.sql` logger as it finishes, with its `sql`, `subject` and duration as attributes (see [Logging](../connections.md#logging)); the log is for watching, and the report is still the structured answer. +`changed` names every table whose changes committed. On SQLite the pass applies its statements one at a time, so a failure can leave earlier statements of the same table applied; when a change must land whole, make it a [migration](migrations.md) instead. `connect(..., auto_migrate=True)` runs the same pass and returns nothing; its failure carries the same `.report`. Each statement is also a record on the `ferro.sql` logger as it finishes, with its `sql`, `subject` and duration as attributes (see [Logging](../connections.md#logging)); the log is for watching, and the report is still the structured answer. -## Two processes at once +## Concurrent passes and the run lock Every auto-migrate pass takes the same run lock `ferro migrate up` takes, so two processes booting together never collide: the second waits, saying so with a `UserWarning`, and sees the first one's DDL: + ```text -connect(auto_migrate=…) is waiting: another ferro migration run or auto-migrate pass holds the run lock on this database. It goes on once that one finishes. +ferro auto-migrate: connect(auto_migrate=…) is waiting: another ferro migration run or auto-migrate pass holds the run lock on this database. It goes on once that one finishes. ``` -It waits up to the project's [`lock_timeout`](../../reference/configuration.md#lock_timeout) (default `30s`, the same wait `ferro migrate up` gives another run; `"0"` waits without a limit), then refuses: +It waits up to the project's [`lock_timeout`](../../reference/configuration.md#lock_timeout) (default `30s`; the setting's `"0"` waits without a limit, unlike the CLI's `--lock-timeout 0`, which refuses at once), then refuses. With `lock_timeout = "1s"`, on SQLite (whose schema is `main`): + ```text -connect(auto_migrate=…) gave up waiting for the run lock on public: another ferro migration run or auto-migrate pass held it longer than lock_timeout (30s). Nothing was applied. Wait for that run to finish and try again, or raise lock_timeout; `ferro migrate status` shows a migration run while it holds the lock. +connect(auto_migrate=…) gave up waiting for the run lock on main: another ferro migration run or auto-migrate pass held it longer than lock_timeout (1s). Nothing was applied. Wait for that run to finish and try again, or raise lock_timeout; `ferro migrate status` shows a migration run while it holds the lock. ``` -A database ferro migrations govern is refused before the pass waits at all, so a boot never sits behind a `ferro migrate up` only to be refused once it finishes. +The refused process applied nothing. A database Ferro migrations govern is refused before the pass waits at all, so a boot never sits behind a `ferro migrate up` only to be refused once it finishes. On Postgres the lock is a session-level advisory lock, so auto-migrate is refused behind a transaction-mode connection pooler; connect to the database directly to migrate. ## Safety guidance -!!! danger "Never use destructive auto-migration in production" +!!! warning "Destructive auto-migration deletes data" `auto_migrate` and its extension flags are for development and tests, while the schema is still moving. `migrate_destructive` deletes data the moment a field is removed from a model. For production, use [migrations](migrations.md): renames, data transforms and the changes SQLite can only make by rebuilding a table live there, reviewed before they run. -## See Also +## See also -- [Schema Management overview](overview.md) — the ladder and one door per database -- [Connections & Databases](../connections.md) — `connect()` options -- [Adopting migrations on an existing database](../../howto/adopting-migrations.md) — moving a database auto-migrate built onto migrations +- [Schema management overview](overview.md): the ladder and one door per database +- [Connections & Databases](../connections.md): `connect()` options +- [Adopting migrations on an existing database](adopting-migrations.md): moving a database auto-migrate built onto migrations +- [Connection API reference](../../api/connection.md#ferro.PassReport): `connect()`, `migrate()`, `create_tables()` and `PassReport` diff --git a/docs/pages/guide/schema/data-steps.md b/docs/pages/guide/schema/data-steps.md index 841439bc..f2e8c6ec 100644 --- a/docs/pages/guide/schema/data-steps.md +++ b/docs/pages/guide/schema/data-steps.md @@ -1,6 +1,10 @@ -# Data Steps and Backfills +--- +description: "Write the Python step that gives existing rows their values when a migration adds or changes a column." +--- -Some changes need more than DDL. Add a required `slug` to a table that already has authors, and every existing row needs a slug before the column can be `NOT NULL`. Only you know what that value is, so the migration holds a Python **data step** where you write it: +# Data steps and backfills + +A data step is a Python step inside a migration, for the part of a change only you can write: the value existing rows get. Add a required `slug` to a table that already has authors, and every existing row needs a slug before the column can be `NOT NULL`: === "Assignment" @@ -40,7 +44,7 @@ ok Wrote migrations/0002_author_slug/ - 02_backfill_author.py needs writing where it says todo(...); if no author needs a value, delete 0002_author_slug/ and run: ferro migrate new author_slug --no-backfill author.slug ``` -This whole page runs in [`docs/examples/migrations_backfill.py`](https://github.com/syn54x/ferro-orm/blob/main/docs/examples/migrations_backfill.py): the guard refusing, the backfill written, the recovery from a late row. +The `02_backfill_author.py` step is the one you write; the rest is generated and complete. This whole page runs in [`docs/examples/migrations_backfill.py`](https://github.com/syn54x/ferro-orm/blob/main/docs/examples/migrations_backfill.py): the guard refusing, the backfill written, the step reverted and run again. ## What the generator writes @@ -53,10 +57,10 @@ The migration is split around the rows: | `03_add_constraint` | Postgres: `ALTER TABLE "author" ADD CONSTRAINT "_ferro_notnull_author_slug" CHECK ("slug" IS NOT NULL) NOT VALID;`, which refuses every new `NULL` from this point without scanning the table. SQLite: `-- ferro: not-applicable`. | | `04_contract` | What the rows had to be prepared for, marked `-- ferro: data-dependent`. | -The contract on each dialect: +The contract on Postgres validates the staged check, sets `NOT NULL` and drops the check: ```sql --- 04_contract.up.postgres.sql +-- 0002_author_slug/04_contract.up.postgres.sql -- ferro: data-dependent ALTER TABLE "author" VALIDATE CONSTRAINT "_ferro_notnull_author_slug"; @@ -66,8 +70,12 @@ ALTER TABLE "author" ALTER COLUMN "slug" SET NOT NULL; ALTER TABLE "author" DROP CONSTRAINT "_ferro_notnull_author_slug"; ``` +Only the validation scans the table, and it blocks no writer while it does. + +SQLite cannot make a column `NOT NULL` in place, so its contract is a [table rebuild](migrations.md#sqlite-table-rebuilds): + ```sql --- 04_contract.up.sqlite.sql +-- 0002_author_slug/04_contract.up.sqlite.sql -- ferro: foreign-keys-off -- ferro: data-dependent @@ -75,18 +83,26 @@ CREATE TABLE IF NOT EXISTS "_ferro_new_author" ( "id" integer NOT NULL PRIMARY K INSERT INTO "_ferro_new_author" ("id", "name", "slug") SELECT "id", "name", "slug" FROM "author"; +DELETE FROM sqlite_sequence WHERE name = '_ferro_new_author'; + +INSERT INTO sqlite_sequence (name, seq) SELECT '_ferro_new_author', seq FROM sqlite_sequence WHERE name = 'author'; + DROP TABLE "author"; ALTER TABLE "_ferro_new_author" RENAME TO "author"; + +CREATE INDEX IF NOT EXISTS "idx_author_name" ON "author" ("name"); ``` +The new table takes over the rows, the `AUTOINCREMENT` counter and the name, and the index on `name` is built again on it. + The same split happens for a column that stops accepting `NULL` and for an [enum label rows still carry](#removing-an-enum-label). A migration with no data step has no expand or contract: its DDL is one schema step. ## Writing the backfill -The generated file: +`new` writes the backfill with a `todo(...)` where the value goes: -```python +```python {test=compile reason="a generated step file; the docs tests regenerate it and compare"} # migrations/0002_author_slug/02_backfill_author.py # Gives every existing author a value for slug. # Write each value marked todo, then run `ferro migrate up`. @@ -110,9 +126,9 @@ async def up(ctx, batch): def down(ctx): ... ``` -Until you replace the `todo(...)`, `up` refuses the migration before running any of it, naming the file and line: +The query selects only the rows that still need a value, so running the step again touches only what is left. The runner calls `up` once per batch, inside that batch's own transaction: the step opens no session of its own. -`ferro migrate up` prints: +Until you replace the `todo(...)`, `ferro migrate up` refuses the migration before running any of it, naming the file and line: ```text @@ -121,12 +137,14 @@ default · sqlite:///./app.db Write each step where it says todo(...), then run `ferro migrate up` again. Nothing was applied. ``` -Write the value in place of the `todo(...)`, then run `up` again: +Write the value in place of the `todo(...)`: -```python +```python {test=compile reason="one line of the generated backfill above; the example script runs it"} author.slug = author.name.lower().replace(" ", "-") ``` +Then `ferro migrate up` applies the four steps: + ```text default · sqlite:///./app.db @@ -136,12 +154,15 @@ default · sqlite:///./app.db 0002_author_slug 04_contract applied (0 ms) ``` -The query selects only the rows that still need a value, so running the step again touches only what is left. A `default_factory` from the standard library (`uuid.uuid4`) is pre-filled as its call instead of a `todo`. +Ada Lovelace's row now has the slug `ada-lovelace`, and the column is `NOT NULL`. A `default_factory` from the standard library (`uuid.uuid4`) is pre-filled as its call instead of a `todo`. ## Backfill or guard When you know no existing row needs a value (the table is new in production, or every writer already sets the column), skip the backfill: `--no-backfill
.` writes a **guard step** in its place. A guard is generated and complete; it changes nothing and fails the migration if a row turns out to need a value. +The two migrations side by side: + + ```text ferro migrate new author_slug ferro migrate new author_slug --no-backfill author.slug @@ -153,7 +174,9 @@ ferro migrate new author_slug ferro migrate new author_slug --n ir.json ir.json ``` -```python +Only step 02 differs. The guard step `--no-backfill` writes counts the rows that would need a value: + +```python {test=compile reason="a generated step file; the docs tests regenerate it and compare"} # migrations/0002_author_slug/02_guard_author.py # Generated by `ferro migrate new --no-backfill author.slug`: the migration claims no # existing author needs a value for slug, and this step checks it on @@ -178,11 +201,13 @@ async def up(ctx): def down(ctx): ... ``` -Pick the backfill when rows need values, the guard when you can say none do. Never delete a backfill file to skip it: the step number would be a gap, and the contract would fail on the rows. Delete the unapplied migration and regenerate it with `--no-backfill` instead (step numbers stay dense either way, ADR-0037). +On a database that holds Ada Lovelace's row, the guard refuses before the contract runs, and nothing after it is applied. + +Pick the backfill when rows need values, the guard when you can say none do. Never delete a backfill file to skip it: the step number would be a gap, and the contract would fail on the rows. Delete the unapplied migration and regenerate it with `--no-backfill` instead. A project can replace either skeleton with `_templates/backfill.py` or `_templates/guard.py` in its migrations directory, used verbatim with `{model}`, `{columns}` and `{query}` filled in (and `_templates/data_step.py` for `new --data-step`). -## A data step +## Data step declarations {#a-data-step} Every data step file defines `up` and `down`, and each carries exactly one declaration: @@ -193,46 +218,27 @@ Every data step file defines `up` and `down`, and each carries exactly one decla | `@nothing_to_reverse("")` | `down` | Reverting removes the step's record and runs nothing. | | `@irreversible("")` | `down` | A run that would revert this step refuses before reverting anything. | -An undeclared or doubly declared function is refused when the file loads, naming the file and the function. `todo("…")` marks what only a person can supply; it is read from the file before anything runs, so a migration holding one never starts. - -### Atomic or chunked +An undeclared or doubly declared function is refused when the file loads, naming the file and the function. `todo("…")` marks what only a person can supply; it is read from the file before anything runs, so a migration holding one never starts. Each declaration has its entry in the [API reference](../../api/migrations.md#data-step-declarations). -Pick `@atomic` for work small enough to hold in one transaction: the step lands whole or not at all. Pick `@chunked` for a big table: each batch commits with its **cursor** (the last row's order-key values), so an interrupted step resumes at its last committed batch and never replays a row. +### `@atomic` and `@chunked` {#atomic-or-chunked} -```python -from ferro.migrations import chunked, nothing_to_reverse +Pick `@atomic` for work small enough to hold in one transaction: the step lands whole or not at all, as the guard above does. Pick `@chunked` for a big table, as the generated backfill does: each batch commits with its **cursor** (the last row's order-key values), so an interrupted step resumes at its last committed batch and never replays a row. - -@chunked( - lambda models: models.Author.where(lambda author: author.slug == None) - .order_by(lambda author: author.id), - batch_size=1000, -) -async def up(ctx, batch): - for author in batch: - author.slug = author.name.lower().replace(" ", "-") - await author.save() - - -@nothing_to_reverse("01_expand.down.sql drops the column") -def down(ctx): ... -``` - -The query is a function of `ctx.models`. It must be ordered by columns of the model itself including its primary key, select only the rows that still need the step, and leave `limit`, `offset`, `after` and `before` to the runner; anything else is refused before the run starts, naming the fix. A many-to-many join table has no single primary key to page by: page its parent instead and change each parent's join rows inside the batch. +The `@chunked` query is a function of `ctx.models`. It must be ordered by columns of the model itself, including its primary key, select only the rows that still need the step, and leave `limit`, `offset`, `after` and `before` to the runner; anything else is refused before the run starts, naming the fix. A many-to-many join table has no single primary key to page by: page its parent instead, and change each parent's join rows inside the batch. ### Historical models A data step never sees the models in your codebase today, which will have moved on by the time someone runs an old migration. `ctx.models` holds a **historical model** for every table, built from the migration's schema snapshots: - `ctx.models.Author` reaches a model by its class name; `ctx.models.table("author_tags")` reaches any table by name, a many-to-many join table included. -- Each class is the table as it stands between the migration's expand and contract steps: the union of the previous snapshot's columns and this migration's. A column only this migration adds is nullable there, and a renamed column appears once, under its new name. +- Each class is the table as it stands between the migration's expand and contract steps: the union of the previous snapshot's columns and this migration's. A column only this migration adds is nullable there (`slug` is `str | None` in the backfill above), and a renamed column appears once, under its new name. - It carries columns only: no relations, methods or validators. A foreign key is its plain `*_id` column (`book.author_id`). -- A json-family column is typed `dict[str, Any] | list[Any]` with its `db_type`, and a column of a type Ferro cannot name is `Any`. Your own models may not declare those shapes (ADR-0004); historical models carry them because they describe what the database holds, not what a model should declare. +- A json-family column is typed `dict[str, Any] | list[Any]` with its `db_type`, and a column of a type Ferro cannot name is `Any`. Your own models may not declare those shapes; historical models carry them because they describe what the database holds, not what a model should declare. - Querying a model from your codebase inside a step raises, naming the historical model to use instead. ### The step context -`ctx` is all a data step can reach (ADR-0035): +`ctx` is all a data step can reach: | Member | What it is | | :--- | :--- | @@ -242,13 +248,11 @@ A data step never sees the models in your codebase today, which will have moved | `ctx.dialect` | `"postgres"` or `"sqlite"`: the database this run is on. | | `ctx.log` | A logger named for the step (`ferro.migrations.0002.02_backfill_author`). | -It has no way to open, commit or leave the step's transaction. A nested `ferro.transaction()` inside a step is a savepoint, as in application code. +It has no way to open, commit or leave the step's transaction, so a step cannot commit half of its work behind the runner's back. A nested `ferro.transaction()` inside a step is a savepoint, as in application code. -## When a late row arrives +## Late rows and contract recovery {#when-a-late-row-arrives} -A backfill fills the rows that exist when it runs. On a live database, a writer that does not know the column yet (code from before this release) can insert a `NULL` after the backfill and before the contract. The contract then fails loudly and says how to recover: - -`ferro migrate up` applies the first three steps and stops at the contract: +A backfill fills the rows that exist when it runs. On a live database, a writer that does not know the column yet (code from before this release) can insert a `NULL` after the backfill and before the contract. The contract then fails loudly and says how to recover. With one such row, `ferro migrate up` applies the first three steps and stops at the contract: ```text @@ -260,9 +264,7 @@ ferro migrate: 0002_author_slug/04_contract.up.sqlite.sql failed: 1 row still ha The step was rolled back; fix the file or the database and run `ferro migrate up` again to resume at it. Nothing after it ran. ``` -`down --to 0002:01` leaves the expand step applied and reverts the rest; `up` re-runs the backfill (which selects only the rows still missing a value) and the contract: - -`ferro migrate down --to 0002:01 --yes` reverts what the migration did after its first step: +`ferro migrate down --to 0002:01 --yes` leaves the expand step applied and reverts the rest: ```text @@ -276,7 +278,7 @@ down reverts, in this order: 0002_author_slug 02_backfill_author nothing to reverse: 01_expand.down.sql drops the column ``` -Then `ferro migrate up` runs the rest again: +The `slug` column and the values the backfill wrote stay. Then `ferro migrate up` runs the backfill again, which selects only the rows still missing a value, and the contract: ```text @@ -290,9 +292,17 @@ On Postgres the add-constraint step already refuses new `NULL`s, so the window i ## Removing an enum label -Removing a member from a `StrEnum` whose rows may still hold it is the same shape. The backfill pages the rows holding the removed label and asks which label each gets instead: +Removing a member from a `StrEnum` whose rows may still hold it takes the same shape. Say `Order.status` is a `Status` with the labels `paid` and `canceled`, and you delete `CANCELED = "canceled"`. `ferro migrate new drop_canceled` writes a backfill that pages the rows holding the removed label and asks which label each gets instead: + +```python {test=compile reason="a generated step file; the docs tests regenerate it and compare"} +# migrations/0002_drop_canceled/01_backfill_order.py +# Gives every existing order a value for status. +# Write each value marked todo, then run `ferro migrate up`. +# If no row needs a value, delete this unapplied migration and run +# `ferro migrate new --no-backfill order.status` for a guard step instead. +from ferro.migrations import chunked, nothing_to_reverse, todo + -```python @chunked( lambda models: models.Order.where(lambda order: order.status == "canceled") .order_by(lambda order: order.id), @@ -302,12 +312,34 @@ async def up(ctx, batch): for order in batch: order.status = type(order.status)(todo("the label to use instead of 'canceled'")) await order.save() + + +@nothing_to_reverse("the contract's down restores the label") +def down(ctx): ... +``` + +Postgres cannot drop an enum label in place, so the contract creates the type anew without it, moves the column over, drops the old type and renames the new one into its place: + +```sql +-- 0002_drop_canceled/02_contract.up.postgres.sql +-- ferro: data-dependent + +CREATE TYPE "status_new" AS ENUM ('paid'); + +ALTER TABLE "order" ALTER COLUMN "status" DROP DEFAULT; + +ALTER TABLE "order" ALTER COLUMN "status" TYPE "status_new" USING "status"::text::"status_new"; + +DROP TYPE "status"; + +ALTER TYPE "status_new" RENAME TO "status"; ``` -The contract then removes the label. Postgres cannot drop an enum label in place, so the contract creates the type anew without it, moves the column over (`ALTER COLUMN … TYPE "orderstatus_new" USING "status"::text::"orderstatus_new"`), drops the old type and renames the new one into its place; its down adds the label back. On SQLite, where labels are text in the rows, the contract is `-- ferro: not-applicable`. +A row still holding `canceled` fails the cast, so the contract cannot run before the backfill has moved every such row. Its down adds the label back with `ALTER TYPE "status" ADD VALUE IF NOT EXISTS 'canceled' AFTER 'paid'`. On SQLite, where labels are text in the rows, the contract is a [table rebuild](migrations.md#sqlite-table-rebuilds) that narrows the column to the remaining labels. -## See Also +## See also -- [Migrations](migrations.md) — the loop, headers and renames -- [Deploying migrations](deploying.md) — rolling deploys and the two-release contract -- [Testing migrations](testing.md) — seed rows through historical models and prove a backfill +- [Migrations](migrations.md): the loop, headers and renames +- [Deploying migrations](deploying.md): rolling deploys and the two-release contract +- [Testing migrations](testing.md): seed rows through historical models and prove a backfill +- [Migrations API reference](../../api/migrations.md#data-step-declarations): `atomic`, `chunked`, `irreversible`, `nothing_to_reverse`, `todo` diff --git a/docs/pages/guide/schema/deploying.md b/docs/pages/guide/schema/deploying.md index 964fde3b..3191d3c9 100644 --- a/docs/pages/guide/schema/deploying.md +++ b/docs/pages/guide/schema/deploying.md @@ -1,22 +1,26 @@ -# Deploying Migrations +--- +description: "Apply migrations in a deploy step or at application start-up, and refuse to serve a database they have not reached." +--- + +# Deploying migrations Migrations reach a database in one of two ways: a deploy step runs `ferro migrate up` before the new code starts, or the application applies them itself at start-up. Either way the application should refuse to serve a database its migrations have not reached. -## In CI +## Checks in CI `ferro migrate check` reads the models and the migrations directory and nothing else, so it needs no database and no credentials: ```text default · migrations -ok: models match 0002_author_slug +ok: models match 0003_author_slug_required ``` -It exits 3, naming each problem, when a model change has no migration, the snapshot chain is broken, a number is duplicated or missing, a step lacks a target dialect's rendering, or a data step still says `todo(...)`. Run it on every pull request. +The models match the newest migration, so the check passes. It exits 3, naming each problem, when a model change has no migration, the snapshot chain is broken, a number is duplicated or missing, a step lacks a target dialect's rendering, or a data step still says `todo(...)`. Run it on every pull request. ### A pre-commit hook -Ferro ships no pre-commit hook repository; a local hook runs the same check: +Ferro ships no pre-commit hook repository; a local hook in `.pre-commit-config.yaml` runs the same check: ```yaml # .pre-commit-config.yaml @@ -30,24 +34,26 @@ repos: pass_filenames: false ``` -## On a server +The commit is refused while `ferro migrate check` exits non-zero. + +## Servers that migrate in a deploy step The deploy step applies every pending migration, in order, under the run lock: ```text default · sqlite:///./app.db -0002_author_slug 01_expand applied (0 ms) -0002_author_slug 02_backfill_author applied (0 ms) -0002_author_slug 03_add_constraint applied (0 ms) -0002_author_slug 04_contract applied (1 ms) +0002_author_slug_nullable 01_schema applied (0 ms) +0003_author_slug_required 01_backfill_author applied (2 ms) +0003_author_slug_required 02_add_constraint applied (0 ms) +0003_author_slug_required 03_contract applied (1 ms) ``` It exits 0 when it applied everything (or there was nothing to apply), 1 when it refused or a step failed; the message says why and how to go on, and the next `up` resumes at the failed step. The URL comes from `$DATABASE_URL` (or the variable the configuration's `url_env` names), or `--url`. The application then checks at start-up that the database stands at the head of its migrations, without changing anything and without taking a lock: -```python +```python {test=compile reason="needs a deployed project: its configuration, migrations and database"} import ferro import ferro.migrations @@ -59,9 +65,10 @@ async def start() -> None: Behind its migrations, it raises `PendingMigrationsError`: + ```text -ferro.migrations.PendingMigrationsError: ferro.migrations: this database is behind its migrations. - pending 0002_author_slug +PendingMigrationsError: ferro.migrations: this database is behind its migrations. + pending 0002_author_slug_nullable Run `ferro migrate up` (or `await ferro.migrations.up()`) before serving. ``` @@ -69,9 +76,9 @@ Run `ferro migrate up` (or `await ferro.migrations.up()`) before serving. ### Concurrent pre-deploys -Two services that share one database often run their pre-deploy steps at the same time. When that database is being [adopted](../../howto/adopting-migrations.md), both pre-deploys find no migration records and both call `baseline()`. The run lock lets the first one record. The second then finds a tracked database (its tracking table holds records now) and raises `AlreadyTrackedError`. Catch it by type and go on to `up()`: +Two services that share one database often run their pre-deploy steps at the same time. When that database is being [adopted](adopting-migrations.md), both pre-deploys find no migration records and both call `baseline()`. The run lock lets the first one record. The second then finds a tracked database (its tracking table holds records now) and raises `AlreadyTrackedError`. Catch it by type and go on to `up()`: -```python +```python {test=compile reason="needs a deployed project: its configuration, migrations and database"} import ferro import ferro.migrations @@ -86,15 +93,15 @@ async def pre_deploy() -> None: await ferro.migrations.up() ``` -The target is pinned to the migration that describes what the previous release built, as in the [start-up recipe](../../howto/adopting-migrations.md#an-app-that-migrates-at-start-up). An unattended baseline must never record a step it did not run. Without a target it records every migration through the head, and a later data step would be marked applied without its backfill ever running; `up()` then applies those later migrations for real. +The target is pinned to the migration that describes what the previous release built, as in the [start-up recipe](adopting-migrations.md#an-app-that-migrates-at-start-up). An unattended baseline must never record a step it did not run. Without a target it records every migration through the head, and a later data step would be marked applied without its backfill ever running; `up()` then applies those later migrations for real. `AlreadyTrackedError` is a `MigrationRefused`, and its message is the one `ferro migrate baseline` prints. `.applied` names the recorded migrations, `.head` the newest of them, and `.report` is the database's status report. Every other refusal from `baseline()` is still a plain `MigrationRefused`: a target the directory lacks, an empty directory, a lock wait that outlasts `lock_timeout`, or a configuration that names no single database. Two `up()` calls need no such care, because the second waits for the lock and finds nothing pending. ## Apps that migrate at start-up -A desktop or local-first app that ships a SQLite file has no deploy step: it applies its own migrations when it starts. +A desktop or local-first app that ships a SQLite file has no deploy step, so it applies its own migrations when it starts: -```python +```python {test=compile reason="needs a deployed project: its configuration, migrations and database"} import ferro import ferro.migrations @@ -104,9 +111,9 @@ async def start() -> None: await ferro.migrations.up() ``` -`up()` takes the same run lock as `ferro migrate up`, so two instances starting together apply the migrations once. It raises instead of returning a refusal: an app must not go on serving a database the run did not reach. When a user opens a file a newer build migrated, it raises `DatabaseAheadError` (see below). A file that existed before the app had migrations is [baselined](../../howto/adopting-migrations.md#an-app-that-migrates-at-start-up) in the same start-up code. +`up()` takes the same run lock as `ferro migrate up`, so two instances starting together apply the migrations once. It raises instead of returning a refusal: an app must not go on serving a database the run did not reach. When a user opens a file a newer build migrated, it raises `DatabaseAheadError` (see below). A file that existed before the app had migrations is [baselined](adopting-migrations.md#an-app-that-migrates-at-start-up) in the same start-up code. -`up()` and `require_applied()` take no target: a migration is applied whole, always to the head (ADR-0040). Both take `settings=` and `database=` (which configured database; needed when several are configured) and `using=` (which open connection; the default one otherwise). +`up()` and `require_applied()` take no target: a migration is applied whole, always to the head, so an application never stops a database between two migrations. Both take `database=` when several databases are configured, and `using=` or `url=` to name the connection; the [API reference](../../api/migrations.md#running-migrations) lists every parameter. ## Rolling deploys and the two-release contract @@ -114,72 +121,74 @@ In a rolling deploy, old code keeps running while the migration applies. A migra | Release | Model | Migration | Old code during the deploy | | :--- | :--- | :--- | :--- | -| 1 | `slug: str \| None = None` | `0008_author_slug`: one schema step, `ADD COLUMN`, nullable | runs against a database one migration ahead of it | -| 2 | `slug: str` | `0009_author_slug_required`: backfill, add-constraint, contract | release 1 code already writes `slug` | +| 1 | `slug: str \| None = None` | `0002_author_slug_nullable`: one schema step, `ADD COLUMN`, nullable | runs against a database one migration ahead of it | +| 2 | `slug: str` | `0003_author_slug_required`: backfill, add-constraint, contract | release 1 code already writes `slug` | Between the two, release 0 code meets a database that has applied a migration its checkout does not have. `require_applied()` and `up()` refuse that by default: + ```text -ferro.migrations.DatabaseAheadError: ferro migrate: this database has applied 0008_author_slug, which is not in migrations/. +DatabaseAheadError: ferro migrate: this database has applied 0002_author_slug_nullable, which is not in migrations/. The directory is behind the database: check out the branch that holds it. Nothing was applied. Pass allow_ahead=True to run beside them (a rolling deploy). ``` So the old release starts with `allow_ahead="expand-only"`: -```python +```python {test=compile reason="needs a database ahead of the checkout"} await ferro.migrations.require_applied(allow_ahead="expand-only") ``` -It lets the database through when every step the old checkout lacks is an *expand* step: one the old release's writes keep succeeding beside. `0008_author_slug` adds a nullable column, so release 0 starts. A database ahead by a *contract* step is refused, naming the first one (`ferro migrate up --allow-ahead=expand-only` does the same): +It lets the database through when every step the old checkout lacks is an *expand* step: one the old release's writes keep succeeding beside. `0002_author_slug_nullable` adds a nullable column, so release 0 starts. Release 2's deploy is different: `0003_author_slug_required` ends in a *contract* step, and a database ahead by one is refused, naming the first (`ferro migrate up --allow-ahead=expand-only` does the same): + ```text -ferro.migrations.DatabaseAheadError: ferro migrate: this database has applied 0008_drop_bio, which is not in migrations/. +DatabaseAheadError: ferro migrate: this database has applied 0003_author_slug_required, which is not in migrations/. The directory is behind the database: check out the branch that holds it. -It is ahead by contract steps, starting with 0008_drop_bio:01 (contract: a write the previous release makes can fail beside it), so allow_ahead="expand-only" does not let it through. Nothing was applied. +It is ahead by contract steps, starting with 0003_author_slug_required:03 (contract: a write the previous release makes can fail beside it), so allow_ahead="expand-only" does not let it through. Nothing was applied. Pass allow_ahead=True to run beside them (a rolling deploy). ``` -`.contract_steps` lists every such step (`["0008_drop_bio:01"]`). Each step's class is decided from what the step does: - -| Expand | Contract | -| :--- | :--- | -| adding a table, or a nullable column | dropping a table or a column | -| `DROP NOT NULL` | `SET NOT NULL`: a required column's add-constraint and contract steps | -| a plain index; dropping an index, a check, a foreign key or a unique | a unique index; a type change | -| adding an enum label | renaming a column, a table, an enum type or a label; removing a label | -| a data step, a guard step | adding a check or a foreign key (`NOT VALID`) and its validate step | -| a hand-written SQL step whose file says `-- ferro: expand` | any other hand-written SQL step | - -A SQLite table rebuild takes the class of what it carries, and a step that runs nothing on a dialect (`not-applicable`) is expand there. A generated step you edit by hand is classed like a hand-written one: `contract` unless its file says `-- ferro: expand`. +`.contract_steps` lists every such step (`["0003_author_slug_required:03"]`). Each step's class is decided from what the step does: adding a nullable column is expand, making it required is contract, since a write that leaves it empty then fails. Release 1 code writes `slug` on every save, so it can run beside the contract; a deployment that knows this starts it with `allow_ahead=True` (below). The [step classes](../../reference/migrations.md#step-classes) table lists the class of every kind of step. The class is about **writes**. `expand` means the previous release's writes keep succeeding beside the step, not that its reads only meet values it knows. Adding a label is `expand`, yet release 0 reading a row that holds the new label gets a value its enum does not have. So add a label in one release and write it in the next. `allow_ahead=True` is the unbounded form: it admits any distance and any step, for a deployment that runs a fixed old version beside a newer schema on purpose. Either way the deployment says so in code; nothing in the database or the configuration turns the check off. -The class is recorded in each step's record when the step is applied, so a rolled-back checkout that lacks the files still knows what the database is ahead by. A database last migrated by ferro 0.22 has records without a class, and they read as `contract`. The first run of a newer ferro against it (`up`, `down`, `baseline` or `rerecord`) records each class from the migration files when they are the files the record applied, and `contract` where they are gone, edited or replaced since. A migration whose snapshots no longer regenerate refuses `up` and `baseline`, naming the migration, rather than recording a class the planner did not decide; `down`, `rerecord` and `baseline --remove` read no class, so they run and warn, leaving its records without one. So `"expand-only"` is answerable only after the first run since ferro was upgraded. The upgrade that added step classes changed nothing a 0.22 runner refuses, so that release can still roll back to its predecessor. The release with `squash` is different; see [Upgrading to the release with `squash`](#upgrading-to-the-release-with-squash). The one-migration shape (expand, backfill and contract together) is for deploys where writers stop, or where every writer already supplies the value. If a row slips in behind a backfill anyway, the contract fails and names [the recovery](data-steps.md#when-a-late-row-arrives). +The class is recorded in each step's record when the step is applied, so a rolled-back checkout that lacks the files still knows what the database is ahead by. -## Upgrading to the release with `squash` +The one-migration shape (expand, backfill and contract together) is for deploys where writers stop, or where every writer already supplies the value. If a row slips in behind a backfill anyway, the contract fails and names [the recovery](data-steps.md#when-a-late-row-arrives). -*Written 2026-10-18, for the first release after 0.24.* +!!! note "Databases last migrated by ferro 0.22" + Records written by ferro 0.22 carry no class, and read as `contract`. The first run of a newer ferro against such a database (`up`, `down`, `baseline` or `rerecord`) records each class from the migration files when they are the files the record applied, and `contract` where they are gone, edited or replaced since. A migration whose snapshots no longer regenerate refuses `up` and `baseline`, naming the migration, rather than recording a class the planner did not decide; `down`, `rerecord` and `baseline --remove` read no class, so they run and warn, leaving its records without one. So `"expand-only"` is answerable only after the first run since the upgrade. That upgrade changed nothing a 0.22 runner refuses, so it can still roll back; the upgrade from 0.24 to 0.25 cannot (below). -Take a team with two ferro versions on one migrations directory and one database during a rolling deploy: some instances still on 0.24, some on the new release. Two things in the new release are not readable by 0.24, so the old instances stop where the new ones carry on. + -1. **Every snapshot the new ferro writes is schema IR version 3.** A 0.24 checkout refuses the first migration it meets that carries one, with the upgrade message. Finish the rollout to the new ferro before anyone runs `ferro migrate new`. -2. **A database that inherited a squash holds rows with `origin = inherited`.** A 0.24 reader cannot parse that value, and every verb (and `require_applied` at start-up) fails with `the tracking table holds a row ferro cannot read (origin)`. No tracking-format bump is made: a squash is pre-release, so only development databases meet one, once, under the new ferro. If a 0.24 checkout does meet one, there are three ways back, each run once on SQLite: - - Check out the squash commit, which reads the inherited rows. This is the ordinary path. - - To stay on the pre-squash commit, delete every row of the tracking table and run `ferro migrate baseline` there. The live schema matches the last original's snapshot, so the baseline records every migration through it, and `up` then has nothing to apply. The earlier migrations' records become baseline records too, so their downs no longer run. - - For a throwaway development database, recreate it and run `up`. +!!! note "Upgrading from 0.24 to 0.25" + Ferro 0.25 writes two things 0.24 cannot read. During a rolling deploy with both versions on one migrations directory and one database, the 0.24 instances stop where the 0.25 ones carry on: - Deleting only the `inherited` rows and running `up` is no recovery: it re-runs the originals' DDL on a schema that already holds it, and the run fails part-way (`column bio already exists`). [ADR-0056](https://github.com/syn54x/ferro-orm/blob/main/docs/adr/0056-a-squash-is-planned-across-snapshots-and-a-database-that-ran-the-originals-inherits-it.md) records the reasoning. + 1. **Every snapshot 0.25 writes is schema IR version 3.** A 0.24 checkout refuses the first migration it meets that carries one, with the upgrade message. Finish the rollout to 0.25 before anyone runs `ferro migrate new`. + 2. **A database that inherited a [squash](../../reference/cli.md#ferro-migrate-squash-name-from-target) holds rows with `origin = inherited`.** 0.24 cannot parse that value, and every verb (and `require_applied` at start-up) fails with `the tracking table holds a row ferro cannot read (origin)`. A squash is made before a branch merges, so only development databases meet one. If a 0.24 checkout does meet one, there are three ways back, each run once on SQLite: + - Check out the squash commit, which reads the inherited rows. This is the ordinary path. + - To stay on the pre-squash commit, delete every row of the tracking table and run `ferro migrate baseline` there. The live schema matches the last original's snapshot, so the baseline records every migration through it, and `up` then has nothing to apply. The earlier migrations' records become baseline records too, so their downs no longer run. + - For a throwaway development database, recreate it and run `up`. -Before you roll out, check both places: + Deleting only the `inherited` rows and running `up` is no recovery: it re-runs the originals' DDL on a schema that already holds it, and the run fails part-way (`column bio already exists`). -```sql -SELECT origin, count(*) FROM _ferro_migrations GROUP BY origin; -``` + Before you roll out, check both places. In each database: + + ```sql + SELECT origin, count(*) FROM _ferro_migrations GROUP BY origin; + ``` -Any `inherited` row means a 0.24 binary can no longer read that database. In the repository, list which `ir.json` files are still version 1 or 2 and which carry `replaces` (a squashed migration); the version-3 ones are the files a 0.24 checkout will refuse. + Any `inherited` row means 0.24 can no longer read that database. In the repository: + + ```bash + grep -L '"ir_version": 3' migrations/*/ir.json + grep -l '^ "replaces"' migrations/*/ir.json + ``` + + The first command lists the version 1 and 2 snapshots; every `ir.json` it leaves out is one a 0.24 checkout refuses. The second lists the squashed migrations. ## Locks and timeouts @@ -194,46 +203,42 @@ Two different waits apply: ferro migrate: another ferro migration run held the run lock on this database for longer than the lock timeout (1s). Nothing was applied; run again once it has finished, or raise --lock-timeout. ``` - `--lock-timeout` bounds the wait: `30s` (the default), `500ms`, `1m`, a number of seconds, or `0` to refuse at once. The lock dies with the process that holds it, never by a timeout, so a crashed run never leaves it stuck (ADR-0029). + `--lock-timeout` bounds the wait: `30s` (the default), `500ms`, `1m`, a number of seconds, or `0` to refuse at once. The lock dies with the process that holds it, never by a timeout, so a crashed run never leaves it stuck. + - **The DDL lock timeout.** On Postgres, each statement waits for its table lock under the database's [`ddl_lock_timeout`](../../reference/configuration.md#ddl_lock_timeout) (default `5s`), so one statement queued behind a long query never queues every other query behind itself. A statement that times out is retried with its step from the first statement, each attempt a progress line. With `ddl_lock_timeout = "1s"` and another session holding `author`, a step that adds a column to it prints (Postgres only; the docs test runs this one when `FERRO_POSTGRES_URL` is set): ```text default · postgresql://app:***@localhost/app - 0002_author_slug 01_schema waiting for a lock on "author" (attempt 1 of 10, retry in 1s) - 0002_author_slug 01_schema waiting for a lock on "author" (attempt 2 of 10, retry in 2s) - 0002_author_slug 01_schema applied (2214 ms) + 0002_author_slug_nullable 01_schema waiting for a lock on "author" (attempt 1 of 10, retry in 1s) + 0002_author_slug_nullable 01_schema waiting for a lock on "author" (attempt 2 of 10, retry in 2s) + 0002_author_slug_nullable 01_schema applied (2214 ms) ``` It does not apply to data steps. -## Where a database stands +## Database status -`ferro migrate status` reads the tracking table without a lock and changes nothing. Here a contract step failed on a [late row](data-steps.md#when-a-late-row-arrives), so `0002_author_slug` is `partial` and the failed step carries its error: +`ferro migrate status` reads the tracking table without a lock and changes nothing. Here a contract step failed on a [late row](data-steps.md#when-a-late-row-arrives), so `0003_author_slug_required` is `partial` and the failed step carries its error: ```text default · sqlite:///./app.db !! 1 failed · 1 pending main._ferro_migrations - ok 0001_create_author applied 2026-10-15 04:30Z + ok 0001_create_author applied 2026-10-15 04:30Z 01_schema.up.sqlite.sql applied expand - ~~ 0002_author_slug partial, 3 of 4 steps 2026-10-15 04:30Z - 01_expand.up.sqlite.sql applied expand - 02_backfill_author.py applied expand - 03_add_constraint.up.sqlite.sql applied expand - 04_contract.up.sqlite.sql failed [data-dependent] contract - 1 row still has NULL "slug" in "author"; run ferro migrate down --to 0002:01 then ferro migrate up to re-run the backfill + ok 0002_author_slug_nullable applied 2026-10-15 04:30Z + 01_schema.up.sqlite.sql applied expand + ~~ 0003_author_slug_required partial, 2 of 3 steps 2026-10-15 04:30Z + 01_backfill_author.py applied expand + 02_add_constraint.up.sqlite.sql applied expand + 03_contract.up.sqlite.sql failed [data-dependent] contract + 1 row still has NULL "slug" in "author"; run ferro migrate down --to 0002 then ferro migrate up to re-run the backfill ``` With `--steps`, each step's [class](#rolling-deploys-and-the-two-release-contract) follows its flags. Without it, only the migrations that need a look list their steps. -| Exit code | Meaning | -| :--- | :--- | -| 0 | Everything is applied. | -| 3 | Something is pending: run `up`. | -| 4 | Something needs a person: a failed or interrupted step, an edited file, a migration applied that the directory does not have. | - -`--json` prints the same report as a JSON document; `await ferro.migrations.status()` returns it. +`status` exits 0 when everything is applied, 3 when something is pending, and 4 when something needs a person: a failed or interrupted step, an edited file, a migration applied that the directory does not have ([exit codes](../../reference/cli.md#exit-codes)). `--json` prints the same report as a JSON document; `await ferro.migrations.status()` returns it. ## Edited files and `rerecord` @@ -279,9 +284,9 @@ ferro migrate up --database billing Each database has its own migrations directory and its own lineage. In code, pass `database="billing"` to `up()`, `require_applied()`, `status()` and `check()`. The three that read the database (`up()`, `require_applied()`, `status()`) also take `using=`, naming the open connection that reaches it; `check()` reads only the models and the migrations directory, so it takes no connection. See [Multiple Databases](../../howto/multiple-databases.md). -## See Also +## See also -- [Migrations](migrations.md) — the loop and going back with `down` -- [Data steps and backfills](data-steps.md) — what a late row does to a contract -- [CLI reference](../../reference/cli.md) — every verb, flag and exit code -- [Migrations API reference](../../api/migrations.md) — `up`, `require_applied`, `status`, `check` +- [Migrations](migrations.md): the loop and going back with `down` +- [Data steps and backfills](data-steps.md): what a late row does to a contract +- [CLI reference](../../reference/cli.md): every verb, flag and exit code +- [Migrations API reference](../../api/migrations.md): `up`, `require_applied`, `status`, `check` diff --git a/docs/pages/guide/schema/migrations.md b/docs/pages/guide/schema/migrations.md index 1f439f70..08b626b1 100644 --- a/docs/pages/guide/schema/migrations.md +++ b/docs/pages/guide/schema/migrations.md @@ -1,16 +1,26 @@ +--- +description: "Write, review and apply migration files with ferro migrate new, up, status and down." +--- + # Migrations A migration is a numbered directory of steps that changes a database from one version of your models to the next. `ferro migrate new` writes it from the difference between your models and the previous migration; you read it, commit it, and `ferro migrate up` applies it. This is rung 3 of [the ladder](overview.md#the-ladder), and the door Ferro recommends for any database whose data you would mind losing. -## Install and set up +## Installation and setup {#install-and-set-up} + +Install the CLI extra: ```bash pip install "ferro-orm[cli]" ``` -`ferro migrate init` writes the [project configuration](../../reference/configuration.md) and creates the migrations directory. On a terminal it asks for what its flags do not answer (the config file, the models module, the target dialects, whether there is another database); with every answer given as a flag it asks nothing and prints what it wrote: +The `ferro` command is now on your path. `ferro migrate init` writes the [project configuration](../../reference/configuration.md) and creates the migrations directory. On a terminal it asks for what its flags do not answer (the config file, the models module, the target dialects, whether there is another database). Pass every answer as a flag and `init` asks nothing: + +```bash +ferro migrate init --models blog.models --dialects postgres,sqlite --config-file pyproject.toml +``` -`ferro migrate init --models blog.models --dialects postgres,sqlite --config-file pyproject.toml` +It prints what it wrote: ```text @@ -20,7 +30,7 @@ dialects = ["postgres", "sqlite"] Wrote [tool.ferro] to pyproject.toml and created migrations/ ``` -The questions, and what each answer means: +The project now has its configuration and an empty `migrations/`. The questions, and what each answer means: - **Config file**: `pyproject.toml` (a `[tool.ferro]` table, the default when the project has one) or `ferro.toml`. - **Models module**: the dotted module(s) defining your models. Ferro imports them to read the models; a model belongs to a database by the module that defines it. @@ -46,7 +56,7 @@ Edit a model, generate, read, apply, commit. Starting from an `Author` with `id` --8<-- "docs/examples/migrations_quickstart_annotated.py:models" ``` -`check` notices the edit, offline, and says what to run: +The `# new` comment marks the field the edit adds; `0001_create_author`, the first migration, created the table without it. `ferro migrate check` notices the edit, offline, and says what to run: ```text @@ -54,7 +64,7 @@ default · migrations ungenerated: the models changed since 0001_create_author and no migration records it (changed models: Author); run `ferro migrate new ` and commit the migration it writes ``` -`ferro migrate new author_nickname` writes the migration, one file per step and dialect, and lists what changed: +It exits 3, so a CI job running it fails until the migration exists. `ferro migrate new author_nickname` writes the migration, one file per step and dialect, and lists what changed: ```text @@ -74,7 +84,7 @@ ok Wrote migrations/0002_author_nickname/ Author ``` -`ferro migrate up` applies it, one line per step: +The column is one step and its index a second, each rendered for both dialects. Read the files, commit them, and `ferro migrate up` applies them, one line per step: ```text @@ -83,12 +93,15 @@ default · sqlite:///./app.db 0002_author_nickname 02_idx_author_nickname applied (0 ms) ``` -`new` diffs the models against the newest migration's [schema snapshot](#reading-a-migration), never against a database, so it needs no connection and gives every developer the same files. A change that renders no DDL (a Python-side default, a back-reference) writes nothing and says `no schema change: nothing written`. +The database now has the `nickname` column and its index. `new` diffs the models against the newest migration's [schema snapshot](#reading-a-migration), never against a database, so it needs no connection and gives every developer the same files. A change that renders no DDL (a Python-side default, a back-reference) writes nothing and says `no schema change: nothing written`. -The whole loop runs in [`docs/examples/migrations_quickstart.py`](https://github.com/syn54x/ferro-orm/blob/main/docs/examples/migrations_quickstart.py): `init`, `new`, `up`, `check`, `drift`, `down`, on SQLite. +The example `docs/examples/migrations_quickstart.py` in the repository runs the whole loop on SQLite: `init`, `new`, `up`, `check`, `drift`, `down`. ## Reading a migration +`new` writes one directory per migration: + + ```text migrations/ .gitattributes written once: the directory is byte-exact (no line-ending conversion) @@ -102,6 +115,8 @@ migrations/ ... ``` +Each directory holds a migration's steps and the schema snapshot it was generated from: + - **Migrations are numbered** `0001`, `0002`, …; numbers count migrations, never steps. A migration is applied whole and recorded step by step, so a failure resumes at the step that failed. - **Steps are numbered within a migration** (`01_`, `02_`, …) and share their number across dialects. A DDL step has an `.up` and a `.down` file per target dialect. Where a dialect has nothing to do, its rendering says so with `-- ferro: not-applicable`. - **A data step** is a Python file, `NN_.py`, that serves every dialect; see [Data steps and backfills](data-steps.md). @@ -111,29 +126,24 @@ The SQL is exactly what the auto-migrate passes run for the same change; Postgre ```sql -- 0001_create_author/01_schema.up.postgres.sql -CREATE TABLE IF NOT EXISTS "author" ( "id" serial PRIMARY KEY NOT NULL, "name" varchar NOT NULL ); +CREATE TABLE IF NOT EXISTS "author" ( "email" varchar NOT NULL, "id" serial PRIMARY KEY NOT NULL, "name" varchar NOT NULL ); + +CREATE UNIQUE INDEX IF NOT EXISTS "uq_author_email" ON "author" ("email"); -- 0001_create_author/01_schema.up.sqlite.sql -CREATE TABLE IF NOT EXISTS "author" ( "id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, "name" varchar NOT NULL ); +CREATE TABLE IF NOT EXISTS "author" ( "email" varchar NOT NULL, "id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, "name" varchar NOT NULL ); + +CREATE UNIQUE INDEX IF NOT EXISTS "uq_author_email" ON "author" ("email"); -- 0001_create_author/01_schema.down.postgres.sql DROP TABLE "author"; ``` -### Headers +The table and the unique index on `email` are created together, and the down drops the table. -A step file may open with `-- ferro: …` lines. The generator writes them, except `expand` and `irreversible`, which only a person writes; you read them in review. +### Step file headers -| Header | Meaning | -| :--- | :--- | -| `-- ferro: destructive` | The step discards data: it drops a column, a table or an enum type. Review is the gate; nothing refuses it. | -| `-- ferro: data-dependent` | The step discards nothing but fails on rows that do not satisfy it: a `NOT NULL`, a unique over existing rows, a type cast, the validation of a staged constraint, a removed enum label. | -| `-- ferro: no-transaction` | The runner opens no transaction around the step, so a statement such as `CREATE INDEX CONCURRENTLY` can run. The step is written to be safe to re-run from its first statement. | -| `-- ferro: foreign-keys-off` | A SQLite [table rebuild](#sqlite-table-rebuilds): the runner turns foreign-key enforcement off while the step runs. | -| `-- ferro: not-applicable` | This dialect has nothing to do at this step number. | -| `-- ferro: expand` | In a [hand-written SQL step](#hand-written-steps): its author declares it *expand*, so the previous release's writes keep succeeding beside it ([step class](deploying.md#rolling-deploys-and-the-two-release-contract)). A hand-written step without it is *contract*. The generator never writes it: a generated step's class comes from what the step does. A ferro older than this header refuses the file, as it refuses any header it does not know. | -| `-- ferro: nothing-to-reverse ` | In a `.down` file: reverting removes the step's record and runs nothing. | -| `-- ferro: irreversible ` | In a `.down` file: a run that would revert this step refuses before reverting anything. Only a person declares one. | +A step file may open with `-- ferro: …` lines that say how the runner treats it: whether it discards data, runs outside a transaction, or turns SQLite's foreign-key checks off. The generator writes them, except `expand` and `irreversible`, which only a person writes; you read them in review. The [header table](../../reference/migrations.md#step-file-headers) lists every one. A new column with an index on a table that already exists shows the shape most changes take on Postgres: the column in one step, the index in its own [index step](#index-steps) that builds without blocking writers: @@ -156,144 +166,23 @@ On a table that already exists, every index, and every unique, gets a step of it On a Postgres table that already exists, a generated migration never scans the table under a lock that blocks writers. A foreign key or a check is added `NOT VALID` first (refusing every new write that violates it) and validated in a later step; a column becoming `NOT NULL` gets a temporary `CHECK ("slug" IS NOT NULL) NOT VALID` first, and the contract step validates it, sets `NOT NULL` and drops it. [Data steps and backfills](data-steps.md#what-the-generator-writes) shows a whole one. A table the same migration creates needs none of this, and SQLite has no equivalent: there, the same changes are table rebuilds. -## Renames - -Without a hint, a renamed column is a dropped column and an added one, and `new` writes exactly that, marked destructive, and says what it saw. With `nickname` renamed to `handle` and no hint, `ferro migrate new rename_writer` ends with the line that names the fix: - - -```text -default · migrations -ok Wrote migrations/0002_rename_writer/ - 01_schema.up.postgres.sql schema step, up (postgres) - 01_schema.down.postgres.sql schema step, down (postgres) - 01_schema.up.sqlite.sql schema step, up (sqlite) - 01_schema.down.sqlite.sql schema step, down (sqlite) - 02_idx_author_handle.up.postgres.sql schema step, up (postgres) - 02_idx_author_handle.down.postgres.sql schema step, down (postgres) - 02_idx_author_handle.up.sqlite.sql schema step, up (sqlite) - 02_idx_author_handle.down.sqlite.sql schema step, down (sqlite) - ir.json snapshot - - 1 changed model - Author - - author: if "nickname" became "handle", declare renamed_from="nickname" -``` - -With a hint, it writes a rename and every row keeps its value. A hint says what a column, a table or an enum label was called before: - -=== "Assignment" - - ```python - --8<-- "docs/examples/migrations_rename.py:models" - ``` - -=== "Annotated" - - ```python - --8<-- "docs/examples/migrations_rename_annotated.py:models" - ``` - -(The `# before:` and `# new` comments show the previous version of the models; they are not part of the declaration.) - -```sql --- 0002_rename_writer/01_schema.up.postgres.sql -ALTER TABLE "author" RENAME TO "writer"; - -ALTER TABLE "writer" RENAME COLUMN "nickname" TO "handle"; - -ALTER INDEX "idx_author_nickname" RENAME TO "idx_writer_handle"; - -ALTER TYPE "status" RENAME VALUE 'canceled' TO 'cancelled'; -``` - -- `Field(renamed_from="nickname")` renames a column; on a relation, `ForeignKey(..., renamed_from="writer")` renames its `*_id` column. -- `__ferro_renamed_from__ = "author"` on a model renames its table, and every index and constraint named after it. -- `__ferro_renamed_labels__ = {"cancelled": "canceled"}` on a `StrEnum` maps each new label to its old one. On SQLite, where labels are text in the rows, the rows are rewritten in a table rebuild. -- An enum *type*'s rename needs no hint: it is read off the columns that moved to it. - -A hint is live only while the previous snapshot still holds the old name and lacks the new one. Once its migration is generated it is inert and may be deleted. A hint whose old name is still declared, or two hints naming one old name, is refused. Auto-migrate honours the column and label hints but not a table's: its create pass builds the new table name as a new, empty table ([what each door covers](overview.md#what-each-door-covers)). - -The rename example runs end to end, including the revert, in [`docs/examples/migrations_rename.py`](https://github.com/syn54x/ferro-orm/blob/main/docs/examples/migrations_rename.py). - -A rename hint renames in one migration, so the previous release's code, still naming `nickname`, fails the moment it runs. When old and new code serve side by side (a rolling deploy), rename online instead, over three releases: add the new column, write both in app code and copy the existing rows across in a [data step](data-steps.md#a-data-step); [retire](#retiring-a-column) the old one; delete its declaration, which drops it. - -## Retiring a column - -Dropping a column in the release that stops using it breaks the release still serving: its code names the column in every `INSERT` and `UPDATE`, so it fails from the moment the drop runs until it is replaced. A column leaves in two releases instead. In the first, declare it `retired`: - -=== "Assignment" - - ```python - --8<-- "docs/examples/migrations_retire.py:models" - ``` - -=== "Annotated" +## Backfills - ```python - --8<-- "docs/examples/migrations_retire_annotated.py:models" - ``` - -From this release on `bio` is gone from the model: `author.bio` raises `AttributeError`, `Author(bio="x")` raises `TypeError`, a query, `order_by`, `update()` or `save(only=...)` naming it fails with "retired in this release", and no `SELECT`, `INSERT` or `UPDATE` the model issues names it. The column stays in the schema snapshot, nullable, with its index, so the one change is the relaxation the previous release needs to keep writing beside this one: - -```sql --- 0002_retire_bio/01_schema.up.postgres.sql -ALTER TABLE "author" ALTER COLUMN "bio" DROP NOT NULL; -``` - -On SQLite that is a [table rebuild](#sqlite-table-rebuilds). Retiring a column that already accepts `NULL` writes nothing: `new` says "no schema change" and `check` is clean. In the next release, delete the declaration: `new` writes the `DROP COLUMN`, marked destructive, and, when it was the last column of a Postgres enum type, the `DROP TYPE` after it. - -| Release | The model | The migration | Old code beside it | -|---|---|---|---| -| A | `bio: str = Field(index=True)` | creates `bio` | | -| B | `bio: str \| None = Field(index=True, retired=True)` | `DROP NOT NULL` | A writes `bio`; works | -| C | no `bio` | `DROP COLUMN "bio"` (destructive) | B never names `bio`; works | - -- `ForeignKey(retired=True)` retires a relation and its `*_id` column; its foreign key constraint stays until the column is dropped. It needs no `related_name`, and a `BackRef` reading it back is refused: delete it. -- Nothing else about the column changes while it is retired: its index, unique, check and foreign key, and every table check, row policy, composite index or unique that names it, stay as declared. -- A [historical model](data-steps.md#historical-models) sees a retired column as an ordinary nullable column, so a data step can still read it. -- Auto-migrate creates the column on a fresh database and, under `migrate_updates`, relaxes its `NOT NULL`; it never drops it. Alembic autogenerate writes the same `alter_column(..., nullable=True)`. -- Refused at class definition, each naming the fix: retiring a field whose annotation does not allow `None` (declare it `str | None`), a field declared `nullable=False` (a retired column stays nullable), a primary key (drop `retired=True` and change the key first, see [Changing a primary key](#changing-a-primary-key)), a field that also declares `renamed_from` (retire it or rename it, not both), a `BackRef` or a `ManyToMany`. - -The example runs all three releases end to end in [`docs/examples/migrations_retire.py`](https://github.com/syn54x/ferro-orm/blob/main/docs/examples/migrations_retire.py). - -## Retiring an enum label - -Removing a label in the release that stops writing it breaks the release still serving: its code may still write the label, and a removed label is gone from the Postgres type and from any `db_check`. Stop writing the label first, and remove it a release later. In the first release, declare it retired on the `StrEnum`, beside [`__ferro_renamed_labels__`](#renames): - -=== "Assignment" +Adding a required column to a table with rows, making a column required, or removing an enum label that rows still hold cannot be done by DDL alone: someone has to say what the existing rows get. `new` splits such a migration into an **expand step** (the new column, nullable), a **backfill** per model (a Python data step you write), and a **contract step** (the `NOT NULL`, the label removal), and `up` refuses the migration until the backfill is written. When you know no row needs a value, `--no-backfill
.` writes a **guard step** instead, which fails the migration if one does. Both are on [Data steps and backfills](data-steps.md). - ```python - --8<-- "docs/examples/migrations_retire_label.py:models" - ``` +## Renames and retirements -=== "Annotated" +A column, a table or an enum label renamed in the models is a drop and an add unless you declare a rename hint, and a column or a label leaves over two releases so the release still serving keeps working. [Renaming and retiring columns and labels](renames-and-retirements.md) shows both. - ```python - --8<-- "docs/examples/migrations_retire_label_annotated.py:models" - ``` +## Hand-written steps -The member stays, so a row that holds `canceled` still loads as `Status.CANCELED`. Writing the label is refused, at construction and by attribute assignment, with a validation error: +`new` can add a step for what the generator does not write. Add a SQL step: -```text -Order(status="canceled") - Status.CANCELED ('canceled') is retired in this release: the previous release may still write it, this one does not +```bash +ferro migrate new seed --sql-step seed_admins ``` -A row loaded with the label can be saved after another field changes: loading does not validate, and `save()` writes every column, so the unchanged label goes back as it was. That is not a write of the label. - -Retiring a label is not a schema change. The Postgres type and any `db_check` keep listing it, so `new` says "no schema change", writes nothing, and `check` is clean. In the next release, delete the member and the declaration together: `new` then writes the ordinary [label removal](data-steps.md#what-the-generator-writes), the backfill for rows that still hold it, and on Postgres the type swap. - -Each label in `__ferro_retired_labels__` must be a member's value: a set of anything else, or a label that names no member, raises `TypeError` at class definition. The example runs in [`docs/examples/migrations_retire_label.py`](https://github.com/syn54x/ferro-orm/blob/main/docs/examples/migrations_retire_label.py). - -## Changes existing rows need a value for - -Adding a required column to a table with rows, making a column required, or removing an enum label that rows still hold cannot be done by DDL alone: someone has to say what the existing rows get. `new` splits such a migration into an **expand step** (the new column, nullable), a **backfill** per model (a Python data step you write), and a **contract step** (the `NOT NULL`, the label removal), and `up` refuses the migration until the backfill is written. When you know no row needs a value, `--no-backfill
.` writes a **guard step** instead, which fails the migration if one does. Both are on [Data steps and backfills](data-steps.md). - -## Hand-written steps - -`new` can add a step for what the generator does not write: - -`ferro migrate new seed --sql-step seed_admins` writes a SQL step beside the snapshot: +It writes the step beside the snapshot: ```text @@ -304,7 +193,13 @@ ok Wrote migrations/0002_seed/ ir.json snapshot ``` -`ferro migrate new fix_slugs --data-only --data-step Author` writes a data step and nothing else: +The two SQL files are yours to write, and the snapshot is a copy of the previous one. Add a data step and nothing else: + +```bash +ferro migrate new fix_slugs --data-only --data-step Author +``` + +It writes one Python step and a copy of the snapshot: ```text @@ -314,6 +209,8 @@ ok Wrote migrations/0003_fix_slugs/ ir.json snapshot ``` +The data step says `todo("write this step")` where your code goes. The flags: + - `--sql-step ` adds `NN_.up.sql` / `.down.sql` without a dialect suffix: one file serves every dialect, so write portable SQL. The files start as `-- write this step`. - `--data-step ` adds a Python data step over that model (`NN_backfill_.py`), with `todo("write this step")` in its `up` and `down`. - `--data-only` writes only the data step: no DDL, and a copy of the previous snapshot. @@ -322,46 +219,39 @@ ok Wrote migrations/0003_fix_slugs/ ## SQLite table rebuilds -SQLite cannot add a constraint, change a type or nullability, or drop a constraint on a table that exists. A migration does it the only way SQLite allows: it creates the table again in its new shape under a temporary name, copies the rows, drops the old table and renames the new one into place, inside the step whose change needs it: - -```sql --- 0002_author_slug/04_contract.up.sqlite.sql --- ferro: foreign-keys-off --- ferro: data-dependent - -CREATE TABLE IF NOT EXISTS "_ferro_new_author" ( "id" integer NOT NULL PRIMARY KEY AUTOINCREMENT, "name" varchar NOT NULL, "slug" varchar NOT NULL ); +SQLite cannot add a constraint, change a type or nullability, or drop a constraint on a table that exists. A migration does it the only way SQLite allows, inside the step whose change needs it: -INSERT INTO "_ferro_new_author" ("id", "name", "slug") SELECT "id", "name", "slug" FROM "author"; +1. It creates the table again in its new shape, under a temporary name. +2. It copies the rows and the table's `AUTOINCREMENT` counter across. +3. It drops the old table and renames the new one into place. +4. It recreates the table's indexes. -DROP TABLE "author"; +[Data steps and backfills](data-steps.md#what-the-generator-writes) quotes a whole rebuild, the contract that makes `author.slug` required. -ALTER TABLE "_ferro_new_author" RENAME TO "author"; -``` - -The copy carries only what the schema snapshot declares, so before a rebuild runs, the runner refuses a live table that holds anything else, naming each object and its fix. Here `Post.title` became required while the table holds an index and a trigger ferro never declared; the steps before the rebuild have run, the rebuild has not: +The copy carries only what the schema snapshot declares, so before a rebuild runs, the runner refuses a live table that holds anything else, naming each object and its fix. Make `nickname` required (`nickname: str = Field(index=True)`) on a database whose `author` table also holds an index and a trigger ferro never declared, generate the change with `ferro migrate new nickname_required --no-backfill author.nickname`, and `ferro migrate up` stops at the rebuild: ```text default · sqlite:///./app.db -0002_post_title_required 01_guard_post applied (2 ms) -0002_post_title_required 02_add_constraint applied (0 ms) -ferro migrate: a SQLite rebuild of table "post" copies only the columns and recreates only the indexes the schema snapshot declares, and the live table also holds: +0003_nickname_required 01_guard_author applied (0 ms) +0003_nickname_required 02_add_constraint applied (0 ms) +ferro migrate: a SQLite rebuild of table "author" copies only the columns and recreates only the indexes the schema snapshot declares, and the live table also holds: - index "my_idx": drop it, or declare it on the model in a migration (ferro builds the indexes it declares, named idx_/uq_) - trigger "t": drop it; a rebuild's DROP TABLE would drop it, and ferro cannot carry it across Nothing was applied. ``` -An undeclared column is refused the same way (`declare it on the model in a migration, or drop it`). +The steps before the rebuild have run; the rebuild has not, and the table is as it was. An undeclared column is refused the same way (`declare it on the model in a migration, or drop it`). + +## `check`, `status` and `drift` -## Checking +Three verbs tell you where the models, the directory and a database stand: -| Command | Needs a database | What it answers | -| :--- | :--- | :--- | -| `ferro migrate check` | no | Does every model change have a migration, and is the directory intact? Exit 0, or 3 naming each problem: an ungenerated change, a broken snapshot chain, a duplicate or missing number, a step missing a target dialect's rendering, an unwritten step. | -| `ferro migrate status` | yes | Which migrations this database has applied. Exit 0 up to date, 3 pending, 4 needs attention. | -| `ferro migrate drift` | yes | Does the live schema match the snapshot of the last migration applied? Exit 0, or 4 with one line per difference. | +- `ferro migrate check` needs no database. It answers whether every model change has a migration and the directory is intact: no broken snapshot chain, no duplicate or missing number, no step missing a target dialect's rendering, no unwritten step. `check` belongs in CI and in a [pre-commit hook](deploying.md#a-pre-commit-hook). +- `ferro migrate status` reads a database and lists which migrations it has applied ([Database status](deploying.md#database-status)). +- `ferro migrate drift` reads a database and compares its live schema with the snapshot of the last migration applied. -With the index `up` built dropped by hand, `ferro migrate drift` exits 4 and lists the difference: +Drop by hand the index `up` built, and `ferro migrate drift` lists the difference: ```text @@ -370,11 +260,17 @@ xx 1 difference against 0002_author_nickname - idx_author_nickname index is missing ``` -Drift is reported, never repaired: write the fix as a migration, or put the object back. Tables the migrations never declared are not drift. `check` belongs in CI and in a [pre-commit hook](deploying.md#a-pre-commit-hook). +It exits 4. Drift is reported, never repaired: write the fix as a migration, or put the object back. Tables the migrations never declared are not drift. The [CLI reference](../../reference/cli.md#exit-codes) lists every verb's exit codes. + +## Reverting with `down` -## Going back +`ferro migrate down` prints what it will revert and, on a terminal, asks `Revert 0002_author_nickname (2 steps)? [y/N]`. Skip the question with `--yes`: -`ferro migrate down` prints what it will revert and, on a terminal, asks `Revert 0002_author_nickname (2 steps)? [y/N]`. With `--yes` it goes straight on: +```bash +ferro migrate down --yes +``` + +It reverts the latest migration, newest step first: ```text @@ -386,222 +282,96 @@ down reverts, in this order: 0002_author_nickname 01_schema reverted (0 ms) ``` +The `nickname` column and its index are gone, and `0002_author_nickname` is pending again. + - With no flag, `down` reverts the latest applied migration; `--to 0005` leaves `0005` fully applied, `--to 0007:02` leaves steps `01`–`02` of `0007` applied, `--to 0000` (or `--all`) reverts everything. -- It prints the plan and asks first. `--yes` skips the question; without a terminal and without `--yes` it prints the plan and refuses. +- Without a terminal and without `--yes`, it prints the plan and refuses. - A down restores the schema of the previous snapshot, never the rows a drop removed. A step declared irreversible stops the whole run before anything is reverted. -- A migration that `baseline` recorded created nothing on that database, so `down` refuses to go below it ([Adopting migrations](../../howto/adopting-migrations.md#undoing-a-baseline)). +- A migration that `baseline` recorded created nothing on that database, so `down` refuses to go below it ([Adopting migrations](adopting-migrations.md#undoing-a-baseline)). -## Editing a migration +## Edited migrations A migration no database has applied is yours to change: edit it, or delete its directory and run `new` again. Once a database has applied a step, `up` checks the step's file against the checksum it recorded, and an edited file is refused; see [Edited files and `rerecord`](deploying.md#edited-files-and-rerecord). -## Two branches, one number +## Duplicate migration numbers and `--replace` -Your branch adds `bio` to `Author` and generates `0002_add_x`, with a hand-written step `02_fix_rows` you filled in. Meanwhile a teammate merges `0002_add_y`, which adds `nick`. Both were generated on top of `0001`, so after you rebase, the directory holds two `0002`s, and every command refuses it: +A teammate merges `0002_author_nickname`, the migration the loop above wrote. Meanwhile your branch added `bio` to `Author` and generated `0002_author_bio`, with a hand-written SQL step `02_fix_rows`. Both were generated on top of `0001`, so after you rebase, the directory holds two `0002`s, and every command refuses it. `ferro migrate check` says: ```text default · migrations -duplicate_number: two migrations are numbered 0002: 0002_add_x and 0002_add_y. Keep the one that is applied anywhere and regenerate the other on top of it: - ferro migrate new add_x --replace 0002_add_x - ferro migrate new add_y --replace 0002_add_y +duplicate_number: two migrations are numbered 0002: 0002_author_bio and 0002_author_nickname. Keep the one that is applied anywhere and regenerate the other on top of it: + ferro migrate new author_bio --replace 0002_author_bio + ferro migrate new author_nickname --replace 0002_author_nickname (each line regenerates the migration it names) ``` -`0002_add_y` is on main, so keep it and regenerate yours: +`0002_author_nickname` is on main, so keep it and regenerate yours: + +```bash +ferro migrate new author_bio --replace 0002_author_bio +``` -`ferro migrate new add_x --replace 0002_add_x` writes: +It writes the migration again under the next number: ```text default · migrations -ok Wrote migrations/0003_add_x/ (replaces 0002_add_x, now deleted) - 01_schema.up.postgres.sql schema step, up (postgres) moved (was 0002_add_x/01_schema.up.postgres.sql) - 01_schema.down.postgres.sql schema step, down (postgres) moved (was 0002_add_x/01_schema.down.postgres.sql) - 01_schema.up.sqlite.sql schema step, up (sqlite) moved (was 0002_add_x/01_schema.up.sqlite.sql) - 01_schema.down.sqlite.sql schema step, down (sqlite) moved (was 0002_add_x/01_schema.down.sqlite.sql) - 02_fix_rows.up.sql SQL step, up carried, review it (was 0002_add_x/02_fix_rows.up.sql) - 02_fix_rows.down.sql SQL step, down carried, review it (was 0002_add_x/02_fix_rows.down.sql) - ir.json snapshot new snapshot, chained from 0002_add_y +ok Wrote migrations/0003_author_bio/ (replaces 0002_author_bio, now deleted) + 01_schema.up.postgres.sql schema step, up (postgres) moved (was 0002_author_bio/01_schema.up.postgres.sql) + 01_schema.down.postgres.sql schema step, down (postgres) moved (was 0002_author_bio/01_schema.down.postgres.sql) + 01_schema.up.sqlite.sql schema step, up (sqlite) moved (was 0002_author_bio/01_schema.up.sqlite.sql) + 01_schema.down.sqlite.sql schema step, down (sqlite) moved (was 0002_author_bio/01_schema.down.sqlite.sql) + 02_fix_rows.up.sql SQL step, up carried, review it (was 0002_author_bio/02_fix_rows.up.sql) + 02_fix_rows.down.sql SQL step, down carried, review it (was 0002_author_bio/02_fix_rows.down.sql) + ir.json snapshot new snapshot, chained from 0002_author_nickname 1 changed model Author ``` -`0003_add_x` is what `new` would have written had `0002_add_y` been there first: a new schema snapshot chained from `0002_add_y`'s, and the DDL regenerated from the difference, so it adds `bio` and nothing of `nick`. +`0003_author_bio` is what `new` would have written had `0002_author_nickname` been there first: a new schema snapshot chained from `0002_author_nickname`'s, and the DDL regenerated from the difference, so it adds `bio` and nothing of `nickname`. - **What is carried.** Every data step (`NN_.py`) and hand-written SQL step (`NN_.up.sql` / `.down.sql`, headers included) keeps its body byte for byte, under the number its place in the regenerated migration gives it. A carried step wins over a step of the same name the regeneration scaffolds again (a backfill's `todo`), and the line says so. The DDL steps, the files with a dialect suffix, are regenerated. Read every step marked `carried, review it`: it was written against the old parent. -- **The number.** It is kept when it is free, and the next one is taken when it is not. A lone `0002_add_x` whose parent changed is regenerated in place as `0002_add_x`. +- **The number.** It is kept when it is free, and the next one is taken when it is not. A lone `0002_author_bio` whose parent changed is regenerated in place as `0002_author_bio`. - **Naming the migration.** `--replace` takes `NNNN_`, or `NNNN` when one directory carries the number. A number two directories share is refused, naming both. - **Only the newest.** A migration a later one chains from is refused: `replace the later one first, or delete it`. A regeneration that holds nothing (the models add nothing it does not already hold, and nothing is carried) is refused too: delete the directory. - **Offline.** Like `new`, it reads the models and the directory, never a database; `--url` is refused. `--sql-step`, `--data-step`, `--data-only` and `--no-backfill` mean what they mean to `new`. -- **Interrupted.** Until the new directory is written, the old one waits under a hidden name: `.0002_add_x.0003_add_x.replaced` (`.0002_add_x.replaced` when the number is kept). A run killed in between leaves it there, and every command refuses the directory, naming it and what to do: rename it back to `0002_add_x` when the replacement was never written, delete it when the replacement is complete. +- **Interrupted.** Until the new directory is written, the old one waits under a hidden name: `.0002_author_bio.0003_author_bio.replaced` (`.0002_author_bio.replaced` when the number is kept). A run killed in between leaves it there, and every command refuses the directory, naming it and what to do: rename it back to `0002_author_bio` when the replacement was never written, delete it when the replacement is complete. -A database that applied the old `0002_add_x` meets the replaced migration as a changed one. `up` refuses it, naming the replacement as the second cause: +A database that applied the old `0002_author_bio` meets the replaced migration as a changed one. `ferro migrate up` refuses it, naming the replacement as the second cause: ```text default · sqlite:///./app.db -ferro migrate: this database has applied 0002_add_x, which is not in migrations/. +ferro migrate: this database has applied 0002_author_bio, which is not in migrations/. The directory is behind the database: check out the branch that holds it, or 0002 was replaced: revert it with the files that were applied (check out the commit that applied it, `ferro migrate down --to 0001`), then `up`. Nothing was applied. ``` -When the number was free and the migration was regenerated in place, the same cause follows the snapshot refusal (`0002_add_x/ir.json is not the snapshot this database applied 0002_add_x under`). Either way, a down needs the files that went up: +Nothing was applied. When the number was free and the migration was regenerated in place, the same cause follows the snapshot refusal (`0002_author_bio/ir.json is not the snapshot this database applied 0002_author_bio under`). Either way, a down needs the files that went up: ```bash -git checkout +git checkout ferro migrate down --to 0001 git checkout - ferro migrate up ``` -## Squashing an epic's migrations - -An epic's branch collects one migration per task. Main stands at `0004_author_nick`, and the teams epic has added four on top of it: - -- `0005_add_teams` creates `Team`, with a `tmp_code` column one task needed for a while; -- `0006_team_slug` adds a required `slug`, with the backfill `02_backfill_team.py` written by hand; -- `0007_drop_tmp` drops `tmp_code` again; -- `0008_fix_rows` is a hand-written SQL step, `fix_rows`, that trims team names. - -Four migrations, three of them `ALTER TABLE`s on a table the first one created, is the history of the work, not the change main needs. Before the epic's PR to main, replace them with the one migration `new` would have written across them: - -`ferro migrate squash teams_epic --from 0005` writes: - - -```text -default · migrations -ok Wrote migrations/0005_teams_epic/ (replaces 0005_add_teams … 0008_fix_rows, now deleted) - 01_schema.up.postgres.sql schema step, up (postgres) regenerated - 01_schema.down.postgres.sql schema step, down (postgres) regenerated - 01_schema.up.sqlite.sql schema step, up (sqlite) regenerated - 01_schema.down.sqlite.sql schema step, down (sqlite) regenerated - 02_fix_rows.up.sql SQL step, up carried, review it (was 0008_fix_rows/01_fix_rows.up.sql) - 02_fix_rows.down.sql SQL step, down carried, review it (was 0008_fix_rows/01_fix_rows.down.sql) - 03_backfill_team.py data step carried, review it (was 0006_team_slug/02_backfill_team.py) - ir.json snapshot 0008_fix_rows's, chained from 0004_author_nick, replaces 0005_add_teams … 0008_fix_rows - - 1 new model - Team - - team.tmp_code existed only inside the range (0005_add_teams through 0007_drop_tmp): review the carried steps for it -``` - -`0005_teams_epic` creates `team` with its `slug` in one `CREATE TABLE`; `tmp_code` never appears. The four directories are gone. A squash is a one-way step for older ferro releases; read [Upgrading to the release with `squash`](deploying.md#upgrading-to-the-release-with-squash) before rolling it out beside them. - -- **What is regenerated.** The DDL, every file with a dialect suffix, is planned once, from `0004_author_nick`'s snapshot to `0008_fix_rows`'s, by the planner `new` uses. A squash reads those snapshots and the directory: it never imports the models (an ungenerated model change is still `new`'s to write) and never opens a database (`--url` is refused). Its `ir.json` is the last snapshot under a new parent, and it records which migrations it replaced: the squash's *witness*. -- **What is carried.** Every data step and hand-written SQL step of the range keeps its body byte for byte, in chain order, placed as `--replace` places it (a SQL step before the data steps, a data step before the contract), and is marked `carried, review it` with the file it was. Read each one: `fix_rows` now runs before the backfill it used to follow, and a step written after one task's contract now runs before the one contract. A backfill still byte for byte as the generator scaffolded it, `todo(...)` and all, that the squashed plan does not scaffold again is dropped with a note (`02_backfill_team.py from 0006_team_slug was the generator's unwritten scaffold and the squashed plan needs no backfill of team.slug: dropped`); a step with any hand-written byte is always carried, and a carried unwritten step still refuses to load until it is written. -- **The dead-name note.** A table, column, enum label or constraint the range added and then removed (`team.tmp_code` here) does not exist in the squashed migration, so a carried step that touches it now fails. Where a carried step's declaration proves it, the squash refuses and writes nothing: a `@chunked` step whose query names the dead name in its model, order keys or filter columns or labels. Had `02_backfill_team.py` selected its rows by `tmp_code` (`lambda team: team.tmp_code == None`), the squash would say: - - - ```text - default · migrations - ferro migrate squash: 0006_team_slug/02_backfill_team.py names team.tmp_code, which existed only inside the range (added by 0005_add_teams, removed by 0007_drop_tmp): delete the step, or start the range at 0007_drop_tmp, the migration that removed it (pass --from 0007_drop_tmp). Nothing was written. - ``` - - An `@atomic` step declares nothing the squash can read, and a SQL step is SQL, so neither is ever refused: each dead name gets the note above instead, and reading the carried steps for it is yours. -- **The range.** It is always a tail, `--from` through the newest migration, because the migration after a middle range would chain from a snapshot that no longer exists. The squash takes the first migration's number and the name you give it. A name that would rewrite a replaced migration's own directory (`ferro migrate squash add_teams --from 0005` writing `0005_add_teams`) is refused, since every reader would then take the squash for the original. A range that adds nothing and carries nothing is refused: delete the directories. A migration that shares the first number of the range, because two branches each generated it, is refused too: resolve the shared number with `--replace` ([two branches, one number](#two-branches-one-number)) before squashing. The [CLI reference](../../reference/cli.md#ferro-migrate-squash-name-from-target) lists every flag and refusal, and what to do with the hidden directory a squash killed halfway leaves. - -### A rename inside the range - -A column renamed in one task, whose hint a later task deleted, is still a rename after the squash. Task one renames `nickname` to `handle` and generates `0002_rename_handle`: - -=== "Assignment" - - ```python - --8<-- "docs/examples/migrations_squash.py:models" - ``` - -=== "Annotated" - - ```python - --8<-- "docs/examples/migrations_squash_annotated.py:models" - ``` - -Its migration generated, the hint is inert, so task two deletes it and adds `bio` (`0003_add_bio`): - -=== "Assignment" - - ```python - --8<-- "docs/examples/migrations_squash.py:later" - ``` - -=== "Annotated" - - ```python - --8<-- "docs/examples/migrations_squash_annotated.py:later" - ``` - -Planned from `0001` to `0003`'s models alone, `nickname` would be dropped and `handle` added, and every author's nickname lost. The squash composes the hints of every snapshot in the range into its own, so `ferro migrate squash author_profile --from 0002` writes one rename: - -```sql --- 0002_author_profile/01_schema.up.postgres.sql -ALTER TABLE "author" RENAME COLUMN "nickname" TO "handle"; - -ALTER TABLE "author" ADD COLUMN "bio" varchar; -``` - -`a→b` then `b→c` is `a→c`; a renamed column the range later drops is a drop of the old name; a column added and then renamed is an add of the new name. A rename whose old name a later task adds again cannot be one migration, and is refused naming the `--from` that starts the range at the task that adds it again, leaving the rename below the squash. The example runs end to end, inheritance and revert included, in [`docs/examples/migrations_squash.py`](https://github.com/syn54x/ferro-orm/blob/main/docs/examples/migrations_squash.py). - -### Every database that applied the originals inherits the squash - -Your epic database applied `0005_add_teams` through `0008_fix_rows`. After you pull the squash, `status` shows the one migration it has no records for, and what `up` will do with it: - - -```text -default · sqlite:///./app.db --- 1 pending main._ferro_migrations - ok 0001_create_author applied 2026-10-15 04:31Z - ok 0002_author_email applied 2026-10-15 04:31Z - ok 0003_author_bio applied 2026-10-15 04:31Z - ok 0004_author_nick applied 2026-10-15 04:31Z - -- 0005_teams_epic pending (inheritable) run `ferro migrate up`; this database applied 0005_add_teams … 0008_fix_rows and nothing will be run -``` - -`ferro migrate up` runs nothing: +The database is back at `0001`, and `up` applies `0002_author_nickname` and `0003_author_bio`. - -```text -default · sqlite:///./app.db -inherited 0005_teams_epic: this database had applied 0005_add_teams … 0008_fix_rows; nothing was run -``` +## Squashing -That is *inheritance*. The run reads the tracking table and finds a finished record for every step of every migration the squash's witness names, each under the snapshot the witness names; it swaps those records for the squash's own, in one transaction, and `status` shows `0005_teams_epic` as `applied (inherited)`. There is no flag and no prompt: the committed squash is the consent. Nothing checks the live schema (`ferro migrate drift` is the verb for doubting it); the records already say the database stands at the snapshot the squash ends at, and its rows are kept. `down` inherits the same way before it reverts, and an inherited step's down runs as a run's does. +A feature branch that wrote one migration per task can replace them with the one migration `new` would have written across them, before it merges. [Squashing a feature branch's migrations](squash.md) shows when and how, and what a database that applied the originals does with the squash. -Until that `up`, the squashed migration is pending. `require_applied()` refuses to start with `PendingMigrationsError`, whatever `allow_ahead` says, because a migration the database has no record of is pending, never ahead. Each developer's database, and the epic's integration database, meets the squash once and runs `ferro migrate up`. +## Changing a primary key -A database that applied only part of the range (`0005` through `0007`, not `0008_fix_rows`) cannot be inherited: it does not stand at the squash's end. `up` and `down` refuse it, and `status` reports the same refusal under `needs attention`, naming both ways out: +No door changes a primary key in place: the key moved to another column, the key column's type changed, or a key made composite. `new` refuses it before writing anything, on every dialect. Change `Author`'s key and run: - -```text -default · sqlite:///./app.db -ferro migrate: 0005_teams_epic replaces migrations this database applied only part of: it applied 0005_add_teams, 0006_team_slug and 0007_drop_tmp but not 0008_fix_rows, so it cannot inherit 0005_teams_epic. To keep its rows, check out the commit that holds 0008_fix_rows, run `ferro migrate up` there to finish it, then run `ferro migrate up` here, which inherits 0005_teams_epic. Or revert instead: on the commit that applied them run `ferro migrate down --to 0004`, then `ferro migrate up` here; the revert drops what they created and fails on an irreversible step. Nothing was changed. +```bash +ferro migrate new rekey ``` -Take the first if you can: finishing the range keeps every row. The second drops the epic's tables and their rows, and stops at an irreversible step. - -### A squash is pre-release - -A squash rewrites the history of the branch's own migrations, the way a rebase rewrites its commits. Do it once, before the PR to main, not each time a task lands on the integration branch: the databases that applied the originals are your team's development databases, and each inherits it once. A production database never meets a squash, because the migrations it runs are the ones main already holds. - -### Squash or `--replace` - -Both regenerate migrations and carry their hand-written steps; they answer different situations. - -| | `ferro migrate squash NAME --from …` | `ferro migrate new NAME --replace …` | -| :--- | :--- | :--- | -| Plans | from the snapshot below the range to the range's last snapshot | from the new parent's snapshot to the models | -| Replaces | a tail of migrations, `--from` through the newest | the newest migration | -| A database that applied the originals | inherits the squash, running nothing | reverts with the files that went up, then runs the new migration ([two branches, one number](#two-branches-one-number)) | - -So a squash keeps what the range did and shortens it; `--replace` puts your migration back on top of someone else's. - -Two pins in `tests/test_cross_emitter_parity.py` hold a squash to this page over every casebook change on both dialects: [(h)](https://github.com/syn54x/ferro-orm/blob/main/AGENTS.md#the-nine-pins), a squash writes the files `new` writes across its ends, and [(i)](https://github.com/syn54x/ferro-orm/blob/main/AGENTS.md#the-nine-pins), a database that inherited it has the live schema and the tracking rows of one that ran it. - -## Changing a primary key - -No door changes a primary key in place: the key moved to another column, the key column's type changed, or a key made composite. `new` refuses it before writing anything, on every dialect: - -`ferro migrate new rekey` writes nothing and says: +It writes nothing and says: ```text @@ -617,9 +387,9 @@ The recipe, for an `author` table keyed by an integer `id` that should be keyed ferro migrate new add_writer --data-step Writer ``` - The data step lands after the generated schema steps, as `03_backfill_writer.py` here, with `todo(...)` where you write the copy. + The data step lands after the generated schema steps, as `03_backfill_writer.py` here, with `todo(...)` where you write the copy. Written, it reads: - ```python + ```python {test=compile reason="a data step file; the runner calls it with a step context during ferro migrate up"} # migrations/0002_add_writer/03_backfill_writer.py from uuid import uuid4 @@ -640,7 +410,7 @@ The recipe, for an `author` table keyed by an integer `id` that should be keyed def down(ctx): ... ``` - On a large table, write it as a [`@chunked`](data-steps.md#atomic-or-chunked) step instead. + Every author now has a writer, and every book points at it. The step runs inside the step's own transaction, so it opens no session. On a large table, write it as a [`@chunked`](data-steps.md#atomic-or-chunked) step instead. 2. **Drop the old table.** Remove `Author` and `Book.author`, and make `Book.writer` required. Every book already has a writer, so generate the change with a guard rather than a backfill: @@ -650,15 +420,18 @@ The recipe, for an `author` table keyed by an integer `id` that should be keyed The drop is marked `-- ferro: destructive`. Its down restores the schema, not the rows: if reverting past it must be impossible, declare its down `-- ferro: irreversible `. -3. **Optionally take the old name back.** Rename `Writer` to `Author` with `__ferro_renamed_from__ = "writer"` and the relation with `ForeignKey(..., renamed_from="writer")`, and generate the rename. +3. **Take the old name back, if you want it.** Rename `Writer` to `Author` with `__ferro_renamed_from__ = "writer"` and the relation with `ForeignKey(..., renamed_from="writer")`, and generate the rename ([rename hints](renames-and-retirements.md#rename-hints)). On a server that deploys with rolling restarts, ship each step as its own release, as the [two-release contract](deploying.md#rolling-deploys-and-the-two-release-contract) describes: code running during step 1 must write both keys. A native enum column moved to another enum type, while other columns stay on the old type, is refused the same way, with its own recipe: add a column of the new type, copy the values across in a data step (`ferro migrate new --data-step …`), then drop the old column. -## See Also +## See also -- [Data steps and backfills](data-steps.md) — what to write where `new` says `todo(...)` -- [Deploying migrations](deploying.md) — `up` in a deploy, `require_applied()`, rolling deploys, edited files -- [Testing migrations](testing.md) — prove a migration in your test suite -- [CLI reference](../../reference/cli.md) — every verb, flag and exit code +- [Data steps and backfills](data-steps.md): what to write where `new` says `todo(...)` +- [Renaming and retiring columns and labels](renames-and-retirements.md): rename hints and two-release retirements +- [Squashing a feature branch's migrations](squash.md): one migration in place of a branch's many +- [Deploying migrations](deploying.md): `up` in a deploy, `require_applied()`, rolling deploys, edited files +- [Testing migrations](testing.md): prove a migration in your test suite +- [Migrations reference](../../reference/migrations.md): step file headers, step classes, door coverage +- [CLI reference](../../reference/cli.md): every verb, flag and exit code diff --git a/docs/pages/guide/schema/overview.md b/docs/pages/guide/schema/overview.md index 68e03937..7281d0ed 100644 --- a/docs/pages/guide/schema/overview.md +++ b/docs/pages/guide/schema/overview.md @@ -1,6 +1,10 @@ -# Schema Management +--- +description: "How a database comes to match your models: auto-migrate for development, migration files for production." +--- -Your models are the source of truth for your schema. Ferro gives you two doors for getting a database to match them: +# Overview of schema management + +Schema management is how a database comes to match your models, which are the source of truth for its schema. Ferro gives you two doors for it: - **Auto-migrate** changes the database straight from the models when you call `connect()`. No files, nothing to review. - **Migrations** write each change down as a numbered directory of SQL and Python steps that you review, commit and apply. Ferro writes them for you from the difference between your models and the last migration. @@ -19,9 +23,9 @@ Say you add one field: --8<-- "docs/examples/migrations_quickstart_annotated.py:models" ``` -With auto-migrate, the next `connect(url, migrate_updates=True)` runs `ALTER TABLE "author" ADD COLUMN "nickname" varchar` and creates its index, and carries on. With migrations, `ferro migrate new author_nickname` writes those same statements into `migrations/0002_author_nickname/`, once per database dialect you target, and `ferro migrate up` runs them when you say so. +With auto-migrate, the next `connect(url, migrate_updates=True)` runs `ALTER TABLE "author" ADD COLUMN "nickname" varchar` and creates its index, and carries on. With migrations, `ferro migrate new author_nickname` writes the same change into `migrations/0002_author_nickname/`, rendered once per database dialect you target (on Postgres the index builds `CONCURRENTLY`, in a step of its own), and `ferro migrate up` runs them when you say so. -## The ladder +## The ladder: `auto_migrate`, `migrate_updates`, migrations {#the-ladder} | Rung | How you ask for it | What it does | Reach for it when | | :--- | :--- | :--- | :--- | @@ -38,11 +42,12 @@ Rungs 1 and 2 are the first door ([Auto-migrate](auto-migrate.md)); each flag im A database is managed by auto-migrate **or** by migrations, never both. Once a database has run a migration, it carries the tracking table `_ferro_migrations`, and `connect()` refuses every auto-migrate flag on it before running any DDL: + ```text -connect(auto_migrate=…) is refused: main is governed by ferro migrations (main._ferro_migrations). Use ferro migrate up, or drop the tracking tables to leave migrations. +RunRefused: connect(auto_migrate=…) is refused: main is governed by ferro migrations (main._ferro_migrations). Use ferro migrate up, or drop the tracking tables to leave migrations. ``` -(`main` is SQLite's schema; on Postgres it names the schema the connection works in, such as `public`.) +Nothing has run, and the message names the two ways out. The error is a [`RunRefused`](../../api/migrations.md#ferro.migrations.RunRefused), a subclass of `MigrationRefused`, so `except ferro.migrations.MigrationRefused` catches it too. Only some refusals carry a kind, and this one does not: its `kind` is `None`, so match it on its type or its message. `main` is SQLite's schema; on Postgres the message names the schema the connection works in, such as `public`. The rule is per database, not per project. A throwaway test database built with `connect(url, auto_migrate=True)` still works in a project that has migrations, because that database has never run one. @@ -50,46 +55,28 @@ The rule is per database, not per project. A throwaway test database built with | :--- | :--- | :--- | | A database with migrations applied | `connect(..., auto_migrate=True)` (or either stronger flag), `create_tables()`, `migrate()` | Refused before any DDL, text above | | A database with migrations applied | `alembic revision --autogenerate` over Ferro models | Refused, naming `ferro migrate new` ([Alembic](alembic.md#what-autogenerate-refuses)) | -| A database auto-migrate or Alembic built | `ferro migrate up` | Refused, naming `ferro migrate baseline` ([adopting migrations](../../howto/adopting-migrations.md)) | +| A database auto-migrate or Alembic built | `ferro migrate up` | Refused, naming `ferro migrate baseline` ([adopting migrations](adopting-migrations.md)) | | A database auto-migrate built | `alembic revision --autogenerate` | Works: the first revision is empty when the database matches the models, since both doors plan with the same planner | ## What each door covers -| Change | Auto-update | Migrations | Alembic bridge | -| :--- | :--- | :--- | :--- | -| Add a table, a nullable column, an index, an enum label | ✅ | ✅ | ✅ | -| Add a check, a foreign key, change a type or nullability, on Postgres | ✅ | ✅ | ✅ | -| The same on SQLite (needs a table rebuild) | ⚠️ warns, no DDL | ✅ generated [table rebuild](migrations.md#sqlite-table-rebuilds) | ❌ refused at autogenerate | -| Rename a column (`renamed_from`) or an enum label (`__ferro_renamed_labels__`) | ✅ | ✅ [declared on the model](migrations.md#renames) | ✅ same hints | -| Rename a table (`__ferro_renamed_from__`) | ✅ `RENAME TABLE` under `migrate_updates`, indexes and checks renamed with it; without `migrate_updates` the pass warns and creates nothing | ✅ | ✅ | -| Drop a column | with `migrate_destructive` | ✅ marked `-- ferro: destructive` | ✅ marked `# ferro: destructive` | -| Drop a table | ❌ | ✅ marked `-- ferro: destructive` | ✅ marked `# ferro: destructive` | -| Add a new required column to a table with rows | only with a literal default (backfills existing rows) | ✅ [expand, backfill, contract](data-steps.md) generated | ⚠️ the plain op, marked `# ferro: data-dependent`; the backfill is yours to write | -| Make a nullable column required | Postgres: `SET NOT NULL`, fails if any row is `NULL` (write it as a migration with a backfill); SQLite: warns, no DDL | ✅ expand, backfill, contract generated | ⚠️ the plain op, marked `# ferro: data-dependent` | -| Remove an enum label rows may hold | warns, no DDL | ✅ [backfill and contract](data-steps.md#removing-an-enum-label) generated | ❌ not written; [Migrations](data-steps.md#removing-an-enum-label) generate it | -| Python data steps over the models as they were | ❌ | ✅ [data steps](data-steps.md) | hand-written `op.execute(...)` | -| Change a primary key | ❌ | refused, with [the recipe](migrations.md#changing-a-primary-key) | ❌ refused, with the same recipe | - -## Choosing +The three doors plan with one planner, so a change two of them make leaves the same schema. They differ in which changes they make at all: + +- **Auto-update** alters a table in place, as far as one statement can. It never drops a table, warns rather than rebuilding a SQLite table, and adds a required column to a table with rows only when the field has a literal default. +- **Migrations** cover every change the models can express, on Postgres and SQLite: [SQLite table rebuilds](migrations.md#sqlite-table-rebuilds), [backfills](data-steps.md), [renames](renames-and-retirements.md#rename-hints) and label removals. A primary-key change is refused with [the recipe](migrations.md#changing-a-primary-key). +- **The Alembic bridge** writes the same statements as a revision, refuses at autogenerate a change that needs a SQLite table rebuild, and leaves backfills to you. + +The [door coverage table](../../reference/migrations.md#what-each-door-covers) lists every change, door by door. + +## Choosing a door {#choosing} - **Starting out, or writing tests:** `auto_migrate=True`. Tests keep this posture even in a project with migrations; see [Testing migrations](testing.md). - **Developing, the schema still moving, the data disposable:** `migrate_updates=True`. -- **The first time you would mind losing the data:** `ferro migrate init`, then `ferro migrate new initial`. If the database already exists, follow [Adopting migrations on an existing database](../../howto/adopting-migrations.md); it runs no DDL. +- **The first time you would mind losing the data:** `ferro migrate init`, then `ferro migrate new initial`. If the database already exists, follow [Adopting migrations on an existing database](adopting-migrations.md); it runs no DDL. - **A server:** apply migrations in the deploy step and refuse to start behind them; see [Deploying migrations](deploying.md). - **A local-first app that ships a SQLite file to its users:** migrations, applied at start-up with `await ferro.migrations.up()` ([Deploying migrations](deploying.md#apps-that-migrate-at-start-up)). -## Where the old page went - -This group replaces the single *Schema Migrations* page. Its sections now live here: - -| Old section | Now | -| :--- | :--- | -| Three Ways to Manage Schema | [The ladder](#the-ladder) | -| Auto-Migration, `migrate_updates`, label addition, `migrate_destructive`, `migrate()`, safety guidance | [Auto-migrate](auto-migrate.md) | -| Alembic for Production | [Alembic](alembic.md) | -| Choosing a Workflow | [Choosing](#choosing) | - -## See Also +## See also - [Auto-migrate](auto-migrate.md) · [Migrations](migrations.md) · [Data steps and backfills](data-steps.md) · [Deploying migrations](deploying.md) · [Testing migrations](testing.md) · [Alembic](alembic.md) - [CLI reference](../../reference/cli.md) and [Configuration reference](../../reference/configuration.md) diff --git a/docs/pages/guide/schema/renames-and-retirements.md b/docs/pages/guide/schema/renames-and-retirements.md new file mode 100644 index 00000000..f22e40ad --- /dev/null +++ b/docs/pages/guide/schema/renames-and-retirements.md @@ -0,0 +1,186 @@ +--- +description: "Rename a table, column or enum label without losing rows, and retire a column or label over two releases." +status: new +--- + +# Renaming and retiring columns and labels + +This page shows how to rename a table, a column or an enum label without losing the rows it holds, and how to retire a column or a label over two releases so the release still serving keeps working. Every section starts from this `Author`: + +=== "Assignment" + + ```python + --8<-- "docs/examples/migrations_rename.py:base" + ``` + +=== "Annotated" + + ```python + --8<-- "docs/examples/migrations_rename_annotated.py:base" + ``` + +Its first migration, `0001_create_author`, is applied, and each section below changes these models once. The `# before:` and `# new` comments in each section mark the change against them; they are not part of the declaration. + +## Rename hints + +Without a hint, a renamed column is a dropped column and an added one, and `new` writes exactly that, marked destructive, and says what it saw. Rename `nickname` to `handle` with no hint, and `ferro migrate new rename_handle` ends with the line that names the fix: + + +```text +default · migrations +ok Wrote migrations/0002_rename_handle/ + 01_schema.up.postgres.sql schema step, up (postgres) + 01_schema.down.postgres.sql schema step, down (postgres) + 01_schema.up.sqlite.sql schema step, up (sqlite) + 01_schema.down.sqlite.sql schema step, down (sqlite) + 02_idx_author_handle.up.postgres.sql schema step, up (postgres) + 02_idx_author_handle.down.postgres.sql schema step, down (postgres) + 02_idx_author_handle.up.sqlite.sql schema step, up (sqlite) + 02_idx_author_handle.down.sqlite.sql schema step, down (sqlite) + ir.json snapshot + + 1 changed model + Author + - author: if "nickname" became "handle", declare renamed_from="nickname" +``` + +Every author's nickname would be lost. With a hint, `new` writes a rename instead, and every row keeps its value. A hint says what a table, a column or an enum label was called before; here all three are renamed at once: + +=== "Assignment" + + ```python + --8<-- "docs/examples/migrations_rename.py:models" + ``` + +=== "Annotated" + + ```python + --8<-- "docs/examples/migrations_rename_annotated.py:models" + ``` + +`ferro migrate new rename_writer` writes the renames, and nothing is dropped: + +```sql +-- 0002_rename_writer/01_schema.up.postgres.sql +ALTER TABLE "author" RENAME TO "writer"; + +DO $$ DECLARE seq regclass := pg_get_serial_sequence('"writer"', 'id')::regclass; holder regclass; kind text; BEGIN IF seq IS NOT NULL AND (SELECT relname FROM pg_class WHERE oid = seq) <> 'writer_id_seq' THEN SELECT c.oid, CASE c.relkind WHEN 'S' THEN 'sequence' WHEN 'r' THEN 'table' WHEN 'p' THEN 'table' WHEN 'i' THEN 'index' WHEN 'I' THEN 'index' WHEN 'v' THEN 'view' WHEN 'm' THEN 'materialized view' ELSE 'relation' END INTO holder, kind FROM pg_class c WHERE c.relname = 'writer_id_seq' AND c.relnamespace = (SELECT relnamespace FROM pg_class WHERE oid = seq); IF holder IS NOT NULL THEN RAISE EXCEPTION USING ERRCODE = 'duplicate_table', MESSAGE = format('Cannot rename sequence %s, owned by %s, to %I, the name a table created as %s gives it: %s %s already holds that name. Rename or drop that %s, then run the change again.', seq, '"writer"."id"', 'writer_id_seq', '"writer"', kind, holder, kind); END IF; EXECUTE format('ALTER SEQUENCE %s RENAME TO %I', seq, 'writer_id_seq'); END IF; END $$; + +ALTER TABLE "writer" RENAME COLUMN "nickname" TO "handle"; + +ALTER INDEX "idx_author_bio" RENAME TO "idx_writer_bio"; + +ALTER INDEX "idx_author_nickname" RENAME TO "idx_writer_handle"; + +ALTER INDEX "uq_author_email" RENAME TO "uq_writer_email"; + +ALTER TYPE "status" RENAME VALUE 'canceled' TO 'cancelled'; +``` + +The table, the column, every index named after them and the label are renamed in place. The hints: + +- `Field(renamed_from="nickname")` renames a column; on a relation, `ForeignKey(..., renamed_from="writer")` renames its `*_id` column. +- `__ferro_renamed_from__ = "author"` on a model renames its table, and every index and constraint named after it. On Postgres the `DO` block renames the table's `serial` sequence too, to `writer_id_seq`, the name a table created as `writer` gives it; it refuses, naming the object, if something else already holds that name. +- `__ferro_renamed_labels__ = {"cancelled": "canceled"}` on a `StrEnum` maps each new label to its old one. On SQLite, where labels are text in the rows, the rows are rewritten in a [table rebuild](migrations.md#sqlite-table-rebuilds). +- An enum *type*'s rename needs no hint: it is read off the columns that moved to it. + +A hint is live only while the previous snapshot still holds the old name and lacks the new one. Once its migration is generated it is inert and may be deleted. A hint whose old name is still declared, or two hints naming one old name, is refused. + +Auto-migrate honours every hint under `migrate_updates` (on SQLite a label hint only warns, since the labels live in the rows). Without `migrate_updates`, a table hint makes the create pass hold the table back with a warning rather than create it empty. The [door coverage table](../../reference/migrations.md#what-each-door-covers) lists what each door does with a hint. + +The example `docs/examples/migrations_rename.py` in the repository runs this rename end to end, the revert included. + +## Renaming under a rolling deploy + +A rename hint renames in one migration, so the previous release's code, still naming `nickname`, fails the moment the migration runs. When old and new code serve side by side, rename over three releases instead: + +1. Add `handle` beside `nickname`, write both in the application code, and copy the existing rows across in a [data step](data-steps.md#a-data-step). +2. [Retire](#retiring-a-column) `nickname`. +3. Delete its declaration, which drops it. + +Each release is a migration the release before it can run beside; the [step classes](../../reference/migrations.md#step-classes) say which steps that holds for. + +## Retiring a column + +Dropping a column in the release that stops using it breaks the release still serving: its code names the column in every `INSERT` and `UPDATE`, so it fails from the moment the drop runs until it is replaced. A column leaves in two releases instead. In the first, declare it `retired`: + +=== "Assignment" + + ```python + --8<-- "docs/examples/migrations_retire.py:models" + ``` + +=== "Annotated" + + ```python + --8<-- "docs/examples/migrations_retire_annotated.py:models" + ``` + +From this release on, `bio` is gone from the model: + +- `author.bio` raises `AttributeError`, and `Author(bio="x")` raises `TypeError`. +- A query, `order_by`, `update()` or `save(only=...)` naming it fails with "retired in this release". +- No `SELECT`, `INSERT` or `UPDATE` the model issues names it. + +The column stays in the schema snapshot, nullable, with its index, so `ferro migrate new retire_bio` writes only the relaxation the previous release needs to keep writing beside this one: + +```sql +-- 0002_retire_bio/01_schema.up.postgres.sql +ALTER TABLE "author" ALTER COLUMN "bio" DROP NOT NULL; +``` + +The previous release can still insert an author with a `bio`, and this one inserts authors without it. On SQLite the same change is a [table rebuild](migrations.md#sqlite-table-rebuilds). Retiring a column that already accepts `NULL` writes nothing: `new` says "no schema change" and `check` is clean. + +In the next release, delete the declaration: `new` writes the `DROP COLUMN`, marked destructive, and, when it was the last column of a Postgres enum type, the `DROP TYPE` after it. + +| Release | `bio` in the model | The migration | Old code beside it | +| :--- | :--- | :--- | :--- | +| A | required, indexed | creates `bio` | | +| B | nullable, indexed, `retired=True` | `DROP NOT NULL` | A writes `bio`; works | +| C | deleted | `DROP COLUMN "bio"` (destructive) | B never names `bio`; works | + +- `ForeignKey(retired=True)` retires a relation and its `*_id` column; its foreign key constraint stays until the column is dropped. It needs no `related_name`, and a `BackRef` reading it back is refused: delete it. +- Nothing else about the column changes while it is retired: its index, unique, check and foreign key, and every table check, row policy, composite index or unique that names it, stay as declared. +- A [historical model](data-steps.md#historical-models) sees a retired column as an ordinary nullable column, so a data step can still read it. +- Auto-migrate creates the column on a fresh database and, under `migrate_updates`, relaxes its `NOT NULL`; it never drops it. Alembic autogenerate writes the same `alter_column(..., nullable=True)`. +- Refused at class definition, each naming the fix: retiring a field whose annotation does not allow `None` (declare it `str | None`), a field declared `nullable=False` (a retired column stays nullable), a primary key (drop `retired=True` and change the key first, see [Changing a primary key](migrations.md#changing-a-primary-key)), a field that also declares `renamed_from` (retire it or rename it, not both), a `BackRef` or a `ManyToMany`. + +The example `docs/examples/migrations_retire.py` runs all three releases end to end. + +## Retiring an enum label + +Removing a label in the release that stops writing it breaks the release still serving: its code may still write the label, and a removed label is gone from the Postgres type and from any `db_check`. Stop writing the label first, and remove it a release later. In the first release, declare it retired on the `StrEnum`, beside the [`__ferro_renamed_labels__`](#rename-hints) hint: + +=== "Assignment" + + ```python + --8<-- "docs/examples/migrations_retire_label.py:models" + ``` + +=== "Annotated" + + ```python + --8<-- "docs/examples/migrations_retire_label_annotated.py:models" + ``` + +The member stays, so a row that holds `paused` still loads as `Status.PAUSED`. Writing the label is refused, at construction and by attribute assignment. `Author(name="Grace Hopper", email="grace@example.com", bio="Admiral", status="paused")` raises a validation error: + + +```text +ValidationError: 1 validation error for Author +status + Status.PAUSED ('paused') is retired in this release: the previous release may still write it, this one does not [type=retired_enum_label, input_value='paused', input_type=str] +``` + +Nothing was saved. A row loaded with the label can still be saved after another field changes: loading does not validate, and `save()` writes every column, so the unchanged label goes back as it was. That is not a write of the label. + +Retiring a label is not a schema change. The Postgres type and any `db_check` keep listing it, so `ferro migrate new retire_paused` says "no schema change", writes nothing, and `check` is clean. In the next release, delete the member and the declaration together: `new` then writes the ordinary [label removal](data-steps.md#removing-an-enum-label), the backfill for rows that still hold it, and on Postgres the type swap. + +Each label in `__ferro_retired_labels__` must be a member's value: a set of anything else, or a label that names no member, raises `TypeError` at class definition. The example `docs/examples/migrations_retire_label.py` runs both releases. + +## See also + +- [Migrations](migrations.md): the loop these changes run through +- [Data steps and backfills](data-steps.md): the copy step of an online rename, and a label's removal +- [Deploying migrations](deploying.md#rolling-deploys-and-the-two-release-contract): the two-release contract +- [Migrations reference](../../reference/migrations.md): step classes and what each door covers diff --git a/docs/pages/guide/schema/squash.md b/docs/pages/guide/schema/squash.md new file mode 100644 index 00000000..09a1c270 --- /dev/null +++ b/docs/pages/guide/schema/squash.md @@ -0,0 +1,184 @@ +--- +description: "Replace a feature branch's migrations with the one migration ferro migrate new would write across them." +status: new +--- + +# Squashing a feature branch's migrations + +`ferro migrate squash` replaces a range of migrations, from the one you name through the newest, with the one migration `new` would have written across them. A feature branch collects one migration per task. Main stands at `0001_create_author`, the [Migrations](migrations.md) loop's `Author`, and the teams branch has added four on top of it: + +- `0002_add_teams` creates `Team`, with a `tmp_code` column one task needed for a while; +- `0003_team_slug` adds a required `slug`, with the backfill `02_backfill_team.py` written by hand; +- `0004_drop_tmp` drops `tmp_code` again; +- `0005_fix_rows` is a hand-written SQL step, `fix_rows`, that trims team names. + +Four migrations, three of them `ALTER TABLE`s on a table the first one created, are the history of the work, not the change main needs. Before the branch's pull request to main, replace them with one. `ferro migrate squash teams --from 0002` writes: + + +```text +default · migrations +ok Wrote migrations/0002_teams/ (replaces 0002_add_teams … 0005_fix_rows, now deleted) + 01_schema.up.postgres.sql schema step, up (postgres) regenerated + 01_schema.down.postgres.sql schema step, down (postgres) regenerated + 01_schema.up.sqlite.sql schema step, up (sqlite) regenerated + 01_schema.down.sqlite.sql schema step, down (sqlite) regenerated + 02_fix_rows.up.sql SQL step, up carried, review it (was 0005_fix_rows/01_fix_rows.up.sql) + 02_fix_rows.down.sql SQL step, down carried, review it (was 0005_fix_rows/01_fix_rows.down.sql) + 03_backfill_team.py data step carried, review it (was 0003_team_slug/02_backfill_team.py) + ir.json snapshot 0005_fix_rows's, chained from 0001_create_author, replaces 0002_add_teams … 0005_fix_rows + + 1 new model + Team + - team.tmp_code existed only inside the range (0002_add_teams through 0004_drop_tmp): review the carried steps for it +``` + +`0002_teams` creates `team` with its `slug` in one `CREATE TABLE`, and `tmp_code` never appears. The four directories are gone. + +## What is regenerated + +The DDL, every file with a dialect suffix, is planned once, from `0001_create_author`'s snapshot to `0005_fix_rows`'s, by the planner `new` uses. A squash writes exactly the files `new` would write across the range. + +A squash reads those snapshots and the migrations directory. It never imports the models, so a model change no migration records is still `new`'s to write, and it never opens a database (`--url` is refused). Its `ir.json` is the last snapshot under a new parent, and it records which migrations it replaced: the squash's *witness*. + +## What is carried + +Every data step and hand-written SQL step of the range keeps its body byte for byte, in chain order, placed as `--replace` places it (a SQL step before the data steps, a data step before the contract). Each is marked `carried, review it` with the file it was. + +Read each one. Here `fix_rows` now runs before the backfill it used to follow, and a step written after one task's contract now runs before the one contract. + +A backfill still byte for byte as the generator scaffolded it, `todo(...)` and all, that the squashed plan does not scaffold again is dropped with a note. Had `02_backfill_team.py` been left unwritten, the note would read `02_backfill_team.py from 0003_team_slug was the generator's unwritten scaffold and the squashed plan needs no backfill of team.slug: dropped`, since a `slug` on a table the squash creates needs no backfill. A step with any hand-written byte is always carried, and a carried unwritten step still refuses to load until it is written. + +## Names that existed only inside the range + +A table, column, enum label or constraint the range added and then removed (`team.tmp_code` here) does not exist in the squashed migration, so a carried step that touches it now fails. Where a carried step's declaration proves it, the squash refuses and writes nothing: a `@chunked` step whose query names the dead name in its model, order keys or filter columns or labels. Had `02_backfill_team.py` selected its rows by `tmp_code` (`lambda team: team.tmp_code == None`), the squash would say: + + +```text +default · migrations +ferro migrate squash: 0003_team_slug/02_backfill_team.py names team.tmp_code, which existed only inside the range (added by 0002_add_teams, removed by 0004_drop_tmp): delete the step, or start the range at 0004_drop_tmp, the migration that removed it (pass --from 0004_drop_tmp). Nothing was written. +``` + +Nothing was written, and the refusal names both ways on. An `@atomic` step declares nothing the squash can read, and a SQL step is SQL, so neither is ever refused: each dead name gets the note in the first output instead, and reading the carried steps for it is yours. + +## The range + +- The range is always a tail, `--from` through the newest migration, because the migration after a middle range would chain from a snapshot that no longer exists. +- The squash takes the first migration's number and the name you give it. +- A name that would rewrite a replaced migration's own directory (`ferro migrate squash add_teams --from 0002` writing `0002_add_teams`) is refused, since every reader would then take the squash for the original. +- A range that adds nothing and carries nothing is refused: delete the directories. +- A migration that shares the first number of the range, because two branches each generated it, is refused too: resolve the shared number with [`--replace`](migrations.md#duplicate-migration-numbers-and-replace) before squashing. + +The [CLI reference](../../reference/cli.md#ferro-migrate-squash-name-from-target) lists every flag and refusal, and what to do with the hidden directory a squash killed halfway leaves. + +## Renames inside the range + +A column renamed in one task, whose hint a later task deleted, is still a rename after the squash. Say the branch's first task renamed `nickname` to `handle` and generated `0002_rename_handle`: + +=== "Assignment" + + ```python + --8<-- "docs/examples/migrations_squash.py:models" + ``` + +=== "Annotated" + + ```python + --8<-- "docs/examples/migrations_squash_annotated.py:models" + ``` + +Its migration generated, the hint is inert, so the second task deleted it and added `Team` (`0003_add_teams`): + +=== "Assignment" + + ```python + --8<-- "docs/examples/migrations_squash.py:later" + ``` + +=== "Annotated" + + ```python + --8<-- "docs/examples/migrations_squash_annotated.py:later" + ``` + +Planned from `0001`'s snapshot to `0003`'s models alone, `nickname` would be dropped and `handle` added, and every author's nickname lost. The squash composes the hints of every snapshot in the range into its own, so `ferro migrate squash teams --from 0002` writes one rename: + +```sql +-- 0002_teams/01_schema.up.postgres.sql +ALTER TABLE "author" RENAME COLUMN "nickname" TO "handle"; + +ALTER INDEX "idx_author_nickname" RENAME TO "idx_author_handle"; + +CREATE TABLE IF NOT EXISTS "team" ( "id" serial PRIMARY KEY NOT NULL, "name" varchar NOT NULL, "slug" varchar NOT NULL ); +``` + +The column and its index are renamed, and every author keeps a nickname, now under `handle`. Renames compose: + +- `a→b` then `b→c` is `a→c`. +- A renamed column the range later drops is a drop of the old name. +- A column added and then renamed is an add of the new name. +- A rename whose old name a later task adds again cannot be one migration. It is refused, naming the `--from` that starts the range at the task that adds it again, which leaves the rename below the squash. + +The example `docs/examples/migrations_squash.py` in the repository runs this end to end, inheritance and revert included. + +## Inheritance on databases that ran the originals + +Your development database applied `0002_add_teams` through `0005_fix_rows`. After you pull the squash, `ferro migrate status` shows the one migration it has no records for, and what `up` will do with it: + + +```text +default · sqlite:///./app.db +-- 1 pending main._ferro_migrations + ok 0001_create_author applied 2026-10-15 04:31Z + -- 0002_teams pending (inheritable) run `ferro migrate up`; this database applied 0002_add_teams … 0005_fix_rows and nothing will be run +``` + +`0002_teams` is pending and *inheritable*. `ferro migrate up` runs nothing: + + +```text +default · sqlite:///./app.db +inherited 0002_teams: this database had applied 0002_add_teams … 0005_fix_rows; nothing was run +``` + +That is *inheritance*. The run reads the tracking table and finds a finished record for every step of every migration the squash's witness names, each under the snapshot the witness names. It swaps those records for the squash's own, in one transaction, and `status` shows `0002_teams` as `applied (inherited)`. + +- There is no flag and no prompt: the committed squash is the consent. +- Nothing checks the live schema (`ferro migrate drift` is the verb for doubting it). The records already say the database stands at the snapshot the squash ends at, and its rows are kept. +- `down` inherits the same way before it reverts, and an inherited step's down runs as a run's does. + +Until that `up`, the squashed migration is pending. `require_applied()` refuses to start with `PendingMigrationsError`, whatever `allow_ahead` says, because a migration the database has no record of is pending, never ahead. Each developer's database, and the branch's integration database, meets the squash once and runs `ferro migrate up`. + +A database that applied only part of the range (`0002` through `0004`, not `0005_fix_rows`) cannot be inherited: it does not stand at the squash's end. `up` and `down` refuse it, and `status` reports the same refusal under `needs attention`, naming both ways out: + + +```text +default · sqlite:///./app.db +ferro migrate: 0002_teams replaces migrations this database applied only part of: it applied 0002_add_teams, 0003_team_slug and 0004_drop_tmp but not 0005_fix_rows, so it cannot inherit 0002_teams. To keep its rows, check out the commit that holds 0005_fix_rows, run `ferro migrate up` there to finish it, then run `ferro migrate up` here, which inherits 0002_teams. Or revert instead: on the commit that applied them run `ferro migrate down --to 0001`, then `ferro migrate up` here; the revert drops what they created and fails on an irreversible step. Nothing was changed. +``` + +Take the first if you can: finishing the range keeps every row. The second drops the branch's tables and their rows, and stops at an irreversible step. + +## Squash timing + +A squash rewrites the history of the branch's own migrations, the way a rebase rewrites its commits. Do it once, before the pull request to main, not each time a task lands on the integration branch. The databases that applied the originals are your team's development databases, and each inherits the squash once. A production database never meets a squash, because the migrations it runs are the ones main already holds. + +!!! note "Older ferro releases" + A ferro release from before `squash` cannot read a database that inherited a squash. Read [squashes and older releases](deploying.md#squashes-and-older-releases) before you roll one out beside them. + +## `squash` vs `--replace` + +Both regenerate migrations and carry their hand-written steps; they answer different situations: + +| | `ferro migrate squash NAME --from …` | `ferro migrate new NAME --replace …` | +| :--- | :--- | :--- | +| Plans | from the snapshot below the range to the range's last snapshot | from the new parent's snapshot to the models | +| Replaces | a tail of migrations, `--from` through the newest | the newest migration | +| A database that applied the originals | inherits the squash, running nothing | reverts with the files that went up, then runs the new migration ([duplicate migration numbers](migrations.md#duplicate-migration-numbers-and-replace)) | + +A squash keeps what the range did and shortens it; `--replace` puts your migration back on top of someone else's. The [CLI reference](../../reference/cli.md#ferro-migrate-squash-name-from-target) lists every flag of both. + +## See also + +- [Migrations](migrations.md): the loop, and `--replace` for two branches that share a number +- [Data steps and backfills](data-steps.md): the steps a squash carries +- [CLI reference](../../reference/cli.md#ferro-migrate-squash-name-from-target): every `squash` flag and refusal diff --git a/docs/pages/guide/schema/testing.md b/docs/pages/guide/schema/testing.md index 9e1d9e09..78b1e5fe 100644 --- a/docs/pages/guide/schema/testing.md +++ b/docs/pages/guide/schema/testing.md @@ -1,15 +1,10 @@ -# Testing Migrations +--- +description: "Test a migration by seeding a database at the previous migration, applying the next one and reading the rows back." +--- -Two kinds of test touch the schema, and they want different things: - -- **Application tests** want a database shaped like the models, fast and fresh. Keep `connect(url, auto_migrate=True)` for them, as the [Testing how-to](../../howto/testing.md) sets up. A throwaway database that never ran a migration is not managed by migrations, so the [one-door rule](overview.md#one-door-per-database) does not refuse it. -- **Migration tests** want the database at a particular migration, rows seeded the way the table stood then, one migration applied, and the result read back. That is `ferro.migrations.testing.harness`. - -Know the boundary of the first kind: a schema auto-migrate builds from today's models is the schema at the head of your migrations only while the two agree, which `ferro migrate check` proves in CI. It never exercises a migration's own SQL, its backfill, or its down. Those are what the harness is for. - -## A migration test +# Testing migrations -Take a project whose `0002_author_slug` adds a required `slug` and backfills it: +A migration test stands a database at one migration, seeds rows the way the tables stood then, applies the next migration and reads the result back. Take a project whose `0002_author_slug` adds a required `slug` and backfills it from the name: === "Assignment" @@ -23,65 +18,73 @@ Take a project whose `0002_author_slug` adds a required `slug` and backfills it: --8<-- "docs/examples/migrations_testing_annotated.py:models" ``` -The test stands the database at `0001`, seeds an author through `0001`'s historical models, applies exactly `0002`, and reads the row back through `0002`'s: +The test seeds an author through `0001`'s historical models, applies exactly `0002`, and reads the row back through `0002`'s: ```python --8<-- "docs/examples/migrations_testing.py:tests" ``` -The fixture gives each test a fresh, empty database, and the test connects to it; the harness reads the project's configuration from the working directory, as every `ferro migrate` command does: +`pytest` runs both tests and reports `2 passed`. A backfill that wrote the wrong value fails the first one at its `assert`; a down that does not reach its parent fails the second. + +Two kinds of test touch the schema, and they want different things: + +- **Application tests** want a database shaped like the models, fast and fresh. Keep `connect(url, auto_migrate=True)` for them, as the [Testing how-to](../../howto/testing.md) sets up. A throwaway database that never ran a migration is not managed by migrations, so the [one-door rule](overview.md#one-door-per-database) does not refuse it. +- **Migration tests** want the database at a particular migration. That is `ferro.migrations.testing.harness`. + +A schema auto-migrate builds from today's models matches the head of your migrations only while the two agree, which `ferro migrate check` proves in CI. It never runs a migration's own SQL, its backfill or its down; the harness does. + +## The fixture and pytest settings + +Each test gets a fresh, empty database from a fixture in `conftest.py`, and connects to it itself: ```python -import pytest +--8<-- "docs/examples/migrations_testing.py:fixture" +``` + +The fixture closes the connection after the test, so the next test can connect again. The harness reads the project's configuration from the working directory, as every `ferro migrate` command does, so run `pytest` from the project root. +The tests are `async def`. With pytest-asyncio, set its mode to `auto` in `pyproject.toml` so it runs them without a decorator each: -@pytest.fixture -def database_url(tmp_path): - return f"sqlite:{tmp_path / 'test.db'}?mode=rwc" +```toml +[tool.pytest.ini_options] +asyncio_mode = "auto" ``` -For Postgres, give each test its own schema with `?ferro_search_path=...`, as [Testing Against Postgres](../../howto/testing.md#testing-against-postgres) shows, and run the same tests on both dialects your migrations target: the harness never skips one. Both tests run in [`docs/examples/migrations_testing.py`](https://github.com/syn54x/ferro-orm/blob/main/docs/examples/migrations_testing.py). +Without it, pytest-asyncio runs only the tests marked `@pytest.mark.asyncio`. [Configuring pytest-asyncio](../../howto/testing.md#configuring-pytest-asyncio) covers the other settings. The fixture, the tests and these settings run as written in [`docs/examples/migrations_testing.py`](https://github.com/syn54x/ferro-orm/blob/main/docs/examples/migrations_testing.py). + +For Postgres, give each test its own schema with `?ferro_search_path=...`, as [Testing Against Postgres](../../howto/testing.md#testing-against-postgres) shows, and run the same tests on both dialects your migrations target: the harness never skips one. ## The harness -`harness(*, settings=None, database=None, using=None)` binds the project's configuration (`FerroSettings()` by default), the configured database (the only one by default), and the open connection (the default one), and keeps no other state: every call reads where the database stands from its tracking table. +`harness(*, settings=None, database=None, using=None)` binds the project's configuration (`FerroSettings()` by default), the configured database (the only one by default) and the open connection (the default one). It keeps no other state: every call reads where the database stands from its tracking table. | Call | What it does | | :--- | :--- | -| `await h.apply_through("0002")` | Applies every migration up to and including `0002`, from wherever the database stands. | -| `await h.apply("0002")` | Applies exactly `0002`; refused unless the database stands at its parent, `0001`. | -| `await h.revert_to("0001")` | Reverts down to `0001`, leaving it applied, with no prompt. | -| `await h.revert_all()` | Reverts every migration. | -| `async with h.models_at("0001") as models:` | The historical models of `0001`, to seed or read rows the way that migration left the tables. Takes no lock and changes nothing. | -| `await h.round_trip()` | Applies every migration, reverts every one, applies them again, and checks for drift after each stop; returns a `RoundTripResult`. | +| `await migrations.apply_through("0002")` | Applies every migration up to and including `0002`, from wherever the database stands. | +| `await migrations.apply("0002")` | Applies exactly `0002`; refused unless the database stands at its parent, `0001`. | +| `await migrations.revert_to("0001")` | Reverts down to `0001`, leaving it applied, with no prompt. | +| `await migrations.revert_all()` | Reverts every migration. | +| `async with migrations.models_at("0001") as models:` | The historical models of `0001`, to seed or read rows the way that migration left the tables. Takes no lock and changes nothing. | +| `await migrations.round_trip()` | Applies every migration, reverts every one, applies them again, and checks for drift after each stop; returns a `RoundTripResult`. | -Migrations are named by number (`"0002"`) or full name (`"0002_author_slug"`). There is no step-level target (`"0002:03"`): no snapshot describes the state between two steps. Every step runs through the same runner `ferro migrate up` uses, under its run lock (only the apply, revert and round-trip calls take it), and a call that cannot do exactly what it was asked raises `MigrationRefused` naming where the database stands and what was asked. +Migrations are named by number (`"0002"`) or full name (`"0002_author_slug"`). There is no step-level target (`"0002:03"`): no snapshot describes the state between two steps. Every step runs through the same runner `ferro migrate up` uses, under its run lock (only the apply, revert and round-trip calls take it). A call that cannot do exactly what it was asked raises `MigrationRefused`, naming where the database stands and what was asked. Each call has its entry in the [API reference](../../api/migrations.md#the-test-harness). ### The round trip -```python -result = await harness().round_trip() -assert result.irreversible is None -``` - -It proves every down reaches its parent. An irreversible step ends the downward walk at its migration, reported rather than failed: +`round_trip()` walks the whole lineage in one call and proves every down reaches its parent. An irreversible step ends the downward walk at its migration, reported rather than failed. Declare the backfill's down `@irreversible("the old slugs are gone")`, and the round trip stops there: ```python -RoundTripResult( - applied=["0001_create_author", "0002_add_slug"], - reverted_to="0002_add_slug", - irreversible=("0002_add_slug", "01_schema", "the index is shared"), -) +--8<-- "docs/examples/migrations_testing.py:irreversible" ``` -Everything above it round-tripped; everything below was applied only. +The walk down stopped at `0002_author_slug`, which stays applied, so `0001_create_author` below it was applied but never reverted. A migration above the irreversible one would still have round-tripped. ### What the harness does not do -It never makes a database fresh (that is your fixture's job) and has no `reset()`. It carries what the application API refuses on purpose (a target on the way up, a revert with no prompt, historical models outside a data step) because a test is about one migration and is deleted with it (ADR-0045). Those calls stay out of `ferro.migrations.up()`. +It never makes a database fresh (that is your fixture's job) and has no `reset()`. It carries what the application API refuses on purpose: a target on the way up, a revert with no prompt, historical models outside a data step. A test is about one migration and is deleted with it, so those calls stay out of `ferro.migrations.up()`. -## See Also +## See also -- [Testing how-to](../../howto/testing.md) — fixtures for application tests -- [Data steps and backfills](data-steps.md) — historical models and backfills -- [Migrations API reference](../../api/migrations.md#the-test-harness) +- [Testing how-to](../../howto/testing.md): fixtures for application tests +- [Data steps and backfills](data-steps.md): historical models and backfills +- [Migrations API reference](../../api/migrations.md#the-test-harness): `harness`, `Harness`, `RoundTripResult` diff --git a/docs/pages/guide/transactions.md b/docs/pages/guide/transactions.md index ae72016a..1699f1a5 100644 --- a/docs/pages/guide/transactions.md +++ b/docs/pages/guide/transactions.md @@ -1,123 +1,112 @@ +--- +description: "Group database calls into one transaction that commits when its block exits and rolls back when it raises." +--- + # Transactions -Group multiple operations into a single atomic unit with the `ferro.transaction()` async context manager: commit on clean exit, rollback if the block raises. +`transaction()` groups database calls into one unit: the block commits when it exits cleanly and rolls back if it raises. -## Basic Usage +A transaction runs inside a session. Open one, create two accounts, and move 25 from checking to savings in one unit: ```python --8<-- "docs/examples/transactions.py:basic" ``` -Every ORM operation inside the block — `create`, `save`, `delete`, batch `update`, `bulk_create`, queries — runs on the **same database connection** (connection affinity is tracked through `contextvars`, so it is safe under concurrent asyncio tasks: each task gets its own transaction). +Both rows changed together; if either `save()` had failed, neither would have. Outside a session, `async with transaction():` raises `RuntimeError: No database route`. The blocks below run inside this session; see [Sessions](connections.md#sessions). + +The examples use one model, a bank account with a unique account number: + +=== "Assignment" + + ```python + --8<-- "docs/examples/transactions.py:models" + ``` + +=== "Annotated" -## Rollback on Error + ```python + --8<-- "docs/examples/transactions_annotated.py:models" + ``` -If any exception escapes the block, everything performed inside it is rolled back and the exception re-raises: +Every call inside the block, from `create()` and `save()` to batch `update()`, `bulk_create()` and queries, runs on the same database connection. Ferro tracks that connection per asyncio task, so concurrent tasks each get their own transaction. + +## Rollback + +If an exception leaves the block, Ferro rolls back everything the block did and raises the exception again: ```python --8<-- "docs/examples/transactions.py:rollback" ``` -## Using Named Connections +The row still holds 75, but the `checking` instance held `-925` until `refresh()` reloaded it. + +!!! warning "Rollback does not undo changes to Python objects" + A rollback restores the database, not the instances you changed in the block. Saving such an instance later writes the rolled-back value back to the row. Call `refresh()` on each instance you changed before you use it again. Ferro also clears the session's [identity map](../concepts/identity-map.md) on rollback, so a later load returns a new instance rather than the stale one. -A transaction is pinned to one connection. Open it against a [named connection](connections.md#named-connections) with `using=`, and everything inside inherits that connection — unqualified operations included. Trying to route to a *different* connection inside the block raises: +## Catching errors inside a transaction + +Catch an exception outside the block when the whole unit should roll back. A `transfer()` function that refuses to overdraw an account: ```python ---8<-- "docs/examples/multiple_databases.py:transaction" +--8<-- "docs/examples/transactions.py:transfer" ``` -## Sessions + Transactions +The refused transfer left both balances where they were, in the database and in memory, because the `except` branch refreshed `src`. The transfer of 50 went through. -`transaction()` composes with sessions. If a session is active, transactions inherit that session route by default: +If you catch an exception inside the block and carry on, the work that did not raise commits when the block exits. Keep that to work you want to keep in part, inside a nested block, as the next section shows. -```python -import ferro +## Nested transactions -async with ferro.engines.session("analytics"): - async with ferro.transaction(): - await Metric.create(name="requests", value=1) -``` +A `transaction()` inside another one is a savepoint on the outer transaction's connection. If the inner block raises, only its work rolls back, and the outer block carries on. Open three accounts, one of which reuses a taken number: -You can also pass an explicit session handle (`transaction(session=my_session)`) when ambient context is different and you need a deterministic override. +```python +--8<-- "docs/examples/transactions.py:nested" +``` -`Session.close()` raises `RuntimeError` while session-scoped transactions are still open. Exit all `transaction()` blocks before closing the session. +`CHK-001` raised [`IntegrityError`](../api/exceptions.md#ferro.IntegrityError) and rolled back to its savepoint; the other two accounts were committed. If the outer block raises, everything rolls back, including inner blocks that finished. A nested block never commits on its own: only a clean exit from the outermost block commits. -## Raw SQL Inside a Transaction +## Raw SQL in a transaction -`transaction()` yields a `Transaction` handle exposing `execute` / `fetch_all` / `fetch_one` for raw SQL on the transaction's own connection — useful for Postgres GUCs (`set_config`, `SET LOCAL`), advisory locks, or any one-off statement that doesn't fit a model: +`transaction()` yields a handle with `execute`, `fetch_all` and `fetch_one` for raw SQL on the transaction's own connection. Use it for Postgres settings (`set_config`, `SET LOCAL`), advisory locks, or a statement that does not fit a model: ```python --8<-- "docs/examples/transactions.py:handle" ``` -The handle becomes invalid when the block exits — calling it afterwards raises `RuntimeError`. If you don't need it, write `async with transaction():` and the handle is discarded. The top-level `ferro.execute` / `fetch_all` / `fetch_one` functions also automatically join the active transaction — see [Raw SQL](raw-sql.md#raw-sql-in-transactions). +The count ran inside the transaction, so it saw Carol's account before the commit. The handle stops working when the block exits; a later call on it raises `RuntimeError`. The top-level `ferro.execute`, `fetch_all` and `fetch_one` also join the active transaction; see [Raw SQL](raw-sql.md#raw-sql-in-transactions). -## Nesting Behavior +## SQLite write locks: `immediate=True` -Nested `transaction()` blocks map to **savepoints** on the outer transaction's connection: - -- If an **inner** block raises, only its work is rolled back (to the savepoint); the outer transaction can continue and commit. -- If the **outer** block raises, everything rolls back — including inner blocks that completed "successfully". +A SQLite transaction that reads and then writes can fail with `SQLITE_BUSY` when it tries to upgrade its lock while another connection writes. `transaction(immediate=True)` opens it with `BEGIN IMMEDIATE`, which takes the write lock at the start: ```python -from ferro import transaction - - -async def import_rows(rows: list[dict]) -> int: - imported = 0 - async with transaction(): - await AuditLog.create(event="import-started") - for row in rows: - try: - async with transaction(): # savepoint per row - await Record.create(**row) - imported += 1 - except ValueError: - continue # this row rolled back; the import continues - return imported +--8<-- "docs/examples/transactions.py:immediate" ``` -A nested block never commits independently of its parent — only the outermost block's clean exit commits to the database. - -## Patterns +The read and the write ran under one write lock, and savings now holds 85. Another writer waits on SQLite's busy timeout instead of failing. On PostgreSQL `immediate=True` is a plain `BEGIN`, and a nested transaction ignores it. -### Keep transactions short +## Short transactions -A transaction holds a pooled connection (and database locks) for its entire duration. Do slow work — HTTP calls, file I/O, expensive computation — *outside* the block, and keep only the database writes inside: +A transaction holds a pooled connection, and the database's locks, until it ends. Do slow work such as HTTP calls, file I/O or heavy computation before the block, and keep only the writes inside it: ```python -async def publish(post_id: int) -> None: - rendered = await render_markdown(post_id) # slow work first, no tx held - - async with transaction(): - post = await Post.get(post_id) - post.body_html = rendered - post.published = True - await post.save() +--8<-- "docs/examples/transactions.py:short" ``` -### Error handling +`quote_interest()` stands for any slow call; the transaction lasted only as long as the `save()`. -Catch exceptions *outside* the block when the whole unit should roll back, and *inside* it only for work you genuinely want to keep partial (paired with a nested block, as above). Catching an exception inside the block and continuing means the surviving operations **will commit**: +## Transactions in a session -```python -async def transfer(src: Account, dst: Account, amount: int) -> bool: - try: - async with transaction(): - src.balance -= amount - await src.save() - if src.balance < 0: - raise ValueError("insufficient funds") - dst.balance += amount - await dst.save() - except ValueError: - return False # nothing was committed - return True -``` +`transaction()` uses the session you are in. To pin it to a session handle you hold instead, pass `transaction(session=handle)`. `Session.close()` raises `RuntimeError` while a transaction in that session is still open, so leave every `transaction()` block before you close it. + +## Transactions on a named connection + +`transaction(using="analytics")` pins a transaction to a [named connection](connections.md#named-connections). Every call inside the block runs on that connection, unqualified calls included, and routing a call to another connection raises `ValueError`. One transaction spans one connection, so writes to two databases never commit together. [How to use several databases](../howto/multiple-databases.md#transactions-on-a-named-connection) shows it with two connections. + +The [`transaction` reference](../api/transactions.md#ferro.transaction) lists every argument. -## See Also +## See also -- [Raw SQL](raw-sql.md) — placeholders, bind types, and the `Transaction` handle -- [Connections & Databases](connections.md) — named connections and pools -- [Mutations](mutations.md) — the operations you'll wrap in transactions -- [Multiple Databases](../howto/multiple-databases.md) — routing patterns +- [Mutations](mutations.md): the writes you group in a transaction +- [Raw SQL](raw-sql.md): placeholders, bind types and the `Transaction` handle +- [Connections](connections.md): sessions, pools and named connections diff --git a/docs/pages/howto/migrate-from-sqlalchemy.md b/docs/pages/howto/migrate-from-sqlalchemy.md index 86213c2e..ad1f2fd2 100644 --- a/docs/pages/howto/migrate-from-sqlalchemy.md +++ b/docs/pages/howto/migrate-from-sqlalchemy.md @@ -1,27 +1,49 @@ -# Migrating from SQLAlchemy +--- +description: "Translate each SQLAlchemy 2.0 idiom, from models to sessions and queries, into its Ferro equivalent." +--- -Ferro's model-centric API replaces SQLAlchemy's session/statement split: models are Pydantic classes, queries hang off the model, and there is no session to manage. Each section below maps a SQLAlchemy concept to its Ferro equivalent. +# Migrate from SQLAlchemy -## Quick Comparison +This page maps each SQLAlchemy 2.0 idiom to its Ferro equivalent, using one `User` and `Post` pair throughout. Ferro keeps a session, but it only routes calls to a connection and holds an identity map: there is no `add()`, `flush()` or `commit()`. Open one with `async with engines.session():` and write the call on the model: + +=== "SQLAlchemy" + + ```python {test=compile reason="SQLAlchemy side of the comparison"} + async with session_factory() as session: + user = User(username="ada", email="ada@example.com", age=36) + session.add(user) + await session.commit() + ``` + +=== "Ferro" + + ```python + --8<-- "docs/examples/from_sqlalchemy.py:session" + ``` + +The SQLAlchemy block inserts the row on `commit()`. The Ferro block inserts it inside `create()` and returns the instance with its primary key set. The Ferro blocks below run inside a session like this one; see [Connections › Sessions](../guide/connections.md#sessions). + +## Comparison | Concept | SQLAlchemy 2.0 | Ferro | |---|---|---| | Model definition | `DeclarativeBase` + `Mapped` / `mapped_column` | Pydantic `Model` + type annotations | -| Validation | Separate (e.g. Pydantic on top) | Built in — models *are* Pydantic models | -| Querying | `select(User).where(...)` executed via a session | `User.where(...)` awaited directly | -| Get by primary key | `session.get(User, pk)` → `None` if missing | `User.get(pk)` raises; `User.get_or_none(pk)` → `None` | -| Sessions | Required (`async_sessionmaker`, `session.add`, `commit`) | None — connections are managed for you | +| Validation | Separate (for example Pydantic on top) | The model is a Pydantic model | +| Querying | `select(User).where(...)` executed on a session | `User.where(...)` awaited inside a session | +| Get by primary key | `session.get(User, pk)` returns `None` if missing | `User.get(pk)` raises; `User.get_or_none(pk)` returns `None` | +| Sessions | `async_sessionmaker`, `session.add`, `commit` | `async with engines.session():` once per request; no `add` or `commit` | | Transactions | `async with session.begin():` | `async with transaction():` | | Relationships | `relationship()` + `ForeignKey` columns | `Annotated[..., ForeignKey(...)]` + `BackRef` / `ManyToMany` | | Async | Native | Native (async-only) | -| Migrations | Alembic | Alembic (via `ferro.migrations.get_metadata`) | +| Migrations | Alembic | `ferro migrate`, or Alembic through `get_metadata()` and `ferro_options()` | ## Models -SQLAlchemy: +The SQLAlchemy models are: -```python -from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column +```python {test=compile reason="SQLAlchemy side of the comparison"} +from sqlalchemy import ForeignKey +from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship class Base(DeclarativeBase): @@ -34,215 +56,151 @@ class User(Base): id: Mapped[int] = mapped_column(primary_key=True) username: Mapped[str] = mapped_column(unique=True) email: Mapped[str] + age: Mapped[int] + posts: Mapped[list["Post"]] = relationship(back_populates="author") + + +class Post(Base): + __tablename__ = "posts" + + id: Mapped[int] = mapped_column(primary_key=True) + title: Mapped[str] + author_id: Mapped[int] = mapped_column(ForeignKey("users.id")) + author: Mapped["User"] = relationship(back_populates="posts") ``` -Ferro: +The Ferro models, in either declaration style, are: === "Assignment" ```python - from ferro import Field, Model - - - class User(Model): - id: int | None = Field(default=None, primary_key=True) - username: str = Field(unique=True) - email: str + --8<-- "docs/examples/from_sqlalchemy.py:models" ``` === "Annotated" ```python - from typing import Annotated - - from ferro import Field, Model - - - class User(Model): - id: Annotated[int | None, Field(default=None, primary_key=True)] - username: Annotated[str, Field(unique=True)] - email: str + --8<-- "docs/examples/from_sqlalchemy_annotated.py:models" ``` -Differences worth noting: +The two declare the same tables; the later sections reuse these models. What changes: + +- There is no `__tablename__`: the table name comes from the class name (`user`). +- The auto-increment primary key is `int | None` with `default=None`: it is `None` until the row is inserted. +- Validation and serialization come from Pydantic, so the model doubles as your API schema. +- The `ForeignKey` annotation declares both the relation and the `author_id` column; Ferro creates the shadow column. +- Declare fields on each concrete model. A `Model` base class does not pass its fields on; shared behavior goes in plain mixins, as in the [Timestamps how-to](timestamps.md). -- No `__tablename__` — the table name is derived from the class name (`user`). -- The auto-increment primary key is annotated `int | None` with `default=None`: it is `None` until the row is inserted. -- Because Ferro models are Pydantic models, you get validation, serialization, and FastAPI integration for free — no separate schema classes. -- Declare fields on each concrete model. Ferro does not support inheriting fields from a `Model` base class (each model class gets its own table schema registration); shared behavior goes in plain mixins instead — see the [Timestamps how-to](timestamps.md). +See the [Models and fields guide](../guide/models-and-fields.md) for every field option. ## Queries -Fetch all: +Fetch every row: -```python -# SQLAlchemy -from sqlalchemy import select +=== "SQLAlchemy" -async with session_factory() as session: - result = await session.execute(select(User)) - users = result.scalars().all() -``` - -```python -# Ferro -users = await User.all() -``` + ```python {test=compile reason="SQLAlchemy side of the comparison"} + from sqlalchemy import select -Filtering, ordering, limiting: + async with session_factory() as session: + result = await session.execute(select(User)) + users = result.scalars().all() + ``` -```python -# SQLAlchemy -stmt = select(User).where(User.age >= 18).order_by(User.age).limit(10) -result = await session.execute(stmt) -adults = result.scalars().all() -``` +=== "Ferro" -```python -# Ferro -adults = await User.where(lambda t: t.age >= 18).order_by(lambda t: t.age).limit(10).all() -``` + ```python + --8<-- "docs/examples/from_sqlalchemy.py:fetch-all" + ``` -Get by primary key — the semantics differ. `session.get(User, pk)` returns `None` when the row is missing; Ferro's `User.get(pk)` raises `ModelDoesNotExist`, and `User.get_or_none(pk)` is the optional variant: +Both return a list of `User` instances; the assert shows the Ferro list in insertion order. Filter, order and limit: -```python -# SQLAlchemy -user = await session.get(User, 1) -``` +=== "SQLAlchemy" -```python -# Ferro — raises if missing -from ferro import ModelDoesNotExist + ```python {test=compile reason="SQLAlchemy side of the comparison"} + stmt = select(User).where(User.age >= 18).order_by(User.age).limit(2) + result = await session.execute(stmt) + adults = result.scalars().all() + ``` -try: - user = await User.get(1) -except ModelDoesNotExist: - user = None +=== "Ferro" -# Ferro — optional (like session.get when no row) -user = await User.get_or_none(1) -``` + ```python + --8<-- "docs/examples/from_sqlalchemy.py:filter" + ``` -See the [Queries guide](../guide/queries.md) for the full predicate and builder API. +Both return the two youngest adults as a list. Ferro takes lambda predicates named after the model; see the [Queries guide](../guide/queries.md) for the other styles. -## Creating Records +Get by primary key. `session.get(User, pk)` returns `None` when the row is missing. Ferro's `User.get(pk)` raises `ModelDoesNotExist`, and `get_or_none` is the optional variant: -```python -# SQLAlchemy -async with session_factory() as session: - user = User(username="alice", email="alice@example.com") - session.add(user) - await session.commit() -``` +=== "SQLAlchemy" -```python -# Ferro -user = await User.create(username="alice", email="alice@example.com") -``` + ```python {test=compile reason="SQLAlchemy side of the comparison"} + user = await session.get(User, 1) + ``` -There is no unit of work to flush: `create()` inserts immediately and returns the instance with its primary key set. Ferro also covers the common session idioms directly: `bulk_create([...])` for batch inserts, `get_or_create(...)` and `update_or_create(...)` for upsert-style flows, `upsert(...)` for a true insert-or-update on the primary key (the analog of a PostgreSQL `ON CONFLICT DO UPDATE`), and `instance.refresh()` to re-read from the database (the rough analog of `session.refresh`). +=== "Ferro" -## Relationships + ```python + --8<-- "docs/examples/from_sqlalchemy.py:get" + ``` -```python -# SQLAlchemy -from sqlalchemy import ForeignKey -from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship +A hit returns the `User`. A miss raises, or returns `None` from `get_or_none`, which matches `session.get`. +See the [Queries guide](../guide/queries.md) for the full predicate and builder API. -class Base(DeclarativeBase): - pass +## Creating records +`create()` inserts at once and returns the instance, so there is nothing to flush. Ferro also covers the common session idioms directly: -class User(Base): - __tablename__ = "users" +- `bulk_create([...])` inserts a batch. +- `get_or_create(...)` and `update_or_create(...)` cover upsert-style flows. +- `upsert(...)` is an insert-or-update on the primary key, like PostgreSQL `ON CONFLICT DO UPDATE`. +- `instance.refresh()` re-reads the row, like `session.refresh`. - id: Mapped[int] = mapped_column(primary_key=True) - posts: Mapped[list["Post"]] = relationship(back_populates="author") +Look up a user, or create one: +```python +--8<-- "docs/examples/from_sqlalchemy.py:upserts" +``` -class Post(Base): - __tablename__ = "posts" +The user `ada` exists, so `created` is `False` and the stored row is unchanged. See the [Mutations guide](../guide/mutations.md) for each call. - id: Mapped[int] = mapped_column(primary_key=True) - author_id: Mapped[int] = mapped_column(ForeignKey("users.id")) - author: Mapped["User"] = relationship(back_populates="posts") -``` +## Relationships -=== "Assignment" +Relations are lazy and awaitable. A forward foreign key returns the instance, and a `BackRef` returns a chainable query: - ```python - # Ferro - from typing import Annotated +```python +--8<-- "docs/examples/from_sqlalchemy.py:relationships" +``` - from ferro import BackRef, Field, ForeignKey, Model, Relation +`post.author` returns the `User`, and `ada.posts` returns her posts, newest first. Many-to-many uses `ManyToMany(related_name=...)` on one side and `BackRef()` on the other; see the [Relationships guide](../guide/relationships.md). +## Transactions - class User(Model): - id: int | None = Field(default=None, primary_key=True) - posts: Relation[list["Post"]] = BackRef() +SQLAlchemy commits when `session.begin()` exits. Ferro's `transaction()` runs everything inside it on one connection and commits or rolls back together: +=== "SQLAlchemy" - class Post(Model): - id: int | None = Field(default=None, primary_key=True) - author: Annotated[User, ForeignKey(related_name="posts")] + ```python {test=compile reason="SQLAlchemy side of the comparison"} + async with session.begin(): + session.add(User(username="margaret", email="m@example.com", age=30)) + raise RuntimeError("abort") # rolls back ``` -=== "Annotated" +=== "Ferro" ```python - # Ferro - from typing import Annotated - - from ferro import BackRef, Field, ForeignKey, Model, Relation - - - class User(Model): - id: Annotated[int | None, Field(default=None, primary_key=True)] - posts: Relation[list["Post"]] = BackRef() - - - class Post(Model): - id: Annotated[int | None, Field(default=None, primary_key=True)] - author: Annotated[User, ForeignKey(related_name="posts")] + --8<-- "docs/examples/from_sqlalchemy.py:transactions" ``` -The `ForeignKey` annotation declares both the relation and the underlying `author_id` column (Ferro creates the shadow column for you). Accessing relations is lazy and awaitable: - -```python -author = await post.author # forward FK → instance -posts = await user.posts.all() # BackRef → chainable query -recent = await user.posts.order_by(lambda t: t.id, "desc").limit(5).all() -``` - -Many-to-many uses `ManyToMany(related_name=...)` on one side and `BackRef()` on the other — see the [Relationships guide](../guide/relationships.md). - -## Transactions - -```python -# SQLAlchemy -async with session.begin(): - session.add(User(username="alice")) - # commits on exit, rolls back on exception -``` - -```python -# Ferro -from ferro import transaction - -async with transaction(): - await User.create(username="alice") - # commits on exit, rolls back on exception -``` - -Same shape, no session: everything inside the block runs on one connection and commits or rolls back together. See the [Transactions guide](../guide/transactions.md). +In both, the raised error rolls the insert back, and no `margaret` row exists afterwards. See the [Transactions guide](../guide/transactions.md). ## Migrations Your Alembic chain keeps working through the switch. Ferro ships a bridge that builds a SQLAlchemy `MetaData` from your Ferro models, and its options keep Alembic's own comparator off Ferro's tables, so `alembic revision --autogenerate` writes the same changes Ferro's planner decides. Pass both in `env.py`: -```python -# alembic/env.py +```python {test=compile reason="an Alembic env.py: Alembic runs it, not the docs harness"} from alembic import context from ferro.migrations import ferro_options, get_metadata @@ -254,23 +212,23 @@ context.configure( ) ``` -Install the extra with `pip install "ferro-orm[alembic]"`. The bridge is the right door while the project is in transition, or while Alembic still manages SQLAlchemy tables beside your Ferro models; see [Alembic](../guide/schema/alembic.md). +The block hands Alembic Ferro's metadata. Install the extra with `pip install "ferro-orm[alembic]"`. The bridge fits while the project is in transition, or while Alembic still manages SQLAlchemy tables beside your Ferro models; see [Alembic](../guide/schema/alembic.md). -Once the models are all Ferro, you can move to Ferro's own [migrations](../guide/schema/migrations.md) (`pip install "ferro-orm[cli]"`), which also write backfills and SQLite table rebuilds. Bring each database to `alembic upgrade head` and [baseline it](adopting-migrations.md#coming-from-alembic); no DDL runs. For development and tests, `connect(url, auto_migrate=True)` creates tables without any migration files. See [Schema Management](../guide/schema/overview.md) and the [Migrations API](../api/migrations.md). +Once the models are all Ferro, move to Ferro's own [migrations](../guide/schema/migrations.md) (`pip install "ferro-orm[cli]"`), which also write backfills and SQLite table rebuilds. Bring each database to `alembic upgrade head` and [baseline it](../guide/schema/adopting-migrations.md#coming-from-alembic); no DDL runs. For development and tests, `connect(url, auto_migrate=True)` creates tables without migration files. See [Schema management](../guide/schema/overview.md) and the [Migrations API](../api/migrations.md). -## What Has No Ferro Equivalent Yet +## Features without a Ferro equivalent Some SQLAlchemy features map differently, or have no Ferro counterpart today: -- **Eager loading** (`selectinload` / `joinedload`) — forward-FK paths populate with [`include()`](../guide/queries.md#populating-relations-with-include) in one statement; `BackRef`/M2M collections still load lazily per access (reverse population is a future mechanism). -- **Partial column selects** (`select(User.id, User.name)`) — [`select(lambda u: (u.id, u.name))`](../guide/queries.md#selecting-a-column-subset) returns projected records; traversal and output aliases use the dict form. -- **`func.sum` / `avg` / `min` / `max` and `group_by()`** — aggregate methods on columns (`t.amount.sum()`), with [grouping derived from the projection](../guide/aggregations.md) — there is deliberately no `group_by()` chainer. `having()` is not built yet ([#291](https://github.com/syn54x/ferro-orm/issues/291)); filter groups in Python or use [raw SQL](../guide/raw-sql.md). -- **Atomic update expressions** — recipe form, not SQLAlchemy's `update().values(...)`: `update(lambda counter: {"n": counter.n + 1, "updated_at": now})`. Keyword `update()` stays literals-only. Postgres `.merge()` is shipped on the recipe door; `.concat()` is still coming. +- **Eager loading** (`selectinload` / `joinedload`): forward-FK paths populate with [`include()`](../guide/queries.md#populating-relations-with-include) in one statement; `BackRef` and many-to-many collections still load lazily per access. +- **Partial column selects** (`select(User.id, User.name)`): [`select(lambda user: (user.id, user.username))`](../guide/queries.md#selecting-a-column-subset) returns projected records; traversal and output aliases use the dict form. +- **`func.sum` / `avg` / `min` / `max` and `group_by()`**: aggregate methods on columns (`order.amount.sum()`), with [grouping derived from the projection](../guide/aggregations.md). There is deliberately no `group_by()` chainer. `having()` is not built yet; filter groups in Python or use [raw SQL](../guide/raw-sql.md). +- **Atomic update expressions**: written as a recipe, not SQLAlchemy's `update().values(...)`: `update(lambda counter: {"n": counter.n + 1, "updated_at": now})`. Keyword `update()` accepts literals only. `update()` recipes support Postgres `.merge()`; `.concat()` is planned. -For what's planned, see the [Roadmap](../roadmap.md). Where you hit a gap, `execute()` / `fetch_all()` give you full SQL with bound parameters. +Where you hit a gap, `execute()` and `fetch_all()` run full SQL with bound parameters. For what is planned, see the [Roadmap](../roadmap.md). -## See Also +## See also -- [Quickstart Tutorial](../getting-started/quickstart.md) — Ferro end to end in a few minutes -- [Queries guide](../guide/queries.md) — the full query-building API -- [Schema Management](../guide/schema/overview.md) — auto-migrate, migrations and the Alembic bridge +- [Quickstart](../getting-started/quickstart.md): Ferro end to end in a few minutes +- [Queries guide](../guide/queries.md): the full query-building API +- [Schema management](../guide/schema/overview.md): auto-migrate, migrations and the Alembic bridge diff --git a/docs/pages/howto/multiple-databases.md b/docs/pages/howto/multiple-databases.md index 189bbc29..dc57684a 100644 --- a/docs/pages/howto/multiple-databases.md +++ b/docs/pages/howto/multiple-databases.md @@ -1,73 +1,127 @@ -# Multiple Databases +--- +description: "Connect one process to two databases and route each model call to the right one." +--- -One process can hold several named connections — a primary application database, a read replica, an analytics warehouse — each with its own pool. The common case still works untouched: `await connect(url)` registers and selects `"default"`. Named connections make everything else explicit. +# How to use several databases in one app -## Registering Connections +This how-to connects one process to two databases, an application database and an analytics database, and routes each call to the right one. -Give each connection a `name`; mark one as `default` for unqualified operations: +Give each connection a name, mark one as the default, and route calls to each: ```python --8<-- "docs/examples/multiple_databases.py:connect" ``` -Each `connect()` call creates an independent pool, configurable per connection with `pool=PoolConfig(max_connections=...)`. You can change which named connection is the default later with [`set_default_connection(name)`](../api/connection.md). +It prints: -## Routing Queries + +```text +1 1 +``` + +Each database holds one metric: `signups` went to the default connection, `app`, through the session, and `page_views` went to `analytics` through `.using()`. [Named connections](../guide/connections.md#named-connections) in the Connections guide explains how naming and the default work, and [Sessions](../guide/connections.md#sessions) explains the session. + +The examples use one model, a named metric: + +=== "Assignment" + + ```python + --8<-- "docs/examples/multiple_databases.py:models" + ``` + +=== "Annotated" -Operations need a route: either an open session (ambient or explicit `session=`) or an explicit `using=`. Inside `async with engines.session():`, unqualified calls resolve to that session's connection — the default connection if you opened the session with no name. `Model.using(name)` returns a handle exposing the same API (`create`, `all`, `select`, `where`, `get`, `get_or_none`, `bulk_create`, `get_or_create`, `update_or_create`) pinned to the named connection, and it runs on its own — no session required: + ```python + --8<-- "docs/examples/multiple_databases_annotated.py:models" + ``` + +Each `connect()` call opens its own pool, sized with `pool=PoolConfig(...)`, and `auto_migrate=True` creates the `metric` table on each database. + +## Route calls to a connection + +Every call needs a route. Inside `async with engines.session():`, unqualified calls use the session's connection: the default connection when you open the session with no name. [`Model.using(name)`](../api/model.md#ferro.Model.using) routes one call and needs no session. + +To run a whole block on one named connection, open a session on it: ```python ---8<-- "docs/examples/multiple_databases.py:routing" +--8<-- "docs/examples/multiple_databases.py:session" ``` -Routing is per-call: `using()` doesn't change any global state, so two coroutines can talk to different databases concurrently without interfering. A `using()` call made without a session, as `analytics_metrics` is above, runs with no identity map — every load is a fresh instance, with no `a is b` guarantee across calls. See [Identity Map](../concepts/identity-map.md#no-session-no-identity) for what that trades away. +It prints: + + +```text +['page_views'] +``` -An explicit `using=` that names a connection different from the session you're ambiently inside of raises `ValueError` rather than silently running outside the session — and an operation with no session and no `using=` at all raises `RuntimeError`. There is no implicit default-connection fallback. +Only the analytics database has a metric above 100. -For session-first workflows, use `engines.session(name)` to pin a full block: +Inside a session, `.using()` must name the session's connection. Naming another one raises `ValueError` rather than running the call outside the session: ```python -import ferro +--8<-- "docs/examples/multiple_databases.py:conflict" +``` + +It prints: -async with ferro.engines.session("analytics") as s: - metrics = await s.query(Metric).where(lambda t: t.value > 0).all() + +```text +ValueError: Explicit `using='analytics'` conflicts with the ambient session bound to connection 'app'. Pass an explicit `session=` for that connection, or open a session on 'analytics'. ``` -## Transactions on Named Connections +A call with no session and no `using=` raises `RuntimeError`; Ferro never falls back to the default connection on its own. -`transaction(using=...)` pins a transaction to one named connection: +Routing is per call: `.using()` changes no global state, so two coroutines can use different databases at once. A `.using()` call outside a session has no identity map, so every load returns a new instance; see [Sessionless operations](../concepts/identity-map.md#sessionless-operations). + +## Transactions on a named connection + +`transaction(using=name)` pins a transaction to one connection. Unqualified calls inside the block run on it, and routing a call elsewhere raises `ValueError`: ```python --8<-- "docs/examples/multiple_databases.py:transaction" ``` -Everything inside the block — including raw SQL via `execute()` / `fetch_all()` — inherits that connection. Nested `transaction()` blocks inherit it too. Ferro does not support distributed transactions: one `transaction()` spans exactly one named connection, so writes to two databases are never atomic together. +It prints: -## Per-Connection Schema Setup + +```text +ValueError: Operations inside a transaction inherit the transaction connection +2 +``` -Schema creation targets one connection at a time: +The `clicks` metric landed on `analytics`, which now holds two metrics; the call routed to `app` was refused. Raw SQL through `execute()` and `fetch_all()`, and nested `transaction()` blocks, inherit the connection too. Ferro has no distributed transactions: one transaction spans one connection, so writes to two databases never commit together. -- `connect(url, name=..., auto_migrate=True)` runs auto-migration **on that connection** as part of connecting — each database gets tables for all registered models, as the example above shows. -- [`create_tables(using=...)`](../api/connection.md) creates tables explicitly on a named connection after the fact: +## Create tables on one connection + +`connect(url, name=..., auto_migrate=True)` creates the tables on that connection as it connects. To create them later, call `create_tables(using=name)`: ```python -from ferro import create_tables +--8<-- "docs/examples/multiple_databases.py:create-tables" +``` + +It prints: -await create_tables(using="analytics") + +```text +['metric'] ``` -Don't run schema creation concurrently through multiple names that point at the same physical database. For production schema changes, use [migrations](../guide/schema/migrations.md): a project whose models live in separate databases [configures each one](../reference/configuration.md#several-databases) with its own migrations directory, and every command names it with `--database` ([Deploying migrations](../guide/schema/deploying.md#several-databases)). +`create_tables()` returns a [`PassReport`](../api/connection.md#ferro.PassReport) of the statements it ran, here one `CREATE TABLE` for `metric`. Two connections that name the same database share its run lock, so their passes run one after the other ([Concurrent passes and the run lock](../guide/schema/auto-migrate.md#concurrent-passes-and-the-run-lock)). + +For production schema changes, use [migrations](../guide/schema/migrations.md): a project whose models live in separate databases [configures each one](../reference/configuration.md#several-databases) with its own migrations directory, and every command names it with `--database` ([Deploying migrations](../guide/schema/deploying.md#several-databases)). + +## Security and routing rules -## Practical Notes +!!! warning "Never route from untrusted input" + Choose connection names from constants or trusted server-side logic. A name taken from a request parameter or header lets the caller pick the database. -- **Typical roles.** A read replica for expensive list endpoints (`User.using("replica")`), an analytics database that receives event rows, or a separate service database owned by another team. -- **Keep credentials server-side.** Elevated service-role connections belong in configuration, not source control — and never make a service-role connection the default in a user-facing runtime. -- **Never route from untrusted input.** Don't pick the `using` name from request data. -- **Pools isolate roles, not request context.** A named connection isolates credentials and pooling; it does not provide per-request RLS/JWT context inside one shared pool. Objects loaded through an elevated connection can contain elevated data — filter before returning them to users. -- **No automatic routing.** Read/write splitting, cross-connection joins, and two-phase commit are not features; routing is always an explicit `using=`, `session=`, or `engines.session(name)` you name yourself — never inferred from query shape or request data. +- **Typical roles.** A read replica for expensive list endpoints (`Metric.using("replica")`), an analytics database that receives event rows, or a database another team owns. +- **Keep credentials server-side.** Elevated service-role connections belong in configuration, not source control, and a user-facing runtime never makes one its default. +- **Pools isolate roles, not request context.** A named connection separates credentials and pooling; it does not give each request its own row-security context inside one shared pool. Objects loaded through an elevated connection can hold elevated data, so filter them before returning them to users. +- **No automatic routing.** Ferro has no read/write splitting, cross-connection joins or two-phase commit. Every route is a `using=`, `session=` or `engines.session(name)` you write yourself. -## See Also +## See also -- [Connections & Databases guide](../guide/connections.md) — URLs, pools, and connection lifecycle -- [Transactions guide](../guide/transactions.md) — transaction semantics and nesting -- [Connection & Registry API](../api/connection.md) — `connect`, `PoolConfig`, `set_default_connection`, `create_tables` +- [Connections](../guide/connections.md): URLs, pools, sessions and the default connection +- [Transactions](../guide/transactions.md): commit, rollback and savepoints +- [Connection reference](../api/connection.md): `connect`, `PoolConfig`, `set_default_connection` and `create_tables` diff --git a/docs/pages/howto/pagination.md b/docs/pages/howto/pagination.md index 33c8546f..a2065276 100644 --- a/docs/pages/howto/pagination.md +++ b/docs/pages/howto/pagination.md @@ -1,122 +1,107 @@ -# Pagination +--- +description: "Split query results into pages by page number with limit() and offset(), or by position with after() and before()." +--- -Ferro supports the two standard pagination strategies: offset pagination with `limit()` / `offset()`, and keyset (cursor) pagination with a `where` filter on the sort key. +# How to paginate query results -## Offset Pagination +This page splits a query's results into pages two ways: by page number with `limit()` and `offset()`, and by position with `after()` and `before()`. Use page numbers when users jump to page 7 of a small table; use positions for feeds, infinite scroll, APIs and large tables. -The simplest approach: order the rows, skip `(page - 1) * per_page`, take `per_page`. +The examples page through seven articles, newest first. Declare the model, connect, and add the rows inside a session: -```python ---8<-- "docs/examples/pagination.py:offset" -``` +=== "Assignment" -Offset pagination is easy to implement and lets users jump to an arbitrary page number. It has two well-known costs: + ```python + --8<-- "docs/examples/pagination.py:model" + --8<-- "docs/examples/pagination.py:session" + ``` + +=== "Annotated" -- **Drift.** If rows are inserted or deleted between requests, page boundaries shift — a user paging through results can see duplicates or miss rows. -- **Deep offsets are expensive.** The database still scans and discards every skipped row, so `OFFSET 100000` does real work before returning anything. Latency grows with page depth. + ```python + --8<-- "docs/examples/pagination_annotated.py:model" + --8<-- "docs/examples/pagination_annotated.py:session" + ``` -For small datasets and admin-style page numbers, neither matters much. For feeds and large tables, use keyset pagination instead. +`published_on` carries an index because both kinds of paging read the table in `published_on` order. The blocks below run inside this session; [Sessions](../guide/connections.md#sessions) explains why every call needs one. -## Keyset (Cursor) Pagination +## Pages by number with `offset()` -Instead of skipping rows, remember the last value seen and filter past it: +Order the rows, skip `(page - 1) * per_page` of them, and take `per_page`: ```python ---8<-- "docs/examples/pagination.py:keyset" +--8<-- "docs/examples/pagination.py:offset" ``` -The client passes back the `id` of the last item it received (the cursor); the next query is a plain indexed range scan (`WHERE id > ? ORDER BY id LIMIT ?`). This makes keyset pagination: - -- **Stable** — inserts and deletes elsewhere in the table don't shift the window, so no duplicates or gaps. -- **Fast at any depth** — page 1 and page 10,000 cost the same, since the database seeks directly to the cursor instead of scanning skipped rows. +Each call reads one page. Page 2, three to a page, holds the fourth to sixth newest articles: -The trade-off: clients can only walk forward (or backward) from a cursor — there is no "jump to page 57". - -## Using It in an API - -A keyset endpoint returns the items plus the cursor for the next request: - -=== "Assignment" +```python +--8<-- "docs/examples/pagination.py:offset-call" +``` - ```python - from fastapi import FastAPI, Query +Two articles share 12 January, so the second `order_by()` on `id` decides which comes first. Without it the database may return tied rows in either order, and a row can move from one page to the next between two requests. - from ferro import Field, Model +Offset paging has two costs: +- **Drift.** A row added or deleted between two requests shifts every later page by one, so the reader sees a row twice or misses one. +- **Deep pages are slow.** The database reads and discards every skipped row, so `offset(100_000)` reads 100,000 rows before it returns any. - class Article(Model): - id: int | None = Field(default=None, primary_key=True) - title: str +## Keyset (cursor) pagination +[`after()`][ferro.query.builder.Query.after] starts a page just after a row you already have. Build the ordered query once, read the first page, then pass its last row to `after()`: - app = FastAPI() +```python +--8<-- "docs/examples/pagination.py:after" +``` +The second page starts with "Keyset paging", the article after "Soft deletes". The database seeks straight to that position, so page 1,000 costs what page 2 costs, and rows added or deleted elsewhere do not shift the page. - @app.get("/articles") - async def list_articles( - cursor: int | None = Query(None), - limit: int = Query(20, ge=1, le=100), - ): - query = Article.select() if cursor is None else Article.where(lambda t: t.id > cursor) - items = await query.order_by(lambda t: t.id).limit(limit).all() - return { - "items": items, - "next_cursor": items[-1].id if items else None, - "has_more": len(items) == limit, - } - ``` +[`before()`][ferro.query.builder.Query.before] goes the other way. Pass it the first row of a page to get the page before it, still newest first: -=== "Annotated" +```python +--8<-- "docs/examples/pagination.py:before" +``` - ```python - from typing import Annotated +`previous_page` holds the same three articles as `first_page`. The order keys must include the primary key, so that every row has exactly one position; `after()` and `before()` raise `ValueError` when it is missing. [Paging with `after()` and `before()`](../guide/queries.md#paging-with-after-and-before) lists the other rules. - from fastapi import FastAPI, Query +## A paginated API endpoint - from ferro import Field, Model +An API hands the client a cursor: a string that encodes the last row's position, which the client sends back for the next page. [`position_of()`][ferro.query.builder.Query.position_of] reads a row's position as a tuple of its order-key values, and `after()` accepts that tuple in place of the row: +```python +--8<-- "docs/examples/pagination.py:cursor" +``` - class Article(Model): - id: Annotated[int | None, Field(default=None, primary_key=True)] - title: str +`article_page()` asks for one row more than the page holds. If that row comes back, there is a next page and `next_cursor` points at the page's last row; if it does not, `next_cursor` is `None`, even when the last page is exactly full. Three calls walk all seven articles: +```python +--8<-- "docs/examples/pagination.py:cursor-calls" +``` - app = FastAPI() +The third page holds the one remaining article, and its `next_cursor` is `None`. +A FastAPI endpoint opens a session for the request and returns `article_page()`'s result. Ferro models are Pydantic models, so FastAPI serializes the articles directly: - @app.get("/articles") - async def list_articles( - cursor: int | None = Query(None), - limit: int = Query(20, ge=1, le=100), - ): - query = Article.select() if cursor is None else Article.where(lambda t: t.id > cursor) - items = await query.order_by(lambda t: t.id).limit(limit).all() - return { - "items": items, - "next_cursor": items[-1].id if items else None, - "has_more": len(items) == limit, - } - ``` +```python {test=compile reason="needs FastAPI, which the docs test environment does not install; article_page() runs in docs/examples/pagination.py"} +from fastapi import FastAPI, Query -Clients call `/articles`, then `/articles?cursor=` until `has_more` is false. Ferro models are Pydantic models, so FastAPI serializes them directly. +from ferro import engines +from myapp.articles import article_page -## Tips +app = FastAPI() -- **Always `order_by`.** Without an explicit ordering, the database returns rows in whatever order it likes, and `limit`/`offset` windows become non-deterministic. Order by a unique column (or end the ordering with one) so ties can't straddle a page boundary. -- **Index your sort columns.** Both strategies turn into an ordered scan over the sort key. Declare an index on non-primary-key sort columns, e.g. `created_at: datetime = Field(index=True)`. Primary keys are already indexed. -- **Cap page sizes.** Enforce a maximum `limit` at the API boundary (as the FastAPI example does with `le=100`) so a single request can't ask for the whole table. -## Choosing a Strategy +@app.get("/articles") +async def list_articles( + cursor: str | None = None, + limit: int = Query(20, ge=1, le=100), +): + async with engines.session(): + return await article_page(cursor=cursor, limit=limit) +``` -| | Offset | Keyset (cursor) | -|---|---|---| -| Jump to arbitrary page | Yes | No | -| Stable under concurrent writes | No (drift) | Yes | -| Cost of deep pages | Grows with depth | Constant | -| Implementation effort | Trivial | Small (track a cursor) | -| Best for | Small tables, admin UIs, page numbers | Feeds, infinite scroll, large tables, APIs | +Clients call `/articles`, then `/articles?cursor=` until `next_cursor` is `null`. `le=100` caps the page size, so one request cannot ask for the whole table. [A session per request](../guide/connections.md#a-session-per-request) shows other ways to open the session. -## See Also +## See also -- [Queries guide](../guide/queries.md) — filtering, ordering, `limit` and `offset` -- [Queries API](../api/queries.md) — the full `Query` builder reference +- [Paging with `after()` and `before()`](../guide/queries.md#paging-with-after-and-before): the rules for positions, `None` order keys and related columns. +- [`Query.after()`][ferro.query.builder.Query.after], [`Query.before()`][ferro.query.builder.Query.before], [`Query.limit()`][ferro.query.builder.Query.limit] and [`Query.offset()`][ferro.query.builder.Query.offset] in the API reference. diff --git a/docs/pages/howto/soft-deletes.md b/docs/pages/howto/soft-deletes.md index 2e5e3d47..c5da579d 100644 --- a/docs/pages/howto/soft-deletes.md +++ b/docs/pages/howto/soft-deletes.md @@ -1,63 +1,82 @@ -# Soft Deletes +--- +description: "Mark rows as deleted instead of removing them, and restore or filter them later with a mixin." +--- -Soft deletes mark rows as deleted instead of removing them, so records can be audited or restored later. In Ferro this is two flag fields plus a small mixin that supplies the lifecycle methods. +# How to soft-delete rows -## The Pattern +This page marks rows as deleted instead of removing them, so they can be audited or restored later: two fields on the model hold the mark, and a mixin supplies `soft_delete()`, `restore()` and `active()`. + +## The `SoftDeleteMixin` + +Declare `is_deleted` and `deleted_at` on the model, and put the mixin first among its bases: === "Assignment" ```python --8<-- "docs/examples/soft_deletes.py:model" + --8<-- "docs/examples/soft_deletes.py:session" ``` === "Annotated" ```python --8<-- "docs/examples/soft_deletes_annotated.py:model" + --8<-- "docs/examples/soft_deletes_annotated.py:session" ``` -- **Fields on the concrete model.** `is_deleted` and `deleted_at` are declared on `Invoice` itself. Every model that wants soft deletes repeats these two declarations. -- **`SoftDeleteMixin` for behavior.** `soft_delete()` and `restore()` flip the flags and `save()`; `active()` is a classmethod returning a normal [`Query`](../api/queries.md) filtered to non-deleted rows, so it chains like any other query. +The table now holds three live invoices. The blocks below run inside this session; [Sessions](../guide/connections.md#sessions) explains why every call needs one. + +`soft_delete()` and `restore()` set the two fields and save. `active()` returns an ordinary [`Query`][ferro.query.builder.Query] filtered to rows that are not deleted, so it chains like any other query. Its lambda names its parameter `row` rather than after a model, because the mixin serves every model that uses it. + +The mixin is a plain class, not a `Model` subclass, so every model that uses it declares the two fields itself. A `Model` base class would get a table of its own; [Fields on a model base class](../guide/models-and-fields.md#fields-on-a-model-base-class) shows that table. -The mixin is a plain class, not a `Model` subclass. Ferro registers a table schema for each model class as it is defined, so a `Model` base class cannot contribute fields to subclasses — declare fields on each concrete model and keep shared behavior in the mixin. (See the [Timestamps how-to](timestamps.md) for the same pattern.) +## Deleting and restoring -## Usage +Soft-delete an invoice, count, then restore it: ```python --8<-- "docs/examples/soft_deletes.py:usage" ``` -`soft_delete()` keeps the row in the table — `Invoice.select().count()` still sees it — while `Invoice.active()` excludes it. `restore()` brings it back. +After `soft_delete()`, `active()` counts two invoices while `select()` still counts three: the row stays in the table. `restore()` brings it back to `active()`. -## Querying +## Querying live rows -Use `active()` as the entry point everywhere you would otherwise use `select()` or `where()`: +Start every query that should skip deleted rows from `active()`, and chain from there. With INV-001 soft-deleted again: ```python -unpaid = await Invoice.active().where(lambda t: t.number.like("INV-%")).all() -trashed = await Invoice.where(lambda t: t.is_deleted == True).all() # noqa: E712 +--8<-- "docs/examples/soft_deletes.py:querying" ``` -Two things to remember: +`live` holds the two live invoices from INV-002 on, and `trashed` holds only the soft-deleted INV-001. -- **Nothing is filtered automatically.** `Invoice.all()`, `Invoice.select()`, `Invoice.get(pk)` and relationship traversals still return soft-deleted rows. The `active()` discipline is a convention your code must follow. -- **Batch and instance deletes bypass soft delete.** `await invoice.delete()` and `await Invoice.where(...).delete()` issue real `DELETE` statements — the mixin only adds `soft_delete()`, it does not intercept the built-in delete paths. Reach for the hard delete deliberately (e.g. retention cleanup), not by accident. +!!! warning "Nothing is filtered automatically" + `Invoice.all()`, `Invoice.select()`, `Invoice.get(pk)` and relationship traversals still return soft-deleted rows. Only queries that start from `active()` skip them. + +`await invoice.delete()` and `Invoice.where(...).delete()` still issue a real `DELETE`: the mixin adds `soft_delete()` beside them and changes neither. + +## Trade-offs -If you want soft-deleted rows to age out, a periodic job can purge them for real: +**A unique column still holds a soft-deleted row's value.** `number` is unique, so while INV-001 is soft-deleted, a new invoice cannot take its number: ```python -async def purge_deleted() -> int: - return await Invoice.where(lambda t: t.is_deleted == True).delete() # noqa: E712 +--8<-- "docs/examples/soft_deletes.py:unique" ``` -## Trade-offs +It prints `INV-001 is still taken`. When a flow must reuse the value, hard-delete the row in that flow, or change the value as part of the soft delete (append the primary key, for example). Checking for a live duplicate in application code before inserting is not safe: two requests can both pass the check and both insert. + +**Soft-deleted rows keep their space.** They stay in the table and its indexes, so scans, backups and indexes grow. Pair soft deletes with a retention period, and a job that purges the rows past it with a real `DELETE`: + +```python +--8<-- "docs/examples/soft_deletes.py:purge" +``` + +The purge removes INV-001 for good and leaves the two live invoices. -- **Unique constraints see soft-deleted rows.** A unique column like `number: str = Field(unique=True)` still holds the value after a soft delete, so creating a replacement with the same number fails. Options: hard-delete in that flow, rename the value on soft delete (e.g. suffix the primary key), or drop the database-level constraint and enforce uniqueness among active rows in application code. -- **Data growth.** Soft-deleted rows stay in the table and its indexes, so table scans, backups, and index sizes grow forever unless you purge. Pair soft deletes with a retention policy. -- **Privacy.** "Deleted" data is still data. If users expect deletion to remove personal information, soft delete alone does not satisfy that — schedule a real purge. +**Soft-deleted data is still stored.** If users expect deletion to remove their personal data, a soft delete does not do that; purge the row for real. -## See Also +## See also -- [Timestamps how-to](timestamps.md) — the mixin pattern explained in detail -- [Queries guide](../guide/queries.md) — building filtered queries -- [Mutations guide](../guide/mutations.md) — `save()`, `delete()`, and batch operations +- [How to record created and updated times](timestamps.md): the same mixin pattern for timestamps. +- [Fields on a model base class](../guide/models-and-fields.md#fields-on-a-model-base-class): why the fields go on each model. +- [Batch updates and deletes](../guide/mutations.md#batch-updates-and-deletes) and [`Query.delete()`][ferro.query.builder.Query.delete] in the API reference. diff --git a/docs/pages/howto/testing.md b/docs/pages/howto/testing.md index 9635f22c..d84c3790 100644 --- a/docs/pages/howto/testing.md +++ b/docs/pages/howto/testing.md @@ -1,37 +1,26 @@ -# Testing +--- +description: "Set up pytest so each test gets a fresh in-memory database with a session open, plus factories and a PostgreSQL fixture." +--- -Connect to a fresh in-memory SQLite database per test, run your code, and reset the engine on teardown. The rest of this page is the pytest fixture, a factory for test data, and how to isolate tests against a real PostgreSQL database. +# How to test a Ferro app with pytest -## Test Setup +This page sets up pytest so that each test gets a fresh in-memory SQLite database with a session open, then shows a first test, factories for test data, and a fixture for tests against PostgreSQL. -Put two fixtures in your `conftest.py`: +## The `db` fixture + +Put two fixtures in `tests/conftest.py`: ```python --8<-- "docs/examples/testing_conftest.py:fixtures" ``` -How this works: - -- **`db`** connects to a fresh in-memory SQLite database for each test and holds an [`engines.session()`](../guide/connections.md#sessions-recommended) open around the test body, so ORM calls inside the test resolve against that session without extra boilerplate. `auto_migrate=True` creates tables for every registered model, so there is no schema setup to maintain. On teardown, [`reset_engine()`](../api/connection.md) closes the pool and clears the identity map, guaranteeing no state leaks between tests. -- **`db_transaction`** layers a [`transaction()`](../guide/transactions.md) on top. Everything inside the test shares one connection, which gives you connection affinity for the duration of the test. Use it when a test mixes ORM calls with raw SQL that must observe the same uncommitted state. +`db` connects to a new in-memory SQLite database and holds an [`engines.session()`](../guide/connections.md#sessions) open while the test runs, so the test's ORM calls need no session of their own. `auto_migrate=True` creates a table for every model your code has imported, so there is no schema setup to keep up to date. On teardown, [`reset_engine()`][ferro.reset_engine] closes the connection and clears the identity map, so no rows or cached instances carry over to the next test. -Since each test gets its own database, most tests only need `db`. +`db_transaction` runs the test inside one [`transaction()`](../guide/transactions.md) and yields its handle. The handle's `fetch_one()`, `fetch_all()` and `execute()` run raw SQL on the test's own connection, so they see what the test's ORM calls wrote before anything is committed. When the test ends, the fixture raises `RollBack` inside the transaction, which rolls back every write. Each test already has a database of its own, so most tests need only `db`. ## Configuring pytest-asyncio -Ferro is async, so tests are `async def` functions. With `asyncio_mode = auto`, pytest-asyncio runs them without per-test decorators: - -```ini -# pytest.ini -[pytest] -asyncio_mode = auto -testpaths = tests -python_files = test_*.py -python_classes = Test* -python_functions = test_* -``` - -Or in `pyproject.toml`: +Ferro is async, so tests are `async def` functions, run by [pytest-asyncio](https://pytest-asyncio.readthedocs.io/). Set its mode to `auto` in `pyproject.toml`, and it runs every async test and fixture without a decorator: ```toml [tool.pytest.ini_options] @@ -39,139 +28,143 @@ asyncio_mode = "auto" testpaths = ["tests"] ``` -If you prefer `asyncio_mode = strict`, mark each test with `@pytest.mark.asyncio`. +`pytest.ini` takes the same `asyncio_mode = auto` line under `[pytest]`. In `strict` mode, decorate each async fixture with `@pytest_asyncio.fixture` instead of `@pytest.fixture`, and mark each test with `@pytest.mark.asyncio`. -## Writing Your First Test +## The models -Request the `db` fixture and use your models directly: +The tests on this page use two models, kept in `myapp/models.py`: -```python +=== "Assignment" + + ```python + --8<-- "docs/examples/testing_models.py:models" + ``` + +=== "Annotated" + + ```python + --8<-- "docs/examples/testing_models_annotated.py:models" + ``` + +`username` and `email` are unique, which the tests below rely on. + +## A first test + +Request the `db` fixture and use the models directly. A second insert of the same email raises [`UniqueViolationError`][ferro.UniqueViolationError], which `pytest.raises` expects: + +```python {test=pytest path="tests/test_users.py"} +import pytest + +from ferro import UniqueViolationError from myapp.models import User async def test_create_user(db): - user = await User.create(username="testuser", email="test@example.com") + user = await User.create(username="ada", email="ada@example.com") - assert user.id is not None - assert user.username == "testuser" - - # Verify it round-trips through the database - found = await User.where(lambda t: t.username == "testuser").first() + found = await User.where(lambda user: user.username == "ada").first() assert found is not None assert found.id == user.id + + +async def test_email_is_unique(db): + await User.create(username="ada", email="ada@example.com") + + with pytest.raises(UniqueViolationError): + await User.create(username="grace", email="ada@example.com") ``` -Constraint violations surface as exceptions from the engine: +Run the file: -```python -import pytest + +```text +$ pytest -q tests/test_users.py +.. [100%] +2 passed in 0.04s +``` + +Both tests create `ada@example.com`, and neither sees the other's row: each starts with an empty database. + +## Raw SQL in the test's transaction + +Request `db_transaction` instead of `db` when a test checks its work with raw SQL. The handle it yields reads on the test's connection: +```python {test=pytest path="tests/test_raw_sql.py"} from myapp.models import User -async def test_user_unique_email(db): - await User.create(username="user1", email="same@example.com") +async def test_raw_count_sees_new_user(db_transaction): + await User.create(username="ada", email="ada@example.com") - with pytest.raises(Exception): - await User.create(username="user2", email="same@example.com") + row = await db_transaction.fetch_one('SELECT count(*) AS total FROM "user"') + assert row == {"total": 1} ``` +The raw count sees the new user inside the transaction, and the fixture rolls the row back when the test ends. + ## Factories -For tests that need realistic object graphs, a small factory class keeps setup terse without pulling in a library: +A test that needs a post also needs its author. A factory class per model fills in the fields a test does not care about: -```python +```python {test=pytest path="tests/test_posts.py"} from typing import Any from myapp.models import Post, User class UserFactory: - _counter = 0 + count = 0 @classmethod - async def create(cls, **kwargs: Any) -> User: - cls._counter += 1 + async def create(cls, **fields: Any) -> User: + cls.count += 1 defaults = { - "username": f"user_{cls._counter}", - "email": f"user{cls._counter}@example.com", + "username": f"user{cls.count}", + "email": f"user{cls.count}@example.com", } - defaults.update(kwargs) - return await User.create(**defaults) + return await User.create(**(defaults | fields)) class PostFactory: - _counter = 0 + count = 0 @classmethod - async def create(cls, **kwargs: Any) -> Post: - cls._counter += 1 - - # Auto-create an author when none is provided - if "author" not in kwargs: - kwargs["author"] = await UserFactory.create() + async def create(cls, **fields: Any) -> Post: + cls.count += 1 + if "author" not in fields: + fields["author"] = await UserFactory.create() + defaults = {"title": f"Post {cls.count}", "content": "Test content"} + return await Post.create(**(defaults | fields)) - defaults = {"title": f"Post {cls._counter}", "content": "Test content"} - defaults.update(kwargs) - return await Post.create(**defaults) +async def test_post_gets_an_author(db): + post = await PostFactory.create(title="Custom title") -async def test_post_with_author(db): - post = await PostFactory.create(title="Custom Title") - assert (await post.author) is not None + author = await post.author + assert post.title == "Custom title" + assert author.username.startswith("user") ``` -Override only the fields the test cares about; the counters keep unique-constrained fields distinct. - -## Testing Against Postgres - -SQLite in memory covers most logic, but behavior that depends on native Postgres types, casts, or constraints should run against a real PostgreSQL database. +The post keeps the title the test passed and gets a generated author. The counters keep the unique `username` and `email` values distinct across every call in the run. -### Schema isolation with `?ferro_search_path=...` +## Testing against Postgres -Appending `ferro_search_path=` to a Postgres connection URL makes Ferro run `SET search_path TO ` on every pooled connection. All tables created by `auto_migrate` (and all queries) then live in that schema, so many test runs can share one physical database without colliding. - -The schema must already exist — create it before connecting, and drop it on teardown: +In-memory SQLite covers most logic. Behaviour that depends on Postgres types, casts or constraints needs a real PostgreSQL database. This fixture gives each test a schema of its own in one shared database, through the URL's [`ferro_search_path`](../guide/connections.md#postgres-schema-isolation) parameter: ```python -import uuid - -import pytest - -from ferro import connect, engines, execute, reset_engine - -POSTGRES_URL = "postgresql://localhost:5432/app_test" - - -@pytest.fixture -async def pg_db(): - schema = f"test_{uuid.uuid4().hex[:8]}" - - # Create the schema with a throwaway connection - await connect(POSTGRES_URL) - async with engines.session(): - await execute(f'CREATE SCHEMA "{schema}"') - reset_engine() - - # Reconnect with the schema as the search path - await connect(f"{POSTGRES_URL}?ferro_search_path={schema}", auto_migrate=True) - async with engines.session(): - yield - - await execute(f'DROP SCHEMA "{schema}" CASCADE') - reset_engine() +--8<-- "docs/examples/testing_conftest_postgres.py:postgres" ``` -Schema names passed through `ferro_search_path` must contain only ASCII letters, digits, and underscores; anything else is rejected at connect time. +`pg_db` creates the schema on a first connection, reconnects with `ferro_search_path` set to it so that `auto_migrate=True` creates the tables there, and drops the schema with everything in it when the test ends. `with_search_path()` adds the parameter beside any the URL already has, such as `sslmode=require`. Set `TEST_POSTGRES_URL` to point the fixture at your test database. -## Tests in a Project with Migrations +## Projects with migrations The fixtures above stay as they are when a project adopts [migrations](../guide/schema/migrations.md). A fresh test database has never run a migration, so `auto_migrate=True` is not refused on it ([one door per database](../guide/schema/overview.md#one-door-per-database)), and it builds the schema the models declare. -That schema is the head of your migrations only while the models and the migrations agree, so run `ferro migrate check` in CI beside the tests. And `auto_migrate=True` never runs a migration's own SQL, its backfill or its down: test those with the migration test harness, on a database it stands at a named migration. See [Testing migrations](../guide/schema/testing.md). +That schema is the schema your migrations end at only while the models and the migrations agree, so run `ferro migrate check` in CI beside the tests. `auto_migrate=True` never runs a migration's own SQL, backfill or down step. Test those with the migration test harness, which stands a database at a named migration; see [Testing migrations](../guide/schema/testing.md). -## See Also +## See also -- [Transactions guide](../guide/transactions.md) — semantics of `transaction()` and connection affinity -- [Connections & Databases guide](../guide/connections.md) — connection URLs and `auto_migrate` -- [Connection & Registry API](../api/connection.md) — `connect`, `reset_engine`, and friends +- [Transactions](../guide/transactions.md): what `transaction()` commits and rolls back. +- [Connections](../guide/connections.md): connection URLs, sessions and `auto_migrate`. +- [`connect()`][ferro.connect], [`reset_engine()`][ferro.reset_engine] and [`transaction()`][ferro.transaction] in the API reference. diff --git a/docs/pages/howto/timestamps.md b/docs/pages/howto/timestamps.md index e760a17f..a434d3de 100644 --- a/docs/pages/howto/timestamps.md +++ b/docs/pages/howto/timestamps.md @@ -1,76 +1,72 @@ -# Timestamps +--- +description: "Give a model created_at and updated_at fields that Ferro sets on create and on every save()." +--- -Track when rows are created and last modified with `created_at` / `updated_at` fields: `created_at` is filled by a field default, and `updated_at` is refreshed by a small mixin that hooks `save()`. +# How to record created and updated times -## The Pattern +This page gives a model `created_at` and `updated_at` fields: a field default sets both when the row is created, and a mixin sets `updated_at` again in every `save()`. + +## The `TimestampMixin` + +Declare both fields on the model with a default of the current UTC time, and put the mixin first among the bases so its `save()` runs before `Model.save()`: === "Assignment" ```python --8<-- "docs/examples/timestamps.py:model" + --8<-- "docs/examples/timestamps.py:session" ``` === "Annotated" ```python --8<-- "docs/examples/timestamps_annotated.py:model" + --8<-- "docs/examples/timestamps_annotated.py:session" ``` -Two pieces work together: - -- **Field defaults on the concrete model.** `created_at` and `updated_at` are declared on `Note` itself with `default_factory=utcnow`, so both are set when an instance is constructed. -- **`TimestampMixin` for behavior.** The mixin overrides `save()` to touch `updated_at` before delegating to `Model.save()`. It is a *plain class* — not a `Model` subclass — and contributes only behavior, never fields. +The new note's `created_at` is an aware datetime in UTC. `create()` saves through `save()`, so the mixin runs for it too. The blocks below run inside this session; [Sessions](../guide/connections.md#sessions) explains why every call needs one. -Every model that wants timestamps repeats the same two field declarations and adds the mixin to its bases (mixin first, so its `save()` wins in the MRO). +The mixin is a plain class, not a `Model` subclass, so every model that uses it declares the two fields itself. A `Model` base class would get a table of its own; [Fields on a model base class](../guide/models-and-fields.md#fields-on-a-model-base-class) shows that table. -!!! note "Why a mixin instead of a `Model` base class?" +## Updating a note - You might expect to declare the fields once on a shared base, e.g. `class Timestamped(Model)` with `created_at` / `updated_at`, and inherit from it. Ferro does not support this: the ORM registers a table schema for each model class as it is defined, so fields declared on a `Model` base class are not contributed to its subclasses' tables. Keep shared *behavior* in a plain mixin and declare *fields* on each concrete model. - -## Usage +Change a field and save, and the mixin moves `updated_at` forward: ```python --8<-- "docs/examples/timestamps.py:usage" ``` -`created_at` is set once when the instance is created; every subsequent `save()` advances `updated_at`. Note that the mixin only hooks instance `save()` (which `create()` paths go through) — batch updates like `Note.where(...).update(...)` write columns directly and will not touch `updated_at` unless you set it explicitly. Use a recipe with the imported `now` singleton for the database clock: - -```python -from ferro import now +`created_at` keeps the time the note was created, and `updated_at` is now later than before the save. -await Note.where(lambda note: note.id == nid).update( - lambda note: {"updated_at": now} -) -``` +## Batch updates -## Timezone Notes +!!! warning "Batch updates skip the mixin" + `Note.where(...).update(...)` writes its columns in one `UPDATE` statement without loading the rows, so no `save()` runs and `updated_at` keeps its old value unless the update sets it. -Store UTC, always. The example's `utcnow()` helper returns a timezone-aware datetime: +Pass `updated_at=utcnow()` beside the columns the update changes: ```python -from datetime import UTC, datetime +--8<-- "docs/examples/timestamps.py:batch" +``` +The archived note now has the latest `updated_at`, so it sorts first, and it reads back as an aware datetime that compares with `utcnow()`. Ferro stores `utcnow()` in an update exactly as `save()` stores it, so rows touched either way sort together. -def utcnow() -> datetime: - return datetime.now(UTC) -``` +!!! warning "`now` on SQLite" + The database-clock recipe `update(lambda note: {"updated_at": now})` writes SQLite's own time format, whole seconds with no time zone (`2026-10-19 14:56:27`), while `save()` writes `2026-10-19T14:56:27.736246Z`. SQLite compares the two as text, so such a row sorts before every row `save()` wrote on the same day, however much earlier. It also reads back as a naive datetime, which raises `TypeError` when compared with `utcnow()`. Use `utcnow()` on SQLite; on Postgres, `now` stores a `timestamptz` like any other. + +## Storing UTC -Avoid naive `datetime.now()` — it captures the server's local clock, which makes values ambiguous and breaks comparisons across hosts and DST changes. Keep storage in UTC and convert to the user's timezone only at the display layer. +`utcnow()` returns `datetime.now(UTC)`, an aware datetime. A naive `datetime.now()` reads the server's local clock, so its values change meaning between hosts and across daylight-saving changes. Store UTC and convert to the reader's time zone when you display the value. -### Naive vs timezone-aware columns +## Naive and aware columns -A `datetime` field maps to Postgres `timestamptz` (SQLite stores both the same -way). If you point a model at a pre-existing plain `timestamp` (no time zone) -column, Ferro will **not** silently convert it — auto-migrate warns and leaves the -column untouched, because a `timestamp` → `timestamptz` cast reinterprets stored -values under the connection's timezone and can shift your data. +A `datetime` field is a `timestamptz` column on Postgres; SQLite stores both kinds as text. If a model's `datetime` field points at an existing `timestamp` column (no time zone), auto-migrate warns and leaves the column as it is: casting `timestamp` to `timestamptz` reads the stored values in the connection's time zone and can shift them. - To keep the column naive, declare the field with `db_type="timestamp"`. -- To convert it intentionally, run a reviewed migration (Alembic) with an explicit - source timezone, e.g. `USING occurred_at AT TIME ZONE 'UTC'`. +- To convert it, write the cast yourself with an explicit source time zone, such as `USING occurred_at AT TIME ZONE 'UTC'`, in a [hand-written migration step](../guide/schema/migrations.md#hand-written-steps) or an Alembic revision. -## See Also +## See also -- [Models & Fields guide](../guide/models-and-fields.md) — field defaults and `default_factory` -- [Soft Deletes how-to](soft-deletes.md) — the same mixin pattern applied to deletion -- [Mutations guide](../guide/mutations.md) — instance `save()` vs batch `update()` +- [Fields on a model base class](../guide/models-and-fields.md#fields-on-a-model-base-class): why the fields go on each model. +- [How to soft-delete rows](soft-deletes.md): the same mixin pattern for deletion. +- [Batch updates and deletes](../guide/mutations.md#batch-updates-and-deletes) and [`Query.update()`][ferro.query.builder.Query.update] in the API reference. diff --git a/docs/pages/howto/upgrade-guide.md b/docs/pages/howto/upgrade-guide.md deleted file mode 100644 index 6034fd33..00000000 --- a/docs/pages/howto/upgrade-guide.md +++ /dev/null @@ -1,490 +0,0 @@ - - -# Upgrade Guide - -This is the curated path for upgrading Ferro across releases that changed -behavior. The [changelog](../changelog.md) is the exhaustive per-version record; -this guide is the task-oriented "here is the work" companion for the boundaries -that need it. - -Find your current version below, then work **top to bottom** through every -boundary section you cross. Each section describes the settled end state — what -the API does now, and what you change to get there. - -## Which sections apply to you - -| You are on | Read | -| --- | --- | -| `0.14.x`, upgrading to `0.15` | [Upgrading to 0.15](#upgrading-to-015) | -| `0.12.x` or `0.13.x`, upgrading to `0.15` | Both sections, top to bottom | -| `0.12.x` or `0.13.x`, upgrading to `0.14` | [Upgrading to 0.14](#upgrading-to-014) | -| `0.11.x` or earlier, upgrading to `0.14` | Both sections, top to bottom | -| `0.11.x` or earlier, stopping at `0.12` | [Upgrading to 0.12](#upgrading-to-012) | -| `0.11.x` or earlier, stopping at `0.13` | [Upgrading to 0.12](#upgrading-to-012), plus the [connection-routing note](#connection-routing-requires-a-session) in Upgrading to 0.14 | - -Each section includes a copy/paste prompt for a coding agent. If you are crossing -more than one boundary, run the prompts **newest first, one at a time**, and -verify your suite between them. - -## Before you start - -Turn deprecation warnings into failures on a throwaway branch so nothing slips -through: - -```bash -uv run pytest -W error::DeprecationWarning -``` - -This catches every **deprecation-based** change. It does **not** catch the silent -behavioral changes in `0.14` (`create()`/`save()` no longer upsert; -`limit()`/`offset()` on a mutation now raise) — those emit no warning and must be -reviewed by reading the relevant section below. - -## Upgrading to 0.15 - -`0.15` adds JSONB support and makes **JSONB the default storage** for derived -json-family fields (`dict`/`list`/nested-model with no explicit `db_type`) on -PostgreSQL. Your model definitions do not change; your PostgreSQL columns might. -SQLite users are unaffected — both JSON tokens store identically there. - - - -??? example "Automate this: copy this prompt to your coding agent" - - ```text - You are helping upgrade a Python codebase from Ferro ORM 0.14.x to 0.15. - Reference: https://ferro-orm.x54.sh/howto/upgrade-guide/#upgrading-to-015 - - The one breaking change: derived dict/list/nested-model fields now store as - JSONB on PostgreSQL (previously plain json). On the first connect with - migrate_updates=True, each existing plain-json column with a derived - declaration is rewritten in place with ALTER ... TYPE jsonb USING ... — - values survive, but jsonb does not preserve dict key order. - - For each model field whose annotation is dict/list/a nested Pydantic model - and which has NO explicit db_type: - 1. If the code iterates that field's keys relying on insertion order, or - compares stored JSON text byte-for-byte, add db_type="json" to keep - plain-json storage (zero migration). - 2. Otherwise leave it — the one-time ALTER upgrades it to JSONB. - - List every affected field as file:line with your keep/upgrade call and let - me review before connecting to production with migrate_updates=True. - Alembic users: autogenerate will propose the same type changes; review the - migration instead. - ``` - -### Derived JSON storage is JSONB on PostgreSQL - -A bare `payload: dict` field now creates (and migrates to) a `jsonb` column. -`db_type="json"` is the explicit opt-out and produces no migration for -existing plain-json columns; `db_type="jsonb"` stays valid and simply states -the default. Key-order-sensitive workloads should opt out **before** upgrading. -See [JSON storage](../guide/models-and-fields.md#json-storage-json-and-jsonb) -and ADR-0005. - -## Upgrading to 0.14 - -`0.14` makes the mutation surface strict — no write silently does more (or -different) work than you asked — routes every database failure through a typed -exception hierarchy, unifies how the Rust runtime emitter and the Alembic bridge -derive column types and names, and **removes** the operator-style predicates and -`col()` bridge that `0.12` deprecated. Your model definitions do not change. - - -??? example "Automate this: copy this prompt to your coding agent" - - ```text - You are helping upgrade a Python codebase from Ferro ORM 0.13.x to 0.14. - Reference: https://ferro-orm.x54.sh/howto/upgrade-guide/#upgrading-to-014 - - Work in two passes. Do NOT open a PR until I have reviewed Pass 2. - - PASS 1 — mechanical replacements (safe to apply directly): - 1. Query predicates: replace operator-style `where(Model.field == value)` and any - `col(...)` bridges with lambda predicates, e.g. `where(lambda m: m.field == value)`. - `order_by` takes a lambda or a validated column-name string. Operator style and - `col()` were removed in 0.14. - 2. Connection routing: any ORM or raw call that ran outside a session - (`User.all()`, `ferro.execute(...)` with no active session) must run inside - `async with ferro.engines.session("name"):`. Unsessioned calls now raise. - 3. Exceptions: replace `except RuntimeError` / `except ConnectionError` handlers - around Ferro calls with the typed hierarchy from `ferro` (e.g. - `UniqueViolationError`, `OperationalError`, `InterfaceError`). Match on type, - not on driver-message text. - - PASS 2 — semantic changes (DO NOT rewrite; list each site as file:line and ask me): - 4. `Model.create(...)` and `instance.save()` no longer upsert. For each call site, - tell me whether the code RELIED on the old overwrite-on-conflict behavior. If it - did, I decide between `Model.upsert(...)` / `save(on_conflict="update")` and - catching `UniqueViolationError`. Do not guess. - 5. `.limit()` / `.offset()` chained into `.update()` / `.delete()` now raise. List - each site — bounding a mutation requires fetching primary keys first, and only I - know whether the bound was intentional. - 6. Schema-emission defaults changed for derived types (plain `datetime` -> - timestamptz, Enum -> native enum, `datetime.time` -> time). Auto-migrate refuses - to rewrite existing columns and warns. For each affected model, list it and ask - me to choose keep (`db_type=...`) vs a reviewed ALTER migration. - - VERIFY after Pass 1: - - Run the test suite. - - Run `pytest -W error::DeprecationWarning`. This catches the deprecation-based - changes ONLY — items 4, 5, and 6 are silent and will not appear here. - - Grep helper: grep -rn "\.limit(\|\.offset(" --include="*.py" src/ | grep "delete()\|update(" - - Report a summary: what you changed in Pass 1, and the Pass 2 sites awaiting my decision. - ``` - -### Predicates: operator style and `col()` are gone - -`where()` accepts only lambda predicates. Operator-style predicates -(`Model.field == value`) and the `col()` bridge — both deprecated in `0.12` — -were removed. Class attributes are no longer replaced with `FieldProxy`, so -`Model.field` at class scope raises `AttributeError` under normal Pydantic -semantics. `order_by()` takes a lambda (`order_by(lambda t: t.created_at, "desc")`) -or a validated column-name string (`order_by("created_at", "desc")`). See -[Typed Query Predicates](../concepts/query-typing.md). - -=== "Before (removed)" - - ```python - adults = await User.where(User.age >= 18).all() - admins = await User.where(User.role == "admin").all() - ``` - -=== "After" - - ```python - adults = await User.where(lambda t: t.age >= 18).all() - admins = await User.where(lambda t: t.role == "admin").all() - ``` - -### Connection routing requires a session - -Unqualified operations (`User.all()`, `ferro.execute(...)`) once fell back to -implicit default-connection routing. That fallback is gone (it was removed in -`0.13`, one release ahead of the other deprecations, as part of the identity-map -and routing redesign). Wrap request- or task-scoped work in a session so routing, -the identity map, and transactions are explicit and isolated under concurrency. A -call with no active session and no `using=` now raises `RuntimeError`; an explicit -`using=` that names a different connection than the ambient session raises -`ValueError`. See [Identity Map](../concepts/identity-map.md). - -=== "Before (removed)" - - ```python - import ferro - - users = await User.where(lambda t: t.active == True).all() # noqa: E712 - await ferro.execute("UPDATE users SET active = TRUE") - ``` - -=== "After (ambient session)" - - ```python - import ferro - - async with ferro.engines.session("app"): - users = await User.where(lambda t: t.active == True).all() # noqa: E712 - await ferro.execute("UPDATE users SET active = TRUE") - ``` - -### create() no longer overwrites on primary-key conflict - -Previously `create()` compiled to `INSERT ... ON CONFLICT (pk) DO UPDATE`: a -duplicate primary key silently **replaced** the existing row. It is now a plain -INSERT that raises [`UniqueViolationError`](../api/exceptions.md) and leaves -the existing row untouched. - -=== "Before (v0.13: silent clobber)" - - ```python - # Overwrote the stored row when id 1 already existed - user = await User.create(id=1, email="taylor@example.com") - ``` - -=== "After (insert-or-update, explicit)" - - ```python - user = await User.upsert(id=1, email="taylor@example.com") - ``` - -=== "After (insert only)" - - ```python - from ferro import UniqueViolationError - - try: - user = await User.create(id=1, email="taylor@example.com") - except UniqueViolationError: - ... # duplicate — the stored row is untouched - ``` - -### save() no longer upserts - -`save()` now tracks whether the instance has been persisted: a new instance is -INSERTed, a fetched or previously saved instance is UPDATEd by primary key. Two -consequences for code written against the old behavior: - -- Saving a *new* instance whose primary key is already taken raises - `UniqueViolationError` instead of overwriting the stored row. -- Saving a *persisted* instance whose row has been deleted underneath raises - `ModelDoesNotExist` instead of silently re-inserting it. - -Where you deliberately used `save()` to insert-or-update, say so explicitly: - -=== "Before (v0.13: every save upserted)" - - ```python - profile = Profile(id=user_id, theme="dark") - await profile.save() # inserted or replaced, whichever applied - ``` - -=== "After (explicit upsert)" - - ```python - profile = Profile(id=user_id, theme="dark") - await profile.save(on_conflict="update") - - # or the one-call convenience: - profile = await Profile.upsert(id=user_id, theme="dark") - ``` - -The conflict target is the **primary key only**, and the whole row is written -on update — see [Upsert](../guide/mutations.md#upsert) for the details. - -### limit()/offset() on a mutating query raise - -Portable SQL has no `DELETE ... LIMIT`; previously the pagination was silently -dropped and the mutation touched **every** matching row. `update()` and -`delete()` now raise `ValueError` when the query carries `limit()` or -`offset()`. To mutate a bounded subset, fetch primary keys first: - -=== "Before (v0.13: limit silently ignored!)" - - ```python - # Looked bounded — actually deleted every matching row - await Job.where(lambda job: job.done == True).limit(100).delete() # noqa: E712 - ``` - -=== "After (bounded by primary-key set)" - - ```python - batch = await Job.where(lambda job: job.done == True).limit(100).all() # noqa: E712 - ids = [job.id for job in batch] - await Job.where(lambda job: job.id.in_(ids)).delete() - ``` - -### Catch typed exceptions, not RuntimeError - -Database failures now raise a DBAPI-shaped hierarchy rooted at -[`ferro.FerroError`](../api/exceptions.md) instead of bare `RuntimeError` / -`ConnectionError`. Broad handlers keep working during the transition -(`FerroError` is still an `Exception`), but move them over: - -=== "Before" - - ```python - try: - await order.save() - except RuntimeError as exc: - if "unique" in str(exc).lower(): # matching driver text - ... - ``` - -=== "After" - - ```python - from ferro import UniqueViolationError - - try: - await order.save() - except UniqueViolationError as exc: - ... # exc.sqlstate / exc.constraint / exc.driver_message for detail - ``` - -`connect()` failures are now `ferro.OperationalError` (server or environment) -or `ferro.InterfaceError` (bad scheme or configuration) instead of -`ConnectionError`. - -### Schema-emission default changes - -`0.14` also unifies how the Rust runtime emitter and the Alembic bridge derive -column types and artifact names — one decision table, consumed by both (see -`docs/solutions/patterns/derived-type-and-naming-decision-table.md`). Three -storage defaults change for *derived* types (fields with no explicit -`db_type`). **Auto-migrate never rewrites an existing column silently**: where -your database was bootstrapped by an older emitter, Ferro warns and skips, and -you choose between the keep-recipe and the convert-recipe below. - -#### Plain `datetime` fields are timezone-aware (`timestamptz`) - -The Alembic bridge previously mapped a plain `datetime.datetime` field to a -timezone-naive `DateTime`; the runtime already used `timestamptz`. Both now -emit `timestamptz`. If your schema has naive `timestamp` columns (bootstrapped -via Alembic), auto-migrate refuses the conversion and warns. - -=== "Keep the column naive" - - ```python - class Event(Ferro): - occurred_at: datetime.datetime = FerroField(db_type="timestamp") - ``` - -=== "Convert intentionally (reviewed Alembic migration)" - - ```sql - ALTER TABLE event - ALTER COLUMN occurred_at TYPE timestamptz - USING occurred_at AT TIME ZONE 'UTC'; -- your source timezone - ``` - -#### Enum fields default to native Postgres enum types - -The runtime emitter previously stored Enum fields as `varchar`; the Alembic -bridge already emitted a native enum type. Both now emit the native type -(created idempotently; on SQLite the column is `varchar()`, -matching SQLAlchemy). If your schema has varchar enum columns (bootstrapped by -the runtime), auto-migrate refuses the conversion and warns. - -=== "Keep varchar storage" - - ```python - class Account(Ferro): - role: Role = FerroField(db_type="varchar", db_check=True) - ``` - -=== "Convert intentionally (reviewed Alembic migration)" - - ```sql - CREATE TYPE role AS ENUM ('admin', 'member'); - ALTER TABLE account - ALTER COLUMN role TYPE role USING role::role; - ``` - -#### `datetime.time` fields store as `time` - -The runtime emitter previously stored `datetime.time` fields as `varchar` (the -bridge already said `TIME`). Both now emit `time`. Existing varchar columns: -auto-migrate refuses the conversion and warns — keep with -`db_type="varchar"`, or convert with a reviewed migration using an explicit -`USING` cast. - -#### Constraint and index names are single-sourced - -- **Foreign keys are now named** (`fk_
__`) in the DDL of - both emitters. Existing databases need **no action**: Alembic autogenerate - compares foreign keys by signature, not name, and auto-migrate never alters - existing foreign keys. -- **Single-column uniques become named unique indexes** (`uq_
_`), - the same shape composite uniques always had. On an existing database - auto-migrate adds the named index (`CREATE UNIQUE INDEX IF NOT EXISTS` — - additive and idempotent); the old inline artifact (e.g. `account_email_key` - on Postgres) remains as a redundant duplicate until you drop it: - - ```sql - ALTER TABLE account DROP CONSTRAINT account_email_key; - ``` - -- Identifiers longer than 63 characters now truncate deterministically on both - emitters (previously only some name kinds guarded, and PostgreSQL truncated - the rest silently). -- **SQLite declared types now use SQLAlchemy's spellings** (`DATETIME`, - `DATE`, `TIME`, `CHAR(32)`, `JSON`, `NUMERIC` instead of sea-query's - `timestamp_with_timezone_text`-style names), and primary-key columns carry - an explicit `NOT NULL`. Existing SQLite databases need **no action** — - SQLite's type affinity makes the storage identical, and auto-migrate's - affinity-class comparison treats both spellings as equivalent. Only newly - created tables show the new spellings. - -### Changed surfaces at a glance - -| Surface | v0.13 behavior | v0.14 behavior | Migration | -| --- | --- | --- | --- | -| `Model.create(**fields)` on existing PK | Silently replaced the row | Raises `UniqueViolationError` | `Model.upsert(...)` if you wanted replace | -| `instance.save()` on a new instance with taken PK | Silently replaced the row | Raises `UniqueViolationError` | `save(on_conflict="update")` | -| `instance.save()` after the row was deleted | Silently re-inserted | Raises `ModelDoesNotExist` | Catch and re-`create()` if intended | -| `Query.limit(...).update()/.delete()` | Pagination silently ignored | Raises `ValueError` | Fetch PKs, mutate by PK set | -| Database failures | `RuntimeError(...)` with driver text | Typed `FerroError` subclasses | Catch by type, not message | -| Plain `datetime` field (Alembic bridge) | Naive `timestamp` | `timestamptz` | Keep via `db_type="timestamp"`, or convert with `USING ... AT TIME ZONE` | -| Enum field (runtime emitter) | `varchar` | Native PG enum type | Keep via `db_type="varchar"`, or convert with `USING col::` | -| `datetime.time` field (runtime emitter) | `varchar` | `time` | Keep via `db_type="varchar"`, or convert with `USING` | -| FK constraints | Anonymous | Named `fk_
__` | None (names not compared) | -| Single-column unique | Inline column `UNIQUE` | Named `uq_
_` unique index | Optional: drop the old inline constraint | - -## Upgrading to 0.12 - -`0.12` is the first release built on Ferro's IR-first architecture: query -execution, schema/migration planning, codecs, hydration, and connection routing -all flow through one shared intermediate representation instead of several -independent code paths. A single source of truth removes whole classes of drift -bugs — predictable schema diffs, identical bind/null semantics across SQLite and -PostgreSQL, and explicit session-scoped runtime state. Your model definitions do -not change; this is about how a few APIs are called. - -`0.12` introduced these as **deprecation warnings**; operator-predicate, routing, -and private Alembic JSON helper removals later landed (see -[Upgrading to 0.14](#upgrading-to-014)). - - -??? example "Automate this: copy this prompt to your coding agent" - - ```text - You are helping upgrade a Python codebase from Ferro ORM 0.11.x to 0.12. - Reference: https://ferro-orm.x54.sh/howto/upgrade-guide/#upgrading-to-012 - - All 0.12 changes are deprecation-based and safe to apply mechanically. Apply them, - then verify. - - 1. Query predicates: replace operator-style `where(Model.field == value)` with - lambda predicates `where(lambda m: m.field == value)`. (These were removed - outright in 0.14 — if you are upgrading past 0.12, prefer lambdas now.) - 2. Connection routing: wrap request/task-scoped ORM and raw operations in - `async with ferro.engines.session("name"):` instead of relying on implicit - default-connection routing. - 3. Alembic metadata: replace - `from ferro.migrations.alembic import _build_sa_table, _map_to_sa_type` - with `from ferro.migrations import get_metadata; target_metadata = get_metadata()` - in your Alembic env.py. - - VERIFY: - - Run the test suite. - - Run `pytest -W error::DeprecationWarning`; a clean run means no deprecated - 0.12-era surfaces remain on the paths you exercise. - ``` - -### Alembic metadata: build from `get_metadata()` - -The private JSON-derivation helpers `ferro.migrations.alembic._build_sa_table` -and `ferro.migrations.alembic._map_to_sa_type` were removed in `0.14`. Schema -metadata derives from SchemaIR through the public `get_metadata()` entry point; -use it directly in your Alembic `env.py`. - -=== "Before (removed)" - - ```python - from ferro.migrations.alembic import _build_sa_table, _map_to_sa_type - ``` - -=== "After" - - ```python - from ferro.migrations import get_metadata - - target_metadata = get_metadata() - ``` - -### Deprecations introduced in 0.12, at a glance - -| Deprecated in 0.12 | Replacement | Status | -| --- | --- | --- | -| `Model.where(Model.field OP value)` / `col()` | `where(lambda t: ...)` | Removed in `0.14` — see [Predicates](#predicates-operator-style-and-col-are-gone) | -| Unqualified ops outside an active session | `async with ferro.engines.session("name")` | Removed in `0.13` — see [Connection routing](#connection-routing-requires-a-session) | -| `ferro.migrations.alembic._build_sa_table` | `ferro.migrations.get_metadata()` | Removed in `0.14` — see [Alembic metadata](#alembic-metadata-build-from-get_metadata) | -| `ferro.migrations.alembic._map_to_sa_type` | `ferro.migrations.get_metadata()` | Removed in `0.14` — see [Alembic metadata](#alembic-metadata-build-from-get_metadata) | diff --git a/docs/pages/index.md b/docs/pages/index.md index 1551de0a..d3b62b4b 100644 --- a/docs/pages/index.md +++ b/docs/pages/index.md @@ -1,12 +1,55 @@ +--- +description: "Ferro is an async Python ORM with Pydantic models and a Rust core, for SQLite and PostgreSQL." +--- + # Ferro ORM ---8<-- "README.md:main" +--8<-- "README.md:intro" + +Declare a model once. The two tabs show the same model in Ferro's two equivalent field-declaration styles: + +=== "Assignment" + + ```python + --8<-- "docs/examples/readme_quickstart.py:models" + ``` + +=== "Annotated" + + ```python + --8<-- "docs/examples/readme_quickstart_annotated.py:models" + ``` + +Connect, open a session, create two users and query the active ones: + +=== "Assignment" + + ```python + --8<-- "docs/examples/readme_quickstart.py:query" + ``` + +=== "Annotated" + + ```python + --8<-- "docs/examples/readme_quickstart_annotated.py:query" + ``` + +Only `alice` is active, so the script prints: + + +```text +Found 1 active users. +``` + +Every ORM call runs inside `async with engines.session():`; see [Sessions](guide/connections.md#sessions). `auto_migrate=True` creates the table on connect, which suits development and tests. ## Where to go next -- **[Quickstart Tutorial](getting-started/quickstart.md)** — Build a working app with models, queries, relationships, and transactions in about 10 minutes. -- **[Why Ferro](why-ferro.md)** — How Ferro differs from SQLAlchemy, Django ORM, and Tortoise, and the trade-offs of a Rust-core ORM. -- **[Guide](guide/models-and-fields.md)** — Models, fields, queries, mutations, relationships, transactions, and migrations. -- **[How-To](howto/testing.md)** — Recipes for testing, pagination, timestamps, soft deletes, and multiple databases. -- **[API Reference](api/model.md)** — Public classes and methods. -- **[Roadmap](roadmap.md)** — What's implemented today and what's coming next. +- [Installation](getting-started/installation.md) covers Python versions, extras and building from source. +- [Quickstart](getting-started/quickstart.md) builds a small blog backend with models, queries, relationships and transactions. +- [Why Ferro](why-ferro.md) compares Ferro with SQLAlchemy, Django ORM and Tortoise, and lists the trade-offs. +- [Schema management](guide/schema/overview.md) explains auto-migrate and `ferro migrate`. +- [Models and fields](guide/models-and-fields.md), [Queries](guide/queries.md) and [Relationships](guide/relationships.md) are the main guides. +- [How-to guides](howto/testing.md) hold recipes for testing, pagination, timestamps, soft deletes and multiple databases. +- [API reference](api/model.md) lists every public class and method. +- [Roadmap](roadmap.md) lists the features that are not implemented yet. diff --git a/docs/pages/reference/cli.md b/docs/pages/reference/cli.md index c7e695c3..a91a63bf 100644 --- a/docs/pages/reference/cli.md +++ b/docs/pages/reference/cli.md @@ -1,3 +1,7 @@ +--- +description: "Every ferro migrate command, with its options, exit codes and output." +--- + # CLI The `ferro` command ships with the core package; its command line needs the `cli` extra: @@ -6,7 +10,7 @@ The `ferro` command ships with the core package; its command line needs the `cli pip install "ferro-orm[cli]" ``` -Without the extra, `ferro` prints `ferro's CLI needs the cli extra: pip install "ferro-orm[cli]"` and exits 2. Every verb lives under `ferro migrate`; the guides show them in use ([Migrations](../guide/schema/migrations.md), [Deploying](../guide/schema/deploying.md), [Adopting](../howto/adopting-migrations.md)). +Without the extra, `ferro` prints `ferro's CLI needs the cli extra: pip install "ferro-orm[cli]"` and exits 2. Every verb lives under `ferro migrate`; the guides show them in use ([Migrations](../guide/schema/migrations.md), [Deploying](../guide/schema/deploying.md), [Adopting](../guide/schema/adopting-migrations.md)). ## Global options @@ -106,7 +110,7 @@ On a terminal the heading is bold with a green tick, and each file's stem is col | `--data-only` | Write only the `--data-step`: no DDL, and a full copy of the previous migration's snapshot. | | `--no-backfill TABLE.COLUMN` | Replace the generated backfill of this column with a guard step that fails the migration while any row still needs a value. Repeatable. | | `--verbose` | Show every name in a long list of the change summary; a terminal otherwise shows the first few (a list past 12 names shows its first 6). | -| `--replace NNNN_NAME` | Regenerate this migration (or `NNNN`, when one migration carries the number) on top of the newest snapshot below it, carrying its data steps and hand-written SQL steps across byte for byte; the number is kept when free. Only the newest migration can be replaced. See [Two branches, one number](../guide/schema/migrations.md#two-branches-one-number). | +| `--replace NNNN_NAME` | Regenerate this migration (or `NNNN`, when one migration carries the number) on top of the newest snapshot below it, carrying its data steps and hand-written SQL steps across byte for byte; the number is kept when free. Only the newest migration can be replaced. See [Duplicate migration numbers and `--replace`](../guide/schema/migrations.md#duplicate-migration-numbers-and-replace). | Exit 0 when written (or nothing to write), 1 when refused. A primary-key change is refused, writing nothing: @@ -119,12 +123,12 @@ That section is [Changing a primary key](../guide/schema/migrations.md#changing- ## `ferro migrate squash NAME --from TARGET` -Replace a range of migrations, from `TARGET` through the newest, with the one migration `new` would have written across it. A feature branch wrote `0002_add_bio` (with a hand-written SQL step `fix_rows`) and `0003_add_team`; `ferro migrate squash epic --from 0002` writes: +Replace a range of migrations, from `TARGET` through the newest, with the one migration `new` would have written across it. A feature branch wrote `0002_add_bio` (with a hand-written SQL step `fix_rows`) and `0003_add_team`; `ferro migrate squash teams --from 0002` writes: ```text default · migrations -ok Wrote migrations/0002_epic/ (replaces 0002_add_bio … 0003_add_team, now deleted) +ok Wrote migrations/0002_teams/ (replaces 0002_add_bio … 0003_add_team, now deleted) 01_schema.up.postgres.sql schema step, up (postgres) regenerated 01_schema.down.postgres.sql schema step, down (postgres) regenerated 01_schema.up.sqlite.sql schema step, up (sqlite) regenerated @@ -139,7 +143,7 @@ ok Wrote migrations/0002_epic/ (replaces 0002_add_bio … 0003_add_team, now de Author ``` -The DDL is planned once, from the snapshot below the range to the range's last snapshot, with every rename the range declared composed into it: a column renamed in `0002` whose hint `0003` dropped is still a `RENAME COLUMN`, never a drop and an add. Every data step and hand-written SQL step of the range is carried byte for byte, in chain order, each marked `carried, review it` with the file it was; a carried data step still holding `todo(...)` still needs writing, except one that is byte for byte the generator's unwritten scaffold when the squashed plan does not scaffold it again: that one is dropped with a note (` from was the generator's unwritten scaffold and the squashed plan needs no backfill of
.: dropped`), and any hand-written byte means it is carried. The squashed migration takes the first migration's number, and its `ir.json` records which migrations it replaced. The range's directories are deleted. It reads the migrations directory only: never the models, never a database. A database that applied the originals inherits the squash on its next `up`, running nothing; [Squashing an epic's migrations](../guide/schema/migrations.md#squashing-an-epics-migrations) walks through the recipe, inheritance and when to squash. +The DDL is planned once, from the snapshot below the range to the range's last snapshot, with every rename the range declared composed into it: a column renamed in `0002` whose hint `0003` dropped is still a `RENAME COLUMN`, never a drop and an add. Every data step and hand-written SQL step of the range is carried byte for byte, in chain order, each marked `carried, review it` with the file it was; a carried data step still holding `todo(...)` still needs writing, except one that is byte for byte the generator's unwritten scaffold when the squashed plan does not scaffold it again: that one is dropped with a note (` from was the generator's unwritten scaffold and the squashed plan needs no backfill of
.: dropped`), and any hand-written byte means it is carried. The squashed migration takes the first migration's number, and its `ir.json` records which migrations it replaced. The range's directories are deleted. It reads the migrations directory only: never the models, never a database. A database that applied the originals inherits the squash on its next `up`, running nothing; [Squashing a feature branch's migrations](../guide/schema/squash.md) walks through the recipe, inheritance and when to squash. On a terminal the file table is laid out as `new`'s is, with the last column, what became of each file, in the default colour. @@ -195,6 +199,8 @@ Apply every pending migration, in order, under the run lock. Prints one line per | Flag | What it does | | :--- | :--- | | `--lock-timeout DURATION` | How long to wait for another run's lock: `30s`, `500ms`, `1m`, a number of seconds, or `0` to refuse at once. Defaults to the database's [`lock_timeout`](configuration.md#lock_timeout) (`30s`); the flag wins. | +| `--allow-ahead=expand-only` | Run beside a database that has applied migrations this directory lacks, when every step they hold is an expand step: the previous release of a [rolling deploy](../guide/schema/deploying.md#rolling-deploys-and-the-two-release-contract). Refused, naming the first contract step, otherwise. | +| `--allow-ahead` | Run beside a database that has applied migrations this directory lacks, whatever steps they hold. Without either form, `up` refuses a database ahead of its directory. | ```text @@ -240,6 +246,8 @@ Show which migrations this database has applied, without changing it or taking a | :--- | :--- | | `--steps` | Print every migration's steps, not only those needing attention. | | `--json` | Print the report as a JSON document, and nothing else on stdout. It carries `url` (the database's URL, password masked), `applied_at` on each migration and `finished_at` on each step, beside the keys it always had. | +| `--no-steps` | Print only the steps needing attention. This is the default. | +| `--no-json` | Print the plain report. This is the default. | Exit 0 when everything is applied, 3 when something is pending, 4 when something needs attention. @@ -328,7 +336,7 @@ recorded 0001_create_author … 0002_author_slug as baseline (2 steps) baseline checked what `ferro migrate drift` checks: objects ferro does not own (functions, triggers) were not compared ``` -On a terminal the first line is a green tick and a bold `recorded …`, with the step count dim, and the note under it is dim. [When baseline says no](../howto/adopting-migrations.md#when-baseline-says-no) shows the differences it prints instead. +On a terminal the first line is a green tick and a bold `recorded …`, with the step count dim, and the note under it is dim. [When baseline says no](../guide/schema/adopting-migrations.md#when-baseline-says-no) shows the differences it prints instead. ## `ferro migrate rerecord MIGRATION:STEP` diff --git a/docs/pages/reference/configuration.md b/docs/pages/reference/configuration.md index a7e2a0e2..77a3c505 100644 --- a/docs/pages/reference/configuration.md +++ b/docs/pages/reference/configuration.md @@ -1,3 +1,7 @@ +--- +description: "Every key of ferro.toml or [tool.ferro], which tells Ferro's tooling where the models are and which dialects to target." +--- + # Configuration A project commits one file that tells Ferro's tooling where its models are and which dialects its migrations target. `ferro migrate init` writes it; every `ferro migrate` verb, `ferro.migrations.*`, the Alembic bridge's `get_metadata()`, and `connect()` with an auto-migrate flag read it through one class, `ferro.FerroSettings`. @@ -81,10 +85,11 @@ How long a DDL statement Ferro runs on Postgres, in a migration step or in an au ### `lock_timeout` -How long a run waits for the run lock while another run holds it: `"500ms"`, `"30s"`, `"1m"`, a number of seconds, or `"0"` to wait without a limit. It is the default of `--lock-timeout` on `ferro migrate up`, `down`, `baseline` and `rerecord` (and of `lock_timeout=` on their Python calls); the flag wins. An auto-migrate pass (`connect(auto_migrate=…)`, `ferro.create_tables()`, `ferro.migrate()`) waits for it too, so a replica booting while another one's pass runs waits instead of failing: +How long a run waits for the run lock while another run holds it: `"500ms"`, `"30s"`, `"1m"`, a number of seconds, or `"0"` to wait without a limit. It is the default of `--lock-timeout` on `ferro migrate up`, `down`, `baseline` and `rerecord` (and of `lock_timeout=` on their Python calls); the flag wins. An auto-migrate pass (`connect(auto_migrate=…)`, `ferro.create_tables()`, `ferro.migrate()`) waits for it too, so a replica booting while another one's pass runs waits instead of failing. Once the wait passes `lock_timeout` (here `"1s"`, on SQLite) it refuses: + ```text -connect(auto_migrate=…) gave up waiting for the run lock on public: another ferro migration run or auto-migrate pass held it longer than lock_timeout (30s). Nothing was applied. Wait for that run to finish and try again, or raise lock_timeout; `ferro migrate status` shows a migration run while it holds the lock. +connect(auto_migrate=…) gave up waiting for the run lock on main: another ferro migration run or auto-migrate pass held it longer than lock_timeout (1s). Nothing was applied. Wait for that run to finish and try again, or raise lock_timeout; `ferro migrate status` shows a migration run while it holds the lock. ``` Unlike the flag, whose `0` refuses at once, the setting's `"0"` waits without a limit. As with `ddl_lock_timeout`, every configured database must set the same `lock_timeout` for an auto-migrate pass. @@ -106,7 +111,7 @@ Unlike the flag, whose `0` refuses at once, the setting's `"0"` waits without a The file found is used alone: nothing is layered on it, no environment variable overrides a key, and no `.env` file is read. These keys decide what a migration contains, so they come from the committed file only. No file is not an error for `FerroSettings()`: it returns an empty object listing the directories it searched, and refuses only when a database is asked for. -```python +```python {test=compile reason="needs a ferro.toml naming one database"} from ferro import FerroSettings settings = FerroSettings() diff --git a/docs/pages/reference/migrations.md b/docs/pages/reference/migrations.md new file mode 100644 index 00000000..4ae48661 --- /dev/null +++ b/docs/pages/reference/migrations.md @@ -0,0 +1,152 @@ +--- +description: "Step-file headers, step classes, what each schema door covers, what auto-migrate does per backend and what a PassReport holds." +status: new +--- + +# Migrations reference + +This page lists the facts the schema-management guides link to: the headers a step file can carry, the class of each step, what each door covers, what auto-migrate does on each backend, and what a `PassReport` holds. A generated index step shows a header at work: + +```sql +-- 0002_author_nickname/02_idx_author_nickname.up.postgres.sql +-- ferro: no-transaction + +DROP INDEX CONCURRENTLY IF EXISTS "idx_author_nickname"; + +CREATE INDEX CONCURRENTLY "idx_author_nickname" ON "author" ("nickname"); +``` + +The `no-transaction` header tells the runner to open no transaction around the step, because Postgres refuses `CREATE INDEX CONCURRENTLY` inside one. The [Migrations](../guide/schema/migrations.md) guide walks through the files a migration holds. + +## Step file headers + +A step file may open with `-- ferro: …` lines. The generator writes them, except `expand` and `irreversible`, which only a person writes. + +| Header | Meaning | +| :--- | :--- | +| `-- ferro: destructive` | The step discards data: it drops a column, a table or an enum type. Review is the gate; nothing refuses it. | +| `-- ferro: data-dependent` | The step discards nothing but fails on rows that do not satisfy it: a `NOT NULL`, a unique over existing rows, a type cast, the validation of a staged constraint, a removed enum label. | +| `-- ferro: no-transaction` | The runner opens no transaction around the step, so a statement such as `CREATE INDEX CONCURRENTLY` can run. The step is written to be safe to re-run from its first statement. | +| `-- ferro: foreign-keys-off` | A SQLite [table rebuild](../guide/schema/migrations.md#sqlite-table-rebuilds): the runner turns foreign-key enforcement off while the step runs. | +| `-- ferro: not-applicable` | This dialect has nothing to do at this step number. | +| `-- ferro: expand` | In a [hand-written SQL step](../guide/schema/migrations.md#hand-written-steps): its author declares it *expand*, so the previous release's writes keep succeeding beside it (see [step classes](#step-classes)). A hand-written step without it is *contract*. The generator never writes it: a generated step's class comes from what the step does. | +| `-- ferro: nothing-to-reverse ` | In a `.down` file: reverting removes the step's record and runs nothing. | +| `-- ferro: irreversible ` | In a `.down` file: a run that would revert this step refuses before reverting anything. Only a person declares one. | + +!!! note "Headers an older release does not know" + A ferro release refuses a step file that carries a header it does not know, naming the file. A file with `-- ferro: expand` is refused by a release that predates the header. + +## Step classes + +Each step is *expand* or *contract*, decided from what the step does and recorded with the step when it is applied. `require_applied(allow_ahead="expand-only")` lets an older release start on a database that is ahead of it only by expand steps; the [two-release contract](../guide/schema/deploying.md#rolling-deploys-and-the-two-release-contract) shows it in a deploy. + +| Expand | Contract | +| :--- | :--- | +| adding a table, or a nullable column | dropping a table or a column | +| `DROP NOT NULL` | `SET NOT NULL`: a required column's add-constraint and contract steps | +| a plain index; dropping an index, a check, a foreign key or a unique | a unique index; a type change | +| adding an enum label | renaming a column, a table, an enum type or a label; removing a label | +| a data step, a guard step | adding a check or a foreign key (`NOT VALID`) and its validate step | +| a hand-written SQL step whose file says `-- ferro: expand` | any other hand-written SQL step | + +- A SQLite table rebuild takes the class of what it carries. +- A step that runs nothing on a dialect (`not-applicable`) is expand there. +- A generated step you edit by hand is classed like a hand-written one: contract unless its file says `-- ferro: expand`. + +## What each door covers + +The three doors are [auto-migrate](../guide/schema/auto-migrate.md) (here its update rung, `migrate_updates`), [migrations](../guide/schema/migrations.md) and the [Alembic bridge](../guide/schema/alembic.md). + +| Change | Auto-update | Migrations | Alembic bridge | +| :--- | :--- | :--- | :--- | +| Add a table, a nullable column, an index, an enum label | ✅ | ✅ | ✅ | +| Add a check, a foreign key, change a type or nullability, on Postgres | ✅ | ✅ | ✅ | +| The same on SQLite (needs a table rebuild) | ⚠️ warns, no DDL | ✅ generated [table rebuild](../guide/schema/migrations.md#sqlite-table-rebuilds) | ❌ refused at autogenerate | +| Rename a column (`renamed_from`) or an enum label (`__ferro_renamed_labels__`) | ✅ | ✅ [declared on the model](../guide/schema/renames-and-retirements.md#rename-hints) | ✅ same hints | +| Rename a table (`__ferro_renamed_from__`) | ✅ `ALTER TABLE … RENAME TO` under `migrate_updates`, indexes and checks renamed with it; without `migrate_updates` the pass warns and creates nothing | ✅ | ✅ | +| Drop a column | with `migrate_destructive` | ✅ marked `-- ferro: destructive` | ✅ marked `# ferro: destructive` | +| Drop a table | ❌ | ✅ marked `-- ferro: destructive` | ✅ marked `# ferro: destructive` | +| Add a new required column to a table with rows | only with a literal default (backfills existing rows) | ✅ [expand, backfill, contract](../guide/schema/data-steps.md) generated | Postgres: ⚠️ the plain op, marked `# ferro: data-dependent`; the backfill is yours to write. SQLite: ❌ refused at autogenerate | +| Make a nullable column required | Postgres: `SET NOT NULL`, fails if any row is `NULL` (write it as a migration with a backfill); SQLite: warns, no DDL | ✅ expand, backfill, contract generated | Postgres: ⚠️ the plain op, marked `# ferro: data-dependent`. SQLite: ❌ refused at autogenerate | +| Remove an enum label rows may hold | warns, no DDL | ✅ [backfill and contract](../guide/schema/data-steps.md#removing-an-enum-label) generated | ❌ not written; [migrations](../guide/schema/data-steps.md#removing-an-enum-label) generate it | +| Python data steps over the models as they were | ❌ | ✅ [data steps](../guide/schema/data-steps.md) | hand-written `op.execute(...)` | +| Change a primary key | ❌ | refused, with [the recipe](../guide/schema/migrations.md#changing-a-primary-key) | ❌ refused, with the same recipe | + +## Auto-migrate changes per backend + +What `connect(..., migrate_updates=True)` and `ferro.migrate()` do to an existing table, change by change. `migrate_destructive=True` adds the drops marked so. + +| Change | SQLite | PostgreSQL | +| :--- | :--- | :--- | +| Add missing column | ✅ `ADD COLUMN` | ✅ `ADD COLUMN` | +| Rename a column declared `renamed_from` | ✅ `RENAME COLUMN` | ✅ `RENAME COLUMN` | +| Add the column's index (`index=True`) | ✅ `CREATE INDEX` | ✅ `CREATE INDEX` | +| Add composite index (`__ferro_composite_indexes__`) to existing columns | ✅ `CREATE INDEX` | ✅ `CREATE INDEX` | +| Add table check (`__ferro_checks__`) on CREATE TABLE | ✅ inline CHECK | ✅ inline CHECK | +| Add table check to existing table | ⚠️ `UserWarning`, no DDL; [migrations](../guide/schema/migrations.md#sqlite-table-rebuilds) generate the table rebuild | ✅ `ADD CONSTRAINT` | +| Rebuild table check on body drift | ⚠️ `UserWarning`, no DDL; migrations generate the table rebuild | ✅ rebuild: `DROP CONSTRAINT` + `ADD CONSTRAINT` | +| Add column `db_check=True` on existing column | ⚠️ `UserWarning`, no DDL; migrations generate the table rebuild | ✅ `ADD CONSTRAINT` | +| Leftover ferro check (`ck_*` live, removed from model) under `migrate_updates` | ⚠️ `UserWarning`, constraint stays | ⚠️ `UserWarning`, constraint stays | +| Drop orphaned ferro check (`ck_*`) | ⚠️ `UserWarning`, no DDL; migrations generate the table rebuild | ✅ with `migrate_destructive=True` | +| Add unique column (`unique=True`) | ✅ via explicit unique index + warning | ✅ inline `UNIQUE` | +| Add foreign-key column | ✅ column only, no FK constraint + warning | ✅ column + FK constraint | +| Add missing FK constraint to an existing column | ⚠️ `UserWarning`, no DDL; migrations generate the table rebuild | ✅ `ADD CONSTRAINT` | +| Drop a ferro foreign key (`fk_*`) from a column the model keeps (`team: Annotated[Team, ForeignKey(...)]` became `team_id: int`) | ⚠️ `UserWarning`, constraint stays; migrations generate the table rebuild | ✅ with `migrate_destructive=True`: `DROP CONSTRAINT` | +| Change a foreign key's `on_delete` (or target) | ⚠️ `UserWarning`, no DDL; migrations generate the table rebuild | ✅ rebuild: `DROP CONSTRAINT` + `ADD CONSTRAINT` | +| Change column type | ⚠️ `UserWarning`, no DDL (SQLite type affinity makes drift mostly cosmetic); migrations generate the table rebuild | ✅ `ALTER COLUMN ... TYPE ... USING` cast | +| Change nullability | ⚠️ `UserWarning`, no DDL; migrations generate the table rebuild | ✅ `SET NOT NULL` / `DROP NOT NULL`. `SET NOT NULL` backfills nothing, whatever the default: it fails the connect if any row holds `NULL`. A [migration](../guide/schema/data-steps.md) writes the backfill first | +| A server default no model declares (a *foreign default*, see [destructive drops](../guide/schema/auto-migrate.md#destructive-drops-with-migrate_destructive)) | ⚠️ `UserWarning`, default stays; only a hand rebuild drops it (an expression, or any default on a nullable column; a literal on a `NOT NULL` column is ferro's own) | ⚠️ `UserWarning` under `migrate_updates`; ✅ with `migrate_destructive=True`: `DROP DEFAULT` | +| Drop orphaned Ferro-named index (`idx_*` / `uq_*`) | ✅ with `migrate_destructive=True` | ✅ with `migrate_destructive=True` | +| Redefine an index that keeps its name (a long name cut to 63 characters, two column groups that join to one name, or a live `idx_*` / `uq_*` index written another way) | ✅ `DROP INDEX` + `CREATE INDEX` under `migrate_updates`. A unique one over duplicate values fails the connect, counting them | ✅ same, in the table's transaction | +| Add a missing enum label (a `StrEnum` grew a member) | ✅ nothing to do: enums store as text | ✅ `ALTER TYPE ... ADD VALUE` | +| Rename an enum label declared with `__ferro_renamed_labels__` | ✅ nothing to do | ✅ `ALTER TYPE ... RENAME VALUE` | +| Remove an enum label | ✅ nothing to do | ⚠️ `UserWarning`, no DDL. [Migrations](../guide/schema/migrations.md) generate it with its [backfill](../guide/schema/data-steps.md#removing-an-enum-label) | +| Inline single-column `UNIQUE` on an existing column, index option changes | ❌ never here; [migrations](../guide/schema/migrations.md) generate it | ❌ never here; migrations generate it | +| Rename a table | ✅ `ALTER TABLE … RENAME TO` under `migrate_updates` when the database holds the old name from `__ferro_renamed_from__` and not the new one; derived index and check names follow. Without `migrate_updates` a `UserWarning` names the table, the hint and both doors, and nothing is created. [Migrations](../guide/schema/renames-and-retirements.md#rename-hints) are the reviewed path | ✅ same | +| Drop a column | ✅ with `migrate_destructive=True` | ✅ with `migrate_destructive=True` | +| Drop a table | ❌ never here; [migrations](../guide/schema/migrations.md#reading-a-migration) generate it, marked destructive | ❌ same | +| Change a primary key | ❌ never; no door generates it. Migrations refuse it with [the recipe](../guide/schema/migrations.md#changing-a-primary-key) | ❌ same | + +## `PassReport` contents + +`ferro.migrate()` and `ferro.create_tables()` return a [`PassReport`](../api/connection.md#ferro.PassReport): every statement the pass executed, in order, as an `ExecutedStatement`, and every warning it raised, as a `Report`. [On-demand `migrate()`](../guide/schema/auto-migrate.md#on-demand-migrate) shows one read in code. + +An `ExecutedStatement`'s `role` says what the statement was for: + +| `role` | The statement | +| :--- | :--- | +| `"schema"` | DDL: the create pass, the enum type statements, the reconciliation. | +| `"lock_timeout"` | The `SET LOCAL lock_timeout`, `SET` and `RESET` lines that bound each statement's wait for its table lock on Postgres. | +| `"probe"` | The SQLite query that looks for rows still holding a label the model renamed. | + +A `Report`'s `kind` names the warning, so code can match on it rather than on the sentence. `subject` says what it is about (the modelset, a table, a column or an enum type), and `recurs` is `True` when the warning comes back on every pass until someone acts. + +| `kind` | Raised when | +| :--- | :--- | +| `HintRefused` | A declared rename hint cannot be meant; nothing is renamed. | +| `LeftoverChecks` | Live ferro-owned checks (`ck_*`) the model no longer declares. | +| `ExtraEnumLabels` | Live labels of an enum type the model no longer declares; the labels stay. | +| `ForeignFkDrift` | A foreign key drifts from the model under a name ferro does not own; it is left untouched. | +| `DroppedRowSecurity` | The live table carries row security the model no longer asks for; it stays. | +| `ForeignPolicies` | Live policies ferro does not own; they are never touched. | +| `UnverifiablePolicy` | A raw policy body ferro cannot verify; it is left as it is. | +| `PolicyBodyReplaced` | A raw policy was rebuilt for its metadata, replacing its live body. | +| `RowSecurityTeardown` | `migrate_destructive` tore down row security on a table, naming the policies dropped. | +| `ExtraPolicies` | Live ferro-owned policies the model no longer declares; they stay. | +| `ForeignDefault` | A live server default no model declares; it stays until `migrate_destructive` or a hand rebuild drops it. | +| `RefusedConversion` | A storage conversion that could reinterpret or destroy stored values; it is not executed. | +| `SqliteInPlace` | A change SQLite cannot make to an existing table in place: it needs the table rebuild only a migration writes. | +| `PrimaryKeyKept` | The declared primary key differs from the live one; no door changes a primary key in place. | +| `EnumTypeMove` | A column moves to or from a native Postgres enum type, which no statement converts in place. | +| `RowSecuritySkipped` | Row security a table declares that the database does not apply: SQLite has none, or the create pass met the table already there. | +| `PendingTableRename` | A table rename hint is live, and a pass without `migrate_updates` holds the table back. | +| `StrandedLabelRename` | A SQLite column whose rows still hold a label the model renamed. | +| `RowSecurityUnderMigrator` | The migrating role is itself bound by the `FORCE` policies it maintains. | +| `RunLockWait` | The pass waited for another run's lock. | +| `DdlLockRetry` | A statement timed out waiting for its table lock and is retried. | + +Every warning is also raised as a `UserWarning`. The [API reference](../api/connection.md#ferro.PassReport) documents each field. + +## Exit codes + +Every `ferro migrate` verb shares one set of exit codes; the [CLI reference](cli.md#exit-codes) lists them. diff --git a/docs/pages/reference/row-level-security.md b/docs/pages/reference/row-level-security.md new file mode 100644 index 00000000..7dd9ce86 --- /dev/null +++ b/docs/pages/reference/row-level-security.md @@ -0,0 +1,213 @@ +--- +description: "API reference for RowSecurity, RowPolicy and the session settings their policies read." +status: new +--- + +# Row-level security + +This page lists Ferro's row-level security API: `RowSecurity` and `RowPolicy`, which declare a table's policies, and the session settings the policies read. + +A model declares its policies on `__ferro_rls__`, and a session's `settings=` supplies the value they compare against. With the `Invoice` model below, a session scoped to one tenant reads that tenant's invoices: + +```python +--8<-- "docs/examples/row_level_security.py:scoped" +``` + +Postgres returns Acme's two invoices; the policy filtered out Globex's. The examples on this page run against Postgres in the docs test suite. [Scope rows by tenant](../guide/row-level-security.md) walks through the setup, and [Row-level security and session settings](../concepts/row-level-security.md) explains how it works. + +## `RowSecurity` + +`RowSecurity` declares a table's row security, its policies and its table flags: + +```python {test=compile reason="a signature"} +class RowSecurity: + def __init__(self, *policies: RowPolicy, force: bool = True) -> None: ... +``` + +Assign it to the model's `__ferro_rls__` class variable, annotated `ClassVar`: + +=== "Assignment" + + ```python + --8<-- "docs/examples/row_level_security.py:models" + ``` + +=== "Annotated" + + ```python + --8<-- "docs/examples/row_level_security_annotated.py:models" + ``` + +`Invoice` gets `ENABLE` and `FORCE ROW LEVEL SECURITY` and one policy, `rls_invoice_tenant_id`. + +| Parameter | Default | Meaning | +| :--- | :--- | :--- | +| `*policies` | none | The table's `RowPolicy` values, in any number. Permissive policies combine with OR, restrictive ones with AND (see [Composing policies](#composing-policies)). | +| `force` | `True` | Also emit `FORCE ROW LEVEL SECURITY`, so the table's owner is filtered too. With `False`, a role that owns the table skips every policy. | + +Raises `TypeError` when a positional argument is not a `RowPolicy` or `force` is not a `bool`, and, when the model is defined, when two policies share a name. + +## `RowPolicy` + +`RowPolicy` declares one policy, the rows a command may see or write: + +```python {test=compile reason="a signature"} +@dataclass(frozen=True) +class RowPolicy: + name: str | None = None + command: str = "all" + restrictive: bool = False + column: str | None = None + setting: str | None = None + using: str | None = None + with_check: str | None = None +``` + +It takes one of two forms, the shorthand (`column=` and `setting=`) or SQL (`using=` and `with_check=`); passing both, or neither, raises `TypeError`. + +| Parameter | Default | Meaning | +| :--- | :--- | :--- | +| `name` | the column, in the shorthand | The policy's name suffix; the live policy is `rls_
_`. Required in the SQL form. | +| `command` | `"all"` | The command the policy covers: `"all"`, `"select"`, `"insert"`, `"update"` or `"delete"` (see [Commands](#commands)). | +| `restrictive` | `False` | `True` makes the policy `AS RESTRICTIVE`: it combines with every other policy by AND instead of OR. | +| `column` | `None` | Shorthand: the model column holding the scope value. | +| `setting` | `None` | Shorthand: the session setting to compare it with, a dotted name such as `"app.tenant_id"`. | +| `using` | `None` | SQL form: the `USING` expression, which decides the rows a command sees. | +| `with_check` | `None` | SQL form: the `WITH CHECK` expression, which decides the rows a write may leave. | + +### The shorthand + +`RowPolicy(column="tenant_id", setting="app.tenant_id")` renders one comparison and uses it for both `USING` and `WITH CHECK`: + +```sql +"tenant_id" = NULLIF(current_setting('app.tenant_id', true), '')::uuid +``` + +The cast comes from the column's storage type on Postgres: + +| Column storage | Cast | +| :--- | :--- | +| `uuid` | `::uuid` | +| `text`, `varchar` | none | +| `smallint`, `integer`, `bigint` | `::smallint`, `::integer`, `::bigint` | +| a native enum, or any other type (`timestamptz`, `jsonb`, ...) | not supported: defining the model raises `TypeError`, naming the SQL form as the alternative | + +`setting` must be dotted identifiers, matching `[A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z0-9_]+)+`, or the constructor raises `ValueError`. A `column` the model does not have raises `TypeError` when the model is defined, listing the valid columns. [Policy expressions](../concepts/row-level-security.md#policy-expressions) explains the `NULLIF`. + +### The SQL form + +Pass `using=`, `with_check=` or both, with `name=`, when one column compared with one setting is not enough: a function call, a subquery, boolean logic. Each expression must be a non-empty string. The constructor checks it against the clauses Postgres accepts for the command, and raises `TypeError`: + +- when `name=` is missing; +- for `using=` on `"insert"`, or `with_check=` on `"select"` or `"delete"` (see [Commands](#commands)); +- when an `"insert"` policy has no `with_check=`, or any other command has no `using=`. + +Write the same `NULLIF(current_setting(key, true), '')` wrapper around a setting you read in SQL, so an unset setting matches no rows instead of raising. + +!!! note "A policy that reads another table" + Postgres resolves the tables a policy's SQL reads when it creates the policy, so they must exist by then. Ferro orders the tables it creates by their foreign keys, not by what a policy reads: a policy on `document` that reads `membership`, created in the same pass, fails with `relation "membership" does not exist`. Create the table the policy reads in an earlier migration, or declare the policy once that table exists. + +### Policy names + +The live name is `rls_
_`: `RowPolicy(name="tenant", ...)` on `Document` is `rls_document_tenant`. The `name` must match `[a-z][a-z0-9_]*` and must not start with `rls_`, or the constructor raises `TypeError`. + +Names are unique per model. Postgres limits identifiers to 63 bytes, so Ferro truncates a long name; two names that are equal after truncation raise `TypeError` when the model is defined, as a repeated name does. + +Ferro manages only the policies whose names start with `rls_`. It reports any other policy on a table and never changes it. + +### Commands + +`command=` scopes a policy to one command, and each command takes the clauses Postgres accepts for it: + +| `command` | `using=` | `with_check=` | +| :--- | :--- | :--- | +| `"all"` | required | accepted | +| `"select"` | required | refused | +| `"insert"` | refused | required | +| `"update"` | required | accepted | +| `"delete"` | required | refused | + +The table holds for the SQL form. The shorthand fills in every clause its command accepts. + +## Composing policies + +Postgres combines a table's policies the same way Ferro declares them: permissive policies (the default) with OR, and restrictive ones (`restrictive=True`) with AND against the rest. A restrictive tenant fence plus permissive grants gives "within the tenant, the owner has full access and everyone reads published documents": + +=== "Assignment" + + ```python + --8<-- "docs/examples/row_level_security.py:composed" + ``` + +=== "Annotated" + + ```python + --8<-- "docs/examples/row_level_security_annotated.py:composed" + ``` + +`tenant` is restrictive, so the two permissive policies apply only inside the tenant. The three policies render as: + +```sql +CREATE POLICY "rls_document_tenant" ON "document" AS RESTRICTIVE FOR ALL USING ("tenant_id" = NULLIF(current_setting('app.tenant_id', true), '')::uuid) WITH CHECK ("tenant_id" = NULLIF(current_setting('app.tenant_id', true), '')::uuid); + +CREATE POLICY "rls_document_owner_all" ON "document" FOR ALL USING ("owner" = NULLIF(current_setting('app.user', true), '')) WITH CHECK ("owner" = NULLIF(current_setting('app.user', true), '')); + +CREATE POLICY "rls_document_published_read" ON "document" FOR SELECT USING ("published"); +``` + +With four documents (Ada's "Q3 plan", Grace's published "Handbook" and her unpublished draft in Acme, and Ada's published "Globex roadmap" in Globex), Ada in Acme reads her own document and the published one: + +```python +--8<-- "docs/examples/row_level_security.py:composed_query" +``` + +Grace's draft fails both permissive policies, and the Globex roadmap passes `published_read` but fails the restrictive tenant fence. + +## Session settings + +The session settings API lives on `engines` and `Session`; the [API reference](../api/connection.md) has the full signatures. + +| Call | What it does | +| :--- | :--- | +| `engines.session(name=None, *, settings=None)` | Opens a session whose operations carry `settings`, a mapping of setting name to value. | +| `Session.set_config(key, value)` | Sets one setting on an open session (see [Visibility of a new value](../concepts/row-level-security.md#visibility-of-a-new-value)). | +| `Session.effective_settings` | The settings the session runs with, inherited ones included. | +| `ferro.current_session()` | The session open in the current task, or `None`. | +| `PoolConfig(settings_delivery="transaction")` | How a connection delivers settings: `"transaction"` (the default) or `"connection"` (see [Connection delivery](../concepts/row-level-security.md#connection-delivery)). | + +Both `settings=` and `set_config` check their arguments before touching a connection: + +- a key or value that is not a `str` raises `TypeError` (convert a UUID with `str()`); +- a key with no dot raises `ValueError`: built-in settings such as `timezone` or `role` cannot be set this way; +- `settings=` on a non-Postgres connection, or `set_config` on a session whose connection is not Postgres, raises `RuntimeError`; `set_config` on a session that is not open raises `RuntimeError` too. + +Keys and values travel as bound parameters, never as SQL text. + +## SQLite + +SQLite has no row-level security. A model with `__ferro_rls__` still registers, and its table is created without policies, with one warning per table: + +```python +--8<-- "docs/examples/row_level_security.py:sqlite" +``` + +Rows on SQLite are not filtered. Declaring settings on a SQLite session raises, since nothing could honour them: + +```python +--8<-- "docs/examples/row_level_security.py:sqlite_settings" +``` + +A session that only inherits settings from an outer Postgres session opens on SQLite without them. + +## Warnings + +`connect(migrate_updates=True)` (or `migrate_destructive=True`) warns when the connecting role is neither a superuser nor `BYPASSRLS` and an existing table it migrates declares `RowSecurity(force=True)`: + + +```text +ferro auto-migrate: The connected role is neither a superuser nor BYPASSRLS, and this migration touches table(s) 'document', 'invoice' with FORCE ROW LEVEL SECURITY. Row policies apply to the migrating role too, so a backfill or data step can silently see and update zero rows. Migrate as a role with BYPASSRLS if this pass moves data. +``` + +It does not check whether the table's live `FORCE` flag was already set. A table created by the same connect has no rows to miss, so `auto_migrate=True` and `create_tables()` never print it. + +The reconciliation pass also warns about a drifted policy written in SQL, a policy Ferro does not own, a removed declaration whose flags are still on, and an orphaned `rls_*` policy; [Drift and ownership](../concepts/row-level-security.md#drift-and-ownership) explains each. diff --git a/docs/pages/roadmap.md b/docs/pages/roadmap.md index e2363c3d..fbe69e83 100644 --- a/docs/pages/roadmap.md +++ b/docs/pages/roadmap.md @@ -1,22 +1,117 @@ +--- +description: "Features Ferro does not have yet, each with the workaround to use today." +--- + # Roadmap -Ferro is pre-1.0 and under active development. The items below are known gaps we intend to close; priorities are driven by what users actually hit, so [issues](https://github.com/syn54x/ferro-orm/issues) move things up this list. +This page lists the features Ferro does not have yet, each with the workaround to use today. A filter on grouped results is the first gap; with the `Author` and `Post` models below, you filter the groups in Python after `all()`: + +=== "Assignment" + + ```python + --8<-- "docs/examples/faq.py:models" + ``` + +=== "Annotated" + + ```python + --8<-- "docs/examples/faq_annotated.py:models" + ``` + +Each block below runs inside `async with engines.session():`; see [Connections › Sessions](guide/connections.md#sessions). + +## Query features + +### `having()` + +Post-aggregation filtering for [grouped queries](guide/aggregations.md) does not exist, and `where()` rejects an aggregate predicate. Until `having()` ships ([tracked here](https://github.com/syn54x/ferro-orm/issues/291)), filter the groups in Python: + +```python {test=run prelude=reference} +from ferro import engines + +async with engines.session(): + rows = await Post.select( + lambda post: {"author_id": post.author_id, "total": post.views.sum()} + ).all() + busy = [row.author_id for row in rows if row.total > 1000] + assert busy == [1] +``` + +The query sums views per author, and the list comprehension keeps the groups above 1000. + +### Typed record fields + +A projected `Row` types its fields as `Any` on access, so `row.total` above does not check as `int | None`. Inferring a type per field from the selector is [tracked here](https://github.com/syn54x/ferro-orm/issues/290). + +### Reverse and many-to-many population + +[`include()`](guide/queries.md#populating-relations-with-include) populates forward foreign keys in one statement. Populating `BackRef` collections and many-to-many sets is not built: it needs a separate mechanism, a batched second query stitched onto the results. Until then, each awaited collection is its own query: + +```python {test=run prelude=reference} +authors = await Author.all() +counts = [len(await author.posts.all()) for author in authors] +assert counts == [1, 1] +``` -## Query Features +The comprehension issues one query per author. *Filtering* on reverse or many-to-many membership already works; see [existence tests](guide/queries.md#existence-tests-on-reverse-many-to-many-relations). -- **`having()`** — post-aggregation filtering for [grouped queries](guide/aggregations.md); `where()` rejects aggregate predicates pointing at it ([#291](https://github.com/syn54x/ferro-orm/issues/291)). Workaround: filter groups in Python after `all()`. -- **Typed record fields** — a projected `Row`'s fields type as `Any` on access today; inferring per-field types from the selector (so `row.total` checks as `int | None`) is [#290](https://github.com/syn54x/ferro-orm/issues/290). -- **Reverse and many-to-many population** — [`include()`](guide/queries.md#populating-relations-with-include) populates forward-FK paths in one statement; populating `BackRef` collections and M2M sets is a separate future mechanism (a batched second query stitched onto the results). Today each awaited collection is its own query. (*Filtering* on reverse/M2M membership already works — the [existence test](guide/queries.md#existence-tests-on-reverse-many-to-many-relations), `t.lines.exists(...)`.) -- **Cross-scope correlation in existence tests** — comparing an inner-lambda column against the outer scope (`t.lines.exists(lambda line: line.category_id == t.category_id)`) is rejected at build time today; correlated column-to-column comparison is [#309](https://github.com/syn54x/ferro-orm/issues/309). -- **`ilike()`** — case-insensitive pattern matching. Workaround: `like()` with normalized case. -- **Richer update recipes** — `+` / `-`, `now`, and Postgres `.merge()` already run in the database (`update(lambda counter: {"n": counter.n + 1, "updated_at": now})`, `update(lambda conversation: {"turns": conversation.turns.merge(patch)})`). `.concat()` and other operators (`*`, `/`) are still load–modify–`save()` or [raw SQL](guide/raw-sql.md). +### Cross-scope correlation in existence tests + +An inner lambda cannot compare its column to the outer scope. Ferro rejects the comparison when the query is built instead of rendering it wrongly: + +```python {test=run prelude=reference} +try: + Author.where( + lambda author: author.posts.exists(lambda post: post.views > author.id) + ) +except TypeError as error: + assert "cross-scope correlation" in str(error) +else: + raise AssertionError("expected TypeError") +``` + +The `TypeError` names the comparison and says to compare against a value. Column-to-column correlation is [tracked here](https://github.com/syn54x/ferro-orm/issues/309). + +### `ilike()` + +Case-insensitive pattern matching has no operator, and no issue tracks it yet. Store a lowercased copy of the value and match it with `like()`. + +### `.concat()`, `*` and `/` in update recipes + +An update recipe supports `+`, `-`, `now` and Postgres `.merge()`, but not list `.concat()`, `*` or `/`. No issue tracks them yet. Load the row, change it and `save()` it, or use [raw SQL](guide/raw-sql.md): + +```python {test=run prelude=reference} +post = await Post.get(1) +post.views = post.views * 2 +await post.save() +assert (await Post.get(1)).views == 3686 +``` + +The multiplication happens in Python, so the read and the write are two statements. See [Mutations](guide/mutations.md#expression-updates) for the recipes that run in the database. ## Connections -- **`disconnect()`** — graceful pool shutdown for application shutdown hooks. Today cleanup happens at process exit. -- **Health checks** — a `check_connection()`-style probe for readiness endpoints. Workaround: run a trivial query and catch the failure. -- **Richer pool configuration** — `PoolConfig` covers `max_connections`/`min_connections` today; acquire timeouts, idle timeouts, and max connection lifetime are future work. +### `disconnect()` + +There is no graceful pool shutdown for an application's shutdown hook; pools are cleaned up at process exit. No issue tracks it yet. In a test suite, `ferro.reset_engine()` tears the pools down. + +### Health checks + +There is no `check_connection()` probe for readiness endpoints, and no issue tracks one yet. Run a trivial query and catch the failure: + +```python {test=run prelude=reference} +from ferro import fetch_one + +row = await fetch_one("SELECT 1 AS ok") +assert row == {"ok": 1} +``` + +A readiness endpoint treats an exception from this query as unhealthy. + +### Richer pool configuration + +`PoolConfig` has `max_connections`, `min_connections` and `settings_delivery`. Acquire timeouts, idle timeouts and a maximum connection lifetime are future work, and no issue tracks them yet. See [Connections](guide/connections.md#pool-size). -## Influencing Priorities +## Influencing priorities -None of this is on a promised schedule. If one of these gaps blocks you, say so on the [issue tracker](https://github.com/syn54x/ferro-orm/issues) — a concrete use case is the strongest signal we get, and [contributions](contributing.md) are welcome. +None of this has a promised schedule. If a gap blocks you, say so on the [issue tracker](https://github.com/syn54x/ferro-orm/issues): a concrete use case is the strongest signal, and [contributions](contributing.md) are welcome. diff --git a/docs/pages/upgrading.md b/docs/pages/upgrading.md new file mode 100644 index 00000000..a806df8e --- /dev/null +++ b/docs/pages/upgrading.md @@ -0,0 +1,703 @@ +--- +description: "What upgrading Ferro asks of your code, release by release, and how to make each change." +--- + + + +# Upgrading between versions + +This guide lists what upgrading Ferro asks of your code, release by release from `0.11` to `0.26`; the [changelog](changelog.md) is the exhaustive record. After an upgrade, a query reads like this, inside a session: + +```python {test=run prelude=upgrade} +from ferro import engines + +async with engines.session(): + adults = await Account.where(lambda account: account.age >= 40).order_by("name").all() + assert [account.name for account in adults] == ["Alan Turing", "Grace Hopper"] +``` + +The query returns the two accounts aged 40 or over, in name order. Every recipe on this page reads one model, `Account`, declared here once: + +=== "Assignment" + + ```python + --8<-- "docs/examples/upgrade_account.py:models" + ``` + +=== "Annotated" + + ```python + --8<-- "docs/examples/upgrade_account_annotated.py:models" + ``` + +Every block below that calls a model runs inside a session on four accounts, as the first block does; see [Connections › Sessions](guide/connections.md#sessions). Each section shows only what changed. + +## Which sections apply + +Sections run newest first. From version X, read every section for a release newer than X, up to the release you are moving to, top to bottom within that span: + +| You are on | Moving to | Read | +| --- | --- | --- | +| `0.25.x` | `0.26` | [Upgrading to 0.26](#upgrading-to-026) | +| `0.24.x` | `0.26` | [Upgrading to 0.26](#upgrading-to-026) and [Upgrading to 0.25](#upgrading-to-025) | +| `0.15.x` | `0.26` | Every `0.16` to `0.26` section, and the [no-action table](#releases-that-need-no-action) | +| `0.14.x` | `0.26` | The `0.15` section as well | +| `0.12.x` or `0.13.x` | `0.26` | The `0.14` section as well | +| `0.11.x` or earlier | `0.26` | Every section | +| `0.11.x` or earlier | `0.13` | [Upgrading to 0.12](#upgrading-to-012) only | + +Each section with code to change includes a copy/paste prompt for a coding agent. If you cross more than one boundary, run the prompts **newest first, one at a time**, and run your tests between them. + +## Before you start + +Turn deprecation warnings into failures on a throwaway branch, so nothing that still warns slips through: + +```bash +uv run pytest -W error::DeprecationWarning +``` + +The run catches every change a release announced with a `DeprecationWarning`. It does not catch the silent behaviour changes in `0.14` (`create()` and `save()` stop upserting, and `limit()` or `offset()` on a mutation raise), the `NULL` ordering change in `0.21`, or the Postgres timeouts of `0.22`: those emit no warning, so read their sections. + +If you are moving from `0.12` or earlier straight past `0.14`, the deprecated APIs are already removed there and fail with `AttributeError` or `RuntimeError`, not a warning, so the run above catches nothing. Run your suite on `0.13` first, or apply the prompts before you upgrade. + +## Upgrading to 0.26 + +Five changes in `0.26` can reach code that runs today. Two show up in a type checker or an import, two in text you compare, and one in the values a secret column stores: + +- `Session.query` is now typed: it takes `type[M]` and returns `Query[M]`, so a type checker may flag a call that passes a class that is not a `Model`. +- The undocumented module `ferro.migrations.chunked` is now the private `ferro.migrations._chunk_loop`. `from ferro.migrations import chunked` still imports the [`chunked`](api/migrations.md#ferro.migrations.chunked) decorator, unchanged. +- The `RuntimeError` an operation raises with no database route ends differently. It said "Implicit default-connection routing was removed in v0.13 (deprecated since v0.12); see the sessions migration guide." and now says "Implicit default-connection routing was removed in v0.14 (deprecated since v0.12); see the upgrade guide." Code that matches the message on its old ending needs the new one; a match on its start, `No database route`, still holds. +- `save()` of a new or fetched record, `save(only=...)`, `save(exclude=...)` and `bulk_create()` list their columns in the model's field order. Before, a `bytes` column came last, a fetched record's `save()` followed the order its `SELECT` returned, and `save(only=...)` or `save(exclude=...)` could change order from one run to the next. The order changes no value written; the statement text changes, so a test or a log check that compares captured SQL (the [statement log](guide/connections.md#logging) or `.sql()`) needs its expected text updated. `update()` is unchanged: it lists the columns in the order the call passes them, a `bytes` column included. +- A `SecretStr` or `SecretBytes` field now stores its real value, and a fetched record loads it back as a `SecretStr` or `SecretBytes`. `0.25` wrote the mask `**********` in its place on `save()`, `bulk_create()` and `update()`, and loaded the `str` `**********` back. A row `0.25` saved holds the mask, and the real value cannot be recovered from it: write the value again from its source. A `SecretBytes` column is now binary (`BYTEA` on Postgres, `BLOB` on SQLite); `0.25` created it as `varchar`. Change such a column to the binary type with a migration (`ferro migrate new` writes it) before `0.26` saves to it: on Postgres a `SecretBytes` value saved into the `varchar` column is stored as its hex text (`\xff006b` for `b"\xff\x00k"`) and loads back as those characters. + +## Upgrading to 0.25 + +`0.25` adds [`ferro migrate squash`](reference/cli.md#ferro-migrate-squash-name-from-target), which asks nothing of you. It also writes two things `0.24` cannot read, which matters only while both versions share one migrations directory and one database, as in a rolling deploy: + +- **Schema snapshots.** Every snapshot `0.25` writes uses a newer snapshot format. A `0.24` checkout refuses the first migration that carries one, so finish rolling out `0.25` before anyone runs `ferro migrate new`. +- **Inherited rows.** A database that inherited a squash holds rows with `origin = inherited`. A `0.24` runner fails every verb on them with `the tracking table holds a row ferro cannot read (origin)`. A squash is made before a branch merges, so only development databases meet this. + +[Deploying migrations](guide/schema/deploying.md) lists the three ways back from the second. Reference: [Squashing a feature branch's migrations](guide/schema/squash.md). + +## Upgrading to 0.24 + +`0.24` rewrites the output of `ferro migrate`. Every verb but `init` now opens with one line naming the database it works on (`default · sqlite:///./app.db`), a terminal gets a headed layout with colour, and a pipe or a CI log gets a plain form with ASCII markers. A script that reads the output needs a look in three places: + +- The first line is new. Skip it, or read the exit code instead of the text. +- The one-line results scripts grep for (`ok: models match`, `no drift against`, `nothing to apply`), every refusal sentence and every `warning:` line are unchanged. +- A connection error masks the password as `***`; it used to print ``. + +`rich` now ships in the `cli` extra, so `pip install "ferro-orm[cli]"` brings it. Reference: [What the output looks like](reference/cli.md#what-the-output-looks-like). + +## Upgrading to 0.23 + +`0.23` makes a server default that no model declares count as drift, and makes the migrations door tighter about concurrent deploys. Two things ask something of you: + +- **A hand-set server default is reported on every start.** Ferro keeps no server default of its own, so a column default someone added with raw SQL is now a *foreign default*. Under `migrate_updates` the pass warns and leaves it; under `migrate_destructive` it drops it on Postgres. `ferro migrate drift` lists it and `ferro migrate baseline` refuses it. Say a hand-set default sits on `seen_at`: + + ```python + --8<-- "docs/examples/upgrade_foreign_default.py:warning" + ``` + + The pass reports one warning and changes nothing. To silence it, drop the default yourself (`ALTER TABLE "account" ALTER COLUMN "seen_at" DROP DEFAULT` on Postgres), or run the pass with `migrate_destructive=True`. On SQLite only a table rebuild drops a default, so the warning names the hand-written step that does it. + +- **A start-up `baseline()` can branch on a type.** Two services that baseline one database at the same moment used to get a plain `MigrationRefused` from the loser. The loser now gets `AlreadyTrackedError`, a subclass, so `except MigrationRefused` still works and you can catch the narrower type. See [An app that migrates at start-up](guide/schema/adopting-migrations.md#an-app-that-migrates-at-start-up). + +Also new, and opt-in: [`Field(retired=True)`](guide/schema/renames-and-retirements.md#retiring-a-column) and retired enum labels, which let a column or a label leave in two releases, and [`allow_ahead="expand-only"`](guide/schema/deploying.md), which lets an older release start beside a newer schema. A database whose last migration ran on `0.22` has its steps classed on the first run of `0.23`; [Deploying migrations](guide/schema/deploying.md) says how. + +??? example "Automate this: copy this prompt to your coding agent" + + + ```text + You are helping upgrade a Python codebase from Ferro ORM 0.22.x to 0.23. + Reference: https://ferro-orm.x54.sh/upgrading/#upgrading-to-023 + + 1. Server defaults: search the repo's SQL and migration files for + `SET DEFAULT` and `DEFAULT` clauses on columns no Ferro model declares a + default for. List each as file:line. Do not drop any: I decide between + dropping the default and keeping it. + 2. Start-up code that calls ferro.migrations.baseline(): if it catches + MigrationRefused and matches the message text to detect "already + adopted", change it to catch ferro.migrations.AlreadyTrackedError. + ``` + +## Upgrading to 0.22 + +`0.22` adds Ferro's own migrations door: the `ferro` command, a project configuration file and numbered migration directories (see [Schema management](guide/schema/overview.md)). Nothing about adopting it is forced, but four things change for code that stays on auto-migrate: + +- **A database that has run a migration refuses auto-migrate.** `connect(auto_migrate=…)`, `create_tables()` and `migrate()` raise before any DDL on a database that carries the `_ferro_migrations` tracking table, and the message names the two ways out. A database that never ran a migration, such as a test database, is unaffected. See [One door per database](guide/schema/overview.md#one-door-per-database). +- **Postgres passes wait under timeouts.** Each DDL statement waits up to `ddl_lock_timeout` (default `5s`) for its table lock and is retried up to ten times. A pass waits up to `lock_timeout` (default `30s`) for another pass or run that holds the run lock, then fails. Set both in the [project configuration](reference/configuration.md#ddl_lock_timeout) if your deploys hold locks longer. +- **`migrate()` and `create_tables()` return a report.** They returned `None`; they now return a `PassReport` of the statements the pass ran. Code that ignored the return value needs no change: + + ```python {test=run prelude=upgrade} + import ferro + + report = await ferro.migrate() + assert report.statements == () and report.warnings == () + ``` + + The database already matched the models, so the pass ran nothing and warned of nothing. + +- **A cancelled transaction rolls back.** A `transaction()` interrupted by task cancellation or `KeyboardInterrupt` now rolls back and releases its connection; before, the connection stayed pinned to the dead transaction: + + ```python {test=run prelude=upgrade} + import asyncio + + from ferro import transaction + + + async def work(): + async with transaction(): + await Account.create(name="Ghost", email="ghost@example.com", age=1) + await asyncio.sleep(10) + + + task = asyncio.ensure_future(work()) + await asyncio.sleep(0.3) + task.cancel() + try: + await task + except asyncio.CancelledError: + pass + assert await Account.where(lambda account: account.name == "Ghost").count() == 0 + ``` + + The cancelled task left no `Ghost` row behind. + +The `ferro` command is in an extra: `pip install "ferro-orm[cli]"`. Reference: [Configuration](reference/configuration.md) and [CLI](reference/cli.md). + +## Upgrading to 0.21 + +`0.21` changes how `NULL` sorts and how a datetime, date, time, UUID or decimal in a query matches what `save()` stored. Both change results without an error. + +### `NULL` ordering + +An `order_by()` with no `nulls=` now puts `NULL` **last** on every database. It followed each database's own default before: Postgres sorted `NULL` first under `DESC`, and SQLite sorted it first under `ASC`. Name the placement you want, or ask for the database's own: + +=== "Before (database default)" + + ```python {test=compile reason="shows the pre-0.21 call, which still runs but now sorts NULL last"} + accounts = await Account.select().order_by(lambda account: account.seen_at).all() + ``` + +=== "After (NULL first, as SQLite sorted it)" + + ```python {test=run prelude=upgrade} + accounts = await Account.select().order_by(lambda account: account.seen_at, nulls="first").all() + assert [account.name for account in accounts] == [ + "Grace Hopper", + "Alan Turing", + "Ada Lovelace", + "Linus Torvalds", + ] + ``` + +Grace Hopper has never been seen, so her `seen_at` is `NULL` and she sorts first, ahead of the other three in date order. `nulls="native"` keeps each database's own behaviour, which differs between Postgres and SQLite for `"desc"`, so a query that must return the same pages on both should name `"first"` or `"last"`. + +### Query literals + +A `datetime`, `date`, `time`, `UUID` or `Decimal` in `where()`, `after()` or `update()` is now sent in the form `save()` writes. On SQLite a `datetime` is stored as text, and the old literal (`2026-03-01T15:00:00+00:00`) never equalled the stored one (`2026-03-01T15:00:00Z`), so an equality or a keyset page could miss its row. Pass the value itself, and drop any hand-formatted string you used to dodge the miss: + +```python {test=run prelude=upgrade} +seen = datetime(2026, 3, 1, 15, 0, tzinfo=UTC) +by_value = await Account.where(lambda account: account.seen_at == seen).all() +assert [account.name for account in by_value] == ["Ada Lovelace"] +by_text = await Account.where(lambda account: account.seen_at == seen.isoformat()).all() +assert by_text == [] +``` + +The value matches Ada's row. A string from `isoformat()` still misses, because the query sends a string as it is and the stored text ends in `Z`. + +??? example "Automate this: copy this prompt to your coding agent" + + + ```text + You are helping upgrade a Python codebase from Ferro ORM 0.20.x to 0.21. + Reference: https://ferro-orm.x54.sh/upgrading/#upgrading-to-021 + + 1. Find every order_by() over a nullable column that omits `nulls=`. List + each as file:line with the column, and ask me whether NULL should sort + first, last, or follow the database ("native"). Do not guess. + 2. Find every where()/after()/update() that passes a datetime, date, time, + UUID or Decimal, or a string built from one by isoformat()/str(). Pass the + value itself. Query lambdas name their parameter after the model in + lowercase singular: `lambda account: account.seen_at == seen`. + ``` + +## Upgrading to 0.18 + +`0.18` makes a grown enum reach an existing Postgres database. Under `migrate_updates=True`, the pass appends each label the model declares and the live type lacks (`ALTER TYPE … ADD VALUE`), and warns of each live label the model no longer declares. Under plain `auto_migrate=True` an existing enum type is never changed, so a new member passes every test that builds a fresh schema and fails in production with `invalid input value for enum`. If your enums evolve, connect with `migrate_updates=True` or generate the change as a migration. SQLite stores enums as text and needs nothing. Reference: [Evolving enums](guide/schema/auto-migrate.md#evolving-enums-label-addition). + +## Upgrading to 0.17.1 + +`0.17.1` makes the create pass leave existing tables alone: a plain `auto_migrate=True` no longer adds an index to a table that already exists. Add it under `migrate_updates=True`, or in a migration. The same release makes `migrate_updates` read live foreign keys, so a changed `on_delete` is now acted on: Postgres rebuilds the constraint, and SQLite warns, since it cannot alter one in place. Reference: [Applying column changes with `migrate_updates`](guide/schema/auto-migrate.md#applying-column-changes-with-migrate_updates). + +## Upgrading to 0.16.3 + +`0.16.3` refuses a model that declares more than one `primary_key=True` field. The model used to save by its first key and raise when you traversed a relation; it now raises a `TypeError` naming the fields when the class is defined: + +=== "Assignment" + + ```python + --8<-- "docs/examples/upgrade_primary_key.py:refused" + ``` + +=== "Annotated" + + ```python + --8<-- "docs/examples/upgrade_primary_key_annotated.py:refused" + ``` + +The class is refused at definition. Keep one primary key and let the other column be unique, as `Account` does: + +=== "Assignment" + + ```python + --8<-- "docs/examples/upgrade_account.py:primary-key" + ``` + +=== "Annotated" + + ```python + --8<-- "docs/examples/upgrade_account_annotated.py:primary-key" + ``` + +If the pair of columns together must be unique, declare a [composite unique](guide/models-and-fields.md#composite-uniques). Reference: [Models and fields](guide/models-and-fields.md). + +## Releases that need no action + +These releases from `0.13` onward change nothing you have to do. Releases with a section above or below ask something of you. + +| Release | What it adds | Action | +| --- | --- | --- | +| `0.21.2` | Alembic revisions create enum types a revision only adds a column of | None | +| `0.21.1` | Alembic downgrades drop the enum types their `create_table` made | None | +| `0.20.0` | `order_by(..., nulls=)` | None; the default moved in `0.21` | +| `0.19.0` | Named table checks (`__ferro_checks__`); Postgres 18's `RESTRICT` error is a `ForeignKeyViolationError` | None | +| `0.17.0` | `.exists()` on a reverse foreign key and `~` negation of any predicate | None | +| `0.16.2` | A self-referential foreign key no longer breaks `CREATE TABLE` ordering | None | +| `0.16.1` | `bulk_create()` splits batches under the backend's bind-parameter limit | None | +| `0.16.0` | Aggregates and grouped aggregates | None | +| `0.13.0` | Ferro warns and skips, rather than reinterpreting, a Postgres column whose `timestamp` or `timestamptz` storage differs from the model | None; the [`0.14` schema defaults](#schema-emission-defaults) build on it | + +## Upgrading to 0.15 + +`0.15` adds JSONB support and makes **JSONB the default storage** for derived json-family fields (`dict`, `list` or a nested model with no `db_type`) on PostgreSQL. Your model definitions do not change; your PostgreSQL columns might. SQLite is unaffected, because both JSON tokens store the same there. + +On the first connect with `migrate_updates=True`, each existing plain-`json` column with a derived declaration is rewritten in place (`ALTER … TYPE jsonb USING …`). The values survive, but `jsonb` does not preserve the order of a dict's keys. If your code iterates a field's keys in insertion order, or compares stored JSON text byte for byte, keep plain `json` before you upgrade: + +=== "Assignment" + + ```python + --8<-- "docs/examples/upgrade_keep.py:keep-json" + ``` + +=== "Annotated" + + ```python + --8<-- "docs/examples/upgrade_keep_annotated.py:keep-json" + ``` + +The column stays plain `json`, and `0.15` writes no migration for it. `db_type="jsonb"` stays valid and states the default. Alembic autogenerate proposes the same type changes; review the migration instead. Reference: [JSON storage](guide/models-and-fields.md#json-storage-json-and-jsonb). + +??? example "Automate this: copy this prompt to your coding agent" + + + ```text + You are helping upgrade a Python codebase from Ferro ORM 0.14.x to 0.15. + Reference: https://ferro-orm.x54.sh/upgrading/#upgrading-to-015 + + The one breaking change: derived dict/list/nested-model fields now store as + JSONB on PostgreSQL (previously plain json). On the first connect with + migrate_updates=True, each existing plain-json column with a derived + declaration is rewritten in place with ALTER ... TYPE jsonb USING ... — + values survive, but jsonb does not preserve dict key order. + + For each model field whose annotation is dict/list/a nested Pydantic model + and which has NO explicit db_type: + 1. If the code iterates that field's keys relying on insertion order, or + compares stored JSON text byte-for-byte, add db_type="json" to keep + plain-json storage (zero migration). + 2. Otherwise leave it — the one-time ALTER upgrades it to JSONB. + + List every affected field as file:line with your keep/upgrade call and let + me review before connecting to production with migrate_updates=True. + Alembic users: autogenerate will propose the same type changes; review the + migration instead. + ``` + +## Upgrading to 0.14 + +`0.14` makes the mutation surface strict, so no write silently does more or different work than you asked. It routes every database failure through a typed exception hierarchy, derives column types and names in one place for the runtime emitter and the Alembic bridge, and **removes** the operator-style predicates, the `col()` bridge and implicit connection routing that `0.12` deprecated. Your model definitions do not change. + +??? example "Automate this: copy this prompt to your coding agent" + + + ```text + You are helping upgrade a Python codebase from Ferro ORM 0.13.x to 0.14. + Reference: https://ferro-orm.x54.sh/upgrading/#upgrading-to-014 + + Work in two passes. Do NOT open a PR until I have reviewed Pass 2. + + PASS 1 — mechanical replacements (safe to apply directly): + 1. Query predicates: replace operator-style `where(Model.field == value)` and any + `col(...)` bridges with lambda predicates named after the model in lowercase + singular, e.g. `where(lambda account: account.field == value)`. `order_by` + takes a lambda or a validated column-name string. Operator style and + `col()` were removed in 0.14. + 2. Connection routing: any ORM or raw call that ran outside a session + (`Account.all()`, `ferro.execute(...)` with no active session) must run + inside `async with ferro.engines.session("name"):`. Unsessioned calls now + raise. + 3. Exceptions: replace `except RuntimeError` / `except ConnectionError` handlers + around Ferro calls with the typed hierarchy from `ferro` (e.g. + `UniqueViolationError`, `OperationalError`, `InterfaceError`). Match on type, + not on driver-message text. + + PASS 2 — semantic changes (DO NOT rewrite; list each site as file:line and ask me): + 4. `Model.create(...)` and `instance.save()` no longer upsert. For each call site, + tell me whether the code RELIED on the old overwrite-on-conflict behavior. If it + did, I decide between `Model.upsert(...)` / `save(on_conflict="update")` and + catching `UniqueViolationError`. Do not guess. + 5. `.limit()` / `.offset()` chained into `.update()` / `.delete()` now raise. List + each site — bounding a mutation requires fetching primary keys first, and only I + know whether the bound was intentional. + 6. Schema-emission defaults changed for derived types (plain `datetime` -> + timestamptz, Enum -> native enum, `datetime.time` -> time). Auto-migrate refuses + to rewrite existing columns and warns. For each affected model, list it and ask + me to choose keep (`db_type=...`) vs a reviewed migration. + + VERIFY after Pass 1: + - Run the test suite. + - Run `pytest -W error::DeprecationWarning`. This catches the deprecation-based + changes ONLY — items 4, 5, and 6 are silent and will not appear here. + - Grep helper: grep -rn "\.limit(\|\.offset(" --include="*.py" src/ | grep "delete()\|update(" + + Report a summary: what you changed in Pass 1, and the Pass 2 sites awaiting my decision. + ``` + +### Operator predicates and `col()` {#predicates-operator-style-and-col-are-gone} + +`where()` accepts only lambda predicates. Operator-style predicates (`Account.age >= 18`) and the `col()` bridge, both deprecated in `0.12`, are removed. Class attributes are no longer replaced with a field proxy, so `Account.age` at class scope raises `AttributeError` under normal Pydantic semantics. `order_by()` takes a lambda (`order_by(lambda account: account.seen_at, "desc")`) or a validated column-name string (`order_by("seen_at", "desc")`): + +=== "Before (removed)" + + ```python {test=compile reason="shows the pre-0.14 operator style, which now raises"} + adults = await Account.where(Account.age >= 40).all() + admins = await Account.where(Account.role == "admin").all() + ``` + +=== "After" + + ```python {test=run prelude=upgrade} + adults = await Account.where(lambda account: account.age >= 40).order_by("name").all() + assert [account.name for account in adults] == ["Alan Turing", "Grace Hopper"] + admins = await Account.where(lambda account: account.role == Role.ADMIN).all() + assert [account.name for account in admins] == ["Ada Lovelace"] + ``` + +Both queries return the same rows they always did; only the predicate spelling changed. Reference: [Typed query predicates](concepts/query-typing.md). + +### Session routing {#connection-routing-requires-a-session} + +Unqualified operations (`Account.all()`, `ferro.execute(...)`) used to fall back to an implicit default connection. That fallback is removed. Wrap request- or task-scoped work in a session, so routing, the identity map and transactions are explicit and isolated under concurrency. A call with no active session and no `using=` raises `RuntimeError`; an explicit `using=` that names a different connection than the ambient session raises `ValueError`: + +=== "Before (removed)" + + ```python {test=compile reason="shows the pre-0.14 implicit routing, which now raises RuntimeError"} + import ferro + + accounts = await Account.where(lambda account: account.active == True).all() + await ferro.execute("UPDATE account SET active = 1") + ``` + +=== "After (ambient session)" + + ```python {test=run prelude=upgrade} + import ferro + + async with ferro.engines.session(): + accounts = await Account.where(lambda account: account.active == True).all() + await ferro.execute("UPDATE account SET active = 1") + assert len(accounts) == 3 + ``` + +The session routes both calls to its connection, and the query returns the three active accounts. Reference: [Identity map](concepts/identity-map.md) and [Sessions](guide/connections.md#sessions). + +### `create()` on a primary-key conflict + +`create()` used to compile to `INSERT … ON CONFLICT (pk) DO UPDATE`, so a duplicate primary key silently **replaced** the existing row. It is a plain `INSERT` now: a duplicate raises [`UniqueViolationError`](api/exceptions.md) and the stored row stays as it was: + +=== "Before (silent clobber)" + + ```python {test=compile reason="shows the pre-0.14 create(), which overwrote the stored row"} + await Account.create(id=1, name="Ada", email="ada@example.com", age=1) + ``` + +=== "After (insert or update, explicit)" + + ```python {test=run prelude=upgrade} + account = await Account.upsert(id=1, name="Ada Lovelace", email="ada@example.com", age=37) + assert account.age == 37 + ``` + +=== "After (insert only)" + + ```python {test=run prelude=upgrade} + from ferro import UniqueViolationError + + try: + await Account.create(id=1, name="Ada", email="ada@example.com", age=1) + except UniqueViolationError: + ada = await Account.where(lambda account: account.id == 1).first() + assert ada.age == 36 # the stored row is untouched + else: + raise AssertionError("create() overwrote the stored row") + ``` + +`upsert()` replaces the row; `create()` raises and leaves Ada at 36. Reference: [Mutations](guide/mutations.md). + +### `save()` and upserts + +`save()` now tracks whether the instance has been persisted: a new instance is inserted, and a fetched or previously saved instance is updated by primary key. Two consequences for code written against the old behaviour: + +- Saving a *new* instance whose primary key is taken raises `UniqueViolationError` instead of overwriting the stored row. +- Saving a *persisted* instance whose row was deleted underneath raises `ModelDoesNotExist` instead of silently re-inserting it. + +Where you used `save()` on purpose to insert or update, say so: + +=== "Before (every save upserted)" + + ```python {test=compile reason="shows the pre-0.14 save(), which upserted every call"} + account = Account(id=2, name="Grace Hopper", email="grace@example.com", age=46) + await account.save() # inserted or replaced, whichever applied + ``` + +=== "After (explicit upsert)" + + ```python {test=run prelude=upgrade} + account = Account(id=2, name="Grace Hopper", email="grace@example.com", age=46) + await account.save(on_conflict="update") + grace = await Account.where(lambda account: account.id == 2).first() + assert grace.age == 46 + ``` + +The save updated Grace's row. The conflict target is the primary key only, and the whole row is written on update; see [Upsert](guide/mutations.md#upsert). + +### `limit()` and `offset()` on mutations + +Portable SQL has no `DELETE … LIMIT`. The pagination used to be silently dropped, so the mutation touched **every** matching row. `update()` and `delete()` now raise `ValueError` when the query carries `limit()` or `offset()`. To mutate a bounded subset, fetch primary keys first: + +=== "Before (limit silently ignored)" + + ```python {test=compile reason="shows the pre-0.14 behaviour, which deleted every active account"} + await Account.where(lambda account: account.active == True).limit(2).delete() + ``` + +=== "After (bounded by a primary-key set)" + + ```python {test=run prelude=upgrade} + try: + await Account.where(lambda account: account.active == True).limit(2).delete() + except ValueError: + pass # the bounded delete is refused + else: + raise AssertionError("delete() accepted limit()") + + batch = await Account.where(lambda account: account.active == True).order_by("id").limit(2).all() + ids = [account.id for account in batch] + await Account.where(lambda account: account.id.in_(ids)).delete() + remaining = await Account.select().order_by("id").all() + assert [account.name for account in remaining] == ["Alan Turing", "Linus Torvalds"] + ``` + +The delete removed Ada and Grace, the first two active accounts, and no others. Reference: [Mutations](guide/mutations.md). + +### Typed exceptions + +Database failures raise a hierarchy rooted at [`ferro.FerroError`](api/exceptions.md) instead of bare `RuntimeError` or `ConnectionError`. Broad handlers keep working during the move (`FerroError` is still an `Exception`), but match on the type: + +=== "Before" + + ```python {test=compile reason="shows the pre-0.14 handler, which matched driver text"} + try: + await account.save() + except RuntimeError as error: + if "unique" in str(error).lower(): # matching driver text + ... + ``` + +=== "After" + + ```python {test=run prelude=upgrade} + from ferro import UniqueViolationError + + try: + await Account(id=1, name="Dup", email="dup@example.com", age=1).save() + except UniqueViolationError as error: + assert error.driver_message # error.sqlstate and error.constraint carry detail + else: + raise AssertionError("a taken primary key was accepted") + ``` + +The handler runs on the type, with the driver's text kept as `driver_message`. `connect()` failures are `ferro.OperationalError` (server or environment) or `ferro.InterfaceError` (bad scheme or configuration) instead of `ConnectionError`. Reference: [Exceptions](api/exceptions.md). + +### Schema-emission defaults {#schema-emission-defaults} + +`0.14` also derives column types and artifact names in one place for the runtime emitter and the Alembic bridge. Three storage defaults change for *derived* types (fields with no explicit `db_type`). **Auto-migrate never rewrites an existing column silently**: where an older emitter built your database, Ferro warns and skips, and you choose between keeping the old storage and converting with a reviewed migration (a hand-written step in `ferro migrate`, or an Alembic revision). + +#### Plain `datetime` fields are timezone-aware + +The Alembic bridge used to map a plain `datetime` field to a naive `timestamp`; the runtime already used `timestamptz`. Both now emit `timestamptz`. If your schema has naive `timestamp` columns, auto-migrate refuses the conversion and warns. Keep the column naive by declaring it: + +=== "Assignment" + + ```python + --8<-- "docs/examples/upgrade_keep.py:keep-timestamp" + ``` + +=== "Annotated" + + ```python + --8<-- "docs/examples/upgrade_keep_annotated.py:keep-timestamp" + ``` + +The column stays `timestamp` and the warning goes away. To convert it instead, run this in a reviewed migration, with your source time zone in place of `UTC`: + +```sql +ALTER TABLE "account" + ALTER COLUMN "seen_at" TYPE timestamptz + USING "seen_at" AT TIME ZONE 'UTC'; +``` + +The statement reads each stored value as UTC and rewrites the column as `timestamptz`. + +#### Enum fields are native Postgres enums + +The runtime emitter used to store an `Enum` field as `varchar`; the Alembic bridge already emitted a native enum type. Both now emit the native type, created idempotently (on SQLite the column is `varchar()`). If your schema has `varchar` enum columns, auto-migrate refuses the conversion and warns. Keep the text storage, with its check, by declaring it: + +=== "Assignment" + + ```python + --8<-- "docs/examples/upgrade_keep.py:keep-varchar" + ``` + +=== "Annotated" + + ```python + --8<-- "docs/examples/upgrade_keep_annotated.py:keep-varchar" + ``` + +The column stays text, and `db_check=True` keeps its `CHECK` over the labels. To convert it, run this in a reviewed migration: + +```sql +CREATE TYPE "role" AS ENUM ('admin', 'member'); +ALTER TABLE "account" + ALTER COLUMN "role" TYPE "role" USING "role"::"role"; +``` + +The statement creates the enum type and casts each stored label into it. + +#### `datetime.time` fields store as `time` + +The runtime emitter used to store a `datetime.time` field as `varchar` (the bridge already said `TIME`). Both now emit `time`. A `time` annotation accepts no text `db_type`, so there is no declaration that keeps the old column: auto-migrate refuses the conversion and warns, and you convert with a reviewed migration that casts explicitly, as in `ALTER COLUMN "starts" TYPE time USING "starts"::time`. + +#### Constraint and index names + +- **Foreign keys are named** (`fk_
__`) in the DDL of both emitters. Existing databases need no action: Alembic autogenerate compares foreign keys by signature, not name. +- **Single-column uniques become named unique indexes** (`uq_
_`), the shape composite uniques always had. On an existing database auto-migrate adds the named index (`CREATE UNIQUE INDEX IF NOT EXISTS`, additive and idempotent); the old inline artifact (on Postgres, `account_email_key`) stays as a redundant duplicate until you drop it: + + ```sql + ALTER TABLE "account" DROP CONSTRAINT account_email_key; + ``` + +- **Identifiers longer than 63 characters truncate the same way** on both emitters. PostgreSQL used to truncate some of them silently. +- **SQLite declared types use SQLAlchemy's spellings** (`DATETIME`, `DATE`, `TIME`, `CHAR(32)`, `JSON`, `NUMERIC`), and primary-key columns carry an explicit `NOT NULL`. Existing SQLite databases need no action, because type affinity makes the storage identical; only new tables show the new spellings. + +### Changed surfaces + +| Surface | `0.13` behaviour | `0.14` behaviour | What you do | +| --- | --- | --- | --- | +| `Model.create(**fields)` on an existing primary key | Replaced the row | Raises `UniqueViolationError` | `Model.upsert(...)` if you wanted replace | +| `instance.save()` on a new instance with a taken primary key | Replaced the row | Raises `UniqueViolationError` | `save(on_conflict="update")` | +| `instance.save()` after the row was deleted | Re-inserted it | Raises `ModelDoesNotExist` | Catch it and `create()` again if intended | +| `query.limit(...).update()` or `.delete()` | Ignored the pagination | Raises `ValueError` | Fetch primary keys, mutate by that set | +| Database failures | `RuntimeError` with driver text | Typed `FerroError` subclasses | Catch by type | +| Plain `datetime` field (Alembic bridge) | Naive `timestamp` | `timestamptz` | Keep with `db_type="timestamp"`, or convert with `USING … AT TIME ZONE` | +| Enum field (runtime emitter) | `varchar` | Native Postgres enum | Keep with `db_type="text"`, or convert with `USING col::` | +| `datetime.time` field (runtime emitter) | `varchar` | `time` | Convert with a reviewed `USING` cast | +| Foreign keys | Anonymous | Named `fk_
__` | Nothing; names are not compared | +| Single-column unique | Inline column `UNIQUE` | Named `uq_
_` unique index | Optionally drop the old inline constraint | + +## Upgrading to 0.12 + +`0.12` is the first release built on Ferro's IR-first architecture: query execution, schema and migration planning, codecs, hydration and connection routing all flow through one shared intermediate representation instead of several independent code paths. One source of truth removes whole classes of drift bugs, and gives predictable schema diffs, identical bind and null semantics across SQLite and PostgreSQL, and explicit session-scoped runtime state. Your model definitions do not change; a few APIs are called differently. + +`0.12` introduced these as **deprecation warnings**; the removals landed in `0.14`, see [Upgrading to 0.14](#upgrading-to-014). + +??? example "Automate this: copy this prompt to your coding agent" + + + ```text + You are helping upgrade a Python codebase from Ferro ORM 0.11.x to 0.12. + Reference: https://ferro-orm.x54.sh/upgrading/#upgrading-to-012 + + All 0.12 changes are deprecation-based and safe to apply mechanically. Apply them, + then verify. + + 1. Query predicates: replace operator-style `where(Model.field == value)` with + lambda predicates named after the model in lowercase singular: + `where(lambda account: account.field == value)`. (These were removed + outright in 0.14 — if you are upgrading past 0.12, prefer lambdas now.) + 2. Connection routing: wrap request/task-scoped ORM and raw operations in + `async with ferro.engines.session("name"):` instead of relying on implicit + default-connection routing. + 3. Alembic metadata: replace + `from ferro.migrations.alembic import _build_sa_table, _map_to_sa_type` + with `from ferro.migrations import get_metadata; target_metadata = get_metadata()` + in your Alembic env.py. + + VERIFY: + - Run the test suite. + - Run `pytest -W error::DeprecationWarning`; a clean run means no deprecated + 0.12-era surfaces remain on the paths you exercise. + ``` + +### Alembic metadata from `get_metadata()` + +The private helpers `ferro.migrations.alembic._build_sa_table` and `ferro.migrations.alembic._map_to_sa_type` were removed in `0.14`. Schema metadata comes from the public `get_metadata()` entry point; use it in your Alembic `env.py`: + +=== "Before (removed)" + + ```python {test=compile reason="shows the removed private helpers"} + from ferro.migrations.alembic import _build_sa_table, _map_to_sa_type + ``` + +=== "After" + + ```python {test=run prelude=upgrade} + from ferro.migrations import get_metadata + + target_metadata = get_metadata() + assert target_metadata is not None + ``` + +`target_metadata` is the SQLAlchemy metadata Alembic compares against. Reference: [Alembic](guide/schema/alembic.md). + +### Deprecations of 0.12 + +| Deprecated in `0.12` | Replacement | Status | +| --- | --- | --- | +| `Model.where(Model.field OP value)` and `col()` | `where(lambda account: …)` | Removed in `0.14`, see [Operator predicates](#predicates-operator-style-and-col-are-gone) | +| Unqualified operations outside an active session | `async with ferro.engines.session("name")` | Removed in `0.14`, see [Session routing](#connection-routing-requires-a-session) | +| `ferro.migrations.alembic._build_sa_table` | `ferro.migrations.get_metadata()` | Removed in `0.14`, see [Alembic metadata](#alembic-metadata-from-get_metadata) | +| `ferro.migrations.alembic._map_to_sa_type` | `ferro.migrations.get_metadata()` | Removed in `0.14`, see [Alembic metadata](#alembic-metadata-from-get_metadata) | diff --git a/docs/pages/why-ferro.md b/docs/pages/why-ferro.md index 4d7c240e..1d6a4d6c 100644 --- a/docs/pages/why-ferro.md +++ b/docs/pages/why-ferro.md @@ -1,81 +1,90 @@ -# Why Ferro? +--- +description: "What Ferro's Rust engine buys you, what it costs and when another ORM is the better choice." +--- -## The Problem +# Why Ferro -Python ORMs are convenient, but they come with a performance tax. Traditional ORMs like SQLAlchemy, Django ORM, and Tortoise spend significant CPU time in Python on SQL generation (query strings, escaping, JOINs), row parsing (turning database rows into objects), and object instantiation (`__init__`, validators, attributes). All of that happens while holding the Global Interpreter Lock. +Ferro is an async Python ORM that moves SQL generation, execution and row parsing into a Rust engine, so Python only builds a small filter description and receives finished model instances. This page explains what that buys, what it costs and when another ORM is the better choice. -For simple CRUD this overhead is acceptable. But when you process thousands of rows per request, run high-concurrency workloads, or care about tail latency in services, the Python tax becomes the bottleneck. +Here is a query on a blog's `Post` model. It runs inside an `async with engines.session():` block (see [Sessions](guide/connections.md#sessions)): -## How Ferro is Different +```python {test=run prelude=reference} +popular = await Post.where(lambda post: post.views >= 1000).all() +assert [post.title for post in popular] == ["Notes on the Analytical Engine"] +``` -Ferro moves the expensive parts out of Python and into a Rust engine, connected to Python through a PyO3 FFI bridge. +Python builds the predicate from the lambda. Rust renders the parameterized SQL, runs it and hands back `Post` objects, here the one post with at least 1000 views. -### Rust Core +## Where Python ORMs spend time -Sea-Query builds parameterized SQL in Rust. SQLx executes queries and parses rows GIL-free. Data flows from the database through Rust into Python with zero-copy intent. SQLite and PostgreSQL support is compiled into the engine, so there are no separate driver packages. +Python ORMs are convenient and carry a performance tax. SQLAlchemy, Django ORM and Tortoise spend CPU time in Python on SQL generation (query strings, escaping, JOINs), row parsing (turning rows into objects) and object instantiation (`__init__`, validators, attributes). All of it runs while holding the Global Interpreter Lock. -When you call `User.where(lambda t: t.age >= 18).all()`, Python only builds a small filter AST. SQL generation, execution, and row parsing all happen in Rust; Python receives hydrated `User` objects at the end. +For simple CRUD this overhead is acceptable. When a request processes thousands of rows, a service runs many concurrent queries or tail latency matters, it becomes the bottleneck. -### Pydantic-Native +## Rust core -Unlike ORMs that wrap Pydantic or use it as a serialization layer, Ferro models *are* Pydantic models: +Sea-Query builds the SQL in Rust, and SQLx runs it and parses rows outside the GIL. Rust then fills each model's fields directly, without calling `__init__`, so rows do not pass through Pydantic's constructor. SQLite and PostgreSQL support is compiled into the engine, so there are no driver packages. -- Models inherit directly from `pydantic.BaseModel` -- Validation runs in `pydantic-core` (also Rust) -- Type hints work exactly as your IDE and type checker expect -- No adapter layer between your ORM models and your API schemas +## Pydantic models -If you already use FastAPI or any Pydantic-heavy stack, your database models and your validation models are the same objects. +Ferro models are Pydantic models: -### Async-First +- Models inherit from `pydantic.BaseModel`. +- Validation runs in `pydantic-core`, which is also Rust. +- Type hints work as your IDE and type checker expect. +- There is no adapter layer between your database models and your API schemas. + +If you already use FastAPI or another Pydantic-heavy stack, your database models and your validation models are the same objects. + +## Async API Ferro is built on `sqlx-core` and `pyo3-async-runtimes`: -- True async from Rust to Python — no sync wrappers or thread pools -- Connection pooling handled by SQLx -- Concurrent query execution without blocking the event loop +- The async path runs from Rust to Python, with no sync wrappers or thread pools. +- SQLx handles connection pooling. +- Queries run concurrently without blocking the event loop. -## What You Give Up +## Trade-offs -Ferro is not the right choice for every project. The trade-offs: +Ferro is not the right choice for every project: -- **Python 3.13+ only.** Ferro targets modern Python and does not support older interpreters. -- **Async-only API.** There is no synchronous interface. If your application is sync (e.g., classic Flask or scripts without an event loop), Ferro is a poor fit. -- **Young feature set.** Ferro covers models, queries, mutations, relationships (including forward-FK population via `include()`), partial selects, aggregations with grouped queries, transactions, and Alembic-based migrations — but some features common in mature ORMs are not implemented yet, including `having()`, reverse/M2M collection population, and atomic update expressions. See the [Roadmap](roadmap.md) for what's planned. -- **Smaller ecosystem.** Fewer third-party integrations, plugins, and Stack Overflow answers than SQLAlchemy or Django. -- **Rust at the bottom.** You never need Rust to *use* Ferro, but contributing to or extending the engine requires it, and building from source needs a Rust toolchain. +- Ferro supports Python 3.13 and newer only. +- The API is async-only. If your application is synchronous, such as classic Flask or scripts without an event loop, Ferro is a poor fit. +- The feature set is young. Ferro covers models, queries, mutations, relationships (including forward-FK population with `include()`), partial selects, grouped aggregations, transactions and migrations. Some features of mature ORMs are missing: `having()`, reverse and many-to-many collection population, and the `*`, `/` and `.concat()` operators in update expressions. The [Roadmap](roadmap.md) lists what is planned. +- The ecosystem is smaller: fewer integrations, plugins and Stack Overflow answers than SQLAlchemy or Django. +- Contributing to the engine, or building from source, needs a Rust toolchain. Using Ferro does not. -## Comparison +## Comparison with other ORMs | | Ferro | SQLAlchemy 2.0 | Django ORM | Tortoise ORM | |---|---|---|---|---| -| **Core** | Rust (SQLx + Sea-Query) | Python | Python | Python | -| **Async support** | Native, async-only | Native (opt-in) | Limited | Native | -| **Type safety** | Pydantic models | Typed declarative API | Dynamic | Basic Pydantic integration | -| **Learning curve** | Low | High | Low | Low | -| **Migrations** | Alembic (optional extra) | Alembic | Built-in | Aerich | -| **Runtime dependencies** | Pydantic only | Several | Django | Several | -| **Ecosystem maturity** | Young | Very mature | Very mature | Moderate | -| **Backends** | SQLite, PostgreSQL | Many dialects | Many | Several | +| Core | Rust (SQLx and Sea-Query) | Python | Python | Python | +| Async support | Native, async-only | Native (opt-in) | Limited | Native | +| Sync API | No | Yes | Yes | No | +| Type safety | Pydantic models | Typed declarative API | Dynamic | Basic Pydantic integration | +| Migrations | Built in (`ferro migrate`), plus an Alembic bridge | Alembic | Built in | Aerich | +| Runtime dependencies | Pydantic and pydantic-settings | Several | Django | Several | +| Ecosystem maturity | Young | Very mature | Very mature | Moderate | +| Backends | SQLite, PostgreSQL | Many dialects | Many | Several | -Ferro's architecture is designed to make bulk reads, large result sets, and row hydration fast by keeping that work in Rust and outside the GIL. For single-row operations, network and disk latency dominate and every ORM performs similarly — choose based on ergonomics and ecosystem, not microbenchmarks. +Ferro keeps bulk reads, large result sets and row hydration in Rust and outside the GIL. For single-row operations, network and disk latency dominate and every ORM performs similarly, so choose on ergonomics and ecosystem rather than microbenchmarks. -## When to Choose Ferro +## When to choose Ferro Choose Ferro when: -- You're building **async services** — FastAPI, Starlette, Litestar, or anything on asyncio -- Your codebase is **Pydantic-heavy** and you want one model class for validation and persistence -- You move **lots of rows** — data pipelines, bulk ingestion, read-heavy APIs -- You want a **small dependency footprint** (Pydantic is the only runtime dependency) -- You're on **SQLite or PostgreSQL** and Python 3.13+ +- You are building async services on FastAPI, Starlette, Litestar or anything on asyncio. +- Your codebase is Pydantic-heavy and you want one model class for validation and persistence. +- You move many rows: data pipelines, bulk ingestion, read-heavy APIs. +- You want a small dependency footprint: Pydantic and pydantic-settings. +- You are on SQLite or PostgreSQL and Python 3.13 or newer. Choose something else when: -- You need a **sync API** or support for Python < 3.13 -- You need **dialects beyond SQLite/PostgreSQL** (MySQL, MSSQL, Oracle) — use SQLAlchemy -- You're inside a **Django project** — the integrated Django ORM is the pragmatic choice -- You rely on features Ferro hasn't shipped yet — check the [Roadmap](roadmap.md) before committing -- You need **maximum query flexibility** for deeply complex SQL — SQLAlchemy Core is hard to beat +- You need a sync API or Python older than 3.13. +- You need dialects beyond SQLite and PostgreSQL (MySQL, MSSQL, Oracle). Use SQLAlchemy. +- You work inside a Django project. The Django ORM is the pragmatic choice. +- You rely on features Ferro has not shipped. Check the [Roadmap](roadmap.md) first. +- You need maximum query flexibility for deeply complex SQL. SQLAlchemy Core is hard to beat. -Migrating from SQLAlchemy? There's a [dedicated guide](howto/migrate-from-sqlalchemy.md). Otherwise start with the [Quickstart Tutorial](getting-started/quickstart.md) (about 10 minutes). +Migrating from SQLAlchemy? Read [Migrate from SQLAlchemy](howto/migrate-from-sqlalchemy.md). Otherwise start with the [Quickstart](getting-started/quickstart.md). diff --git a/docs/solutions/issues/docs-block-declaring-a-model.md b/docs/solutions/issues/docs-block-declaring-a-model.md new file mode 100644 index 00000000..a86d11cc --- /dev/null +++ b/docs/solutions/issues/docs-block-declaring-a-model.md @@ -0,0 +1,50 @@ +--- +title: A runnable docs block that declares a model inherits the runner's context +type: issue +tags: [docs, python, pydantic, pytest, gotcha] +related_files: + - docs/examples/_preludes/__init__.py + - tests/test_docstring_examples.py + - src/ferro/metaclass.py +related_issues: [704, 719] +related_prs: [] +captured: 2026-10-19 +--- + +## Problem + +A `{test=run prelude=...}` block that declared its own model failed in two +ways that had nothing to do with the block. Declaring any model raised +`SyntaxError` from pydantic, and a model with a `ForeignKey` got a `JSON` +column named after the relation instead of a `_id` foreign key. + +## Takeaway + +The runner compiles and runs each block, and anything the runner's own +module carries leaks into the block unless it is cut off. Run the block as a +module of its own whose `__file__` is the page or docstring, and compile it +with `dont_inherit=True`. + +## What happened + +- **`SyntaxError`**: `Model` sets `use_attribute_docstrings=True`, so pydantic + reads each model's source with `inspect.getsourcelines(cls)`. The block ran + with `__name__ == "__main__"`, so the source lookup found the runner file + and parsed its docstring as the class body. +- **`JSON` column**: `compile()` inherits the caller's `__future__` flags. + The runner has `from __future__ import annotations`, so every annotation in + the block became a string, and the metaclass never saw the `ForeignKey` + inside `Annotated[...]`. + +Ferro has the same gap outside the harness: a module with +`from __future__ import annotations` that declares +`author: Annotated[Author, ForeignKey(related_name="posts")]` gets a `JSON` +column `author` and no error. Until the metaclass resolves string +annotations, keep that import out of model modules. + +## How to recognize + +- `SyntaxError` with a line of English prose from + `pydantic/_internal/_docs_extraction.py`. +- `CREATE TABLE ... "" JSON` where a `"_id"` column and a + `FOREIGN KEY` constraint belong. diff --git a/docs/solutions/issues/pytest-examples-prefix-settings-need-quotes.md b/docs/solutions/issues/pytest-examples-prefix-settings-need-quotes.md new file mode 100644 index 00000000..77e74a00 --- /dev/null +++ b/docs/solutions/issues/pytest-examples-prefix-settings-need-quotes.md @@ -0,0 +1,35 @@ +--- +title: pytest-examples ignores bare fence settings like {test=run} +type: issue +tags: [pytest, docs, gotcha] +related_files: + - tests/_docs_harness.py + - tests/test_docs_examples.py + - docs/examples/_preludes/__init__.py +related_issues: [704] +captured: 2026-10-19 +--- + +A docs fence marked ` ```python {test=run} ` never ran, because +`CodeExample.prefix_settings()` from pytest-examples only reads quoted values +(`test="run"`). + +Read fence settings with `fence_settings()` in `tests/_docs_harness.py` +instead. It accepts both forms: + +```python +fence_settings(example) # 'python {test=run prelude=reference}' +# {'test': 'run', 'prelude': 'reference'} +``` + +The docs harness used `prefix_settings()` from the start, so the +`{test=run}` and `{lint=check}` opt-ins it documented did nothing for the bare +spelling. No page used them, so nothing failed. Every other check in the +harness reads settings through `fence_settings()`, which also refuses a +`prelude=` without `test=run`. + +## How to recognize + +A fence carries a setting and its test passes, but deliberately breaking the +block also passes. Print `example.prefix_settings()`: an empty dict for a +prefix that clearly has `key=value` pairs is this bug. diff --git a/docs/solutions/patterns/docs-page-conventions.md b/docs/solutions/patterns/docs-page-conventions.md new file mode 100644 index 00000000..1e5c9fd1 --- /dev/null +++ b/docs/solutions/patterns/docs-page-conventions.md @@ -0,0 +1,308 @@ +--- +title: Docs page conventions (kind, running example, results, printed output, session, SQL, glossary, Annotated twins) +type: pattern +tags: [docs, convention, zensical, pydantic] +related_files: + - AGENTS.md + - docs/examples/ + - docs/includes/abbreviations.md + - tests/_docs_harness.py + - tests/_docs_sql.py + - tests/_docs_twins.py + - tests/test_docs_examples.py + - tests/test_docs_output.py + - tests/test_docs_sql.py + - tests/test_docs_twins.py + - zensical.toml +related_issues: [698, 700, 701, 703, 704, 725] +related_prs: [] +captured: 2026-10-19 +--- + +## Problem + +The 2026 docs audit found pages a reader could not copy from: snippets that +stop just above the answer, three to fourteen unrelated models per page, no +session wrapper anywhere, no SQL, and Annotated tabs that differ from their +Assignment twins in more than declaration style. + +## Takeaway + +A page is one kind, uses one running example, shows every result, shows the +session it runs in, and shows the SQL Ferro sends. Annotated twins use one +idiom: `name: Annotated[T, Field(...)] = default`. + +## The conventions + +### One kind per page + +Each page is a tutorial, a how-to, a reference or an explanation, and opens +with one sentence saying what it covers, then an example on the first screen. +Material of another kind moves to its own page and is linked. Option tables, +exit codes and other reference tables live once under Reference; guides link +to them. + +### One running example per page + +A page declares one model set near the top and reuses it to the end; each +later section shows only what changed. Example data is about four rows that +read like real records, with no tied values, so the reader can check a +result by eye. Docstrings and the API reference share one running example, +the `reference` prelude (`docs/examples/_preludes/reference.py`, `Author` and +`Post`). + +### Snippet `end:` markers go below the result + +The embedded range ends after the `assert` or `print` that holds the answer, +so the page shows what the call returns. + +Before (the page shows a query and never its result): + +```python +# --8<-- [start:filtering] +adults = await User.where(lambda user: user.age >= 18).all() +# --8<-- [end:filtering] +assert [user.name for user in adults] == ["Ada", "Grace"] +``` + +After: + +```python +# --8<-- [start:filtering] +adults = await User.where(lambda user: user.age >= 18).all() +assert [user.name for user in adults] == ["Ada", "Grace"] +# --8<-- [end:filtering] +``` + +### Printed output is regenerated, never typed + +A ```text block shows what something printed, and says where that came from +in an HTML comment on the line directly above it, at the fence's indentation: + +````markdown + +```text +Rent -1450 +To savings -500 +``` +```` + +| Note | The block is | Checked by | +| --- | --- | --- | +| `` | What that snippet region prints | `tests/test_docs_output.py`, byte for byte | +| `` | What the region prints on PostgreSQL; the script runs as `.py ` | `tests/test_docs_output.py` under `postgres_only`, byte for byte | +| `` | What the nearest Python fence above prints; that fence is marked `{test=run prelude=}` | `tests/test_docs_output.py`, byte for byte | +| ``, `` | What `ferro migrate` prints | `tests/test_docs_migrations.py` | +| `` | Text Ferro does not print: a sketch, a file tree, another tool's shell session, a prompt to paste | Nothing; the note says what it is | + +`tests/test_docs_output.py` refuses a ```text block with no note. To add one +under an embedded example, put the `print` inside the snippet region (a +region that catches an error prints `f"{type(exc).__name__}: {exc}"`), write +the note and an empty fence, and fill the fence from the failure message. A +printed line belongs to the innermost line of the example inside a region on +the call stack, so a helper defined outside the region can do the printing. +Every error, warning and message Ferro prints is a `prints:` block: write the +scenario that raises it as an example region, however small (a model, a +connect, a held lock, a migrations project from `_migrations_project.py`). +`docs/examples/auto_migrate_errors.py` and `migration_errors.py` are the +pattern; a region not shown on any page is fine, since the page shows the +text. A `shows:` block that reads like Ferro's output (`ferro.`, +`ferro auto-migrate:`, `SomethingError:`, `N validation error for`, +`connect(`) fails the test. The one exception, in `VARIES`, is the statement +log on the Connections guide, whose lines carry timings that change on every run. + +### The session wrapper + +Every ORM call runs inside `async with engines.session():` and raises +`No database route` outside it. The first code block on a page shows the +wrapper. Every later block either shows it too or sits under a sentence +saying it runs inside one, linking to Connections › Sessions. An inline block +that does real work runs under the docs harness with a prelude that opens the +session (`{test=run prelude=}`, from #704). + +### Generated SQL beside each query + +Each query and each mutation an example runs shows the SQL Ferro sends, in a +`sql` fence below the example's result sentence. A page never carries SQL a +person typed: a hand-simplified block drifts (unquoted reserved words, +invented aliases), and the four such blocks the audit found on the Queries +guide had. The block's first line is a marker naming where its SQL comes from: + +```sql +-- predicates.py:first-query (sqlite) +SELECT "transaction"."account_id", ... +FROM "transaction" +WHERE "transaction"."amount" <= ? +ORDER BY "transaction"."amount" ASC NULLS LAST; +``` + +`.py` is a script in `docs/examples/`, `first-query` one of its +snippet regions (`# --8<-- [start:first-query]`), and `(sqlite)` or +`(postgres)` the dialect. The block holds every statement the region sends, +in order: a query terminal's statement as its `.sql()` renders it, and any +other statement (`save()`, `create()`, `bulk_create()`, an instance's +`delete()`, a many-to-many `add()`, a raw `execute()`) as the statement log +records it. Schema statements, the Postgres catalog probe, session settings +and a `transaction()` block's own `BEGIN` / `COMMIT` are left out. A region +that sends nothing but prints statements (a `create_tables()` report's +`statement.sql`) shows what it prints. + +`tests/test_docs_sql.py` regenerates every `sql` fence on the pages it lists +(`PAGES`: Queries, Aggregations, Relationships, Mutations) and compares byte +for byte, failing with the page, the line and the example. The capture and +the checks live in `tests/_docs_sql.py` (`captured(example, dialect, url)`, +`check_block`, `check_printed`), which a new gate reuses; one run of an +example records both the SQL and the printed output of every region. To add +a block: + +1. Write the query in a snippet region of an example script; the region is + what the block shows, so keep one example per region. +2. Below the example's result sentence, write a lead-in naming the call and + saying whether only the placeholders differ on PostgreSQL ("The query + sends this; on PostgreSQL only the placeholders differ:"), then a `sql` + fence holding just the marker line. +3. Fill the fence from the test's failure message, which prints what the + region sends, then write the sentence after it: what in the SQL the reader + should notice. +4. Where PostgreSQL differs in more than placeholders (casts, `RETURNING`, + `serial`), show both as `=== "SQLite"` / `=== "PostgreSQL"` tabs, the + second marked `(postgres)`. Under `--db-backends=sqlite,postgres` the test + checks a `(postgres)` block exactly and a lone SQLite block once both + sides read each placeholder as `?`, so "only the placeholders differ" + stays true. + +Each statement is laid out one clause per line (`FROM`, `JOIN`, `WHERE`, +`GROUP BY`, `ORDER BY`, `LIMIT` and the rest, at the top level only), a +`CREATE TABLE` one column per line, and ends with `;`. Unindenting the lines +and joining them with single spaces gives back the statement Ferro sent. The +example runs under the interpreter's own hash seed, so output that varies from +run to run fails the gate. An example that a `(postgres)` block +or a lone SQLite block names must run on PostgreSQL too: `ferro.connect` is +pointed at the test database, so raw SQL in it uses `$1` placeholders, which +SQLite also accepts. + +### Quoted migration files are regenerated, never pasted + +A block whose first line is `-- 0002_author_slug/04_contract.up.sqlite.sql` +(or, for a generated data step, `# migrations/0002_author_slug/02_backfill_author.py`) +quotes a file `ferro migrate new` writes. `tests/test_docs_migrations.py` +regenerates it from the page's example models (`STEP_FILE_SOURCES` names the +example or scenario) and compares byte for byte, so a quoted step file cannot +go stale the way the hand-pasted `0001` `CREATE TABLE` and SQLite rebuild did +(they lost a column, a unique index, the `sqlite_sequence` copy and a +recreated index). A data step the page shows written by hand is listed with +`None`. Alembic revisions on the bridge page are pinned the same way, under +``. + +### Compile-only blocks say why + +An inline Python block that does not run under the harness says why in its +fence: `{test=compile reason="needs Litestar, which the test groups do not install"}`. +Generated files, `env.py` excerpts and model fragments whose imports are left +out are the usual cases; a block that can run, runs. + +Every Python fence on a page, in Markdown a page pulls in with `--8<--` +(`CONTRIBUTING.md`, `README.md` sections, `docs/includes`) and in a +docstring carries one of three markers; `tests/test_docs_examples.py` and +`tests/test_docstring_examples.py` fail on a fence with none: + +| Marker | What the harness does | +| --- | --- | +| `{test=run prelude=}` | Runs the block after `docs/examples/_preludes/.py`. | +| `{test=run prelude= backend=postgres}` | Runs it on PostgreSQL under the `postgres_only` marker, every `connect()` in the prelude and the block pointed at the test database whatever URL it names, keeping the PostgreSQL query options that URL carries (a `ferro_search_path` schema is created first); without Postgres the test is skipped and the block only compiles. A `` block under it is compared on PostgreSQL. | +| `{test=compile reason="..."}` | Compiles the block; the reason says why it cannot run. | +| `{test=pytest path="tests/.py"}` | Writes the block at that path in a scratch project and runs pytest there (`tests/test_docs_testing_fixtures.py`, for the testing how-to). | + +A block that needs Postgres runs with `backend=postgres`; `test=compile` is for a block that cannot run anywhere the suite can (a web framework the test groups do not install, a generated file). An option the test server cannot honour (`sslmode=require` needs a TLS-enabled server) makes the block compile-only, saying so. + +The redirect (`point_connect_at`) and the snippet-region reader (`snippet_regions`) live in `tests/_docs_shared.py`, stdlib-only so the prelude runner and `docs/examples/_migrations_project.py` import them too; `tests/_docs_harness.py` re-exports both. Nothing else patches `ferro.connect` or parses `--8<-- [start:]` markers. + +`lint=check` can be added to any of them. Any other setting is refused with +the marker to write instead, so `{compile-only="..."}` or a bare +`{test=run}` fails the test rather than being ignored. + +### The glossary file + +Acronyms in prose (DDL, RLS, FK, PK, GUC, FFI, IR, ...) get a hover +definition from one glossary, `docs/includes/abbreviations.md`, appended to +every page by `pymdownx.snippets.auto_append` (#701). Its wording follows +`GLOSSARY.md` where `GLOSSARY.md` defines the term. A new term goes into that +file, never into a page's own abbreviation list. + +### One Annotated idiom for I-7 twins + +AGENTS.md I-7 shows every field-declaring example as an Assignment tab and an +Annotated tab. The two tabs differ only in where the field options sit: the +Annotated tab moves the same `Field(...)` call inside `Annotated[...]` and +keeps the default as the assigned value. A field with no default has no +assignment in either tab. + +```python +name: Annotated[T, Field(...)] = default +``` + +Before (two idioms in today's twins, neither matching the Assignment tab +line for line): + +```python +id: Annotated[int | None, Field(default=None, primary_key=True)] +id: Annotated[int | None, FerroField(primary_key=True)] = None +``` + +After (the Assignment tab, then its Annotated twin): + +```python +id: int | None = Field(default=None, primary_key=True) +email: str = Field(unique=True) +nickname: str | None = Field(default=None, index=True) +``` + +```python +id: Annotated[int | None, Field(primary_key=True)] = None +email: Annotated[str, Field(unique=True)] +nickname: Annotated[str | None, Field(index=True)] = None +``` + +Pydantic accepts `Field(default=...)` inside `Annotated`, but type checkers +do not see a default there (`ty` reports `User(email="ada@example.com")` as +missing `id`); the +assigned default is the one both read, and it keeps the two tabs line for +line. Use `Field`, the name the Assignment tab imports; +`FerroField` stays out of twins. Forward foreign keys +(`Annotated[Target, ForeignKey(...)]`) and `BackRef()` / `ManyToMany()` +assignments have one form and appear the same in both tabs. + +A `default_factory` is a default too, and type checkers do not see it +inside `Annotated` either (`ty` reports `missing-argument` for +`id: Annotated[UUID, Field(primary_key=True, default_factory=uuid4)]`). It +stays the assigned value, as a `Field` of its own; only the other options +move: + +```python +id: uuid.UUID = Field(default_factory=uuid.uuid4, primary_key=True) +created_at: datetime = Field(default_factory=lambda: datetime.now(UTC)) +``` + +```python +id: Annotated[uuid.UUID, Field(primary_key=True)] = Field(default_factory=uuid.uuid4) +created_at: datetime = Field(default_factory=lambda: datetime.now(UTC)) +``` + +The twin is generated, never written by hand. `tests/_docs_twins.py` +rewrites the Assignment script into the Annotated idiom and leaves every +other line alone, so both tabs run the same data, queries and prints. After +editing `.py`, run `python -m tests._docs_twins docs/examples/.py` +to rewrite `_annotated.py`. `tests/test_docs_twins.py` fails when a +twin is not its script rewritten, or when the two declare a different schema +or print different output. + +Link the tabs with the labels `Assignment` and `Annotated`, exactly; the +`content.tabs.link` feature only syncs tabs whose labels match. + +## When to apply + +Every new or edited page under `docs/pages/`, every script under +`docs/examples/`, and every twin pair. Prose rules (lead-in and result +sentences, sentence-case headings, no minimising words) come from the +writing-for-humans skill and are checked in review; lambda naming (I-8) and +error checks that can fail are checked by `python -m tests._docs_checks`. diff --git a/docs/solutions/patterns/docstring-conventions.md b/docs/solutions/patterns/docstring-conventions.md new file mode 100644 index 00000000..d0b2d830 --- /dev/null +++ b/docs/solutions/patterns/docstring-conventions.md @@ -0,0 +1,252 @@ +--- +title: Docstring conventions for the rendered API reference (I-14) +type: pattern +tags: [docs, python, convention, invariant, mkdocstrings, griffe] +related_files: + - AGENTS.md + - src/ferro/ + - docs/examples/_preludes/reference.py + - tests/test_docstring_examples.py + - zensical.toml +related_issues: [698, 703, 704, 719, 720, 721] +related_prs: [] +captured: 2026-10-19 +--- + +## Problem + +The API reference is rendered from docstrings, and the 2026 audit found it +showing about 280 literal `:class:` roles, a `Returns:` section split into +five broken table rows, about 180 members with no docstring, a `#294` line +rendered as a heading, and 90-line essays opening `connect` and `PoolConfig`. + +## Takeaway + +A docstring is a short Google-style reference entry: a one-line summary, +every parameter in `Args:`, Markdown autorefs, references the reader can +follow, a runnable lambda-style example, and a link for anything longer than +a paragraph. AGENTS.md I-14 holds the rules; this note shows each as a +before/after pair. + +## The rules, before and after + +### Google style as griffe parses it + +griffe recognises a section by its title and a colon, with entries indented +under it. A `Returns:` written as prose lines becomes table rows, one per +line. + +Before: + +```python +"""... + +Returns: + PassReport with: + - statements: what ran + - warnings: what was skipped +""" +``` + +After: + +```python +"""... + +Returns: + The statements the pass ran and the warnings it raised, as a + [`PassReport`][ferro.PassReport]. +""" +``` + +### A one-line summary in the infinitive, ending with a period + +A function or method summary starts with a verb in the infinitive. A class +summary is a noun phrase. + +Before: + +```python +"""Returns the first matching record or None if the query has no rows""" +"""This class represents a lazily-evaluated query that ...""" +``` + +After: + +```python +"""Return the first matching record, or `None` when there is none.""" +"""A lazy query over one model's rows.""" +``` + +### Every parameter in `Args:` + +Keyword-only parameters such as `session`, `using` and `nulls` belong in the +table too, with the default when the signature does not make it obvious. + +Before: + +```python +async def get(cls, pk: Any, *, session: Session | None = None) -> Self: + """Fetch one record by primary key. + + Args: + pk: Primary key value. + """ +``` + +After: + +```python +async def get(cls, pk: Any, *, session: Session | None = None) -> Self: + """Fetch one record by primary key. + + Args: + pk: The primary key value to look up. + session: The session to run in. Defaults to the one opened by the + enclosing `async with engines.session():`. + """ +``` + +### A one-line docstring on every public dataclass or Pydantic field + +griffe reads a string literal on the line below a field as its docstring. +Without one, the field renders as a bare name. + +Before: + +```python +@dataclass(frozen=True) +class PassReport: + statements: tuple[ExecutedStatement, ...] + warnings: tuple[Report, ...] +``` + +After: + +```python +@dataclass(frozen=True) +class PassReport: + """What one reconciliation pass ran and reported.""" + + statements: tuple[ExecutedStatement, ...] + """Every statement the pass sent, in execution order.""" + warnings: tuple[Report, ...] + """Every warning the pass raised, in the order raised.""" +``` + +### Markdown autorefs, never Sphinx roles + +Sphinx roles render as literal text in mkdocstrings. + +Before: + +```python +"""Use :meth:`get_or_none` to get ``None`` instead. See :class:`~ferro.Model`.""" +``` + +After: + +```python +"""Use [`get_or_none`][ferro.Model.get_or_none] to get `None` instead. +See [Model][ferro.Model].""" +``` + +### No ADR numbers, issue numbers, codenames or Rust internals + +The reader cannot follow `ADR-0007`, `#412` or a Rust function name. +Say what the behaviour is, and link the docs page that explains why. + +Before: + +```python +"""Test membership with ``t.posts.exists()``; negate it with +``~t.posts.exists()`` (ADR-0007, #412), lowered by ``render_exists_subquery``.""" +``` + +After: + +```python +"""Test membership with `author.posts.exists()`; negate it with +`~author.posts.exists()`.""" +``` + +### Examples in the lambda style, against the reference prelude + +Examples use lambda predicates named after the model (I-8) and the shared +`reference` prelude (`Author` and `Post`, connected, inside a session). The +fence marker `{test=run prelude=reference}` makes the docs harness run it. + +Before: + +```python +"""Examples: + >>> users = await User.where(lambda u: u.age >= 18).all() +""" +``` + +After: + +````python +"""Examples: + ```python {test=run prelude=reference} + authors = await Author.where(lambda author: author.id <= 2).all() + print([author.id for author in authors]) + ``` +""" +```` + +### Explanations longer than a paragraph live in a guide + +A docstring states what the object does and how to call it. The reasoning, +trade-offs and walk-throughs go on a guide or concepts page, and the +docstring links there. + +Before: `connect()` opening with 90 lines on pools, routing and migration +modes. + +After: + +```python +"""Open a connection pool and register it as a database route. + +See [Connections](../guide/connections.md) for routing and pool sizing. + +Args: + ... +""" +``` + +### A docstring link to a guide heading is checked by the site test, not the build + +`zensical build --strict` reports a dead anchor in a docs page, but not in a +link that mkdocstrings renders from a docstring. Three docstrings linked guide +headings that had been renamed (`#composite-constraints`, +`#overriding-the-column-type-with-db_type`, +`#every-traversed-hop-is-an-inner-join`) and the strict build stayed green. +`test_links_from_api_pages_land_on_existing_anchors` in +`tests/test_docs_site.py` reads every link on the built API pages and fails on +one whose anchor does not exist. Before renaming a guide heading, grep +`src/ferro` as well as `docs/pages` for its anchor. + +### Every export has one entry, and nothing renders empty + +`test_every_export_has_an_api_entry` checks that each name in `ferro.__all__`, +`ferro.query.__all__`, `ferro.migrations.__all__` and +`ferro.migrations.testing.__all__` has exactly one entry on the API pages, +under its public path (`::: ferro.query.Query`, never +`::: ferro.query.builder.Query`). A type a public call returns but no +`__all__` exports (`RunReport`, `ModelConnection`) is listed in +`RETURNED_TYPES` there and documented under its defining module. +`show_if_no_docstring` is off, so a public member without a docstring would +vanish from the reference without a warning; +`test_every_public_object_and_member_has_a_docstring` reads the package with +griffe and fails naming it. + +## How to recognize + +- Literal `:class:` or `:meth:` text on a rendered API page. +- A `Returns:` or `Attributes:` table whose rows are sentence fragments. +- A field listed with no description, or a parameter in the signature missing + from the table. +- A summary that wraps, starts with "Returns" or "This", or has no period. +- `lambda t:` or a doctest `>>>` in an `Examples:` section. diff --git a/pyproject.toml b/pyproject.toml index 90eb8c95..b08824da 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -61,6 +61,7 @@ docs = [ "zensical>=0.0.71", "mkdocstrings-python>=2.0.9", "pymdown-extensions>=12.3", + "griffe-inherited-docstrings>=1.1.4", ] release = [ "python-semantic-release>=9.0.0", @@ -89,6 +90,7 @@ dev = [ "greenlet>=3.3.1", "ty>=0.0.56", "cyclopts>=5,<6", + "griffe-inherited-docstrings>=1.1.4", ] [tool.pytest.ini_options] @@ -142,3 +144,9 @@ keep_trailing_newline = true allowed_tags = ["build", "chore", "ci", "docs", "feat", "fix", "perf", "style", "refactor", "test"] minor_tags = ["feat"] patch_tags = ["fix", "perf", "refactor"] + +[tool.ruff.lint.per-file-ignores] +# Query lambdas compare with `== None` and `== True` on purpose: the lambda +# builds SQL, and Python lets a column overload `==` but not `is`. The docs +# embed these files, so the ignore lives here, not in a `# noqa` on the page. +"docs/examples/*" = ["E711", "E712"] diff --git a/src/ferro/__init__.py b/src/ferro/__init__.py index 21c1b644..1dd5a47e 100644 --- a/src/ferro/__init__.py +++ b/src/ferro/__init__.py @@ -1,9 +1,4 @@ -""" -Ferro: A High-Performance Rust-Backed Python ORM. - -Ferro combines the speed of a Rust engine with the ergonomics of Pydantic models -to provide a seamless, high-performance database experience. -""" +"""Ferro, an async Python ORM on Pydantic models with a Rust core.""" import logging import threading @@ -19,9 +14,9 @@ clear_registry as _core_clear_registry, create_tables as _core_create_tables, migrate as _core_migrate, - reset_engine, - set_default_connection, - version, + reset_engine as _core_reset_engine, + set_default_connection as _core_set_default_connection, + version as _core_version, ) from ._core import ( _install_registration, @@ -95,18 +90,20 @@ async def _ensure_rust_registration_synced_for_operation() -> None: def ensure_resolved_modelset() -> dict[str, Any]: - """Ensure the SchemaIR modelset reflects the current registry. + """Return the schema of every registered model, relationships resolved. - Clean path (O(1)): one in-process generation comparison, then assemble - already-compiled envelopes without recompiling. Dirty path: run - ``resolve_relationships()`` (which clears the generation counter on success), - then assemble (#245). + It is the schema [`connect`][ferro.connect], + [`create_tables`][ferro.create_tables] and `ferro migrate new` read, + as a JSON-ready dict. Relationships between models are resolved first, + so a model declared since the last call is included. Declaring models + in another thread while it runs is unsupported and can raise. - Defining model classes concurrently with ``connect``/``create_tables``/``migrate`` - is unsupported: a registry entry can become visible before its envelope is - persisted, producing a transient missing-envelope error on the unlocked fast - path. + Returns: + The schema as a dict, ready for `json.dumps`. """ + # Clean registry: assemble the already-compiled envelopes. Dirty: + # resolve relationships (which clears the generation counter), then + # assemble, under the lock so two threads resolve once. from .ir.compiler import compile_registry_schema_ir from .registry import REGISTRY from .relations import resolve_relationships @@ -132,34 +129,106 @@ def _ensure_rust_registration_synced() -> None: def clear_registry() -> None: - """Reset the compiled/registered schema state. - - Delegates to the Rust core (which clears the Rust model registry and the - pushed SchemaIR modelset) and additionally clears the Python **join-table - registry**. That registry must be reset here because - ``connect``/``create_tables``/``migrate`` compile the *full* registry via - ``compile_registry_schema_ir()``: a join table left behind by a prior run - would be re-created with foreign keys to tables that no longer exist — - tolerated by SQLite but rejected by Postgres (``relation ... does not - exist``). (#153) - - The Python model registry is intentionally **not** cleared here: clearing - the Rust registry while keeping the declared Python models is what allows - cold re-hydration after ``reset_engine`` (see - ``tests/test_enum_cold_hydration.py``). Callers that want a full - Python-side reset use ``REGISTRY.reset_for_test()``. - - Each purged join table's compiled SchemaIR envelope is evicted with it — - ``Registry.clear_join_tables`` owns that agreement (#153): a lingering - join envelope would let a future assemble step resurrect the stale join - table. + """Drop the schema the engine holds for the registered models. + + Call it in a test fixture, after [`reset_engine`][ferro.reset_engine], + when the next test declares its own models: the next `connect` then + builds the schema only from what is registered at that point, and + leaves out the many-to-many join tables of earlier tests. Model classes + you have declared stay declared and keep working after the next + `connect`. + + Examples: + ```python {test=run prelude=connections} + from ferro import clear_registry, connect, reset_engine + + await connect("sqlite::memory:", auto_migrate=True) + reset_engine() + clear_registry() + await connect("sqlite::memory:", auto_migrate=True) + ``` """ + # The Rust core forgets its model registry and pushed modelset; the + # Python join-table registry (and each join table's compiled envelope) + # is cleared here, since a join table left behind would be recreated + # with foreign keys to tables that may no longer exist. The Python model + # registry is kept on purpose, so models re-hydrate after reset_engine(); + # tests that need a full reset use REGISTRY.reset_for_test(). _core_clear_registry() from .registry import REGISTRY REGISTRY.clear_join_tables() +def version() -> str: + """Return the version of the installed Ferro. + + Returns: + The version string, such as `"0.25.0"`. + + Examples: + ```python {test=run prelude=reference} + from importlib.metadata import version as installed + + from ferro import version + + assert version() == installed("ferro-orm") + ``` + """ + return _core_version() + + +def reset_engine() -> None: + """Close every connection and drop every open session and transaction. + + Call it between tests to start each one from a clean engine. The next + [`connect`][ferro.connect] without a name registers as the default + again. Declared models stay registered; see + [`clear_registry`][ferro.clear_registry] to drop their schema as well. + + Raises: + RuntimeError: The engine's state could not be locked. + + Examples: + ```python {test=run prelude=connections} + from ferro import connect, reset_engine + + await connect("sqlite::memory:", auto_migrate=True) + reset_engine() + await connect("sqlite::memory:", auto_migrate=True) + ``` + """ + _core_reset_engine() + + +def set_default_connection(name: str) -> None: + """Set the connection `engines.session()` opens when called without a name. + + See [The default connection](../guide/connections.md#the-default-connection) + for the other two ways to set it. + + Args: + name: A connection registered with `connect(url, name=...)`. + + Raises: + ValueError: No connection is registered under `name`, or `name` holds + a character other than an ASCII letter, digit or underscore. + RuntimeError: The engine's state could not be locked. + + Examples: + ```python {test=run prelude=connections} + from ferro import connect, engines, set_default_connection + + await connect("sqlite::memory:", name="app", default=True) + await connect("sqlite::memory:", name="archive") + set_default_connection("archive") + async with engines.session() as session: + assert session.connection_name == "archive" + ``` + """ + _core_set_default_connection(name) + + def _push_registration_to_rust( envelope: dict[str, Any] | None = None, *, @@ -196,106 +265,57 @@ def _push_registration_to_rust( class PoolConfig(BaseModel): - """Connection pool settings for a named Ferro connection. - - ``settings_delivery`` chooses how sessions on this connection get their - session settings (the Postgres GUCs row-level-security policies read) onto - the database. The two modes send different SQL for the very same session: - - async with engines.session(settings={"myapp.tenant_id": "acme"}): - invoices = await Invoice.where(lambda invoice: invoice.paid).all() - - ``"transaction"`` (the default) wraps the query in a transaction of its own - and scopes the value to it:: - - BEGIN - SELECT set_config($1, $2, true) -- 'myapp.tenant_id', 'acme' - SELECT "invoice".* FROM "invoice" WHERE "paid" - COMMIT - - ``"connection"`` sets the value once, on a connection it keeps for the - session, and then sends your statements bare:: - - SELECT set_config($1, $2, false), set_config($3, $4, false) - -- 'myapp.tenant_id', 'acme', 'ferro.pinned_keys', 'myapp.tenant_id' - SELECT "invoice".* FROM "invoice" WHERE "paid" - ... -- every later query, no wrap - SELECT set_config($1, NULL, false), set_config($2, NULL, false) - -- at session close: exactly the keys above, reset - - The difference that matters is ``true`` vs ``false`` — Postgres' - ``is_local`` flag. ``true`` makes the value die with the transaction, which - is why the default is safe even behind a transaction pooler like PgBouncer, - where consecutive statements can land on different backends. ``false`` - makes it live for the whole database session, which is only safe if that - session belongs to one Ferro session and nobody else — hence the pinned - connection. - - So ``settings_delivery="connection"`` is a promise about your deployment: - **this pool talks to Postgres directly.** Ferro never guesses it. A pooler - is invisible to its clients, and guessing wrong means one tenant's value - answering another tenant's query. - - What you get for the promise: no per-operation wrap at all, so a - settings-bearing session costs one extra round trip for its whole life - instead of about two per operation outside ``transaction()``. - - What it costs, and there are four costs worth knowing before you opt in. - - **One connection per scoped session.** A settings-bearing session holds a - pool connection from its first operation until it closes, so **no more - settings-bearing sessions can run at once than the pool has - connections**. Number ``max_connections`` + 1's first query waits for a - connection, exactly like any other pool checkout. Size the pool for peak - concurrent scoped sessions. - - **One session, one connection, therefore one thing at a time.** Everything - a pinned session does is serialized: two sibling tasks sharing the session - run one after the other, and while a ``transaction()`` block is open a - sibling task's operation waits for it rather than running inside it. - That is the honest meaning of a session that owns a single connection — - the same fact as the cap above, seen from inside one session. Operations - in the same task *inside* a ``transaction()`` block are unaffected; they - already own the block. - - **A marker check on release while anyone is pinned.** The pool verifies - that a connection coming back is not still carrying a session's settings. - While no session is pinned this costs nothing at all. While *any* session - on the pool is pinned, every connection release on that pool performs one - marker check — including releases by settings-less sessions and by - sessionless operations. - - **Schema changes wait for scoped sessions.** ``connect(migrate_updates=…)`` - and ``migrate()`` refresh the pool afterwards, and a refresh cannot finish - until every connection comes back — including the pinned ones. Migrating - while long-lived tenant sessions are open therefore blocks until they - close; Ferro warns, naming how many it is waiting on. Run schema changes - before opening tenant-scoped sessions. - - Sessions *without* settings never pin: same statements, same connections, - no wrap, no serialization. - - Two smaller behaviours worth knowing: - - * A session that opens with no ``settings`` and gains them later via - ``set_config`` has nothing pinned yet, so it pins on its **next - operation** and applies the values then. - * Closing a session resets its keys to the value the connection *started* - with. For an ordinary custom setting that is the empty string, which is - what fail-closed policies rely on. If an operator has set a default with - ``ALTER ROLE ... SET myapp.tenant_id = ...`` or ``ALTER DATABASE ... SET - ...``, resetting brings that value *back* rather than clearing it — so - never configure a tenancy key as a role or database default. + """Pool size and settings delivery for one connection. + + Pass it to [`connect`][ferro.connect] as `pool=`. `settings_delivery` + chooses how a session sends its + [session settings](../concepts/row-level-security.md#session-settings) + to Postgres; read + [Connection delivery](../concepts/row-level-security.md#connection-delivery) + for what `"connection"` saves and what it costs before you opt in. + + Raises: + pydantic.ValidationError: A size is out of range, or + `min_connections` exceeds `max_connections`. + + Examples: + ```python {test=run prelude=reference} + from ferro import PoolConfig, connect + + pool = PoolConfig(max_connections=10, min_connections=1) + await connect("sqlite::memory:", name="pooled", pool=pool) + assert pool.settings_delivery == "transaction" + ``` """ model_config = ConfigDict(frozen=True) max_connections: int = PydanticField(default=5, ge=1) + """The most connections the pool opens at once. Defaults to 5; at least 1.""" min_connections: int = PydanticField(default=0, ge=0) + """The connections the pool keeps open while idle. Defaults to 0.""" settings_delivery: Literal["transaction", "connection"] = "transaction" + """How sessions send their settings to Postgres. Defaults to `"transaction"`. + + `"transaction"` sets them inside each transaction, which is safe behind a + transaction-mode pooler. `"connection"` sets them once on a connection the + session holds until it closes; use it only when the pool reaches Postgres + directly. + """ @model_validator(mode="after") def validate(self) -> "PoolConfig": + """Check that `min_connections` does not exceed `max_connections`. + + Pydantic runs it on construction, so a `PoolConfig` you hold has + already passed it. + + Returns: + This configuration, unchanged. + + Raises: + ValueError: `min_connections` exceeds `max_connections`. + """ if self.min_connections > self.max_connections: raise ValueError("min_connections cannot exceed max_connections") return self @@ -312,103 +332,68 @@ async def connect( migrate_updates: bool = False, migrate_destructive: bool = False, ) -> None: - """ - Establish a connection to the database. + """Open a connection pool and register it under a name. + + Call it once at startup. ORM calls then run inside a session opened on + the connection; see [Sessions](../guide/connections.md#sessions). + + The three migrate flags make the database match the registered models + as the connection opens, each one going further than the last. They run + under the run lock `ferro migrate up` takes and read the project's + `lock_timeout` and `ddl_lock_timeout` settings. + [Auto-migrate](../guide/schema/auto-migrate.md) lists what each flag + changes on each backend and how the lock waits. A change the pass + refuses, such as a primary-key change, is a migration's job: see + [Changing a primary key](../guide/schema/migrations.md#changing-a-primary-key). Args: - url: The database connection string (e.g., "sqlite:example.db?mode=rwc"). - auto_migrate: If True, automatically create tables for all registered models. - Existing tables are left completely untouched — whatever their shape — - unless ``migrate_updates`` / ``migrate_destructive`` are also set. - name: Optional connection name. Omitted connections register as "default". - default: If True, make this named connection the default for unqualified operations. - pool: Optional per-connection pool configuration, including - ``settings_delivery`` — how sessions on this connection deliver - their session settings (see ``PoolConfig``). - identity_map: If True (default), sessions opened on this connection keep an identity - map so the same primary key maps to a single Python instance within a session. - Identity maps are session-scoped: operations outside a session never cache or - dedup instances. If False, loads on this connection return fresh instances even - inside a session (lower memory use; no ``a is b`` guarantees across loads). - migrate_updates: If True, additionally update existing tables to match the - registered models. Implies ``auto_migrate``. What this covers is - capability-relative per backend: - - - **Both backends**: ``ALTER TABLE ... ADD COLUMN`` for model fields - missing from the live table, using the same column DDL ``CREATE TABLE`` - would emit (including single-column indexes and, on Postgres, CHECK - constraints and foreign keys). NOT NULL fields need a literal default - to backfill existing rows — a json-family object/array (`Field(default={})`, - `default_factory=dict`) counts; connecting fails with a clear error - otherwise. - - **Postgres only**: column type changes - (``ALTER COLUMN ... TYPE ... USING`` cast) and nullability changes - (``SET/DROP NOT NULL``) when the live column disagrees with the model. - Also foreign-key reconciliation: a ferro-owned (``fk_``-named) - constraint whose definition drifts from the declared FK - (``on_delete``, target) is rebuilt (``DROP CONSTRAINT`` + - ``ADD CONSTRAINT``), and a declared FK missing entirely from an - existing column is added. A drifting constraint ferro does not - own is warned about, never altered. - - **SQLite**: type/nullability drift cannot be changed in place; ferro - emits a ``UserWarning`` naming the column and pointing at Alembic. - (SQLite's type affinity makes declared-type drift mostly cosmetic.) - FK constraints likewise cannot be added or altered on an existing - table; any foreign-key drift warns loudly instead of diverging - silently. - - **Transactionality**: on Postgres each table's migration plan - runs inside a single transaction — a mid-plan failure rolls the - table back to exactly its pre-migration state. On SQLite, - statements apply one at a time; a mid-run failure can leave - earlier statements of that table applied (SQLite ALTERs are - single-statement operations). For transactional multi-step - SQLite migrations, use the Alembic bridge. - - After any schema change, the connection pool is refreshed so no cached - statement can observe the pre-migration schema. - migrate_destructive: If True, additionally **drop** live columns that no - longer exist on the model (never whole tables). Implies - ``migrate_updates``. Dropping is dependency-aware: explicit indexes - covering the column are dropped first; columns that are primary keys or - enforced by table constraints fail with a clear error instead. - - Auto-migrate and ferro migrations are exclusive per database: with any of - the three flags, a database whose schema ferro migrations govern (it - carries a ``_ferro_migrations`` tracking table) is refused before any - DDL, and the passes run under the same run lock ``ferro migrate up`` - takes, so two processes booting together never collide. Only then is the - project's ``FerroSettings()`` read (for its ``tracking_schema``s, its - ``ddl_lock_timeout`` and its ``lock_timeout``); a plain ``connect(url)`` - reads no config and runs no extra query. - - A database ferro migrations govern is refused at once, before the pass - waits for the run lock. Otherwise a pass that finds the lock held warns - that it waits, and waits up to ``lock_timeout`` (default ``30s``; ``"0"`` - waits without a limit) for the other run or pass to finish, then refuses - with ``MigrationRefused`` naming ``lock_timeout`` and ``ferro migrate - status``. When several databases are configured they must all set the - same ``lock_timeout`` and the same ``ddl_lock_timeout``. - - On Postgres every reconciliation statement waits for a table lock under - ``ddl_lock_timeout`` (default ``5s``; ``"0"`` waits without a limit): a - table's plan whose statement times out is rolled back and retried, up to - ten attempts, each one a ``UserWarning`` naming the attempt, and then - fails with ``OperationalError`` naming ``ddl_lock_timeout``. - - ``connect()`` returns nothing; the pass's ``PassReport`` (every statement - it ran, every warning) is what ``ferro.migrate()`` returns. A pass that - fails raises with ``.report`` set to what committed before the failure. + url: The database URL, such as `"sqlite:app.db?mode=rwc"`, + `"sqlite::memory:"` or `"postgresql://user:password@localhost/app"`. + auto_migrate: Create the missing tables of every registered model. + Existing tables are left as they are. Defaults to `False`. + name: The connection's name, made of ASCII letters, digits and + underscores. Defaults to `None`, which registers it as + `"default"`. + default: Make this named connection the default, the one + `engines.session()` opens when given no name. Defaults to `False`. + pool: The pool's size and how sessions send their settings. Defaults + to `None`, which means `PoolConfig()`. + identity_map: Keep an identity map in each session on this + connection, so one primary key loads as one instance within the + session. Defaults to `True`. + migrate_updates: Also change existing tables to match the models: + add columns and indexes, and on Postgres change types, + nullability and foreign keys. Implies `auto_migrate`. Defaults to + `False`. + migrate_destructive: Also drop columns the models no longer declare, + never whole tables. Implies `migrate_updates`. Defaults to + `False`. Raises: - ValueError: A connection with this name (or a default connection, - when ``name`` is omitted) is already registered. Use ``name=...`` - for additional connections or ``reset_engine()`` to tear down. - ferro.migrations.MigrationRefused: an auto-migrate flag on a database - governed by ferro migrations (use ``ferro migrate up``), or behind - a transaction-mode pooler on Postgres. - - For schema changes beyond these (renames, primary-key changes, complex - transforms), use the Alembic bridge — see ``docs/guide/migrations.md``. + ValueError: A connection with this name is already registered (or a + default one, when `name` is omitted), or `name` holds a + character other than an ASCII letter, digit or underscore. Pass + another `name`, or call [`reset_engine`][ferro.reset_engine] + first. + ferro.migrations.MigrationRefused: A migrate flag was given for a + database that migrations govern (run `ferro migrate up` instead), + the run lock stayed held longer than `lock_timeout`, or the + Postgres URL goes through a transaction-mode pooler. + OperationalError: On Postgres, a statement of the pass waited longer + than `ddl_lock_timeout` for its table lock on ten attempts. Its + `.report` lists what committed before the failure. + SettingsError: The configured databases set different `lock_timeout` + or `ddl_lock_timeout` values. + + Examples: + ```python {test=run prelude=connections} + from ferro import connect, engines + + await connect("sqlite::memory:", auto_migrate=True) + async with engines.session(): + await Author.create(name="Ada Lovelace", email="ada@example.com") + assert len(await Author.all()) == 1 + ``` """ _ensure_rust_registration_synced() @@ -497,26 +482,42 @@ def _agreed_seconds( async def create_tables(using: str | None = None) -> PassReport: - """ - Manually create the *missing* tables for registered models on a connected - engine. A table that already exists is left completely untouched; altering - existing tables belongs to ``migrate(updates=True)``. - - Compiles and pushes the current registry SchemaIR to the Rust runtime - before delegating to the Rust create entrypoint, so a model defined after - ``connect()`` (and thus absent from the connect-time snapshot) is still - created. The runtime emits each ``CREATE TABLE`` from this SchemaIR via the - shared emitter. + """Create the missing tables of every registered model. - Like ``connect()``'s auto-migrate flags, it runs under the run lock and - refuses a database governed by ferro migrations. + A table that already exists is left as it is; changing one is + [`migrate`][ferro.migrate]'s job. A model defined after + [`connect`][ferro.connect] is created too. Like `connect`'s migrate + flags, it runs under the run lock and refuses a database that + migrations govern. Args: - using: Named connection to create tables on, or None for the default. + using: The connection to create the tables on. Defaults to `None`, + the default connection. Returns: - The ``PassReport``: every statement the pass ran and every warning it - raised. A failed pass's error carries it as ``.report``. + Every statement the pass ran and every warning it raised, as a + [`PassReport`][ferro.PassReport]. + + Raises: + ferro.migrations.MigrationRefused: Migrations govern the database, + or the run lock stayed held longer than `lock_timeout`. + + Examples: + ```python {test=run prelude=reference} + from ferro import Field, Model, create_tables + + + class Note(Model): + id: int | None = Field(default=None, primary_key=True) + body: str + + + report = await create_tables() + assert [statement.sql for statement in report.statements] == [ + 'CREATE TABLE IF NOT EXISTS "note" ( "body" varchar NOT NULL, ' + '"id" integer NOT NULL PRIMARY KEY AUTOINCREMENT )' + ] + ``` """ _ensure_rust_registration_synced() return await _run_pass(_core_create_tables(using=using, **_auto_migrate_settings())) @@ -525,27 +526,52 @@ async def create_tables(using: str | None = None) -> PassReport: async def migrate( using: str | None = None, updates: bool = True, destructive: bool = False ) -> PassReport: - """ - Manually run the auto-migrate pass against a connected engine. + """Run the auto-migrate pass on an open connection. - Compiles and pushes the current registry SchemaIR to the Rust runtime, - then delegates to the Rust migrate entrypoint. + It runs the same pass as `connect(migrate_updates=True)`: it creates the + missing tables, then changes existing ones to match the models. See + [On-demand `migrate()`](../guide/schema/auto-migrate.md#on-demand-migrate) + for what it changes on each backend. Like `connect`'s migrate flags, it + runs under the run lock and refuses a database that migrations govern. Args: - using: Named connection to migrate, or None for the default. - updates: If True (default), add missing columns and reconcile - type, nullability, and foreign-key definition drift (see ``connect``). - destructive: If True, also drop live columns absent from the model. Implies ``updates``. - - Like ``connect()``'s auto-migrate flags, it runs under the run lock and - refuses a database governed by ferro migrations. + using: The connection to migrate. Defaults to `None`, the default + connection. + updates: Change existing tables as well as creating missing ones. + Defaults to `True`; `False` runs only the create pass, as + [`create_tables`][ferro.create_tables] does. + destructive: Also drop columns the models no longer declare. Implies + `updates`. Defaults to `False`. Returns: - The ``PassReport``: every statement the pass sent to the database, in - order (``ExecutedStatement(subject, sql, role)``), and every warning - it raised (also raised as a ``UserWarning``). A pass that fails - partway raises its usual error with ``.report`` set to what committed - before the failure. + Every statement the pass sent, in order, and every warning it raised, + as a [`PassReport`][ferro.PassReport]. Each warning is also raised + as a `UserWarning`. + + Raises: + ferro.migrations.MigrationRefused: Migrations govern the database, + or the run lock stayed held longer than `lock_timeout`. + OperationalError: A statement failed partway through the pass. Its + `.report` lists what committed before the failure. + + Examples: + ```python {test=run prelude=reference} + from ferro import Field, Model, execute, migrate + + await execute('CREATE TABLE "tag" ("id" integer PRIMARY KEY, "label" varchar NOT NULL)') + + + class Tag(Model): + id: int | None = Field(default=None, primary_key=True) + label: str + color: str | None = None + + + report = await migrate() + assert [statement.sql for statement in report.statements] == [ + 'ALTER TABLE "tag" ADD COLUMN "color" varchar' + ] + ``` """ _ensure_rust_registration_synced() return await _run_pass( diff --git a/src/ferro/_bind_payload.py b/src/ferro/_bind_payload.py index 95b49e98..d97d441c 100644 --- a/src/ferro/_bind_payload.py +++ b/src/ferro/_bind_payload.py @@ -148,16 +148,28 @@ def canonicalize_wire_scalar(value: Any) -> Any: def save_bind_payload(instance: Any) -> dict[str, Any]: - """Column->value map for ``save``/``bulk_create``. + """Column->value map for ``save``/``bulk_create``, in field order. Columns go through pydantic ``model_dump(mode="json")`` (honoring field - serializers/aliases), except bytes and secret columns, which are overlaid + serializers/aliases), except bytes and secret columns, which are bound raw: bytes verbatim, a secret as ``unwrap_secret`` gives it. + + The map is in field declaration order, so a statement lists its columns + the same way on every call: the dump follows the instance's ``__dict__``, + which a fetch fills in the ``SELECT``'s column order, and the raw columns + are bound apart from it. """ raw = _raw_bound_fields(instance) payload: dict[str, Any] = instance.model_dump(mode="json", exclude=set(raw)) - payload.update(raw) - return payload + ordered: dict[str, Any] = {} + for name in type(instance).model_fields: + if name in raw: + ordered[name] = raw[name] + elif name in payload: + ordered[name] = payload[name] + for name, value in payload.items(): + ordered.setdefault(name, value) + return ordered def normalize_save_columns(names: object, *, param: str) -> set[str]: @@ -255,7 +267,8 @@ def _keep_write_set( f"save({param}=...) bind payload is missing " f"{sorted(missing)!r}. Legal persisted columns: {legal}." ) - return {key: payload[key] for key in keep} + # The payload is in field order; a set's order changes with the hash seed. + return {key: value for key, value in payload.items() if key in keep} def apply_save_only( diff --git a/src/ferro/base.py b/src/ferro/base.py index aa707278..0a9ce4d2 100644 --- a/src/ferro/base.py +++ b/src/ferro/base.py @@ -1,6 +1,7 @@ -"""Define field and relationship metadata primitives for Ferro models. +"""Field and relation options that model annotations carry. -This module provides lightweight configuration objects used by model annotations to describe column constraints and inter-model relationships. +[`FerroField`][ferro.FerroField] holds a column's database options and +[`ForeignKey`][ferro.ForeignKey] the owning side of a relation. """ import warnings @@ -13,9 +14,15 @@ from ._annotation_utils import annotation_allows_none T = TypeVar("T") +"""A type variable for generic helpers.""" FerroNullable = Literal["infer"] | bool -"""Alembic column nullability: ``'infer'`` from the field type, or forced bool.""" +"""Column nullability: `"infer"` follows the annotation, `True` or `False` forces it. + +With `"infer"`, a column is nullable when its annotation allows `None`. +`True` and `False` force `NULL` and `NOT NULL` in every schema Ferro emits, +whatever the annotation says. +""" DbTypeToken = Literal[ "text", @@ -30,22 +37,41 @@ "json", "jsonb", ] -"""Fixed canonical SQL storage tokens accepted by :func:`~ferro.Field` ``db_type=``.""" +"""SQL storage type names accepted by `db_type=` on [`Field`][ferro.Field]. + +See [Column type overrides with `db_type`](../guide/models-and-fields.md#column-type-overrides-with-db_type) +for the types each name maps to on SQLite and Postgres. +""" DbType = DbTypeToken | str -"""Canonical ``db_type`` value: a :data:`DbTypeToken` or ``varchar(N)`` (see :func:`varchar`).""" +"""A `db_type=` value: a [`DbTypeToken`][ferro.DbTypeToken] or a `varchar(N)` string. + +Build the `varchar(N)` form with [`varchar`][ferro.varchar]. +""" def varchar(length: int) -> str: - """Build the canonical ``varchar(N)`` token for :func:`~ferro.Field` ``db_type=``. + """Build the `varchar(N)` type name for `db_type=`. + + A call to `varchar` reads as a deliberate type choice, where a raw + `"varchar(80)"` string could be a typo. Ferro still checks the value + against the field's annotation when the model class is defined. + + Args: + length: The maximum length in characters. Must be positive. - Prefer this over a raw string so IDEs and type checkers see a deliberate - Ferro vocabulary choice. Runtime validation still applies at model - definition time. + Returns: + The type name, `"varchar()"`. + + Raises: + ValueError: `length` is zero or negative. Examples: - >>> varchar(255) - 'varchar(255)' + ```python {test=run prelude=reference} + from ferro import varchar + + assert varchar(80) == "varchar(80)" + ``` """ if length <= 0: raise ValueError(f"varchar length must be positive, got {length}") @@ -76,42 +102,30 @@ def _validate_retired(retired: bool, owner: str) -> bool: class FerroField: - """Store database column metadata for a model field - - Attributes: - - primary_key: Mark the field as the table primary key. - autoincrement: Override automatic increment behavior for primary key columns. - unique: Enforce a **single-column** uniqueness constraint for this column only. - For uniqueness on multiple columns together, declare - ``__ferro_composite_uniques__`` on the :class:`ferro.models.Model` subclass - (see the models guide). - index: Request an index for the column. - nullable: Alembic ``Column.nullable`` when using :func:`~ferro.migrations.get_metadata`. - ``'infer'`` (default) uses whether the annotation allows ``None``. - ``False`` / ``True`` force NOT NULL / NULL regardless of the type (for - advanced cases such as ``int | None`` used only for static typing). - db_type: Optional canonical SQL-type override (:data:`DbType`). Use a - :data:`DbTypeToken` literal or :func:`varchar` for ``varchar(N)``. - Strict validation against the Python annotation runs at class-definition - time. Default ``None`` preserves today's behavior. - db_check: When ``True``, emit a DB-side ``CHECK`` constraint enforcing the - allowed values for closed-domain types (``enum.Enum`` subclasses). - Only valid in combination with ``db_type``; combining with default - native-enum storage is rejected as redundant. Default ``False``. - retired: When ``True``, the column is retired (ADR-0053): it leaves the - model (no attribute, no keyword, no statement the model issues names - it) and stays in the schema snapshot, nullable, until the - declaration is deleted in a later release. The annotation must - allow ``None``. Default ``False``. + """Database options for one model field, given inside `Annotated[...]`. + + [`Field`][ferro.Field] takes the same options and is the usual spelling. + `FerroField` carries only the database options, for an `Annotated` + declaration that keeps them apart from Pydantic's own `Field`. See + [Field options](../guide/models-and-fields.md#field-options) for each + option at work. Examples: - >>> from typing import Annotated - >>> from ferro.models import Model - >>> - >>> class User(Model): - ... id: Annotated[int, FerroField(primary_key=True)] - ... email: Annotated[str, FerroField(unique=True, index=True)] + ```python {test=run prelude=reference} + from typing import Annotated + + from ferro import FerroField, Model, create_tables + + + class Tag(Model): + id: Annotated[int | None, FerroField(primary_key=True)] = None + label: Annotated[str, FerroField(unique=True, index=True)] + + + await create_tables() + tag = await Tag.create(label="history") + assert tag.id == 1 + ``` """ def __init__( @@ -126,69 +140,90 @@ def __init__( renamed_from: str | None = None, retired: bool = False, ): - """Initialize field metadata options + """Declare the database options for a field. Args: - - primary_key: Set to True when the field is the primary key. - autoincrement: Control whether the database auto-increments the value. - When not provided, the backend infers a default for integer primary keys. - unique: Set to True to enforce **single-column** uniqueness only. - index: Set to True to create a database index. - nullable: See :class:`FerroField` attribute ``nullable`` in the class docstring. - renamed_from: The column's previous name. ``ferro migrate new`` - renders ``RENAME COLUMN`` while the previous migration still - holds that name; afterwards the hint is inert and may stay or - be deleted (ADR-0032). - retired: Retire the column (ADR-0053); see the class docstring. - - Examples: - >>> from typing import Annotated - >>> from ferro.models import Model - >>> - >>> class User(Model): - ... id: Annotated[int, FerroField(primary_key=True)] - ... created_at: Annotated[int, FerroField(index=True)] + primary_key: Make the column the table's primary key. + autoincrement: Whether the database generates the key. Defaults to + `None`, which lets an integer primary key auto-increment. + unique: Add a unique constraint on this column alone. For + uniqueness across several columns, declare + `__ferro_composite_uniques__` on the model (see + [Composite uniques](../guide/models-and-fields.md#composite-uniques)). + index: Add an index on the column. + nullable: `"infer"` (the default) makes the column nullable when + the annotation allows `None`; `True` or `False` forces `NULL` + or `NOT NULL` whatever the annotation says. + db_type: The column's SQL type, overriding the one Ferro derives + from the annotation: a [`DbTypeToken`][ferro.DbTypeToken] or + a [`varchar(N)`][ferro.varchar] name. Defaults to `None`, the + derived type. A type the annotation cannot be stored as is + refused when the class is defined. + db_check: Add a `CHECK` constraint that limits the column to an + enum's values. Needs `db_type`, since the default native enum + type already enforces them. Defaults to `False`. + renamed_from: The column's previous name, so that + `ferro migrate new` renames the column instead of dropping + and adding it. See [Rename hints](../guide/schema/renames-and-retirements.md#rename-hints). + retired: Take the column out of the model while it stays in the + schema, nullable, until a later release deletes the + declaration. The annotation must allow `None`. Defaults to + `False`. See + [Retiring a column](../guide/schema/renames-and-retirements.md#retiring-a-column). + + Raises: + TypeError: `nullable`, `renamed_from` or `retired` has a value of + the wrong type. """ self.primary_key = primary_key + """Whether the column is the table's primary key.""" self.autoincrement = autoincrement + """Whether the database generates the key; `None` lets Ferro decide.""" self.unique = unique + """Whether the column has a single-column unique constraint.""" self.index = index + """Whether the column has an index.""" self.nullable = _validate_nullable_option(nullable, "FerroField") + """`"infer"`, or a forced `True` or `False`.""" self.db_type = db_type + """The SQL type override, or `None` for the derived type.""" self.db_check = db_check + """Whether the column has a `CHECK` constraint on an enum's values.""" self.renamed_from = _validate_renamed_from(renamed_from, "FerroField") + """The column's previous name, or `None`.""" self.retired = _validate_retired(retired, "FerroField") + """Whether the column is retired from the model.""" class ForeignKey: - """Describe a forward foreign-key relationship between models - - Attributes: - - to: Target model class resolved during model binding. - related_name: Name of the reverse relationship attribute on the target model. - on_delete: Referential action applied when the parent row is deleted. - unique: Treat the relation as one-to-one when True. - index: Request a non-unique index on the shadow ``*_id`` column. - nullable: Alembic nullability for the shadow ``*_id`` column (see - :class:`FerroField` ``nullable``). When ``'infer'``, uses whether the - **relation** annotation allows ``None``. - retired: Retire the relation and its shadow ``*_id`` column - (ADR-0053): both leave the model, and the column stays in the schema - snapshot, nullable, with its foreign key, until the declaration is - deleted in a later release. + """The owning side of a relation: a foreign key to another model. + + Declare it as `Annotated[Target, ForeignKey(...)]`. The table gets a + `_id` column holding the target's primary key, and the target + model reads the relation back through the [`BackRef`][ferro.BackRef] + that `related_name` names. The models these examples run against declare + the pair like this, other fields left out: + + ```python {test=compile reason="the reference prelude already declares these models"} + class Author(Model): + ... + posts: Relation[list["Post"]] = BackRef() + + + class Post(Model): + ... + author: Annotated[Author, ForeignKey(related_name="posts")] + ``` + + See [Relationships](../guide/relationships.md) for one-to-one, nullable + and self-referential relations. Examples: - >>> from typing import Annotated - >>> from ferro.models import Model - >>> - >>> class User(Model): - ... id: Annotated[int, FerroField(primary_key=True)] - >>> - >>> class Post(Model): - ... id: Annotated[int, FerroField(primary_key=True)] - ... author: Annotated[int, ForeignKey("posts", on_delete="CASCADE")] + ```python {test=run prelude=reference} + post = await Post.where(lambda post: post.views > 1000).first() + author = await post.author + assert author.name == "Ada Lovelace" + ``` """ def __init__( @@ -201,50 +236,55 @@ def __init__( renamed_from: str | None = None, retired: bool = False, ): - """Initialize foreign-key relationship metadata + """Declare a foreign key to the model in the annotation. Args: - - related_name: Name for reverse access from the related model. - Required unless ``retired=True``: a retired relation has no - reverse access. - on_delete: Referential action for parent deletion. - Common values include "CASCADE", "RESTRICT", "SET NULL", "SET DEFAULT", and "NO ACTION". - unique: Set to True to enforce one-to-one behavior. Implies an - index, so combining ``unique=True`` with ``index=True`` is - redundant; ``index=True`` will be ignored and a ``UserWarning`` - will be raised. - index: Set to True to create a non-unique index on the shadow - ``*_id`` column. Useful for tenant FKs queried on every list - endpoint where Postgres does not auto-index the FK column. - nullable: See :class:`ForeignKey` class docstring. - renamed_from: The relation field's previous name. Its shadow - ``_id`` column is renamed with it (``press`` → - ``house`` renames ``press_id`` to ``house_id``), together with - its ``fk_`` constraint (ADR-0032). - retired: Retire the relation (ADR-0053); see the class docstring. - - Examples: - >>> from typing import Annotated - >>> from ferro import BackRef, ForeignKey, Relation - >>> from ferro.models import Model - >>> - >>> class Org(Model): - ... id: Annotated[int, FerroField(primary_key=True)] - ... projects: Relation[list["Project"]] = BackRef() - >>> - >>> class Project(Model): - ... id: Annotated[int, FerroField(primary_key=True)] - ... org: Annotated[Org, ForeignKey("projects", index=True)] + related_name: The name of the [`BackRef`][ferro.BackRef] on the + target model that reads this relation back. Required unless + `retired=True`. + on_delete: What the database does to this row when the target row + is deleted: `"CASCADE"` (the default), `"RESTRICT"`, + `"SET NULL"`, `"SET DEFAULT"` or `"NO ACTION"`. + unique: Make the relation one-to-one with a unique constraint on + the `_id` column. The constraint indexes the column, so + `index=True` beside it is ignored with a `UserWarning`. + index: Add a non-unique index on the `_id` column. Postgres + does not index a foreign key column on its own. + nullable: `"infer"` (the default) makes the `_id` column + nullable when the relation's annotation allows `None` or + `on_delete` is `"SET NULL"`; `True` or `False` forces it. + renamed_from: The relation's previous field name. The + `_id` column and its foreign key constraint are renamed + with it (`writer` to `author` renames `writer_id` to + `author_id`). See [Rename hints](../guide/schema/renames-and-retirements.md#rename-hints). + retired: Take the relation and its `_id` column out of the + model while the column and its foreign key stay in the schema, + nullable, until a later release deletes the declaration. + Defaults to `False`. See + [Retiring a column](../guide/schema/renames-and-retirements.md#retiring-a-column). + + Raises: + TypeError: `related_name` is missing on a relation that is not + retired, or an option has a value of the wrong type. + ValueError: `on_delete="SET NULL"` with `nullable=False`; the + database could not set the column to `NULL`. """ self.to = None # Resolved later + """The target model class, set once Ferro resolves the relation.""" self.related_name = related_name + """The name of the reverse relation on the target model.""" self.on_delete = on_delete + """The referential action when the target row is deleted.""" self.unique = unique + """Whether the relation is one-to-one.""" self.index = index + """Whether the `_id` column has a non-unique index.""" self.nullable = _validate_nullable_option(nullable, "ForeignKey") + """`"infer"`, or a forced `True` or `False`.""" self.renamed_from = _validate_renamed_from(renamed_from, "ForeignKey") + """The relation's previous field name, or `None`.""" self.retired = _validate_retired(retired, "ForeignKey") + """Whether the relation is retired from the model.""" if related_name is None and not self.retired: raise TypeError( "ForeignKey requires related_name, the BackRef on the target " @@ -263,13 +303,23 @@ def __init__( stacklevel=2, ) self.index = False - #: First type argument of ``Annotated[..., ForeignKey]``; set by the metaclass - #: for Alembic nullability inference (forward fields are not in ``model_fields``). + # Set by the metaclass: forward fields are not in ``model_fields``, so + # nullability inference reads the relation's annotation from here. self.relation_annotation: Any | None = None + """The relation's annotation, set when the model class is defined.""" def foreign_key_allows_none(metadata: "ForeignKey") -> bool | None: - """Effective FK nullability from explicit override, delete action, and relation type.""" + """Decide whether a foreign key's `_id` column is nullable. + + Args: + metadata: The relation's foreign key options. + + Returns: + `True` or `False` when `nullable` forces it or `on_delete` is + `"SET NULL"`; otherwise whether the relation's annotation allows + `None`, or `None` when the annotation is not known yet. + """ if metadata.nullable is True: return True if metadata.nullable is False: @@ -283,24 +333,11 @@ def foreign_key_allows_none(metadata: "ForeignKey") -> bool | None: class ManyToManyRelation: - """Describe internal metadata for a many-to-many relationship - - Attributes: + """The options of a many-to-many relation, as Ferro stores them on the model. - to: Target model class resolved during model binding. - related_name: Name of the reverse relationship attribute on the target model. - through: Optional join table name used for the association. - - Examples: - >>> from typing import Annotated - >>> from ferro.models import Model - >>> - >>> class Tag(Model): - ... id: Annotated[int, FerroField(primary_key=True)] - >>> - >>> class Post(Model): - ... id: Annotated[int, FerroField(primary_key=True)] - ... tags: Relation[list["Tag"]] = ManyToMany(related_name="posts") + Declare the relation with [`ManyToMany`][ferro.ManyToMany], which builds + this object; see + [Many-to-many](../guide/relationships.md#many-to-many). """ def __init__( @@ -309,27 +346,22 @@ def __init__( through: str | None = None, reverse_index: bool = True, ): - """Initialize many-to-many relationship metadata + """Store the options of a many-to-many relation. Args: - - related_name: Name for reverse access from the related model. - through: Explicit join table name. - When omitted, Ferro generates a join table name automatically. - reverse_index: When True (default), the synthesized join table - gets a non-unique composite index on ``(target_col, source_col)`` - to optimize back-ref queries. Set to False to opt out (e.g., - write-heavy join tables where the extra index cost is unwanted). - - Examples: - >>> from typing import Annotated - >>> from ferro.models import Model - >>> - >>> class User(Model): - ... id: Annotated[int, FerroField(primary_key=True)] - ... teams: Relation[list["Team"]] = ManyToMany(related_name="members", through="team_members") + related_name: The name of the relation on the target model that + reads this one back. + through: The join table's name. Defaults to `None`, a name Ferro + derives from the two models. + reverse_index: Add an index on the join table's + `(target, source)` columns for queries from the other side. + Defaults to `True`; turn it off for a write-heavy join table. """ self.to = None # Resolved later + """The target model class, set once Ferro resolves the relation.""" self.related_name = related_name + """The name of the reverse relation on the target model.""" self.through = through + """The join table's name, or `None` for the derived one.""" self.reverse_index = reverse_index + """Whether the join table has an index for reverse queries.""" diff --git a/src/ferro/checks.py b/src/ferro/checks.py index 7593cdbe..ccc33b85 100644 --- a/src/ferro/checks.py +++ b/src/ferro/checks.py @@ -38,6 +38,7 @@ class definition. The lowering below is the single place new arms attach. from .query.nodes import FieldProxy, Predicate, QueryNode, QueryProxy FERRO_CHECKS = "__ferro_checks__" +"""The class attribute a model declares its table checks in.""" _SUFFIX_RE = re.compile(r"^[a-z][a-z0-9_]*\Z") _SUFFIX_SHAPE = "[a-z][a-z0-9_]*" @@ -72,19 +73,55 @@ class definition. The lowering below is the single place new arms attach. @dataclass(frozen=True) class Check: - """One table check: a name suffix plus a ferro predicate lambda. + """A `CHECK` constraint across the columns of one row, declared in `__ferro_checks__`. - The live constraint name is ``ck_
_`` — ferro owns the - prefix, so ``suffix`` is a bare identifier (``[a-z][a-z0-9_]*``), unique - per model. + Ferro names the constraint `ck_
_` and writes it inside + `CREATE TABLE`, on SQLite and Postgres alike. The predicate is a lambda + written like a [`where`][ferro.Model.where] filter. See + [Table checks](../guide/models-and-fields.md#table-checks) for the + predicates it accepts. + + Args: + suffix: The end of the constraint's name: a lowercase identifier + (`[a-z][a-z0-9_]*`), unique within the model, without the + `ck_
_` prefix Ferro adds. + predicate: A lambda over the model's own columns, its parameter named + after the model. + + Raises: + TypeError: `suffix` is not a lowercase identifier or starts with + `ck_`, or `predicate` is not callable. Examples: - >>> Check("at_most_one_outflow", lambda transfer: transfer.outflow_transaction == None) - Check(suffix='at_most_one_outflow') + ```python {test=run prelude=reference} + from typing import ClassVar + + from ferro import Check, CheckViolationError, Field, Model, create_tables + + + class Payment(Model): + __ferro_checks__: ClassVar[tuple[Check, ...]] = ( + Check("positive_amount", lambda payment: payment.amount > 0), + ) + + id: int | None = Field(default=None, primary_key=True) + amount: int + + + await create_tables() + try: + await Payment.create(amount=0) + except CheckViolationError as error: + assert "ck_payment_positive_amount" in str(error) + else: + raise AssertionError("expected CheckViolationError") + ``` """ suffix: str + """The constraint name's suffix, after `ck_
_`.""" predicate: Predicate[Any] + """The lambda the constraint's body is rendered from.""" def __post_init__(self) -> None: if not isinstance(self.suffix, str): @@ -118,15 +155,15 @@ def __repr__(self) -> str: @dataclass(frozen=True) class TableCheckSpec: - """One compiled table check: its suffix and its ``CheckExpr`` payload. + """One compiled table check: its suffix and its predicate as plain data. - The SchemaIR compiler turns the suffix into the canonical constraint name; - ``predicate`` is already in the ``CheckExpr`` wire shape pinned by - ``crates/ferro-schema-ir``. + The schema compiler turns the suffix into the constraint name. """ suffix: str + """The constraint name's suffix, after `ck_
_`.""" predicate: dict[str, Any] + """The predicate, lowered to the JSON shape the schema carries.""" class _CheckProxy(QueryProxy[Any]): @@ -344,7 +381,7 @@ def _check_literal_token(value: Any) -> str: """Render one Python value as a pre-quoted SQL literal token. Mirrors the ``db_check`` enum-label quoting in - :func:`ferro.ir.compiler.compile_schema_ir_payload` — bools as + `ferro.ir.compiler.compile_schema_ir_payload` — bools as ``true``/``false``, numbers as ``str``, everything else single-quoted with ``'`` escaped. """ @@ -591,7 +628,7 @@ def compile_table_checks( columns: The compile's column specs, keyed by column name. Returns: - One :class:`TableCheckSpec` per declared check, in declaration order. + One `TableCheckSpec` per declared check, in declaration order. Raises: TypeError: For any declaration error — a malformed ``__ferro_checks__``, diff --git a/src/ferro/cli/__init__.py b/src/ferro/cli/__init__.py index 5c829982..7e5e4383 100644 --- a/src/ferro/cli/__init__.py +++ b/src/ferro/cli/__init__.py @@ -5,19 +5,19 @@ $ ferro migrate init ``` -The ``ferro`` console script ships with the core wheel and calls -:func:`main`. The command line itself is a cyclopts app that needs the -``cli`` extra; without it :func:`main` prints the install line and exits 2. -Importing this module never imports cyclopts: :data:`app` is built on first -access, so ``import ferro`` costs nothing for the CLI. - -Every verb lives on the ``migrate`` sub-app (:mod:`ferro.cli.migrate`) and -receives the three global options as one :class:`Global`, given before or -after the verb (``ferro --database app migrate up`` or -``ferro migrate up --database app``). A :class:`~ferro.exceptions.FerroError` -raised by a verb is a refusal: :func:`render_refusal` prints its message, -which names the fix, and the command exits :data:`exit_codes.REFUSED`. -A bad flag or a missing ``cli`` extra exits :data:`exit_codes.USAGE`. +The `ferro` console script ships with the core wheel and calls `main`. +The command line itself is a cyclopts app that needs the `cli` extra; +without it `main` prints the install line and exits 2. Importing this module +never imports cyclopts: `app` is built on first access, so `import ferro` +costs nothing for the CLI. + +Every verb lives on the `migrate` sub-app (`ferro.cli.migrate`) and +receives the three global options as one `Global`, given before or after +the verb (`ferro --database app migrate up` or +`ferro migrate up --database app`). A [`FerroError`][ferro.FerroError] +raised by a verb is a refusal: `render_refusal` prints its message, which +names the fix, and the command exits `exit_codes.REFUSED`. A bad flag or a +missing `cli` extra exits `exit_codes.USAGE`. """ import logging @@ -41,24 +41,33 @@ @dataclass(frozen=True) class Global: - """The global options every verb receives (ADR-0036: the whole external - surface is these three flags and two environment variables).""" + """The global options every verb receives. + + These three flags, with the `FERRO_CONFIG` and `url_env` environment + variables, are the whole surface outside the verbs' own flags. + """ config: Path | None = None - """``--config``: the config file to read, in place of the lookup.""" + """`--config`: the config file to read, in place of the lookup.""" database: str | None = None - """``--database``: which configured database the verb acts on.""" + """`--database`: which configured database the verb acts on.""" url: str | None = None - """``--url``: the database URL, in place of the ``url_env`` variable.""" + """`--url`: the database URL, in place of the `url_env` variable.""" def render_refusal(err: FerroError, verb: str | None = None) -> int: - """Print a refusal's message to stderr and return :data:`exit_codes.REFUSED`. + """Print a refusal's message to stderr and return the refused exit code. The message is printed as it is; on a terminal it sits under a red - heading naming what refused (``new refused``, or ``ferro refused`` when - no ``verb`` was running), through :meth:`Output.refusal`, the one - refusal path every verb prints through. + heading naming what refused (`new refused`, or `ferro refused` when no + verb was running). Every verb prints its refusals through here. + + Args: + err: The refusal to print; its message names the fix. + verb: The verb that refused, or `None` when no verb was running. + + Returns: + `exit_codes.REFUSED` (1). """ from .output import Output @@ -69,9 +78,16 @@ def render_refusal(err: FerroError, verb: str | None = None) -> int: def main(argv: Sequence[str] | None = None) -> int: - """Run the ``ferro`` command on ``argv`` (``sys.argv[1:]`` when ``None``). + """Run the `ferro` command line and return its exit code. + + Args: + argv: The arguments after the program name. Defaults to + `sys.argv[1:]`. - Returns the exit code; the console script passes it to ``sys.exit``. + Returns: + The exit code, which the console script passes to `sys.exit`: 0 on + success, 1 for a refusal, 2 for a bad flag or a missing `cli` + extra, and the verb's own code otherwise. """ try: import cyclopts @@ -141,8 +157,9 @@ def _build() -> "App": ferro_version = "unknown" root = App( + help_format="plaintext", name="ferro", - help="ferro's command line: schema migrations for ferro models.", + help="Ferro's command line: schema migrations for Ferro models.", version=ferro_version, result_action="return_value", exit_on_error=False, diff --git a/src/ferro/cli/migrate.py b/src/ferro/cli/migrate.py index 7e6ab222..3f958079 100644 --- a/src/ferro/cli/migrate.py +++ b/src/ferro/cli/migrate.py @@ -1,21 +1,23 @@ -"""``ferro migrate``: the sub-app every migration verb registers on. +"""`ferro migrate`: the sub-app every migration verb registers on. -This module defines ``init``. Later verbs register with ``@migrate.command`` -and take ``glob: Annotated[Global, Parameter(parse=False)]`` to receive the -global options. +Each verb registers with `@migrate.command` and takes +`glob: Annotated[Global, Parameter(parse=False)]` to receive the global +options. -``init`` writes the project configuration :class:`~ferro.FerroSettings` -reads (ADR-0036, ADR-0039):: +`init` writes the project configuration that +[`FerroSettings`][ferro.FerroSettings] reads: - $ ferro migrate init - Config file [pyproject.toml]: - Models module (dotted) [myapp.models]: - Target dialects [postgres]: - Another database (separate tables, own migration history)? [y/N]: - Wrote [tool.ferro] to pyproject.toml and created migrations/ +```text +$ ferro migrate init +Config file [pyproject.toml]: +Models module (dotted) [myapp.models]: +Target dialects [postgres]: +Another database (separate tables, own migration history)? [y/N]: +Wrote [tool.ferro] to pyproject.toml and created migrations/ +``` Every prompt has a flag that answers it. The config is appended to an -existing ``pyproject.toml`` as text, so the user's other tables are never +existing `pyproject.toml` as text, so the user's other tables are never reformatted. """ @@ -48,6 +50,7 @@ __all__ = ["migrate"] migrate = App( + help_format="plaintext", name="migrate", help="Write, apply and check schema migrations.", ) @@ -62,8 +65,7 @@ negative="", ), ] -"""``--verbose`` on ``new`` and ``check``, the verbs that print a change -summary.""" +"""The `--verbose` flag of `new` and `check`, the verbs that print a change summary.""" DIALECTS = ("postgres", "sqlite") _DATABASE_NAME = re.compile(r"^[A-Za-z0-9_-]+$") @@ -207,10 +209,10 @@ def new( str | None, Parameter( help=( - "Regenerate this migration (NNNN_, or NNNN when one " - "migration carries the number) on top of the newest snapshot " - "below it, carrying its data steps and hand-written SQL steps " - "across: two branches that each generated the same number." + "When two branches generated the same number, regenerate this " + "migration (NNNN_, or NNNN when one migration carries the " + "number) on top of the newest snapshot below it, carrying its " + "data steps and hand-written SQL steps across." ) ), ] = None, @@ -317,7 +319,7 @@ def squash( ) -> int: """Replace a range of migrations, from --from through the newest, with one. - Writes the migration `new` would have written across the range: planned + Writes the migration new would have written across the range: planned once from the snapshot below the range to the range's last snapshot (its renames composed), carrying every data step and hand-written SQL step of the range byte for byte, under the first migration's number. Deletes the @@ -356,8 +358,7 @@ def check( verbose: Verbose = False, glob: Annotated[Global, Parameter(parse=False)], ) -> int: - """Check, offline, that every model change has a migration and the - migrations directory is intact. + """Check, offline, that every model change has a migration and the directory is intact. Exits 0 when it is, 3 naming each problem otherwise: an ungenerated model change, a broken snapshot chain, a duplicate or missing number, a diff --git a/src/ferro/cli/output.py b/src/ferro/cli/output.py index b56a2cc0..a14b105d 100644 --- a/src/ferro/cli/output.py +++ b/src/ferro/cli/output.py @@ -1,4 +1,4 @@ -"""How every ``ferro migrate`` verb prints: one presentation module, two forms. +"""How every `ferro migrate` verb prints: one presentation module, two forms. ```text $ ferro migrate new add_team --data-step Author @@ -13,28 +13,27 @@ ``` Rich's console decides the form, from the stream and the environment; there -is no flag. The **plain form** (stdout is not a terminal: a pipe, a file, a -CI log) prints ASCII glyphs, no colour, and every name on its own line, and -keeps the one-liners (``ok: models match 0001_initial``), every refusal and -every ``warning: …`` line on stderr byte for byte. The **TTY form** (a -terminal, or ``FORCE_COLOR`` set) is headed: a glyph and a bold heading, the -details indented beneath, name lists in columns sized to the terminal, a long -list of the change summary collapsed behind ``--verbose``. ``NO_COLOR`` strips every escape -and keeps the layout; a dumb terminal gets the ASCII glyphs, and a stream -that cannot encode Unicode (``PYTHONIOENCODING=ascii``) gets ASCII -throughout: the glyphs, and ``-``, ``->`` and ``...`` for ``·``, ``→`` and -``…`` (:data:`ASCII_FORMS`); ``COLUMNS`` sets the width. +is no flag. The plain form (stdout is not a terminal: a pipe, a file, a CI +log) prints ASCII glyphs, no colour, and every name on its own line, and +keeps the one-liners (`ok: models match 0001_initial`), every refusal and +every `warning: …` line on stderr byte for byte. The TTY form (a terminal, +or `FORCE_COLOR` set) is headed: a glyph and a bold heading, the details +indented beneath, name lists in columns sized to the terminal, a long list +of the change summary collapsed behind `--verbose`. `NO_COLOR` strips every +escape and keeps the layout; a dumb terminal gets the ASCII glyphs, and a +stream that cannot encode Unicode (`PYTHONIOENCODING=ascii`) gets ASCII +throughout: the glyphs, and `-`, `->` and `...` for `·`, `→` and `…` +(`ASCII_FORMS`); `COLUMNS` sets the width. Words carry every meaning in both forms (a file's role, a list's count heading, a problem's kind); colour only reinforces them. The tables below are the one glyph table and the one colour table every verb reads. -The module reads report objects (:class:`GeneratedMigration`, -:class:`CheckReport`, :class:`StatusReport`, :class:`DriftReport`, -:class:`BaselineReport`, :class:`RerecordReport`, :class:`DownPlan`) and a run's -events (:class:`StepEvent`, :class:`RunNote`, a :class:`RunReport`'s -:class:`StepFailure`) and prints; it never changes them and never takes a -report's text apart. +The module reads report objects (`GeneratedMigration`, `CheckReport`, +`StatusReport`, `DriftReport`, `BaselineReport`, `RerecordReport`, +`DownPlan`) and a run's events (`StepEvent`, `RunNote`, a `RunReport`'s +`StepFailure`) and prints; it never changes them and never takes a report's +text apart. """ from __future__ import annotations @@ -105,8 +104,7 @@ "bullet": ("•", "-"), "arrow": ("→", "->"), } -"""Each glyph as ``(TTY, ASCII)``: the ASCII one in the plain form and on a -terminal that cannot show the other.""" +"""Each glyph as `(TTY, ASCII)`: the ASCII one in the plain form and on a terminal that cannot show the other.""" GLYPH_STYLE = { "ok": "bold green", @@ -117,6 +115,7 @@ "bullet": "dim", "arrow": "dim", } +"""Each glyph's style, by the same keys as `GLYPHS`.""" KIND_STYLE = { "ddl": "cyan", @@ -124,10 +123,12 @@ "portable_sql": "blue", "snapshot": "dim", } -"""A step file's stem, by the kind its generated step records (``ddl``, a -schema step; ``data``, a Python step; ``portable_sql``, a hand-written SQL -step) or ``snapshot`` for ``ir.json``. A down file keeps its kind's hue, -dimmed; the suffix is always dim.""" +"""A step file's stem style, by the kind its generated step records, or `snapshot` for `ir.json`. + +The kinds are `ddl` (a schema step), `data` (a Python step) and +`portable_sql` (a hand-written SQL step). A down file keeps its kind's hue, +dimmed; the suffix is always dim. +""" CHANGE_STYLE = { "new": "green", @@ -136,8 +137,10 @@ "removed": "red", "renamed": "blue", } -"""A change summary's names, by the generator's verb: the colours ``git -status`` taught. The count heading above them stays plain bold.""" +"""A change summary's names' style, by the generator's verb: the colours `git status` taught. + +The count heading above them stays plain bold. +""" STATES: dict[str, tuple[str, str]] = { "applied": ("ok", "green"), @@ -155,52 +158,66 @@ "nothing to reverse": ("ok", "dim"), "partial": ("partial", "yellow"), } -"""A migration's or a step's state word, as ``status``, ``up`` and ``down`` -print it: the glyph its row opens with (a key of :data:`GLYPHS`) and its -style. ``partial`` stands for a migration's ``partial, X of N steps``, the -one state word that carries numbers. :data:`STATE_GLYPH` and -:data:`STATE_STYLE` are its two columns.""" +"""Each state word `status`, `up` and `down` print, with its row's glyph (a key of `GLYPHS`) and style. + +`partial` stands for a migration's `partial, X of N steps`, the one state +word that carries numbers. `STATE_GLYPH` and `STATE_STYLE` are its two +columns. +""" STATE_GLYPH = {word: glyph for word, (glyph, _) in STATES.items()} -"""The glyph each state word's row opens with, from :data:`STATES`.""" +"""The glyph each state word's row opens with, from `STATES`.""" STATE_STYLE = {word: style for word, (_, style) in STATES.items()} -"""Each state word's style, from :data:`STATES`.""" +"""Each state word's style, from `STATES`.""" ASCII_FORMS = {"·": "-", "→": "->", "…": "..."} -"""What a stream that cannot encode Unicode prints for each such character -a line carries (the database line's separator, a rename's arrow, an -elision); the glyphs fall back through :data:`GLYPHS`.""" +"""What a stream that cannot encode Unicode prints for each such character a line carries. + +These are the database line's separator, a rename's arrow and an elision; +the glyphs fall back through `GLYPHS`. +""" LONG_LIST = 12 """A list of the change summary of up to this many names is shown whole on a TTY.""" SHOWN_FIRST = 6 -"""A longer one shows its first names, then how many more ``--verbose`` -shows.""" +"""How many names a longer list shows on a TTY before saying how many more `--verbose` shows.""" class Output: """The two streams of one command, as Rich sees them. - Build it once per command, inside the verb, with :meth:`for_streams`, so - it wraps the streams the command really writes to. + Build it once per command, inside the verb, with `for_streams`, so it + wraps the streams the command really writes to. """ def __init__(self, out: Console, err: Console) -> None: self.out = out + """The console on stdout.""" self.err = err + """The console on stderr.""" self.plain: bool = not out.is_terminal """The plain form: stdout is not a terminal.""" self.width: int = out.width + """The width stdout lays out at.""" self._ascii = self.plain or _ascii_only(out) or out.is_dumb_terminal @classmethod def for_streams( cls, out: TextIO | None = None, err: TextIO | None = None ) -> Output: - """The output of a command writing to ``out`` and ``err`` - (``sys.stdout`` and ``sys.stderr`` when omitted, read at the call).""" + """Build the output of a command writing to two streams. + + Args: + out: The stdout stream. Defaults to `sys.stdout`, read at the + call. + err: The stderr stream. Defaults to `sys.stderr`, read at the + call. + + Returns: + The output. + """ return cls( _console(sys.stdout if out is None else out), _console(sys.stderr if err is None else err), @@ -209,14 +226,29 @@ def for_streams( # -- pieces ------------------------------------------------------------------ def glyph(self, key: str) -> str: - """The glyph ``key`` names in :data:`GLYPHS`, in this output's form.""" + """Return a glyph in this output's form. + + Args: + key: A key of `GLYPHS`. + + Returns: + The TTY glyph, or its ASCII form. + """ tty, ascii_ = GLYPHS[key] return ascii_ if self._ascii else tty def file_name(self, name: str, *, kind: str) -> Text: - """A step file's name: its stem coloured by ``kind`` (a key of - :data:`KIND_STYLE`), its suffix dim, a down file its kind's hue - dimmed.""" + """Style a step file's name: its stem coloured by kind, its suffix dim. + + A down file takes its kind's hue, dimmed. + + Args: + name: The file name. + kind: A key of `KIND_STYLE`. + + Returns: + The styled name. + """ stem, dot, suffix = name.partition(".") style = KIND_STYLE[kind] if suffix.startswith("down.") and kind != "snapshot": @@ -230,11 +262,20 @@ def file_name(self, name: str, *, kind: str) -> Text: # -- lines ------------------------------------------------------------------- def fit(self, text: str, *, err: bool = False) -> str: - """``text`` as the stream can carry it: itself on a stream that - encodes Unicode; else each :data:`ASCII_FORMS` character replaced, - and any other character the stream cannot encode (a name's accent) - written as its backslash escape (``\\xe9``), so nothing is lost and - nothing raises.""" + """Fit text to what the stream can carry, losing nothing and raising nothing. + + On a stream that encodes Unicode the text is unchanged. Otherwise + each `ASCII_FORMS` character is replaced, and any other character + the stream cannot encode (a name's accent) is written as its + backslash escape (`\\xe9`). + + Args: + text: The text. + err: Fit it to stderr rather than stdout. + + Returns: + The fitted text. + """ console = self.err if err else self.out if not _ascii_only(console): return text @@ -244,14 +285,19 @@ def fit(self, text: str, *, err: bool = False) -> str: return text.encode(encoding, "backslashreplace").decode(encoding) def print(self, text: str | Text = "", *, err: bool = False) -> None: - """Print one line, through :meth:`fit`. No form wraps it: a script - reads what was said, and a command quoted in a note or a refusal is - copied whole; a terminal folds a long line on its own. + """Print one line, fitted to the stream and never wrapped. - A stream that is not a terminal gets the line's text written to it - as it is, never laid out by Rich: no width (``COLUMNS=0`` included) + A script reads what was said, and a command quoted in a note or a + refusal is copied whole; a terminal folds a long line on its own. A + stream that is not a terminal gets the line's text written to it as + it is, never laid out by Rich: no width (`COLUMNS=0` included) decides whether a line prints, and a tab or a carriage return in a - refusal reaches the pipe unchanged.""" + refusal reaches the pipe unchanged. + + Args: + text: The line, plain or styled. Defaults to an empty line. + err: Print it on stderr rather than stdout. + """ console = self.err if err else self.out if not console.is_terminal: line = text if isinstance(text, str) else text.plain @@ -267,7 +313,7 @@ def print(self, text: str | Text = "", *, err: bool = False) -> None: console.print(text) def _fit_text(self, text: Text, *, err: bool) -> Text: - """:meth:`fit` over a styled line, each run keeping its styles.""" + """`fit` over a styled line, each run keeping its styles.""" cuts = sorted( {span.start for span in text.spans} | {span.end for span in text.spans} ) @@ -282,8 +328,17 @@ def _fit_text(self, text: Text, *, err: bool) -> Text: def heading( self, key: str, title: str | Text, where: str = "", *, err: bool = False ) -> None: - """A result's heading: its glyph, a bold title (red for an error) - and a dim ``where`` clause (``✔ Wrote migrations/0002_add_team/``).""" + """Print a result's heading: its glyph, a bold title and a dim clause. + + For example `✔ Wrote migrations/0002_add_team/`. An error's title is + red. + + Args: + key: The glyph's key in `GLYPHS`. + title: The title. + where: The dim clause after it. Defaults to none. + err: Print it on stderr rather than stdout. + """ text = Text() text.append(self.glyph(key), style=GLYPH_STYLE[key]) text.append(" ") @@ -296,11 +351,15 @@ def heading( self.print(text, err=err) def database_line(self, name: str, where: str) -> None: - """The first line of every verb: the database's name, bold, then - what it touches, dim: ``main · postgresql://pinch:***@db:5432/app`` - (``where`` the masked URL), or ``main · migrations/main`` for - ``new`` and ``check``, which never connect (``where`` the - directory).""" + """Print the first line of every verb: the database's name, bold, then what it touches, dim. + + For example `main · postgresql://app:***@db:5432/app`, or + `main · migrations/main` for `new` and `check`, which never connect. + + Args: + name: The configured database's name. + where: The masked URL, or the migrations directory. + """ text = Text(f"{name} · {where}") text.stylize("bold", 0, len(name)) text.stylize("dim", len(text) - len(where)) @@ -315,11 +374,20 @@ def names( verbose: bool, err: bool = False, ) -> None: - """One list of the change summary: its count heading (``2 new models``), plain - bold, then its names coloured by ``verb``. One name per line in the - plain form; columns on a TTY, where a list longer than - :data:`LONG_LIST` shows its first :data:`SHOWN_FIRST` unless - ``verbose``.""" + """Print one list of the change summary: its count heading, then its names. + + The heading (`2 new models`) is plain bold, the names coloured by + `verb`. One name per line in the plain form; columns on a TTY, where + a list longer than `LONG_LIST` shows its first `SHOWN_FIRST` unless + `verbose`. + + Args: + label_count: The count heading. + names: The names. + verb: The generator's verb, a key of `CHANGE_STYLE`. + verbose: Show every name of a long list. + err: Print it on stderr rather than stdout. + """ self.print(Text(f" {label_count}", style="bold"), err=err) style = CHANGE_STYLE[verb] names = [self.fit(name, err=err) for name in names] @@ -349,15 +417,26 @@ def names( ) def note(self, text: str, *, err: bool = False) -> None: - """A note after a result, behind a bullet.""" + """Print a note after a result, behind a bullet. + + Args: + text: The note. + err: Print it on stderr rather than stdout. + """ line = Text(" ") line.append(self.glyph("bullet"), style=GLYPH_STYLE["bullet"]) line.append(f" {text}") self.print(line, err=err) def warning(self, text: str) -> None: - """A warning, on stderr: ``warning: `` in the plain form, the - warning glyph and the word ``warning`` before the text on a TTY.""" + """Print a warning on stderr. + + The plain form prints `warning: `; a TTY prints the warning + glyph and the word `warning` before the text. + + Args: + text: The warning. + """ if self.plain: self.print(f"warning: {text}", err=True) return @@ -368,17 +447,30 @@ def warning(self, text: str) -> None: self.print(line, err=True) def problems(self, heading: str, lines: Sequence[str]) -> None: - """Problems, on stderr, each line as it is: under a red ``heading`` - on a TTY, alone in the plain form.""" + """Print problems on stderr, each line as it is. + + A TTY prints them under a red heading; the plain form prints them + alone. + + Args: + heading: The heading. + lines: The problems. + """ if not self.plain: self.heading("error", heading, err=True) for line in lines: self.print(line, err=True) def refusal(self, heading: str, sentence: str) -> None: - """A verb's refusal, on stderr: the sentence as it is, under a red - ``heading`` naming what refused (``drift refused``) on a TTY, alone - in the plain form. The one refusal path every verb prints through.""" + """Print a verb's refusal on stderr: the one refusal path every verb prints through. + + A TTY prints the sentence under a red heading naming what refused; + the plain form prints it alone. + + Args: + heading: What refused, such as `drift refused`. + sentence: The refusal, as it is. + """ self.problems(heading, [sentence]) @@ -416,12 +508,13 @@ def _console(file: TextIO) -> Console: FALLBACK_WIDTH = 80 -"""The terminal width the TTY form lays out at when the reported one is -below one column.""" +"""The terminal width the TTY form lays out at when the reported one is below one column.""" NAME_INDENT = " " +"""The indent of a change summary's names.""" NAME_GAP = 2 +"""The spaces between two columns of names.""" def _columns(names: Sequence[str], width: int) -> list[list[tuple[str, int]]]: @@ -441,7 +534,15 @@ def _columns(names: Sequence[str], width: int) -> list[list[tuple[str, int]]]: def count(n: int, noun: str) -> str: - """``n`` and ``noun``, pluralised by an ``s`` unless ``n`` is one.""" + """Count a noun: `1 step`, `2 steps`. + + Args: + n: The count. + noun: The noun, singular. + + Returns: + The count and the noun, pluralised by an `s` unless `n` is one. + """ return f"{n} {noun}{'' if n == 1 else 's'}" @@ -451,9 +552,18 @@ def count(n: int, noun: str) -> str: def print_new( out: Output, migration: GeneratedMigration, shown_dir: str, *, verbose: bool ) -> None: - """What ``ferro migrate new`` wrote: the directory, each file with its - role (and, under ``--replace``, what became of it), the change summary, - the notes, and the warnings on stderr.""" + """Print what `ferro migrate new` wrote. + + That is the directory, each file with its role (and, under `--replace`, + what became of it), the change summary, the notes, and the warnings on + stderr. + + Args: + out: The command's output. + migration: The written migration. + shown_dir: The migration's directory, as the heading shows it. + verbose: Show every name of a long change list. + """ from ..migrations.report import migration_span where = "" @@ -493,9 +603,16 @@ def print_new( def print_changes( out: Output, changes: Sequence[Change], *, verbose: bool, err: bool = False ) -> None: - """A change summary laid out from its data, as ``new`` and ``check`` - print it: each change's count heading (``2 new models``), then its - names (:meth:`Output.names`).""" + """Print a change summary laid out from its data, as `new` and `check` print it. + + Each change prints its count heading (`2 new models`), then its names. + + Args: + out: The command's output. + changes: The changes. + verbose: Show every name of a long list. + err: Print it on stderr rather than stdout. + """ for change in changes: noun = f"{change.verb} {change.kind.removesuffix('s')}" out.names( @@ -508,13 +625,15 @@ def print_changes( DATA_ROLE = {"backfill": "backfill", "guard": "guard", "data_step": "data step"} -"""A data step's role words, by the role the generator gave it; a carried -step, a person's whatever it began as, is a ``data step``.""" +"""A data step's role words, by the role the generator gave it. + +A carried step, a person's whatever it began as, is a `data step`. +""" def _file_rows(migration: GeneratedMigration) -> Iterator[tuple[str, str, str, str]]: """Each file of ``migration``, ``ir.json`` last: its name, its kind (a - key of :data:`KIND_STYLE`), its role words, and under ``--replace`` + key of `KIND_STYLE`), its role words, and under ``--replace`` what became of the replaced migration's file (else ``""``).""" from ..migrations.layout import SNAPSHOT_FILE @@ -530,7 +649,7 @@ def _file_rows(migration: GeneratedMigration) -> Iterator[tuple[str, str, str, s def _role(step: GeneratedStep, name: str) -> str: """A file's role words: a data step's by the role the generator gave it; a SQL file's by its direction and dialect, as the core reads its - name (:func:`step_file_role`).""" + name (`step_file_role`).""" if step.kind == "data": return DATA_ROLE.get(step.role or "data_step", "data step") role = step_file_role(name) @@ -580,14 +699,21 @@ def _became(migration: GeneratedMigration, name: str) -> str: def print_check(out: Output, report: CheckReport, *, verbose: bool) -> None: - """What ``ferro migrate check`` found: ``ok: models match NNNN`` (the - plain one-liner) or a heading on a TTY; else every problem on stderr, - never dropped. In the plain form each problem is ``kind: message``, the - change summary inside the sentence as the core wrote it. On a TTY the - problems sit under a red heading, and an ``ungenerated`` problem prints - its sentence without the summary, then the summary laid out from its - data as ``new`` lays it out (:func:`print_changes`: a long list - collapses unless ``verbose``), then its notes.""" + """Print what `ferro migrate check` found. + + A clean check prints `ok: models match NNNN` (the plain one-liner) or a + heading on a TTY; otherwise every problem goes to stderr, never dropped. + In the plain form each problem is `kind: message`, the change summary + inside the sentence as the core wrote it. On a TTY the problems sit + under a red heading, and an `ungenerated` problem prints its sentence + without the summary, then the summary laid out as `new` lays it out, + then its notes. + + Args: + out: The command's output. + report: The check's report. + verbose: Show every name of a long change list. + """ head = report.head or "no migration" if report.ok: if out.plain: @@ -616,9 +742,14 @@ def print_check(out: Output, report: CheckReport, *, verbose: bool) -> None: def applied_at_short(iso: str | None) -> str: - """A stored finish time to the minute in UTC with an explicit ``Z`` - (``2026-10-14T09:40:12.123Z`` is ``2026-10-14 09:40Z``); ``""`` for - ``None``.""" + """Shorten a stored finish time to the minute, in UTC with an explicit `Z`. + + Args: + iso: The stored time, such as `2026-10-14T09:40:12.123Z`, or `None`. + + Returns: + The short time (`2026-10-14 09:40Z`), or `""` for `None`. + """ if iso is None: return "" return datetime.fromisoformat(iso).astimezone(UTC).strftime("%Y-%m-%d %H:%MZ") @@ -628,26 +759,42 @@ class StepFileRole(NamedTuple): """What a step file's name says it is, as the core reads it.""" kind: str - """``ddl``, ``data`` or ``portable_sql``.""" + """`ddl`, `data` or `portable_sql`.""" direction: str - """``up`` or ``down``.""" + """`up` or `down`.""" dialect: str - """``postgres``, ``sqlite`` or ``portable``.""" + """`postgres`, `sqlite` or `portable`.""" def step_file_role(name: str) -> StepFileRole: - """The kind, direction and dialect a step file's name says, read by the - directory reader's own parse (``_core._step_file_role``); ``ValueError`` - for a name that is not a step file's.""" + """Read the kind, direction and dialect a step file's name says, as the directory reader parses it. + + Args: + name: The file name. + + Returns: + The file's kind, direction and dialect. + + Raises: + ValueError: The name is not a step file's. + """ from .._core import _step_file_role return StepFileRole(**json.loads(_step_file_role(name))) def kind_of_file(name: str) -> str: - """A step file's kind (a key of :data:`KIND_STYLE`): the core's reading - of its name (:func:`step_file_role`), or ``snapshot`` for ``ir.json``; - ``ValueError`` for any other name.""" + """Read a file's kind from its name, as a key of `KIND_STYLE`. + + Args: + name: The file name. + + Returns: + The step file's kind, or `snapshot` for `ir.json`. + + Raises: + ValueError: The name is neither a step file's nor `ir.json`. + """ from ..migrations.layout import SNAPSHOT_FILE if name == SNAPSHOT_FILE: @@ -656,7 +803,7 @@ def kind_of_file(name: str) -> str: def _state_key(word: str) -> str: - """The key of :data:`STATE_STYLE` and :data:`STATE_GLYPH` for a state + """The key of `STATE_STYLE` and `STATE_GLYPH` for a state word: itself, or ``partial`` for ``partial, X of N steps``.""" return word if word in STATE_STYLE else "partial" @@ -693,13 +840,20 @@ def having(state: str) -> int: def print_status(out: Output, report: StatusReport, *, steps: bool) -> None: - """Where the database stands: a heading counting what is failed, - pending and ahead beside the tracking table; one row per migration - (glyph, name, state word, ``applied_at``), never collapsed; the steps - of a migration that needs attention, or of every one with ``steps``; - the migrations applied but not in the directory; and what ``up`` would - refuse with, under ``needs attention``. All of it on stdout: the - refusal is part of the report.""" + """Print where the database stands, all on stdout: the refusal is part of the report. + + That is a heading counting what is failed, pending and ahead beside the + tracking table; one row per migration (glyph, name, state word, + `applied_at`), never collapsed; the steps of a migration that needs + attention, or of every one with `steps`; the migrations applied but not + in the directory; and what `up` would refuse with, under + `needs attention`. + + Args: + out: The command's output. + report: The status report. + steps: Print every migration's steps. + """ key, title = _status_title(report) out.heading(key, title, report.table) names = [m.name for m in report.migrations] + report.ahead @@ -757,7 +911,7 @@ def _status_steps(out: Output, migration: MigrationStatus, *, steps: bool) -> No """A migration's steps, six spaces in: each file coloured by its kind, its state, ``[flag]``s and (with ``--steps``) its class; beneath, an edited step's two checksums or an unfinished step's error, where - :meth:`StatusReport.render` shows them.""" + `StatusReport.render` shows them.""" width = max((len(step.file) for step in migration.steps), default=0) for step in migration.steps: line = Text(" ") @@ -784,12 +938,17 @@ def _status_steps(out: Output, migration: MigrationStatus, *, steps: bool) -> No def print_drift(out: Output, report: DriftReport) -> None: - """What ``ferro migrate drift`` found. Its warnings first, on stderr; - then, when it refused, the refusal sentence on stderr as it is (under a - red ``drift refused`` heading on a TTY, :meth:`Output.refusal`); else - ``no drift against NNNN`` (the plain one-liner, an ok heading on a TTY) - or ``N differences against NNNN`` and one bullet per finding, never - collapsed.""" + """Print what `ferro migrate drift` found. + + Its warnings come first, on stderr; then, when it refused, the refusal + sentence on stderr as it is; else `no drift against NNNN` (the plain + one-liner, an ok heading on a TTY) or `N differences against NNNN` and + one bullet per finding, never collapsed. + + Args: + out: The command's output. + report: The drift report. + """ for warning in report.warnings: out.warning(warning) if report.refusal is not None: @@ -824,13 +983,20 @@ def _drift_findings(out: Output, report: DriftReport) -> None: def print_step( out: Output, event: StepEvent, *, name_width: int, stem_width: int ) -> None: - """One step a run finished, as it finishes. The plain form is the line - ``progress`` receives (:meth:`StepEvent.line`); on a TTY a row two + """Print one step a run finished, as it finishes. + + The plain form is the line `progress` receives; a TTY prints a row two spaces in: the glyph, the migration, the step coloured by its kind, the - state word and the duration in seconds, dim (``nothing to reverse`` - gives its reason instead). A failed step is a red row on a TTY and - nothing in the plain form, where its message alone says it - (:func:`print_run_failure`).""" + state word and the duration in seconds, dim (`nothing to reverse` gives + its reason instead). A failed step is a red row on a TTY and nothing in + the plain form, where its message alone says it. + + Args: + out: The command's output. + event: The finished step. + name_width: The migration column's width. + stem_width: The step column's width. + """ if out.plain: line = event.line(name_width, stem_width) if line is not None: @@ -851,12 +1017,18 @@ def print_step( def print_run_failure(out: Output, refused: MigrationRefused) -> None: - """Why ``up`` or ``down`` stopped, on stderr: its report's refusal (or, - with no run report, the exception's text) as it is in the plain form. - On a TTY a blank line, then a failed step under a red `` - failed`` heading (``after N rows`` for a chunked step's committed - batches) with its error red and its hint plain, indented two; any other - refusal under ``up refused`` / ``down refused``.""" + """Print why `up` or `down` stopped, on stderr. + + The plain form prints the report's refusal (or, with no run report, the + exception's text) as it is. A TTY prints a blank line, then a failed + step under a red ` failed` heading (`after N rows` for a chunked + step's committed batches) with its error red and its hint plain, + indented two; any other refusal under `up refused` / `down refused`. + + Args: + out: The command's output. + refused: The refusal the run raised. + """ from ..migrations.runner import RunReport report = refused.report @@ -877,10 +1049,16 @@ def print_run_failure(out: Output, refused: MigrationRefused) -> None: def print_down_plan(out: Output, plan: DownPlan) -> None: - """What ``down`` would revert, newest first: :meth:`DownPlan.describe` - in the plain form; on a TTY a warning heading naming the migrations and - the step count, then a row per step, its stem coloured by kind and a - nothing-to-reverse reason dim.""" + """Print what `down` would revert, newest first. + + The plain form prints the plan's `describe()`; a TTY prints a warning + heading naming the migrations and the step count, then a row per step, + its stem coloured by kind and a nothing-to-reverse reason dim. + + Args: + out: The command's output. + plan: The down plan. + """ if out.plain: out.print(plan.describe()) return @@ -902,10 +1080,15 @@ def print_down_plan(out: Output, plan: DownPlan) -> None: def print_reverted(out: Output, reverted: Sequence[AppliedStep]) -> None: - """A finished ``down``'s closing heading, on a TTY only: ``reverted - N steps``. Where the database now stands is ``status``'s - to say: neither the plan nor the run report names the migration below - the lowest step reverted.""" + """Print a finished `down`'s closing heading, on a TTY only: `reverted N steps`. + + Where the database now stands is `status`'s to say: neither the plan nor + the run report names the migration below the lowest step reverted. + + Args: + out: The command's output. + reverted: The steps reverted. + """ if out.plain: return migrations = list(dict.fromkeys(step.migration for step in reverted)) @@ -917,11 +1100,17 @@ def print_reverted(out: Output, reverted: Sequence[AppliedStep]) -> None: def print_inherited(out: Output, event: InheritedEvent) -> None: - """A squash the run inherited, before any step (ADR-0056): the line - ``progress`` receives in the plain form (``inherited 0005_teams_epic: this - database had applied 0005_add_teams … 0009_fix_rows; nothing was - run``); on a TTY an ok heading, ``inherited `` bold and the rest - dim.""" + """Print a squash the run inherited, before any step. + + The plain form prints the line `progress` receives + (`inherited 0005_teams_epic: this database had applied 0005_add_teams … + 0009_fix_rows; nothing was run`); a TTY prints an ok heading, + `inherited ` bold and the rest dim. + + Args: + out: The command's output. + event: The inherited squash. + """ from ..migrations.report import migration_span if out.plain: @@ -935,9 +1124,15 @@ def print_inherited(out: Output, event: InheritedEvent) -> None: def print_event(out: Output, event: RunEvent) -> None: - """One line a run says, as it says it: a finished step through - :func:`print_step` at the run's widths, an inherited squash through - :func:`print_inherited`; any other line as it is.""" + """Print one line a run says, as it says it. + + A finished step prints through `print_step` at the run's widths, an + inherited squash through `print_inherited`, and any other line as it is. + + Args: + out: The command's output. + event: The run's event. + """ from ..migrations.runner import InheritedEvent, StepEvent if isinstance(event, StepEvent): @@ -949,9 +1144,17 @@ def print_event(out: Output, event: RunEvent) -> None: def print_done(out: Output, title: str, where: str) -> None: - """A run with nothing to do: ``: <where>`` in the plain form - (``nothing to apply: the database is up to date``), an ok heading on a - TTY; ``title`` alone when ``where`` is empty (``nothing to revert``).""" + """Print a run with nothing to do. + + The plain form prints `<title>: <where>` + (`nothing to apply: the database is up to date`), or `title` alone when + `where` is empty; a TTY prints an ok heading. + + Args: + out: The command's output. + title: What the run found, such as `nothing to revert`. + where: Why, or an empty string. + """ if out.plain: out.print(f"{title}: {where}" if where else title) else: @@ -962,11 +1165,18 @@ def print_done(out: Output, title: str, where: str) -> None: def print_baseline(out: Output, report: BaselineReport) -> None: - """What ``ferro migrate baseline`` did. Its warnings on stderr first. - The plain form is :meth:`BaselineReport.render`. On a TTY: the drift it - found (and that nothing was recorded); or an ok heading with the step - count, one line per data step it listed without running (its file - coloured by kind), and the not-compared note, dim.""" + """Print what `ferro migrate baseline` did. + + Its warnings come first, on stderr. The plain form prints the report's + `render()`. A TTY prints the drift it found (and that nothing was + recorded); or an ok heading with the step count, one line per data step + it listed without running (its file coloured by kind), and the + not-compared note, dim. + + Args: + out: The command's output. + report: The baseline's report. + """ from ..migrations._baseline import NOT_COMPARED, _span for warning in report.warnings: @@ -992,8 +1202,12 @@ def print_baseline(out: Output, report: BaselineReport) -> None: def print_removed(out: Output, removed: list[str]) -> None: - """What ``baseline --remove`` removed: :func:`render_removed` in the - plain form, under an ok glyph on a TTY.""" + """Print what `baseline --remove` removed, under an ok glyph on a TTY. + + Args: + out: The command's output. + removed: The migrations whose baseline was removed. + """ from ..migrations._baseline import render_removed if out.plain: @@ -1006,10 +1220,17 @@ def print_removed(out: Output, removed: list[str]) -> None: def print_rerecord(out: Output, report: RerecordReport) -> None: - """What ``ferro migrate rerecord`` changed: :meth:`RerecordReport.render` - in the plain form; on a TTY an ok heading naming the file (coloured by - kind) and that nothing was run, the two checksums with an arrow, and - what the mode leaves the next ``up``, dim.""" + """Print what `ferro migrate rerecord` changed. + + The plain form prints the report's `render()`. A TTY prints an ok + heading naming the file (coloured by kind) and that nothing was run, the + two checksums with an arrow, and what the mode leaves the next `up`, + dim. + + Args: + out: The command's output. + report: The re-record's report. + """ if out.plain: out.print(report.render()) return @@ -1029,9 +1250,17 @@ def print_rerecord(out: Output, report: RerecordReport) -> None: def print_init(out: Output, block: str, written: str, created: str) -> None: - """What ``ferro migrate init`` wrote: the config ``block`` as written, - then ``Wrote <written> and created <created>`` in the plain form; on a - TTY the block dim, then an ok heading.""" + """Print what `ferro migrate init` wrote. + + The config block prints as written (dim on a TTY), then + `Wrote <written> and created <created>` (an ok heading on a TTY). + + Args: + out: The command's output. + block: The config block appended. + written: The file written, as the line names it. + created: The directories created, as the line names them. + """ for line in block.splitlines(): out.print(Text(line, style="dim") if not out.plain else line) if out.plain: diff --git a/src/ferro/columns.py b/src/ferro/columns.py index 016c7a50..6b3fee97 100644 --- a/src/ferro/columns.py +++ b/src/ferro/columns.py @@ -1,4 +1,4 @@ -"""Column specs: each column fact derived exactly once (GLOSSARY.md "Column spec"). +"""Column specs: each column fact derived exactly once. One frozen ``ColumnSpec`` per column, built from the field declaration at class-body time (provisional) and replaced during relationship resolution @@ -56,13 +56,19 @@ def primary_key_field_name(model_name: str, specs: dict[str, ColumnSpec]) -> str is legal; zero is a valid declaration (``None``) and the operations that need a PK raise their own errors. + Args: + model_name: The model's name, for the error message. + specs: The model's column specs, keyed by column name. + + Returns: + The primary key's field name, or `None` when no column is one. + Raises: - TypeError: If more than one column declares ``primary_key=True`` — - at class definition time, naming the model and offending fields, - rather than resolving the ambiguity silently by declaration order - (the follow-up ADR-0002 asked for). ``TypeError`` is the - declaration-error contract the metaclass surfaces unwrapped - (ADR-0003's db_type validation sets the precedent). + TypeError: More than one column declares `primary_key=True`. It is + raised at class definition, naming the model and the fields, + rather than settling the ambiguity by declaration order; + `TypeError` is the declaration error the metaclass surfaces + unwrapped. """ pks = [name for name, spec in specs.items() if spec.primary_key] if len(pks) > 1: @@ -75,93 +81,128 @@ def primary_key_field_name(model_name: str, specs: dict[str, ColumnSpec]) -> str @dataclass(frozen=True, slots=True) class ForeignKeyRef: + """The target of a column's foreign key.""" + to_table: str + """The referenced table's name.""" on_delete: str | None + """The `ON DELETE` action, such as `"CASCADE"`, or `None` for the database's default.""" @dataclass(frozen=True, slots=True) class RelationSpec: - """One declared forward-FK relation's traversal facts (#268). + """One declared forward foreign-key relation's traversal facts. - Built alongside ``ColumnSpec`` at the same compile choke point - (``compile_model_schema_ir``) and exposed as - ``cls.__ferro_relation_specs__``. Relation-traversal query planning (PRD - #267) resolves an attribute access like ``t.account`` to its target model - and shadow FK column via this lookup, hop by hop. + Built alongside `ColumnSpec` at the same compile choke point + (`compile_model_schema_ir`) and exposed as + `cls.__ferro_relation_specs__`. Relation-traversal query planning + resolves an attribute access like `post.author` to its target model and + shadow foreign-key column through this lookup, hop by hop. """ field_name: str + """The relation field's name on the declaring model.""" target: type + """The model the relation points at.""" shadow_column: str + """The `<field>_id` column holding the target's primary key.""" nullable: bool + """Whether the shadow column is nullable.""" @dataclass(frozen=True, slots=True) class ReverseSpec: - """One reverse (BackRef/M2M) relation's existence-test facts (ADR-0007). - - Built beside :class:`RelationSpec` at the same compile choke point and - exposed as ``cls.__ferro_reverse_specs__``. A reverse or M2M relation - supports exactly one predicate form — the existence test - ``t.rel.exists()`` — so the spec carries only what a correlated EXISTS - needs: the related model plus the correlation columns (the child-side FK - for a reverse FK; the join-table triple for M2M, #316). Reverse relations - are *tested*, never *traversed*; traversal facts stay on - :class:`RelationSpec`. + """One reverse (`BackRef` or many-to-many) relation's existence-test facts. + + Built beside `RelationSpec` at the same compile choke point and exposed + as `cls.__ferro_reverse_specs__`. A reverse or many-to-many relation + supports one predicate form, the existence test + `author.posts.exists()`, so the spec carries only what a correlated + `EXISTS` needs: the related model and the correlation columns (the + child-side foreign key for a reverse relation, the join-table triple for + a many-to-many one). Reverse relations are tested, never traversed; + traversal facts stay on `RelationSpec`. """ field_name: str - #: The related model — the FK-declaring child (reverse FK) or the M2M - #: target: the model the inner lambda's parameter resolves against. + """The relation field's name on this model.""" target: type - #: Shadow FK column on the child table the EXISTS correlates against - #: (reverse FK only; ``None`` for M2M). + """The related model, which the inner lambda's parameter resolves against. + + It is the child that declares the foreign key for a reverse relation, + or the far side of a many-to-many one. + """ child_fk_column: str | None - #: True when the child FK is ``unique=True`` (one-to-one BackRef). The - #: rendering is identical at every cardinality (always correlated EXISTS); - #: carried as a compile-side fact, not a render switch. + """The child table's foreign-key column the `EXISTS` correlates on; `None` for many-to-many.""" is_one_to_one: bool = False - #: True for an M2M edge — the existence test correlates through the join - #: table (two hops) instead of a child FK (one hop). + """Whether the child's foreign key is `unique=True`, a one-to-one `BackRef`. + + The rendering is the same correlated `EXISTS` at every cardinality; this + is a compile-side fact, not a render switch. + """ is_m2m: bool = False - #: M2M join-table triple (``None`` for reverse FK): the association - #: table, its column referencing THIS side, and its column referencing - #: the target side — exactly as the descriptor orients them per side. + """Whether the test correlates through a join table (two hops) instead of a child's foreign key.""" join_table: str | None = None + """Many-to-many only: the join table's name.""" source_col: str | None = None + """Many-to-many only: the join table's column referencing this side.""" target_col: str | None = None + """Many-to-many only: the join table's column referencing the target side.""" @dataclass(frozen=True, slots=True) class ColumnSpec: + """Every fact about one column, derived once from the field declaration.""" + name: str + """The column's name.""" logical_type: str + """The JSON-schema type the column's storage is derived from, such as `"integer"`.""" nullable: bool + """Whether the column accepts `NULL`.""" primary_key: bool + """Whether the column is the table's primary key.""" autoincrement: bool + """Whether the database generates the column's values.""" unique: bool + """Whether the column has a single-column unique constraint.""" index: bool + """Whether the column has a single-column index.""" default: Any + """The JSON-schema default, or `None` when the field has none.""" format: str | None - # The scalar Python type, ``Annotated``-stripped. NOTE: producers differ — - # ``build_column_specs`` stores the stripped annotation, which may still be - # a union (e.g. ``int | None`` for an optional PK); ``pk_spec`` / - # ``fk_shadow_spec`` store the fully-unwrapped scalar. Consumers that need a - # bare scalar must unwrap (``_scalar_part_of_annotation`` / union handling). + """The JSON-schema format, such as `"uuid"` or `"date-time"`, or `None`.""" python_type: Any + """The field's Python type, with `Annotated` stripped. + + Producers differ: `build_column_specs` stores the stripped annotation, + which may still be a union (`int | None` for an optional primary key); + `pk_spec` and `fk_shadow_spec` store the bare scalar. A consumer that + needs the scalar unwraps it (`_scalar_part_of_annotation`, or union + handling). + """ enum_values: tuple[Any, ...] | None = None + """An enum column's values, or `None`.""" enum_type_name: str | None = None + """An enum column's Postgres type name, or `None`.""" enum_class: type[Enum] | None = None - #: ``SecretStr`` / ``SecretBytes`` when the field is declared as one: it - #: stores the raw value and hydrates wrapped in this class again. + """An enum column's `Enum` class, or `None`.""" secret_class: type[SecretStr] | type[SecretBytes] | None = None + """The `SecretStr` or `SecretBytes` class of a secret column, or `None`: the column stores the raw value and hydrates it wrapped in this class again.""" db_type: str | None = None + """The column's SQL type token, or `None` for the derived type.""" db_type_explicit: bool = False + """Whether `db_type` was declared rather than derived.""" db_check: bool = False + """Whether the column has a `CHECK` constraint on an enum's values.""" foreign_key: ForeignKeyRef | None = None - #: A retired column (ADR-0053): in the schema snapshot, nullable, never in - #: ``__ferro_columns__`` (the mapped columns the model reads and writes). + """The column's foreign key, or `None`.""" retired: bool = False + """Whether the column is retired. + + A retired column stays in the schema snapshot, nullable, and is never in + `__ferro_columns__`, the mapped columns the model reads and writes. + """ def _derive_autoincrement( @@ -379,11 +420,17 @@ def _enum_values(col_info: dict[str, Any]) -> list[Any] | None: def build_column_specs(model_cls: type[Any]) -> dict[str, ColumnSpec]: - """Build one ColumnSpec per schema property, in model-field order. + """Build one `ColumnSpec` per schema property, in model-field order. - Reads Pydantic's ``model_json_schema()`` exactly once — the only place in - Ferro that consumes it (grilling Q2). Returns a plain dict; callers store - it on ``cls.__ferro_columns__`` and must replace, never mutate. + Reads Pydantic's `model_json_schema()` exactly once, the only place in + Ferro that consumes it. Callers store the result on + `cls.__ferro_columns__` and must replace it, never mutate it. + + Args: + model_cls: The model class. + + Returns: + The column specs, keyed by column name. """ schema = model_cls.model_json_schema() properties = schema.get("properties") @@ -549,7 +596,7 @@ def build_column_specs(model_cls: type[Any]) -> dict[str, ColumnSpec]: def build_relation_specs( model_cls: type[Any], columns: dict[str, ColumnSpec] ) -> dict[str, RelationSpec]: - """Build one RelationSpec per declared forward-FK relation (#268). + """Build one `RelationSpec` per declared forward foreign-key relation. Reads ``ferro_relations`` (the same class-body-registered relation metadata ``build_column_specs`` reads via ``fk_by_shadow``) and pairs each @@ -565,6 +612,13 @@ def build_relation_specs( Callers store the result on ``cls.__ferro_relation_specs__`` and must replace, never mutate — same convention as ``__ferro_columns__``. + + Args: + model_cls: The model class. + columns: The model's compiled column specs, keyed by column name. + + Returns: + One spec per resolved forward relation, keyed by field name. """ ferro_relations = getattr(model_cls, "ferro_relations", {}) or {} specs: dict[str, RelationSpec] = {} @@ -588,24 +642,30 @@ def build_relation_specs( def build_reverse_specs(model_cls: type[Any]) -> dict[str, ReverseSpec]: - """Build one ReverseSpec per resolved reverse (BackRef) relation (#314). + """Build one `ReverseSpec` per resolved reverse (`BackRef`) relation. - Reads the :class:`~ferro.relations.descriptors.RelationshipDescriptor` + Reads the `RelationshipDescriptor` instances ``resolve_relationships`` installs on the class — the facts a correlated EXISTS needs (related model, child FK column or join-table triple, one-to-one flag) already live there, so this derives, never re-computes. At provisional class-body time no descriptors exist yet and the map is empty; the resolved second pass recompiles with descriptors installed, so the lookup is complete once resolution finishes (same - lifecycle as :func:`build_relation_specs`). + lifecycle as `build_relation_specs`). - M2M descriptors (#316) map to a join-table spec; each side's descriptor + Many-to-many descriptors map to a join-table spec; each side's descriptor already orients ``source_col``/``target_col`` for that side, so the spec is a straight copy of resolution's facts on both the declaring and the ``related_name`` side. Callers store the result on ``cls.__ferro_reverse_specs__`` and must replace, never mutate — same convention as ``__ferro_columns__``. + + Args: + model_cls: The model class. + + Returns: + One spec per resolved reverse relation, keyed by field name. """ # Late imports: columns.py is a low-level leaf module; the relations # package (and the registry it pulls in) imports back into it. @@ -654,10 +714,16 @@ def build_reverse_specs(model_cls: type[Any]) -> dict[str, ReverseSpec]: def pk_spec(name: str, python_type: Any) -> ColumnSpec: - """Spec for a column typed like a primary key (join-table columns). + """Build the spec of a column typed like a primary key, for join tables. + + A `None` or unknown type falls back to a string column. + + Args: + name: The column's name. + python_type: The primary key's Python type. - Mirrors the historical ``schema_fragment_for_pk`` mapping, including the - fall-back-to-string for None/unknown types. + Returns: + A non-null, non-key column spec of that type. """ scalar = ( _scalar_part_of_annotation(python_type) if python_type is not None else None @@ -692,7 +758,22 @@ def fk_shadow_spec( index: bool = False, nullable: bool = False, ) -> ColumnSpec: - """FK column spec at a known target — join tables and resolve-time rebuilds.""" + """Build a foreign-key column's spec at a known target. + + Join tables and the resolve-time rebuilds use it. + + Args: + name: The column's name. + python_type: The target primary key's Python type. + to_table: The target table's name. + on_delete: The `ON DELETE` action. Defaults to `"CASCADE"`. + unique: Add a unique constraint. Defaults to `False`. + index: Add an index. Defaults to `False`. + nullable: Allow `NULL`. Defaults to `False`. + + Returns: + The column spec. + """ return replace( pk_spec(name, python_type), foreign_key=ForeignKeyRef(to_table=to_table, on_delete=on_delete), @@ -703,12 +784,18 @@ def fk_shadow_spec( def retired_column_specs(model_cls: type[Any]) -> dict[str, ColumnSpec]: - """The retired columns of ``model_cls`` (ADR-0053), flagged and nullable. + """Return the retired columns of `model_cls`, flagged and nullable. A retired scalar field's spec was derived once at class creation, before - the field left the Pydantic model (``__ferro_retired_fields__``); a retired - ``ForeignKey``'s shadow column is derived here on every compile, as the + the field left the Pydantic model (`__ferro_retired_fields__`); a retired + `ForeignKey`'s shadow column is derived here on every compile, as the mapped relation's would be, so it follows its target's resolution. + + Args: + model_cls: The model class. + + Returns: + The retired columns' specs, keyed by column name. """ specs: dict[str, ColumnSpec] = dict( model_cls.__dict__.get("__ferro_retired_fields__", {}) @@ -741,10 +828,19 @@ def retired_column_specs(model_cls: type[Any]) -> dict[str, ColumnSpec]: def retired_clause(model_cls: type[Any], name: str) -> str: - """`` 'bio' is retired in this release.`` when ``name`` is one of - ``model_cls``'s retired fields or columns (ADR-0053), else ``""``: the - clause the usual unknown-column errors carry, so the reader learns the - name is gone on purpose, not misspelled.""" + """Return the clause an unknown-column error adds for a retired name. + + The usual unknown-column errors carry it, so the reader learns the name + is gone on purpose, not misspelled. + + Args: + model_cls: The model class. + name: The field or column name the error is about. + + Returns: + A sentence saying `name` is retired, or `""` when it is not one of + the model's retired fields or columns. + """ retired = getattr(model_cls, "__ferro_retired_names__", frozenset()) if name not in retired: return "" @@ -757,15 +853,20 @@ def retired_clause(model_cls: type[Any], name: str) -> str: def refuse_back_ref_over_retired_relation( model_cls: type[Any], field_name: str, meta: ForeignKey ) -> None: - """Refuse a ``BackRef`` that reads a retired relation back (ADR-0053). + """Refuse a `BackRef` that reads a retired relation back. A retired ``ForeignKey`` leaves the model, so nothing fills the target's ``BackRef``; it renders no DDL, and the fix is to delete it. Checked when the target is a concrete class: at class creation when the target already exists, and again by relationship resolution for a forward reference. + Args: + model_cls: The model declaring the retired relation. + field_name: The retired relation's field name. + meta: The retired relation's `ForeignKey`. + Raises: - TypeError: The target declares ``meta.related_name`` as a ``BackRef``. + TypeError: The target declares `meta.related_name` as a `BackRef`. """ target = meta.to if meta.related_name is None or not isinstance(target, type): diff --git a/src/ferro/composite_indexes.py b/src/ferro/composite_indexes.py index 2423595c..c076422a 100644 --- a/src/ferro/composite_indexes.py +++ b/src/ferro/composite_indexes.py @@ -11,12 +11,13 @@ from typing import Any FERRO_COMPOSITE_INDEXES = "__ferro_composite_indexes__" +"""The class attribute a model declares its composite indexes in.""" def _normalized_groups(cls: type[Any]) -> tuple[tuple[str, ...], ...]: """Return validated, de-duplicated composite-index column groups. - ``cls`` must be a :class:`ferro.models.Model` subclass; the base defines + ``cls`` must be a `ferro.models.Model` subclass; the base defines ``__ferro_composite_indexes__`` so normal access is always valid. Duplicate identical ordered column tuples are dropped; the first occurrence @@ -78,10 +79,23 @@ def _validate_groups_against_columns( def normalized_composite_indexes( cls: type[Any], column_names: frozenset[str] ) -> tuple[tuple[str, ...], ...]: - """Return validated, de-duplicated composite-index column groups for ``cls``. + """Return the validated, de-duplicated composite-index column groups of `cls`. The single validate-and-normalize choke point for composite indexes: both - the SchemaIR compiler and the legacy dict pipeline route through this. + the schema compiler and the legacy dict pipeline route through it. + + Args: + cls: The model class. + column_names: The model's column names, which every group must use. + + Returns: + The column groups, in declaration order. + + Raises: + TypeError: The declaration is not a tuple of tuples of non-empty + strings. + ValueError: A group names fewer than two columns, or a column the + model does not have. """ groups = _normalized_groups(cls) if not groups: @@ -93,12 +107,31 @@ def normalized_composite_indexes( def validate_composite_indexes_against_properties( cls: type[Any], properties: dict[str, Any] ) -> None: - """Ensure each referenced column exists on the JSON schema ``properties``.""" + """Check that every column a group names is a JSON-schema property. + + Args: + cls: The model class. + properties: The model's JSON-schema properties. + + Raises: + TypeError: The declaration is not a tuple of tuples of non-empty + strings. + ValueError: A group names fewer than two columns, or a column the + model does not have. + """ normalized_composite_indexes(cls, frozenset(properties.keys())) def ferro_composite_indexes_for_json(cls: type[Any]) -> list[list[str]] | None: - """Serialize composite groups for the Rust registry / Alembic, or ``None`` to omit.""" + """Serialize the composite-index groups of `cls` for the Rust registry and Alembic. + + Args: + cls: The model class. + + Returns: + The groups as nested lists, since JSON has no tuple, or `None` when + the model declares none. + """ groups = _normalized_groups(cls) if not groups: return None @@ -115,6 +148,14 @@ def drop_overlap_with_uniques( Same ordered column tuple in both the composite-unique and composite-index groups is redundant: the unique constraint already creates an underlying index. Reordered tuples are kept (different leftmost-prefix optimization). + + Args: + indexes: The model's composite-index groups. + uniques: The model's composite-unique groups. + model_name: The model's name, for the warning. + + Returns: + The index groups that no unique group duplicates. """ if not indexes or not uniques: return tuple(indexes) diff --git a/src/ferro/composite_uniques.py b/src/ferro/composite_uniques.py index d9bba24a..62e873d0 100644 --- a/src/ferro/composite_uniques.py +++ b/src/ferro/composite_uniques.py @@ -10,12 +10,13 @@ from typing import Any FERRO_COMPOSITE_UNIQUES = "__ferro_composite_uniques__" +"""The class attribute a model declares its composite uniques in.""" def _normalized_groups(cls: type[Any]) -> tuple[tuple[str, ...], ...]: """Return validated, de-duplicated composite-unique column groups. - ``cls`` must be a :class:`ferro.models.Model` subclass; the base defines + ``cls`` must be a `ferro.models.Model` subclass; the base defines ``__ferro_composite_uniques__`` so normal access is always valid. Duplicate identical column tuples are dropped; the first occurrence is kept. @@ -76,10 +77,23 @@ def _validate_groups_against_columns( def normalized_composite_uniques( cls: type[Any], column_names: frozenset[str] ) -> tuple[tuple[str, ...], ...]: - """Return validated, de-duplicated composite-unique column groups for ``cls``. + """Return the validated, de-duplicated composite-unique column groups of `cls`. The single validate-and-normalize choke point for composite uniques: both - the SchemaIR compiler and the legacy dict pipeline route through this. + the schema compiler and the legacy dict pipeline route through it. + + Args: + cls: The model class. + column_names: The model's column names, which every group must use. + + Returns: + The column groups, in declaration order. + + Raises: + TypeError: The declaration is not a tuple of tuples of non-empty + strings. + ValueError: A group names fewer than two columns, or a column the + model does not have. """ groups = _normalized_groups(cls) if not groups: @@ -91,12 +105,31 @@ def normalized_composite_uniques( def validate_composite_uniques_against_properties( cls: type[Any], properties: dict[str, Any] ) -> None: - """Ensure each referenced column exists on the JSON schema ``properties``.""" + """Check that every column a group names is a JSON-schema property. + + Args: + cls: The model class. + properties: The model's JSON-schema properties. + + Raises: + TypeError: The declaration is not a tuple of tuples of non-empty + strings. + ValueError: A group names fewer than two columns, or a column the + model does not have. + """ normalized_composite_uniques(cls, frozenset(properties.keys())) def ferro_composite_uniques_for_json(cls: type[Any]) -> list[list[str]] | None: - """Serialize composite groups for the Rust registry / Alembic, or ``None`` to omit.""" + """Serialize the composite-unique groups of `cls` for the Rust registry and Alembic. + + Args: + cls: The model class. + + Returns: + The groups as nested lists, since JSON has no tuple, or `None` when + the model declares none. + """ groups = _normalized_groups(cls) if not groups: return None diff --git a/src/ferro/exceptions.py b/src/ferro/exceptions.py index 32849666..a5e69c76 100644 --- a/src/ferro/exceptions.py +++ b/src/ferro/exceptions.py @@ -3,45 +3,57 @@ The tree is DBAPI-shaped so callers can catch database failures by type instead of string-matching driver messages: -- :class:`FerroError` — root of everything Ferro raises for database work. - - :class:`InterfaceError` — the database interface was misused - (unknown connection name, protocol/configuration problems). - - :class:`OperationalError` — the database or its environment failed +- `FerroError`: the root of everything Ferro raises for database work. + - `InterfaceError`: the database interface was misused (unknown + connection name, protocol or configuration problems). + - `OperationalError`: the database or its environment failed (connection refused, pool timeout, I/O or TLS errors). - - :class:`DataError` — a value could not be decoded or converted. - - :class:`IntegrityError` — a constraint rejected the statement. - - :class:`UniqueViolationError` - - :class:`ForeignKeyViolationError` - - :class:`NotNullViolationError` - - :class:`CheckViolationError` - - :class:`ModelDoesNotExist` — lookup by primary key found no row - (also a :class:`LookupError` for backwards compatibility). - -Exceptions raised from the Rust core preserve the original driver -message and, where the backend provides them, the SQLSTATE code and the -violated constraint name as structured attributes. + - `DataError`: a value could not be decoded or converted. + - `IntegrityError`: a constraint rejected the statement. + - `UniqueViolationError` + - `ForeignKeyViolationError` + - `NotNullViolationError` + - `CheckViolationError` + - `ModelDoesNotExist`: no row matched the primary key (also a + `LookupError`). + +Exceptions raised from the Rust core keep the driver's message and, where +the backend reports them, the SQLSTATE code and the violated constraint's +name as attributes. """ from typing import Any class FerroError(Exception): - """Root of Ferro's database exception hierarchy. - - Attributes: - driver_message: Original message from the database driver, when the - error originated in the database. ``None`` otherwise. - sqlstate: Backend error code as reported by the driver: the - five-character SQLSTATE (e.g. ``"23505"``) on Postgres, the - extended result code (e.g. ``"2067"``) on SQLite. ``None`` when - the driver reports none. - constraint: Name of the violated constraint when the backend reports - it (Postgres does; SQLite does not). ``None`` otherwise. + """The root of every exception Ferro raises for database work. + + Catch it to handle any database failure, or a subclass to handle one + kind. + + Examples: + ```python {test=run prelude=reference} + from ferro import UniqueViolationError + + try: + await Author.create(name="Ada King", email="ada@example.com") + except UniqueViolationError as error: + assert error.sqlstate == "2067" + else: + raise AssertionError("expected UniqueViolationError") + ``` """ driver_message: str | None + """The database driver's own message, or `None` when the error did not come from the database.""" sqlstate: str | None + """The backend's error code, or `None` when the driver reports none. + + On Postgres it is the five-character SQLSTATE, such as `"23505"`; on + SQLite the extended result code, such as `"2067"`. + """ constraint: str | None + """The violated constraint's name on Postgres, or `None` (SQLite does not report it).""" def __init__( self, @@ -99,34 +111,52 @@ class IntegrityError(FerroError): class UniqueViolationError(IntegrityError): - """A UNIQUE or PRIMARY KEY constraint rejected the statement.""" + """A `UNIQUE` or `PRIMARY KEY` constraint rejected the statement.""" class ForeignKeyViolationError(IntegrityError): - """A FOREIGN KEY constraint rejected the statement. + """A `FOREIGN KEY` constraint rejected the statement. - Includes a parent delete blocked by ``ForeignKey(on_delete="RESTRICT")``. - PostgreSQL 18 reports SQLSTATE ``23001`` for that case; PostgreSQL 17 - reports ``23503``. ``sqlstate`` is the raw driver code. + Includes a parent delete blocked by `ForeignKey(on_delete="RESTRICT")`. + PostgreSQL 18 reports SQLSTATE `23001` for that case; PostgreSQL 17 + reports `23503`. `sqlstate` is the raw driver code. """ class NotNullViolationError(IntegrityError): - """A NOT NULL constraint rejected the statement.""" + """A `NOT NULL` constraint rejected the statement.""" class CheckViolationError(IntegrityError): - """A CHECK constraint rejected the statement.""" + """A `CHECK` constraint rejected the statement.""" class ModelDoesNotExist(FerroError, LookupError): - """Raised when :meth:`~ferro.models.Model.get` finds no row for the primary - key, and when :meth:`~ferro.models.Model.save` on a persisted instance - matches no row (the row was deleted underneath, or the primary key was - mutated before saving).""" + """No row matched the primary key. + + [`Model.get`][ferro.Model.get] raises it when no row has the key, and + [`save`][ferro.Model.save] on a saved instance raises it when its row is + gone (deleted, or its primary key changed before saving). It is also a + `LookupError`. Use [`get_or_none`][ferro.Model.get_or_none] to get `None` + instead. + + Examples: + ```python {test=run prelude=reference} + from ferro import ModelDoesNotExist + + try: + await Author.get(99) + except ModelDoesNotExist as error: + assert (error.model, error.pk) == (Author, 99) + else: + raise AssertionError("expected ModelDoesNotExist") + ``` + """ model: type + """The model class that was looked up.""" pk: Any + """The primary key that matched no row.""" def __init__(self, model_cls: type, pk: Any) -> None: self.model = model_cls diff --git a/src/ferro/fields.py b/src/ferro/fields.py index 846fbe02..dd0c5dad 100644 --- a/src/ferro/fields.py +++ b/src/ferro/fields.py @@ -1,4 +1,4 @@ -"""Expose Ferro's wrapped Field helper on top of Pydantic.""" +"""`Field`, `BackRef` and `ManyToMany`: Pydantic's `Field` with Ferro's database options.""" from __future__ import annotations @@ -27,6 +27,7 @@ from pydantic.fields import Deprecated, FieldInfo FERRO_FIELD_EXTRA_KEY = "ferro_field" +"""The `json_schema_extra` key under which `Field` stores its database options.""" _T = TypeVar("_T") @@ -415,32 +416,57 @@ def Field( fail_fast: bool | None = _Unset, **extra: Unpack[_EmptyKwargs], ) -> Any: - """Build field metadata with Pydantic and Ferro options + """Declare a model field with Pydantic's options and Ferro's database options. + + `Field` takes every argument of Pydantic's own `Field` and adds the + database options Ferro reads: primary key, uniqueness, indexes, column + type, relations and rename hints. Use it as the field's default, or + inside `Annotated[...]`. See + [Field options](../guide/models-and-fields.md#field-options) for each + database option at work. Args: - default: Default value used when the field is not set. - primary_key: Mark this column as the table primary key in Ferro. - autoincrement: Override automatic increment behavior for primary key columns. - When not provided, Ferro infers this for integer primary keys. - unique: Add a **single-column** uniqueness constraint for this column in Ferro. - Multi-column uniqueness is declared with ``__ferro_composite_uniques__`` on the model. - index: Request an index for this column in Ferro. - back_ref: Mark this field as a reverse relationship. This is the lower-level - equivalent of assigning ``BackRef()`` as the field default. - many_to_many: Mark this field as a many-to-many relationship. - related_name: Reverse relationship field name used by many-to-many relationships. - through: Optional join table name used by many-to-many relationships. - nullable: Alembic ``Column.nullable`` override for :func:`~ferro.migrations.get_metadata`. - ``\"infer\"`` (default) derives nullability from the field annotation. - renamed_from: The column's previous name. ``ferro migrate new`` renders - ``RENAME COLUMN`` (and renames every index and check named after the - column) while the previous migration still holds that name; after - that migration the hint is inert and may stay or be deleted. - retired: Retire the column (ADR-0053): the field leaves the model (no - attribute, no keyword, no statement the model issues names it) and - the column stays in the schema snapshot, nullable, until the - declaration is deleted in a later release. The annotation must - allow ``None``. + default: The value the field takes when it is not set. + primary_key: Make the column the table's primary key. + autoincrement: Whether the database generates the key. When not + given, an integer primary key auto-increments. + unique: Add a unique constraint on this column alone. For uniqueness + across several columns, declare `__ferro_composite_uniques__` on + the model (see + [Composite uniques](../guide/models-and-fields.md#composite-uniques)). + index: Add an index on the column. + back_ref: Mark the field as the reading side of a relation. Assigning + [`BackRef()`][ferro.BackRef] is the usual spelling. + many_to_many: Mark the field as a many-to-many relation. + [`ManyToMany(...)`][ferro.ManyToMany] is the usual spelling. + related_name: The name of the relation on the target model that reads + this one back. + through: The many-to-many join table's name. When not given, Ferro + derives one from the two models. + reverse_index: Add an index on the many-to-many join table's + `(target, source)` columns for queries from the other side. When + not given, the index is added. + nullable: `"infer"` (the default) makes the column nullable when the + annotation allows `None`; `True` or `False` forces `NULL` or + `NOT NULL` in every schema Ferro emits, whatever the annotation + says. + db_type: The column's SQL type, overriding the one Ferro derives from + the annotation: a [`DbTypeToken`][ferro.DbTypeToken] or a + [`varchar(N)`][ferro.varchar] name. When not given, the column + takes the derived type. A type the annotation cannot be stored + as is refused when the class is defined. + db_check: Add a `CHECK` constraint, named `ck_<table>_<column>`, that + limits the column to an enum's values. Needs `db_type`, since the + default native enum type already enforces them. Defaults to + `False`. + renamed_from: The column's previous name, so that `ferro migrate new` + renames the column, and every index and check named after it, + instead of dropping and adding it. See + [Rename hints](../guide/schema/renames-and-retirements.md#rename-hints). + retired: Take the column out of the model while it stays in the + schema, nullable, until a later release deletes the declaration. + The annotation must allow `None`. Defaults to `False`. See + [Retiring a column](../guide/schema/renames-and-retirements.md#retiring-a-column). default_factory: A callable to generate the default value. The callable can either take 0 arguments (in which case it is called as is) or a single argument containing the already validated data. alias: The name to use for the attribute when validating or serializing by alias. @@ -470,7 +496,7 @@ def Field( (Only applies to dataclasses.) coerce_numbers_to_str: Whether to enable coercion of any `Number` type to `str` (not applicable in `strict` mode). strict: If `True`, strict validation is applied to the field. - See [Strict Mode](../concepts/strict_mode.md) for details. + See [Strict Mode](https://docs.pydantic.dev/latest/concepts/strict_mode/). gt: Greater than. If set, value must be greater than this. Only applicable to numbers. ge: Greater than or equal. If set, value must be greater than or equal to this. Only applicable to numbers. lt: Less than. If set, value must be less than this. Only applicable to numbers. @@ -480,33 +506,53 @@ def Field( max_length: Maximum length for iterables. pattern: Pattern for strings (a regular expression). allow_inf_nan: Allow `inf`, `-inf`, `nan`. Only applicable to float and [`Decimal`][decimal.Decimal] numbers. - max_digits: Maximum number of allow digits for strings. + max_digits: Maximum number of allowed digits for strings. decimal_places: Maximum number of decimal places allowed for numbers. union_mode: The strategy to apply when validating a union. Can be `smart` (the default), or `left_to_right`. - See [Union Mode](../concepts/unions.md#union-modes) for details. + See [Union Mode](https://docs.pydantic.dev/latest/concepts/unions/#union-modes). fail_fast: If `True`, validation will stop on the first error. If `False`, all validation errors will be collected. This option can be applied only to iterable types (list, tuple, set, and frozenset). - extra: (Deprecated) Extra fields that will be included in the JSON schema. - - !!! warning Deprecated - The `extra` kwargs is deprecated. Use `json_schema_extra` instead. + **extra: Deprecated: extra fields to include in the JSON schema. Use + `json_schema_extra` instead. Returns: - A new [`FieldInfo`][pydantic.fields.FieldInfo]. The return annotation is `Any` so `Field` can be used on - type-annotated fields without causing a type error. + A new [`FieldInfo`][pydantic.fields.FieldInfo]. The return annotation + is `Any` so that `Field` type-checks as any field's default. Raises: - TypeError: If Ferro kwargs are provided together with callable `json_schema_extra`. + TypeError: A database option is given together with a callable + `json_schema_extra`, or `nullable`, `renamed_from` or `retired` + has a value of the wrong type. Examples: - >>> from typing import Annotated - >>> from ferro import Field, Model - >>> class User(Model): - ... id: int | None = Field(default=None, primary_key=True) - ... username: str = Field(unique=True, min_length=3) - >>> class UserAnnotated(Model): - ... id: Annotated[int | None, Field(default=None, primary_key=True)] - ... username: Annotated[str, Field(unique=True, min_length=3)] + ```python {test=run prelude=reference} + from ferro import Field, Model, create_tables + + + class Reader(Model): + id: int | None = Field(default=None, primary_key=True) + handle: str = Field(unique=True, min_length=3) + nickname: str | None = Field(default=None, index=True) + + + await create_tables() + reader = await Reader.create(handle="ada") + assert reader.id == 1 + ``` + + The same model with the options inside `Annotated`: + + ```python {test=run prelude=reference} + from typing import Annotated + + from ferro import Field, Model + + + class Reader(Model): + id: Annotated[int | None, Field(primary_key=True)] = None + handle: Annotated[str, Field(unique=True, min_length=3)] + nickname: Annotated[str | None, Field(index=True)] = None + ``` """ ferro_kwargs: dict[str, Any] = {} if primary_key is not _Unset: @@ -597,9 +643,20 @@ def Field( class BackRef: - """Declare a reverse relationship field. + """The reading side of a relation, assigned as the field's default. + + `BackRef()` is `Field(back_ref=True)`. Pair it with the + [`ForeignKey`][ferro.ForeignKey] or [`ManyToMany`][ferro.ManyToMany] + whose `related_name` names this field; the field adds no column. It + takes the keyword arguments of [`Field`][ferro.Field] except + `reverse_index`, which belongs on the `ManyToMany` side. - ``BackRef()`` is a convenience wrapper around ``Field(back_ref=True)``. + Examples: + ```python {test=run prelude=reference} + ada = await Author.where(lambda author: author.name == "Ada Lovelace").first() + titles = [post.title for post in await ada.posts.all()] + assert titles == ["Notes on the Analytical Engine"] + ``` """ def __new__(cls, **kwargs: Any) -> Any: @@ -625,15 +682,49 @@ def ManyToMany( reverse_index: bool = True, **kwargs: Any, ) -> Any: - """Declare a many-to-many relationship field. + """Declare a many-to-many relation, assigned as the field's default. + + Ferro creates the join table. Annotate the field + `Relation[list[Target]]`, and give the target a + [`BackRef`][ferro.BackRef] named by `related_name`. See + [Many-to-many](../guide/relationships.md#many-to-many). Args: - related_name: Name for reverse access from the related model. - through: Optional explicit join table name. When omitted, Ferro - generates a join table name automatically. - reverse_index: When True (default), the synthesized join table gets - a non-unique composite index on ``(target_col, source_col)`` to - optimize back-ref queries. Set to False to opt out. + related_name: The name of the relation on the target model that reads + this one back. + through: The join table's name. Defaults to `None`, a name Ferro + derives from the two models. + reverse_index: Add an index on the join table's `(target, source)` + columns for queries from the other side. Defaults to `True`; turn + it off for a write-heavy join table. + **kwargs: Other keyword arguments of [`Field`][ferro.Field]. + + Returns: + The field's metadata, as [`Field`][ferro.Field] returns it. + + Examples: + ```python {test=run prelude=reference} + from ferro import BackRef, Field, ManyToMany, Model, Relation, create_tables + + + class Tag(Model): + id: int | None = Field(default=None, primary_key=True) + label: str + posts: Relation[list["Article"]] = BackRef() + + + class Article(Model): + id: int | None = Field(default=None, primary_key=True) + title: str + tags: Relation[list[Tag]] = ManyToMany(related_name="posts") + + + await create_tables() + history = await Tag.create(label="history") + article = await Article.create(title="Babbage's engines") + await article.tags.add(history) + assert [tag.label for tag in await article.tags.all()] == ["history"] + ``` """ return Field( diff --git a/src/ferro/metaclass.py b/src/ferro/metaclass.py index ac02b36b..c4894211 100644 --- a/src/ferro/metaclass.py +++ b/src/ferro/metaclass.py @@ -1,3 +1,5 @@ +"""The metaclass that turns a model class declaration into a registered table.""" + import re from dataclasses import replace from functools import partial @@ -144,20 +146,26 @@ def _written_retired_label(value: Any, labels: frozenset[str]) -> str | None: def refuse_retired_label( enum_cls: type[Enum], labels: frozenset[str], value: Any ) -> Any: - """Refuse ``value`` if it writes one of ``enum_cls``'s retired ``labels``. + """Refuse a value that would write one of an enum's retired labels. + + Every path that writes an enum column refuses a retired label the same + way: Pydantic validation calls this function, and + [`refuse_retired_label_write`][ferro.metaclass.refuse_retired_label_write] + calls it for the writes that skip validation. Each surfaces as a + `ValidationError` of type `retired_enum_label`. See + [Retiring an enum label](../guide/schema/renames-and-retirements.md#retiring-an-enum-label). - The one retired-label refusal (ADR-0053), whichever path writes: the - field's ``AfterValidator`` raises it from inside Pydantic validation - (construction, and assignment under ``validate_assignment``), and - :func:`refuse_retired_label_write` carries it out of the paths that run - no validation. Every one surfaces as a ``ValidationError`` whose error - type is ``retired_enum_label``. + Args: + enum_cls: The column's enum class. + labels: The enum's retired labels. + value: The value being written: a member or a raw label. Returns: - ``value`` unchanged when it writes no retired label. + `value`, unchanged, when it writes no retired label. Raises: - PydanticCustomError: ``retired_enum_label``, naming the member. + PydanticCustomError: The value writes a retired label; the error + names the member. """ label = _written_retired_label(value, labels) if label is None: @@ -171,15 +179,20 @@ def refuse_retired_label( def refuse_retired_label_write(model_cls: type, column: str, value: Any) -> None: - """Refuse a write of a retired label to ``column`` that runs no validation. + """Refuse a retired enum label written without Pydantic validation. - Attribute assignment without ``validate_assignment`` and ``update()`` - (keyword and recipe) set a column without Pydantic validating it, so - they ask here; the refusal is :func:`refuse_retired_label`'s, raised as - the ``ValidationError`` validation would have raised. + Assigning an attribute (without `validate_assignment`) and `update()` + write a column without validating it, so they check here and raise the + `ValidationError` that validation would have raised. + + Args: + model_cls: The model class being written. + column: The column being written. + value: The value being written. Raises: - ValidationError: ``retired_enum_label`` at ``(column,)``. + ValidationError: The value writes a retired label of the column's + enum, with error type `retired_enum_label` at `(column,)`. """ enum_cls = getattr(model_cls, "_enum_fields", {}).get(column) if enum_cls is None: @@ -203,12 +216,11 @@ def __setattr__(self: BaseModel, name: str, value: Any) -> None: class ModelMetaclass(type(BaseModel)): - """ - Metaclass for Ferro models that provisionally registers schema in Python caches. + """The metaclass of [`Model`][ferro.Model], which records each model's schema when the class is defined. - Class-body registration writes the Python registry, SchemaIR envelope cache, - and generation counter only. Rust registration is installed at the first - bulk sync (``connect``/``create_tables``/``migrate``) — ADR #242 / #246. + Defining a model class checks its declaration and records its table in + the model registry. The database learns about it on the next + `connect()`, `create_tables()` or `migrate()`. """ def __new__(mcs, name, bases, namespace, **kwargs): @@ -631,7 +643,7 @@ def _retire_relation( The relation gets no descriptor and no shadow ``*_id`` field, so the model has neither attribute; its column is compiled from the - ``ForeignKey`` alone (:func:`ferro.columns.retired_column_specs`). + ``ForeignKey`` alone (`ferro.columns.retired_column_specs`). """ if metadata.renamed_from is not None: raise _retired_and_renamed(model_name, field_name) diff --git a/src/ferro/migrations/__init__.py b/src/ferro/migrations/__init__.py index 6e327a52..105bd7cf 100644 --- a/src/ferro/migrations/__init__.py +++ b/src/ferro/migrations/__init__.py @@ -1,27 +1,26 @@ """Ferro migrations: the in-process calls and the Alembic bridge. -Each call here is the CLI verb's own function: ``up()`` is ``ferro migrate -up``, ``status()`` is ``ferro migrate status``, ``check()`` is ``ferro -migrate check``. Each takes ``settings=None, database=None`` (the project's -configuration and which of its databases) and, when it works on a live -database, ``using=`` (an open connection) or ``url=`` (a private one); the -default connection otherwise (:class:`~ferro.migrations.target.Target`). -``down`` and ``rerecord`` stay CLI verbs: they are operator decisions with -prompts. A refusal is raised, never returned: an application that called -``up()`` must not go on serving a database the run did not reach. +Each call here is the CLI verb's own function: `up()` is `ferro migrate up`, +`status()` is `ferro migrate status`, `check()` is `ferro migrate check`. +Each takes `settings=None, database=None` (the project's configuration and +which of its databases) and, when it works on a live database, `using=` (an +open connection) or `url=` (a private one); the default connection +otherwise. `down` and `rerecord` stay CLI verbs: they are operator +decisions with prompts. A refusal is raised, never returned: an application +that called `up()` must not go on serving a database the run did not reach. -```python +```python {test=compile reason="needs a project with a migrations directory"} await ferro.connect(url) await ferro.migrations.up() # a desktop app with no deploy step await ferro.migrations.require_applied() # a server that migrates in its deploy step ``` -A data step declares its shape with ``atomic`` / ``chunked`` (and its down -with ``irreversible`` / ``nothing_to_reverse``), and marks what only a -person can supply with ``todo("…")`` (ADR-0035). +A data step declares its shape with `atomic` / `chunked` (and its down with +`irreversible` / `nothing_to_reverse`), and marks what only a person can +supply with `todo("…")`. -``get_metadata``, ``ferro_options`` and ``render_item`` (the Alembic bridge) are loaded on -first use, so the migration calls never import Alembic. +`get_metadata`, `ferro_options` and `render_item` (the Alembic bridge) are +loaded on first use, so the migration calls never import Alembic. """ from __future__ import annotations @@ -32,7 +31,7 @@ from ._drift import DriftReport, drift, render_op from .errors import DatabaseAheadError, MigrationRefused, PendingMigrationsError from .generate import check -from .report import AlreadyTrackedError, AppliedAboveBaselineError +from .report import AlreadyTrackedError, AppliedAboveBaselineError, RunRefused from .runner import require_applied, status, up from .steps import atomic, chunked, irreversible, nothing_to_reverse, todo @@ -47,6 +46,7 @@ "DriftReport", "MigrationRefused", "PendingMigrationsError", + "RunRefused", "atomic", "baseline", "check", diff --git a/src/ferro/migrations/_baseline.py b/src/ferro/migrations/_baseline.py index 07d97956..4d32f2d7 100644 --- a/src/ferro/migrations/_baseline.py +++ b/src/ferro/migrations/_baseline.py @@ -1,5 +1,4 @@ -"""``ferro migrate baseline``: adopt migrations on a database that already -has their schema (ADR-0031). +"""`ferro migrate baseline`: adopt migrations on a database that already has their schema. ```text $ ferro migrate baseline @@ -13,19 +12,18 @@ 0007_nickname pending ``` -A database that ``connect(auto_migrate=True)`` or Alembic built already has -the tables ``0001`` creates. ``baseline`` records every step of every -migration through the target as applied without running it, and only after -checking: under the run lock, with no step records yet, the live database -is planned against the target migration's schema snapshot exactly as -``ferro migrate drift`` plans it against the last applied one: both call -:func:`ferro.migrations._drift.against`, so the lines are the same. Any -line means nothing is recorded; there is no flag that records past one. -Live tables the snapshot does not declare (``alembic_version``, a later -migration's tables) are never drift. - -``baseline --remove`` deletes the baseline's records again, refused while a -run has applied a migration above them; ``down`` never reverts a baselined +A database that `connect(auto_migrate=True)` or Alembic built already has +the tables `0001` creates. `baseline` records every step of every migration +through the target as applied without running it, and only after checking: +under the run lock, with no step records yet, the live database is planned +against the target migration's schema snapshot exactly as +`ferro migrate drift` plans it against the last applied one, so the lines +are the same. Any line means nothing is recorded; there is no flag that +records past one. Live tables the snapshot does not declare +(`alembic_version`, a later migration's tables) are never drift. + +`baseline --remove` deletes the baseline's records again, refused while a +run has applied a migration above them; `down` never reverts a baselined migration, whose down would drop tables it never created. """ @@ -52,28 +50,26 @@ "baseline checked what `ferro migrate drift` checks: objects ferro does not " "own (functions, triggers) were not compared" ) -"""What a successful baseline says it did not check (ADR-0031).""" +"""What a successful baseline says it did not check.""" @dataclass(frozen=True) class BaselineReport: - """What ``baseline`` did. - - ``recorded`` names every migration it recorded (``NNNN_<name>``), empty - when the drift check found something; ``data_steps_listed`` every data - step it recorded without running (``NNNN_<name>/<file>``); ``drift`` the - drift check's report when it found something, ``None`` when the - database matched the target's snapshot. - """ + """What [`baseline`][ferro.migrations.baseline] did: what it recorded, or the drift that stopped it.""" recorded: list[str] + """Every migration it recorded (`NNNN_<name>`); empty when the drift check found something.""" data_steps_listed: list[str] + """Every data step it recorded without running (`NNNN_<name>/<file>`).""" drift: DriftReport | None + """The drift check's report when it found something; `None` when the database matched the target's snapshot.""" steps: int = 0 """How many step records were written.""" reports: list[dict[str, Any]] = field(default_factory=list) - """What the drift check reported without counting it as drift (a - foreign or unverifiable row policy, ...), as ``drift`` reports it.""" + """What the drift check reported without counting it as drift, as `drift` reports it. + + A foreign or unverifiable row policy, for example. + """ @property def warnings(self) -> list[str]: @@ -81,7 +77,11 @@ def warnings(self) -> list[str]: return [report["text"] for report in self.reports] def render(self) -> str: - """The text ``ferro migrate baseline`` prints on stdout.""" + """Render the text `ferro migrate baseline` prints on stdout. + + Returns: + The drift lines and `nothing was recorded`, or what was recorded. + """ if self.drift is not None: return f"{self.drift.render()}\nnothing was recorded" data = len(self.data_steps_listed) @@ -94,15 +94,25 @@ def render(self) -> str: return "\n".join(lines) def raise_for_problems(self) -> None: - """Raise :class:`MigrationRefused` naming every drift line unless the - baseline was recorded.""" + """Raise unless the baseline was recorded. + + Raises: + MigrationRefused: The drift check found something; the message + names every drift line. + """ if self.drift is not None: raise MigrationRefused(self.render()) def render_removed(removed: list[str]) -> str: - """The text ``ferro migrate baseline --remove`` prints for the migrations - whose baseline it removed.""" + """Render the text `ferro migrate baseline --remove` prints. + + Args: + removed: The migrations whose baseline was removed. + + Returns: + `no baseline to remove`, or the span of migrations removed. + """ if not removed: return "no baseline to remove" return f"removed the baseline of {_span(removed)}" @@ -191,25 +201,47 @@ async def baseline( url: str | None = None, lock_timeout: str | float | None = None, ) -> BaselineReport: - """Record every migration through ``target`` (the head when ``None``; - ``"0006"`` or ``"0006_add_teams"``) as applied on a database that already - has their schema, under the run lock (``ferro migrate baseline``). - - The database is checked against the target's schema snapshot first, as - :func:`~ferro.migrations.drift` checks it; when that finds anything, - nothing is recorded and the report's ``drift`` lists it - (``.raise_for_problems()`` raises with every line). There is no - override (ADR-0031). Works on ``using`` (an open connection), on a - private connection to ``url``, or on the default connection. + """Record migrations as applied on a database that already has their schema. + + This is `ferro migrate baseline`. Under the run lock, the database is + checked against the target's schema snapshot first, as + [`drift`][ferro.migrations.drift] checks it; when that finds anything, + nothing is recorded and the report's `drift` lists it. There is no + override. See + [Adopting migrations](../guide/schema/adopting-migrations.md). + + Args: + settings: The project's configuration. Defaults to `FerroSettings()`. + database: The configured database. Defaults to the only one + configured. + target: The last migration to record, `"0006"` or + `"0006_add_teams"`. Defaults to the newest migration. + using: An open connection to work on. Defaults to the default + connection. + url: A URL to open a private connection to, closed after the call. + Refused together with `using`. + lock_timeout: How long to wait for another run's lock: `"30s"`, + `"500ms"`, `"1m"` or a number of seconds; `0` refuses at once. + Defaults to the database's `lock_timeout`. + + Returns: + What was recorded, or the drift that stopped it; its + `raise_for_problems()` raises with every drift line. Raises: - AlreadyTrackedError: the database is already tracked (its tracking - table holds step records): ``applied`` names them, ``head`` the - newest, ``report`` is its status. Another pre-deploy may have - adopted it first; carry on to ``up()``. - MigrationRefused: the target is not in the migrations directory, the + AlreadyTrackedError: The database is already tracked (its tracking + table holds step records). Another pre-deploy may have adopted + it first; carry on to `up()`. + MigrationRefused: The target is not in the migrations directory, the directory holds no migration, the lock wait outlasts - ``lock_timeout``, or the configuration names no single database. + `lock_timeout`, or the configuration names no single database. + + Examples: + ```python {test=compile reason="needs a migrations project and a database"} + report = await ferro.migrations.baseline(target="0006") + report.raise_for_problems() + print(report.recorded[-1]) # 0006_add_teams + ``` """ where = Target.resolve(settings, database, using=using, url=url) async with where.open() as name: @@ -224,16 +256,32 @@ async def remove_baseline( url: str | None = None, lock_timeout: str | float | None = None, ) -> list[str]: - """Delete every record a baseline wrote, under the run lock (``ferro - migrate baseline --remove``). Returns the migrations whose baseline was - removed (``NNNN_<name>``), empty when there was none; they are pending + """Delete every record a baseline wrote, under the run lock. + + This is `ferro migrate baseline --remove`. The migrations are pending afterwards. + Args: + settings: The project's configuration. Defaults to `FerroSettings()`. + database: The configured database. Defaults to the only one + configured. + using: An open connection to work on. Defaults to the default + connection. + url: A URL to open a private connection to, closed after the call. + Refused together with `using`. + lock_timeout: How long to wait for another run's lock: `"30s"`, + `"500ms"`, `"1m"` or a number of seconds; `0` refuses at once. + Defaults to the database's `lock_timeout`. + + Returns: + The migrations whose baseline was removed (`NNNN_<name>`); empty + when there was none. + Raises: - AppliedAboveBaselineError: a run applied a migration above the - baseline (``above`` names each; revert them with ``ferro migrate - down`` first, as the message says). - MigrationRefused: the lock wait outlasts ``lock_timeout``, or the + AppliedAboveBaselineError: A run applied a migration above the + baseline; revert it with `ferro migrate down` first, as the + message says. + MigrationRefused: The lock wait outlasts `lock_timeout`, or the configuration names no single database. """ target = Target.resolve(settings, database, using=using, url=url) diff --git a/src/ferro/migrations/chunked.py b/src/ferro/migrations/_chunk_loop.py similarity index 60% rename from src/ferro/migrations/chunked.py rename to src/ferro/migrations/_chunk_loop.py index fbbacfb6..db47f109 100644 --- a/src/ferro/migrations/chunked.py +++ b/src/ferro/migrations/_chunk_loop.py @@ -1,6 +1,9 @@ -"""The chunk loop: a chunked data step run in batches that resumes (ADR-0024). +"""The chunk loop: a chunked data step run in batches that resumes. -```python +The public declaration is [`chunked`][ferro.migrations.chunked]; this +private module runs it. + +```python {test=compile reason="a data step file; the migration runner calls it"} @chunked( lambda models: models.Author.where(lambda author: author.slug == None) .order_by(lambda author: author.id), @@ -13,36 +16,38 @@ async def up(ctx, batch): ``` Over 2,500 authors the runner opens three transactions. Each reads the next -1,000 rows after the cursor (keyset ``after()`` over the query's order -keys), hands them to ``up(ctx, batch)``, and commits, in that same -transaction, the step record's cursor and ``rows_done``:: - - batch 1 rows 1-1000 resume_cursor {"keys": [1000], "order_by": ["author.id"], "rows_done": 1000} - batch 2 rows 1001-2000 resume_cursor {"keys": [2000], "order_by": ["author.id"], "rows_done": 2000} - batch 3 rows 2001-2500 the finished record, rows_done 2500 +1,000 rows after the cursor (keyset `after()` over the query's order keys), +hands them to `up(ctx, batch)`, and commits, in that same transaction, the +step record's cursor and `rows_done`: + +```text +batch 1 rows 1-1000 resume_cursor {"keys": [1000], "order_by": ["author.id"], "rows_done": 1000} +batch 2 rows 1001-2000 resume_cursor {"keys": [2000], "order_by": ["author.id"], "rows_done": 2000} +batch 3 rows 2001-2500 the finished record, rows_done 2500 +``` A batch that fails rolls back alone: the batches before it stay committed with their cursor, and the next run starts after it, so no row is replayed. -The last batch (fewer rows than ``batch_size``, possibly none) writes the +The last batch (fewer rows than `batch_size`, possibly none) writes the finished record going up, or removes the record going down. A chunked down -pages its own query the same way, marking the record ``reverting`` with its -own cursor (``revert_cursor``) until that last batch (ADR-0033). +pages its own query the same way, marking the record `reverting` with its +own cursor (`revert_cursor`) until that last batch. A failed down is recorded only where the database moved (the tracking table says where it stands now). A down that rolled back completely (an atomic down, a transactional SQL down, a chunked down failing on its first -batch) leaves the record unchanged: the step stays ``applied`` and the -error is in the run's output. A down that left part of itself applied -writes the error onto the record: a no-transaction SQL down, and a chunked -down failing after a committed batch, which stays ``reverting`` at its -cursor so ``up`` refuses until a ``down`` finishes it. +batch) leaves the record unchanged: the step stays `applied` and the error +is in the run's output. A down that left part of itself applied writes the +error onto the record: a no-transaction SQL down, and a chunked down +failing after a committed batch, which stays `reverting` at its cursor so +`up` refuses until a `down` finishes it. The cursor is the last row's order-key tuple as JSON, each value in the form -``canonicalize_wire_scalar`` gives a query literal (I-13), so ``after()`` -compares a resumed cursor exactly as ``save()`` wrote the row. It names the -order keys it is a position in (``order_by``, :func:`order_keys`), so an -edited step can be continued from it only while its query still pages over -the same ones (``ferro migrate rerecord --continue``, ADR-0030). +a query literal takes, so `after()` compares a resumed cursor exactly as +`save()` wrote the row. It names the order keys it is a position in +(`order_by`, from `order_keys`), so an edited step can be continued from it +only while its query still pages over the same ones +(`ferro migrate rerecord --continue`). """ from __future__ import annotations @@ -90,10 +95,10 @@ class ChunkOutcome: class BatchFailed(Exception): - """A batch raised and rolled back; everything before it stays committed. + """The error for a batch that raised and rolled back; everything before it stays committed. - Carries what the record holds now, so the runner can write the failure - beside the committed cursor without moving it. + It carries what the record holds now, so the runner can write the + failure beside the committed cursor without moving it. """ def __init__( @@ -107,15 +112,19 @@ def __init__( ) -> None: super().__init__(str(error)) self.error = error + """The error the batch raised.""" self.cursor = cursor - """The last committed cursor (``None`` before any row).""" + """The last committed cursor (`None` before any row).""" self.rows_done = rows_done + """The rows the walk has committed, across every run of it.""" self.batches = batches """The batches this run committed before the failing one.""" self.committed = committed - """Whether the walk has committed anything on the record: going up - always (the started record); going down once the record is - ``reverting``, from this run or an earlier one.""" + """Whether the walk has committed anything on the record. + + Going up always (the started record); going down once the record is + `reverting`, from this run or an earlier one. + """ # -- the cursor codec -------------------------------------------------------------- @@ -136,12 +145,23 @@ def _cursor_value(value: Any) -> Any: def encode_cursor( keys: Sequence[Any], rows_done: int, *, order_by: Sequence[str] ) -> str: - """The cursor JSON for the last row's order-key values ``keys`` under the - order keys ``order_by`` (:func:`order_keys`): ``{"keys": [...], - "order_by": ["author.id"], "rows_done": N}``, each value in its - query-literal form (``canonicalize_wire_scalar``, I-13). ``order_by`` - is what ``ferro migrate rerecord --continue`` compares an edited step's - query against (ADR-0030).""" + """Encode a cursor as JSON: `{"keys": [...], "order_by": ["author.id"], "rows_done": N}`. + + Each value is in its query-literal form. `order_by` is what + `ferro migrate rerecord --continue` compares an edited step's query + against. + + Args: + keys: The last row's order-key values. + rows_done: The rows the walk has committed. + order_by: The order keys, as `order_keys` spells them. + + Returns: + The cursor JSON. + + Raises: + TypeError: An order-key value is bytes, which a cursor cannot hold. + """ return json.dumps( { "keys": [_cursor_value(key) for key in keys], @@ -154,10 +174,23 @@ def encode_cursor( def decode_cursor( text: str, types: Sequence[Any], order_by: Sequence[str] | None = None ) -> tuple[tuple[Any, ...], int]: - """The order-key values and ``rows_done`` of cursor JSON ``text``, each - value validated back to its order key's annotation in ``types``. - ``order_by`` (the query's :func:`order_keys`) must be the cursor's own - when the cursor records them.""" + """Decode cursor JSON into its order-key values and `rows_done`. + + Args: + text: The cursor JSON. + types: Each order key's annotation; every value is validated back + to its key's annotation. + order_by: The query's order keys, as `order_keys` spells them. When + given and the cursor records its own, they must match. Defaults + to no check. + + Returns: + The order-key values and the rows the walk has committed. + + Raises: + StepRefused: The query's order keys changed since the cursor was + committed. + """ cursor = json.loads(text) keys = cursor["keys"] recorded = cursor.get("order_by") @@ -181,11 +214,19 @@ def decode_cursor( def order_keys(query: Query[Any]) -> list[str]: - """The order keys a chunked query pages over, as a cursor records them: - ``<table>.<column>``, then `` desc`` for a descending key and - `` nulls first`` where nulls sort first (``["author.created_at desc", - "author.id"]``). Two queries with the same list page in the same order, - so a cursor committed under one is a position in the other.""" + """Spell the order keys a chunked query pages over, as a cursor records them. + + Each key is `<table>.<column>`, then ` desc` for a descending key and + ` nulls first` where nulls sort first (`["author.created_at desc", + "author.id"]`). Two queries with the same list page in the same order, + so a cursor committed under one is a position in the other. + + Args: + query: The chunked query. + + Returns: + The spelled order keys, in order. + """ model = query.model_cls table = getattr(model, "__ferro_table__", None) or model.__name__.lower() spelled = [] @@ -216,30 +257,37 @@ async def run_chunked( direction: Literal["up", "down"], using: str | None = None, ) -> ChunkOutcome: - """Run ``declared`` (a ``@chunked`` step function) to its last batch. - - ``ctx_factory(tx)`` is the step's context on one batch's transaction; - ``step`` is the step's handle on the locked ``run``, which carries the - cursor an earlier attempt committed (``step.resume_cursor``: going up the - started record's, going down a reverting record's). Batches run on - connection ``using``. Each batch's record moves through the run's - transitions inside the batch's transaction, each verifying the run lock - first (ADR-0029 as amended): a lost lock commits nothing and propagates - untouched. - - Going up a batch advances ``resume_cursor`` (``run.advance``) and the - last batch writes the finished record (``run.finish``); going down a - batch advances ``revert_cursor`` with the record marked ``reverting`` - (``run.advance_revert``), and the last batch removes the record - (``run.remove``). Each batch is ``transaction(immediate=True)``: on - SQLite a ``BEGIN IMMEDIATE`` that holds the write lock before the batch - reads, so its read-then-write cannot fail upgrading into - ``SQLITE_BUSY`` (ADR-0024); on Postgres a plain ``BEGIN``. + """Run a `@chunked` step function to its last batch. + + Each batch's record moves through the run's transitions inside the + batch's transaction, each verifying the run lock first: a lost lock + commits nothing and propagates untouched. Going up, a batch advances + `resume_cursor` and the last batch writes the finished record; going + down, a batch advances `revert_cursor` with the record marked + `reverting`, and the last batch removes the record. Each batch is + `transaction(immediate=True)`: on SQLite a `BEGIN IMMEDIATE` that holds + the write lock before the batch reads, so its read-then-write cannot + fail upgrading into `SQLITE_BUSY`; on Postgres a plain `BEGIN`. + + Args: + ctx_factory: Builds the step's context on one batch's transaction. + declared: The `@chunked` step function and its shape. + run: The locked database the run holds. + step: The step's handle on `run`, carrying the cursor an earlier + attempt committed (going up the started record's, going down a + reverting record's). + direction: `"up"` or `"down"`. + using: The connection the batches run on. Defaults to the default + connection. + + Returns: + The rows the walk has committed and the batches this run committed. Raises: - BatchFailed: a batch raised; it rolled back, and what the record + TypeError: `declared` is not a `@chunked` step. + BatchFailed: A batch raised; it rolled back, and what the record holds is on the exception. - RunRefused: the run lock was lost; the batch rolled back. + RunRefused: The run lock was lost; the batch rolled back. """ shape = declared.shape if not isinstance(shape, Chunked): diff --git a/src/ferro/migrations/_drift.py b/src/ferro/migrations/_drift.py index fe0674ac..6be48573 100644 --- a/src/ferro/migrations/_drift.py +++ b/src/ferro/migrations/_drift.py @@ -1,4 +1,4 @@ -"""``ferro migrate drift``: the live database against its last applied snapshot. +"""`ferro migrate drift`: the live database against its last applied snapshot. ```text $ ferro migrate drift @@ -10,23 +10,23 @@ no drift against 0007_nickname ``` -Drift is what the one planner would change to turn the live database into the -schema snapshot of the last migration applied to it (ADR-0023). Nothing here -compares live and declared state: one core call, ``_core._plan_drift``, reads -the live database into the planner's input, the snapshot's tables by the -pass's own rule (ADR-0047), and plans it against the snapshot with the one -planner, destructive changes on, so extra objects are reported too; each -planned op is printed as one line (:func:`render_op`). That comparison of the live database with one snapshot -is :func:`against`; ``baseline`` makes the same call with its target's -snapshot, so the two check exactly the same things. +Drift is what the planner would change to turn the live database into the +schema snapshot of the last migration applied to it. Nothing here compares +live and declared state: one core call reads the snapshot's tables from the +live database the way the auto-migrate pass reads them, and plans them +against the snapshot with destructive changes on, so extra objects are +reported too; each planned op is printed as one line (`render_op`). That +comparison of the live database with one snapshot is `against`; `baseline` +makes the same call with its target's snapshot, so the two check exactly +the same things. Only the snapshot's tables are read, so a live table the snapshot does not -declare (``alembic_version``, an extension's tables, the tracking tables) is -never drift (ADR-0031). A database mid-migration (a ``failed``, ``running``, -``interrupted`` or ``reverting`` step, or a run holding the lock) is at -neither snapshot, so drift refuses rather than report the run's own partial -work (ADR-0043); so does a database with no migration records, which has no -applied snapshot to drift from. ``drift`` takes no lock and writes nothing. +declare (`alembic_version`, an extension's tables, the tracking tables) is +never drift. A database mid-migration (a `failed`, `running`, `interrupted` +or `reverting` step, or a run holding the lock) is at neither snapshot, so +drift refuses rather than report the run's own partial work; so does a +database with no migration records, which has no applied snapshot to drift +from. `drift` takes no lock and writes nothing. """ from __future__ import annotations @@ -50,25 +50,23 @@ @dataclass(frozen=True) class DriftReport: - """What ``drift`` found. - - ``lines`` holds one plain-language line per planned op; ``operations`` - the planner's ops themselves (``{"kind": ..., <fields>}``). ``refusal`` - is why there was nothing to compare: a run is unfinished, or the - database has no migration records. Either way ``clean`` is false. - """ + """What [`drift`][ferro.migrations.drift] found: one line per difference, or why it could not compare.""" against: str | None - """The applied migration whose snapshot the database was compared with - (``NNNN_<name>``), or ``None`` when drift refused.""" + """The applied migration the database was compared with (`NNNN_<name>`), or `None` when drift refused.""" lines: list[str] = field(default_factory=list) + """One plain-language line per difference, such as `author.slug column is missing`.""" operations: list[dict[str, Any]] = field(default_factory=list) + """The planner's ops behind the lines, each `{"kind": ..., <fields>}`.""" refusal: str | None = None + """Why there was nothing to compare: a run is unfinished, or the database has no migration records.""" reports: list[dict[str, Any]] = field(default_factory=list) - """What the planner reports without planning an op for it (a foreign - or unverifiable row policy, ...), each ``{"kind", "subject", "text", - "recurs", "blocks"}``, in planning order. Never makes the report - unclean.""" + """What the planner reports without planning an op for it, in planning order. + + A foreign or unverifiable row policy, for example; each is + `{"kind", "subject", "text", "recurs", "blocks"}`. A report never makes + the drift report unclean. + """ @property def warnings(self) -> list[str]: @@ -77,11 +75,16 @@ def warnings(self) -> list[str]: @property def clean(self) -> bool: - """The database matches the snapshot it was compared with.""" + """Whether the database matches the snapshot it was compared with; false after a refusal.""" return self.refusal is None and not self.lines def render(self) -> str: - """The text ``ferro migrate drift`` prints on stdout (empty for a refusal).""" + """Render the text `ferro migrate drift` prints on stdout. + + Returns: + The drift lines under `drift against <migration>:`, `no drift + against <migration>`, or an empty string for a refusal. + """ if self.refusal is not None: return "" if not self.lines: @@ -91,8 +94,12 @@ def render(self) -> str: ) def raise_for_problems(self) -> None: - """Raise :class:`MigrationRefused` naming every line (or the refusal) - unless the report is clean.""" + """Raise unless the report is clean. + + Raises: + MigrationRefused: The report has drift lines (the message names + every one) or a refusal (the message is the refusal). + """ if self.refusal is not None: raise MigrationRefused(self.refusal) if self.lines: @@ -221,15 +228,34 @@ def _redefined(op: dict[str, Any]) -> str: def render_op(op: dict[str, Any]) -> str: - """One planner op as one plain-language line (``team.name column is - missing``). An op kind this module does not know still renders, as - ``<kind> on <table>``; a known kind missing one of its fields raises - ``ValueError`` naming both. - - ``AlterColumnType`` reads ``live_type`` / ``snapshot_type`` and - ``AlterColumnNullability`` reads ``live_nullable`` when the op carries - them (:func:`against` adds them); without them the line says only that the - two differ. + """Render one planner op as one plain-language line. + + An op kind this function does not know still renders, as + `<kind> on <table>`. `AlterColumnType` reads `live_type` / + `snapshot_type` and `AlterColumnNullability` reads `live_nullable` when + the op carries them; without them the line says only that the two + differ. + + Args: + op: The op, as `DriftReport.operations` holds it: + `{"kind": ..., <fields>}`. + + Returns: + The line, such as `author.slug column is missing`. + + Raises: + ValueError: A known kind is missing one of its fields; the message + names both. + + Examples: + ```python {test=run prelude=reference} + from ferro.migrations import render_op + + print(render_op({"kind": "AddColumn", "table": "author", "column": "slug"})) + # author.slug column is missing + print(render_op({"kind": "DropTable", "table": "post"})) + # post table is extra + ``` """ kind = op.get("kind", "unknown op") renderer = _RENDERERS.get(kind) @@ -340,21 +366,31 @@ def _refused(why: str) -> DriftReport: async def against( snapshot: dict[str, Any], *, migration: str, using: str ) -> DriftReport: - """Drift of the live database on open connection ``using`` against one - schema snapshot (the ``["snapshot"]["ir"]`` of a migration in the - directory), reported as against ``migration`` (``NNNN_<name>``). + """Compare the live database with one schema snapshot. - The check :func:`drift` makes against the last applied migration and - ``baseline`` makes against its target, and nothing else: only the - snapshot's tables are read (ADR-0031), destructive changes count, each - planned op is one :func:`render_op` line, and nothing is refused, locked - or written:: + This is the check `drift` makes against the last applied migration and + `baseline` makes against its target, and nothing else: only the + snapshot's tables are read, destructive changes count, each planned op + is one `render_op` line, and nothing is refused, locked or written. - report = await against(snapshot, migration="0007_nickname", using="default") - report.lines # ["user.nickname column is missing"] + Args: + snapshot: The schema snapshot, the `["snapshot"]["ir"]` of a + migration in the directory. + migration: The migration the report says it compared against + (`NNNN_<name>`). + using: The open connection to read. + + Returns: + The drift report. Raises: - MigrationRefused: ``using`` is not an open connection. + MigrationRefused: `using` is not an open connection. + + Examples: + ```python {test=compile reason="needs a migrations project and a database"} + report = await against(snapshot, migration="0007_nickname", using="default") + report.lines # ["user.nickname column is missing"] + ``` """ if _core.connection_backend(using) is None: raise MigrationRefused(f"connection `{using}` is not open; connect it first") @@ -433,22 +469,35 @@ async def drift( using: str | None = None, url: str | None = None, ) -> DriftReport: - """Compare the live database with the schema snapshot of the last - migration applied to it (``ferro migrate drift``). Takes no lock and - changes nothing. + """Compare the live database with the schema snapshot of the last migration applied to it. - Works on ``using`` (an open connection), on a private connection to - ``url``, or on the default connection. Raises nothing for drift or for - a database it cannot compare (mid-migration, no records): the report - says so, and ``.raise_for_problems()`` raises :class:`MigrationRefused` - with every line, so a fixture aborts in one line:: + This is `ferro migrate drift`. It takes no lock and changes nothing, and + raises nothing for drift or for a database it cannot compare + (mid-migration, no records): the report says so. - (await ferro.migrations.drift()).raise_for_problems() + Args: + settings: The project's configuration. Defaults to `FerroSettings()`. + database: The configured database. Defaults to the only one + configured. + using: An open connection to work on. Defaults to the default + connection. + url: A URL to open a private connection to, closed after the call. + Refused together with `using`. + + Returns: + The drift report; its `raise_for_problems()` raises with every line. Raises: - MigrationRefused: the configuration names no single database, there + MigrationRefused: The configuration names no single database, there is no connection to work on, or the migrations directory is unreadable. + + Examples: + A test fixture that aborts on drift, in one line: + + ```python {test=compile reason="needs a migrations project and a database"} + (await ferro.migrations.drift()).raise_for_problems() + ``` """ target = Target.resolve(settings, database, using=using, url=url) async with target.open() as name: diff --git a/src/ferro/migrations/alembic.py b/src/ferro/migrations/alembic.py index c27c3d1a..24cd8bde 100644 --- a/src/ferro/migrations/alembic.py +++ b/src/ferro/migrations/alembic.py @@ -1,21 +1,20 @@ -"""The Alembic bridge (ADR-0041): ferro's models as Alembic metadata, and one -comparator that writes what the one planner decides. +"""The Alembic bridge: Ferro's models as Alembic metadata, and one comparator that writes what the planner decides. -```python +```python {test=compile reason="an Alembic env.py fragment; Alembic runs it"} # env.py from ferro.migrations import ferro_options, get_metadata context.configure(connection=connection, target_metadata=get_metadata(), **ferro_options()) ``` -``alembic revision --autogenerate`` reads the live database through the -reconciliation pass's converter (``_core._live_schema_ir``), asks the core -for the whole revision in one call (``_core._plan_revision``: the upgrade -planned against the models with destructive changes on, the downgrade the -planner run back to the live database, and every refusal and marker) and -hands it to the translator (:mod:`ferro.migrations.translate`), which -decides nothing. An empty revision and "no drift" are the same statement. ``ferro_options()`` hides ferro's tables and both tracking -tables from Alembic's own comparator, so every op on a ferro table is the +`alembic revision --autogenerate` reads the live database the way the +auto-migrate pass reads it, asks the core for the whole revision in one +call (the upgrade planned against the models with destructive changes on, +the downgrade planned back to the live database, and every refusal and +marker) and hands it to the translator (`ferro.migrations.translate`), +which decides nothing. An empty revision and "no drift" are the same +statement. `ferro_options()` hides Ferro's tables and both tracking tables +from Alembic's own comparator, so every op on a Ferro table is the planner's; a project's own SQLAlchemy tables keep Alembic's comparison. """ @@ -76,7 +75,7 @@ def tables(self) -> list[str]: """The models' tables and every table a rename hint names as the old one: the tables Alembic's own comparator is kept off. Which of them the planner reads live is the live read's own rule - (``_live_schema_ir``, ADR-0047).""" + (``_live_schema_ir``).""" names = set() for model in self.envelope["payload"]["models"]: names.add(model["table_name"]) @@ -104,30 +103,42 @@ def _declared_modelset(database: "str | None") -> Dict[str, Any]: def get_metadata(database: "str | None" = None) -> "sa.MetaData": - """ - Generate a SQLAlchemy MetaData object representing the project's Ferro - models, for alembic's env.py together with :func:`ferro_options`. - - With a ferro configuration (``ferro.toml`` or ``[tool.ferro]``, read - through ``FerroSettings``) it imports the configured database's - ``models`` modules and returns only that database's tables; name the - database with ``database=`` when several are configured. A registered - model no configured database claims is refused by name. With no - configuration it renders every registered model. - - Enum columns are mapped to named ``sqlalchemy.Enum`` types so PostgreSQL - autogenerate and DDL compilation succeed (anonymous enums are rejected). - When the field annotation is a Python ``enum.Enum`` subclass, the database - type name defaults to the enum class name in lowercase; otherwise the - column name is used as the type name. - - **Column nullability:** ``Column.nullable`` follows :class:`~ferro.base.FerroField` - / :class:`~ferro.base.ForeignKey` ``nullable`` when set to a boolean (force - NULL / NOT NULL). The default ``nullable='infer'`` uses whether the Python - annotation allows ``None`` (after unwrapping ``Annotated``). Shadow ``*_id`` - columns infer from the **forward relation** field's annotation, not from the - synthetic ``*_id`` field. Primary key columns are always ``nullable=False``. - Pydantic "required" and JSON-schema defaults do not change inferred nullability. + """Build SQLAlchemy `MetaData` for the project's Ferro models, for Alembic's `env.py`. + + Pass it as `target_metadata` together with + [`ferro_options`][ferro.migrations.alembic.ferro_options]. With a Ferro + configuration (`ferro.toml` or `[tool.ferro]`) it imports the configured + database's `models` modules and returns only that database's tables; a + registered model no configured database claims is refused by name. With + no configuration it renders every registered model. + + Enum columns map to named `sqlalchemy.Enum` types, named after the enum + class in lowercase (or after the column when the annotation is not an + `enum.Enum`), so Postgres autogenerate and DDL compilation succeed. + + A column's `nullable` follows the `nullable` of its + [`FerroField`][ferro.FerroField] or [`ForeignKey`][ferro.ForeignKey] + when that is a boolean. The default, `nullable="infer"`, uses whether + the annotation allows `None`; a shadow `*_id` column infers from its + forward relation's annotation. Primary key columns are never nullable. + + Args: + database: The configured database whose tables to return. Defaults + to the only one configured; name it when several are. + + Returns: + The metadata, one table per model. + + Examples: + ```python {test=compile reason="an Alembic env.py"} + from ferro.migrations import ferro_options, get_metadata + + context.configure( + connection=connection, + target_metadata=get_metadata(), + **ferro_options(), + ) + ``` """ metadata = _naming_metadata() envelope = _declared_modelset(database) @@ -370,22 +381,24 @@ def _db_type_to_sa_type(token: str) -> "sa.types.TypeEngine | None": def render_item( type_: str, obj: Any, autogen_context: "AutogenContext" ) -> "str | Literal[False]": - """Alembic ``render_item`` hook for ferro's bridge, carried by - :func:`ferro_options` (which ``env.py`` passes to ``context.configure``). - - It renders one thing Alembic cannot: a native Postgres enum type the - revision must **not** create. Every enum type a revision needs is - created by the planner's own ``CREATE TYPE`` (or already exists), so a - ``create_table`` / ``add_column`` column of it is written - ``postgresql.ENUM(..., create_type=False)`` — or SQLAlchemy would issue - ``CREATE TYPE`` again and the upgrade would fail with ``DuplicateObject`` - (#443). ``repr()`` of that type silently drops the flag, and Alembic's - default type renderer is that ``repr``; this hook renders the flag and - adds the ``from sqlalchemy.dialects import postgresql`` import. - - Everything else returns ``False`` so Alembic's own renderers keep their - say. The signature is Alembic's ``RenderItemFn`` exactly — - ``(str, Any, AutogenContext) -> str | Literal[False]`` (#446). + """Render a native Postgres enum type the revision must not create, for Alembic. + + The planner creates every enum type a revision needs, so a column of one + is written `postgresql.ENUM(..., create_type=False)`. Everything else + returns `False`, so Alembic's own renderers keep their say. You do not + call this directly: + [`ferro_options`][ferro.migrations.alembic.ferro_options] passes it to + Alembic. + + Args: + type_: The kind of item Alembic is rendering, such as `"type"`. + obj: The item. + autogen_context: Alembic's autogenerate context, which collects the + revision's imports. + + Returns: + The rendered type, or `False` for anything this hook does not + render. """ if type_ != "type" or sa is None: return False @@ -414,7 +427,7 @@ def render_item( class _FerroObjectFilter: - """Alembic's ``include_object`` hook from :func:`ferro_options`: hides + """Alembic's ``include_object`` hook from `ferro_options`: hides ferro's tables (and everything on them) and both tracking tables from Alembic's own comparator, then asks the project's own filter, if any. @@ -448,23 +461,34 @@ def ferro_options( include_object: "_IncludeObject | None" = None, render_item: "_RenderItem | None" = None, ) -> dict[str, Any]: - """The options ``env.py`` passes to ``context.configure`` beside - ``get_metadata()``:: + """Return the options `env.py` passes to `context.configure` beside `get_metadata()`. + + The returned `include_object` keeps Alembic's own comparator off Ferro's + tables and both tracking tables, so every op on a Ferro table is the + planner's; the returned `render_item` writes a native enum type the + revision must not create. Autogenerate over `get_metadata()` without + these options is refused, naming this line. + + Args: + include_object: The project's own `include_object` hook. Ferro's + filter asks it about every object Ferro does not hide. + render_item: The project's own `render_item` hook. Ferro's renderer + falls through to it whenever Ferro's returns `False`. + + Returns: + The `include_object` and `render_item` keyword arguments for + `context.configure`. + + Examples: + ```python {test=compile reason="an Alembic env.py"} + from ferro.migrations import ferro_options, get_metadata context.configure( connection=connection, target_metadata=get_metadata(), **ferro_options(), ) - - ``include_object`` keeps Alembic's own comparator off ferro's tables and - both tracking tables, so every op on a ferro table is the planner's; - ``render_item`` writes a native enum type the revision must not create - (``postgresql.ENUM(..., create_type=False)``). A project with its own - hooks passes them here: ferro's filter asks the project's for every - object it does not hide, and ferro's renderer falls through to the - project's on ``False``. Autogenerate over ``get_metadata()`` without - these options is refused, naming this line. + ``` """ def rendered( diff --git a/src/ferro/migrations/context.py b/src/ferro/migrations/context.py index cc3be4db..46b2958d 100644 --- a/src/ferro/migrations/context.py +++ b/src/ferro/migrations/context.py @@ -1,6 +1,6 @@ -"""What a data step is handed: :class:`StepContext` and :class:`HistoricalModels`. +"""What a data step is handed: `StepContext` and `HistoricalModels`. -```python +```python {test=compile reason="a data step body; the migration runner calls it"} @atomic async def up(ctx): for author in await ctx.models.Author.where(lambda author: author.slug == None).all(): @@ -11,11 +11,11 @@ async def up(ctx): ctx.log.info("backfilled %d authors", ...) ``` -The context is closed (ADR-0035): the historical models, raw SQL on the -step's own transaction, the target dialect and a logger. It has no -``transaction()``, no database name, migration number or direction, and no -path to the models in the codebase today. A nested ``ferro.transaction()`` -inside a step is a savepoint, as in application code. +The context is closed: the historical models, raw SQL on the step's own +transaction, the target dialect and a logger. It has no `transaction()`, no +database name, migration number or direction, and no path to the models in +the codebase today. A nested `ferro.transaction()` inside a step is a +savepoint, as in application code. """ from __future__ import annotations @@ -34,10 +34,10 @@ async def up(ctx): class HistoricalModels: """The historical model of every table in a migration's snapshot union. - ``models.Author`` is a model by class name; ``models.table("author_tags")`` + `models.Author` is a model by class name; `models.table("author_tags")` reaches any table by table name, a many-to-many join table included (it never had a class of its own). A name the union lacks raises and lists - what it holds. Built by :func:`ferro.migrations.historical.build`. + what it holds. """ __slots__ = ("_by_name", "_by_table", "_registry_state", "_rev") @@ -48,8 +48,13 @@ def __init__( classes: dict[str, type[Model]], registry_state: RegistrySnapshot, ) -> None: - """``classes`` maps each table name to its historical class; - ``registry_state`` is the registry those classes registered into.""" + """Hold a migration's historical classes. + + Args: + rev: The migration these models belong to. + classes: Each table name mapped to its historical class. + registry_state: The registry those classes registered into. + """ self._rev = rev self._by_table = dict(sorted(classes.items())) self._by_name: dict[str, type[Model]] = {} @@ -63,25 +68,43 @@ def __init__( @property def rev(self) -> str: - """The migration these models belong to (``0011_backfill_slugs``).""" + """The migration these models belong to (`0011_backfill_slugs`).""" return self._rev @property def registry_state(self) -> RegistrySnapshot: - """The registry holding exactly these classes (what - :meth:`ferro.registry.Registry.swap` installs).""" + """The registry holding exactly these classes, which a step run installs.""" return self._registry_state def names(self) -> list[str]: - """The class names, sorted (a join table's class is named after it).""" + """List the class names, sorted (a join table's class is named after it). + + Returns: + The class names. + """ return sorted(self._by_name) def tables(self) -> list[str]: - """The table names, sorted.""" + """List the table names, sorted. + + Returns: + The table names. + """ return list(self._by_table) def table(self, name: str) -> type[Model]: - """The historical model of table ``name``.""" + """Return the historical model of a table, by table name. + + Args: + name: The table name, such as `"author_tags"`. + + Returns: + The table's historical model class. + + Raises: + LookupError: The union has no such table; the message lists the + tables it holds. + """ try: return self._by_table[name] except KeyError: @@ -91,8 +114,14 @@ def table(self, name: str) -> type[Model]: ) from None def unreachable(self, cls: FerroModelClass) -> str: - """Why today's ``cls`` cannot be queried while these models are - installed, naming the historical model to use instead.""" + """Explain why a model from today's codebase cannot be queried in a data step. + + Args: + cls: The model class from the codebase today. + + Returns: + The sentence naming the historical model to use instead. + """ name = cls.__name__ return ( f"{name} is the model in the codebase today, which a data step cannot " @@ -125,13 +154,11 @@ def __repr__(self) -> str: class StepContext: - """``ctx``: everything a data step can reach (ADR-0035). + """The `ctx` a data step receives: everything the step can reach. - ``models`` are the historical models; ``execute`` / ``fetch_all`` / - ``fetch_one`` are ferro's raw API on the step's own transaction (no - ``using``, ``session`` or ``autocommit``); ``dialect`` is the target - dialect (``"postgres"`` or ``"sqlite"``); ``log`` is a logger named for - the step. + `execute`, `fetch_all` and `fetch_one` are Ferro's + [raw SQL](../guide/raw-sql.md) calls on the step's own transaction, + without `using`, `session` or `autocommit`. """ __slots__ = ("_dialect", "_log", "_historical", "_tx") @@ -150,29 +177,54 @@ def __init__( @property def models(self) -> HistoricalModels: - """The historical models: ``ctx.models.Author``, ``ctx.models.table(...)``.""" + """The historical models: `ctx.models.Author`, `ctx.models.table(...)`.""" return self._historical @property def dialect(self) -> str: - """The target dialect: ``"postgres"`` or ``"sqlite"``.""" + """The target dialect: `"postgres"` or `"sqlite"`.""" return self._dialect @property def log(self) -> logging.Logger: - """A logger named for the step (``ferro.migrations.0011.01_backfill_author``).""" + """A logger named for the step (`ferro.migrations.0011.01_backfill_author`).""" return self._log async def execute(self, sql: str, *args: Any) -> int: - """Run a raw statement on the step's transaction; rows affected.""" + """Run a raw statement on the step's transaction. + + Args: + sql: The statement. + *args: Values bound to the statement's placeholders, in order. + + Returns: + The number of rows affected. + """ return await self._tx.execute(sql, *args) async def fetch_all(self, sql: str, *args: Any) -> list[dict[str, Any]]: - """Run a raw query on the step's transaction; every row as a dict.""" + """Run a raw query on the step's transaction and return every row. + + Args: + sql: The query. + *args: Values bound to the query's placeholders, in order. + + Returns: + Every row, as a dict keyed by column name. + """ return await self._tx.fetch_all(sql, *args) async def fetch_one(self, sql: str, *args: Any) -> dict[str, Any] | None: - """Run a raw query on the step's transaction; the first row, or ``None``.""" + """Run a raw query on the step's transaction and return its first row. + + Args: + sql: The query. + *args: Values bound to the query's placeholders, in order. + + Returns: + The first row as a dict keyed by column name, or `None` when there + is none. + """ return await self._tx.fetch_one(sql, *args) def __repr__(self) -> str: diff --git a/src/ferro/migrations/errors.py b/src/ferro/migrations/errors.py index f4fddeaf..4e2d99b2 100644 --- a/src/ferro/migrations/errors.py +++ b/src/ferro/migrations/errors.py @@ -1,6 +1,6 @@ -"""What the migration calls raise (ADR-0038). +"""What the migration calls raise. -```python +```python {test=compile reason="needs a database behind its migrations directory"} await ferro.connect(url) await ferro.migrations.require_applied() # PendingMigrationsError: ferro.migrations: this database is behind its migrations. @@ -8,14 +8,14 @@ # Run `ferro migrate up` (or `await ferro.migrations.up()`) before serving. ``` -Every refusal a migration call makes is a :class:`MigrationRefused`, so one -``except`` catches them all and the CLI renders any of them as exit 1. The +Every refusal a migration call makes is a `MigrationRefused`, so one +`except` catches them all and the CLI renders any of them as exit 1. The states an application branches on have their own subclasses: -:class:`PendingMigrationsError` (the database is behind the checkout) and -:class:`DatabaseAheadError` (the database holds migrations the checkout -lacks), here; a run planner refusal with a class of its own -(``AlreadyTrackedError``, ``AppliedAboveBaselineError``) is a ``RunRefused``, -beside it in :mod:`ferro.migrations.report`. +`PendingMigrationsError` (the database is behind the checkout) and +`DatabaseAheadError` (the database holds migrations the checkout lacks), +here; a run planner refusal with a class of its own (`AlreadyTrackedError`, +`AppliedAboveBaselineError`) is a `RunRefused`, beside it in +`ferro.migrations.report`. """ from __future__ import annotations @@ -34,20 +34,11 @@ class MigrationRefused(FerroError): - """A migration call refused, naming the fix. - - The base of every refusal: a run's (``RunRefused``), ``ferro migrate - check``'s problems (``MigrationsCheckError``), a configuration that - names no single database, and the subclasses an application branches - on. - - ``report`` is the report of the call that refused, when it has one: the - :class:`~ferro.migrations.runner.RunReport` of a refused ``up`` or - ``down`` (what it did before it stopped, and the refusal itself as - ``report.refused``), the ``CheckReport`` whose problems - ``raise_for_problems()`` raised, or the - :class:`~ferro.migrations.report.StatusReport` of the database a - baseline refusal is about. ``None`` otherwise. + """A migration call's refusal, naming the fix. + + The base of every refusal: a run's, `ferro migrate check`'s problems, a + configuration that names no single database, and the subclasses an + application branches on. """ def __init__( @@ -58,20 +49,26 @@ def __init__( ) -> None: super().__init__(message) self.report = report + """The report of the call that refused, or `None` when it has none. + The `RunReport` of a refused `up` (what it did before it stopped, + and the refusal itself as `report.refused`), the `CheckReport` whose + problems `raise_for_problems()` raised, or the `StatusReport` of the + database a baseline refusal is about. + """ -class PendingMigrationsError(MigrationRefused): - """``require_applied()``: the database is not at the head of its migrations. - ``pending`` names each migration with a step still to apply - (``NNNN_<name>``); ``refusals`` carries the tracking table's own - refusals (an interrupted run, an edited applied file, ...), each the - text ``ferro migrate up`` would print. - """ +class PendingMigrationsError(MigrationRefused): + """The refusal `require_applied()` raises for a database behind its migrations.""" def __init__(self, pending: Sequence[str], refusals: Sequence[str] = ()) -> None: self.pending = list(pending) + """Each migration with a step still to apply, as `NNNN_<name>`.""" self.refusals = list(refusals) + """The tracking table's own refusals, each the text `ferro migrate up` would print. + + An interrupted run or an edited applied file, for example. + """ lines = ["ferro.migrations: this database is behind its migrations."] lines += [f" pending {name}" for name in self.pending] lines += self.refusals @@ -84,15 +81,15 @@ def __init__(self, pending: Sequence[str], refusals: Sequence[str] = ()) -> None class DatabaseAheadError(MigrationRefused): - """``up()`` / ``require_applied()``: the database has applied migrations - this checkout does not have (``ahead``, ``NNNN_<name>``). - - ``contract_steps`` names each of their steps whose class is not - ``expand`` (``NNNN_<name>:NN``, ADR-0054): a step the previous release's - writes may fail beside. ``allow_ahead="expand-only"`` lets the database - through when it is empty, and its message names the first; pass - ``allow_ahead=True`` to run beside any of them, as a rolling deploy with - a fixed old version does. + """The refusal `up()` and `require_applied()` raise for a database ahead of the checkout. + + The database has applied migrations this checkout does not have. + `allow_ahead="expand-only"` lets the database through when + `contract_steps` is empty, and its message names the first of them; + pass `allow_ahead=True` to run beside any of them, as a rolling deploy + with a fixed old version does. See + [Deploying migrations](../guide/schema/deploying.md). Its `report` is + the `RunReport` of the refused `up`, or `None` for `require_applied()`. """ def __init__( @@ -104,7 +101,12 @@ def __init__( contract_steps: Sequence[str] = (), ) -> None: self.ahead = list(ahead) + """The migrations the database has applied that the checkout lacks, as `NNNN_<name>`.""" self.contract_steps = list(contract_steps) + """Each of their steps whose class is not `expand`, as `NNNN_<name>:NN`. + + Each is a step the previous release's writes may fail beside. + """ super().__init__( f"{message}\nPass allow_ahead=True to run beside them (a rolling deploy).", report=report, diff --git a/src/ferro/migrations/generate.py b/src/ferro/migrations/generate.py index add311d4..6824aeb1 100644 --- a/src/ferro/migrations/generate.py +++ b/src/ferro/migrations/generate.py @@ -1,22 +1,20 @@ -"""``ferro migrate new`` and ``ferro migrate check``: the offline half. - -A developer declares a model and asks for a migration:: - - class Author(Model): - id: Annotated[int | None, FerroField(primary_key=True)] = None - name: str - - $ ferro migrate new create_author - migrations/0001_create_author/ - 01_schema.up.postgres.sql CREATE TABLE "author" (...); - 01_schema.down.postgres.sql -- ferro: destructive / DROP TABLE "author"; - ir.json - -:func:`new` compiles the database's declared modelset, diffs it against the -head migration's schema snapshot through the one planner (in the Rust core, -which renders every statement), and writes the migration. It never opens a -database (ADR-0023). :func:`check` asks the same question without writing: -does every model change have a migration, and is the directory intact? +"""`ferro migrate new` and `ferro migrate check`: the offline half. + +A developer declares a model and asks for a migration: + +```text +$ ferro migrate new create_author +migrations/0001_create_author/ + 01_schema.up.postgres.sql CREATE TABLE "author" (...); + 01_schema.down.postgres.sql -- ferro: destructive / DROP TABLE "author"; + ir.json +``` + +`new` compiles the database's declared modelset, diffs it against the head +migration's schema snapshot through the planner (which renders every +statement), and writes the migration. It never opens a database. `check` +asks the same question without writing: does every model change have a +migration, and is the directory intact? """ from __future__ import annotations @@ -79,7 +77,7 @@ class Author(Model): class MigrationsCheckError(MigrationRefused): - """Raised by :meth:`CheckReport.raise_for_problems`; carries the report.""" + """The refusal `CheckReport.raise_for_problems()` raises; its `report` is the check's report.""" def __init__(self, report: CheckReport) -> None: lines = "\n".join(f" {p.kind}: {p.message}" for p in report.problems) @@ -88,50 +86,71 @@ def __init__(self, report: CheckReport) -> None: @dataclass(frozen=True) class Problem: - """One thing :func:`check` found wrong.""" + """One thing [`check`][ferro.migrations.check] found wrong.""" kind: str - """A stable identifier: ``ungenerated``, ``broken_chain``, - ``duplicate_number``, ``missing_number``, ``missing_rendering``, ...""" + """A stable identifier, such as `ungenerated`, `broken_chain`, `duplicate_number` or `unwritten_step`.""" message: str """What is wrong and how to fix it.""" changes: tuple[Change, ...] = () - """An ``ungenerated`` problem's change summary, as data: what the - migration ``ferro migrate new`` would write changes. Empty for every - other kind.""" + """An `ungenerated` problem's change summary, as data; empty for every other kind. + + It says what the migration `ferro migrate new` would write changes. + """ headline: str | None = None - """An ``ungenerated`` problem's message without the summary and notes it - quotes (``the models changed since 0001_initial and no migration records - it; run ...``), as the core builds it: what a terminal prints above the - summary it lays out from :attr:`changes`. ``None`` for every other - problem.""" + """An `ungenerated` problem's message without the summary and notes it quotes; `None` otherwise. + + For example `the models changed since 0001_initial and no migration + records it; run ...`: what a terminal prints above the summary it lays + out from `changes`. + """ notes: tuple[str, ...] = () - """The notes an ``ungenerated`` problem's message quotes after its - summary, as ``new`` prints them. Empty for every other problem.""" + """The notes an `ungenerated` problem's message quotes after its summary; empty otherwise.""" @dataclass(frozen=True) class CheckReport: - """What :func:`check` found.""" + """What [`check`][ferro.migrations.check] found: whether the migrations are complete, and each problem. + + Examples: + ```python {test=compile reason="needs a migrations project"} + report = await ferro.migrations.check() + for problem in report.problems: + print(problem.kind, problem.message) + report.raise_for_problems() + ``` + """ ok: bool + """Whether `check` found no problem.""" head: str | None = None """The newest migration's directory name, when there is one.""" problems: list[Problem] = field(default_factory=list) + """Every problem found, in the order found.""" def raise_for_problems(self) -> None: - """Raise :class:`MigrationsCheckError` carrying this report when it - has any problem; do nothing otherwise.""" + """Raise when the report has any problem; do nothing otherwise. + + Raises: + MigrationsCheckError: The report has a problem; the message + lists each, and its `report` is this report. + """ if self.problems: raise MigrationsCheckError(self) def declared_modelset(database: DatabaseSettings) -> dict[str, Any]: - """The SchemaIR modelset envelope ``database`` declares. + """Compile the schema a configured database's models declare. - Imports the database's ``models`` modules and keeps, from the resolved + Imports the database's `models` modules and keeps, from the resolved registry modelset, the models this database claims and the join tables between them (a join table rides its models). + + Args: + database: The configured database. + + Returns: + The modelset, in the form a migration's `ir.json` stores. """ from .. import ensure_resolved_modelset from ..registry import REGISTRY @@ -168,35 +187,40 @@ def new( data_only: bool = False, no_backfill: Sequence[str] = (), ) -> Path | None: - """Write the migration that brings ``database``'s migrations up to its models. - - Returns the new migration's directory, or ``None`` when the models - change nothing that renders DDL (a Python default, a back-reference, a - method edit: ADR-0027) and no hand step (``sql_step``, ``data_step``) - was asked for; a migration of hand steps alone stores the declared - modelset as its snapshot. ``sql_step`` - appends a hand-written portable step (``NN_<sql_step>.up.sql`` / - ``.down.sql``) holding ``-- write this step``. ``data_step`` (a model's - class name, ``Author``) adds a Python data step - ``NN_backfill_<model>.py`` whose ``up`` and ``down`` hold - ``todo("write this step")`` (or the project's ``_templates/data_step.py``), - after every generated step but the contract: a column or table the - migration drops is dropped by the contract, after the data step, which - can still read it (ADR-0025); with ``data_only`` the migration holds that - step alone, whatever the models changed, and a full copy of its parent's - snapshot. A change that - asks existing rows for values gets a generated backfill per model - (``NN_backfill_<model>.py``, ``todo`` where a value is missing); - ``no_backfill`` (``["author.slug"]``) replaces a model's backfill with a - generated guard step that fails the migration while any row still needs - a value. Rendering warnings (a - dialect that skips a declaration, such as row security on SQLite) are - issued as ``UserWarning``. + """Write the migration that brings a database's migrations up to its models. + + This is `ferro migrate new`. A change that asks existing rows for values + gets a generated backfill per model (`NN_backfill_<model>.py`, with + `todo` where a value is missing). A migration of hand steps alone stores + the declared modelset as its snapshot. Rendering warnings (a dialect + that skips a declaration, such as row security on SQLite) are issued as + `UserWarning`. + + Args: + database: The configured database. + name: The migration's name: lowercase letters, digits and `_`. + sql_step: Also add a hand-written portable SQL step + `NN_<sql_step>.up.sql` / `.down.sql` holding `-- write this step`. + data_step: Also add a Python data step `NN_backfill_<model>.py` over + this model (its class name, `Author`), whose `up` and `down` hold + `todo("write this step")` (or the project's + `_templates/data_step.py`). It lands after every generated step + but the contract, so it can still read what the contract drops. + data_only: Write only the `data_step`: no DDL, and a full copy of the + parent's snapshot. + no_backfill: Columns (`"author.slug"`) whose generated backfill is + replaced by a guard step that fails the migration while any row + still needs a value. + + Returns: + The new migration's directory, or `None` when the models change + nothing that renders DDL (a Python default, a back-reference, a + method edit) and no hand step was asked for. Raises: - MigrationsDirectoryError: the directory is malformed, a name is not - usable, or the models change something this generator does not - generate yet (``not generated yet: <op> on <table> (ticket #N)``). + MigrationsDirectoryError: The directory is malformed, a name is not + usable, or the models change something the generator does not + generate yet. """ migration = prepare( database, @@ -223,28 +247,37 @@ def replace( data_only: bool = False, no_backfill: Sequence[str] = (), ) -> Path: - """Regenerate the migration ``target`` names as ``name`` on top of the - newest schema snapshot below it (``ferro migrate new <name> --replace - <target>``), and return its directory. - - Two branches each generated ``0007``; after the rebase the directory - holds ``0007_add_x`` and ``0007_add_y``. ``replace(db, "add_x", - "0007_add_x")`` writes ``0008_add_x`` chained from ``0007_add_y``'s - snapshot and deletes ``0007_add_x``: a new snapshot, regenerated DDL - steps, and every data step and hand-written SQL step carried across by - name with its body intact, renumbered when the regenerated steps move - it. A carried step wins over a rescaffolded step of its name. The number - is kept when it is free. ``target`` is ``NNNN_<name>``, or ``NNNN`` when - one migration carries the number. The other options mean what they mean - to :func:`new`. It never opens a database (ADR-0023); a database that - applied the old files meets the replaced migration as a changed one. + """Regenerate a migration on top of the newest schema snapshot below it. + + This is `ferro migrate new <name> --replace <target>`. Two branches each + generated `0007`; after the rebase the directory holds `0007_add_x` and + `0007_add_y`. `replace(db, "add_x", "0007_add_x")` writes `0008_add_x` + chained from `0007_add_y`'s snapshot and deletes `0007_add_x`: a new + snapshot, regenerated DDL steps, and every data step and hand-written + SQL step carried across by name with its body intact, renumbered when + the regenerated steps move it. A carried step wins over a rescaffolded + step of its name. The number is kept when it is free. It never opens a + database; a database that applied the old files meets the replaced + migration as a changed one. + + Args: + database: The configured database. + name: The regenerated migration's name. + target: The migration to regenerate, `NNNN_<name>`, or `NNNN` when + one migration carries the number. + sql_step: As for `new`. + data_step: As for `new`. + data_only: As for `new`. + no_backfill: As for `new`. + + Returns: + The regenerated migration's directory. Raises: - MigrationsDirectoryError: the target is not in the directory, is a - number two migrations share, or a later migration chains from it - ("replace the later one first, or delete it"); the regeneration - is empty and carries nothing ("delete the directory"); or what - :func:`new` raises. + MigrationsDirectoryError: The target is not in the directory, is a + number two migrations share, or a later migration chains from it; + the regeneration is empty and carries nothing; or what `new` + raises. """ migration = prepare( database, @@ -272,29 +305,39 @@ def squash( data_only: bool = False, no_backfill: Sequence[str] = (), ) -> Path: - """Replace the range from the migration ``from_`` names through the head - with the one migration ``new`` would have written across it (``ferro - migrate squash <name> --from <from_>``, ADR-0056), and return its - directory. - - A feature branch wrote ``0005_add_teams`` through ``0009_fix_rows``; - ``squash(db, "teams_epic", "0005")`` writes ``0005_teams_epic``, planned - once by the one planner from the snapshot below the range (none for a range - from ``0001``) to the range's composed target, with every data step and - hand-written SQL step of the range carried byte for byte in chain order, - and deletes the five. Its snapshot stores the composed target under the - parent's checksum, with the witness naming each replaced migration. - ``from_`` is ``NNNN_<name>``, or ``NNNN`` when one migration carries the - number. The other options mean what they mean to :func:`new`. It reads + """Replace a range of migrations, through the head, with the one migration `new` would have written. + + This is `ferro migrate squash <name> --from <from_>`. A feature branch + wrote `0005_add_teams` through `0009_fix_rows`; + `squash(db, "teams_epic", "0005")` writes `0005_teams_epic`, planned + once from the snapshot below the range (none for a range from `0001`) to + the range's composed target, with every data step and hand-written SQL + step of the range carried byte for byte in chain order, and deletes the + five. Its snapshot stores the composed target under the parent's + checksum, with a witness naming each replaced migration. It reads snapshots and the directory only: the models are never imported, no database is opened. + Args: + database: The configured database. + name: The squashed migration's name. + from_: The first migration of the range, `NNNN_<name>`, or `NNNN` + when one migration carries the number. + sql_step: As for `new`. + data_step: As for `new`. + data_only: As for `new`, with a full copy of the snapshot below the + range. + no_backfill: As for `new`. + + Returns: + The squashed migration's directory. + Raises: - MigrationsDirectoryError: ``from_`` is not in the directory or is a + MigrationsDirectoryError: `from_` is not in the directory or is a number two migrations share; the range does not reach the head; - the range's rename hints do not compose; the squash renders nothing - and carries nothing ("delete the directories"); an aside of an - earlier squash or replace is present; or what :func:`new` raises. + the range's rename hints do not compose; the squash renders + nothing and carries nothing; an aside of an earlier squash or + replace is present; or what `new` raises. """ migration = prepare( database, @@ -313,23 +356,30 @@ def squash( def write(database: DatabaseSettings, migration: GeneratedMigration) -> Path: - """Write a migration :func:`prepare` returned into ``database``'s - directory (with its ``.gitattributes``); return the migration's directory. - - A replacement (``migration.replaces``) sets the replaced migration's - directory aside under a hidden name naming both (``.0007_add_x.replaced`` - in place, ``.0007_add_x.0008_add_x.replaced`` renumbered), writes the new - one, and deletes the aside; a write that raises puts it back. A run killed - in between leaves the aside, which every directory reader refuses by - name with its recovery (panel F6): rename it back while the replacement - is absent, delete it once the replacement is in place. - - A squash moves the range's directories one by one into one aside named - after both ends and the replacement - (``.0005_add_teams..0009_fix_rows.0005_teams_epic.squashed``), writes the - new migration, and deletes the aside; a write that raises moves them - back. Every reader refuses a leftover aside the same way, naming any - original still outside it.""" + """Write a prepared migration into a database's migrations directory. + + A replacement sets the replaced migration's directory aside under a + hidden name naming both (`.0007_add_x.replaced` in place, + `.0007_add_x.0008_add_x.replaced` renumbered), writes the new one, and + deletes the aside; a write that raises puts it back. A run killed in + between leaves the aside, which every directory reader refuses by name + with its recovery: rename it back while the replacement is absent, + delete it once the replacement is in place. A squash moves the range's + directories one by one into one aside named after both ends and the + replacement (`.0005_add_teams..0009_fix_rows.0005_teams_epic.squashed`) + and recovers the same way. + + Args: + database: The configured database. + migration: The migration `prepare` returned. + + Returns: + The written migration's directory. + + Raises: + MigrationsDirectoryError: An aside of another replace or squash is + present. + """ directory = database.directory ensure_gitattributes(directory) if not migration.replaces: @@ -359,7 +409,7 @@ def write(database: DatabaseSettings, migration: GeneratedMigration) -> Path: def _write_squash(directory: Path, migration: GeneratedMigration) -> Path: - """:func:`write` for a squash: the originals into one aside, one by one, + """`write` for a squash: the originals into one aside, one by one, the replacement written, the aside deleted; a raise moves them back.""" first, last = migration.replaces[0], migration.replaces[-1] aside = directory / f".{first}..{last}.{migration.dir_name}.squashed" @@ -396,11 +446,27 @@ def prepare( replace: str | None = None, squash_from: str | None = None, ) -> GeneratedMigration | None: - """The migration :func:`new` would write, without writing it; ``None`` - for no schema change. Raises what :func:`new` raises. With ``replace``, - the migration :func:`replace` would write (never ``None``), raising what - it raises. With ``squash_from``, the migration :func:`squash` would write - (never ``None``), raising what it raises; the models are never read.""" + """Build the migration `new`, `replace` or `squash` would write, without writing it. + + Args: + database: The configured database. + name: The migration's name. + sql_step: As for `new`. + data_step: As for `new`. + data_only: As for `new`. + no_backfill: As for `new`. + replace: Build what `replace` would write for this target. + squash_from: Build what `squash` would write from this migration; + the models are never read. + + Returns: + The migration, or `None` for no schema change (never `None` with + `replace` or `squash_from`). + + Raises: + ValueError: Both `replace` and `squash_from` are given. + MigrationsDirectoryError: What `new`, `replace` or `squash` raises. + """ if replace is not None and squash_from is not None: raise ValueError("name either replace or squash_from, not both") check_name(name, "migration name") @@ -488,7 +554,7 @@ def _prepare_squash( from_: str, options: dict[str, Any], ) -> GeneratedMigration: - """:func:`prepare` for :func:`squash`: the range read, its snapshots + """`prepare` for `squash`: the range read, its snapshots composed into the target (``_core._compose_range``), the migration planned once from the parent snapshot to that target through the generator ``new`` uses, carrying the range's data steps and hand-written SQL steps and @@ -601,7 +667,7 @@ def _unneeded_scaffolds( dialects: list[str], ) -> dict[int, str]: """The carried data steps a squash drops, by their place in ``carry``, - each with the note that names it (ADR-0056, amended 2026-10-18). + each with the note that names it. A carried data step is dropped when it is still, byte for byte, the backfill the generator scaffolded for its original migration (that @@ -690,12 +756,12 @@ def _with_dead_name_notes( names: list[str], ) -> GeneratedMigration: """``migration`` with a note for each dead name (a table, column, enum - label or constraint that existed only inside the range, ADR-0056) no carried + label or constraint that existed only inside the range) no carried data step's declaration touches, or the refusal of the first carried step whose declaration does. A squash cannot read what a step's body does, but it can read what the - step declares (ADR-0035): a chunked step's query, built over the + step declares: a chunked step's query, built over the historical models of the squashed migration widened by the dead names. The query's table, the columns its filter and order keys read and the enum labels its filter compares are names the step provably touches. A range @@ -749,7 +815,7 @@ def _widened_models( names: list[str], ) -> HistoricalModels: """The squashed migration's historical models (the snapshot below the range - over the composed target, ADR-0025) plus every table and column any + over the composed target) plus every table and column any snapshot of the range held that the parent does not, so a declaration naming a dead name still builds and can be read. @@ -921,8 +987,8 @@ def _refuse_data_only_over_ddl( name: str, options: dict[str, Any], ) -> None: - """Refuse ``squash --data-only`` over a range that renders DDL (panel F2 - on #690): ``--data-only`` writes the snapshot below the range, while the + """Refuse ``squash --data-only`` over a range that renders DDL: + ``--data-only`` writes the snapshot below the range, while the witness lets every database that ran the originals inherit it, so they would stand on a snapshot that says the range's DDL never ran. A range that renders none (what it adds it drops again) loses nothing. @@ -1140,7 +1206,7 @@ def _with_data_steps( template_dir: Path, ) -> GeneratedMigration: """``migration`` with each data step's file written from the step record - the generator produced (:func:`scaffold.render`), and each backfill that + the generator produced (`scaffold.render`), and each backfill that still needs writing named in a note after the generator's. The step's name and place are the generator's.""" by_ordinal = {step["ordinal"]: step for step in generated["steps"]} @@ -1191,18 +1257,31 @@ def _data_role(record: Mapping[str, Any]) -> str: async def check( settings: FerroSettings | None = None, database: str | None = None ) -> CheckReport: - """Check, offline, that every model change has a migration and the - directory is intact (``ferro migrate check``), reading files only: it - takes no connection. + """Check, offline, that every model change has a migration and the directory is intact. + + This is `ferro migrate check`. It reads files only and takes no + connection. It reports an ungenerated model change (naming the models), + a broken snapshot chain (naming both files), a duplicate or missing + number, a DDL step without the rendering a target dialect needs, and a + data step still holding `todo(...)`. It raises nothing for a problem. - Reports an ungenerated model change (naming the models), a broken - snapshot chain (naming both files), a duplicate or missing number, a - DDL step without the rendering a target dialect needs, and a data step - still holding ``todo(...)``. Raises nothing for a problem: call - :meth:`CheckReport.raise_for_problems` to fail on one. + Args: + settings: The project's configuration. Defaults to `FerroSettings()`. + database: The configured database. Defaults to the only one + configured. + + Returns: + The report; its `raise_for_problems()` raises on any problem. Raises: - MigrationRefused: the configuration names no single database. + MigrationRefused: The configuration names no single database. + + Examples: + In CI, fail the build on an ungenerated change: + + ```python {test=compile reason="needs a migrations project"} + (await ferro.migrations.check()).raise_for_problems() + ``` """ chosen = Target.resolve(settings, database).database declared = declared_modelset(chosen) diff --git a/src/ferro/migrations/historical.py b/src/ferro/migrations/historical.py index 763dfe99..96d1db6e 100644 --- a/src/ferro/migrations/historical.py +++ b/src/ferro/migrations/historical.py @@ -1,33 +1,34 @@ """Historical models: the classes a data step queries, built from snapshots. -Migration ``0011_backfill_slugs`` adds ``slug`` to ``author`` and renames -``name`` to ``full_name``. Its data step runs between the expand and the -contract, when the live table holds both the old columns and the new ones, -so its historical ``Author`` is the **union** of the parent's snapshot and -the migration's own (ADR-0025):: - - parent 0010/ir.json author: id, name, bio - own 0011/ir.json author: id, full_name (renamed_from "name"), slug - union author: id, full_name, bio, slug (nullable) +Migration `0011_backfill_slugs` adds `slug` to `author` and renames `name` +to `full_name`. Its data step runs between the expand and the contract, when +the live table holds both the old columns and the new ones, so its +historical `Author` is the union of the parent's snapshot and the +migration's own: + +```text +parent 0010/ir.json author: id, name, bio +own 0011/ir.json author: id, full_name (renamed_from "name"), slug +union author: id, full_name, bio, slug (nullable) +``` - a column only the migration's own snapshot has is nullable, whatever it - declares: the expand created it so and every row holds ``NULL`` until the + declares: the expand created it so and every row holds `NULL` until the backfill runs; - a live rename hint (the own snapshot names an old column the parent has, and the parent lacks the new name) is one column under its new name; - a same-named column whose storage differs between the two snapshots (its - type, ``db_type`` or enum type) is refused, naming both declarations: the + type, `db_type` or enum type) is refused, naming both declarations: the safe expand/contract shape never produces it. A column that only stops (or - starts) accepting ``NULL`` is not that: the union keeps it nullable, which - is what lets a backfill fill the ``NULL``s before the contract; + starts) accepting `NULL` is not that: the union keeps it nullable, which + is what lets a backfill fill the `NULL`s before the contract; - an enum's labels are the union of both snapshots' labels. -Each table becomes a columns-only class built through ferro's own metaclass -(no relations, methods or validators; a foreign key is its plain ``*_id`` -column), under the module identity ``ferro.migrations.historical.<rev>``, -registered into a registry of its own (:meth:`ferro.registry.Registry.capture`) -so today's registry never sees it. A many-to-many join table gets a class -like any other table. +Each table becomes a columns-only class built through Ferro's own metaclass +(no relations, methods or validators; a foreign key is its plain `*_id` +column), under the module identity `ferro.migrations.historical.<rev>`, +registered into a registry of its own so today's registry never sees it. A +many-to-many join table gets a class like any other table. """ from __future__ import annotations @@ -63,7 +64,7 @@ "unknown": Any, } """Each SchemaIR ``logical_type`` and the annotation that compiles back to it -(``ferro.columns._logical_type`` read in reverse). :func:`_check_compiled` +(``ferro.columns._logical_type`` read in reverse). `_check_compiled` pins every class against its union, so a gap here fails loudly.""" # The column facts that decide storage: a same-named column differing in one @@ -86,12 +87,19 @@ class HistoricalModelError(MigrationRefused): def build( previous: dict[str, Any] | None, current: dict[str, Any], *, rev: str ) -> HistoricalModels: - """The historical models of migration ``rev``: the union of its parent's - snapshot ``previous`` (``None`` for the first migration) and its own - ``current`` (SchemaIR envelopes, as ``ir.json`` stores them). + """Build the historical models of a migration: the union of its parent's snapshot and its own. + + Args: + previous: The parent's snapshot, as `ir.json` stores it; `None` for + the first migration. + current: The migration's own snapshot, as `ir.json` stores it. + rev: The migration, such as `0011_backfill_slugs`. + + Returns: + The historical models, one class per table of the union. Raises: - HistoricalModelError: a same-named column is declared with different + HistoricalModelError: A same-named column is declared with different storage in the two snapshots (naming both). """ tables = _union(_tables(previous), _tables(current), rev) @@ -99,7 +107,15 @@ def build( def build_single(snapshot: dict[str, Any], rev: str) -> HistoricalModels: - """The historical models of one snapshot alone, as it declares them.""" + """Build the historical models of one snapshot alone, as it declares them. + + Args: + snapshot: The snapshot, as `ir.json` stores it. + rev: The migration the snapshot belongs to. + + Returns: + The historical models, one class per table. + """ return _models(_tables(snapshot), rev) @@ -288,7 +304,7 @@ def _field(model: dict[str, Any], column: dict[str, Any]) -> tuple[Any, Any]: def _enum(column: dict[str, Any]) -> type[enum.Enum]: """The historical enum: every label either snapshot declares. A value - the live type holds beyond them fails hydration naming it (ADR-0035).""" + the live type holds beyond them fails hydration naming it.""" type_name = column["enum_type_name"] values: Iterable[Any] = column["enum_values"] # Member names only label the values; a sunder-looking label gets a prefix. diff --git a/src/ferro/migrations/layout.py b/src/ferro/migrations/layout.py index 98c533fe..bec69803 100644 --- a/src/ferro/migrations/layout.py +++ b/src/ferro/migrations/layout.py @@ -1,23 +1,25 @@ """Where a migration's files go, and how a new one is written. -A database's migrations directory holds one directory per migration:: - - migrations/ - .gitattributes * -text (byte-exact: checksums are of raw bytes) - 0001_create_author/ - 01_schema.up.postgres.sql one rendering per target dialect (ADR-0026, ADR-0037) - 01_schema.down.postgres.sql - 01_schema.up.sqlite.sql - 01_schema.down.sqlite.sql - 02_fix_rows.up.sql a hand-written step: unsuffixed, serves every dialect - 02_fix_rows.down.sql - 03_backfill_author.py a data step: Python, up() and down() in one file - ir.json the schema snapshot (ADR-0023) - -The directory is read and verified by the Rust core (numbers dense from -``0001``, a duplicate refused naming both directories, the snapshot chain -intact); this module names paths, numbers the next migration and writes one -whole, so a crash never leaves a half-written migration behind. +A database's migrations directory holds one directory per migration: + +```text +migrations/ + .gitattributes * -text (byte-exact: checksums are of raw bytes) + 0001_create_author/ + 01_schema.up.postgres.sql one rendering per target dialect + 01_schema.down.postgres.sql + 01_schema.up.sqlite.sql + 01_schema.down.sqlite.sql + 02_fix_rows.up.sql a hand-written step: unsuffixed, serves every dialect + 02_fix_rows.down.sql + 03_backfill_author.py a data step: Python, up() and down() in one file + ir.json the schema snapshot +``` + +The directory is read and verified by the core (numbers dense from `0001`, +a duplicate refused naming both directories, the snapshot chain intact); +this module names paths, numbers the next migration and writes one whole, +so a crash never leaves a half-written migration behind. """ from __future__ import annotations @@ -53,24 +55,37 @@ ] SNAPSHOT_FILE = "ir.json" +"""The schema snapshot's file name in each migration's directory.""" GITATTRIBUTES = "* -text\n" """Mark every migration file byte-exact: a line-ending conversion on checkout would change an applied step's checksum.""" SQL_STEP_PLACEHOLDER = "-- write this step\n" -"""The body of a hand-written step ``new --sql-step`` adds.""" +"""The body of a hand-written step `new --sql-step` adds.""" _NAME = re.compile(r"^[a-z0-9_]+$") class MigrationsDirectoryError(FerroError): - """A migrations directory ``new`` or ``check`` cannot use, or a migration - it cannot write; the message names the files involved and the fix.""" + """The refusal for a migrations directory `new` or `check` cannot use, or a migration it cannot write. + + The message names the files involved and the fix. + """ def check_name(name: str, what: str) -> None: - """Refuse a migration or step name that is not ``[a-z0-9_]+``.""" + """Refuse a migration or step name that is not `[a-z0-9_]+`. + + Args: + name: The name to check. + what: What the name is for, as the refusal says it + (`"migration name"`). + + Raises: + MigrationsDirectoryError: The name is not lowercase letters, digits + and `_`. + """ if not _NAME.match(name): raise MigrationsDirectoryError( f"{what} {name!r} is not usable in a file name; use lowercase " @@ -79,8 +94,7 @@ def check_name(name: str, what: str) -> None: class Moved(StrEnum): - """What ``ferro migrate new --replace`` or ``ferro migrate squash`` did - with one file of the migrations it replaced.""" + """What `ferro migrate new --replace` or `ferro migrate squash` did with one file it replaced.""" KEPT = "kept" """A regenerated file under the name it had.""" @@ -89,8 +103,7 @@ class Moved(StrEnum): CARRIED = "carried" """A data step or hand-written SQL step written back byte for byte.""" REGENERATED = "regenerated" - """A squash's generated DDL file: planned once across the whole range, - so it is no one original file's counterpart.""" + """A squash's generated DDL file, planned once across the whole range: no one original file's counterpart.""" @dataclass(frozen=True) @@ -98,29 +111,39 @@ class GeneratedStep: """One step of a migration about to be written.""" ordinal: int + """The step's number within its migration.""" name: str + """The step's name, without its number.""" kind: str - """``ddl`` (one rendering per target dialect), ``portable_sql`` or - ``data`` (a Python data step).""" + """`ddl` (one rendering per target dialect), `portable_sql` or `data` (a Python data step).""" files: dict[str, str] = field(default_factory=dict) """File name to its exact text.""" carry: int | None = None - """Carried across from the migration ``--replace`` regenerates: the - position of the step it carries in the generator's ``carry`` option, as - the generator names it.""" + """For a step carried across from the migration `--replace` regenerates: its position in the carried steps. + + `None` for a step the generator wrote. + """ over_namesake: bool = False - """Carried, and in the place of a step of its name and kind the - regeneration laid out, which it wins over.""" + """Whether a carried step takes the place of a step of its name and kind the regeneration laid out.""" role: str | None = None - """Which kind of data step the generator laid out: ``backfill`` (a - generated backfill), ``guard`` (``--no-backfill``) or ``data_step`` - (``--data-step``). ``None`` for every other step, and for a carried one, - whose file is a person's whatever the generator first made it.""" + """Which kind of data step the generator laid out: `backfill`, `guard` or `data_step`. + + `guard` is from `--no-backfill`, `data_step` from `--data-step`. `None` + for every other step, and for a carried one, whose file is a person's + whatever the generator first made it. + """ def portable_sql_step(ordinal: int, name: str) -> GeneratedStep: - """The hand-written portable step ``NN_<name>`` (``new --sql-step``) at - ``ordinal``: its up and down files holding the placeholder.""" + """Build the hand-written portable SQL step `new --sql-step` adds. + + Args: + ordinal: The step's number within its migration. + name: The step's name. + + Returns: + The step `NN_<name>`, its up and down files holding the placeholder. + """ stem = f"{ordinal:02d}_{name}" return GeneratedStep( ordinal, @@ -135,44 +158,58 @@ def portable_sql_step(ordinal: int, name: str) -> GeneratedStep: @dataclass(frozen=True) class Change: - """One entry of a :class:`ChangeSummary`: what happened to which names - of one kind (``new models: Team, TeamMember``), as the generator decided - it.""" + """One entry of a change summary: what happened to which names of one kind (`new models: Team, TeamMember`).""" verb: str - """``new``, ``changed``, ``dropped``, ``renamed`` or ``removed``.""" + """`new`, `changed`, `dropped`, `renamed` or `removed`.""" kind: str - """``models``, ``enum types`` or ``enum labels``.""" + """`models`, `enum types` or `enum labels`.""" names: tuple[str, ...] - """The names, sorted; an enum label is ``type.label``, a rename - ``old → new``.""" + """The names, sorted; an enum label is `type.label`, a rename `old → new`.""" @classmethod def from_json(cls, raw: Mapping[str, Any]) -> Change: - """Build from one entry of the JSON the core returns.""" + """Build a change from one entry of the JSON the core returns. + + Args: + raw: The entry, with `verb`, `kind` and `names`. + + Returns: + The change. + """ return cls(raw["verb"], raw["kind"], tuple(raw["names"])) @dataclass(frozen=True) class ChangeSummary: - """What a migration adds, changes, drops or renames (ADR-0055): the - changes as data, in the generator's order, beside the one plain - rendering the Rust generator produced. Lay the changes out from - ``changes``; never take ``text`` apart.""" + """What a migration adds, changes, drops or renames, as data and as one plain rendering. + + Lay the changes out from `changes`; never take `text` apart. + """ changes: tuple[Change, ...] = () + """The changes, in the generator's order.""" text: str = "" - """The plain rendering (``new models: Team; changed models: Account``), - the one the core quotes inside ``check``'s ``ungenerated`` message and - so the library's error text. ``new`` (in both forms) and ``check`` on a - terminal lay out ``changes`` instead; nothing in Python prints this.""" + """The plain rendering (`new models: Team; changed models: Account`). + + The core quotes it inside `check`'s `ungenerated` message, so it is the + library's error text. `new` and `check` on a terminal lay out `changes` + instead; nothing in Python prints this. + """ def __bool__(self) -> bool: return bool(self.changes) @classmethod def from_json(cls, raw: Mapping[str, Any]) -> ChangeSummary: - """Build from the ``summary`` the core returns.""" + """Build the summary from the `summary` the core returns. + + Args: + raw: The core's summary, with `changes` and `text`. + + Returns: + The summary. + """ return cls(tuple(Change.from_json(c) for c in raw["changes"]), raw["text"]) @@ -181,35 +218,44 @@ class GeneratedMigration: """A whole migration about to be written: its steps and its snapshot.""" number: int + """The migration's number.""" name: str + """The migration's name, without its number.""" steps: tuple[GeneratedStep, ...] + """Its steps, in order.""" snapshot_json: str - """The ``ir.json`` text exactly as it is to be stored.""" + """The `ir.json` text exactly as it is to be stored.""" summary: ChangeSummary = ChangeSummary() + """What the migration adds, changes, drops or renames.""" notes: tuple[str, ...] = () - """The lines ``new`` prints after the summary, in order: the generator's - rename hints, then each backfill that still needs writing.""" + """The lines `new` prints after the summary: the rename hints, then each backfill that still needs writing.""" warnings: tuple[str, ...] = () + """The rendering warnings, such as a dialect that skips a declaration.""" replaces: list[str] = field(default_factory=list) - """``--replace`` and ``squash``: the replaced migrations' directory names - (``NNNN_<name>``) in chain order, deleted once this one is written; one - for ``--replace``, the range for a squash, empty for ``new``.""" + """The migrations a replace or squash deletes once this one is written, as `NNNN_<name>` in chain order. + + One for `--replace`, the range for a squash, empty for `new`. + """ chained_from: str | None = None - """``--replace`` and ``squash``: the directory name of the migration - whose snapshot this one's links to (``None`` for the first).""" + """For a replace or squash, the migration whose snapshot this one's links to; `None` for the first.""" moves: tuple[tuple[str, str, Moved], ...] = () - """``--replace`` and ``squash``: ``(old file, new file, what happened)`` - for every file of this migration a replaced one had a counterpart for, - ``old file`` relative to the migrations directory, ``new file`` to this - migration's.""" + """For a replace or squash, `(old file, new file, what happened)` for every file with a counterpart. + + `old file` is relative to the migrations directory, `new file` to this + migration's. + """ witness: list[dict[str, Any]] = field(default_factory=list) - """``squash``: the witness its snapshot stores (ADR-0056), one entry per - replaced migration in chain order, ``{"migration", "snapshot_checksum", - "steps", "replaces"}``. Empty for ``new`` and ``--replace``.""" + """For a squash, the witness its snapshot stores: one entry per replaced migration, in chain order. + + Each is `{"migration", "snapshot_checksum", "steps", "replaces"}`. + Empty for `new` and `--replace`. + """ dead_names: list[dict[str, Any]] = field(default_factory=list) - """``squash``: the names born and gone inside the range, as - ``_core._compose_range`` reports them (``{"kind", "name", "appeared", - "vanished"}``). Empty for ``new`` and ``--replace``.""" + """For a squash, the names born and gone inside the range. + + Each is `{"kind", "name", "appeared", "vanished"}`. Empty for `new` and + `--replace`. + """ @property def squash(self) -> bool: @@ -218,6 +264,7 @@ def squash(self) -> bool: @property def dir_name(self) -> str: + """The migration's directory name, `NNNN_<name>`.""" return f"{self.number:04d}_{self.name}" @classmethod @@ -229,8 +276,18 @@ def from_generated( name: str, carry: Sequence[Mapping[str, Any]] = (), ) -> GeneratedMigration: - """Build from the JSON ``_core._generate_migration`` returns; a - carried step's files are the texts of the ``carry`` entry it names.""" + """Build the migration from the JSON the generator returns. + + Args: + raw: The generator's output. + number: The migration's number. + name: The migration's name. + carry: The steps carried across from a replaced migration; a + carried step's files are the texts of the entry it names. + + Returns: + The migration, ready to write. + """ steps = [] for step in raw["steps"]: files: dict[str, str] = {} @@ -266,7 +323,11 @@ def from_generated( ) def files(self) -> dict[str, str]: - """Every file of the migration, by name, ``ir.json`` last.""" + """List every file of the migration, by name, `ir.json` last. + + Returns: + Each file name mapped to its exact text. + """ out: dict[str, str] = {} for step in self.steps: out.update(step.files) @@ -275,11 +336,17 @@ def files(self) -> dict[str, str]: def read_migrations(directory: Path) -> list[dict[str, Any]]: - """Every migration in ``directory``, read and verified by the Rust core. + """Read and verify every migration in a migrations directory. + + Args: + directory: The migrations directory. + + Returns: + Each migration as the core reads it, in order. Raises: - MigrationsDirectoryError: the directory is malformed (a duplicate or - missing number, a broken snapshot chain, ...), naming the fix. + MigrationsDirectoryError: The directory is malformed (a duplicate or + missing number, a broken snapshot chain), naming the fix. """ try: raw = _read_migrations_dir(str(directory)) @@ -291,15 +358,23 @@ def read_migrations(directory: Path) -> list[dict[str, Any]]: def read_replacing( directory: Path, target: str ) -> tuple[list[dict[str, Any]], dict[str, Any]]: - """For ``ferro migrate new --replace <target>``: every migration in - ``directory`` but the one ``target`` names (``NNNN_<name>``, or ``NNNN`` - when one migration carries the number), read and verified by the Rust - core with the duplicate at that number admitted, and that migration. + """Read a migrations directory for `ferro migrate new --replace <target>`. + + The directory is read and verified with the duplicate at the target's + number admitted. + + Args: + directory: The migrations directory. + target: The migration to replace, `NNNN_<name>`, or `NNNN` when one + migration carries the number. + + Returns: + Every other migration, in order, and the target migration. Raises: - MigrationsDirectoryError: the target is not in the directory, is a - number two migrations share, or a later migration chains from it; - or the rest of the directory is malformed. + MigrationsDirectoryError: The target is not in the directory, is a + number two migrations share, or a later migration chains from + it; or the rest of the directory is malformed. """ try: raw = _read_migrations_dir(str(directory), replacing=target) @@ -310,23 +385,26 @@ def read_replacing( def read_range_reach(directory: Path, from_: str) -> dict[str, Any]: - """For a squash: every migration in ``directory`` below the one ``from_`` - names (``NNNN_<name>``, or ``NNNN`` when one migration carries the - number), the range from it in chain order, each read and verified by the - Rust core, and what the range leaves off. - - Returns ``{"below", "range", "reaches_head", "off_range", "siblings"}``: - ``range`` follows snapshot links from the migration ``from_`` names and need - not reach the directory head. ``reaches_head`` says whether it does, - ``off_range`` names the later migrations it leaves (a sibling chain's, or - a fork's other branch), each ``{"migration", "chains_from"}``: its - ``NNNN_<name>`` and the migration its snapshot chains from (``None`` for - none in the directory), and ``siblings`` the other - migrations at ``from_``'s number (two branches each generated it), which - are never in ``below``; the caller refuses each with its own fix. + """Read a migrations directory for a squash: the migrations below the range, the range, and what it leaves off. + + The caller refuses each thing the range leaves off with its own fix. + + Args: + directory: The migrations directory. + from_: The first migration of the range, `NNNN_<name>`, or `NNNN` + when one migration carries the number. + + Returns: + `{"below", "range", "reaches_head", "off_range", "siblings"}`. + `range` follows snapshot links from `from_` in chain order and + need not reach the directory head; `reaches_head` says whether + it does. `off_range` names the later migrations it leaves (a + sibling chain's, or a fork's other branch), each + `{"migration", "chains_from"}`. `siblings` are the other + migrations at `from_`'s number, never in `below`. Raises: - MigrationsDirectoryError: ``from_`` is not in the directory or is a + MigrationsDirectoryError: `from_` is not in the directory or is a number two migrations share; or the directory is malformed. """ try: @@ -344,12 +422,23 @@ def read_range_reach(directory: Path, from_: str) -> dict[str, Any]: def next_number(directory: Path) -> int: - """The number the next migration in ``directory`` takes (``1`` when empty).""" + """Return the number the next migration in a directory takes. + + Args: + directory: The migrations directory. + + Returns: + The next number; `1` for an empty directory. + """ return len(read_migrations(directory)) + 1 def ensure_gitattributes(directory: Path) -> None: - """Write ``directory/.gitattributes`` (``* -text``) when it is absent.""" + """Write `.gitattributes` (`* -text`) into a migrations directory when it is absent. + + Args: + directory: The migrations directory; created when missing. + """ path = directory / ".gitattributes" if not path.exists(): directory.mkdir(parents=True, exist_ok=True) @@ -357,15 +446,23 @@ def ensure_gitattributes(directory: Path) -> None: def write_migration(directory: Path, migration: GeneratedMigration) -> Path: - """Write ``migration`` under ``directory`` and return its directory. + """Write a migration under a migrations directory, all at once. The files are written into a hidden sibling first and moved into place with one rename, so the directory reader never sees half a migration. Every file is written as raw UTF-8 bytes, never newline-translated. + Args: + directory: The migrations directory. + migration: The migration to write. + + Returns: + The migration's directory. + Raises: - MigrationsDirectoryError: the migration's directory, or its hidden - staging sibling (another run, or an interrupted one), already exists. + MigrationsDirectoryError: The migration's directory, or its hidden + staging sibling (another run, or an interrupted one), already + exists. """ target = directory / migration.dir_name if target.exists(): diff --git a/src/ferro/migrations/report.py b/src/ferro/migrations/report.py index c5029250..6967e518 100644 --- a/src/ferro/migrations/report.py +++ b/src/ferro/migrations/report.py @@ -1,4 +1,4 @@ -"""What ``ferro migrate status`` reports, and how it prints (#466, #473). +"""What `ferro migrate status` reports, and how it prints. ```text $ ferro migrate status @@ -14,14 +14,14 @@ ``` A fully applied or fully pending migration is one line; its steps expand -where something needs attention, or everywhere with ``--steps``. Every state -is decided in the Rust core (the tracked database's ``status()``); this module names and -prints them (a chunked step's ``rows_done`` comes from its record), and is +where something needs attention, or everywhere with `--steps`. Every state +is decided in the core; this module names and prints them (a chunked +step's `rows_done` comes from its record), and is the one place that reads them into where the database stands: its last fully applied migration, the one a run left unfinished, and whether -anything is pending. ``status`` exits 3 when anything is pending and 4 when +anything is pending. `status` exits 3 when anything is pending and 4 when anything needs attention (a failed, interrupted or reverting step, an edited -file, a database ahead of the checkout, or a refusal ``up`` would meet). +file, a database ahead of the checkout, or a refusal `up` would meet). """ from __future__ import annotations @@ -44,27 +44,26 @@ class RunRefused(MigrationRefused): - """A migration run refused before running anything, naming the fix. - - Raised by the Rust core for every refusal the run planner, the run lock - and the tracking tables make; its message is the operator's text. A - planner refusal also carries what it is about, so a caller matches on - ``kind`` rather than on the text: ``kind`` (``"irreversible"``, - ``"edited_applied"``, ``"edited_chunked"``, ...), ``migration`` and - ``step`` (numbers, when it names them) and ``reason`` (an irreversible - step's declared reason). Each is ``None`` for a refusal that is not the - planner's (a lost lock, a missing schema). ``ahead_only`` is true when - the database holds migrations the directory lacks and ``allow_ahead`` - alone would have let the run through; ``contract_steps`` then names each - step record of those migrations whose class is not ``expand`` - (``NNNN_<name>:NN``, ADR-0054), what ``allow_ahead="expand-only"`` - refuses on. ``names`` lists the migrations - (``NNNN_<name>``) a refusal about a set of them names: a tracked - database's records (``already_tracked``), the run-applied migrations - above a baseline (``applied_above_baseline``); empty otherwise. ``head`` - is the newest of a tracked database's records (``already_tracked``), - ``None`` otherwise. The core builds every planner refusal through - :meth:`from_core`, so a kind with a class of its own arrives as it. + """A migration run's refusal before it ran anything, naming the fix. + + Raised for every refusal the run planner, the run lock and the tracking + tables make, and by `connect(auto_migrate=True)` on a database that + migrations manage; its message is the operator's text. It is a + `MigrationRefused`, so `except MigrationRefused` catches it too. Some + refusals also carry what they are about in `kind`, so a caller matches + on it rather than on the text; on the others `kind` is `None`. A kind with a class of its own + (`AlreadyTrackedError`, `AppliedAboveBaselineError`) arrives as that + class. + + Examples: + ```python {test=compile reason="needs a project with a migrations directory"} + try: + await ferro.migrations.up() + except ferro.migrations.RunRefused as refused: + if refused.kind == "edited_applied": + print(refused) # names the applied migration that was edited + raise + ``` """ def __init__( @@ -82,63 +81,102 @@ def __init__( ) -> None: super().__init__(message) self.kind = kind + """The refusal's kind (`"irreversible"`, `"edited_applied"`, ...), or `None`. + + Only some refusals carry a kind. `connect(auto_migrate=True)` on a + database that migrations manage, a lost run lock and a data step + that does not load are among those whose `kind` is `None`; match + those on the message, or catch the `MigrationRefused` base. + """ self.migration = migration + """The migration number the refusal names, or `None`.""" self.step = step + """The step number the refusal names, or `None`.""" self.reason = reason + """An irreversible step's declared reason, or `None`.""" self.ahead_only = ahead_only + """Whether `allow_ahead` alone would have let the run through. + + True when the database holds migrations the directory lacks and + nothing else stood in the way. + """ self.contract_steps = list(contract_steps or ()) + """With `ahead_only`, each step of those migrations whose class is not `expand`. + + Each is `NNNN_<name>:NN`: what `allow_ahead="expand-only"` refuses + on. Empty otherwise. + """ self.names = list(names or ()) + """The migrations (`NNNN_<name>`) a refusal about a set of them names. + + A tracked database's records, or the run-applied migrations above a + baseline. Empty otherwise. + """ self.head = head + """The newest of a tracked database's records, for an already-tracked refusal; `None` otherwise.""" @classmethod def from_core( cls, message: str, *, kind: str | None = None, **payload: Any ) -> RunRefused: - """The refusal the core raises for ``kind``: the subclass a caller - branches on when the kind has one (:class:`AlreadyTrackedError`, - :class:`AppliedAboveBaselineError`), plain :class:`RunRefused` - otherwise. Chosen by ``kind`` alone, never by ``message``; ``payload`` - is the keyword data ``RunRefused`` takes.""" + """Build the refusal for `kind`, as the subclass a caller branches on when there is one. + + The class is chosen by `kind` alone, never by `message`. + + Args: + message: The refusal's text, naming the fix. + kind: The planner's refusal kind, or `None` for a refusal that is + not the planner's. + **payload: The other keyword data `RunRefused` takes. + + Returns: + An `AlreadyTrackedError` or `AppliedAboveBaselineError` when the + kind has one, a plain `RunRefused` otherwise. + """ return _BY_KIND.get(kind or "", RunRefused)(message, kind=kind, **payload) class AlreadyTrackedError(RunRefused): - """``baseline()``: the database is already tracked; its tracking table - holds step records, and a baseline records only on a database that has - none (ADR-0031). Nothing was recorded. - - Two pre-deploys that adopt one database at once both find no records and - both call ``baseline()``; the run lock lets one record, and the other - raises this. ``applied`` names every migration the records name - (``NNNN_<name>``, in order), ``head`` the newest of them (both as the - core decided the refusal), and ``report`` is the database's - :class:`StatusReport` (what ``status()`` reports), so the caller carries - on to ``up()``. The target is the migration the previous release's models - generated, so an unattended baseline records no step it did not run:: - + """The refusal `baseline()` raises for a database that is already tracked. + + A baseline records only on a database whose tracking table holds no + step records; nothing was recorded. Two pre-deploys that adopt one + database at once both find no records and both call `baseline()`; the + run lock lets one record, and the other raises this, so the caller + carries on to `up()`. The target is the migration the previous + release's models generated, so an unattended baseline records no step + it did not run. Its `head` is the newest of the records, and its + `report` the database's `StatusReport`, what `status()` reports. + + Examples: + ```python {test=compile reason="needs a migrations project and a database"} try: await ferro.migrations.baseline(target="0001") except ferro.migrations.AlreadyTrackedError: pass await ferro.migrations.up() + ``` """ def __init__(self, message: str, **payload: Any) -> None: super().__init__(message, **payload) self.applied = list(self.names) + """Every migration the records name, as `NNNN_<name>`, in order.""" class AppliedAboveBaselineError(RunRefused): - """``remove_baseline()``: a run applied migrations above the baseline - (``above``, ``NNNN_<name>``); removing it under them would leave applied - migrations above pending ones. The message names the ``ferro migrate - down --to`` that reverts them; nothing was removed. ``report`` is the - database's :class:`StatusReport`. + """The refusal `remove_baseline()` raises when a run applied migrations above the baseline. + + Removing the baseline under them would leave applied migrations above + pending ones. The message names the `ferro migrate down --to` that + reverts them; nothing was removed. Its `report` is the database's + `StatusReport`. """ def __init__(self, message: str, **payload: Any) -> None: super().__init__(message, **payload) self.above = list(self.names) + """The run-applied migrations above the baseline, as `NNNN_<name>`.""" _BY_KIND: dict[str, type[RunRefused]] = { @@ -193,7 +231,7 @@ class _State: def _state(*, word: str | None = None, shown: str | None = None) -> _State: - """The row of :data:`_STATES` with this core ``word`` or this ``shown`` word.""" + """The row of `_STATES` with this core ``word`` or this ``shown`` word.""" for state in _STATES: if state.word == word or state.shown == shown: return state @@ -204,8 +242,16 @@ def _state(*, word: str | None = None, shown: str | None = None) -> _State: def migration_span(names: Sequence[str]) -> str: - """``<first> … <last>`` of migration names in chain order, or the one - name: how an inheritance names what the database applied.""" + """Name a run of migrations as `<first> … <last>`, or the one name. + + This is how an inheritance names what the database applied. + + Args: + names: Migration names, in chain order. + + Returns: + The first and last names joined by an ellipsis, or the only name. + """ return names[0] if len(names) == 1 else f"{names[0]} … {names[-1]}" @@ -214,64 +260,70 @@ class StepStatus: """One step of one migration.""" step: int + """The step's number within its migration.""" file: str """The file this database executes (or executed).""" state: str - """``applied``, ``applied (different checksum)``, ``applied - (baseline)``, ``applied (inherited)``, ``pending``, ``pending - (inheritable)`` (a squash the next run inherits, ADR-0056), ``running``, - ``failed``, ``interrupted`` or ``reverting``.""" + """The state `status` prints, such as `applied`, `pending (inheritable)` or `failed`. + + One of `applied`, `applied (different checksum)`, `applied (baseline)`, + `applied (inherited)`, `pending`, `pending (inheritable)` (a squash the + next run inherits), `running`, `failed`, `interrupted` or `reverting`. + """ error: str | None = None """The last attempt's error, when recorded.""" applied_checksum: str | None = None + """The file's checksum when it was applied, or `None` for a step with no record.""" on_disk_checksum: str | None = None + """The file's checksum as it is on disk now.""" flags: list[str] = field(default_factory=list) - """``destructive`` / ``data-dependent`` headers the file carries.""" + """The `destructive` / `data-dependent` headers the file carries.""" step_class: str = "contract" - """``expand`` or ``contract`` (ADR-0054): whether the previous release's - writes keep succeeding beside the step. The class its record holds, read - as ``contract`` when a record an older ferro wrote holds none; for a - step with no record, the class a run would record.""" + """`expand` or `contract`: whether the previous release's writes keep succeeding beside the step. + + The class its record holds, read as `contract` when a record an older + Ferro wrote holds none; for a step with no record, the class a run would + record. + """ rows_done: int | None = None """The rows a chunked step's current walk has committed.""" finished_at: str | None = None - """When the step's record finished, as stored (ISO-8601 UTC, ``Z``); a - baseline record's is the baseline time. ``None`` for a step with no - finished record (pending, running, failed, interrupted).""" + """When the step's record finished, as stored (ISO-8601 UTC, `Z`). + + A baseline record's is the baseline time. `None` for a step with no + finished record (pending, running, failed, interrupted). + """ @property def shown(self) -> str: - """The state as ``status`` prints it: a failed chunked step says how - far it got (``failed at 30,000 rows``).""" + """The state as `status` prints it, with how far a failed chunked step got (`failed at 30,000 rows`).""" if self.state == "failed" and self.rows_done is not None: return f"failed at {self.rows_done:,} rows" return self.state @property def applied(self) -> bool: - """The step has a finished step record (``applied``, ``applied - (different checksum)``, ``applied (baseline)`` or ``applied - (inherited)``).""" + """Whether the step has a finished step record (any of the `applied` states).""" return _state(shown=self.state).applied @property def unfinished(self) -> bool: - """A run left this step part-way, or is in it (``running``, - ``failed``, ``interrupted`` or ``reverting``).""" + """Whether a run left this step part-way, or is in it (`running`, `failed`, `interrupted` or `reverting`).""" return _state(shown=self.state).unfinished @property def running(self) -> bool: - """A run holding the run lock is in this step.""" + """Whether a run holding the run lock is in this step.""" return _state(shown=self.state).running @property def needs_attention(self) -> bool: + """Whether a person has to look before anything runs (exit 4).""" return _state(shown=self.state).attention @property def pending(self) -> bool: - """No finished step record: still to apply.""" + """Whether the step is still to apply (it has no finished step record).""" return not self.applied @@ -280,19 +332,24 @@ class MigrationStatus: """One migration and its steps.""" number: int + """The migration's number.""" name: str - """``NNNN_<name>``.""" + """The migration's directory name, `NNNN_<name>`.""" steps: list[StepStatus] + """Its steps, in order.""" inherits: list[str] = field(default_factory=list) - """For a squash pending its inheritance, ``NNNN_<name>`` of each migration - whose records witness it, in chain order: what the database applied. - Empty otherwise.""" + """For a squash pending its inheritance, each migration the database applied in its place. + + Each is `NNNN_<name>`, in chain order. Empty otherwise. + """ @property def hint(self) -> str | None: - """What to do about this migration, when ``status`` says it on its - row: for a squash pending inheritance, ``run `ferro migrate up`; this - database applied <first> … <last> and nothing will be run``.""" + """What `status` says to do about this migration on its row, or `None`. + + For a squash pending inheritance: ``run `ferro migrate up`; this + database applied <first> … <last> and nothing will be run``. + """ if not self.inherits: return None return ( @@ -302,21 +359,22 @@ def hint(self) -> str | None: @property def applied_at(self) -> str | None: - """When the migration was applied: the last finished step's - ``finished_at``, so a partial migration reports when it stopped. - ``None`` when no step has finished.""" + """When the migration was applied: the last finished step's `finished_at`, or `None`. + + A partial migration reports when it stopped. + """ return next( (s.finished_at for s in reversed(self.steps) if s.finished_at), None ) @property def applied(self) -> bool: - """Every step is applied.""" + """Whether every step is applied.""" return bool(self.steps) and all(step.applied for step in self.steps) @property def pending(self) -> bool: - """A step is still to apply.""" + """Whether a step is still to apply.""" return any(step.pending for step in self.steps) @property @@ -326,15 +384,18 @@ def unfinished_step(self) -> StepStatus | None: @property def unfinished(self) -> bool: - """A run left this migration part-way: a step is unfinished, or only - some of its steps are applied. A wholly pending migration is not.""" + """Whether a run left this migration part-way. + + True when a step is unfinished, or only some of its steps are + applied. A wholly pending migration is not. + """ return self.unfinished_step is not None or ( any(step.applied for step in self.steps) and not self.applied ) @property def running(self) -> bool: - """A run holding the run lock is in this migration.""" + """Whether a run holding the run lock is in this migration.""" return any(step.running for step in self.steps) @property @@ -349,11 +410,14 @@ def _whole(self) -> str | None: @property def state(self) -> str: - """``applied`` / ``applied (baseline)`` / ``applied (inherited)`` / - ``pending`` / ``pending (inheritable)`` when every - step agrees; ``running`` while a run is in it; ``applied (different - checksum)`` when every step is applied and one changed; else - ``partial, X of N steps``.""" + """The migration's state as `status` prints it on its row. + + The steps' one state when every step agrees (`applied`, `applied + (baseline)`, `applied (inherited)`, `pending` or `pending + (inheritable)`); `running` while a run is in it; `applied (different + checksum)` when every step is applied and one changed; else + `partial, X of N steps`. + """ whole = self._whole if whole is not None: return whole @@ -366,62 +430,74 @@ def state(self) -> str: @property def expands(self) -> bool: - """Whether ``status`` prints this migration's steps without ``--steps``.""" + """Whether `status` prints this migration's steps without `--steps`.""" return self._whole is None @dataclass(frozen=True) class StatusReport: - """What ``ferro migrate status`` prints, and what ``status()`` returns. - - It also says where the database stands, for every caller that has to - know (``drift``, the test harness). For the database the module - docstring shows:: - - report.head_applied.name # "0006_add_teams" - report.unfinished.name # "0007_nickname" - report.pending # True + """Where a database stands against its migrations: what `status()` returns. + + `ferro migrate status` prints it. It also says where the database + stands for every caller that has to know (`drift`, the test harness). + For a database with `0006_add_teams` applied and `0007_nickname` + part-way: + + ```python {test=compile reason="needs a database with a migration part-way applied"} + assert report.head_applied.name == "0006_add_teams" + assert report.unfinished.name == "0007_nickname" + assert report.pending + ``` """ database: str + """The configured database's name.""" url: str - """The database's URL, its secrets masked by ``***``.""" + """The database's URL, its secrets masked by `***`.""" dialect: str + """The database's dialect, `postgres` or `sqlite`.""" table: str - """``<schema>._ferro_migrations``.""" + """The tracking table, `<schema>._ferro_migrations`.""" migrations: list[MigrationStatus] + """Every migration in the directory, in order.""" ahead: list[str] = field(default_factory=list) - """Migrations this database has applied that the directory lacks. Never - the originals of a squash it is pending the inheritance of (ADR-0038).""" + """Migrations this database has applied that the directory lacks. + + Never the originals of a squash the database is pending the inheritance + of. + """ inheritable: list[str] = field(default_factory=list) - """Each squash the next run inherits (``NNNN_<name>``), running nothing - (ADR-0056): pending, whatever ``allow_ahead`` says.""" + """Each squash the next run inherits (`NNNN_<name>`), running nothing, whatever `allow_ahead` says.""" refusal: str | None = None - """What ``up`` would refuse with, when it would.""" + """What `up` would refuse with, when it would.""" refusal_needs_attention: bool = False + """Whether `refusal` is one a person has to look at (exit 4) rather than a pending state.""" @property def head_applied(self) -> MigrationStatus | None: - """The last migration every step of which is applied: the one whose - schema snapshot the database is measured against. ``None`` when no - migration is.""" + """The last fully applied migration, or `None` when no migration is. + + Its schema snapshot is the one the database is measured against. + """ applied = [m for m in self.migrations if m.applied] return applied[-1] if applied else None @property def unfinished(self) -> MigrationStatus | None: - """The first migration a run left part-way (or is in): the database - stands at neither its parent's snapshot nor its own.""" + """The first migration a run left part-way (or is in), or `None`. + + The database stands at neither its parent's snapshot nor its own. + """ return next((m for m in self.migrations if m.unfinished), None) @property def pending(self) -> bool: - """Anything still to apply.""" + """Whether anything is still to apply.""" return any(m.pending for m in self.migrations) @property def needs_attention(self) -> bool: - """A person has to look before anything runs.""" + """Whether a person has to look before anything runs.""" return ( bool(self.ahead) or (self.refusal is not None and self.refusal_needs_attention) @@ -430,7 +506,7 @@ def needs_attention(self) -> bool: @property def exit_code(self) -> int: - """4 when anything needs attention, 3 when anything is pending, else 0.""" + """The exit code `status` exits with: 4 when anything needs attention, 3 when anything is pending, else 0.""" # Imported here so the in-process API never loads the CLI package. from ..cli import exit_codes @@ -453,11 +529,23 @@ def from_core( rows_done: dict[tuple[int, int], int] | None = None, finished_at: dict[tuple[int, int], str | None] | None = None, ) -> StatusReport: - """Build the report from the tracked database's ``status()`` - document; ``url`` is the database's URL, masked; ``refusal`` replaces - its refusal when given; ``rows_done`` - holds each chunked step's committed rows by ``(migration, step)`` and - ``finished_at`` each step record's stored finish time.""" + """Build the report from the tracked database's status document. + + Args: + raw: The status document the core returns. + database: The configured database's name. + url: The database's URL, its secrets masked. + dialect: The database's dialect. + table: The tracking table, `<schema>._ferro_migrations`. + refusal: A refusal that replaces the document's own, when given. + rows_done: Each chunked step's committed rows, by + `(migration, step)`. + finished_at: Each step record's stored finish time, by + `(migration, step)`. + + Returns: + The report. + """ rows_done = rows_done or {} finished_at = finished_at or {} migrations = [ @@ -498,8 +586,16 @@ def from_core( ) def render(self, steps: bool = False) -> str: - """The text ``ferro migrate status`` prints (``--steps``: every - migration's steps, each with its class after its flags).""" + """Render the text `ferro migrate status` prints. + + Args: + steps: Print every migration's steps, each with its class after + its flags, as `--steps` does. Defaults to only the + migrations that need attention. + + Returns: + The report as plain text, one line per migration or step. + """ lines = [f"{self.database} ({self.dialect}) · {self.table}", ""] names = [m.name for m in self.migrations] + self.ahead name_width = max((len(name) for name in names), default=0) @@ -528,7 +624,11 @@ def render(self, steps: bool = False) -> str: return "\n".join(lines) def to_json(self) -> str: - """The same report as a JSON document (``status --json``).""" + """Render the report as the JSON document `status --json` prints. + + Returns: + The report as indented JSON. + """ return json.dumps( { "database": self.database, diff --git a/src/ferro/migrations/rerecord.py b/src/ferro/migrations/rerecord.py index 241aa47d..7653d260 100644 --- a/src/ferro/migrations/rerecord.py +++ b/src/ferro/migrations/rerecord.py @@ -1,4 +1,4 @@ -"""``ferro migrate rerecord``: accept a deliberate edit of an applied step (ADR-0030). +"""`ferro migrate rerecord`: accept a deliberate edit of an applied step. ```text $ ferro migrate up @@ -11,24 +11,23 @@ re-recorded 0007_nickname/01_add_nickname.up.postgres.sql (sha384:3f9a…c1 → sha384:b7e2…09); nothing was run ``` -A step record's checksum covers the up file this database executed, in -its own dialect. ``up`` never runs a finished step again, so an edit to -one is refused until the operator says it is deliberate; ``rerecord`` -then changes the record and nothing else: no statement of the step runs. +A step record's checksum covers the up file this database executed, in its +own dialect. `up` never runs a finished step again, so an edit to one is +refused until the operator says it is deliberate; `rerecord` then changes +the record and nothing else: no statement of the step runs. An unfinished chunked step whose batches already committed rows is the -other case that needs a decision (``up`` refuses it): ``--continue`` keeps -the committed rows and resumes from the cursor, allowed only while the -edited query pages over the same order keys the cursor was committed -under; ``--restart`` clears the cursor and ``rows_done`` so the next ``up`` -starts from the first row. - -The Rust core decides (the locked run's ``plan_rerecord``: which record, -which refusal) and writes (its ``rerecord``: one statement under the run -lock); this module sequences the two and reads the facts only Python can: -a data step's declared kind and an edited chunked query's order keys. -``rerecord`` is a CLI verb, never part of the application API -(ADR-0045); the CLI calls :func:`rerecord` with ``url=``, as every verb. +other case that needs a decision (`up` refuses it): `--continue` keeps the +committed rows and resumes from the cursor, allowed only while the edited +query pages over the same order keys the cursor was committed under; +`--restart` clears the cursor and `rows_done` so the next `up` starts from +the first row. + +The core decides (which record, which refusal) and writes (one statement +under the run lock); this module sequences the two and reads the facts only +Python can: a data step's declared kind and an edited chunked query's order +keys. `rerecord` is a CLI verb, never part of the application API; the CLI +calls `rerecord` with `url=`, as every verb. """ from __future__ import annotations @@ -56,23 +55,22 @@ @dataclass(frozen=True) class RerecordReport: - """What ``rerecord`` changed: the record of ``file`` - (``NNNN_<name>/<file>``) now holds ``new_checksum`` where it held - ``old_checksum``.""" + """What `rerecord` changed: the record of `file` now holds `new_checksum` where it held `old_checksum`.""" file: str + """The step, `NNNN_<name>/<file>`.""" old_checksum: str + """The checksum the record held, as SHA-384 hex.""" new_checksum: str + """The checksum the record holds now: the file's on disk.""" mode: Mode = "record" + """`"record"`, `"continue"` or `"restart"`.""" notes: list[str] = field(default_factory=list) - """Warnings taking the lock gave: the records of a migration that does - not regenerate keep no class (ADR-0054).""" + """Warnings taking the lock gave: the records of a migration that does not regenerate keep no class.""" @property def note(self) -> str | None: - """What the re-record leaves the next ``up`` to do, by its mode - (``the next up continues from its cursor``); ``None`` for a plain - ``record``.""" + """What the re-record leaves the next `up` to do, by its mode; `None` for a plain `record`.""" return { "record": None, "continue": "the next up continues from its cursor", @@ -83,7 +81,11 @@ def note(self) -> str | None: }[self.mode] def render(self) -> str: - """The line ``ferro migrate rerecord`` prints.""" + """Render the line `ferro migrate rerecord` prints. + + Returns: + The line, ending `nothing was run`. + """ done = "" if self.note is None else f"; {self.note}" return ( f"re-recorded {self.file} (sha384:{self.old_checksum} → " @@ -101,25 +103,39 @@ async def rerecord( mode: Mode = "record", lock_timeout: str | float | None = None, ) -> RerecordReport: - """Accept a deliberate edit of step ``target`` (``"0007:01"``) under the - run lock: its record takes the file's checksum (and file name and kind), - and nothing of the step runs. - - ``mode`` is ``"continue"`` or ``"restart"`` for an unfinished chunked - step with committed batches, and ``"record"`` for anything else. - Works on the database and connection - :meth:`~ferro.migrations.target.Target.resolve` picks. A second run waits - up to ``lock_timeout`` (the database's configured ``lock_timeout`` when - ``None``). + """Accept a deliberate edit of one applied step, under the run lock. + + This is `ferro migrate rerecord`. The step's record takes the file's + checksum (and file name and kind), and nothing of the step runs. + + Args: + settings: The project's configuration. Defaults to `FerroSettings()`. + database: The configured database. Defaults to the only one + configured. + using: An open connection to work on. Defaults to the default + connection. + url: A URL to open a private connection to, closed after the call. + Refused together with `using`. + target: The step, `"0007:01"`. + mode: `"continue"` or `"restart"` for an unfinished chunked step + with committed batches; `"record"` for anything else. + lock_timeout: How long to wait for another run's lock: `"30s"`, + `"500ms"`, `"1m"` or a number of seconds; `0` refuses at once. + Defaults to the database's `lock_timeout`. + + Returns: + What the re-record changed. Raises: - RunRefused: ``target`` is not one step (a migration alone, the - snapshot: restore the file), names a step with no record (free to - edit) or an unchanged one, ``mode`` does not fit the step, the - edited chunked step needs ``--continue`` or ``--restart``, or - ``--continue`` meets changed order keys. Each names the fix; the - refusal's ``kind`` says which. - MigrationRefused: an edited data step's file does not load. + SettingsError: `mode` is not `"record"`, `"continue"` or + `"restart"`. + RunRefused: `target` is not one step (a migration alone, the + snapshot: restore the file), names a step with no record (free + to edit) or an unchanged one, `mode` does not fit the step, the + edited chunked step needs `continue` or `restart`, or `continue` + meets changed order keys. Each names the fix; the refusal's + `kind` says which. + MigrationRefused: An edited data step's file does not load. """ if mode not in _MODES: raise SettingsError(f"rerecord mode {mode!r} is not one of {', '.join(_MODES)}") diff --git a/src/ferro/migrations/runner.py b/src/ferro/migrations/runner.py index c9d8365e..def620a8 100644 --- a/src/ferro/migrations/runner.py +++ b/src/ferro/migrations/runner.py @@ -1,4 +1,4 @@ -"""``ferro migrate up``, ``down`` and ``status``: the run loop (ADR-0028, ADR-0048). +"""`ferro migrate up`, `down` and `status`: the run loop. ```text $ ferro migrate up @@ -6,59 +6,62 @@ 0002_add_teams 01_schema applied (9 ms) ``` -```python +```python {test=compile reason="needs a project with a migrations directory"} await ferro.migrations.up() # the same run, in-process await ferro.migrations.require_applied() # where it stands, raised when behind ``` Each verb here is one function the CLI and the application both call, -``(settings=None, database=None, *, using=None, url=None, ...)``: the -database and connection come from :meth:`Target.resolve -<ferro.migrations.target.Target.resolve>`. A refused or failed run raises -:class:`~ferro.migrations.errors.MigrationRefused` carrying its -:class:`RunReport` as ``.report``, for every caller; the CLI prints the -report's refusal and exits 1. - -Python sequences a run; the Rust core decides and executes it. :func:`up` -opens the database as one object (``_core._open_tracked``: its records and -one read of the migrations directory, held), takes the run lock as a block -(``tracked.locked(...)``, whose run object re-reads the records under the -lock), asks that run for its plan, and walks the plan's opaque step -handles: ``run.execute(step)`` runs a SQL step from the held bytes and -settles its record. Nothing here decides which step runs, how, or whether -a file may run at all, and nothing here builds a step record. +`(settings=None, database=None, *, using=None, url=None, ...)`: the +database and connection come from `Target.resolve`. A refused or failed run +raises `MigrationRefused` carrying its `RunReport` as `.report`, for every +caller; the CLI prints the report's refusal and exits 1. + +Python sequences a run; the core decides and executes it. `up` opens the +database as one object (its records and one read of the migrations +directory, held), takes the run lock as a block (whose run object re-reads +the records under the lock), asks that run for its plan, and walks the +plan's opaque step handles: `run.execute(step)` runs a SQL step from the +held bytes and settles its record. Nothing here decides which step runs, +how, or whether a file may run at all, and nothing here builds a step +record. Each SQL step waits for table locks under the database's -``ddl_lock_timeout`` (ADR-0044); a step that times out is retried from its -first statement, and each attempt is a progress line:: +`ddl_lock_timeout`; a step that times out is retried from its first +statement, and each attempt is a progress line: - 0003_add_slug 01_expand waiting for a lock on "author" (attempt 1 of 10, retry in 1s) - 0003_add_slug 01_expand applied (2214 ms) +```text +0003_add_slug 01_expand waiting for a lock on "author" (attempt 1 of 10, retry in 1s) +0003_add_slug 01_expand applied (2214 ms) +``` -:func:`down` walks back (ADR-0033) through the same walk over the run's -``Down`` plan, each step's ``.down`` file run by the same executor, which -removes the step's record in the down's own transaction:: +`down` walks back through the same walk over the run's down plan, each +step's `.down` file run by the same executor, which removes the step's +record in the down's own transaction: - $ ferro migrate down - Revert 0002_add_teams (1 step)? [y/N] y - 0002_add_teams 01_schema reverted (8 ms) +```text +$ ferro migrate down +Revert 0002_add_teams (1 step)? [y/N] y +0002_add_teams 01_schema reverted (8 ms) +``` -A data step (``NN_<name>.py``, ADR-0024) is Python's to run: every planned -one is loaded before anything runs (its checksum checked, its declarations -read, every ``todo`` refused by file and line), then each runs on the run's -connection inside one transaction under its migration's historical models -(:meth:`ferro.registry.Registry.swap`), its record moved through the run's -named transitions (``start``, then ``finish`` inside the step's -transaction):: +A data step (`NN_<name>.py`) is Python's to run: every planned one is +loaded before anything runs (its checksum checked, its declarations read, +every `todo` refused by file and line), then each runs on the run's +connection inside one transaction under its migration's historical models, +its record moved through the run's named transitions (`start`, then +`finish` inside the step's transaction): - 0011_backfill_slugs 01_backfill_author applied (40 ms) +```text +0011_backfill_slugs 01_backfill_author applied (40 ms) +``` -A ``@chunked`` step runs one transaction per batch instead, its cursor -committed with each batch (:mod:`ferro.migrations.chunked`), and its query -is checked over the historical models before anything runs. +A `@chunked` step runs one transaction per batch instead, its cursor +committed with each batch, and its query is checked over the historical +models before anything runs. -:func:`status` and :func:`require_applied` read the same object with no -lock and create nothing. +`status` and `require_applied` read the same object with no lock and +create nothing. """ from __future__ import annotations @@ -80,7 +83,7 @@ from ..settings import MAX_LOCK_TIMEOUT_S, SettingsError, parse_lock_timeout from ..state import resolve_operation_scope from . import historical -from .chunked import BatchFailed, order_keys, run_chunked +from ._chunk_loop import BatchFailed, order_keys, run_chunked from .context import HistoricalModels, StepContext from .errors import DatabaseAheadError, MigrationRefused, PendingMigrationsError from .historical import HistoricalModelError @@ -127,10 +130,12 @@ ] AllowAhead: TypeAlias = bool | Literal["expand-only"] -"""How far :func:`up` and :func:`require_applied` let a database ahead of -the checkout through (ADR-0038, ADR-0054): ``False`` refuses any applied -migration the directory lacks, ``"expand-only"`` admits them when every -step they hold is of class ``expand``, ``True`` admits any distance.""" +"""How far `up` and `require_applied` let a database ahead of the checkout through. + +`False` refuses any applied migration the directory lacks, `"expand-only"` +admits them when every step they hold is of class `expand`, `True` admits +any distance. +""" _STEP_STEM = re.compile(r"(\.(up|down)(\.[a-z]+)?\.sql|\.py)$") @@ -138,44 +143,44 @@ class AppliedStep(NamedTuple): - """One step a run applied.""" + """One step a run applied or reverted.""" migration: str - """``NNNN_<name>``.""" + """The migration, `NNNN_<name>`.""" step: str - """``NN_<name>``.""" + """The step, `NN_<name>`.""" ms: int + """How long the step ran, in milliseconds.""" @dataclass(frozen=True) class StepFailure: - """The step a run stopped at: its error and, apart, what the failure - left applied and how to resume. :attr:`RunReport.refusal` is - :attr:`message`.""" + """The step a run stopped at: its error, what the failure left applied, and how to resume. + + The run report's `refusal` is its `message`. + """ migration: str - """``NNNN_<name>``.""" + """The migration, `NNNN_<name>`.""" file: str - """The step's file (``02_backfill_account.py``).""" + """The step's file (`02_backfill_account.py`).""" duration_ms: int + """How long the step ran before it failed, in milliseconds.""" error: str - """What failed (``NotNullViolationError: null value in column ...``).""" + """What failed (`NotNullViolationError: null value in column ...`).""" hint: str - """What stayed applied and how to resume, ending ``Nothing after it - ran.``""" + """What stayed applied and how to resume (its last sentence is `Nothing after it ran.`).""" rows_done: int | None = None - """The rows a chunked step's committed batches did before it failed; - ``None`` for any other step.""" + """The rows a chunked step's committed batches did before it failed; `None` for any other step.""" @property def stem(self) -> str: - """The step's name, its file without the suffix (``01_expand``).""" + """The step's name, its file without the suffix (`01_expand`).""" return _stem(self.file) @property def message(self) -> str: - """``ferro migrate: <migration>/<file> failed: <error>``, then the - hint on its own line.""" + """The failure as the run prints it: `ferro migrate: <migration>/<file> failed: <error>`, then the hint.""" return ( f"ferro migrate: {self.migration}/{self.file} failed: {self.error}\n" f"{self.hint}" @@ -184,20 +189,20 @@ def message(self) -> str: @dataclass(frozen=True) class InheritedMigration: - """A squash a run inherited before it planned (ADR-0056): the database had - applied every migration it replaced, so the run swapped their step - records for the squash's and ran nothing.""" + """A squash a run inherited before it planned. + + The database had applied every migration the squash replaced, so the run + swapped their step records for the squash's and ran nothing. + """ squash: str - """``NNNN_<name>`` of the squash.""" + """The squash, `NNNN_<name>`.""" replaced: list[str] - """``NNNN_<name>`` of each migration whose records it replaced, in chain - order: what the database had applied.""" + """Each migration whose records it replaced (`NNNN_<name>`), in chain order: what the database had applied.""" @property def line(self) -> str: - """``inherited <squash>: this database had applied <first> … <last>; - nothing was run``.""" + """The line a run prints: `inherited <squash>: this database had applied <first> … <last>; nothing was run`.""" return ( f"inherited {self.squash}: this database had applied " f"{migration_span(self.replaced)}; nothing was run" @@ -206,42 +211,57 @@ def line(self) -> str: @dataclass class RunReport: - """What :func:`up` or :func:`down` did.""" + """What a migration run did: what [`up`][ferro.migrations.up] returns. + + A refused run raises `MigrationRefused` with this report as `.report`, + saying what the run applied before it stopped. + + Examples: + ```python {test=compile reason="needs a migrations project and a database"} + report = await ferro.migrations.up() + for step in report.applied: + print(step.migration, step.step, step.ms) + # 0002_add_teams 01_schema 9 + ``` + """ inherited: list[InheritedMigration] = field(default_factory=list) """The squashes the run inherited before anything else, in order.""" applied: list[AppliedStep] = field(default_factory=list) """The steps applied, in order.""" reverted: list[AppliedStep] = field(default_factory=list) - """The steps :func:`down` reverted, newest first.""" + """The steps a down run reverted, newest first.""" declined: bool = False - """:func:`down`'s ``confirm`` said no: nothing was reverted.""" + """Whether a down run's `confirm` said no, so nothing was reverted.""" refusal: str | None = None - """Why the run stopped: a refusal before anything ran, a lost lock, or a - failed step (its error and how to resume). ``None`` when it finished.""" + """Why the run stopped: a refusal before anything ran, a lost lock, or a failed step; `None` when it finished. + + For a failed step it holds the error and how to resume. + """ refused: RunRefused | None = None - """The refusal itself when the run was refused (``refusal`` is its - text): its ``kind``, ``migration``, ``step`` and ``reason`` say what it - is about. ``None`` when the run finished or a step failed.""" + """The refusal itself when the run was refused; `None` when it finished or a step failed. + + `refusal` is its text; its `kind`, `migration`, `step` and `reason` say + what it is about. + """ notes: list[str] = field(default_factory=list) """What the run accepted on the way (an edited unfinished step).""" ahead: list[str] = field(default_factory=list) - """Applied migrations the directory lacks: let through by - ``allow_ahead``, or, when they alone refused the run - (``refused.ahead_only``), the ones that did.""" + """Applied migrations the directory lacks: let through by `allow_ahead`, or the ones that refused the run.""" failed: StepFailure | None = None - """The step the run stopped at when one failed (``refusal`` is its - message); ``None`` when the run finished or was refused.""" + """The step the run stopped at when one failed (`refusal` is its message); `None` otherwise.""" direction: Literal["up", "down"] = "up" - """Which way the run went: :func:`up` or :func:`down`.""" + """Which way the run went: `"up"` or `"down"`.""" def _refuse(self, refused: RunRefused) -> None: self.refusal, self.refused = str(refused), refused def _raise_if_refused(self) -> RunReport: - """This report, or :class:`MigrationRefused` carrying it when the run - refused or a step failed: :class:`DatabaseAheadError` when applied - migrations the directory lacks were the only thing in the way.""" + """Return this report, or raise `MigrationRefused` carrying it when the run refused or a step failed. + + The error is a `DatabaseAheadError` when applied migrations the + directory lacks were the only thing in the way. + """ if self.refusal is None: return self if self.refused is not None and self.refused.ahead_only: @@ -255,19 +275,35 @@ def _raise_if_refused(self) -> RunReport: def tracking_schema_for(database: DatabaseSettings, dialect: str) -> str | None: - """Where ``database``'s tracking tables live on ``dialect``: its - ``tracking_schema`` on Postgres, else the governed schema (``None``).""" + """Return the schema a database's tracking tables live in on a dialect. + + Args: + database: The configured database. + dialect: `"postgres"` or `"sqlite"`. + + Returns: + The database's `tracking_schema` on Postgres; `None` (the governed + schema) otherwise. + """ return database.tracking_schema if dialect == "postgres" else None async def open_tracked(name: str, database: DatabaseSettings) -> TrackedDatabase: - """``database``'s tracking tables and migrations directory on open - connection ``name``: its records and one read of the directory, held - (ADR-0048). Creates nothing and takes no lock. + """Open a database's tracking tables and migrations directory on an open connection. + + The result holds its records and one read of the directory. Creates + nothing and takes no lock. + + Args: + name: The open connection's name. + database: The configured database. + + Returns: + The tracked database. Raises: - SettingsError: ``name`` is not open, or ``database`` does not target - its dialect. + SettingsError: `name` is not open, or `database` does not target its + dialect. """ dialect = connection_dialect(name, database) return await _core._open_tracked( @@ -279,17 +315,28 @@ async def open_tracked(name: str, database: DatabaseSettings) -> TrackedDatabase def say_waiting(text: str) -> None: - """Say on stderr, at once, that a run waits for another run's lock.""" + """Say on stderr, at once, that a run waits for another run's lock. + + Args: + text: The line to print. + """ print(text, file=sys.stderr, flush=True) def warn_unclassified(run: LockedDatabase) -> list[str]: - """Warn, once per migration, of the records taking the lock left without - a class (ADR-0054): a verb that reads no class (``down``, ``rerecord``, - ``baseline --remove``) does not refuse on a migration that does not - regenerate. Those records read as ``contract`` until the next ``up`` or - ``baseline`` classifies them or refuses naming the fix. Returns the - warnings.""" + """Warn, once per migration, of the records taking the lock left without a class. + + A verb that reads no class (`down`, `rerecord`, `baseline --remove`) + does not refuse on a migration that does not regenerate. Those records + read as `contract` until the next `up` or `baseline` classifies them or + refuses naming the fix. + + Args: + run: The locked database. + + Returns: + The warnings, each also raised as a `UserWarning`. + """ deferred = list(run.class_warnings) for text in deferred: warnings.warn(text, UserWarning, stacklevel=3) @@ -302,22 +349,24 @@ def _stem(file: str) -> str: @dataclass(frozen=True) class StepEvent: - """One step a run finished: applied, reverted, reverted with nothing to - reverse, or failed. :meth:`line` is the line ``progress`` receives for - it (``0002_add_team 01_expand applied (412 ms)``); a failed step's is - ``None``, as ``progress`` hears nothing of it (its message is the run's - refusal, :attr:`RunReport.failed`).""" + """One step a run finished: applied, reverted, reverted with nothing to reverse, or failed. + + `line()` is the line `progress` receives for it + (`0002_add_team 01_expand applied (412 ms)`); a failed step's is + `None`, as `progress` hears nothing of it (its message is the run's + refusal, in the report's `failed`). + """ migration: str - """``NNNN_<name>``.""" + """The migration, `NNNN_<name>`.""" file: str - """The step's file (``01_expand.up.postgres.sql``).""" + """The step's file (`01_expand.up.postgres.sql`).""" outcome: Literal["applied", "reverted", "nothing to reverse", "failed"] + """What happened: `"applied"`, `"reverted"`, `"nothing to reverse"` or `"failed"`.""" duration_ms: int | None - """How long the step ran; ``None`` when it had nothing to reverse.""" + """How long the step ran; `None` when it had nothing to reverse.""" reason: str | None - """Why its down runs no statement, when ``outcome`` is ``nothing to - reverse``.""" + """Why its down runs no statement, when `outcome` is `"nothing to reverse"`.""" name_width: int = 0 """The run's migration column width: its longest migration name.""" stem_width: int = 0 @@ -325,13 +374,12 @@ class StepEvent: @property def stem(self) -> str: - """The step's name, its file without the suffix (``01_expand``).""" + """The step's name, its file without the suffix (`01_expand`).""" return _stem(self.file) @property def said(self) -> str: - """What happened, as the line ends it: ``applied (412 ms)``, - ``reverted (8 ms)`` or ``nothing to reverse: <reason>``.""" + """What happened, as the line ends it: `applied (412 ms)`, `reverted (8 ms)` or `nothing to reverse: <reason>`.""" if self.outcome == "nothing to reverse": return f"nothing to reverse: {self.reason}" return f"{self.outcome} ({self.duration_ms} ms)" @@ -339,8 +387,15 @@ def said(self) -> str: def line( self, name_width: int | None = None, stem_width: int | None = None ) -> str | None: - """``<migration> <step> <said>``, the columns padded to the run's - widths unless others are given; ``None`` for a failed step.""" + """Render the progress line `<migration> <step> <said>`. + + Args: + name_width: The migration column's width. Defaults to the run's. + stem_width: The step column's width. Defaults to the run's. + + Returns: + The line, or `None` for a failed step. + """ if self.outcome == "failed": return None names = self.name_width if name_width is None else name_width @@ -350,30 +405,43 @@ def line( @dataclass(frozen=True) class RunNote: - """Any other line a run says: a step waiting for a table lock - (``0003_add_slug 01_add_slug waiting for a lock on "author" ...``), an - edited unfinished step re-recorded before it runs.""" + """Any other line a run says, such as a step waiting for a table lock. + + For example `0003_add_slug 01_add_slug waiting for a lock on "author" ...`, + or an edited unfinished step re-recorded before it runs. + """ text: str + """The line.""" def line(self) -> str: - """The line ``progress`` receives: the text itself.""" + """Return the line `progress` receives: the text itself. + + Returns: + The text. + """ return self.text @dataclass(frozen=True) class InheritedEvent: - """A squash the run inherited, said before any step - (``inherited 0005_teams_epic: this database had applied 0005_add_teams … - 0009_fix_rows; nothing was run``).""" + """A squash the run inherited, said before any step. + + Its line reads `inherited 0005_teams_epic: this database had applied + 0005_add_teams … 0009_fix_rows; nothing was run`. + """ squash: str - """``NNNN_<name>`` of the squash.""" + """The squash, `NNNN_<name>`.""" replaced: list[str] - """``NNNN_<name>`` of each migration whose records it replaced.""" + """Each migration whose records it replaced, `NNNN_<name>`.""" def line(self) -> str: - """The line ``progress`` receives (:attr:`InheritedMigration.line`).""" + """Return the line `progress` receives. + + Returns: + The same line the run report's `InheritedMigration.line` holds. + """ return InheritedMigration(self.squash, list(self.replaced)).line @@ -384,7 +452,7 @@ def line(self) -> str: def _declared_kind(path: Path) -> str: """The shape (``atomic`` / ``chunked``) an inherited data step's ``up`` declares, read from its syntax tree: inheritance runs no step, so its - record holds the declaration, as a baseline's does (ADR-0056).""" + record holds the declaration, as a baseline's does.""" try: return declared_up_kind(Path(path)) except StepRefused as refused: @@ -395,7 +463,7 @@ async def _inherit( run: LockedDatabase, tell: Callable[[RunEvent], None] ) -> list[InheritedMigration]: """Inherit, on ``run``, every squash whose witness its records hold, before - anything is planned (ADR-0056), telling ``tell`` one event per squash + anything is planned, telling ``tell`` one event per squash inherited. Raises ``RunRefused`` (``partial_witness``) on a squash the records hold part of, and (``mixed_witness``) on one a baseline recorded part of, with nothing written.""" @@ -423,42 +491,57 @@ async def up( progress: Callable[[str], Any] | None = None, on_event: Callable[[RunEvent], Any] | None = None, ) -> RunReport: - """Apply every pending step of every pending migration, in order, under - the run lock (``ferro migrate up``). - - Works on ``database`` of ``settings`` over ``using``, a private - connection to ``url``, or the default connection - (:meth:`~ferro.migrations.target.Target.resolve`). A second run waits up - to ``lock_timeout`` (the database's configured ``lock_timeout`` when - ``None``), saying so on stderr at once. ``allow_ahead`` lets a - database holding migrations the directory lacks through (ADR-0038): - ``"expand-only"`` when every step they hold is ``expand`` (ADR-0054), - ``True`` whatever they hold (:data:`AllowAhead`). - ``progress`` receives each line as the run goes: one per squash it - inherits (before any step, ADR-0056), one per applied step, and one per - attempt a step makes while it waits for a table lock under the - database's ``ddl_lock_timeout``. ``on_event`` receives the same lines as - data, each just before ``progress`` receives its ``line()``: an - :class:`InheritedEvent` per squash inherited (its ``squash`` and the - ``replaced`` migrations, with no ``.text`` or ``.step``), a - :class:`StepEvent` per applied step, a :class:`RunNote` for any other; - a failed step is a ``StepEvent`` too, with no line for ``progress``. - - Returns the :class:`RunReport` of what it did: ``.inherited`` the squashes - it inherited, in order (an :class:`InheritedMigration` each), then - ``.applied`` the steps it applied. + """Apply every pending migration, in order, under the run lock. + + This is `ferro migrate up`. A second run waits for the lock, saying so + on stderr at once. See + [Deploying migrations](../guide/schema/deploying.md). + + Args: + settings: The project's configuration. Defaults to `FerroSettings()`. + database: The configured database. Defaults to the only one + configured. + using: An open connection to work on. Defaults to the default + connection. + url: A URL to open a private connection to, closed after the call. + Refused together with `using`. + lock_timeout: How long to wait for another run's lock: `"30s"`, + `"500ms"`, `"1m"` or a number of seconds; `0` refuses at once. + Defaults to the database's `lock_timeout`. + allow_ahead: Run beside a database that has applied migrations the + directory lacks: `"expand-only"` when every step they hold is + `expand` (the previous release of a rolling deploy), `True` + whatever they hold. Defaults to `False`, which refuses. + progress: Receives each line as the run goes: one per squash it + inherits (before any step), one per applied step, and one per + attempt a step makes while it waits for a table lock. + on_event: Receives the same lines as data, each just before + `progress` receives its `line()`: an `InheritedEvent` per squash + inherited, a `StepEvent` per applied step, a `RunNote` for any + other. A failed step is a `StepEvent` too, with no line for + `progress`. + + Returns: + What the run did: the squashes it inherited, in order, then the + steps it applied. Raises: - DatabaseAheadError: the database has applied migrations the - directory lacks, and nothing else is in the way - (``allow_ahead=True`` runs beside them; ``.contract_steps`` names - the steps ``"expand-only"`` refuses on). - MigrationRefused: the run refused or a step failed; the message says - why and how to resume, and ``.report`` is the run's report. Also - the configuration naming no single database, and a connection - the run cannot work on. - TypeError: ``allow_ahead`` is not ``False``, ``True`` or - ``"expand-only"``. + DatabaseAheadError: The database has applied migrations the + directory lacks, and nothing else is in the way. + MigrationRefused: The run refused or a step failed; the message says + why and how to resume, and `.report` is the run's report. Also + raised when the configuration names no single database, or for + a connection the run cannot work on. + TypeError: `allow_ahead` is not `False`, `True` or `"expand-only"`. + + Examples: + ```python {test=compile reason="needs a migrations project and a database"} + await ferro.connect(url) + report = await ferro.migrations.up(progress=print) + # 0002_add_teams 01_schema applied (9 ms) + print([step.migration for step in report.applied]) + # ['0002_add_teams'] + ``` """ allow_ahead = _core._allow_ahead(allow_ahead) target = Target.resolve(settings, database, using=using, url=url) @@ -483,16 +566,16 @@ async def _up( on_event: Callable[[RunEvent], Any] | None = None, through: str | None = None, ) -> RunReport: - """:func:`up` on open connection ``name``, raising as it does. + """`up` on open connection ``name``, raising as it does. ``through`` (``"0007"``, a migration number) stops the run after that migration: only pending steps of migrations up to and including it run. - It is the test harness's target (ADR-0045); the application's ``up()`` - has none (ADR-0040), so it is reachable only here. A number the + It is the test harness's target; the application's ``up()`` + has none, so it is reachable only here. A number the directory lacks is refused, and so is a step (``"0007:02"``): neither is a target. A ``through`` at or below the last applied migration has nothing pending to run: the run applies nothing and reverts nothing - (going down is :func:`down`'s). + (going down is `down`'s). """ timeout = database.lock_wait(lock_timeout) direction: dict[str, Any] = {"direction": "up"} @@ -646,13 +729,20 @@ def _key(step: StepHandle) -> tuple[int, int]: def historical_models(migrations: dict[str, Any], number: int) -> HistoricalModels: - """The historical models of migration ``number`` (ADR-0035), built from - the held directory read (``run.migrations``): its snapshot over its - parent's. The one builder, for a data step's run and for the order keys - of an edited chunked step alike. + """Build the historical models of one migration: its snapshot over its parent's. + + The one builder, for a data step's run and for the order keys of an + edited chunked step alike. + + Args: + migrations: The held read of the migrations directory. + number: The migration's number. + + Returns: + The historical models. Raises: - HistoricalModelError: the snapshots do not build. + HistoricalModelError: The snapshots do not build. """ by_number = {m["number"]: m for m in migrations["migrations"]} own = by_number[number] @@ -667,21 +757,25 @@ def historical_models(migrations: dict[str, Any], number: int) -> HistoricalMode def order_keys_on_disk( tracked: TrackedDatabase | LockedDatabase, ) -> list[tuple[int, int, list[str]]]: - """The order keys each cursor-holding chunked step's file pages over - today, as the run planner reads them (``[(migration, step, ["author.id", - ...]), ...]``; an empty list for a step whose function is no longer - ``@chunked``). - - A fact only Python can read (the query is a function of the migration's - historical models), read for every record that is an unfinished - chunked step holding a cursor: its ``up``'s keys for an up that stopped - part-way, its ``down``'s for a record ``reverting`` at a revert cursor. - The planner decides whether its file was edited and whether its cursor - is still a position in the edited query (ADR-0030). A directory that - does not read gives no facts; the planner refuses it with its own text. + """Read the order keys each cursor-holding chunked step's file pages over today. + + Only Python can read this (the query is a function of the migration's + historical models). It is read for every record that is an unfinished + chunked step holding a cursor: its `up`'s keys for an up that stopped + part-way, its `down`'s for a record `reverting` at a revert cursor. The + planner decides whether its file was edited and whether its cursor is + still a position in the edited query. A directory that does not read + gives no facts; the planner refuses it with its own text. + + Args: + tracked: The tracked database, locked or not. + + Returns: + Each step as `(migration, step, ["author.id", ...])`; the list is + empty for a step whose function is no longer `@chunked`. Raises: - MigrationRefused: such a step's file does not load, or its query + MigrationRefused: Such a step's file does not load, or its query does not build over its historical models. """ wanted = sorted( @@ -726,7 +820,7 @@ def order_keys_on_disk( def _order_keys_for( run: LockedDatabase, nothing: str ) -> list[tuple[int, int, list[str]]]: - """:func:`order_keys_on_disk` for a run that refuses on a file that does + """`order_keys_on_disk` for a run that refuses on a file that does not load (``Nothing was <nothing>.``).""" try: return order_keys_on_disk(run) @@ -746,7 +840,7 @@ def _load_data_steps( the loaded steps, and going down each data step whose down declares ``nothing_to_reverse`` with its reason (it runs no function). A ``@chunked`` query is checked once its historical models are built - (:func:`_check_chunked`).""" + (`_check_chunked`).""" nothing = "applied" if direction == "up" else "reverted" loaded: dict[tuple[int, int], LoadedStep] = {} reasons: dict[tuple[int, int], str] = {} @@ -793,7 +887,7 @@ def _check_chunked( direction: str, ) -> None: """Refuse the run before anything runs when a ``@chunked`` step's query - cannot be paged by keyset (:func:`~ferro.migrations.steps.chunked_query`), + cannot be paged by keyset (`chunked_query`), built over its migration's historical models.""" nothing = "applied" if direction == "up" else "reverted" for step in steps: @@ -812,7 +906,7 @@ def _check_chunked( class _HistoricalSwaps: """The registry swap of the migration whose data step is running: one swap per migration with data steps, held until the run leaves that - migration (ADR-0035), restored on exit, error and cancellation. Built + migration, restored on exit, error and cancellation. Built from the run's held directory read, never from disk again.""" def __init__(self, run: LockedDatabase) -> None: @@ -865,11 +959,11 @@ def _route() -> Any: class _DataStep: """Run one planned data step, its record moved through the run's - transitions (ADR-0024, ADR-0048). + transitions. An atomic step is one transaction on the run's connection, its record finished (going up) or removed (going down) inside that transaction. A - chunked step runs through :func:`~ferro.migrations.chunked.run_chunked`, + chunked step runs through `run_chunked`, one transaction per batch with its cursor committed in it. A failure rolls back what it was running and is recorded only where the @@ -1085,13 +1179,21 @@ async def status_of( *, lock_held: bool | None = None, ) -> StatusReport: - """Where ``tracked`` stands against its held directory read: each step's - state (a run holding the lock makes its step ``running``), and the - refusal ``up`` would meet. Takes no lock and creates nothing. + """Report where a tracked database stands against its held directory read. + + Each step's state (a run holding the lock makes its step `running`), and + the refusal `up` would meet. Takes no lock and creates nothing. - ``lock_held`` is whether a run holds the lock; ``None`` asks the - database. A locked run that executes nothing (a refused baseline) passes - ``False``: its own lock puts no step in progress.""" + Args: + tracked: The tracked database, locked or not. + database: The configured database. + lock_held: Whether a run holds the lock. Defaults to asking the + database. A locked run that executes nothing (a refused + baseline) passes `False`: its own lock puts no step in progress. + + Returns: + The status report. + """ try: keys: list[tuple[int, int, list[str]]] | None = order_keys_on_disk(tracked) except MigrationRefused: @@ -1125,12 +1227,36 @@ async def status( using: str | None = None, url: str | None = None, ) -> StatusReport: - """Where the database stands against its migrations directory - (``ferro migrate status``). + """Report where the database stands against its migrations directory. + + This is `ferro migrate status`. It takes no lock and creates nothing: a + database without the tracking table reports every migration pending. + The refusal `up` would meet (an edited file, a broken chain) is part of + the report. + + Args: + settings: The project's configuration. Defaults to `FerroSettings()`. + database: The configured database. Defaults to the only one + configured. + using: An open connection to work on. Defaults to the default + connection. + url: A URL to open a private connection to, closed after the call. + Refused together with `using`. + + Returns: + The status report: each migration's state, and whether anything is + pending or needs attention. - Takes no lock and creates nothing: a database without the tracking - table reports every migration pending. The refusal ``up`` would meet - (an edited file, a broken chain, ...) is part of the report. + Raises: + MigrationRefused: The configuration names no single database, or + there is no connection to work on. + + Examples: + ```python {test=compile reason="needs a migrations project and a database"} + report = await ferro.migrations.status() + print(report.head_applied.name, report.pending) + # 0006_add_teams True + ``` """ target = Target.resolve(settings, database, using=using, url=url) async with target.open() as name: @@ -1147,32 +1273,45 @@ async def require_applied( url: str | None = None, allow_ahead: AllowAhead = False, ) -> None: - """Return when the database stands at the head of its migrations; raise - otherwise. Takes no lock and changes nothing. - - ``allow_ahead`` lets a database holding migrations this checkout lacks - through (:data:`AllowAhead`): ``"expand-only"`` when every step they hold - is ``expand`` (the previous release of a rolling deploy), ``True`` - whatever they hold. A step's class is read from its step record, written - when it was applied; a record an older ferro wrote holds none and reads - as ``contract`` until the first run of this ferro on the database - classifies it. - - The answer is the run planner's: the plan ``up`` would make, read with - no lock, after the inheritance it would make first (ADR-0056). A database - that applied a squash's originals is pending that squash, never ahead - by them, whatever ``allow_ahead`` says (ADR-0040); one that applied - them in part carries the run's ``partial_witness`` refusal. + """Return when the database stands at the head of its migrations; raise otherwise. + + Takes no lock and changes nothing. The answer is the run planner's: the + plan `up` would make, read with no lock, after the squash inheritance it + would make first. A database that applied a squash's originals is + pending that squash, never ahead by them, whatever `allow_ahead` says. + A step's class is read from its step record; a record an older Ferro + wrote holds none and reads as `contract` until the first run of this + Ferro on the database classifies it. + + Args: + settings: The project's configuration. Defaults to `FerroSettings()`. + database: The configured database. Defaults to the only one + configured. + using: An open connection to work on. Defaults to the default + connection. + url: A URL to open a private connection to, closed after the call. + Refused together with `using`. + allow_ahead: Let a database that has applied migrations this + checkout lacks through: `"expand-only"` when every step they + hold is `expand` (the previous release of a rolling deploy), + `True` whatever they hold. Defaults to `False`, which refuses. Raises: - PendingMigrationsError: a migration is pending (``.pending``), or the - tracking table refuses a run (``.refusals``: an interrupted run, - an edited applied file, ...). - DatabaseAheadError: the database has applied migrations this - checkout lacks (``allow_ahead=True`` lets it through; - ``.contract_steps`` names the steps ``"expand-only"`` refuses on). - TypeError: ``allow_ahead`` is not ``False``, ``True`` or - ``"expand-only"``. + PendingMigrationsError: A migration is pending, or the tracking + table refuses a run (an interrupted run, an edited applied file). + DatabaseAheadError: The database has applied migrations this + checkout lacks. + TypeError: `allow_ahead` is not `False`, `True` or `"expand-only"`. + + Examples: + ```python {test=compile reason="needs a migrations project and a database"} + try: + await ferro.migrations.require_applied(allow_ahead="expand-only") + except ferro.migrations.DatabaseAheadError as ahead: + print(ahead.contract_steps) # ['0008_drop_bio:01'] + else: + print("the database is at the head") + ``` """ allow_ahead = _core._allow_ahead(allow_ahead) target = Target.resolve(settings, database, using=using, url=url) @@ -1208,27 +1347,33 @@ async def require_applied( class DownStep(NamedTuple): - """One step :func:`down` would revert.""" + """One step a down run would revert.""" migration: str - """``NNNN_<name>``.""" + """The migration, `NNNN_<name>`.""" step: str - """``NN_<name>``.""" + """The step, `NN_<name>`.""" nothing_to_reverse: str | None """Why its down runs no statement, when it runs none.""" file: str - """The step's file (``01_schema.up.postgres.sql``), which names its kind.""" + """The step's file (`01_schema.up.postgres.sql`), which names its kind.""" @dataclass class DownPlan: - """What a ``down`` would revert, newest first, or why it would not.""" + """What a down run would revert, newest first, or why it would not.""" steps: list[DownStep] = field(default_factory=list) + """The steps it would revert, newest first.""" refusal: str | None = None + """Why it would be refused, or `None`.""" def describe(self) -> str: - """The plan as the prompt shows it, one line per step.""" + """Render the plan as the prompt shows it, one line per step. + + Returns: + The plan, or `nothing to revert`. + """ if not self.steps: return "nothing to revert" name_width = max(len(step.migration) for step in self.steps) @@ -1241,7 +1386,11 @@ def describe(self) -> str: return "\n".join(lines) def question(self) -> str: - """``Revert 0002_add_teams (1 step)?``""" + """Render the prompt's question, such as `Revert 0002_add_teams (1 step)?`. + + Returns: + The question. + """ migrations = list(dict.fromkeys(step.migration for step in self.steps)) count = len(self.steps) return ( @@ -1250,9 +1399,21 @@ def question(self) -> str: def parse_target(target: str | None = None, *, all: bool = False) -> dict[str, Any]: - """The run planner's target for ``down``: the latest migration, ``"0005"`` - (keep 0005 applied; ``"0000"`` reverts everything), ``"0007:02"`` (keep - steps 01-02 of 0007), or ``all``.""" + """Parse where a down run stops into the run planner's target. + + Args: + target: `"0005"` keeps 0005 applied (`"0000"` reverts everything); + `"0007:02"` keeps steps 01-02 of 0007 applied. Defaults to + reverting the latest migration. + all: Revert everything. Refused together with `target`. + + Returns: + The planner's direction and target. + + Raises: + SettingsError: Both `target` and `all` are given, or `target` is not + a migration or a step. + """ if all and target is not None: raise SettingsError("pass --to or --all, not both") if all: @@ -1279,7 +1440,7 @@ async def _preview_down( ) -> tuple[Plan, DownPlan]: """What a ``down`` would revert, planned on ``tracked`` with no lock, as the run plans it once it has inherited every squash whose witness the - records hold (ADR-0056): a data step's down declares itself in Python, + records hold: a data step's down declares itself in Python, so an irreversible one refuses here too (before any lock), and a nothing-to-reverse one says so. @@ -1315,8 +1476,23 @@ async def plan_down( target: str | None = None, all: bool = False, ) -> DownPlan: - """What :func:`down` would revert, read with no lock, changing nothing. - ``DownPlan.refusal`` says why it would be refused.""" + """Report what a down run would revert, read with no lock, changing nothing. + + Args: + settings: The project's configuration. Defaults to `FerroSettings()`. + database: The configured database. Defaults to the only one + configured. + using: An open connection to work on. Defaults to the default + connection. + url: A URL to open a private connection to, closed after the call. + Refused together with `using`. + target: Where to stop, as `parse_target` reads it. Defaults to the + latest migration. + all: Revert everything. + + Returns: + The plan; its `refusal` says why the run would be refused. + """ direction = parse_target(target, all=all) where = Target.resolve(settings, database, using=using, url=url) async with where.open() as name: @@ -1330,16 +1506,16 @@ async def plan_down( _INHERITANCE_STAMP = ("started_at", "finished_at", "ferro_version") """What an inherited record holds of the moment it was inherited: the preview plans over the records inheritance would write, the run over the ones it -wrote a moment later (ADR-0056).""" +wrote a moment later.""" def _records_as_shown(locked: list[Any], shown: list[Any]) -> bool: """Whether the steps planned under the lock stand on the records the preview showed. Taking the lock gives a record an older ferro wrote - without a class the class ADR-0054 decides for it; that fills a gap and + without a class the class the lock decides for it; that fills a gap and changes nothing the plan was shown, so a class the preview read as missing is not compared. An inherited record is compared without the time - and version it was inherited at (:data:`_INHERITANCE_STAMP`).""" + and version it was inherited at (`_INHERITANCE_STAMP`).""" def standing(step: Any, missing_class: bool) -> dict[str, Any] | None: record = step.standing @@ -1372,42 +1548,53 @@ async def down( progress: Callable[[str], Any] | None = None, on_event: Callable[[RunEvent], Any] | None = None, ) -> RunReport: - """Revert every applied step above ``target``, newest first, under the - run lock (``ferro migrate down``, ADR-0033), on the database and - connection :func:`up` would work on. - - ``target`` is ``None`` (the latest migration with a record, a partly - applied one included), ``"0005"`` (leave 0005 fully applied; ``"0000"`` - reverts everything), or ``"0007:02"`` (leave steps 01-02 of 0007 - applied); ``all`` reverts everything. A step whose down is declared + """Revert applied steps, newest first, under the run lock. + + This is `ferro migrate down`. A step whose down is declared irreversible, or one a baseline recorded, refuses the whole run before - anything is reverted. - - ``confirm`` is shown the plan before the lock is taken; when it returns - false nothing is reverted (``declined``). Once the lock is held the plan - is made again, and a database that changed in between is refused rather - than reverted unseen. Under the lock and before it plans, a ``down`` - inherits every squash whose witness the records hold, as ``up`` does - (ADR-0056). ``progress`` receives one line per squash inherited, then one - per reverted step; ``on_event`` receives each as data just before - ``progress`` receives its ``line()``: an :class:`InheritedEvent` per - squash inherited (its ``squash`` and the ``replaced`` migrations, with no - ``.text`` or ``.step``), a :class:`StepEvent` per reverted step, a - :class:`RunNote` for any other line; a failed down is a ``StepEvent`` - with no line. - - Returns the :class:`RunReport` of what it did: ``.inherited`` the squashes - it inherited, in order (an :class:`InheritedMigration` each), then - ``.reverted`` the steps it reverted, newest first (``.declined`` when - ``confirm`` said no). + anything is reverted. Once the lock is held the plan is made again, and + a database that changed since `confirm` saw it is refused rather than + reverted unseen. Under the lock and before it plans, a down inherits + every squash whose witness the records hold, as `up` does. + + Args: + settings: The project's configuration. Defaults to `FerroSettings()`. + database: The configured database. Defaults to the only one + configured. + using: An open connection to work on. Defaults to the default + connection. + url: A URL to open a private connection to, closed after the call. + Refused together with `using`. + target: Where to stop: `"0005"` leaves 0005 fully applied (`"0000"` + reverts everything), `"0007:02"` leaves steps 01-02 of 0007 + applied. Defaults to the latest migration with a record, a + partly applied one included. + all: Revert everything. Refused together with `target`. + lock_timeout: How long to wait for another run's lock: `"30s"`, + `"500ms"`, `"1m"` or a number of seconds; `0` refuses at once. + Defaults to the database's `lock_timeout`. + confirm: Shown the plan before the lock is taken; when it returns + false nothing is reverted. Defaults to reverting without asking. + progress: Receives one line per squash inherited, then one per + reverted step. + on_event: Receives each line as data just before `progress` + receives its `line()`: an `InheritedEvent` per squash inherited, + a `StepEvent` per reverted step, a `RunNote` for any other. A + failed down is a `StepEvent` with no line. + + Returns: + What the run did: the squashes it inherited, then the steps it + reverted, newest first (`declined` when `confirm` said no). Raises: - MigrationRefused: the run refused or a down failed; ``.report`` is - the run's report. A failed down that rolled back whole leaves its + MigrationRefused: The run refused or a down failed; `.report` is the + run's report. A failed down that rolled back whole leaves its step applied and its record as it was; one that left part of - itself applied (a no-transaction SQL down, a chunked down past its - first batch) leaves the record carrying the error, a chunked one - still ``reverting`` at its cursor. The next ``down`` resumes at it. + itself applied (a no-transaction SQL down, a chunked down past + its first batch) leaves the record carrying the error, a chunked + one still `reverting` at its cursor. The next down resumes at it. + SettingsError: `target` is not a migration or a step, or both + `target` and `all` are given. """ direction = parse_target(target, all=all) report = RunReport(direction="down") diff --git a/src/ferro/migrations/scaffold.py b/src/ferro/migrations/scaffold.py index 4a773adf..a0085a72 100644 --- a/src/ferro/migrations/scaffold.py +++ b/src/ferro/migrations/scaffold.py @@ -1,29 +1,32 @@ -"""The generated data step's text (ADR-0035, ADR-0037, ADR-0040). +"""The generated data step's text. The generator decides every data step of a migration and hands each one over -as a record; :func:`render` turns that record into the step's file. There are +as a record; `render` turns that record into the step's file. There are three shapes, and the record says which. -A **backfill**. ``Author`` gains a required ``slug``; ``ferro migrate new -author_slug`` writes it between the expand and the contract:: +A backfill. `Author` gains a required `slug`; +`ferro migrate new author_slug` writes it between the expand and the +contract: - # migrations/0012_author_slug/02_backfill_author.py - from ferro.migrations import chunked, nothing_to_reverse, todo +```python {test=compile reason="a generated data step file; the migration runner calls it"} +# migrations/0012_author_slug/02_backfill_author.py +from ferro.migrations import chunked, nothing_to_reverse, todo - @chunked( - lambda models: models.Author.where(lambda author: author.slug == None) - .order_by(lambda author: author.id), - batch_size=1000, - ) - async def up(ctx, batch): - for author in batch: - author.slug = todo("the slug for an existing author") - await author.save() +@chunked( + lambda models: models.Author.where(lambda author: author.slug == None) + .order_by(lambda author: author.id), + batch_size=1000, +) +async def up(ctx, batch): + for author in batch: + author.slug = todo("the slug for an existing author") + await author.save() - @nothing_to_reverse("01_expand.down.sql drops the column") - def down(ctx): ... +@nothing_to_reverse("01_expand.down.sql drops the column") +def down(ctx): ... +``` The query always selects the rows that still need a value, so a re-run (the contract's recipe for rows written behind the cursor) touches only what is @@ -32,40 +35,43 @@ def down(ctx): ... single primary key has no keyset to page by: its backfill is one ``@atomic`` ``update`` per column. -A label removed from a ``StrEnum`` (D2) is a backfill over the rows still -holding it, each given another label (the historical model's enum still -declares the removed one, and the label written is looked up in it):: - - @chunked( - lambda models: models.Order.where(lambda order: order.status == "canceled") - .order_by(lambda order: order.id), - batch_size=1000, - ) - async def up(ctx, batch): - for order in batch: - order.status = type(order.status)(todo("the label to use instead of 'canceled'")) - await order.save() - -A **guard step**. ``new --no-backfill author.slug`` writes one in the -backfill's place: an ``@atomic`` step that fails the migration, naming the -count, when any row still needs a value, so a step number is never a gap -(ADR-0037). - -A **hand-requested data step** (``ferro migrate new backfill_slugs ---data-step Author``):: - - from ferro.migrations import atomic, todo +A label removed from a `StrEnum` is a backfill over the rows still holding +it, each given another label (the historical model's enum still declares +the removed one, and the label written is looked up in it): + +```python {test=compile reason="a generated data step body; the migration runner calls it"} +@chunked( + lambda models: models.Order.where(lambda order: order.status == "canceled") + .order_by(lambda order: order.id), + batch_size=1000, +) +async def up(ctx, batch): + for order in batch: + order.status = type(order.status)(todo("the label to use instead of 'canceled'")) + await order.save() +``` + +A guard step. `new --no-backfill author.slug` writes one in the backfill's +place: an `@atomic` step that fails the migration, naming the count, when +any row still needs a value, so a step number is never a gap. + +A hand-requested data step +(`ferro migrate new backfill_slugs --data-step Author`): + +```python {test=compile reason="a generated data step file; the migration runner calls it"} +from ferro.migrations import atomic, todo - @atomic - async def up(ctx): - # ctx.models.Author is the table as this migration leaves it. - todo("write this step") +@atomic +async def up(ctx): + # ctx.models.Author is the table as this migration leaves it. + todo("write this step") - @atomic - async def down(ctx): - todo("write this step") +@atomic +async def down(ctx): + todo("write this step") +``` A project overrides any of the three with a file in ``_templates/`` beside its migrations (``backfill.py``, ``guard.py``, ``data_step.py``), used @@ -123,15 +129,20 @@ async def down(ctx): def render(step: Mapping[str, Any], *, template_dir: Path | None) -> str: - """The text of the data step ``step`` describes. + """Render the text of the data step file a generator step record describes. + + Args: + step: One step record of the generator's output: its `data` (a + backfill, or the guard when `data["guard"]`) or its `hand_model` + (`--data-step`). + template_dir: The project's template directory; a template there + wins when it exists. `None` uses the built-in skeletons. - ``step`` is one step record of ``_core._generate_migration``'s output, as - the generator produced it: its ``data`` (a backfill, or the guard when - ``data["guard"]``) or its ``hand_model`` (``--data-step``). The project's - template in ``template_dir`` wins when it exists. + Returns: + The data step file's text. Raises: - ValueError: ``step`` is not a data step the generator scaffolds. + ValueError: `step` is not a data step the generator scaffolds. """ hand = step.get("hand_model") if hand is not None: @@ -150,11 +161,21 @@ def render(step: Mapping[str, Any], *, template_dir: Path | None) -> str: def note(step: Mapping[str, Any], *, dir_name: str, migration_name: str) -> str | None: - """The note a backfill that still needs writing adds to the - migration (``02_backfill_author.py needs writing where it says - todo(...); …``): where to write, and the command that regenerates the - migration with a guard instead. ``None`` for any other step: a guard, a - hand-requested step, or a backfill pre-filled throughout.""" + """Render the note a backfill that still needs writing adds to its migration. + + The note says where to write, and the command that regenerates the + migration with a guard instead + (`02_backfill_author.py needs writing where it says todo(...); …`). + + Args: + step: One step record of the generator's output. + dir_name: The migration's directory name. + migration_name: The migration's name, for the regenerating command. + + Returns: + The note, or `None` for any other step: a guard, a hand-requested + step, or a backfill pre-filled throughout. + """ data = step.get("data") if data is None or data["guard"]: return None @@ -183,7 +204,7 @@ class _Demand: fills: tuple[tuple[str, str | None], ...] """What the step fills, in order: ``(column, None)`` for a column whose rows hold ``NULL``, ``(column, label)`` for each removed enum ``label`` - its rows may hold (D2).""" + its rows may hold.""" removed: bool """Some column's rows hold a label the migration removes.""" driver: str diff --git a/src/ferro/migrations/steps.py b/src/ferro/migrations/steps.py index efd205b4..4a0ced51 100644 --- a/src/ferro/migrations/steps.py +++ b/src/ferro/migrations/steps.py @@ -1,6 +1,6 @@ -"""Data step declarations, ``todo()``, and the loader that reads a step file. +"""Data step declarations, `todo()`, and the loader that reads a step file. -```python +```python {test=compile reason="a data step file; the migration runner calls it"} # migrations/0011_backfill_slugs/01_backfill_author.py from ferro.migrations import atomic, nothing_to_reverse, todo @@ -14,14 +14,15 @@ async def up(ctx): def down(ctx): ... ``` -Every ``up`` and ``down`` carries exactly one declaration (ADR-0035): -:func:`atomic`, :func:`chunked`, or, on a ``down`` only, :func:`irreversible` -or :func:`nothing_to_reverse`. :func:`load_step` refuses an undecorated or -doubly decorated function naming the file and function, and reads every -:func:`todo` call from the file's syntax tree, so ``up`` refuses the -migration with file, line and message before running anything:: +Every `up` and `down` carries exactly one declaration: `atomic`, `chunked`, +or, on a `down` only, `irreversible` or `nothing_to_reverse`. `load_step` +refuses an undecorated or doubly decorated function naming the file and +function, and reads every `todo` call from the file's syntax tree, so `up` +refuses the migration with file, line and message before running anything: - 0011_backfill_slugs/01_backfill_author.py:7: not written yet: write this step +```text +0011_backfill_slugs/01_backfill_author.py:7: not written yet: write this step +``` """ from __future__ import annotations @@ -68,13 +69,15 @@ def down(ctx): ... class StepRefused(MigrationRefused): - """A data step file that cannot run as written: an undeclared or doubly - declared function, a file that does not load, or one edited since it - was planned. The message names the file and the fix.""" + """The refusal for a data step file that cannot run as written. + + An undeclared or doubly declared function, a file that does not load, or + one edited since it was planned. The message names the file and the fix. + """ class NotWrittenError(RuntimeError): - """A :func:`todo` was reached at run time.""" + """The error a [`todo`][ferro.migrations.todo] raises when a run reaches it.""" # -- the shapes ------------------------------------------------------------------------- @@ -82,38 +85,45 @@ class NotWrittenError(RuntimeError): @dataclass(frozen=True) class Atomic: - """One transaction, the step's record committed inside it (ADR-0024).""" + """The atomic shape: one transaction, the step's record committed inside it.""" kind = "atomic" + """The shape's name, as a step record and a refusal spell it.""" @dataclass(frozen=True) class Chunked: - """The runner pages ``query`` by keyset, ``batch_size`` rows per - transaction (ADR-0024; :func:`ferro.migrations.chunked.run_chunked`). - :func:`chunked_query` builds and checks the query.""" + """The chunked shape: the runner pages `query` by keyset, `batch_size` rows per transaction. + + `chunked_query` builds and checks the query. + """ query: Callable[[HistoricalModels], Query[Any]] + """The declared query, a function of the step's historical models.""" batch_size: int + """The rows each transaction takes.""" kind = "chunked" + """The shape's name, as a step record and a refusal spell it.""" @dataclass(frozen=True) class Irreversible: - """A ``down`` that cannot be reversed: a run that would revert it refuses - before reverting anything (ADR-0033).""" + """The irreversible shape: a run that would revert the step refuses before reverting anything.""" reason: str + """Why the step cannot be reversed.""" kind = "irreversible" + """The shape's name, as a step record and a refusal spell it.""" @dataclass(frozen=True) class NothingToReverse: - """A ``down`` with nothing to do: reverting it removes the record and runs - nothing (ADR-0033).""" + """The nothing-to-reverse shape: reverting the step removes its record and runs nothing.""" reason: str + """Why there is nothing to reverse.""" kind = "nothing_to_reverse" + """The shape's name, as a step record and a refusal spell it.""" Shape = Atomic | Chunked | Irreversible | NothingToReverse @@ -124,7 +134,9 @@ class Declared: """One step function and its declared shape.""" fn: Callable[..., Any] + """The step function.""" shape: Shape + """The shape its one declaration gave it.""" @dataclass(frozen=True) @@ -132,9 +144,11 @@ class LoadedStep: """A loaded data step file.""" up: Declared + """The file's `up` and its shape.""" down: Declared + """The file's `down` and its shape.""" todos: list[tuple[int, str]] - """Every ``todo("…")`` in the file: ``(line, message)``, in file order.""" + """Every `todo("…")` in the file as `(line, message)`, in file order.""" # -- the declarations -------------------------------------------------------------------- @@ -150,27 +164,79 @@ def _declare[F: Callable[..., Any]](fn: F, shape: Shape) -> F: def atomic[F: Callable[..., Awaitable[Any]]](fn: F) -> F: - """Declare a step function atomic: it runs as one transaction, with the - step's record committed inside it. ``async def up(ctx)``.""" + """Declare a step function atomic: one transaction, with the step's record committed inside it. + + The function is `async def up(ctx)` (or `down`); `ctx` is the step's + [context](../guide/schema/data-steps.md). + + Args: + fn: The step function. + + Returns: + The function, unchanged apart from its declaration. + + Raises: + TypeError: `fn` is not callable. + + Examples: + ```python {test=compile reason="a step file in a migrations directory"} + from ferro.migrations import atomic + + + @atomic + async def up(ctx): + for author in await ctx.models.Author.where(lambda author: author.slug == None).all(): + author.slug = author.name.lower().replace(" ", "-") + await author.save() + ``` + """ return _declare(fn, Atomic()) def chunked[F: Callable[..., Awaitable[Any]]]( query: Callable[[HistoricalModels], Query[Any]], *, batch_size: int ) -> Callable[[F], F]: - """Declare a step function chunked: the runner pages ``query`` (given - ``ctx.models``) by keyset, ``batch_size`` rows per transaction, and calls - ``async def up(ctx, batch)`` once per batch, committing the batch's - cursor with it. ``query`` orders by keys that include the model's - primary key, and selects only the rows that still need the step, so a - resumed run re-pages from its cursor:: + """Declare a step function chunked: the runner pages a query in batches, one transaction each. + + The runner calls `async def up(ctx, batch)` once per batch and commits + the batch's cursor with it, so a failed or interrupted run resumes after + the last committed batch. See + [Data steps and backfills](../guide/schema/data-steps.md). + + Args: + query: A function of `ctx.models` that returns the query to page. It + orders by keys that include the model's primary key, and selects + only the rows that still need the step, so a resumed run re-pages + from its cursor. + batch_size: The rows each transaction takes; a positive `int`. + + Returns: + A decorator that declares the step function chunked. + + Raises: + TypeError: `query` is not callable. + ValueError: `batch_size` is not a positive `int`. + StepRefused: At run time, before anything runs: the query does not + build, pages a table without a single primary key, is not + ordered by keys that include the primary key, orders by a + related model's column, or sets its own `limit`, `offset`, + `after` or `before`. + + Examples: + ```python {test=compile reason="a step file in a migrations directory"} + from ferro.migrations import chunked + @chunked( lambda models: models.Author.where(lambda author: author.slug == None) .order_by(lambda author: author.id), batch_size=1000, ) - async def up(ctx, batch): ... + async def up(ctx, batch): + for author in batch: + author.slug = author.name.lower().replace(" ", "-") + await author.save() + ``` """ if not callable(query): raise TypeError("@chunked takes the query as a function of ctx.models") @@ -191,8 +257,29 @@ def decorate(fn: F) -> F: def irreversible[F: Callable[..., Any]](reason: str) -> Callable[[F], F]: - """Declare a ``down`` irreversible, saying why: a run that would revert - it refuses before reverting anything.""" + """Declare a `down` irreversible, saying why. + + A run that would revert the step refuses before reverting anything, and + prints the reason. + + Args: + reason: Why the step cannot be reversed; a non-empty string. + + Returns: + A decorator that declares the `down` irreversible. + + Raises: + TypeError: `reason` is not a non-empty string. + + Examples: + ```python {test=compile reason="a step file in a migrations directory"} + from ferro.migrations import irreversible + + + @irreversible("the old nicknames were dropped with the column") + def down(ctx): ... + ``` + """ shape = Irreversible(_reason(reason, "irreversible")) def decorate(fn: F) -> F: @@ -202,8 +289,28 @@ def decorate(fn: F) -> F: def nothing_to_reverse[F: Callable[..., Any]](reason: str) -> Callable[[F], F]: - """Declare that a ``down`` has nothing to reverse, saying why: reverting - removes the step's record and runs nothing.""" + """Declare that a `down` has nothing to reverse, saying why. + + Reverting the step removes its record and runs nothing. + + Args: + reason: Why there is nothing to reverse; a non-empty string. + + Returns: + A decorator that declares the `down` as having nothing to reverse. + + Raises: + TypeError: `reason` is not a non-empty string. + + Examples: + ```python {test=compile reason="a step file in a migrations directory"} + from ferro.migrations import nothing_to_reverse + + + @nothing_to_reverse("slugs are derived; nothing to put back") + def down(ctx): ... + ``` + """ shape = NothingToReverse(_reason(reason, "nothing_to_reverse")) def decorate(fn: F) -> F: @@ -213,16 +320,25 @@ def decorate(fn: F) -> F: def chunked_query(shape: Chunked, models: HistoricalModels, path: Path) -> Query[Any]: - """Build a ``@chunked`` step's query over ``models`` and check that the - runner can page it by keyset (what the run checks for every chunked step - before anything runs). + """Build a `@chunked` step's query and check that the runner can page it by keyset. + + The run checks this for every chunked step before anything runs. + + Args: + shape: The step's chunked declaration. + models: The step's historical models, passed to the query function. + path: The step file, named in a refusal. + + Returns: + The query, ready to page. Raises: - StepRefused: the declaration does not return a query; the query pages - a table without a single primary key (a many-to-many join table, - #492); it has no ``order_by``, orders by a related model's - column, or its order keys leave out the primary key; or it sets its own ``limit`` / ``offset`` / - ``after`` / ``before``. The message names the file and the fix. + StepRefused: The declaration does not return a query; the query + pages a table without a single primary key (a many-to-many join + table); it has no `order_by`, orders by a related model's + column, or its order keys leave out the primary key; or it sets + its own `limit`, `offset`, `after` or `before`. The message + names the file and the fix. """ from ..query import Query @@ -303,11 +419,29 @@ def _reason(reason: Any, name: str) -> str: def todo(message: str) -> Any: - """Mark what only a person can supply: ``author.slug = todo("…")``. + """Mark a value in a data step that only a person can supply. + + The loader reads every call from the file's syntax tree, and `up` + refuses the migration before running anything, naming the file, the + line and the message. + + Args: + message: What is missing, as a string literal: the refusal prints it. - The loader reads every call from the file's syntax tree and ``up`` - refuses the migration before running anything; reached at run time - anyway, it raises :class:`NotWrittenError`. + Raises: + NotWrittenError: Always, when a run reaches the call anyway. + + Examples: + ```python {test=compile reason="a step file in a migrations directory"} + from ferro.migrations import atomic, todo + + + @atomic + async def up(ctx): + for author in await ctx.models.Author.where(lambda author: author.slug == None).all(): + author.slug = todo("the slug for an existing author") + await author.save() + ``` """ raise NotWrittenError(f"not written yet: {message}") @@ -353,11 +487,18 @@ def _dotted(node: ast.expr) -> str | None: def scan_todos(source: str | bytes, path: Path) -> list[tuple[int, str]]: - """Every ``todo("…")`` call in a step file's source: ``(line, message)``. + """Find every `todo("…")` call in a step file's source. + + Args: + source: The file's source. + path: The file, named in a refusal. + + Returns: + Each call as `(line, message)`, in line order. Raises: - StepRefused: the file does not parse, or a ``todo`` carries no - literal message. + StepRefused: The file does not parse, or a `todo` carries no literal + message. """ try: tree = ast.parse(source, filename=str(path)) @@ -390,7 +531,16 @@ def scan_todos(source: str | bytes, path: Path) -> list[tuple[int, str]]: def unwritten(path: Path, todos: list[tuple[int, str]]) -> list[str]: - """The refusal lines for a step's ``todo`` calls, one per call.""" + """Render the refusal lines for a step's `todo` calls, one per call. + + Args: + path: The step file. + todos: Its `todo` calls as `(line, message)`. + + Returns: + One `<migration>/<file>:<line>: not written yet: <message>` line per + call. + """ return [ f"{_shown(path)}:{line}: not written yet: {message}" for line, message in todos ] @@ -400,13 +550,19 @@ def unwritten(path: Path, todos: list[tuple[int, str]]) -> list[str]: def declared_up_kind(path: Path) -> str: - """The record kind (``atomic`` / ``chunked``) a step file's ``up`` - declares, read from its syntax tree without running the file (what - ``baseline`` records for a step it never runs). + """Read the record kind a step file's `up` declares, without running the file. + + This is what `baseline` records for a step it never runs. + + Args: + path: The step file. + + Returns: + `"atomic"` or `"chunked"`. Raises: - StepRefused: the file has no ``up``, or its ``up`` does not carry - exactly one of ``@atomic`` / ``@chunked``. + StepRefused: The file does not parse, has no `up`, or its `up` does + not carry exactly one of `@atomic` / `@chunked`. """ source = path.read_bytes() try: @@ -463,20 +619,26 @@ def _declaration_problem(path: Path, function: str, kinds: list[str]) -> str: def load_step(path: Path, expected_checksum: str | None) -> LoadedStep: - """Load the data step file at ``path``. + """Load a data step file, checked against the checksum it was planned with. + + The file is compiled from the same bytes that were checked, under the + module name `ferro.migrations.steps_loaded.<migration>_<step>` (never + put on `sys.path`), so the code that runs is the code that was checked. + Its `todo` calls are read from the syntax tree; the caller refuses them. + + Args: + path: The step file. + expected_checksum: The file's SHA-384 as the run planner recorded + it, in lowercase hex; `None` skips the check. - The file's bytes are checked against ``expected_checksum`` (SHA-384, - lowercase hex, as the run planner recorded it; ``None`` skips the - check), then compiled from those same bytes under the module name - ``ferro.migrations.steps_loaded.<migration>_<step>`` (never put on - ``sys.path``), so the code that runs is the code that was checked. Its - ``todo`` calls are read from the syntax tree; the caller refuses them. + Returns: + The file's `up` and `down` with their shapes, and its `todo` calls. Raises: - StepRefused: the file changed since it was planned, does not load, - lacks an ``up`` or ``down``, or a function is undeclared, doubly - declared, not ``async def`` where it must run, or (``up``) - declared irreversible or nothing-to-reverse. + StepRefused: The file changed since it was planned, does not load, + lacks an `up` or `down`, or a function is undeclared, doubly + declared, not `async def` where it must run, or (`up`) declared + irreversible or nothing-to-reverse. """ shown = _shown(path) try: diff --git a/src/ferro/migrations/target.py b/src/ferro/migrations/target.py index 971d5c2c..cdbfab40 100644 --- a/src/ferro/migrations/target.py +++ b/src/ferro/migrations/target.py @@ -1,25 +1,23 @@ """The database a migration verb works on, and the connection that reaches it. -```python +```python {test=compile reason="needs a project with a migrations directory"} await ferro.migrations.up() # the configured database, default connection await ferro.migrations.up(database="analytics", using="reporting") await ferro.migrations.up(url="postgres://…/app") # a private connection, closed after ``` -Every verb (``up``, ``down``, ``status``, ``require_applied``, ``drift``, -``baseline``, ``remove_baseline``, ``rerecord``, ``check``) takes the same -``(settings=None, database=None, *, using=None, url=None)`` and hands them -to :meth:`Target.resolve`, which decides three things once: the project's -configuration (``settings=None`` is ``FerroSettings()``), which configured -database (``database=None`` is the only one, or a refusal naming them all), -and which connection (``using`` names an open one, ``url`` opens a private -one, neither is the default connection; both is refused). The CLI calls the -same verbs with ``url=``: the ``--url`` flag, else the database's -``url_env``. +Every verb (`up`, `down`, `status`, `require_applied`, `drift`, `baseline`, +`remove_baseline`, `rerecord`, `check`) takes the same +`(settings=None, database=None, *, using=None, url=None)` and hands them to +`Target.resolve`, which decides three things once: the project's +configuration (`settings=None` is `FerroSettings()`), which configured +database (`database=None` is the only one, or a refusal naming them all), +and which connection (`using` names an open one, `url` opens a private one, +neither is the default connection; both is refused). The CLI calls the same +verbs with `url=`: the `--url` flag, else the database's `url_env`. A configuration that names no single database, and a connection the verb -cannot work on, are :class:`~ferro.migrations.errors.MigrationRefused` -naming the fix, for every caller. +cannot work on, are a `MigrationRefused` naming the fix, for every caller. """ from __future__ import annotations @@ -37,11 +35,18 @@ def connection_dialect(name: str, database: DatabaseSettings) -> str: - """The dialect of open connection ``name``. + """Return the dialect of an open connection, checked against the database's dialects. + + Args: + name: The open connection's name. + database: The configured database the connection must serve. + + Returns: + `"postgres"` or `"sqlite"`. Raises: - SettingsError: ``name`` is not open, or ``database`` does not target - its dialect. + SettingsError: `name` is not open, or `database` does not target its + dialect. """ dialect = _core.connection_backend(name) if dialect is None: @@ -57,13 +62,17 @@ def connection_dialect(name: str, database: DatabaseSettings) -> str: @dataclass(frozen=True) class Target: - """The database a verb works on: its project's configuration, the - configured database, and the connection choice (``using``, ``url``, or - neither: the default connection). Build it with :meth:`resolve`; work on - it inside :meth:`open`.""" + """The database a verb works on, and the connection that reaches it. + + It holds the project's configuration, the configured database, and the + connection choice (`using`, `url`, or neither: the default connection). + Build it with `resolve`; work on it inside `open`. + """ settings: FerroSettings + """The project's configuration.""" database: DatabaseSettings + """The configured database the verb works on.""" using: str | None = None """An open connection's name.""" url: str | None = None @@ -78,15 +87,26 @@ def resolve( using: str | None = None, url: str | None = None, ) -> Target: - """The target ``settings`` (``None``: ``FerroSettings()``), - ``database`` (``None``: the only one configured) and ``using`` / - ``url`` name. Opens nothing. + """Decide the configuration, the database and the connection choice; open nothing. + + Args: + settings: The project's configuration. Defaults to + `FerroSettings()`, read from the project's config file. + database: The configured database's name. Defaults to the only + one configured. + using: An open connection to work on. + url: A URL to open a private connection to, for the verb's + lifetime. With neither `using` nor `url`, the verb works on + the default connection. + + Returns: + The target. Raises: - MigrationRefused: no configuration is found, several databases - are configured and ``database`` names none, ``database`` - names one that is not configured, or both ``using`` and - ``url`` are given. + MigrationRefused: No configuration is found, several databases + are configured and `database` names none, `database` names + one that is not configured, or both `using` and `url` are + given. """ try: settings = FerroSettings() if settings is None else settings @@ -99,17 +119,20 @@ def resolve( @asynccontextmanager async def open(self) -> AsyncIterator[str]: - """The name of the connection to work on, for the block: ``using``, - a private connection to ``url`` (closed when the block ends), or the - default connection. The connection is checked before the block runs: - it is open, and its dialect is one the database targets. + """Yield the name of the connection to work on, for the block. + + The connection is `using`, a private connection to `url` (closed + when the block ends), or the default connection. It is checked + before the block runs: it is open, and its dialect is one the + database targets. Only reaching the connection is translated; + whatever the block raises reaches the caller unchanged. - Only reaching the connection is translated; whatever the block raises - reaches the caller unchanged. + Yields: + The connection's name. Raises: - MigrationRefused: there is no default connection, ``using`` names - a connection that is not open, or the connection's dialect is + MigrationRefused: There is no default connection, `using` names a + connection that is not open, or the connection's dialect is not one the database targets. """ if self.url is None: diff --git a/src/ferro/migrations/testing.py b/src/ferro/migrations/testing.py index 0469fa7a..263e5b5e 100644 --- a/src/ferro/migrations/testing.py +++ b/src/ferro/migrations/testing.py @@ -1,51 +1,22 @@ -"""The migration test harness: a project's own tests prove its chain (ADR-0045). - -Migration ``0012_add_slug`` adds ``author.slug`` and backfills it in a data -step. The test seeds a row the way the table stood at ``0011``, applies -exactly ``0012``, and reads the row back the way ``0012`` left it:: - - from ferro.migrations.testing import harness - - async def test_slug_backfill(db_url, migrations_settings): - await ferro.connect(db_url) - h = harness(settings=migrations_settings, database="default") - await h.apply_through("0011") - async with h.models_at("0011") as models: - await models.Author(name="Ada").save() # seed against the schema that existed - await h.apply("0012") # refused unless the database stands at 0011 - async with h.models_at("0012") as models: - ada = await models.Author.where(lambda author: author.name == "Ada").first() - assert ada.slug == "ada" - await h.revert_to("0011") - await h.round_trip() - -and the test that every down reaches its parent is one line:: - - async def test_the_chain_round_trips(db_url): - await ferro.connect(db_url) - result = await harness().round_trip() - assert result.irreversible is None - -The harness carries what the application's own calls refuse (ADR-0040, -ADR-0038): a target on the way up, a revert with no prompt, and the -historical models outside a data step. It is one object per fixture, -binding ``settings=``, ``database=`` and ``using=`` once (one -:class:`~ferro.migrations.target.Target`, as every verb resolves) and -keeping no other state: every call reads where the database stands from -its tracking table. It never makes a database fresh and never skips a dialect (both are -the project's fixture), and it has no ``reset()`` and no step-level target -(``"0007:02"``): no snapshot describes the state between two steps. +"""The migration test harness: a project's own tests prove its chain. + +The harness carries what the application's own calls refuse: a target on +the way up, a revert with no prompt, and the historical models outside a +data step. It is one object per fixture, binding `settings=`, `database=` +and `using=` once and keeping no other state: every call reads where the +database stands from its tracking table. It never makes a database fresh +and never skips a dialect (both are the project's fixture), and it has no +`reset()` and no step-level target (`"0007:02"`): no snapshot describes the +state between two steps. Nothing here decides how a migration runs. Every step runs through the -runner (``up`` bounded by its ``through``, :func:`~ferro.migrations.runner.down`) -under its run lock, every comparison is :func:`ferro.migrations.drift`, and -every historical model is built by -:func:`ferro.migrations.historical.build_single` and installed with -:meth:`ferro.registry.Registry.swap`. A harness call that cannot do exactly -what it was asked raises :class:`~ferro.migrations.errors.MigrationRefused` -naming the migration the database stands at and the one asked for; the -runner's own refusals surface unchanged. Only ``apply*``, ``revert*`` and -``round_trip`` take the run lock or create the tracking tables. +runner under its run lock, every comparison is `ferro.migrations.drift`, +and every historical model is the one a data step would get. A harness call +that cannot do exactly what it was asked raises `MigrationRefused` naming +the migration the database stands at and the one asked for; the runner's +own refusals surface unchanged. Only `apply*`, `revert*` and `round_trip` +take the run lock or create the tracking tables. See +[Testing migrations](../guide/schema/testing.md). """ from __future__ import annotations @@ -78,29 +49,37 @@ async def test_the_chain_round_trips(db_url): @dataclass(frozen=True) class RoundTripResult: - """What :meth:`Harness.round_trip` walked. + """What [`Harness.round_trip`][ferro.migrations.testing.Harness.round_trip] walked. - A chain whose every down reaches its parent:: + For a chain whose every down reaches its parent: - RoundTripResult(applied=["0001_create_author", "0002_add_slug"], - reverted_to=None, irreversible=None) + ```python {test=compile reason="a value `round_trip` returns; `RoundTripResult` is not imported here"} + RoundTripResult(applied=["0001_create_author", "0002_add_slug"], + reverted_to=None, irreversible=None) + ``` - and one whose ``0002`` declares a step irreversible:: + and for one whose `0002` declares a step irreversible: - RoundTripResult(applied=["0001_create_author", "0002_add_slug"], - reverted_to="0002_add_slug", - irreversible=("0002_add_slug", "01_schema", "the index is shared")) + ```python {test=compile reason="a value `round_trip` returns; `RoundTripResult` is not imported here"} + RoundTripResult(applied=["0001_create_author", "0002_add_slug"], + reverted_to="0002_add_slug", + irreversible=("0002_add_slug", "01_schema", "the index is shared")) + ``` """ applied: list[str] = field(default_factory=list) - """Every migration of the chain, in order (``NNNN_<name>``): each stood - applied with no drift on the way up, and again on the way back up.""" + """Every migration of the chain, in order (`NNNN_<name>`). + + Each stood applied with no drift on the way up, and again on the way + back up. + """ reverted_to: str | None = None - """Where the downward walk stopped: the migration holding the - irreversible step, still applied. ``None`` when it reverted everything.""" + """Where the downward walk stopped: the migration holding the irreversible step, still applied. + + `None` when the walk reverted everything. + """ irreversible: tuple[str, str, str] | None = None - """``(migration, step, reason)`` of the irreversible step that ended the - downward walk, or ``None``.""" + """The `(migration, step, reason)` of the irreversible step that ended the downward walk, or `None`.""" @dataclass(frozen=True) @@ -119,8 +98,11 @@ def short(self) -> str: class Harness: - """A migration chain driven by name, for a test. Build it with - :func:`harness`.""" + """A migration chain driven by name, for a test. + + Build it with [`harness`][ferro.migrations.testing.harness]. Each + method takes a migration as `"0007"` or `"0007_split_name"`. + """ def __init__(self, target: Target) -> None: self._target = target @@ -134,11 +116,20 @@ def __repr__(self) -> str: # -- the verbs ---------------------------------------------------------------- async def apply_through(self, migration: str) -> RunReport: - """Apply every pending migration up to and including ``migration`` - (``"0007"`` or ``"0007_split_name"``), under the run lock. + """Apply every pending migration up to and including one, under the run lock. + + A database already standing at `migration` is left alone. + + Args: + migration: The last migration to apply, `"0007"` or + `"0007_split_name"`. - A database already standing at ``migration`` is left alone (an empty - report). Refused naming both when the database stands past it. + Returns: + The run's report; empty when nothing was pending. + + Raises: + MigrationRefused: The database stands past `migration` (naming + both), or there is no such migration. """ chain = self._chain() target = _find(chain, migration) @@ -155,12 +146,22 @@ async def apply_through(self, migration: str) -> RunReport: return await self._up_through(name, chain, target) async def apply(self, migration: str) -> RunReport: - """Apply exactly ``migration``, under the run lock. + """Apply exactly one migration, under the run lock. + + A test that seeded at `0006` and applies `0007` knows only `0007` + ran. + + Args: + migration: The migration to apply, `"0007"` or + `"0007_split_name"`. + + Returns: + The run's report. - Refused, naming the migration the database stands at and the parent - it would have to stand at, unless it stands at that parent (for the - first migration: at nothing), so a test that seeded at ``0006`` - knows only ``0007`` ran. + Raises: + MigrationRefused: The database does not stand at the migration's + parent (for the first migration: at nothing); the message + names where it stands and where it would have to. """ chain = self._chain() target = _find(chain, migration) @@ -185,12 +186,21 @@ async def apply(self, migration: str) -> RunReport: return await self._up_through(name, chain, target) async def revert_to(self, migration: str) -> RunReport: - """Revert every migration above ``migration``, newest first, with no - prompt, leaving ``migration`` applied. + """Revert every migration above one, newest first, with no prompt, leaving it applied. The runner's rules hold: an irreversible step in the way refuses - before anything is reverted. Refused naming both when the database - stands below ``migration``. + before anything is reverted. + + Args: + migration: The migration to leave applied, `"0007"` or + `"0007_split_name"`. + + Returns: + The run's report. + + Raises: + MigrationRefused: The database stands below `migration` (naming + both), or an irreversible step is in the way. """ chain = self._chain() target = _find(chain, migration) @@ -205,26 +215,46 @@ async def revert_to(self, migration: str) -> RunReport: return await self._down(name, target=chain[target].short) async def revert_all(self) -> RunReport: - """Revert every applied migration, newest first, with no prompt. On a - database with nothing applied it does nothing and creates nothing.""" + """Revert every applied migration, newest first, with no prompt. + + On a database with nothing applied it does nothing and creates + nothing. + + Returns: + The run's report. + + Raises: + MigrationRefused: An irreversible step is in the way; nothing was + reverted. + """ async with self._target.open() as name: return await self._down(name, all=True) @asynccontextmanager async def models_at(self, migration: str) -> AsyncIterator[HistoricalModels]: - """The historical models of ``migration``'s snapshot alone, installed - for the block: the same ``models`` object a data step's context - carries, with every table as that one snapshot declares it (a column - a later migration adds is absent, not nullable):: + """Install the historical models of one migration's snapshot for the block. + + They are the same `models` object a data step's context carries, + with every table as that one snapshot declares it (a column a later + migration adds is absent, not nullable). The block is a session on + the harness's connection, so the models need no `using=`. While it + is open today's classes are unreachable (`Author` raises naming + `models.Author`); they are restored on exit, error and cancellation. + Reads no database state. + Args: + migration: The migration whose snapshot to install, `"0007"` or + `"0007_split_name"`. + + Yields: + The historical models. + + Examples: + ```python {test=compile reason="needs a migrations project and a database"} async with h.models_at("0011") as models: await models.Author(name="Ada").save() links = await models.table("author_tags").all() - - The block is a session on the harness's connection, so the models - need no ``using=``. While it is open today's classes are unreachable - (``Author`` raises naming ``models.Author``); they are restored on - exit, error and cancellation. Reads no database state. + ``` """ chain = self._chain() this = chain[_find(chain, migration)] @@ -235,19 +265,31 @@ async def models_at(self, migration: str) -> AsyncIterator[HistoricalModels]: yield models async def round_trip(self) -> RoundTripResult: - """Apply every migration, revert every one, and apply them all again, - one migration per stop, raising :class:`MigrationRefused` with the - drift lines at the first stop where the database is not what that - migration's snapshot declares. - - A step whose down is declared irreversible ends the downward walk at - its migration: the result reports it (``irreversible``, - ``reverted_to``) and the walk back up completes; it is not a failure. - - With every migration reverted the database has no records, so - :func:`~ferro.migrations.drift` has no snapshot to compare with and - refuses; that one stop instead checks that none of the first - migration's tables remain (``author table is extra``). + """Apply every migration, revert every one, and apply them all again, checking drift at each stop. + + The walk moves one migration per stop. A step whose down is declared + irreversible ends the downward walk at its migration: the result + reports it and the walk back up completes; it is not a failure. + With every migration reverted there is no snapshot to compare with, + so that one stop instead checks that none of the first migration's + tables remain. + + Returns: + What the walk applied, where it stopped going down, and the + irreversible step that stopped it. + + Raises: + MigrationRefused: At the first stop where the database is not + what that migration's snapshot declares, with the drift + lines. + + Examples: + ```python {test=compile reason="needs a migrations project and a database"} + async def test_the_chain_round_trips(db_url): + await ferro.connect(db_url) + result = await harness().round_trip() + assert result.irreversible is None + ``` """ chain = self._chain() async with self._target.open() as name: @@ -441,18 +483,46 @@ def harness( database: str | None = None, using: str | None = None, ) -> Harness: - """A :class:`Harness` over ``database`` (``None``: the one configured) of - ``settings`` (``None``: ``FerroSettings()``), working on the open - connection ``using`` (``None``: the default connection), which the test - connects and makes fresh itself:: + """Return a [`Harness`][ferro.migrations.testing.Harness] over one configured database. - @pytest.fixture - async def h(db_url): - await ferro.connect(db_url) - return harness() + The test connects the database and makes it fresh itself. + + Args: + settings: The project's configuration. Defaults to `FerroSettings()`. + database: The configured database. Defaults to the only one + configured. + using: The open connection to work on. Defaults to the default + connection. + + Returns: + The harness. Raises: - MigrationRefused: no configuration is found, or ``database`` names + MigrationRefused: No configuration is found, or `database` names none of its databases. + + Examples: + Migration `0012_add_slug` adds `author.slug` and backfills it in a + data step. The test seeds a row the way the table stood at `0011`, + applies exactly `0012`, and reads the row back the way `0012` left + it: + + ```python {test=compile reason="needs a migrations project and a database"} + from ferro.migrations.testing import harness + + + async def test_slug_backfill(db_url, migrations_settings): + await ferro.connect(db_url) + h = harness(settings=migrations_settings, database="default") + await h.apply_through("0011") + async with h.models_at("0011") as models: + await models.Author(name="Ada").save() # the schema as it stood + await h.apply("0012") # refused unless the database stands at 0011 + async with h.models_at("0012") as models: + ada = await models.Author.where(lambda author: author.name == "Ada").first() + assert ada.slug == "ada" + await h.revert_to("0011") + await h.round_trip() + ``` """ return Harness(Target.resolve(settings, database, using=using)) diff --git a/src/ferro/migrations/translate.py b/src/ferro/migrations/translate.py index d6caa1b5..4f157d44 100644 --- a/src/ferro/migrations/translate.py +++ b/src/ferro/migrations/translate.py @@ -1,12 +1,12 @@ -"""The Alembic bridge's translator: the one planner's ops as Alembic ops (ADR-0041). +"""The Alembic bridge's translator: the one planner's ops as Alembic ops. -```python +```python {test=compile reason="a model fragment; its imports are left out"} class Card(Model): flavor: str | None = None # new __ferro_checks__ = (Check("flavor_set", lambda card: card.flavor != None),) ``` -```python +```python {test=compile reason="the revision Alembic writes; Alembic runs it"} def upgrade(): op.add_column('card', sa.Column('flavor', sa.String(), nullable=True)) op.execute(sa.DDL('ALTER TABLE "card" ADD CONSTRAINT "ck_card_flavor_set" CHECK (...)')) @@ -16,8 +16,7 @@ def downgrade(): op.drop_column('card', 'flavor') ``` -Every op comes from ``_core._plan_revision`` (``plan_revision`` in the core), -which decides what the revision holds: each op of ``upgrade()`` and +Every op comes from the core's revision planner, which decides what the revision holds: each op of ``upgrade()`` and ``downgrade()`` in the planner's order, its statements, whether Alembic's own op writes it, its ``# ferro:`` marker and any irreversible reason. This module only writes that answer, and decides nothing. Where the core @@ -53,68 +52,103 @@ def downgrade(): class FerroExecuteOp(ops.MigrateOperation): """One statement the planner rendered, run as written. - Renders as ``op.execute(sa.DDL('<statement>'))``: a DDL construct is never - parsed for bind parameters (an enum label ``':admin'`` would be one in - ``text()``, #449), and its one rule is that ``%`` is written ``%%``. With - ``autocommit`` it runs inside ``op.get_context().autocommit_block()`` - (an enum label addition, which must be committed before a later - statement of the revision can use the label). + Renders as `op.execute(sa.DDL('<statement>'))`: a DDL construct is never + parsed for bind parameters (an enum label `':admin'` would be one in + `text()`), and its one rule is that `%` is written `%%`. With + `autocommit` it runs inside `op.get_context().autocommit_block()` (an + enum label addition, which must be committed before a later statement + of the revision can use the label). """ def __init__(self, statement: str, kind: str, *, autocommit: bool = False) -> None: self.statement = statement + """The statement, exactly as the planner rendered it.""" self.kind = kind + """The planner op's kind the statement belongs to.""" self.autocommit = autocommit + """Whether the statement runs inside an autocommit block.""" def to_diff_tuple(self) -> tuple[Any, ...]: + """Return the op as Alembic's diff tuple: `("ferro_execute", kind, statement)`. + + Returns: + The diff tuple. + """ return ("ferro_execute", self.kind, self.statement) class FerroMarkedOp(ops.MigrateOperation): - """Ops written under one ``# ferro: <marker>`` comment line.""" + """Ops written under one `# ferro: <marker>` comment line.""" def __init__(self, marker: str, wrapped: list[ops.MigrateOperation]) -> None: self.marker = marker + """The comment's text.""" self.wrapped = wrapped + """The ops written under it.""" def to_diff_tuple(self) -> Any: + """Return the wrapped ops' diff tuples, as a list. + + Returns: + One diff tuple per wrapped op. + """ return [op.to_diff_tuple() for op in self.wrapped] class FerroWarningOp(ops.MigrateOperation): - """A report the planner made without planning an op for it (an enum - label the model no longer declares, which ferro never removes), as a - ``# ferro: <warning>`` comment.""" + """A report the planner made without planning an op for it, written as a `# ferro: <warning>` comment. + + For example, an enum label the model no longer declares, which the + auto-migrate pass never removes. + """ def __init__(self, warning: str) -> None: self.warning = warning + """The report's sentence.""" def to_diff_tuple(self) -> tuple[Any, ...]: + """Return the op as Alembic's diff tuple: `("ferro_warning", warning)`. + + Returns: + The diff tuple. + """ return ("ferro_warning", self.warning) class FerroRenameTableOp(ops.RenameTableOp): - """``op.rename_table``: Alembic's own op, which its autogenerate never - proposes and so has no renderer of its own.""" + """Alembic's own `op.rename_table`, which its autogenerate never proposes and so has no renderer for.""" def to_diff_tuple(self) -> tuple[Any, ...]: + """Return the op as Alembic's diff tuple: `("rename_table", old, new)`. + + Returns: + The diff tuple. + """ return ("rename_table", self.table_name, self.new_table_name) class FerroIrreversibleOp(ops.MigrateOperation): - """A step nothing undoes: ``raise RuntimeError("<reason>")``.""" + """A step nothing undoes, written as `raise RuntimeError("<reason>")`.""" def __init__(self, reason: str) -> None: self.reason = reason + """Why nothing undoes it.""" def to_diff_tuple(self) -> tuple[Any, ...]: + """Return the op as Alembic's diff tuple: `("ferro_irreversible", reason)`. + + Returns: + The diff tuple. + """ return ("ferro_irreversible", self.reason) class FerroRevisionOps(ops.OpContainer): - """The whole revision ferro's comparator writes: the upgrade's ops, and - the downgrade's (the planner run back to the live database), which - ``reverse()`` swaps in. Never Alembic's per-op ``reverse()``.""" + """The whole revision Ferro's comparator writes: the upgrade's ops and the downgrade's. + + The downgrade is the planner run back to the live database, which + `reverse()` swaps in; Alembic's per-op `reverse()` is never used. + """ def __init__( self, @@ -123,8 +157,14 @@ def __init__( ) -> None: super().__init__(upgrade) self.downgrade = downgrade + """The downgrade's ops, in order.""" def reverse(self) -> FerroRevisionOps: + """Swap the upgrade and the downgrade. + + Returns: + The revision with the downgrade's ops as its upgrade. + """ return FerroRevisionOps(self.downgrade, list(self.ops)) @@ -210,7 +250,7 @@ def _not_creating_types(column: sa.Column, dialect: str) -> sa.Column: """``column`` (copied) whose native enum type the table op does not create: on Postgres every enum type a revision needs is created by the planner's ``CreateEnumType`` (or already exists), so SQLAlchemy must not - issue its own ``CREATE TYPE`` beside the table (#443).""" + issue its own ``CREATE TYPE`` beside the table.""" copied = column._copy() if dialect == "postgres" and isinstance(column.type, sa.Enum) and column.type.name: from sqlalchemy.dialects import postgresql @@ -397,16 +437,26 @@ def translate( dialect: str, reports: list[dict[str, Any]] | None = None, ) -> list[ops.MigrateOperation]: - """The Alembic ops that write one side of a revision, in its order. - - ``written`` is ``_core._plan_revision``'s ``upgrade`` or ``downgrade`` - list, ``target`` the envelope that side leads to (the models going up, - the live database going down) and ``dialect`` its dialect. The core has - decided every op: whether Alembic's own op writes it (``twin``) or - ``op.execute`` of the pass's statements does, the ``# ferro:`` comment - above it (``marker``) and the reason a ``raise RuntimeError(<reason>)`` - replaces it (``irreversible``). ``reports``, the revision's one-off - reports, lead the upgrade as comments. + """Build the Alembic ops that write one side of a revision, in its order. + + The core has decided every op: whether Alembic's own op writes it or + `op.execute` of the pass's statements does, the `# ferro:` comment above + it and the reason a `raise RuntimeError(<reason>)` replaces it. + + Args: + written: The core's planned upgrade or downgrade list. + target: The schema that side leads to: the models going up, the live + database going down. + dialect: The target's dialect. + reports: The revision's one-off reports, which lead the upgrade as + comments. Defaults to none. + + Returns: + The Alembic ops. + + Raises: + RuntimeError: The planner planned an op the bridge has no + translation for. """ resolved = _Target(target, dialect) out: list[ops.MigrateOperation] = [ diff --git a/src/ferro/models.py b/src/ferro/models.py index 1fc5f228..485fb6e1 100644 --- a/src/ferro/models.py +++ b/src/ferro/models.py @@ -126,17 +126,39 @@ def evict_instance( using: str | None = None, session: "Session | None" = None, ) -> None: - """Remove one instance from the active scope's identity map. + """Remove one instance from the session's identity map. + + Within a session, fetching a row you already hold returns the same + object. After an eviction, the next fetch of that row builds a new + instance and leaves the one you hold detached. A re-fetch already + refreshes a held instance in place, so evict only when you want a second, + separate object. See + [Eviction](../concepts/identity-map.md#eviction). + + Args: + model: The model class, or its name when the name is unambiguous. + pk: The instance's primary key, as a string. + using: The named connection whose map to evict from. Defaults to + `None`, the session's connection. + session: The session whose map to evict from. Defaults to `None`, the + session opened by the enclosing `async with engines.session():`. + + Raises: + RuntimeError: No session, `using=` or transaction gives the call a + database route. - ``model`` is a model class, its qualified identity, or an unambiguous - bare class name (ambiguity raises with the candidates listed). - - Public wrapper around the FFI `evict_instance` (FF-D D3): resolves the - route once via `resolve_operation_scope`, then passes it through. Model - instance methods (`save`/`delete`/`refresh`) call the FFI symbol - directly with their already-resolved route instead of going through this - wrapper, so a route is never resolved twice for one operation. + Examples: + ```python {test=run prelude=reference} + from ferro import evict_instance + + ada = await Author.get(1) + evict_instance(Author, "1") + fresh = await Author.get(1) + assert fresh is not ada + ``` """ + # The instance methods call the core's evict with the route they already + # resolved; this wrapper resolves it once here. from .registry import REGISTRY model_cls = REGISTRY.resolve_reference(model) if isinstance(model, str) else model @@ -151,34 +173,60 @@ async def transaction( session: "Session | None" = None, immediate: bool = False, ): - """Run database operations inside a transaction context. + """Run the operations in the block in one database transaction. + + The transaction commits when the block exits and rolls back when it + raises. A `transaction()` inside another one opens a savepoint. The + yielded [`Transaction`][ferro.Transaction] handle runs raw SQL on the + transaction's connection, for statements no model covers, such as + setting a Postgres setting or taking an advisory lock. See + [Transactions](../guide/transactions.md). + + Args: + using: The named connection to open the transaction on. Defaults to + `None`, the session's connection. + session: The session to open the transaction in. Defaults to `None`, + the session opened by the enclosing `async with engines.session():`. + immediate: On SQLite, open with `BEGIN IMMEDIATE`, which takes the + write lock at once, so a transaction that reads and then writes + never fails with `SQLITE_BUSY` while upgrading its lock. Postgres + and nested transactions ignore it. Defaults to `False`. + + Yields: + tx (ferro.Transaction): A handle with `execute`, `fetch_all` and + `fetch_one` for raw SQL on the transaction's connection. + + Raises: + RuntimeError: No session, `using=` or transaction gives the call a + database route. - Yields a :class:`~ferro.raw.Transaction` handle bound to this transaction's - connection. The handle exposes ``execute`` / ``fetch_all`` / ``fetch_one`` - for raw SQL on the same connection — useful for setting Postgres GUCs, - advisory locks, and any one-off statement that doesn't fit a Model. + Examples: + A failure inside the block rolls back every write in it: - ``immediate=True`` opens a SQLite transaction with ``BEGIN IMMEDIATE``: - it holds the database's write lock from the start, so a transaction that - reads and then writes never fails with ``SQLITE_BUSY`` upgrading its - lock (another writer waits on the busy timeout instead). On Postgres it - is a plain ``BEGIN`` and changes nothing; a nested transaction (a - savepoint) ignores it. + ```python {test=run prelude=reference} + from ferro import transaction - Examples: - >>> async with transaction() as tx: - ... user = await User.create(name="Taylor") - ... await tx.execute( - ... "select set_config('request.jwt.claims', $1, true)", - ... claims_json, - ... ) - - Existing callers that don't bind the yielded value continue to work; the - handle is simply discarded:: - - >>> async with transaction(): - ... user = await User.create(name="Taylor") - ... await user.save() + try: + async with transaction(): + await Author.create(name="Mary Somerville", email="mary@example.com") + raise RuntimeError("abandon the import") + except RuntimeError: + pass + + assert await Author.where(lambda author: author.name == "Mary Somerville").count() == 0 + ``` + + The handle runs raw SQL in the same transaction: + + ```python {test=run prelude=reference} + from ferro import transaction + + async with transaction() as tx: + await Author.create(name="Mary Somerville", email="mary@example.com") + row = await tx.fetch_one("SELECT count(*) AS total FROM author") + + assert row == {"total": 3} + ``` """ from .raw import Transaction @@ -211,54 +259,33 @@ async def transaction( class Model(BaseModel, metaclass=ModelMetaclass): - """Provide the base class for all Ferro models - - Inheriting from this class provisionally registers schema metadata in Python - caches and exposes high-performance CRUD and query entrypoints. The Rust - runtime is populated on first ``connect()`` via bulk install (#246). - - **Composite unique constraints:** declare a ``typing.ClassVar`` named - ``__ferro_composite_uniques__`` as a tuple of tuples of column names - (for example ``(("user_id", "org_id"),)``) to enforce uniqueness on those - columns together. This is separate from per-column uniqueness - (``Field(unique=True)`` on the field, ``Annotated[..., Field(unique=True)]``, - or ``Annotated[..., FerroField(unique=True)]``), each of which applies to a - single column only. Default many-to-many join tables get a - composite unique on their two foreign-key columns automatically. - - **Composite indexes:** declare a ``typing.ClassVar`` named - ``__ferro_composite_indexes__`` as a tuple of tuples of column names - (for example ``(("user_id", "created_at"),)``) for non-unique multi-column - indexes. Validation rules mirror ``__ferro_composite_uniques__``: each - inner tuple must contain at least two columns, columns must exist on the - model, and order is preserved (matters for leftmost-prefix optimization). - For single-column indexes use ``Field(index=True)``. Default many-to-many - join tables get a non-unique reverse-direction composite index - automatically; opt out with ``ManyToMany(reverse_index=False)``. - - **Table checks:** declare a ``typing.ClassVar`` named ``__ferro_checks__`` - as a tuple of :class:`~ferro.checks.Check` objects — each a name suffix - plus a ferro predicate lambda over this model's own columns, e.g. - ``(Check("at_most_one_outflow", lambda transfer: (transfer.outflow_transaction - == None) | (transfer.outflow_activity == None)),)``. The live constraint is - ``ck_<table>_<suffix>``, emitted inline in ``CREATE TABLE`` on both backends - (ADR-0012, ADR-0014). For a single-column closed-domain CHECK use - ``Field(db_check=True)`` instead. - - **Row security:** declare a ``typing.ClassVar`` named ``__ferro_rls__`` as a - :class:`~ferro.rowsecurity.RowSecurity` container of - :class:`~ferro.rowsecurity.RowPolicy` objects, e.g. - ``RowSecurity(RowPolicy(column="tenant_id", setting="app.tenant_id"))``. - A table created with it on gets ``ENABLE``/``FORCE ROW LEVEL SECURITY`` and - its ``rls_<table>_<name>`` policies, so PostgreSQL — not application - discipline — decides which rows a query can see. Row-level security is a - Postgres-only schema object: on SQLite the model registers and the table is - created, with a warning and no policies. + """The base class of every Ferro model: a Pydantic model backed by a table. + + A subclass declares its columns as annotated fields, and its table is + created on [`connect`][ferro.connect] with `auto_migrate=True`. Every + database call runs inside a session, opened with + `async with engines.session():`, and the examples on this page run in + one. See [Models and fields](../guide/models-and-fields.md) to declare a + model and [Connections](../guide/connections.md#sessions) for + sessions. + + A model declares table-wide constraints in class attributes: + `__ferro_composite_uniques__` and `__ferro_composite_indexes__` (see + [Composite uniques](../guide/models-and-fields.md#composite-uniques) and + [Composite indexes](../guide/models-and-fields.md#composite-indexes)), + `__ferro_checks__` (see + [Table checks](../guide/models-and-fields.md#table-checks)) and + `__ferro_rls__` (see [Using with Postgres RLS](../guide/row-level-security.md)). Examples: - >>> class User(Model): - ... id: int | None = None - ... name: str + ```python {test=run prelude=reference} + mary = await Author.create(name="Mary Somerville", email="mary@example.com") + fetched = await Author.get(mary.id) + assert fetched is mary + ``` + + Within a session, fetching a row you already hold returns the same + object. """ __ferro_composite_uniques__: ClassVar[tuple[tuple[str, ...], ...]] = () @@ -298,17 +325,18 @@ def _reregister_ferro(cls) -> None: use_attribute_docstrings=True, arbitrary_types_allowed=True, ) + """Pydantic settings shared by every model; a subclass may extend them.""" def __init__(self, **data: Any): - """Initialize a model instance and normalize relationship inputs + """Build an instance in memory, without saving it. Args: - **data: Field values used to construct the model. + **data: The field values. A relation field takes the related + instance or its primary key. - Examples: - >>> user = User(name="Taylor") - >>> isinstance(user, User) - True + Raises: + TypeError: A value names a retired field. + ValueError: A related instance's model has no primary key. """ retired = self.__class__.__ferro_retired_names__ if retired and not retired.isdisjoint(data): @@ -379,67 +407,67 @@ async def save( only: set[str] | frozenset[str] | list[str] | tuple[str, ...] | None = None, exclude: set[str] | frozenset[str] | list[str] | tuple[str, ...] | None = None, ) -> None: - """Persist the current model instance. - - A transient instance (constructed with ``Model(...)`` and never saved) - is INSERTed — a duplicate primary key or unique value raises - :class:`~ferro.exceptions.UniqueViolationError`. A persistent instance - (fetched from the database, or previously saved) is UPDATEd by primary - key. Pass ``on_conflict="update"`` for insert-or-update semantics - regardless of persistence state (the primitive behind - :meth:`upsert`). - - On a persisted UPDATE, ``only=`` is an explicit column allowlist - and ``exclude=`` is the complementary denylist: - ``await row.save(only={"messages", "updated_at"})`` writes those - columns' current in-memory values (including ``None`` → NULL); - ``await row.save(exclude={"turns"})`` writes every persisted - column except the denylist. The primary key may appear in either - set and is never SET. ``exclude=set()`` is a full write. - Passing both ``only=`` and ``exclude=`` raises. The instance is - not refreshed; omitted in-memory fields may diverge from the row. - Mixins that assign ``updated_at`` then call ``super().save(**kwargs)`` - forward the write-set without expanding it. ``only=`` / ``exclude=`` - are rejected on INSERT and on ``on_conflict="update"``. Use - :meth:`Query.update` for set-oriented or expression writes. - - Note that ``model_copy()`` copies persistence state: saving a copy of - a persisted instance updates the same row. The UPDATE targets the - instance's *current* primary-key value, so mutating the PK of a - persisted instance before ``save()`` matches no row and raises. A row - inserted inside a rolled-back transaction leaves the instance marked - persisted; a later ``save()`` raises ``ModelDoesNotExist``. + """Write the instance to its row: insert it when new, update it otherwise. + + An instance you built and never saved is inserted. One fetched from + the database, or saved before, updates its row by primary key, and + `only=` or `exclude=` limit that update to some columns. For an + update across many rows, use [`Query.update`][ferro.query.builder.Query.update]. + See [Saving: INSERT or UPDATE](../guide/mutations.md#saving-insert-or-update). Args: - using: Connection name override. - session: Session scope for the operation. - on_conflict: ``None`` (default) or ``"update"`` to upsert. - only: Persisted-UPDATE column allowlist (``set`` / ``frozenset`` - / ``list`` / ``tuple`` of column names). ``None`` writes - every column (unless ``exclude=`` is set). - exclude: Persisted-UPDATE column denylist (same containers as - ``only=``). ``exclude=set()`` is a full write. + using: The named connection to write on. Defaults to `None`, the + session's connection, or the one the instance was loaded from. + session: The session to run in. Defaults to `None`, the session + opened by the enclosing `async with engines.session():`. + on_conflict: `"update"` inserts the row or, when its primary key + already exists, updates it, whether or not the instance was + saved before. Defaults to `None`, a plain insert or update. + only: On an update, the columns to write, as a set, list or + tuple of field names. Defaults to `None`, every column. + exclude: On an update, the columns not to write. Defaults to + `None`; `exclude=set()` writes every column. Raises: - UniqueViolationError: A duplicate primary key or unique value on - INSERT. - ModelDoesNotExist: The row behind a persisted instance no longer - exists (deleted underneath, or the PK was mutated). - TypeError: ``only=`` or ``exclude=`` is a bare ``str`` or - ``bytes``. - ValueError: ``on_conflict`` is not ``None`` or ``"update"``, a - persisted instance has no primary-key value, ``only=`` or - ``exclude=`` is used on INSERT / upsert, both write-sets - are passed, the write-set is empty after stripping the PK, - or a name is unknown / a relation. + UniqueViolationError: An insert repeats a primary key or unique + value. + ModelDoesNotExist: The row behind a saved instance no longer + exists, or its primary key was changed in memory. + TypeError: `only` or `exclude` is a bare string. + ValueError: `on_conflict` is neither `None` nor `"update"`; `only` + or `exclude` is given on an insert or with `on_conflict`, both + are given, or one names an unknown field or a relation; or a + saved instance has no primary key value. + + Note: + The instance is not reloaded after a partial update, so a field + left out of `only` keeps its value in memory even if the row + differs. A copy made with `model_copy()` updates the same row as + the original. An instance inserted in a transaction that rolled + back still counts as saved, so its next `save()` raises + `ModelDoesNotExist`. Examples: - >>> user = User(name="Taylor") - >>> await user.save() - >>> row.messages = msgs - >>> row.updated_at = utcnow() - >>> await row.save(only={"messages", "updated_at"}) - >>> await row.save(exclude={"turns"}) + Saving a new instance inserts it: + + ```python {test=run prelude=reference} + mary = Author(name="Mary Somerville", email="mary@example.com") + await mary.save() + assert mary.id == 3 + ``` + + Saving a fetched instance with `only=` updates just those columns: + + ```python {test=run prelude=reference} + post = await Post.get(2) + post.views = 1000 + post.title = "Not written to the row" + await post.save(only={"views"}) + + rows = await Post.select("title", "views").where(lambda post: post.id == 2).all() + assert rows[0].title == "The Education of a Computer" + assert rows[0].views == 1000 + ``` """ if on_conflict not in (None, "update"): raise ValueError( @@ -528,15 +556,28 @@ async def save( async def delete( self, *, using: str | None = None, session: "Session | None" = None ) -> None: - """Delete the current model instance from storage + """Delete the instance's row. - Returns: - None + The instance stays in memory and counts as new again: saving it + inserts a new row. An instance without a primary key value has no + row, and the call does nothing. + + Args: + using: The named connection to delete on. Defaults to `None`, the + session's connection, or the one the instance was loaded from. + session: The session to run in. Defaults to `None`, the session + opened by the enclosing `async with engines.session():`. + + Raises: + ValueError: `using` names a different connection from the one the + instance was loaded from. Examples: - >>> user = await User.get_or_none(1) - >>> if user: - ... await user.delete() + ```python {test=run prelude=reference} + post = await Post.get(2) + await post.delete() + assert await Post.get_or_none(2) is None + ``` """ pk_field_name = self.__class__.__ferro_pk__ pk_val = getattr(self, pk_field_name) if pk_field_name is not None else None @@ -556,37 +597,48 @@ async def delete( async def all( cls, *, using: str | None = None, session: "Session | None" = None ) -> list[Self]: - """Fetch all records for this model class + """Fetch every row of the model's table. + + Args: + using: The named connection to read from. Defaults to `None`, the + session's connection. + session: The session to run in. Defaults to `None`, the session + opened by the enclosing `async with engines.session():`. Returns: - A list of hydrated model instances. + One instance per row, in the order the database returns them. Examples: - >>> users = await User.all() - >>> isinstance(users, list) - True + ```python {test=run prelude=reference} + authors = await Author.all() + assert sorted(author.name for author in authors) == ["Ada Lovelace", "Grace Hopper"] + ``` """ route = await _transaction_or_using(using, session) return await fetch_all(cls, route) @classmethod async def get(cls, pk: Any, *, session: "Session | None" = None) -> Self: - """Fetch one record by primary key value. + """Fetch one record by primary key. Args: - pk: Primary key value to fetch a single record. + pk: The primary key value to look up. + session: The session to run in. Defaults to `None`, the session + opened by the enclosing `async with engines.session():`. Returns: - The matching model instance. + The record whose primary key is `pk`. Raises: - ModelDoesNotExist: When no row exists for this primary key. Use - :meth:`get_or_none` if you need optional lookup without raising. + ModelDoesNotExist: No row has that key. Use + [`get_or_none`][ferro.Model.get_or_none] to get `None` instead. + RuntimeError: The model declares no primary key. Examples: - >>> user = await User.get(1) - >>> isinstance(user, User) - True + ```python {test=run prelude=reference} + author = await Author.get(1) + assert author.name == "Ada Lovelace" + ``` """ instance = await cls.get_or_none(pk, session=session) if instance is None: @@ -597,13 +649,23 @@ async def get(cls, pk: Any, *, session: "Session | None" = None) -> Self: async def get_or_none( cls, pk: Any, *, session: "Session | None" = None ) -> Self | None: - """Fetch one record by primary key, or return None if no row exists. + """Fetch one record by primary key, or `None` when no row has it. Args: - pk: Primary key value to fetch a single record. + pk: The primary key value to look up. + session: The session to run in. Defaults to `None`, the session + opened by the enclosing `async with engines.session():`. Returns: - The matching model instance, or None when no record exists. + The record whose primary key is `pk`, or `None`. + + Raises: + RuntimeError: The model declares no primary key. + + Examples: + ```python {test=run prelude=reference} + assert await Author.get_or_none(99) is None + ``` """ pk_field_name = cls.__ferro_pk__ if pk_field_name is None: @@ -614,17 +676,28 @@ async def get_or_none( async def refresh( self, *, using: str | None = None, session: "Session | None" = None ) -> None: - """Reload this instance from storage using its primary key + """Reload the instance's fields from its row, in place. - Returns: - None + A change made in memory and not saved is lost. See + [`refresh()`](../guide/mutations.md#refresh). + + Args: + using: The named connection to read from. Defaults to `None`, the + session's connection, or the one the instance was loaded from. + session: The session to run in. Defaults to `None`, the session + opened by the enclosing `async with engines.session():`. Raises: - RuntimeError: If no primary key is available or the record no longer exists. + RuntimeError: The instance has no primary key value, or its row no + longer exists. Examples: - >>> user = await User.get(1) - >>> await user.refresh() + ```python {test=run prelude=reference} + post = await Post.get(1) + post.views = 0 + await post.refresh() + assert post.views == 1843 + ``` """ pk_field_name = self.__class__.__ferro_pk__ pk_val = getattr(self, pk_field_name) if pk_field_name is not None else None @@ -653,30 +726,28 @@ async def refresh( def where( cls, predicate: "Predicate[Self]", *, session: "Session | None" = None ) -> Query[Self]: - """Start a fluent query with an initial condition. - - ``predicate`` is a lambda of shape - ``Callable[[QueryProxy[Self]], QueryNode]``, e.g. - ``User.where(lambda user: user.age >= 18)``. The lambda receives a - :class:`QueryProxy` whose attributes build comparisons as - :class:`QueryNode` instances, so predicates type-check cleanly. - Name the parameter after the model in lowercase singular (``user`` for - ``User``, ``post`` for ``Post``). Column names are validated at build - time against the model's declared fields (plus shadow ``{fk}_id`` - columns). + """Start a query filtered by a lambda predicate. + + The lambda receives a stand-in for one row; comparing its fields + builds the SQL condition. Name the parameter after the model, in + lowercase. A field name the model does not declare raises when the + query is built. See [Queries](../guide/queries.md#filtering-with-where). Args: - predicate: A callable that takes a :class:`QueryProxy` and - returns a :class:`QueryNode`. + predicate: A lambda that takes the row stand-in and returns a + condition, such as `lambda post: post.views > 1000`. + session: The session the query runs in. Defaults to `None`, the + session opened by the enclosing `async with engines.session():`. Returns: - A query object scoped to this model class. + A query over the matching rows; nothing runs until you await one + of its methods, such as [`all`][ferro.query.builder.Query.all]. Examples: - >>> q1 = User.where(lambda user: user.archived == False) # noqa: E712 - >>> q2 = User.where(lambda user: user.id == 1) - >>> isinstance(q1, Query) and isinstance(q2, Query) - True + ```python {test=run prelude=reference} + popular = await Post.where(lambda post: post.views > 1000).all() + assert [post.title for post in popular] == ["Notes on the Analytical Engine"] + ``` """ return Query(cls, session=session).where(predicate) @@ -702,79 +773,114 @@ def select( *selectors: "RowSelector[Self] | str", session: "Session | None" = None, ) -> "Query[Self] | ProjectedQuery[Self]": - """Start a fluent query, optionally projected to a column subset. - - Bare ``select()`` starts a full query of complete model instances - (unchanged). With a lambda selector — - ``select(lambda t: (t.id, t.amount))``, or the single-field form - ``select(lambda t: t.amount)`` — the query is a projection: its - results are :class:`~ferro.query.Row` records in the list-like - :class:`~ferro.query.Rows` container, never model instances - (ADR-0007). Column-name strings (``select("id", "amount")``) follow - ``order_by``'s string contract: root columns only, never mixed with - a lambda. Both forms validate at build time with did-you-mean. + """Start a query, of whole records or of some columns. + + With no argument, the query returns model instances. With a lambda + that picks fields, or with column names, it returns + [`Row`][ferro.query.Row] records holding only those columns. See + [Selecting a column subset](../guide/queries.md#selecting-a-column-subset). + + Args: + *selectors: Nothing; one lambda returning a field, a tuple of + fields or a dict naming them; or column names as strings. + session: The session the query runs in. Defaults to `None`, the + session opened by the enclosing `async with engines.session():`. Returns: - A query object scoped to this model class; projected when a - selector is given. + A [`Query`][ferro.query.builder.Query] with no selector, otherwise a + [`ProjectedQuery`][ferro.query.builder.ProjectedQuery]. + + Raises: + AttributeError: A selector names a column the model does not have. + TypeError: The selectors mix a lambda with column names, or are + not a lambda or strings. + ValueError: The selection is empty, two fields share an output + name, or a string names a related column. Examples: - >>> query = User.select().limit(5) - >>> isinstance(query, Query) - True - >>> rows = await Transaction.select(lambda t: (t.id, t.amount)).all() # doctest: +SKIP + ```python {test=run prelude=reference} + rows = await Post.select(lambda post: (post.title, post.views)).order_by(lambda post: post.id).all() + assert rows[0].title == "Notes on the Analytical Engine" + assert rows[0].views == 1843 + ``` """ return Query(cls, session=session).select(*selectors) @classmethod def using(cls, name: str) -> "ModelConnection[Self]": - """Bind ORM operations for this model to a named connection.""" + """Run this model's operations on a named connection. + + Args: + name: The connection's name, as given to `connect(..., name=...)`. + + Returns: + An object with this model's `create`, `get`, `where` and other + class methods, each running on `name`. + + Examples: + ```python {test=run prelude=reference} + from ferro import connect, engines + + await connect("sqlite::memory:", name="archive", auto_migrate=True) + async with engines.session("archive"): + await Author.using("archive").create(name="Mary Somerville", email="mary@example.com") + archived = await Author.using("archive").all() + + assert [author.name for author in archived] == ["Mary Somerville"] + ``` + """ return ModelConnection(cls, name) @classmethod - async def create(cls, *, session: "Session | None" = None, **fields) -> Self: - """Create and persist a new model instance + async def create(cls, *, session: "Session | None" = None, **fields: Any) -> Self: + """Insert a new record and return it. - ``create()`` is a plain INSERT: it never updates an existing row. + `create` always inserts; use [`upsert`][ferro.Model.upsert] to update + a row whose primary key exists. Args: - **fields: Field values to construct the model. + session: The session to run in. Defaults to `None`, the session + opened by the enclosing `async with engines.session():`. + **fields: The field values. A relation field takes the related + instance or its primary key. Returns: - The newly created and persisted model instance. + The saved instance, with a generated primary key filled in. Raises: UniqueViolationError: A row with the same primary key or unique - value already exists — use :meth:`upsert` for - insert-or-update semantics. + value exists. Examples: - >>> user = await User.create(name="Taylor") - >>> isinstance(user, User) - True + ```python {test=run prelude=reference} + mary = await Author.create(name="Mary Somerville", email="mary@example.com") + assert mary.id == 3 + ``` """ instance = cls(**fields) await instance.save(session=session) return instance @classmethod - async def upsert(cls, *, session: "Session | None" = None, **fields) -> Self: - """Insert the row, or update the existing row on primary-key conflict. + async def upsert(cls, *, session: "Session | None" = None, **fields: Any) -> Self: + """Insert a record, or update the row that has its primary key. - Equivalent to ``cls(**fields).save(on_conflict="update")``. With an - autoincrement primary key left unset there is no conflict target, so - this degrades to a plain INSERT. + Without a primary key value, an auto-increment key has nothing to + conflict on and the record is inserted. Args: - **fields: Field values to construct the model. + session: The session to run in. Defaults to `None`, the session + opened by the enclosing `async with engines.session():`. + **fields: The field values, including the primary key. Returns: - The persisted model instance. + The saved instance. Examples: - >>> user = await User.upsert(id=1, name="Taylor") - >>> isinstance(user, User) - True + ```python {test=run prelude=reference} + await Author.upsert(id=1, name="Ada King", email="ada@example.com") + assert (await Author.get(1)).name == "Ada King" + ``` """ instance = cls(**fields) await instance.save(session=session, on_conflict="update") @@ -788,24 +894,33 @@ async def bulk_create( using: str | None = None, session: "Session | None" = None, ) -> int: - """Persist multiple instances in a single bulk operation + """Insert many new records in one call. - Batch size is unbounded: batches larger than the backend's - bind-parameter limit allows in one statement are split internally. - The call is atomic either way — inside an ambient ``transaction()`` - that transaction is the atomicity boundary; a bare call is - all-or-nothing (a failure on any row inserts none). + Every record is inserted or none is. Ferro splits a batch too large + for one statement into several, inside the same all-or-nothing + boundary. See + [Creating rows](../guide/mutations.md#creating-rows). Args: - instances: Model instances to persist. + instances: The instances to insert. + using: The named connection to write on. Defaults to `None`, the + session's connection. + session: The session to run in. Defaults to `None`, the session + opened by the enclosing `async with engines.session():`. Returns: - The number of records inserted. + The number of rows inserted. Examples: - >>> rows = await User.bulk_create([User(name="A"), User(name="B")]) - >>> isinstance(rows, int) - True + ```python {test=run prelude=reference} + inserted = await Author.bulk_create( + [ + Author(name="Mary Somerville", email="mary@example.com"), + Author(name="Hedy Lamarr", email="hedy@example.com"), + ] + ) + assert inserted == 2 + ``` """ if not instances: return 0 @@ -824,21 +939,32 @@ async def get_or_create( defaults: dict[str, Any] | None = None, *, session: "Session | None" = None, - **fields, + **fields: Any, ) -> tuple[Self, bool]: - """Fetch a record by filters or create one when missing + """Fetch the first record whose fields match, or create it. Args: - defaults: Values applied only when creating a new record. - **fields: Exact-match filters used for lookup. + defaults: Extra field values used only when creating. Defaults to + `None`. + session: The session to run in. Defaults to `None`, the session + opened by the enclosing `async with engines.session():`. + **fields: The values to match exactly, also used when creating. Returns: - A tuple of ``(instance, created)`` where ``created`` is True for new records. + The record, and `True` when it was created or `False` when it + was found. Examples: - >>> user, created = await User.get_or_create(email="a@b.com") - >>> isinstance(created, bool) - True + ```python {test=run prelude=reference} + author, created = await Author.get_or_create( + email="mary@example.com", defaults={"name": "Mary Somerville"} + ) + assert created is True + + again, created = await Author.get_or_create(email="mary@example.com") + assert created is False + assert again is author + ``` """ query = Query(cls, session=session) for key, val in fields.items(): @@ -857,16 +983,29 @@ async def update_or_create( defaults: dict[str, Any] | None = None, *, session: "Session | None" = None, - **fields, + **fields: Any, ) -> tuple[Self, bool]: - """Update a matched record or create one when missing + """Update the first record whose fields match, or create it. Args: - defaults: Values applied on update or create paths. - **fields: Exact-match filters used for lookup. + defaults: The field values to write, on the found record or the + new one. Defaults to `None`. + session: The session to run in. Defaults to `None`, the session + opened by the enclosing `async with engines.session():`. + **fields: The values to match exactly, also used when creating. Returns: - A tuple of ``(instance, created)`` where ``created`` is True for new records. + The record, and `True` when it was created or `False` when it + was updated. + + Examples: + ```python {test=run prelude=reference} + author, created = await Author.update_or_create( + email="ada@example.com", defaults={"name": "Ada King"} + ) + assert created is False + assert author.name == "Ada King" + ``` """ query = Query(cls, session=session) for key, val in fields.items(): @@ -884,50 +1023,111 @@ async def update_or_create( class ModelConnection[M: Model]: - """Connection-bound ORM entrypoint returned by ``Model.using(name)``. + """A model's class methods bound to one named connection. - Generic over the concrete model class so that every accessor preserves - the bound type — e.g. ``Transcript.using("service").get(pk)`` resolves - to ``Transcript`` rather than ``Model``. + [`Model.using`][ferro.Model.using] returns one. Each method behaves like + the model's class method of the same name and runs on the bound + connection. """ def __init__(self, model_cls: type[M], connection_name: str) -> None: + """Bind a model class to a connection. + + Args: + model_cls: The model class. + connection_name: The name of the connection to run on. + """ self.model_cls: type[M] = model_cls + """The bound model class.""" self._connection_name: str = connection_name async def create(self, **fields: Any) -> M: - """Create and persist a new instance on this connection. + """Insert a new record on the connection and return it. - A plain INSERT — a duplicate primary key or unique value raises - :class:`~ferro.exceptions.UniqueViolationError`; use :meth:`upsert` - for insert-or-update semantics. + Args: + **fields: The field values. + + Returns: + The saved instance. + + Raises: + UniqueViolationError: A row with the same primary key or unique + value exists. """ instance = self.model_cls(**fields) await instance.save(using=self._connection_name) return instance async def upsert(self, **fields: Any) -> M: - """Insert the row on this connection, or update it on PK conflict.""" + """Insert a record on the connection, or update the row with its primary key. + + Args: + **fields: The field values, including the primary key. + + Returns: + The saved instance. + """ instance = self.model_cls(**fields) await instance.save(using=self._connection_name, on_conflict="update") return instance async def all(self) -> list[M]: + """Fetch every row of the model's table on the connection. + + Returns: + One instance per row. + """ return await self.model_cls.all(using=self._connection_name) def select(self) -> Query[M]: + """Start a query of whole records on the connection. + + Returns: + A query over every row. + """ return Query(self.model_cls, using=self._connection_name) def where(self, predicate: "Predicate[M]") -> Query[M]: + """Start a query on the connection, filtered by a lambda predicate. + + Args: + predicate: A lambda that takes the row stand-in and returns a + condition. + + Returns: + A query over the matching rows. + """ return self.select().where(predicate) async def get(self, pk: Any) -> M: + """Fetch one record by primary key on the connection. + + Args: + pk: The primary key value to look up. + + Returns: + The record whose primary key is `pk`. + + Raises: + ModelDoesNotExist: No row has that key. + """ instance = await self.get_or_none(pk) if instance is None: raise ModelDoesNotExist(self.model_cls, pk) return instance async def get_or_none(self, pk: Any) -> M | None: + """Fetch one record by primary key on the connection, or `None`. + + Args: + pk: The primary key value to look up. + + Returns: + The record whose primary key is `pk`, or `None`. + + Raises: + RuntimeError: The model declares no primary key. + """ pk_field_name = self.model_cls.__ferro_pk__ if pk_field_name is None: raise RuntimeError( @@ -937,11 +1137,29 @@ async def get_or_none(self, pk: Any) -> M | None: return await self.where(_field_eq(pk_field_name, pk)).first() async def bulk_create(self, instances: list[M]) -> int: + """Insert many new records on the connection in one call. + + Args: + instances: The instances to insert. + + Returns: + The number of rows inserted. + """ return await self.model_cls.bulk_create(instances, using=self._connection_name) async def get_or_create( self, defaults: dict[str, Any] | None = None, **fields: Any ) -> tuple[M, bool]: + """Fetch the first matching record on the connection, or create it. + + Args: + defaults: Extra field values used only when creating. Defaults to + `None`. + **fields: The values to match exactly, also used when creating. + + Returns: + The record, and whether it was created. + """ query = Query(self.model_cls, using=self._connection_name) for key, val in fields.items(): query = query.where(_field_eq(key, val)) @@ -956,6 +1174,16 @@ async def get_or_create( async def update_or_create( self, defaults: dict[str, Any] | None = None, **fields: Any ) -> tuple[M, bool]: + """Update the first matching record on the connection, or create it. + + Args: + defaults: The field values to write, on the found record or the + new one. Defaults to `None`. + **fields: The values to match exactly, also used when creating. + + Returns: + The record, and whether it was created. + """ query = Query(self.model_cls, using=self._connection_name) for key, val in fields.items(): query = query.where(_field_eq(key, val)) diff --git a/src/ferro/pass_report.py b/src/ferro/pass_report.py index 5c88a4bd..beb228c8 100644 --- a/src/ferro/pass_report.py +++ b/src/ferro/pass_report.py @@ -1,18 +1,10 @@ -"""What an auto-migrate pass did: the value ``ferro.migrate()`` and -``ferro.create_tables()`` return (ADR-0049). - -```python -report = await ferro.migrate() -[(s.subject, s.sql) for s in report.statements if s.role == "schema"] -# [('author', 'ALTER TABLE "author" ADD COLUMN "slug" TEXT')] -[(w.kind, str(w)) for w in report.warnings] -# [] -``` - -The Rust core builds the report from what its DDL executor actually ran, -never from the plan, so it cannot describe a statement that did not execute. -A pass that fails partway raises its usual error with ``.report`` set: what -committed before the failure, the failing statement left out. +"""What an auto-migrate pass did: the value `ferro.migrate()` and `ferro.create_tables()` return. + +The report is built from what the pass actually executed, never from its +plan, so it cannot list a statement that did not run. A pass that fails +partway raises its usual error with `.report` set to what committed before +the failure, the failing statement left out. See +[Auto-migrate](../guide/schema/auto-migrate.md). """ from __future__ import annotations @@ -31,10 +23,14 @@ ] Role = Literal["schema", "lock_timeout", "probe"] -"""What an executed statement was for: ``"schema"`` (the create pass, the -type statements, the reconciliation), ``"lock_timeout"`` (the ``SET LOCAL -lock_timeout`` / ``SET`` / ``RESET`` lines of ADR-0044) or ``"probe"`` (the -SQLite label row probe of ADR-0047).""" +"""What an executed statement was for: `"schema"`, `"lock_timeout"` or `"probe"`. + +`"schema"` is the create pass, the enum type statements and the +reconciliation. `"lock_timeout"` is a `SET LOCAL lock_timeout` (or `SET` / +`RESET`) the pass wraps each Postgres unit in. `"probe"` is a read that only +decides what the pass reports, such as SQLite looking for rows that hold an +enum label being renamed. +""" _PASS_REPORT_ATTR = "_ferro_pass_report" """The attribute the Rust core sets on a failed pass's error (the report as @@ -50,18 +46,25 @@ class ExecutedStatement: sql: str """The statement, exactly as it ran.""" role: Role - """What it was for (see ``Role``).""" + """What it was for: `"schema"` (DDL), `"lock_timeout"` or `"probe"` (a read that changes nothing).""" @dataclass(frozen=True) class Subject: - """What a ``Report`` is about: the modelset as a whole, one table, one - column, or one enum type. Only the fields of its ``scope`` are set.""" + """What a [`Report`][ferro.Report] is about. + + The modelset as a whole, one table, one column, or one enum type. Only the + fields its `scope` names are set. + """ scope: Literal["modelset", "table", "column", "enum_type"] + """Which of the four it is: `"modelset"`, `"table"`, `"column"` or `"enum_type"`.""" table: str | None = None + """The table, for the `"table"` and `"column"` scopes; `None` otherwise.""" column: str | None = None + """The column, for the `"column"` scope; `None` otherwise.""" type_name: str | None = None + """The Postgres enum type, for the `"enum_type"` scope; `None` otherwise.""" @classmethod def _from_wire(cls, wire: dict[str, Any]) -> Subject: @@ -75,22 +78,20 @@ def _from_wire(cls, wire: dict[str, Any]) -> Subject: @dataclass(frozen=True) class Report: - """One warning, typed (ADR-0052): the planner's, the renderer's, or the - pass's own. ``str(report)`` is the sentence printed. - - ``kind`` is the kind's name: the planner's (``"LeftoverChecks"``, - ``"HintRefused"``, ``"ForeignDefault"``, ...), the renderer's (``"RefusedConversion"``, - ``"SqliteInPlace"``, ``"PrimaryKeyKept"``, ``"EnumTypeMove"``, - ``"RowSecuritySkipped"``) or - the pass's (``"PendingTableRename"``, ``"StrandedLabelRename"``, - ``"RowSecurityUnderMigrator"``, ``"RunLockWait"``, ``"DdlLockRetry"``). - ``recurs`` says whether it is raised on every pass until someone acts. + """One warning a pass raised, with a kind code can match on. + + `str(report)` is the sentence the pass printed. The + [Auto-migrate](../guide/schema/auto-migrate.md) guide lists every kind. """ kind: str + """The warning's kind, such as `"LeftoverChecks"`, `"RefusedConversion"` or `"RunLockWait"`.""" subject: Subject + """What the warning is about.""" text: str + """The sentence the pass printed and raised as a `UserWarning`.""" recurs: bool + """Whether every pass raises it again until someone acts.""" def __str__(self) -> str: return self.text @@ -109,15 +110,27 @@ def _from_wire(cls, wire: dict[str, Any]) -> Report: @dataclass(frozen=True) class PassReport: - """Every statement an auto-migrate pass sent to the database, in - execution order, and every warning it raised, in the order raised. + """What one auto-migrate pass sent to the database and the warnings it raised. + + Every warning is also raised as a Python `UserWarning`. + + Examples: + ```python {test=run prelude=reference} + import ferro - Every warning is also raised as a Python ``UserWarning``, as before; the - report lists it as well. + await ferro.raw.execute('DROP TABLE "post"') + report = await ferro.create_tables() + print([(statement.subject, statement.role) for statement in report.statements]) + # [('post', 'schema')] + print(report.warnings) + # () + ``` """ statements: tuple[ExecutedStatement, ...] = () + """Every statement the pass sent, in execution order.""" warnings: tuple[Report, ...] = () + """Every warning the pass raised, in the order raised.""" @classmethod def _from_json(cls, text: str) -> PassReport: diff --git a/src/ferro/query/__init__.py b/src/ferro/query/__init__.py index 56123300..d5636a3f 100644 --- a/src/ferro/query/__init__.py +++ b/src/ferro/query/__init__.py @@ -1,4 +1,8 @@ -"""Expose query-building primitives used by Ferro models""" +"""Query builders, the pieces query lambdas build, and projected records. + +Start a query from a model with [`Model.where`][ferro.Model.where] or +[`Model.select`][ferro.Model.select]; see [Queries](../guide/queries.md). +""" from .builder import ProjectedQuery, Query, Relation from .nodes import ( @@ -12,7 +16,7 @@ ValueExpr, now, ) -from .rendered import REDACTED, RenderedSql +from .rendered import REDACTED, RenderedSql, Terminal from .rows import Row, Rows __all__ = [ @@ -27,6 +31,7 @@ "Relation", "RelationProxy", "RenderedSql", + "Terminal", "Row", "RowSelector", "Rows", diff --git a/src/ferro/query/builder.py b/src/ferro/query/builder.py index a628a6c4..eadc3e04 100644 --- a/src/ferro/query/builder.py +++ b/src/ferro/query/builder.py @@ -1,11 +1,13 @@ -"""Build fluent query objects; ``ferro.query.wire`` compiles them to QueryIR. +"""Query builders: [`Query`][ferro.query.builder.Query], its projected form and relation collections. -This module owns chainer-time state and validation (predicates, joins, -projection, includes). Everything whose output is wire shape — the payload -dataclasses, hop-fact resolution, join serialization, materialization plans, -verb policy, and the versioned envelope — lives behind ``wire.compile_query``. +Each chained call validates its arguments and returns a new query; nothing +reaches the database until a query is awaited through one of its methods. """ +# This module owns chain-time state and validation (predicates, joins, +# projection, includes). Everything whose output is wire shape lives behind +# ``wire.compile_query``. + import copy from collections.abc import Awaitable from typing import ( @@ -65,7 +67,9 @@ from .._core import RouteHandle T = TypeVar("T") +"""The model class a query returns.""" E = TypeVar("E") +"""The element type of a relation collection.""" # The concrete container type a projected query delivers (parameterized once # at import; `into=` record types would parameterize per call). Construction @@ -199,9 +203,9 @@ def _resolve_join_selector( ) -> tuple[str, ...]: """Resolve a join-chainer selector into a relation path (#272). - ``selector`` is a lambda receiving a validating :class:`QueryProxy` and + ``selector`` is a lambda receiving a validating `QueryProxy` and naming a RELATION path (``lambda t: t.account``, ``lambda t: t.account.owner``) - — i.e. it must return a :class:`RelationProxy`. Each hop is validated against + — i.e. it must return a `RelationProxy`. Each hop is validated against the relevant model at build time (the proxy raises ``AttributeError`` with a did-you-mean for a bad hop, same as ``where()``). @@ -210,7 +214,7 @@ def _resolve_join_selector( Raises: TypeError: If ``selector`` is not callable, resolves to a column - (:class:`FieldProxy`) rather than a relation, or returns any other + (`FieldProxy`) rather than a relation, or returns any other non-relation value — a join selector names a relation path, not a column. """ @@ -271,10 +275,10 @@ def _reverse_population_error(name: str) -> TypeError: """The pinned include()-cannot-populate-a-reverse-relation error (#287). One construction site for the two paths that reach it: a reverse relation - the predicate proxies resolve (a :class:`ReverseRelationProxy` selector + the predicate proxies resolve (a `ReverseRelationProxy` selector result, #314) and one they don't yet (an M2M name, surfaced as an ``AttributeError`` and re-raised by - :func:`_reject_reverse_relation_include`). + `_reject_reverse_relation_include`). """ return TypeError( f"include() cannot populate {name!r}: it is a reverse (BackRef) " @@ -292,10 +296,10 @@ def _resolve_include_selector( ) -> tuple[str, ...]: """Resolve an ``include()`` selector into a relation path (#286). - ``selector`` is a lambda receiving a validating :class:`QueryProxy` and + ``selector`` is a lambda receiving a validating `QueryProxy` and naming a forward-FK RELATION path (``lambda t: t.account``, ``lambda t: t.account.owner``) — the same traversal syntax as ``join()``, - resolving to a :class:`RelationProxy`. Each hop is validated against the + resolving to a `RelationProxy`. Each hop is validated against the relevant model at build time (the proxy raises ``AttributeError`` with a did-you-mean for a bad hop, exactly like ``where()``). @@ -305,7 +309,7 @@ def _resolve_include_selector( Raises: TypeError: If ``selector`` is a string or otherwise not callable (strings never traverse, #280 — include is lambda-selector only), - resolves to a column (:class:`FieldProxy`) rather than a relation, + resolves to a column (`FieldProxy`) rather than a relation, names a BackRef/M2M relation (reverse population is a separate future mechanism, #287), or returns any other non-relation value. """ @@ -375,7 +379,7 @@ def _resolve_projection_selector( ) -> tuple[_ProjectedField, ...]: """Resolve a ``select()`` lambda selector into projected fields (#279/#293). - ``selector`` receives a validating :class:`QueryProxy` and returns one + ``selector`` receives a validating `QueryProxy` and returns one field (``lambda t: t.amount``), a tuple/list of fields (``lambda t: (t.id, t.account.name)``), or a dict whose keys name the output fields (``lambda t: {"account_name": t.account.name}`` — output @@ -435,7 +439,7 @@ def _resolve_projection_strings( Raises: ValueError: For a dotted path, or a column named twice. AttributeError: For a name that is not a queryable column - (did-you-mean, from :func:`validate_query_column`). + (did-you-mean, from `validate_query_column`). """ columns: list[str] = [] for name in names: @@ -605,32 +609,47 @@ def _update_terminal(method: Callable[..., Awaitable[int]]) -> "_UpdateMethod": class Query(Generic[T]): - """Build and execute fluent ORM queries. + """A lazy query over one model's rows. + + Start one from the model with [`Model.where`][ferro.Model.where] or + [`Model.select`][ferro.Model.select]. Each chained method returns a new + query and leaves the original unchanged; nothing runs until you await + [`all`][ferro.query.builder.Query.all], [`first`][ferro.query.builder.Query.first], + [`count`][ferro.query.builder.Query.count], [`exists`][ferro.query.builder.Query.exists], + [`update`][ferro.query.builder.Query.update] or + [`delete`][ferro.query.builder.Query.delete]. See [Queries](../guide/queries.md). - Attributes: - model_cls: Model class used to hydrate results. - where_clause: Accumulated filter nodes for the query. - order_by_clause: Sort definitions sent to the Rust core. + Examples: + ```python {test=run prelude=reference} + query = Post.where(lambda post: post.views > 900) + newest_first = query.order_by(lambda post: post.id, "desc") + + posts = await newest_first.all() + assert [post.id for post in posts] == [2, 1] + assert await query.count() == 2 + ``` """ def __init__( self, model_cls: Type[T], using: str | None = None, session: Any | None = None ): - """Initialize a query for a model class. + """Start a query over every row of a model. Args: - model_cls: Model class that defines the target table. - - Examples: - >>> query = Query(User) - >>> query.model_cls is User - True + model_cls: The model class whose table the query reads. + using: The named connection to run on. Defaults to `None`, the + session's connection. + session: The session to run in. Defaults to `None`, the session + opened by the enclosing `async with engines.session():`. """ self.model_cls = model_cls + """The model class whose rows the query returns.""" self._using = using self._session = session self.where_clause: list["QueryNode"] = [] + """The conditions added by `where()`, combined with `AND`.""" self.order_by_clause: list[OrderByEntry] = [] + """The sort keys added by `order_by()`, in the order added.""" self._limit: int | None = None self._offset: int | None = None self._after: tuple[Any, ...] | None = None @@ -698,39 +717,36 @@ def _m2m( return new def where(self, predicate: "Predicate[T]") -> Self: - """Add a filter condition to the query. - - ``predicate`` is a lambda of shape ``Callable[[QueryProxy[T]], QueryNode]``. - The lambda receives a fresh :class:`QueryProxy` whose attributes - return :class:`FieldProxy` instances, so - ``lambda user: user.archived == False`` builds a comparison. Column - names are validated at build time against the model's declared - fields (plus shadow ``{fk}_id`` columns): a misspelled column raises - ``AttributeError`` naming the closest valid match, before any query - is sent to the database. - - Attribute access on a declared forward-FK field traverses the relation - (``lambda t: t.account.ledger_id == lid``): each hop resolves against - the related model, and every distinct traversed path renders one INNER - join (ADR-0006) when the query runs. Every hop is validated at build - time with the same did-you-mean naming the hop's model. + """Add a condition that every returned row must meet. + + The lambda receives a stand-in for one row, a + [`QueryProxy`][ferro.query.QueryProxy]; comparing its fields builds + the SQL condition. Conditions from several `where()` calls are + combined with `AND`. A field name the model does not declare raises + `AttributeError` with the closest match, before any SQL runs. + + A field that is a foreign key relation reaches into the related + model (`lambda post: post.author.name == "Ada Lovelace"`), and each + relation path adds one `INNER JOIN`. See + [Querying across relationships](../guide/queries.md#querying-across-relationships). Args: - predicate: A callable that takes a :class:`QueryProxy` and - returns a :class:`QueryNode`. + predicate: A lambda that takes the row stand-in and returns a + condition, such as `lambda post: post.views > 1000`. Returns: - A new ``Query`` with the clause added; ``self`` is unchanged. + A new query with the condition added. Raises: - TypeError: If ``predicate`` is not callable, or if it does not - return a ``QueryNode``. + TypeError: `predicate` is not callable or does not return a + condition. + AttributeError: The lambda names a field the model does not have. Examples: - >>> q1 = User.where(lambda user: user.archived == False) # noqa: E712 - >>> q2 = User.where(lambda user: user.id == 1) - >>> isinstance(q1, Query) and isinstance(q2, Query) - True + ```python {test=run prelude=reference} + posts = await Post.where(lambda post: post.author.name == "Grace Hopper").all() + assert [post.title for post in posts] == ["The Education of a Computer"] + ``` """ new = self._clone() node = _resolve_where_node(predicate, self.model_cls) @@ -748,61 +764,51 @@ def select(self, selector: "RowSelector[T]") -> "ProjectedQuery[T]": ... def select(self, *columns: str) -> "ProjectedQuery[T]": ... def select(self, *selectors: "RowSelector[T] | str") -> "Self | ProjectedQuery[T]": - """Project the query to a column subset, or pass through unchanged. - - Bare ``select()`` keeps meaning "full query": it returns an equivalent - query of complete model instances. With a lambda selector — the - documented style — the query becomes a projection: its results are - :class:`Row` records in the list-like :class:`Rows` container, never - model instances (the complete-instance invariant, ADR-0007). The - selector returns one field (``select(lambda t: t.amount)``), a tuple - (``select(lambda t: (t.id, t.amount))``), or a dict whose keys name - the output fields (``select(lambda t: {"account_name": - t.account.name})`` — output aliases). Fields may traverse declared - forward-FK relations at any depth (``t.account.name`` — traversed - projection); traversal narrows exactly like a ``where()`` predicate - on the same path (INNER, one join per path, ADR-0006) and - ``left_join()`` is the opt-out — kept rows decode the traversed - fields as ``None``. Unaliased traversed fields take the bare leaf - column name; two fields resolving to the same output name raise at - build time. Column-name strings (``select("id", "amount")``) follow - ``order_by``'s string contract exactly: root columns only, never - traversal, never mixed with a lambda in one call. All forms validate - at build time with did-you-mean. - - A projected query composes like any other query: ``where()`` - (relation traversal included), ``order_by()`` (by unselected columns - too), ``limit()``/``offset()``, ``first()``, ``count()``, and - ``exists()`` all work; ``update()``/``delete()`` and a second - ``select()`` raise at build time. + """Return only some columns, as records, or the query unchanged. + + With no argument, `select()` returns the same query of whole model + instances. With a lambda or column names, the query returns + [`Row`][ferro.query.Row] records in a [`Rows`][ferro.query.Rows] list, + never model instances. The lambda returns one field, a tuple of + fields, or a dict whose keys name the output fields; a field may + reach into a related model (`post.author.name`). Column names as + strings name the model's own columns only. See + [Selecting a column subset](../guide/queries.md#selecting-a-column-subset). Args: - *selectors: Nothing (full query), one lambda selector, or - column-name strings. + *selectors: Nothing; one lambda returning a field, a tuple of + fields or a dict naming them; or column names as strings. Returns: - A new query; ``self`` is unchanged. Projected when a selector is - given. + The query unchanged with no selector, otherwise a + [`ProjectedQuery`][ferro.query.builder.ProjectedQuery]. Raises: - TypeError: If the selector is not callable/strings, selects - non-fields (a bare relation, a comparison), nests shapes - (a dict inside a tuple, a tuple inside a dict), uses a - non-string dict key, or mixes strings with a lambda in one - call. - ValueError: If the selection is empty, two fields resolve to the - same output name, a string is a dotted path (strings never - traverse), or the query already carries an ``include()`` - (one materialization plan per query — record results are - flat, permanently). + AttributeError: A selector names a column the model does not have. + TypeError: The selectors mix a lambda with column names, select a + relation or a comparison instead of a field, or nest a dict + and a tuple. + ValueError: The selection is empty, two fields share an output + name, a string names a related column, or the query already + has an `include()`. Examples: - >>> rows = await Transaction.select(lambda t: (t.id, t.amount)).all() # doctest: +SKIP - >>> rows[0].amount # doctest: +SKIP - Decimal('12.50') - >>> named = await Transaction.select( # doctest: +SKIP - ... lambda t: {"id": t.id, "account_name": t.account.name} - ... ).all() + A tuple keeps each field's own name: + + ```python {test=run prelude=reference} + rows = await Post.select(lambda post: (post.title, post.views)).order_by(lambda post: post.id).all() + assert rows[0].title == "Notes on the Analytical Engine" + assert rows[0].views == 1843 + ``` + + A dict names the output fields, and may reach into a related model: + + ```python {test=run prelude=reference} + rows = await Post.select( + lambda post: {"post": post.title, "author": post.author.name} + ).order_by(lambda post: post.id).all() + assert rows[1].author == "Grace Hopper" + ``` """ if not selectors: return self._clone() @@ -857,43 +863,36 @@ def order_by( *, nulls: str | None = None, ) -> Self: - """Add an ordering clause and return a new query. - - Accepts a lambda naming the column (``order_by(lambda u: u.created_at, - "desc")`` — the documented style, matching ``where`` predicates) or a - column-name string (``order_by("created_at", "desc")``). Both forms are - validated against the model's queryable columns at build time. - - A lambda selector may traverse a declared forward-FK relation - (``order_by(lambda t: t.account.name)``) exactly like ``where()``: each - hop resolves against the related model, and the traversed path renders - one INNER join (ADR-0006), shared with any ``where()`` traversal of the - same path in the same query — the same path referenced in both yields - exactly one join. String selectors do not traverse: ``"account.name"`` - is looked up as a literal (unqualified) column name on the queried - model. + """Sort the results by a column, after any sort keys added before. + + Name the column with a lambda, the documented form, or a string. Only + the lambda can sort by a related model's column + (`lambda post: post.author.name`), which joins that model the way + `where()` does; a string always names a column of the model itself. + See [Ordering, limit & offset](../guide/queries.md#ordering-limit-offset). Args: - field: Column selector — lambda receiving a :class:`QueryProxy`, - or a column-name string. - direction: ``"asc"`` (default) or ``"desc"``. - nulls: Optional ``"first"``, ``"last"``, or ``"native"`` null - placement (keyword-only). Omitted means ``"last"``; ``"native"`` - keeps each backend's dialect default. + field: A lambda returning the column, such as + `lambda post: post.views`, or the column's name. + direction: `"asc"` (the default) or `"desc"`. + nulls: Where `NULL` values sort: `"first"`, `"last"` or + `"native"`, the database's own rule. Defaults to `None`, + which sorts them last on every database. Returns: - A new ``Query`` with the ordering added; ``self`` is unchanged. + A new query with the sort key added. Raises: - AttributeError: If the column is not a queryable column. - TypeError: If a lambda selector returns something other than a - single column reference (a bare relation, e.g. - ``lambda t: t.account``, is meaningless as a sort key). - ValueError: If ``direction`` is not ``"asc"`` or ``"desc"``, or - ``nulls`` is not ``"first"``, ``"last"``, or ``"native"``. + AttributeError: The column is not one the model has. + TypeError: The lambda returns a relation or anything else that is + not a column. + ValueError: `direction` or `nulls` has a value not listed above. Examples: - >>> newest = await Post.select().order_by(lambda p: p.created_at, "desc").all() + ```python {test=run prelude=reference} + posts = await Post.select().order_by(lambda post: post.views, "desc").all() + assert [post.views for post in posts] == [1843, 952] + ``` """ if direction.lower() not in ("asc", "desc"): raise ValueError("direction must be 'asc' or 'desc'") @@ -944,110 +943,96 @@ def order_by( return new def join(self, selector: "Callable[[QueryProxy[T]], Any]") -> Self: - """Force an INNER join on a relation path and return a new query. - - ``selector`` is a lambda naming a RELATION path - (``lambda t: t.account``, ``lambda t: t.account.owner``) — the same - traversal syntax as ``where()``, but resolving to the relation itself, - not a column on it. A bare ``.join(lambda t: t.account)`` with no - predicate is a meaningful **existence filter** on a nullable relation: - it narrows the result to rows where the relation exists (ADR-0006). + """Join a related model with `INNER JOIN`, keeping only rows that have it. - Every edge of the path is marked explicit-INNER; combining it with an - explicit ``.left_join`` on the same edge is a build-time error (see - :meth:`left_join`). + The lambda names a foreign key relation, not a column on it. On a + nullable relation, a `join()` with no condition drops the rows whose + relation is empty. A `where()` or `order_by()` that reaches into the + same relation shares the join. See + [Join type for traversed relations](../guide/queries.md#join-type-for-traversed-relations). Args: - selector: A lambda receiving a :class:`QueryProxy` and returning a - relation path (a ``RelationProxy``). + selector: A lambda returning a relation, such as + `lambda post: post.author`, or a longer path through several. Returns: - A new ``Query`` with the join registered; ``self`` is unchanged. + A new query with the join added. Raises: - TypeError: If ``selector`` is not callable or does not resolve to a - relation path (a column selector is rejected — join names a - relation, not a column). - ValueError: If an edge of the path is already marked explicit-LEFT. + TypeError: `selector` is not callable, or names a column or a + reverse relation instead of a foreign key relation. + ValueError: A [`left_join`][ferro.query.builder.Query.left_join] + already marked part of the path. Examples: - >>> with_account = QJTransaction.select().join(lambda t: t.account) + ```python {test=run prelude=reference} + query = Post.select().join(lambda post: post.author) + assert await query.count() == 2 + ``` """ return self._add_explicit_join(selector, "inner") def left_join(self, selector: "Callable[[QueryProxy[T]], Any]") -> Self: - """Mark a relation path LEFT (whole-path) and return a new query. + """Join a related model with `LEFT JOIN`, keeping rows that lack it. - ``selector`` names a RELATION path exactly like :meth:`join`. Every edge - of the path is marked LEFT (the **whole-path rule**, ADR-0006), so a - left-marked 2-hop path retains rows missing the relation at either hop. - Relation-less rows are retained (NULL retention), observable both in - ordered results and in traversal predicates on related columns (e.g. - ``where(lambda t: t.account.name == None)``). - - Conflict rules: an explicit LEFT beats implicit ``where()``/``order_by()`` - traversal on a shared edge (the path renders LEFT); an explicit ``.join`` - plus an explicit ``.left_join`` on the same edge is a build-time error. + Every relation along the path is joined with `LEFT JOIN`, so a row + missing the relation at any step is still returned, and a field read + through the relation is `None`. The `LEFT JOIN` also applies to a + `where()`, `order_by()` or `select()` that reaches into the same + path. See + [Keeping rows that have no relation](../guide/queries.md#keeping-rows-that-have-no-relation). Args: - selector: A lambda receiving a :class:`QueryProxy` and returning a - relation path (a ``RelationProxy``). + selector: A lambda returning a relation, such as + `lambda post: post.author`, or a longer path through several. Returns: - A new ``Query`` with the LEFT join registered; ``self`` is unchanged. + A new query with the join added. Raises: - TypeError: If ``selector`` is not callable or does not resolve to a - relation path. - ValueError: If an edge of the path is already marked explicit-INNER. + TypeError: `selector` is not callable, or names a column or a + reverse relation instead of a foreign key relation. + ValueError: A [`join`][ferro.query.builder.Query.join] already + marked part of the path. Examples: - >>> keep_orphans = QJNote.select().left_join(lambda n: n.account) + ```python {test=run prelude=reference} + rows = await ( + Post.select(lambda post: {"post": post.title, "author": post.author.name}) + .left_join(lambda post: post.author) + .order_by(lambda post: post.id) + .all() + ) + assert [row.author for row in rows] == ["Ada Lovelace", "Grace Hopper"] + ``` """ return self._add_explicit_join(selector, "left") def include(self, selector: "Callable[[QueryProxy[T]], Any]") -> Self: - """Populate a relation path on every result and return a new query. - - ``selector`` is a lambda naming a forward-FK RELATION path - (``lambda t: t.account``, ``lambda t: t.account.owner``) — the same - traversal syntax as :meth:`join`. Results are the same ``list[T]`` - the query always returned, in one SQL statement, with each included - relation **populated**: access is a plain attribute - (``txn.account.label`` — no await, no query) returning the complete - related instance, exactly as the field's declared annotation claims - (ADR-0008). Unpopulated relations keep the awaitable contract - unchanged; a nullable FK with no target populates as ``None`` with - the root row retained. - - Include decides attached data only — joins decide membership, - projection decides shape. Adding ``.include(...)`` to any query - returns exactly the rows that query returned without it: include - contributes no join-type opinion on any edge a predicate or explicit - chainer references, and renders LEFT only on edges nothing else - touches. ``count()``/``exists()`` are unaffected. - - Cumulative, order-free, and idempotent: chained includes accumulate, - shared prefixes dedup by path identity, and including a path - populates every hop along it. + """Load a related model with each result, in the same query. + + Each returned instance has the relation filled in: reading it is a + plain attribute that holds the related instance, with no `await` + and no second query. `include()` never changes which rows come back + or how many. A path through several relations fills in each one. See + [Populating relations with include()](../guide/queries.md#populating-relations-with-include). Args: - selector: A lambda receiving a :class:`QueryProxy` and returning - a relation path (a ``RelationProxy``). + selector: A lambda returning a foreign key relation, such as + `lambda post: post.author`. Returns: - A new ``Query`` with the population registered; ``self`` is - unchanged. + A new query that loads the relation. Raises: - TypeError: If ``selector`` is a string or not callable, resolves - to a column rather than a relation, or names a BackRef/M2M - relation (reverse population is a separate future mechanism). + TypeError: `selector` is a string or not callable, or names a + column or a reverse or many-to-many relation. Examples: - >>> txns = await Transaction.select().include(lambda t: t.account).all() # doctest: +SKIP - >>> txns[0].account.label # doctest: +SKIP - 'checking' + ```python {test=run prelude=reference} + posts = await Post.select().include(lambda post: post.author).order_by(lambda post: post.id).all() + assert posts[0].author.name == "Ada Lovelace" + ``` """ path = _resolve_include_selector(selector, self.model_cls) new = self._clone() @@ -1057,7 +1042,7 @@ def include(self, selector: "Callable[[QueryProxy[T]], Any]") -> Self: def _add_explicit_join( self, selector: "Callable[[QueryProxy[T]], Any]", join_type: str ) -> Self: - """Shared body of :meth:`join`/:meth:`left_join` (#272). + """Shared body of `join`/`left_join` (#272). Resolves the selector to a relation path, checks every edge for a contradictory explicit mark (INNER vs LEFT), then records the marks and @@ -1083,40 +1068,46 @@ def _add_explicit_join( return new def limit(self, value: int) -> Self: - """Limit the number of records returned + """Return at most `value` rows. Args: - value: The maximum number of records to return. + value: The largest number of rows to return. Returns: - A new ``Query`` with the clause added; ``self`` is unchanged. + A new query with the limit set. Examples: - >>> query = User.select().limit(10) - >>> query._limit - 10 + ```python {test=run prelude=reference} + posts = await Post.select().order_by(lambda post: post.id).limit(1).all() + assert [post.id for post in posts] == [1] + ``` """ new = self._clone() new._limit = value return new def offset(self, value: int) -> Self: - """Skip a specific number of records + """Skip the first `value` rows. + + For pages deep into a large table, [`after`][ferro.query.builder.Query.after] + stays fast where an offset grows slower with every skipped row. See + [Pagination](../howto/pagination.md). Args: - value: The number of records to skip. + value: The number of rows to skip. Returns: - A new ``Query`` with the clause added; ``self`` is unchanged. + A new query with the offset set. Raises: - ValueError: If ``after()`` or ``before()`` is already set — a query - has one start. + ValueError: `after()` or `before()` is already set; a query has + one starting point. Examples: - >>> query = User.select().offset(20) - >>> query._offset - 20 + ```python {test=run prelude=reference} + posts = await Post.select().order_by(lambda post: post.id).limit(10).offset(1).all() + assert [post.id for post in posts] == [2] + ``` """ if self._after is not None or self._before is not None: raise ValueError( @@ -1166,19 +1157,31 @@ def _assert_position_order_keys(self) -> None: ) def position_of(self, row: T) -> tuple[Any, ...]: - """Read this query's order-key tuple off a model instance. + """Read an instance's values for this query's sort keys. + + The tuple is the instance's place in the query's order, the cursor + that [`after`][ferro.query.builder.Query.after] and + [`before`][ferro.query.builder.Query.before] take. Store it to resume + paging later, for example in an API's next-page token. Args: - row: A hydrated instance of the queried model. + row: An instance of the queried model. Returns: - The ordered tuple of order-key values, matching ``order_by`` - declaration order. + One value per sort key, in the order the keys were added. Raises: - TypeError: If ``row`` is not an instance of the queried model. - ValueError: If the order keys are not a legal position-paging - set, or a traversed order key's relation is unpopulated. + TypeError: `row` is not an instance of the queried model. + ValueError: The sort keys do not include the primary key, or a + key reaches into a relation the instance does not have loaded + with `include()`. + + Examples: + ```python {test=run prelude=reference} + by_views = Post.select().order_by(lambda post: post.views, "desc").order_by(lambda post: post.id) + top = await by_views.first() + assert by_views.position_of(top) == (1843, 1) + ``` """ if not isinstance(row, self.model_cls): raise TypeError( @@ -1232,24 +1235,40 @@ def _resolve_position( return bound def after(self, position: tuple[Any, ...] | T) -> Self: - """Start the page after an exclusive position in the declared order. + """Start the results just after a position in the query's order. - ``position`` is the ordered tuple of this query's order-key values, or - a model instance (sugar for ``after(position_of(row))``). Order keys - may be root or traversed columns and must include the primary key. - ``None`` is legal in every non-PK slot; the PK slot cannot be empty. + This is keyset paging: pass the last row of one page to get the + next. The position is excluded, and rows inserted or deleted + elsewhere do not shift the page. The sort keys must include the + primary key, so that no two rows share a position. See + [Keyset (cursor) pagination](../howto/pagination.md#keyset-cursor-pagination). Args: - position: A tuple of order-key values, or a model instance. + position: An instance of the queried model, or the tuple of its + sort-key values that + [`position_of`][ferro.query.builder.Query.position_of] + returns. Any value but the primary key's may be `None`. Returns: - A new ``Query`` with the bound set; ``self`` is unchanged. + A new query that starts after `position`. Raises: - TypeError: If ``position`` is neither a tuple nor a model instance. - ValueError: If the order keys are illegal for ``after()``, the - tuple has the wrong arity, the PK slot is ``None``, - ``offset()`` is already set, or ``before()`` is already set. + TypeError: `position` is neither a tuple nor an instance of the + queried model. + ValueError: The sort keys do not include the primary key, the + tuple has the wrong length, its primary key value is `None`, + or `offset()` or `before()` is already set. + + Examples: + ```python {test=run prelude=reference} + by_views = Post.select().order_by(lambda post: post.views, "desc").order_by(lambda post: post.id).limit(1) + + page = await by_views.all() + assert [post.views for post in page] == [1843] + + next_page = await by_views.after(page[-1]).all() + assert [post.views for post in next_page] == [952] + ``` """ bound = self._resolve_position(position, op="after") new = self._clone() @@ -1257,29 +1276,41 @@ def after(self, position: tuple[Any, ...] | T) -> Self: return new def before(self, position: tuple[Any, ...] | T) -> Self: - """Start at the rows immediately before an exclusive position. - - With a limit this is the adjacent previous page, yielded in the - declared order. Without a limit it is every earlier row in declared - order (a prefix, not a page). On unbounded ``before()``, ``first()`` - is the adjacent previous row and ``all()[0]`` is the head of the - prefix — they disagree, and that is accepted. + """End the results just before a position in the query's order. - ``before(row)`` is sugar for ``before(position_of(row))``. Same - order-key rules as ``after()`` (root or traversed columns, PK - required). Cannot be combined with ``after()`` or ``offset()``. + With a limit, this is the previous page, still in the query's order. + Without one, it is every row before the position. The position is + excluded, and the sort keys must include the primary key, as for + [`after`][ferro.query.builder.Query.after]. Args: - position: A tuple of order-key values, or a model instance. + position: An instance of the queried model, or the tuple of its + sort-key values that + [`position_of`][ferro.query.builder.Query.position_of] + returns. Any value but the primary key's may be `None`. Returns: - A new ``Query`` with the bound set; ``self`` is unchanged. + A new query that ends before `position`. Raises: - TypeError: If ``position`` is neither a tuple nor a model instance. - ValueError: If the order keys are illegal, the tuple has the - wrong arity, the PK slot is ``None``, ``offset()`` is - already set, or ``after()`` is already set. + TypeError: `position` is neither a tuple nor an instance of the + queried model. + ValueError: The sort keys do not include the primary key, the + tuple has the wrong length, its primary key value is `None`, + or `offset()` or `after()` is already set. + + Note: + Without a limit, [`first`][ferro.query.builder.Query.first] + returns the row just before the position, while `all()[0]` is + the first row of the query. + + Examples: + ```python {test=run prelude=reference} + by_views = Post.select().order_by(lambda post: post.views, "desc").order_by(lambda post: post.id).limit(1) + + previous_page = await by_views.before((952, 2)).all() + assert [post.views for post in previous_page] == [1843] + ``` """ bound = self._resolve_position(position, op="before") new = self._clone() @@ -1288,15 +1319,16 @@ def before(self, position: tuple[Any, ...] | T) -> Self: @terminal async def all(self: "Query[T]") -> list[T]: - """Return all model instances that match the current query + """Run the query and return every matching record. Returns: - A list of model instances. + The matching instances, in the query's order. Examples: - >>> users = await User.where(lambda user: user.active == True).all() # noqa: E712 - >>> isinstance(users, list) - True + ```python {test=run prelude=reference} + posts = await Post.where(lambda post: post.views > 1000).all() + assert [post.title for post in posts] == ["Notes on the Analytical Engine"] + ``` """ compiled = compile_query(self, "fetch") route = await self._transaction_or_using() @@ -1313,15 +1345,18 @@ async def all(self: "Query[T]") -> list[T]: @terminal async def count(self: "Query[T]") -> int: - """Return the number of records that match the current query + """Count the matching rows. + + Limits and offsets are ignored: the count covers every row the + conditions match. Returns: - The count of matching records. + The number of matching rows. Examples: - >>> total = await User.where(lambda user: user.active == True).count() # noqa: E712 - >>> isinstance(total, int) - True + ```python {test=run prelude=reference} + assert await Post.where(lambda post: post.views > 900).count() == 2 + ``` """ return await self._count("count") @@ -1344,46 +1379,48 @@ async def update( /, **fields: Any, ) -> int: - """Update all records matching the current query. + """Update every matching row in one statement. - Two doors, one SET wire: keyword literals (``update(name="x")``) or a - recipe callable (``update(lambda user: {"email": "x", "bonus": - user.score, "n": user.n + 1, "updated_at": now})``). Recipe values - are a literal, a root column copy, ``+`` / ``-`` arithmetic, the - imported ``now`` singleton, or ``.merge()`` (Postgres-only shallow - object merge). ``+`` does not fill empties — ``NULL + 1`` is NULL; - ``.merge()`` treats a NULL object column as ``{}``. The doors - cannot be mixed. + Pass the new values as keywords, or pass one lambda, a recipe, that + returns a dict of new values computed in the database. A recipe + value is a literal, another column of the row, `+` or `-` arithmetic + on columns, [`now`][ferro.now] for the database's clock, or + `.merge()` on a JSON column (Postgres only). `NULL + 1` is `NULL`. + The two forms cannot be mixed. See + [Expression updates](../guide/mutations.md#expression-updates). Args: - recipe: Optional positional recipe callable receiving a - :class:`QueryProxy` and returning a non-empty ``dict`` of - root-column assignments. Positional-only, so a keyword - ``recipe=`` still SETs a column of that name. - **fields: Field names and literal values to update. + recipe: An optional recipe: a lambda that takes the row stand-in + and returns a dict of column names to new values. It is + positional only, so `recipe=` as a keyword still names a + column. + **fields: Column names and the literal values to write. Returns: - The number of records updated. + The number of rows updated. Raises: - TypeError: If both doors are used, a recipe is not callable or + TypeError: Both forms are given, the recipe is not callable or does not return a dict, or a keyword value is not a literal. - ValueError: If the recipe is empty, ``limit()`` or ``offset()`` - was set, or the query traverses a relation — multi-table - mutation has no portable SQL. + ValueError: The recipe assigns nothing, the query has a + `limit()`, `offset()`, `after()` or `before()`, or it reaches + into a related model. Examples: - >>> updated = await User.where(lambda user: user.id == 1).update(name="Taylor") - >>> isinstance(updated, int) - True - >>> from ferro import now - >>> n = await Counter.where(lambda counter: counter.id == cid).update( - ... lambda counter: { - ... "n": counter.n + 1, - ... "total": counter.a + counter.b, - ... "updated_at": now, - ... } - ... ) + Keywords write literal values: + + ```python {test=run prelude=reference} + updated = await Post.where(lambda post: post.id == 2).update(published=False) + assert updated == 1 + ``` + + A recipe computes each row's new value from its current one: + + ```python {test=run prelude=reference} + await Post.where(lambda post: post.views > 900).update(lambda post: {"views": post.views + 1}) + posts = await Post.select().order_by(lambda post: post.id).all() + assert [post.views for post in posts] == [1844, 953] + ``` """ if recipe is not None and fields: raise TypeError( @@ -1406,38 +1443,38 @@ async def update( @terminal async def first(self: "Query[T]") -> T | None: - """Return the first matching record, or None + """Run the query and return its first record, or `None` when nothing matches. Returns: - A model instance or None. + The first matching instance in the query's order, or `None`. Examples: - >>> user = await User.select().order_by("id").first() - >>> user is None or isinstance(user, User) - True + ```python {test=run prelude=reference} + post = await Post.select().order_by(lambda post: post.views).first() + assert post.title == "The Education of a Computer" + ``` """ results = await self.limit(1).all() return results[0] if results else None @terminal async def delete(self: "Query[T]") -> int: - """Delete all records matching the current query + """Delete every matching row in one statement. Returns: - The number of records deleted. + The number of rows deleted. Raises: - ValueError: If ``limit()`` or ``offset()`` was set on this query, or - if the query traverses a relation (a ``where()`` predicate on a - related column, or an explicit ``join()``/``left_join()``) — - multi-table mutation has no portable SQL. Filter by a column on - the target model, or resolve the related primary keys first and - delete by primary-key set. + ValueError: The query has a `limit()`, `offset()`, `after()` or + `before()`, or reaches into a related model through + `where()`, `join()` or `left_join()`. Filter on the model's own columns instead, + such as the foreign key column `author_id`. Examples: - >>> deleted = await User.where(lambda user: user.disabled == True).delete() # noqa: E712 - >>> isinstance(deleted, int) - True + ```python {test=run prelude=reference} + deleted = await Post.where(lambda post: post.views < 1000).delete() + assert deleted == 1 + ``` """ compiled = compile_query(self, "delete") route = await self._transaction_or_using() @@ -1450,32 +1487,31 @@ async def delete(self: "Query[T]") -> int: @terminal async def exists(self: "Query[T]") -> bool: - """Return whether at least one record matches the current query + """Return whether any row matches. Returns: - True if records exist, otherwise False. + `True` when at least one row matches, otherwise `False`. Examples: - >>> found = await User.where(lambda user: user.email == "a@b.com").exists() - >>> isinstance(found, bool) - True + ```python {test=run prelude=reference} + assert await Author.where(lambda author: author.email == "ada@example.com").exists() + ``` """ return await self._count("exists") > 0 async def add(self, *instances: Any) -> None: - """Add links to a many-to-many relationship + """Link instances to the owner of a many-to-many relation. + + Call it on the relation, as in `await post.tags.add(history, science)`. + See [Many-to-many](../guide/relationships.md#many-to-many). Args: - *instances: Target model instances that provide an ``id`` attribute. + *instances: The related instances to link. Each is identified by + its `id` field, so the related model's primary key must be + named `id`. Raises: - RuntimeError: If the query is not bound to a many-to-many context. - - Examples: - >>> user = await User.create(email="taylor@example.com") - >>> admin = await Group.create(name="admin") - >>> staff = await Group.create(name="staff") - >>> await user.groups.add(admin, staff) + RuntimeError: The query is not a many-to-many relation. """ if not self._m2m_context: raise RuntimeError( @@ -1498,18 +1534,18 @@ async def add(self, *instances: Any) -> None: ) async def remove(self, *instances: Any) -> None: - """Remove links from a many-to-many relationship + """Unlink instances from the owner of a many-to-many relation. + + Call it on the relation, as in `await post.tags.remove(history)`. The + instances' rows stay; only the links go. Args: - *instances: Target model instances that provide an ``id`` attribute. + *instances: The related instances to unlink. Each is identified + by its `id` field, so the related model's primary key must be + named `id`. Raises: - RuntimeError: If the query is not bound to a many-to-many context. - - Examples: - >>> user = await User.create(email="taylor@example.com") - >>> admin = await Group.create(name="admin") - >>> await user.groups.remove(admin) + RuntimeError: The query is not a many-to-many relation. """ if not self._m2m_context: raise RuntimeError( @@ -1531,14 +1567,13 @@ async def remove(self, *instances: Any) -> None: ) async def clear(self) -> None: - """Clear all links in a many-to-many relationship + """Unlink every instance from the owner of a many-to-many relation. - Raises: - RuntimeError: If the query is not bound to a many-to-many context. + Call it on the relation, as in `await post.tags.clear()`. The related + rows stay; only the links go. - Examples: - >>> user = await User.create(email="taylor@example.com") - >>> await user.groups.clear() + Raises: + RuntimeError: The query is not a many-to-many relation. """ if not self._m2m_context: raise RuntimeError( @@ -1554,36 +1589,51 @@ async def clear(self) -> None: ) def __repr__(self): - """Return a developer-friendly representation of the query""" + """Show the model and the conditions, for debugging.""" return f"<Query model={self.model_cls.__name__} where={self.where_clause}>" class ProjectedQuery(Query[T]): - """A query projected to a column subset: results are records, not models. - - Created by ``select()`` with a selector; carries a ``record`` - materialization plan (ADR-0007), so ``all()`` delivers :class:`Row` - records in the list-like :class:`Rows` container and ``first()`` a - ``Row | None`` — never model instances (the complete-instance invariant). - Projected records bypass the identity map and carry no persistence - identity. - - Filtering, ordering, and pagination compose exactly like on - :class:`Query`; ``count()``/``exists()`` are unaffected by the - projection. + """A query that returns some columns as records, not model instances. + + [`select`][ferro.query.builder.Query.select] with a lambda or column names + returns one. Awaiting [`all`][ferro.query.builder.ProjectedQuery.all] + gives [`Row`][ferro.query.Row] records in a [`Rows`][ferro.query.Rows] + list. Records are plain values: they are not tracked by the session and + cannot be saved. Filtering, sorting and paging work as on any query. + + A dict selector whose values include an aggregate such as + `post.views.sum()` groups the rows by the plain fields beside it. See + [Aggregations and grouped queries](../guide/aggregations.md). + + Examples: + ```python {test=run prelude=reference} + rows = await Post.select( + lambda post: {"author": post.author.name, "total_views": post.views.sum()} + ).order_by("author").all() + assert [(row.author, row.total_views) for row in rows] == [ + ("Ada Lovelace", 1843), + ("Grace Hopper", 952), + ] + ``` """ def __init__(self, source: Query[T], fields: tuple[_ProjectedField, ...]) -> None: - """Project ``source`` to ``fields`` (selection order preserved). - - Copies the source query's state through ``_clone()`` (fresh mutable - containers, FF-F F-1); ``source`` is unchanged. Every traversed - field's relation path registers a join (#293): projection traversal - narrows exactly like ``where()``/``order_by()`` traversal, sharing - join identity per relation path (ADR-0006) — ``setdefault`` keeps an - already-registered path's entry, and an explicit ``left_join()`` - still wins at serialization via the edge marks. + """Project a query to the given fields. + + Args: + source: The query to project. It is left unchanged. + fields: The fields to return, in order. + + Raises: + ValueError: A sort key added before the projection is not one of + its group keys. """ + # Copies the source's state through ``_clone()`` (fresh mutable + # containers). Every traversed field's relation path registers a join: + # projection traversal narrows like where()/order_by() traversal, + # sharing one join per relation path, and an explicit left_join() + # still wins through the edge marks. self.__dict__.update(source._clone().__dict__) # Immutable, so chained clones share it safely. self._projection: tuple[_ProjectedField, ...] = fields @@ -1645,41 +1695,45 @@ def order_by( *, nulls: str | None = None, ) -> Self: - """Add an ordering clause to a projected query (#295's three rules). + """Sort the records by an output field, a column or an aggregate. - Strings resolve OUTPUT field names first, then root columns — SQL's - own ORDER BY scoping (``order_by("total")`` sorts by the projected - aggregate, even if a root column shares the name). The lambda form - spells SOURCE expressions: a column (traversal included), or an - aggregate (``order_by(lambda t: t.amount.sum(), "desc")``) which must - match a projected aggregate field and sorts by its output name. + A string names an output field first and a column of the model + second, the way SQL's `ORDER BY` reads names. A lambda names a + column, which may reach into a related model, or an aggregate that + the projection returns, such as `lambda post: post.views.sum()`. - On an AGGREGATE projection every sort key must be a group key or an - aggregate — anything else raises at build time (each group would - answer with an arbitrary row's value; SQLite would even permit it). - On a plain projection unselected root columns stay sortable, - unchanged. + When the projection has an aggregate, every sort key must be a + group key or an aggregate: any other column has no single value per + group. See + [Ordering grouped results](../guide/aggregations.md#ordering-grouped-results). Args: - field: Output field name or root column-name string, or a lambda - naming a source column / aggregate expression. - direction: ``"asc"`` (default) or ``"desc"``. - nulls: Optional ``"first"``, ``"last"``, or ``"native"`` null - placement (keyword-only). Omitted means ``"last"``; ``"native"`` - keeps each backend's dialect default. + field: An output field or column name, or a lambda returning a + column or an aggregate. + direction: `"asc"` (the default) or `"desc"`. + nulls: Where `NULL` values sort: `"first"`, `"last"` or + `"native"`, the database's own rule. Defaults to `None`, + which sorts them last on every database. Returns: - A new query with the ordering added; ``self`` is unchanged. + A new query with the sort key added. Raises: - ValueError: If ``direction`` is invalid, ``nulls`` is invalid, or - the sort key is not a group key or aggregate on an aggregate - projection (including an aggregate expression matching no - projected field). - AttributeError: If a string names neither an output field nor a - queryable root column. - TypeError: If a lambda resolves to a bare relation or any other - non-sortable value. + ValueError: `direction` or `nulls` has a value not listed above, + or on an aggregate projection the key is neither a group key + nor an aggregate the projection returns. + AttributeError: A string names neither an output field nor a + column of the model. + TypeError: The lambda returns a relation or anything else that is + not a column or an aggregate. + + Examples: + ```python {test=run prelude=reference} + rows = await Post.select( + lambda post: {"author": post.author.name, "total_views": post.views.sum()} + ).order_by(lambda post: post.views.sum(), "desc").all() + assert [row.author for row in rows] == ["Ada Lovelace", "Grace Hopper"] + ``` """ if direction.lower() not in ("asc", "desc"): raise ValueError("direction must be 'asc' or 'desc'") @@ -1776,16 +1830,22 @@ def _is_group_key_source(self, column: str, path: tuple[str, ...]) -> bool: @terminal def count(self: "ProjectedQuery[T]") -> Awaitable[int]: - """Count matching rows — or raise on an aggregate projection (#295). + """Count the matching rows, whatever columns the projection selects. + + On a projection with an aggregate, "count" could mean rows or + groups, so the call raises before any SQL runs. - On a plain projection ``count()`` stays projection-blind (#279's - verb contract). On an AGGREGATE projection "count" is ambiguous - between rows and groups, so it raises synchronously at the call, - before any coroutine or SQL exists, with both spellings. + Returns: + (int): The number of matching rows, once awaited. Raises: - ValueError: On an aggregate projection — count rows with an - unprojected query, count groups with ``len(await q.all())``. + ValueError: The projection has an aggregate. Count rows with the + query before `select()`, or groups with `len(await query.all())`. + + Examples: + ```python {test=run prelude=reference} + assert await Post.select(lambda post: post.title).count() == 2 + ``` """ if self._has_aggregate(): raise ValueError( @@ -1798,13 +1858,18 @@ def count(self: "ProjectedQuery[T]") -> Awaitable[int]: @terminal def exists(self: "ProjectedQuery[T]") -> Awaitable[bool]: - """Existence check — or raise on an aggregate projection (#295). + """Return whether any row matches, whatever columns the projection selects. + + On a projection with only aggregates there is always exactly one + record, so the call raises before any SQL runs. + + Returns: + (bool): `True` when at least one row matches, once awaited. Raises: - ValueError: On an aggregate projection — a global aggregate - always yields one record, and "a group exists" is - ``len(await q.all()) > 0``; test the unprojected query - instead (``Model.where(...).exists()``). + ValueError: The projection has an aggregate. Test rows with the + query before `select()`, or groups with + `len(await query.all()) > 0`. """ if self._has_aggregate(): raise ValueError( @@ -1817,18 +1882,32 @@ def exists(self: "ProjectedQuery[T]") -> Awaitable[bool]: return super().exists() def position_of(self, row: T | Row) -> tuple[Any, ...]: - """Read this query's order-key tuple off a projected record or instance. + """Read a record's values for this query's sort keys. - Every order key must be in the projection (matched by source column - and path, or by output name). Otherwise pass a tuple to - ``after()``/``before()``. + Every sort key must be among the selected fields, under its own name + or an output name. Otherwise build the tuple yourself for `after()` + or `before()`. + + Args: + row: A record this query returned, or an instance of the queried + model. + + Returns: + One value per sort key, in the order the keys were added. Raises: - TypeError: If ``row`` is neither a :class:`Row` nor a model + TypeError: `row` is neither a [`Row`][ferro.query.Row] nor an instance of the queried model. - ValueError: If an order key is missing from the projection, the - order keys are illegal, or a traversed key on a model - instance is unpopulated. + ValueError: A sort key is not selected, the sort keys do not + include the primary key, or a key reaches into a relation the + instance does not have loaded. + + Examples: + ```python {test=run prelude=reference} + by_views = Post.select(lambda post: (post.id, post.views)).order_by(lambda post: post.views, "desc").order_by(lambda post: post.id) + top = await by_views.first() + assert by_views.position_of(top) == (1843, 1) + ``` """ if isinstance(row, Row): self._assert_position_order_keys() @@ -1862,16 +1941,17 @@ def _read_projected_order_value(self, row: Row, entry: OrderByEntry) -> Any: @terminal async def all(self: "ProjectedQuery[T]") -> Rows[Row]: # type: ignore[override] - """Return the projected records for every matching row. + """Run the query and return a record for every matching row, or group. Returns: - A :class:`Rows` of :class:`Row` records, fields in selection - order, wrapped without validation. + A [`Rows`][ferro.query.Rows] list of [`Row`][ferro.query.Row] + records, each with the selected fields in selection order. Examples: - >>> rows = await Transaction.select(lambda t: (t.id, t.amount)).all() # doctest: +SKIP - >>> [r.amount for r in rows] # doctest: +SKIP - [Decimal('12.50'), Decimal('7.00')] + ```python {test=run prelude=reference} + rows = await Post.select(lambda post: post.views).order_by(lambda post: post.id).all() + assert [row.views for row in rows] == [1843, 952] + ``` """ compiled = compile_query(self, "fetch") route = await self._transaction_or_using() @@ -1889,23 +1969,29 @@ async def all(self: "ProjectedQuery[T]") -> Rows[Row]: # type: ignore[override] @terminal async def first(self: "ProjectedQuery[T]") -> Row | None: # type: ignore[override] - """Return the first matching projected record, or None. + """Run the query and return its first record, or `None` when nothing matches. + + Returns: + The first [`Row`][ferro.query.Row] in the query's order, or `None`. Examples: - >>> row = await Transaction.select(lambda t: t.amount).order_by("id").first() # doctest: +SKIP - >>> row is None or isinstance(row, Row) # doctest: +SKIP - True + ```python {test=run prelude=reference} + row = await Post.select(lambda post: post.title).order_by(lambda post: post.views).first() + assert row.title == "The Education of a Computer" + ``` """ results = await self.limit(1).all() return results[0] if results else None def select(self, *selectors: "RowSelector[T] | str") -> NoReturn: # type: ignore[override] # ty: ignore[invalid-method-override] - """Reject a second projection: it would change the result type - mid-chain (#280). + """Refuse a second `select()`, which would change the result type mid-chain. + + Args: + *selectors: Ignored; the call always raises. Raises: - ValueError: Always — build the projection in one ``select()`` - call, or start a new query from the model. + ValueError: Always. Select every field in one `select()` call, or + start a new query from the model. """ raise ValueError( "select() was already applied to this query; replacing a " @@ -1915,19 +2001,18 @@ def select(self, *selectors: "RowSelector[T] | str") -> NoReturn: # type: ignor ) def include(self: Never, selector: Any) -> NoReturn: # type: ignore[override] - """Reject ``include()`` on a projected query (#287, final at #293). + """Refuse `include()`: a record is flat and cannot hold a related instance. - One materialization plan per query (ADR-0007/ADR-0008), and record - results are flat, permanently (ADR-0009: flattened-as-final) — a - record reaches across a relation with traversed projection, never by - nesting an instance. The ``self: Never`` pin makes this a STATIC - error at the call site (tests/static_fixtures/bad_includes.py), - mirroring the mutation pins. + A type checker reports the call as an error too. To read a related + model's columns, select them through the relation + (`lambda post: {"author": post.author.name}`). + + Args: + selector: Ignored; the call always raises. Raises: - ValueError: Always — reach across the relation with traversed - projection, or populate instances through an unprojected - query. + ValueError: Always. Select the related columns, or call + `include()` on a query without `select()`. """ raise ValueError( "include() cannot be combined with a projection: a query carries " @@ -1951,11 +2036,17 @@ def update( # type: ignore[override] /, **fields: Any, ) -> NoReturn: - """Reject ``update()`` on a projected query (#280). + """Refuse `update()`: a projection only reads. + + A type checker reports the call as an error too. + + Args: + recipe: Ignored; the call always raises. + **fields: Ignored; the call always raises. Raises: - ValueError: Always — mutate through an unprojected query - (``Model.where(...).update(...)``). + ValueError: Always. Update through a query without `select()`, + such as `Model.where(...).update(...)`. """ raise ValueError( "update() is not supported on a projected query: a projection is " @@ -1964,11 +2055,13 @@ def update( # type: ignore[override] ) def delete(self: Never) -> NoReturn: # type: ignore[override] - """Reject ``delete()`` on a projected query (#280). + """Refuse `delete()`: a projection only reads. + + A type checker reports the call as an error too. Raises: - ValueError: Always — mutate through an unprojected query - (``Model.where(...).delete()``). + ValueError: Always. Delete through a query without `select()`, + such as `Model.where(...).delete()`. """ raise ValueError( "delete() is not supported on a projected query: a projection is " @@ -1977,7 +2070,7 @@ def delete(self: Never) -> NoReturn: # type: ignore[override] ) def __repr__(self): - """Return a developer-friendly representation of the projection""" + """Show the model, the selected fields and the conditions, for debugging.""" return ( f"<ProjectedQuery model={self.model_cls.__name__} " f"fields={[field.name for field in self._projection]} " @@ -2021,23 +2114,19 @@ def __get__(self, query: Any, owner: Any) -> Any: ... class Relation(Query[T]): - """Represent lazy collection relationship queries with typing support + """The type of a reverse (`BackRef`) or many-to-many collection on an instance. + + Annotate the field as `Relation[list[Target]]`. On an instance, the field + is a query over the related rows: await `.all()` to load them, or chain + `where()`, `order_by()` and the other query methods first. See + [Relationships](../guide/relationships.md). Examples: - >>> class User(Model): - ... id: Annotated[int, FerroField(primary_key=True)] - ... name: str - ... posts: Relation[list["Post"]] = BackRef() - - >>> class Post(Model): - ... id: Annotated[int, FerroField(primary_key=True)] - ... title: str - ... user: Annotated[User, ForeignKey(related_name="posts")] - - >>> user = await User.get(1) - >>> posts = await user.posts.all() - >>> isinstance(posts, list) - True + ```python {test=run prelude=reference} + ada = await Author.get(1) + posts = await ada.posts.all() + assert [post.title for post in posts] == ["Notes on the Analytical Engine"] + ``` """ # NOTE ON TYPING: @@ -2052,10 +2141,43 @@ class Relation(Query[T]): if TYPE_CHECKING: all: ClassVar["_RelationAll"] # type: ignore[assignment] first: ClassVar["_RelationFirst"] # type: ignore[assignment] + else: + + @terminal + async def all(self): # type: ignore[override] + """Load every related record. + + Returns: + (list): The related instances, in the query's order. + + Examples: + ```python {test=run prelude=reference} + grace = await Author.get(2) + posts = await grace.posts.all() + assert [post.title for post in posts] == ["The Education of a Computer"] + ``` + """ + return await super().all() + + @terminal + async def first(self): # type: ignore[override] + """Load the first related record, or `None` when there is none. + + Returns: + (Any): The first related instance in the query's order, or `None`. + + Examples: + ```python {test=run prelude=reference} + ada = await Author.get(1) + post = await ada.posts.order_by(lambda post: post.id).first() + assert post.title == "Notes on the Analytical Engine" + ``` + """ + return await super().first() @classmethod def __get_pydantic_core_schema__(cls, _source_type, _handler): - """Allow pydantic-core to treat relationships as arbitrary runtime values""" + """Let Pydantic accept any value for a relation field.""" from pydantic_core import core_schema return core_schema.any_schema() diff --git a/src/ferro/query/nodes.py b/src/ferro/query/nodes.py index b80e9181..015b25d3 100644 --- a/src/ferro/query/nodes.py +++ b/src/ferro/query/nodes.py @@ -1,4 +1,9 @@ -"""Define query AST nodes and field proxies for fluent filtering""" +"""The pieces a query lambda builds: row stand-ins, column references and conditions. + +A query lambda receives a [`QueryProxy`][ferro.query.QueryProxy]; reading a +field from it gives a `FieldProxy`, and comparing that builds a `QueryNode` +condition. +""" import difflib from collections.abc import Callable, Mapping @@ -10,7 +15,9 @@ from ..columns import retired_clause TField = TypeVar("TField") +"""The Python type of the column a `FieldProxy` refers to.""" TModel = TypeVar("TModel") +"""The model class a `QueryProxy` stands in for.""" # Aggregate source families (#294, ADR-0009): which column families each # aggregate accepts, validated at build time. `None` = any column. Families @@ -59,24 +66,26 @@ def _json_column_shape(spec: Any) -> str | None: class QueryNode: - """Represent a node in the query expression tree - - Attributes: - column: Column name for leaf nodes. - operator: Comparison or logical operator. - value: Right-hand value for leaf comparisons. - left: Left child node for compound expressions. - right: Right child node for compound expressions. - is_compound: Flag indicating whether the node combines two child nodes. - child: Negated child node for NOT nodes (``~``, ADR-0008). - exists: The existence test's facts for exists nodes (ADR-0007). + """A condition in a query: one comparison, or several combined. + + A comparison in a query lambda returns one, such as + `post.views > 1000`. Combine conditions with `&` (and) and `|` (or), + and negate one with `~`, which renders SQL `NOT (...)`. Python's own + `and`, `or` and `not` raise `TypeError`, since they would keep only one + side. A negated comparison, like SQL `NOT`, leaves out rows where the + column is `NULL`; see + [Negation and NULL values](../guide/queries.md#negation-and-null-values). Examples: - >>> active_filter = FieldProxy("active") == True - >>> admin_filter = FieldProxy("role") == "admin" - >>> expr = active_filter & admin_filter - >>> isinstance(expr, QueryNode) - True + ```python {test=run prelude=reference} + posts = await Post.where( + lambda post: (post.views > 1000) | (post.title == "The Education of a Computer") + ).all() + assert [post.id for post in posts] == [1, 2] + + quiet = await Post.where(lambda post: ~(post.views > 1000)).all() + assert [post.id for post in quiet] == [2] + ``` """ def __init__( @@ -92,101 +101,88 @@ def __init__( exists: "ExistsTest | None" = None, owner: type | None = None, ): - """Initialize a query expression node + """Build a condition; query lambdas build these for you. Args: - column: Column name for a leaf comparison node. - operator: Comparison or logical operator string. - value: Comparison value for leaf nodes. - left: Left child node for compound expressions. - right: Right child node for compound expressions. - is_compound: Set to True for logical expressions with child nodes. - path: Relation field names from the root model to ``column``'s - table; empty (default) means the root model. Plumbed through - so relation traversal (#270) can populate it — this slice - never produces a non-empty path. - child: The negated child for a NOT node (built by ``~``, - ADR-0008); ``None`` for leaf and compound nodes. - exists: The correlation hops and inner condition tree of an - existence test (built by ``t.rel.exists()``, ADR-0007); - ``None`` for every other node kind. - owner: The model class whose scope built this leaf (the proxy's - owner), when known. A compile-side fact, never serialized — - the cross-scope guard on scoped existence tests (#315) reads - it to catch a leaf smuggled in from another lambda's - parameter. + column: The compared column, on a comparison. + operator: The comparison (`"=="`, `"<"`, `"IN"`, `"LIKE"`, ...), + or `"AND"` or `"OR"` on a combination. + value: The value the column is compared with. + left: The first condition of a combination. + right: The second condition of a combination. + is_compound: Whether the node combines `left` and `right`. + path: The relation fields from the queried model to the column's + model. Defaults to `()`, the queried model itself. + child: The negated condition, on a `~` node. + exists: The details of an existence test, on a node built by + `.exists()`. + owner: The model whose stand-in built the comparison, used to + refuse a condition that mixes two lambdas' parameters. """ self.column = column + """The compared column, or `None` on a combination.""" self.operator = operator + """The comparison or combining operator.""" self.value = value + """The value the column is compared with.""" self.left = left + """The first condition of a combination.""" self.right = right + """The second condition of a combination.""" self.is_compound = is_compound + """Whether the node combines `left` and `right`.""" self.path = path + """The relation fields from the queried model to the column's model.""" self.child = child + """The negated condition, on a `~` node.""" self.exists = exists + """The details of an existence test, on a node built by `.exists()`.""" self.owner = owner + """The model whose stand-in built the comparison, when known.""" def __or__(self, other: "QueryNode") -> "QueryNode": - """Combine two nodes with logical OR + """Combine two conditions with `OR`. Args: - other: Another query node to combine. + other: The other condition. Returns: - A compound node representing ``self OR other``. - - Examples: - >>> role = FieldProxy("role") - >>> expr = (role == "admin") | (role == "owner") - >>> expr.is_compound - True + A condition met when either one is. """ if not isinstance(other, QueryNode): return NotImplemented return QueryNode(left=self, operator="OR", right=other, is_compound=True) def __and__(self, other: "QueryNode") -> "QueryNode": - """Combine two nodes with logical AND + """Combine two conditions with `AND`. Args: - other: Another query node to combine. + other: The other condition. Returns: - A compound node representing ``self AND other``. - - Examples: - >>> active = FieldProxy("active") == True - >>> email = FieldProxy("email").like("%@ferro.dev") - >>> expr = active & email - >>> expr.is_compound - True + A condition met when both are. """ if not isinstance(other, QueryNode): return NotImplemented return QueryNode(left=self, operator="AND", right=other, is_compound=True) def __invert__(self) -> "QueryNode": - """Negate this predicate with prefix ``~`` (ADR-0008) + """Negate the condition with `~`, as SQL `NOT (...)`. - One universal rule: ``~`` negates ANY predicate node — leaf - comparison, AND/OR compound, or a negation itself (``~~p`` nests) — - by wrapping it in a NOT node rendered as SQL ``NOT (...)``. There are - no per-operator negative forms: ``~t.col.in_(ids)`` is NOT IN, - ``~t.col.like(p)`` is NOT LIKE. + `~` is the only negation: `~post.id.in_(ids)` is `NOT IN` and + `~post.title.like(pattern)` is `NOT LIKE`. Returns: - A NOT node wrapping ``self``. - - Examples: - >>> node = ~FieldProxy("role").in_(["admin", "owner"]) - >>> node.child.operator - 'IN' + A condition met when this one is not. """ return QueryNode(child=self) def to_ir_dict(self) -> dict[str, Any]: - """Serialize the query node tree into a QueryIR payload shape.""" + """Convert the condition to the dict form Ferro sends with a query. + + Returns: + The condition and everything under it, as plain dicts and lists. + """ if self.child is not None: return {"node_kind": "not", "child": self.child.to_ir_dict()} if self.exists is not None: @@ -223,17 +219,14 @@ def to_ir_dict(self) -> dict[str, Any]: } def __bool__(self) -> bool: - """Reject boolean coercion of a query node (#273). + """Refuse `bool()`, so that `and`, `or` and `not` cannot drop a condition. - Python evaluates ``and``/``or`` by calling ``bool()`` on the operands - (and ``not`` coerces its operand the same way), so ``(u.age >= 18) and - (u.active == True)`` would silently collapse to one branch instead of - building a compound predicate. Raising here turns that mistake into a - pointed error at build time; combine predicates with the bitwise - ``&`` / ``|`` operators and negate with ``~``. + Python's `and` and `or` keep one operand by its truth value, so + `(post.views > 1) and (post.id == 2)` would lose a side. Use `&`, `|` + and `~` instead. Raises: - TypeError: Always — a ``QueryNode`` has no truth value. + TypeError: Always. """ raise TypeError( "QueryNode cannot be used in a boolean context; use & / | to " @@ -242,7 +235,7 @@ def __bool__(self) -> bool: ) def __repr__(self): - """Return a developer-friendly representation of the node""" + """Show the condition, for debugging.""" if self.child is not None: return f"QueryNode(NOT {self.child!r})" if self.exists is not None: @@ -257,27 +250,26 @@ def __repr__(self): @dataclass(frozen=True) class ExistsTest: - """The facts of one existence test (GLOSSARY.md, ADR-0007). - - ``hops`` is the correlation hop path in the ``joins``-section hop-fact - shape (``QueryJoinHop`` — typed loosely here because the wire module - imports this one): one hop for a reverse FK, two for M2M. ``where`` is - the ordinary inner condition tree over the related model — empty for a - bare test; nesting and negation come free from :class:`QueryNode` - recursion. There is no negation flag: NOT EXISTS is ``~`` (a ``not`` - node) over the exists node, like every other predicate (ADR-0008). - - ``joins`` carries the inner tree's forward-traversal hop facts (#315, - ``QueryJoin`` entries, always ``"inner"`` — ADR-0006 semantics inside the - subquery), serialized only when non-empty. ``owner`` is the model whose - proxy built this test — a compile-side scope tag for the cross-scope - guard, never serialized. + """The details of one existence test, as `author.posts.exists()` builds them. + + `NOT EXISTS` is `~` over the condition that holds the test, like any + other negation. """ + # ``hops`` holds ``QueryJoinHop`` entries and ``joins`` ``QueryJoin`` + # entries, typed loosely because the wire module imports this one. hops: tuple[Any, ...] + """The joins from the queried row to the related rows. + + One for a reverse relation, two through the join table for a + many-to-many. + """ where: tuple["QueryNode", ...] + """The condition on the related rows; empty for a bare `exists()`.""" joins: tuple[Any, ...] = () + """The joins the condition needs inside the subquery.""" owner: type | None = None + """The model whose stand-in built the test.""" def _query_value_kind(value: Any) -> str: @@ -299,19 +291,31 @@ def _query_value_kind(value: Any) -> str: class FieldProxy(Generic[TField]): - """Capture field comparisons and build query nodes - - ``FieldProxy`` is generic over the column's Python type so that operator - overloads carry that type into static analysis. At runtime the type - parameter is erased and the proxy works identically for any column type. - - Attributes: - column: Database column name associated with the model field. + """A reference to one column inside a query lambda. + + Reading a field from the lambda's parameter gives one: `post.views` in + `lambda post: post.views > 1000`. It builds: + + - conditions, with `==`, `!=`, `<`, `<=`, `>`, `>=`, + [`in_`][ferro.query.nodes.FieldProxy.in_] (or `<<`) and + [`like`][ferro.query.nodes.FieldProxy.like]; + - aggregates for a `select()` dict, with + [`count`][ferro.query.nodes.FieldProxy.count], + [`sum`][ferro.query.nodes.FieldProxy.sum], + [`avg`][ferro.query.nodes.FieldProxy.avg], + [`min`][ferro.query.nodes.FieldProxy.min] and + [`max`][ferro.query.nodes.FieldProxy.max]; + - new values in an `update()` recipe, with `+`, `-` and + [`merge`][ferro.query.nodes.FieldProxy.merge]. + + The type parameter is the column's Python type, so a type checker can + tell, for example, that `like` needs a text column. Examples: - >>> email_filter = FieldProxy("email") == "taylor@example.com" - >>> isinstance(email_filter, QueryNode) - True + ```python {test=run prelude=reference} + posts = await Post.where(lambda post: post.views >= 952).all() + assert [post.id for post in posts] == [1, 2] + ``` """ def __init__( @@ -320,22 +324,19 @@ def __init__( path: tuple[str, ...] = (), owner: type | None = None, ): - """Initialize a field proxy for a specific column + """Refer to a column; query lambdas build these for you. Args: - column: Database column name to target in expressions. - path: Relation field names from the root model to this column's - table; empty (default) means the root model. Plumbed through - so relation traversal (#270) can populate it — this slice - never produces a non-empty path. - owner: The model class this column belongs to (the root model, or - a traversal's target model), when known. Query proxies always - pass it; aggregate methods (#294) read the column spec off it - to validate source families at build time. A hand-built proxy - without an owner skips that validation. + column: The column's name. + path: The relation fields from the queried model to the column's + model. Defaults to `()`, the queried model itself. + owner: The model the column belongs to. Defaults to `None`, which + skips the column type checks of the aggregate methods. """ self.column = column + """The column's name.""" self.path = path + """The relation fields from the queried model to the column's model.""" # Underscored so the attribute can never statically shadow traversal # to a relation FIELD named "owner" (`t.account.owner.email`) through # the TYPE_CHECKING `__getattr__` chain. @@ -372,31 +373,100 @@ def _aggregate(self, fn: str) -> "AggregateExpr": return AggregateExpr(fn, self.column, self.path) def count(self) -> "AggregateExpr": - """``COUNT(column)`` — counts rows where this column is non-NULL.""" + """Count the rows where this column is not `NULL`, as SQL `COUNT(column)`. + + Returns: + An aggregate to name in a `select()` dict. Its value is an `int`, + `0` over no rows. + + Examples: + ```python {test=run prelude=reference} + row = await Post.select(lambda post: {"posts": post.id.count()}).first() + assert row.posts == 2 + ``` + """ return self._aggregate("count") def sum(self) -> "AggregateExpr": - """``SUM(column)`` — numeric columns only; ``None`` over no rows.""" + """Add up a numeric column, as SQL `SUM(column)`. + + Returns: + An aggregate to name in a `select()` dict. Its value has the + column's type, or is `None` over no rows. + + Raises: + TypeError: The column is not an `int`, `float` or `Decimal` + column. + + Examples: + ```python {test=run prelude=reference} + row = await Post.select(lambda post: {"total": post.views.sum()}).first() + assert row.total == 2795 + ``` + """ return self._aggregate("sum") def avg(self) -> "AggregateExpr": - """``AVG(column)`` — numeric columns only; ``None`` over no rows.""" + """Average a numeric column, as SQL `AVG(column)`. + + Returns: + An aggregate to name in a `select()` dict. Its value is a + `float`, or a `Decimal` for a `Decimal` column, or `None` over + no rows. + + Raises: + TypeError: The column is not an `int`, `float` or `Decimal` + column. + + Examples: + ```python {test=run prelude=reference} + row = await Post.select(lambda post: {"mean": post.views.avg()}).first() + assert row.mean == 1397.5 + ``` + """ return self._aggregate("avg") def min(self) -> "AggregateExpr": - """``MIN(column)`` — numeric, text, or date/time columns.""" + """Find a column's smallest value, as SQL `MIN(column)`. + + Returns: + An aggregate to name in a `select()` dict. Its value has the + column's type, or is `None` over no rows. + + Raises: + TypeError: The column is not a number, text, date or time column. + + Examples: + ```python {test=run prelude=reference} + row = await Post.select(lambda post: {"low": post.views.min()}).first() + assert row.low == 952 + ``` + """ return self._aggregate("min") def max(self) -> "AggregateExpr": - """``MAX(column)`` — numeric, text, or date/time columns.""" + """Find a column's largest value, as SQL `MAX(column)`. + + Returns: + An aggregate to name in a `select()` dict. Its value has the + column's type, or is `None` over no rows. + + Raises: + TypeError: The column is not a number, text, date or time column. + + Examples: + ```python {test=run prelude=reference} + row = await Post.select(lambda post: {"high": post.views.max()}).first() + assert row.high == 1843 + ``` + """ return self._aggregate("max") def __iter__(self) -> "NoReturn": - """Reject iteration — the builtin-``sum`` trap (#294). + """Refuse iteration, so Python's `sum(post.views)` points at `post.views.sum()`. - ``sum(t.amount)`` calls ``iter(t.amount)`` and would otherwise fail - with an opaque message (or hang on an infinite proxy). Aggregation is - a method on the proxy, not a Python builtin over it. + Raises: + TypeError: Always. """ raise TypeError( f"{self._dotted()} is a column reference and cannot be iterated; " @@ -407,64 +477,68 @@ def __iter__(self) -> "NoReturn": if TYPE_CHECKING: def __getattr__(self, name: str) -> "FieldProxy[Any]": - """Statically model relation-traversal chaining (#270). - - Only visible to type checkers: ``t.account.ledger_id`` types each - hop as ``FieldProxy[Any]`` so a traversal comparison still yields a - ``QueryNode`` while a *bare* proxy (``lambda t: t.account``) stays a - non-``QueryNode`` and fails predicate typing. At runtime the actual - resolution lives on :class:`QueryProxy` / :class:`RelationProxy` - (relation → deeper proxy, column → this ``FieldProxy``); this - declaration is never executed. + """Type a field read through a relation for type checkers only. + + `post.author.name` types each step as `FieldProxy[Any]`, so a + comparison at the end is still a condition. At runtime the + relation steps are resolved by the row stand-in. + + Args: + name: The field read through the relation. """ ... def __eq__( # type: ignore[override] # ty: ignore[invalid-method-override] self, other: "TField | FieldProxy[TField]" ) -> QueryNode: - """Build an equality comparison node""" + """Compare for equality; `== None` renders `IS NULL`.""" return QueryNode(self.column, "==", other, path=self.path, owner=self._owner) def __ne__( # type: ignore[override] # ty: ignore[invalid-method-override] self, other: "TField | FieldProxy[TField]" ) -> QueryNode: - """Build an inequality comparison node""" + """Compare for inequality; `!= None` renders `IS NOT NULL`.""" return QueryNode(self.column, "!=", other, path=self.path, owner=self._owner) def __lt__(self, other: "TField | FieldProxy[TField]") -> QueryNode: - """Build a less-than comparison node""" + """Compare with `<`.""" return QueryNode(self.column, "<", other, path=self.path, owner=self._owner) def __le__(self, other: "TField | FieldProxy[TField]") -> QueryNode: - """Build a less-than-or-equal comparison node""" + """Compare with `<=`.""" return QueryNode(self.column, "<=", other, path=self.path, owner=self._owner) def __gt__(self, other: "TField | FieldProxy[TField]") -> QueryNode: - """Build a greater-than comparison node""" + """Compare with `>`.""" return QueryNode(self.column, ">", other, path=self.path, owner=self._owner) def __ge__(self, other: "TField | FieldProxy[TField]") -> QueryNode: - """Build a greater-than-or-equal comparison node""" + """Compare with `>=`.""" return QueryNode(self.column, ">=", other, path=self.path, owner=self._owner) def in_( self, other: "list[TField] | tuple[TField, ...] | set[TField]" ) -> QueryNode: - """Build an ``IN`` comparison node from an iterable + """Match a column against a list of values, as SQL `IN`. + + `post.id << {2, 3}` is the same condition. To test membership in + related rows, use the relation's `exists()` instead of a subquery. Args: - other: Collection of values to match against the field. + other: The values to match, as a list, tuple or set. Returns: - A node using the SQL ``IN`` operator. + A condition met when the column equals one of the values. Raises: - TypeError: If ``other`` is not a list, tuple, or set. + TypeError: `other` is not a list, tuple or set; a query is + refused with a pointer to `exists()`. Examples: - >>> status_filter = FieldProxy("status").in_(["active", "pending"]) - >>> status_filter.operator - 'IN' + ```python {test=run prelude=reference} + posts = await Post.where(lambda post: post.id.in_([2, 3])).all() + assert [post.id for post in posts] == [2] + ``` """ if not isinstance(other, (list, tuple, set)): # Late import: builder imports this module at load time. @@ -492,22 +566,22 @@ def in_( ) def like(self: "FieldProxy[str]", pattern: str) -> QueryNode: - """Build a ``LIKE`` comparison node + """Match a text column against a pattern, as SQL `LIKE`. - The ``self: FieldProxy[str]`` annotation prevents type checkers from - accepting ``.like(...)`` on non-string columns; at runtime the method - is available on any ``FieldProxy``. + In the pattern, `%` matches any run of characters and `_` one + character. A type checker accepts `like` on text columns only. Args: - pattern: SQL LIKE pattern such as ``"%@example.com"``. + pattern: The pattern, such as `"%Engine%"`. Returns: - A node using the SQL ``LIKE`` operator. + A condition met when the column matches. Examples: - >>> email_filter = FieldProxy("email").like("%@example.com") - >>> email_filter.operator - 'LIKE' + ```python {test=run prelude=reference} + posts = await Post.where(lambda post: post.title.like("%Engine%")).all() + assert [post.id for post in posts] == [1] + ``` """ return QueryNode( self.column, "LIKE", pattern, path=self.path, owner=self._owner @@ -516,39 +590,51 @@ def like(self: "FieldProxy[str]", pattern: str) -> QueryNode: def __lshift__( self, other: "list[TField] | tuple[TField, ...] | set[TField]" ) -> QueryNode: - """Use ``<<`` as shorthand syntax for ``IN`` comparisons + """Match a column against a list of values with `<<`, the same as `in_`. Args: - other: Collection of values to match against the field. + other: The values to match, as a list, tuple or set. Returns: - A node using the SQL ``IN`` operator. - - Examples: - >>> role_filter = FieldProxy("role") << {"admin", "owner"} - >>> role_filter.operator - 'IN' + A condition met when the column equals one of the values. """ return self.in_(other) def __add__(self, other: Any) -> "BinaryValueExpr": + """Add to a numeric column in an `update()` recipe.""" return _binary_value("+", self, other) def __sub__(self, other: Any) -> "BinaryValueExpr": + """Subtract from a numeric column in an `update()` recipe.""" return _binary_value("-", self, other) def __radd__(self, other: Any) -> "BinaryValueExpr": + """Add a numeric column to a value in an `update()` recipe.""" return _binary_value("+", other, self) def __rsub__(self, other: Any) -> "BinaryValueExpr": + """Subtract a numeric column from a value in an `update()` recipe.""" return _binary_value("-", other, self) def merge(self, patch: Any) -> "MergeValueExpr": - """Shallow object merge — Postgres ``jsonb ||`` (#379). + """Merge keys into a JSON object column in an `update()` recipe (Postgres only). - The patch mapping wins on key overlap. A NULL object column becomes - ``{}`` at render time. Lists and non-object targets fail here so - SQLite / ``.concat()`` never become a late execute surprise. + The update renders Postgres `jsonb ||`: keys in `patch` replace the + same keys in the column, other keys stay, and a `NULL` column counts + as `{}`. The merge is shallow, so a nested object in `patch` replaces + the stored one whole. For example, + `Post.where(...).update(lambda post: {"meta": post.meta.merge({"pinned": True})})` + sets one key of a `meta` JSON column. + + Args: + patch: The keys and values to merge in, as a mapping. + + Returns: + A new value for the column, to assign in an `update()` recipe. + + Raises: + TypeError: `patch` is not a mapping, or the column does not hold + a JSON object (a list column, or any other type). """ if isinstance(patch, (list, tuple)): raise TypeError( @@ -581,31 +667,49 @@ def merge(self, patch: Any) -> "MergeValueExpr": return MergeValueExpr(self, dict(patch)) def __repr__(self): - """Return a developer-friendly representation of the field proxy""" + """Show the column, for debugging.""" return f"FieldProxy(column={self.column!r})" class AggregateExpr: - """A build-time aggregate expression over one column (#294, ADR-0009). - - Built by the five aggregate methods on :class:`FieldProxy` - (``t.amount.sum()``, traversal included: ``t.account.balance.avg()``). - Carries the aggregate ``fn`` (the closed ``count/sum/avg/min/max`` set), - the source ``column``, and the source's relation ``path`` — the data the - ``select()`` resolver turns into a v5 ``expr`` record field. - - Deliberately opaque: an aggregate expression is a projection source, not - a value — comparing one raises pointedly at build time (post-aggregation - filtering is ``having()``, #291), and it is not a predicate, a column, or - an iterable. + """An aggregate over one column, such as `post.views.sum()`, for a `select()` dict. + + Name each aggregate with a dict key: + `select(lambda post: {"total": post.views.sum()})`. The column may be + reached through a relation (`post.author.id.count()`). With only + aggregates, the query returns one record; with plain fields beside them, + one record per group of those fields. See + [Aggregations and grouped queries](../guide/aggregations.md). + + An aggregate cannot be compared or used as a condition: `where()` + filters rows before they are aggregated, and Ferro has no `having()` + for filtering groups. Filter the rows with `where()`, or compare the + aggregated results in Python. + + Examples: + ```python {test=run prelude=reference} + row = await Post.select(lambda post: {"total": post.views.sum()}).first() + assert row.total == 2795 + ``` """ __slots__ = ("fn", "column", "path") def __init__(self, fn: str, column: str, path: tuple[str, ...]) -> None: + """Build an aggregate; the column methods such as `sum()` build these for you. + + Args: + fn: The aggregate: `"count"`, `"sum"`, `"avg"`, `"min"` or `"max"`. + column: The aggregated column. + path: The relation fields from the queried model to the column's + model. + """ self.fn = fn + """The aggregate: `"count"`, `"sum"`, `"avg"`, `"min"` or `"max"`.""" self.column = column + """The aggregated column.""" self.path = path + """The relation fields from the queried model to the column's model.""" def _dotted(self) -> str: """The selector expression this aggregate came from (errors).""" @@ -715,13 +819,17 @@ def _binary_value(op: str, left: Any, right: Any) -> "BinaryValueExpr": class ValueExpr: - """A SET value-producing expression (#377). Sibling of :class:`AggregateExpr`. + """A new value computed in the database, for an `update()` recipe. + + `post.views + 1`, [`now`][ferro.now] and `.merge()` build one. Assign it + to a column in the dict an `update()` recipe returns. A value expression + cannot be compared, selected or sorted by: it is only a new value. - Not a :class:`QueryNode`. Literals and root :class:`FieldProxy` copies - compile at the ``update()`` recipe door; ``+`` / ``-`` / ``now`` (#378) - and ``.merge()`` (#379) produce instances of this type. ``.concat()`` - is not shipped yet. Comparing one to build a predicate is #327; - projecting or ordering by one is #309. + Examples: + ```python {test=run prelude=reference} + await Post.where(lambda post: post.id == 1).update(lambda post: {"views": post.views + 1}) + assert (await Post.get(1)).views == 1844 + ``` """ def _reject_clause(self, clause: str) -> NoReturn: @@ -765,35 +873,51 @@ def __bool__(self) -> NoReturn: ) def __add__(self, other: Any) -> "BinaryValueExpr": + """Add a number to the value.""" return _binary_value("+", self, other) def __sub__(self, other: Any) -> "BinaryValueExpr": + """Subtract a number from the value.""" return _binary_value("-", self, other) def __radd__(self, other: Any) -> "BinaryValueExpr": + """Add the value to a number.""" return _binary_value("+", other, self) def __rsub__(self, other: Any) -> "BinaryValueExpr": + """Subtract the value from a number.""" return _binary_value("-", other, self) def __repr__(self) -> str: + """Show the expression, for debugging.""" return "ValueExpr()" class BinaryValueExpr(ValueExpr): - """One binary ``+`` / ``-`` SET node (#378). Either side is a value.""" + """A sum or difference in an `update()` recipe, such as `post.views + 1`.""" def __init__(self, op: str, left: Any, right: Any) -> None: + """Build the expression; `+` and `-` on a column build these for you. + + Args: + op: `"+"` or `"-"`. + left: The left operand: a column, a number or another expression. + right: The right operand: a column, a number or another + expression. + """ self.op = op + """`"+"` or `"-"`.""" self.left = left + """The left operand.""" self.right = right + """The right operand.""" def __repr__(self) -> str: return f"BinaryValueExpr({self.op!r}, {self.left!r}, {self.right!r})" class NowExpr(ValueExpr): - """The database clock singleton (#378). Assign ``now``, not ``now()``.""" + """The type of [`now`][ferro.now], the database's clock in an `update()` recipe.""" def __call__(self) -> NoReturn: raise TypeError("now is a singleton, not a function; assign now, not now()") @@ -803,31 +927,71 @@ def __repr__(self) -> str: class MergeValueExpr(ValueExpr): - """One ``.merge(patch)`` SET node (#379). Left is a json object column.""" + """A JSON object merge in an `update()` recipe, built by `.merge()`.""" def __init__(self, left: FieldProxy, patch: dict[str, Any]) -> None: + """Build the merge; [`merge`][ferro.query.nodes.FieldProxy.merge] builds these for you. + + Args: + left: The JSON object column to merge into. + patch: The keys and values to merge in. + """ self.left = left + """The JSON object column to merge into.""" self.patch = patch + """The keys and values to merge in.""" def __repr__(self) -> str: return f"MergeValueExpr({self.left!r}, {self.patch!r})" now = NowExpr() -"""The database clock: a recipe ``update()`` that assigns ``now`` (the -singleton, not ``now()``) to a ``datetime`` column stamps it with the -database's current timestamp.""" +"""The database's clock, for stamping a `datetime` column in an `update()` recipe. + +Assign `now` itself, not `now()`. The update renders SQL +`CURRENT_TIMESTAMP`, so the time comes from the database server, not from +Python. + +Examples: + ```python {test=run prelude=reference} + from datetime import datetime + from typing import Annotated + + from ferro import FerroField, Model, create_tables, now + + + class Visit(Model): + id: Annotated[int | None, FerroField(primary_key=True)] = None + page: str + seen_at: datetime | None = None + + + await create_tables() + visit = await Visit.create(page="/about") + await Visit.where(lambda visit: visit.page == "/about").update(lambda visit: {"seen_at": now}) + await visit.refresh() + assert visit.seen_at is not None + ``` +""" def validate_query_column(model_cls: type, name: str) -> str: - """Validate a queryable column name at build time (FF-F F-2). + """Check that a model has a column a query can name. + + A query can name every declared field and the `<relation>_id` column of + each foreign key. + + Args: + model_cls: The model class. + name: The column name. + + Returns: + `name`, unchanged. Raises: - AttributeError: If ``name`` is not a declared field or shadow - ``{fk}_id`` column of ``model_cls``. The message names the bad - column, suggests the closest valid one across BOTH columns and - declared forward-relation names (#270), and lists the valid - columns and relations. + AttributeError: The model has no such column. The message suggests + the closest column or relation name and lists the valid ones. + TypeError: `model_cls` is not a Ferro model. """ valid = getattr(model_cls, "__ferro_query_columns__", None) if valid is None: @@ -861,39 +1025,49 @@ def validate_query_column(model_cls: type, name: str) -> str: class QueryProxy(Generic[TModel]): - """Validating attribute proxy passed to lambda predicates (FF-F F-2). + """The stand-in for one row that a query lambda receives. - A fresh ``QueryProxy`` is constructed for the queried model each time a - lambda predicate is evaluated. Attribute access validates the name - against the model's queryable columns (declared fields plus shadow - ``{fk}_id`` columns) and returns a :class:`FieldProxy` — so - ``lambda user: user.archived == False`` builds a :class:`QueryNode`, - while ``lambda user: user.archievd == False`` raises ``AttributeError`` - at build time naming the valid columns. - - Attribute types are ``FieldProxy[Any]``: per-field static types for bare - lambda parameters require TypeScript-style mapped types, proposed for - Python in PEP 827 (draft, targeting 3.16) — adopted here when type - checkers support it. + Reading a field gives a `FieldProxy` for that column, so + `lambda post: post.views > 1000` builds a condition. A misspelled name + raises `AttributeError` with the closest match, before any SQL runs. A + foreign key relation gives a stand-in for the related row + (`post.author.name`), and a reverse or many-to-many relation offers + `exists()` (`author.posts.exists()`). See + [Typed query predicates](../concepts/query-typing.md). Examples: - >>> rows = await User.where(lambda user: user.archived == False).all() # noqa: E712 + ```python {test=run prelude=reference} + authors = await Author.where(lambda author: author.posts.exists(lambda post: post.views > 1000)).all() + assert [author.name for author in authors] == ["Ada Lovelace"] + ``` """ + # Every attribute types as ``FieldProxy[Any]``: per-field types for a bare + # lambda parameter need mapped types, proposed for Python in PEP 827. + __slots__ = ("_model_cls",) def __init__(self, model_cls: type) -> None: + """Stand in for a row of a model; queries build these for you. + + Args: + model_cls: The queried model class. + """ self._model_cls = model_cls def __getattr__(self, name: str) -> "FieldProxy[Any]": - """Resolve ``name`` on the root model. - - Relation specs are consulted FIRST (#270): a declared forward-FK field - name yields a :class:`RelationProxy` for traversal (``t.account`` → - proxy → ``.ledger_id``). A reverse (BackRef) relation name yields a - :class:`ReverseRelationProxy` exposing the existence test and nothing - else (#314, ADR-0007). Every other name falls through to column - validation and returns a :class:`FieldProxy`. + """Read a field: a column, a foreign key relation or a reverse relation. + + Args: + name: The field name. + + Returns: + A `FieldProxy` for a column, a `RelationProxy` for a foreign key + relation, or a `ReverseRelationProxy` for a reverse or + many-to-many relation. + + Raises: + AttributeError: The model has no such field. """ relations = getattr(self._model_cls, "__ferro_relation_specs__", None) or {} spec = relations.get(name) @@ -916,47 +1090,56 @@ def __getattr__(self, name: str) -> "FieldProxy[Any]": class RelationProxy: - """Traversal proxy for a declared forward-FK relation path (#270). - - Returned by attribute access on a :class:`QueryProxy` (or a deeper - ``RelationProxy``) when the accessed name is a declared forward relation. - It carries the *root* model, the relation ``path`` walked so far (a tuple - of relation field names), and the ``target`` model that path resolves to. - - Attribute access resolves against the CURRENT TARGET model, recursively at - any depth: - - - a relation name yields a deeper ``RelationProxy`` (``path + (name,)``); - - a column name yields a :class:`FieldProxy` carrying this proxy's ``path``, - so ``t.account.ledger_id == lid`` builds a path-qualified comparison node; - - an unknown name raises ``AttributeError`` naming the hop's model with a - did-you-mean suggestion (same style as :func:`validate_query_column`). - - A bare ``RelationProxy`` returned from a ``where()`` lambda is not a - :class:`QueryNode`; ``where()`` rejects it. Equality sugar against a - persisted target instance or ``None`` desugars **join-free** to a - shadow-FK comparison (#273): ``t.account == acct`` builds - ``QueryNode(column="account_id", operator="==", value=<acct.pk>, path=())`` - — the shadow column of the LAST hop, with the proxy path MINUS that hop, so - a deep proxy (``t.account.owner == o``) compares ``owner_id`` under the - ``("account",)`` prefix joins only. + """The stand-in for a related row, reached through a foreign key relation. + + `post.author` in a query lambda gives one. Reading a field from it + reaches into the related model (`post.author.name`) and joins that + model when the query runs; the path can continue through further + relations. Comparing the relation itself with `==` or `!=` to a saved + instance or `None` compares the foreign key column, with no join. Other + comparisons on the relation itself raise `TypeError` naming a column to + compare instead. Examples: - >>> proxy = QueryProxy(Transaction) # doctest: +SKIP - >>> node = (proxy.account.ledger_id == 7) # doctest: +SKIP - >>> node.path # doctest: +SKIP - ('account',) + ```python {test=run prelude=reference} + ada = await Author.get(1) + by_ada = await Post.where(lambda post: post.author == ada).all() + assert [post.id for post in by_ada] == [1] + + by_name = await Post.where(lambda post: post.author.name == "Grace Hopper").all() + assert [post.id for post in by_name] == [2] + ``` """ __slots__ = ("_root_model", "_path", "_target") def __init__(self, root_model: type, path: tuple[str, ...], target: type) -> None: + """Stand in for a related row; query lambdas build these for you. + + Args: + root_model: The queried model class. + path: The relation fields from the queried model to this one. + target: The related model class. + """ self._root_model = root_model self._path = path self._target = target def __getattr__(self, name: str) -> "RelationProxy | FieldProxy[Any]": - """Resolve ``name`` against the current target model (one hop deeper).""" + """Read a field of the related model: a column or a further relation. + + Args: + name: The field name. + + Returns: + A `FieldProxy` for a column, or a `RelationProxy` one relation + further. + + Raises: + AttributeError: The related model has no such field, or the + field is a reverse relation, which can only be tested with + `exists()` from the queried model. + """ relations = getattr(self._target, "__ferro_relation_specs__", None) or {} spec = relations.get(name) if spec is not None: @@ -1056,13 +1239,13 @@ def _instance_comparison(self, other: object, operator: str) -> QueryNode: def __eq__( # type: ignore[override] # ty: ignore[invalid-method-override] self, other: object ) -> QueryNode: - """Desugar ``t.<relation> == instance`` / ``== None`` (join-free, #273).""" + """Match rows whose relation is a given instance, or empty with `None`.""" return self._instance_comparison(other, "==") def __ne__( # type: ignore[override] # ty: ignore[invalid-method-override] self, other: object ) -> QueryNode: - """Desugar ``t.<relation> != instance`` / ``!= None`` (join-free, #273).""" + """Match rows whose relation is not a given instance, or set with `None`.""" return self._instance_comparison(other, "!=") def _reject_operator(self, symbol: str) -> "QueryNode": @@ -1087,15 +1270,33 @@ def __ge__(self, other: object) -> "QueryNode": return self._reject_operator(">=") def in_(self, other: object) -> "QueryNode": + """Refuse `in_` on a relation; compare one of its columns instead. + + Args: + other: Ignored; the call always raises. + + Raises: + TypeError: Always. + """ return self._reject_operator("in_") def like(self, other: object) -> "QueryNode": + """Refuse `like` on a relation; match one of its columns instead. + + Args: + other: Ignored; the call always raises. + + Raises: + TypeError: Always. + """ return self._reject_operator("like") def __lshift__(self, other: object) -> "QueryNode": + """Refuse `<<` on a relation; compare one of its columns instead.""" return self._reject_operator("<<") def __repr__(self) -> str: + """Show the relation path, for debugging.""" joined = ".".join(self._path) return f"RelationProxy(path={joined!r}, target={self._target.__name__!r})" @@ -1106,7 +1307,7 @@ def _resolve_scoped_predicate( """Evaluate an existence test's inner lambda over the related model (#315). The inner predicate is ordinary ferro — the same validating - :class:`QueryProxy` a root ``where()`` receives, constructed for the + `QueryProxy` a root ``where()`` receives, constructed for the related model — so every operator, combinator, traversal, and nested existence test works unchanged. The rejection shapes mirror ``where()``'s own. @@ -1245,52 +1446,64 @@ def _collect_inner_traversal_paths( class ReverseRelationProxy: - """Predicate proxy for a reverse (BackRef) or M2M relation (ADR-0007). - - Returned by attribute access on a :class:`QueryProxy` when the accessed - name is a resolved reverse or many-to-many relation. It exposes exactly - one verb — the existence test :meth:`exists` — because reverse relations - are *tested*, never *traversed*: column access, comparisons (including - ``!= None`` / ``== None``), and ``in_`` raise at build time with the - supported spelling in the message. Negation is uniform ``~`` over the - returned node (ADR-0008), so NOT EXISTS is ``~t.rel.exists()``. + """The stand-in for a reverse (`BackRef`) or many-to-many relation in a query lambda. + + `author.posts` gives one. Its one method is + [`exists`][ferro.query.nodes.ReverseRelationProxy.exists], which tests + whether related rows exist; reading a column, comparing the relation + or joining it raises with that spelling in the message, since joining + a collection would repeat the queried rows. See + [Existence tests](../guide/queries.md#existence-tests-on-reverse-many-to-many-relations). + + Examples: + ```python {test=run prelude=reference} + authors = await Author.where(lambda author: author.posts.exists()).all() + assert len(authors) == 2 + ``` """ __slots__ = ("_root_model", "_name", "_spec") def __init__(self, root_model: type, name: str, spec: Any) -> None: + """Stand in for a reverse relation; query lambdas build these for you. + + Args: + root_model: The queried model class. + name: The relation's field name. + spec: The relation's details, from the model. + """ self._root_model = root_model self._name = name self._spec = spec def exists(self, inner: "Predicate[Any] | None" = None) -> QueryNode: - """Build the existence test: a correlated EXISTS over the child rows. + """Test whether the row has related rows, as SQL `EXISTS (...)`. - Always a correlated EXISTS at every cardinality (a one-to-one BackRef - renders identically to a to-many one); the result stays root-shaped, - so the node composes with any other predicate, ordering, and paging. + Each queried row comes back at most once, however many related rows + match, so the condition combines with any other, with sorting and + with paging. Negate it with `~` for rows that have none. Args: - inner: Optional scoping predicate — a full ferro predicate over - the related model (#315): every operator, ``&``/``|``/``~``, - forward traversal (joins rendered INSIDE the subquery, - ADR-0006 unchanged), and nested existence tests. It may - reference only its own parameter's scope; cross-scope - references raise at build time (#309). + inner: A lambda over the related model that the related rows + must meet, such as `lambda post: post.views > 1000`. It can + use every condition a `where()` lambda can, but only its own + parameter. Defaults to `None`, any related row. Returns: - An exists :class:`QueryNode` carrying the one-hop correlation - path (child table, child FK column against the root PK) and, - when scoped, the inner condition tree plus its forward-traversal - join facts. + A condition met when a related row exists. Raises: - ValueError: If the root model declares no primary-key column — - the EXISTS correlates child FK to root PK, so a PK-less root - is a loud error, never a guess. - TypeError: If ``inner`` is not a predicate callable, does not - return a predicate, or references a scope other than its own - parameter (cross-scope, #309). + ValueError: The queried model, or a many-to-many relation's + target, has no primary key. + TypeError: `inner` is not callable, does not return a condition, + or uses the outer lambda's parameter. + + Examples: + ```python {test=run prelude=reference} + await Author.create(name="Mary Somerville", email="mary@example.com") + without_posts = await Author.where(lambda author: ~author.posts.exists()).all() + assert [author.name for author in without_posts] == ["Mary Somerville"] + ``` """ # Late import: wire.py imports this module (nodes owns the predicate # shape, wire owns the hop-fact shape). @@ -1375,6 +1588,7 @@ def _reject_operator(self, symbol: str) -> NoReturn: ) def __getattr__(self, name: str) -> NoReturn: + """Refuse a column read: a reverse relation can only be tested with `exists()`.""" raise AttributeError( f"reverse relation {self._name!r} has no queryable column " f"{name!r}: reverse relations are tested, not traversed " @@ -1403,15 +1617,33 @@ def __ge__(self, other: object) -> QueryNode: return self._reject_operator(">=") def in_(self, other: object) -> QueryNode: + """Refuse `in_` on a reverse relation; use `exists()`. + + Args: + other: Ignored; the call always raises. + + Raises: + TypeError: Always. + """ return self._reject_operator("in_") def like(self, other: object) -> QueryNode: + """Refuse `like` on a reverse relation; use `exists()`. + + Args: + other: Ignored; the call always raises. + + Raises: + TypeError: Always. + """ return self._reject_operator("like") def __lshift__(self, other: object) -> QueryNode: + """Refuse `<<` on a reverse relation; use `exists()`.""" return self._reject_operator("<<") def __repr__(self) -> str: + """Show the relation, for debugging.""" return ( f"ReverseRelationProxy(relation={self._name!r}, " f"child={self._spec.target.__name__!r})" @@ -1419,7 +1651,11 @@ def __repr__(self) -> str: Predicate: TypeAlias = Callable[[QueryProxy[TModel]], QueryNode] -"""Type alias for lambda predicates accepted by :meth:`Query.where`.""" +"""The type of a query lambda: it takes a row stand-in and returns a condition. + +`Predicate[Post]` is `Callable[[QueryProxy[Post]], QueryNode]`, as in +`lambda post: post.views > 1000`. +""" RowSelector: TypeAlias = Callable[ [QueryProxy[TModel]], @@ -1428,15 +1664,12 @@ def __repr__(self) -> str: | list[FieldProxy[Any]] | dict[str, "FieldProxy[Any] | AggregateExpr"], ] -"""Type alias for lambda selectors accepted by ``select()`` projections. - -A selector names fields — one (``lambda t: t.amount``), several -(``lambda t: (t.id, t.amount)``), or a dict whose string keys name the -output fields (``lambda t: {"account_name": t.account.name}`` — output -aliases, #293). Fields may traverse forward-FK relations at any depth, and -dict values may be aggregate expressions (``{"total": t.amount.sum()}`` — -#294; aggregates are user-named, so the dict form is their only home). -Returning a comparison (a :class:`QueryNode`), nesting shapes, or using a -non-string dict key fails the static gate: a projection selects fields, -a predicate belongs in ``where()``. +"""The type of a `select()` lambda: it takes a row stand-in and returns the fields to keep. + +It returns one field (`lambda post: post.title`), a tuple of fields +(`lambda post: (post.id, post.title)`), or a dict whose string keys name the +output fields (`lambda post: {"author": post.author.name}`). A field may be +reached through relations, and a dict value may be an aggregate +(`{"total": post.views.sum()}`). A type checker refuses a comparison, a +nested shape or a key that is not a string. """ diff --git a/src/ferro/query/rendered.py b/src/ferro/query/rendered.py index 1c087d7f..1b8fd840 100644 --- a/src/ferro/query/rendered.py +++ b/src/ferro/query/rendered.py @@ -59,8 +59,7 @@ class RenderedSql: sql: str """The SQL text, with the dialect's placeholders (`$1` or `?`).""" params: tuple[object, ...] - """The bound values in placeholder order; a secret value is - [`REDACTED`][ferro.REDACTED].""" + """The bound values in placeholder order, a secret value shown as [`REDACTED`][ferro.REDACTED].""" dialect: Literal["postgres", "sqlite"] """The dialect of the database the call would run on.""" @@ -158,8 +157,8 @@ async def sql(self, *args: P.args, **kwargs: P.kwargs) -> RenderedSql: Returns: The SQL, parameters and dialect the call would send, as the - statement log would record them, with each secret value shown - as [`REDACTED`][ferro.REDACTED]. + statement log records them, with each secret value shown as + [`REDACTED`][ferro.REDACTED]. Raises: RuntimeError: No database route: the call is outside a session @@ -189,7 +188,7 @@ def __repr__(self) -> str: class TerminalMethod(Generic[S, P, R_co]): - """A query terminal on its class: reading it from a query binds a [`Terminal`][ferro.query.rendered.Terminal].""" + """A query terminal on its class: reading it from a query binds a [`Terminal`][ferro.query.Terminal].""" def __init__(self, method: Callable[Concatenate[S, P], Awaitable[R_co]]) -> None: self._method = method @@ -227,6 +226,6 @@ def terminal( Returns: The terminal, read from a query as a bound - [`Terminal`][ferro.query.rendered.Terminal]. + [`Terminal`][ferro.query.Terminal]. """ return TerminalMethod(method) diff --git a/src/ferro/query/rows.py b/src/ferro/query/rows.py index 63ee7ea6..c9185d2d 100644 --- a/src/ferro/query/rows.py +++ b/src/ferro/query/rows.py @@ -1,69 +1,67 @@ -"""Projected-record containers: ``Row`` and the list-like ``Rows`` (ADR-0007). - -A projection (``Transaction.select(lambda t: (t.id, t.amount))``) returns -projected records, never model instances — a model instance always carries a -complete row (the complete-instance invariant). ``Row`` is one record; -``Rows`` is the list-like container a projected query's ``all()`` delivers -them in. - -Both are pydantic-shaped (``model_dump()``, FastAPI ``response_model``) but -never pydantic-constructed: instances come off the same direct-to-dict -hydration path as models (AGENTS.md I-2), and ``Rows`` wraps without -re-validating. Pydantic is the interface, not the constructor. +"""The records a `select()` projection returns: [`Row`][ferro.query.Row] and [`Rows`][ferro.query.Rows]. + +A projection returns records, never model instances, because a model +instance always holds a complete row. Both types behave like Pydantic models +for `model_dump()` and as a FastAPI `response_model`. """ +# Both are built without Pydantic validation: rows come off the same +# direct-to-dict hydration path as model instances (AGENTS.md I-2), and +# ``Rows`` wraps them without re-validating. + from collections.abc import Iterator from typing import TypeVar, overload from pydantic import BaseModel, ConfigDict, RootModel R = TypeVar("R") +"""The record type a `Rows` list holds.""" class Row(BaseModel): - """One projected record: named values from an explicit projection. - - A ``Row`` is not a model instance. It carries no persistence identity — - no ``save()``, no refresh, no identity-map participation — and its field - set is whatever the projection selected, in selection order. + """One record from a `select()` projection: the selected fields, by name. - Fields live in pydantic's extra storage (``model_config`` is - ``extra="allow"``); the Rust hydration path populates them directly - (I-2), so construction never runs pydantic validation. Attribute access, - ``model_dump()``, ``model_dump_json()``, equality, and ``repr`` all work - as on any pydantic model: + A `Row` is not a model instance: it has no `save()` or `refresh()`, the + session does not track it, and its fields are the ones the projection + selected, in that order. Reading a field, `model_dump()`, + `model_dump_json()`, equality and `repr` work as on any Pydantic model. + See + [Selecting a column subset](../guide/queries.md#selecting-a-column-subset). Examples: - >>> rows = await Transaction.select(lambda t: (t.id, t.amount)).all() # doctest: +SKIP - >>> rows[0].amount # doctest: +SKIP - Decimal('12.50') - >>> rows[0].model_dump() # doctest: +SKIP - {'id': 1, 'amount': Decimal('12.50')} + ```python {test=run prelude=reference} + rows = await Post.select(lambda post: (post.id, post.views)).order_by(lambda post: post.id).all() + assert rows[0].views == 1843 + assert rows[0].model_dump() == {"id": 1, "views": 1843} + ``` """ model_config = ConfigDict(extra="allow") + """The selected fields are stored as Pydantic extra fields.""" class Rows(RootModel[list[R]]): - """List-like pydantic container for projected records. + """The list of records a projected query's `all()` returns. - Delivered by a projected query's ``all()`` as ``Rows[Row]``. Indexing, - slicing, iteration, ``len()``, and truthiness work like a list; a slice - returns a ``Rows`` of the same record type. As a pydantic ``RootModel`` - it drops straight into an API response: ``model_dump()`` yields - ``list[dict]`` and ``Rows[Row]`` works as a FastAPI ``response_model``. + It works like a list: indexing, slicing (which returns a `Rows`), + iteration, `len()` and truthiness. As a Pydantic `RootModel`, + `model_dump()` gives a list of dicts and `Rows[Row]` works as a FastAPI + `response_model`. Examples: - >>> len(rows), rows[0].id, [r.amount for r in rows] # doctest: +SKIP - (3, 1, [Decimal('12.50'), Decimal('7.00'), Decimal('99.99')]) - >>> rows[:2].model_dump() # doctest: +SKIP - [{'id': 1, 'amount': Decimal('12.50')}, {'id': 2, 'amount': Decimal('7.00')}] + ```python {test=run prelude=reference} + rows = await Post.select(lambda post: (post.id, post.title)).order_by(lambda post: post.id).all() + assert len(rows) == 2 + assert rows[:1].model_dump() == [{"id": 1, "title": "Notes on the Analytical Engine"}] + ``` """ def __len__(self) -> int: + """Count the records.""" return len(self.root) def __iter__(self) -> Iterator[R]: # type: ignore[override] # ty: ignore[invalid-method-override] + """Iterate over the records.""" # BaseModel.__iter__ yields (field, value) pairs; a Rows iterates its # records, like the list it stands in for. return iter(self.root) @@ -75,16 +73,16 @@ def __getitem__(self, index: int) -> R: ... def __getitem__(self, index: slice) -> "Rows[R]": ... def __getitem__(self, index: int | slice) -> "R | Rows[R]": + """Return one record, or a `Rows` of a slice.""" if isinstance(index, slice): return self.__class__.model_construct(root=self.root[index]) return self.root[index] @classmethod def _wrap(cls, records: list[R]) -> "Rows[R]": - """Wrap already-hydrated records WITHOUT validation (ADR-0007). + """Wrap already-hydrated records without validation. - The records come off the Rust hydration path; re-validating them here - would put pydantic back on the hot path that direct-to-dict - construction exists to avoid (I-2). + Re-validating would put Pydantic back on the hot path that + direct-to-dict construction avoids (AGENTS.md I-2). """ return cls.model_construct(root=records) diff --git a/src/ferro/query/wire.py b/src/ferro/query/wire.py index 0c820f7c..739ccdbc 100644 --- a/src/ferro/query/wire.py +++ b/src/ferro/query/wire.py @@ -1,24 +1,21 @@ -"""The QueryIR payload: Ferro's single query wire contract (GLOSSARY.md). - -Every query the ORM executes crosses the Python→Rust seam as one versioned -JSON envelope. This module is the only place that wire shape exists on the -Python side: :func:`compile_query` is the sole compiler from a built query to -a :class:`QueryIrPayload`, and :func:`to_wire_json` the sole envelope -serializer. The dataclasses mirror ``crates/ferro-schema-ir``'s serde structs -one-for-one; golden vectors in ``tests/fixtures/ir_vectors/`` pin both sides -to the same bytes (builder equality in ``tests/test_query_wire_vectors.py``, -serde round-trips in the Rust crate's tests). - -Friend contract: :func:`compile_query` reads the query object's build state -(``where_clause``, ``order_by_clause``, ``_limit``, ``_offset``, ``_after``, -``_before``, -``_joins``, ``_explicit_edges``, ``_includes``, ``_m2m_context``, -``_projection``) directly — the builder and this module are one package, and -the seam is ``compile_query``, not a snapshot type. The builder owns -chainer-time validation and state; this module owns everything whose output -is wire shape. +"""The query payload: the one form in which a built query reaches Ferro's core. + +Every query the ORM runs is sent as one versioned JSON document. +[`compile_query`][ferro.query.wire.compile_query] turns a built query into a +[`QueryIrPayload`][ferro.query.wire.QueryIrPayload], and +[`to_wire_json`][ferro.query.wire.to_wire_json] serializes it. """ +# The dataclasses mirror the core's payload structs one for one; golden +# vectors in tests/fixtures/ir_vectors/ pin both sides to the same bytes +# (tests/test_query_wire_vectors.py here, serde round-trips in the crate). +# +# Friend contract: compile_query reads the query's build state (where_clause, +# order_by_clause, _limit, _offset, _after, _before, _joins, _explicit_edges, +# _includes, _m2m_context, _projection) directly. The builder owns chain-time +# validation and state; this module owns everything whose output is wire +# shape. + from __future__ import annotations import json @@ -51,9 +48,10 @@ if TYPE_CHECKING: from .builder import Query -# The verb a payload is compiled for. ``update`` and ``delete`` share the -# mutate arm; they stay distinct values so guardrail errors name the caller. +# ``update`` and ``delete`` share the mutate arm; they stay distinct values +# so guardrail errors name the caller. QueryVerb = Literal["fetch", "count", "update", "delete"] +"""What a payload is compiled for: fetching rows, counting, updating or deleting.""" # Always 14 (#395 — unconditional bump). v14 adds an optional ``before`` # position bound on fetch payloads. Python and Rust ship in one wheel, so a @@ -77,33 +75,55 @@ def __repr__(self) -> str: # pragma: no cover - debugging aid def model_identity(model_cls: type) -> str: - """Qualified registry identity of a Ferro model class (FF-E).""" + """Return a model class's qualified name in the model registry. + + Args: + model_cls: The model class. + + Returns: + The name the registry and the core know the model by. + """ return model_cls.__ferro_identity__ # ty: ignore[unresolved-attribute] def path_edges(path: tuple[str, ...]) -> list[tuple[str, ...]]: - """Every edge (prefix of length ≥ 1) of a relation ``path`` (#272).""" + """List every leading part of a relation path, shortest first. + + Args: + path: Relation field names, such as `("author", "team")`. + + Returns: + Each prefix of `path` of length one or more: `[("author",), + ("author", "team")]`. + """ return [path[:i] for i in range(1, len(path) + 1)] @dataclass(frozen=True) class QueryJoinHop: - """One hop fact of a relation path — the shared shape of the ``joins`` - section, ``instances`` plan paths, and aggregate ``expr`` traversals. + """One step of a relation path: the join from one table to the next. - ``target`` is the hop's model class — a compile-side fact riding the hop - (the hop-class map is collected from it), never serialized: the wire - carries ``to_table`` and Rust resolves registration facts from its own - registry. + The same shape describes the payload's joins, the paths an `include()` + fills in and the path of an aggregate's column. """ relation: str + """The relation field the step follows.""" from_column: str + """The column on the source side, the foreign key column.""" to_table: str + """The table the step joins.""" to_column: str + """The column on the joined table, its primary key.""" target: type = field(compare=False, repr=False, kw_only=True) + """The joined model class; kept in Python and never sent.""" def to_ir_dict(self) -> dict[str, str]: + """Convert the step to its payload form. + + Returns: + The step as a dict, without `target`. + """ return { "relation": self.relation, "from_column": self.from_column, @@ -114,12 +134,19 @@ def to_ir_dict(self) -> dict[str, str]: @dataclass(frozen=True) class QueryJoin: - """One ``joins`` entry: a full relation path and its resolved join type.""" + """One join the query needs: a relation path and its join type.""" join_type: str + """`"inner"` or `"left"`.""" path: tuple[QueryJoinHop, ...] + """The steps from the queried table to the joined one.""" def to_ir_dict(self) -> dict[str, Any]: + """Convert the join to its payload form. + + Returns: + The join as a dict. + """ return { "join_type": self.join_type, "path": [hop.to_ir_dict() for hop in self.path], @@ -128,14 +155,23 @@ def to_ir_dict(self) -> dict[str, Any]: @dataclass(frozen=True) class OrderByEntry: - """One ``ORDER BY`` clause: column, direction, relation path, and nulls.""" + """One sort key of a query, as `order_by()` records it.""" column: str + """The sorted column.""" direction: str + """`"asc"` or `"desc"`.""" path: tuple[str, ...] + """The relation fields from the queried model to the column's model.""" nulls: str = "last" + """Where `NULL` values sort: `"first"`, `"last"` or `"native"`.""" def to_ir_dict(self) -> dict[str, Any]: + """Convert the sort key to its payload form. + + Returns: + The sort key as a dict. + """ return { "column": self.column, "direction": self.direction, @@ -146,14 +182,23 @@ def to_ir_dict(self) -> dict[str, Any]: @dataclass(frozen=True) class M2mContext: - """Many-to-many linkage context riding the payload's ``m2m`` section.""" + """The join table details of a query over one instance's many-to-many relation.""" join_table: str + """The join table.""" source_col: str + """The join table column that points at the owning model.""" target_col: str + """The join table column that points at the related model.""" source_id: Any + """The owning instance's primary key.""" def to_ir_dict(self) -> dict[str, Any]: + """Convert the details to their payload form. + + Returns: + The details as a dict. + """ return { "join_table": self.join_table, "source_col": self.source_col, @@ -164,44 +209,71 @@ def to_ir_dict(self) -> dict[str, Any]: @dataclass(frozen=True) class QueryValue: - """One typed literal value carried by a value-expression node.""" + """A literal value and its kind, as the payload carries it.""" kind: str + """The value's kind: `"null"`, `"bool"`, `"int"`, `"float"`, `"string"`, ...""" value: Any + """The value, in its JSON form.""" def to_ir_dict(self) -> dict[str, Any]: + """Convert the value to its payload form. + + Returns: + The value and its kind, as a dict. + """ return {"kind": self.kind, "value": self.value} @dataclass(frozen=True) class LiteralValueExpr: - """A bound literal in the clause-agnostic SET value-expression family.""" + """A literal new value in an update.""" value: QueryValue + """The value.""" def to_ir_dict(self) -> dict[str, Any]: + """Convert the expression to its payload form. + + Returns: + The expression as a dict. + """ return {"kind": "literal", "value": self.value.to_ir_dict()} @dataclass(frozen=True) class ColumnRefValueExpr: - """An unqualified root-column copy in the SET value-expression family.""" + """A new value copied from another column of the same row.""" column: str + """The column copied from.""" def to_ir_dict(self) -> dict[str, Any]: + """Convert the expression to its payload form. + + Returns: + The expression as a dict. + """ return {"kind": "column", "column": self.column} @dataclass(frozen=True) class BinarySetExpr: - """A binary ``+`` / ``-`` SET value expression (#378).""" + """A new value computed with `+` or `-`.""" op: Literal["add", "sub"] + """`"add"` or `"sub"`.""" left: "SetValueExpr" + """The left operand.""" right: "SetValueExpr" + """The right operand.""" def to_ir_dict(self) -> dict[str, Any]: + """Convert the expression to its payload form. + + Returns: + The expression as a dict. + """ return { "kind": self.op, "left": self.left.to_ir_dict(), @@ -211,20 +283,32 @@ def to_ir_dict(self) -> dict[str, Any]: @dataclass(frozen=True) class NowSetExpr: - """The database-clock SET value expression (#378).""" + """A new value read from the database's clock.""" def to_ir_dict(self) -> dict[str, Any]: + """Convert the expression to its payload form. + + Returns: + The expression as a dict. + """ return {"kind": "now"} @dataclass(frozen=True) class MergeSetExpr: - """A Postgres-only shallow object-merge SET value expression (#379).""" + """A new value merging keys into a JSON object column (Postgres only).""" left: "SetValueExpr" + """The column merged into.""" right: "SetValueExpr" + """The keys and values merged in.""" def to_ir_dict(self) -> dict[str, Any]: + """Convert the expression to its payload form. + + Returns: + The expression as a dict. + """ return { "kind": "merge", "left": self.left.to_ir_dict(), @@ -235,33 +319,47 @@ def to_ir_dict(self) -> dict[str, Any]: SetValueExpr = ( LiteralValueExpr | ColumnRefValueExpr | BinarySetExpr | NowSetExpr | MergeSetExpr ) +"""Any new value an update can assign to a column.""" @dataclass(frozen=True) class SetAssignment: - """One ordered root-column assignment in the canonical SET section.""" + """One column an update assigns, and its new value.""" column: str + """The assigned column.""" value: SetValueExpr + """The new value.""" def to_ir_dict(self) -> dict[str, Any]: + """Convert the assignment to its payload form. + + Returns: + The assignment as a dict. + """ return {"column": self.column, "value": self.value.to_ir_dict()} @dataclass(frozen=True) class RecordExpr: - """The expression source of an aggregate record field (v5, ADR-0009). + """The aggregate a projected field reads, such as `SUM(views)`. - Self-contained on the wire: ``fn`` travels as data (the closed aggregate - set) and ``path`` carries the source column's traversal as ordered hop - facts rather than keying into the ``joins`` section. + It carries its own relation path, apart from the query's joins. """ fn: str + """The aggregate: `"count"`, `"sum"`, `"avg"`, `"min"` or `"max"`.""" column: str + """The aggregated column.""" path: tuple[QueryJoinHop, ...] + """The steps from the queried table to the column's table.""" def to_ir_dict(self) -> dict[str, Any]: + """Convert the aggregate to its payload form. + + Returns: + The aggregate as a dict. + """ return { "fn": self.fn, "column": self.column, @@ -271,21 +369,27 @@ def to_ir_dict(self) -> dict[str, Any]: @dataclass(frozen=True) class RecordField: - """One projected field of a ``record`` plan (ADR-0007/ADR-0009). + """One field of a projected record: its output name and what it reads. - ``name`` is declared separately from the source (output aliases are - free). A field reads from exactly one source: a ``column`` + relation - ``path``, or an aggregate ``expr`` whose traversal rides the expression — - never both, never neither (the Rust side enforces this at - deserialization). + A field reads either a column, through `path`, or an aggregate, through + `expr`; never both and never neither. """ name: str + """The field's name on the record.""" column: str | None + """The column read, or `None` for an aggregate.""" path: tuple[str, ...] + """The relation fields from the queried model to the column's model.""" expr: RecordExpr | None + """The aggregate read, or `None` for a column.""" def to_ir_dict(self) -> dict[str, Any]: + """Convert the field to its payload form. + + Returns: + The field as a dict, with `column` or `expr` as it reads. + """ if self.expr is not None: return { "name": self.name, @@ -301,19 +405,30 @@ def to_ir_dict(self) -> dict[str, Any]: @dataclass(frozen=True) class RootInstances: - """Complete root-model instances — the default materialization plan.""" + """Results as whole instances of the queried model, the default.""" def to_ir_dict(self) -> dict[str, Any]: + """Convert the plan to its payload form. + + Returns: + The plan as a dict. + """ return {"kind": "root_instances"} @dataclass(frozen=True) class Record: - """Projected records: a typed column subset decoded into ``Row`` records.""" + """Results as `Row` records holding the selected fields.""" fields: tuple[RecordField, ...] + """The record's fields, in selection order.""" def to_ir_dict(self) -> dict[str, Any]: + """Convert the plan to its payload form. + + Returns: + The plan as a dict. + """ return { "kind": "record", "fields": [field.to_ir_dict() for field in self.fields], @@ -322,14 +437,21 @@ def to_ir_dict(self) -> dict[str, Any]: @dataclass(frozen=True) class Instances: - """Populated instance graphs (joined-row hydration, ADR-0008): one - ordered hop-fact list per include path, deliberately separate from the - ``joins`` section so include can never change membership or a shared - edge's join type.""" + """Results as instances with the `include()` relations filled in. + + The paths are kept apart from the query's joins, so an `include()` never + changes which rows come back or how a shared join is made. + """ paths: tuple[tuple[QueryJoinHop, ...], ...] + """One path of steps per included relation.""" def to_ir_dict(self) -> dict[str, Any]: + """Convert the plan to its payload form. + + Returns: + The plan as a dict. + """ return { "kind": "instances", "paths": [[hop.to_ir_dict() for hop in path] for path in self.paths], @@ -337,32 +459,46 @@ def to_ir_dict(self) -> dict[str, Any]: Materialization = RootInstances | Record | Instances +"""What a query's results are built as: instances, records or filled-in instances.""" @dataclass(frozen=True) class QueryIrPayload: - """The single typed wire artifact a query ships to the Rust runtime. + """Everything one query sends to Ferro's core, field for field. - Mirrors ``ferro_schema_ir::QueryIrPayload`` field-for-field. ``where`` - holds :meth:`QueryNode.to_ir_dict` output — the node module owns the - predicate wire shape. ``limit``/``offset`` may be :data:`_ABSENT` - (mutating payloads omit the keys entirely). ``after`` / ``before`` are - omitted when unset (fetch, count, and mutating payloads alike). + An update or delete leaves out `limit` and `offset` entirely, and any + query leaves out `after` and `before` when they are not set. """ model_name: str + """The queried model's registry name.""" where: tuple[dict[str, Any], ...] + """The conditions, each in the dict form `QueryNode.to_ir_dict()` returns.""" set: tuple[SetAssignment, ...] + """The columns an update assigns; empty for other queries.""" order_by: tuple[OrderByEntry, ...] + """The sort keys.""" limit: int | None | _AbsentType + """The row limit, `None`, or absent on an update or delete.""" offset: int | None | _AbsentType + """The rows skipped, `None`, or absent on an update or delete.""" after: tuple[QueryValue, ...] | None + """The position the results start after, one value per sort key.""" before: tuple[QueryValue, ...] | None + """The position the results end before, one value per sort key.""" m2m: M2mContext | None + """The join table details, on a query over a many-to-many relation.""" joins: tuple[QueryJoin, ...] + """The joins the query needs.""" materialization: Materialization + """What the results are built as.""" def to_ir_dict(self) -> dict[str, Any]: + """Convert the payload to its dict form. + + Returns: + The payload as plain dicts and lists, without the absent keys. + """ payload: dict[str, Any] = { "model_name": self.model_name, "where": list(self.where), @@ -406,16 +542,19 @@ class definition), so only the PK-less case remains to guard. def resolve_join_hops( model_cls: type, path: tuple[str, ...] ) -> tuple[QueryJoinHop, ...]: - """Resolve a relation path into ordered hop facts. + """Turn a relation path into the joins that follow it. - Each hop's ``relation``/``from_column``/``to_table``/``to_column`` is read - from the relation-spec chain starting at ``model_cls`` — ``from_column`` is - the shadow FK column on the source side, ``to_column`` the target's PK. + Args: + model_cls: The model the path starts from. + path: Relation field names, such as `("author",)`. + + Returns: + One step per relation: from the foreign key column on one side to + the primary key of the next table. Raises: - ValueError: If a path element is not a declared relation of the current - model (should not happen — proxies validate at build time), or the - target has no single primary key (:func:`_target_pk_column`). + ValueError: A name in the path is not a relation of the model it is + read from, or a target model has no primary key. """ hops: list[QueryJoinHop] = [] current = model_cls @@ -465,7 +604,7 @@ def _serialize_joins(query: "Query[Any]") -> tuple[QueryJoin, ...]: """Serialize the query's registered relation paths into ``joins`` entries. Emits one entry per registered full path, in insertion order, each - carrying its ordered hop facts (:func:`resolve_join_hops`). The Rust + carrying its ordered hop facts (`resolve_join_hops`). The Rust SELECT walkers assign deterministic ``j{i}_{relation}`` aliases and dedup shared prefixes at render time (#270). @@ -913,30 +1052,22 @@ def _reject_mutation_misuse(query: "Query[Any]", operation: str) -> None: @dataclass(frozen=True) class CompiledQuery: - """One compiled artifact per query — GLOSSARY.md "Compiled query". - - ``wire_json`` and ``hop_classes`` are two views of the same compile: - the map is collected from the hop facts the payload itself carries, so - the wire and the classes the runtime hydrates through can never - disagree (the builder previously re-walked relation-spec chains to - assemble ``hop_classes`` beside the wire, agreeing by hand). - - ``hop_classes`` is plan-scoped, mirroring the Rust ``needs_hop_classes`` - guard exactly (a both-sides double-check, like #272 join edges): ``None`` - unless the materialization plan decodes or hydrates through a hop model's - class — an ``instances`` plan (#286) or a ``record`` plan with traversed - plain fields (#293). When it is a map, it covers every hop the payload - carries, keyed by physical table (one model per table, enforced at - registration). + """A compiled query: its payload, the JSON sent and the related model classes. + + The JSON and the model classes both come from the one payload, so they + always agree. """ payload: QueryIrPayload - """The compiled QueryIR payload.""" + """The query's payload.""" wire_json: str - """The payload serialized as the versioned wire envelope.""" + """The payload serialized as the JSON document sent to the core.""" hop_classes: dict[str, type] | None - """The hop model classes keyed by physical table, or `None` when the plan - decodes through no hop's class.""" + """The related model class for each joined table, keyed by table name. + + `None` unless the results are built through related models: with + `include()`, or a projection that reads a related model's column. + """ secret_values: tuple[object, ...] = () """What each `SecretStr` / `SecretBytes` literal the query compiled with hides, unwrapped, in the order the compile met them; empty when there is @@ -1012,24 +1143,33 @@ def compile_query( assignments: Mapping[str, Any] | None = None, recipe: Mapping[str, Any] | None = None, ) -> CompiledQuery: - """Compile a built query into its wire artifact — the single choke point. - - What each verb carries is policy, stated here once: - - - ``fetch`` carries everything: ordering, paging (``limit``/``offset`` - keys, plus optional ``after`` / ``before`` bounds), m2m context, joins, - and the query's own materialization plan. - - ``count`` is unaffected by ordering, paging, and projection (PRD #277 - verb table): it materializes a scalar, so ordering is dropped, paging - is ``null`` (``after`` / ``before`` omitted), and the plan is ``root_instances`` - even on a projected query — joins and the m2m context still shape - membership. - - ``update``/``delete`` are single-table write shapes: guardrails reject - paging, traversal, and includes (:func:`_reject_mutation_misuse`), and - the payload omits the paging keys entirely. + """Compile a built query into the payload for one operation. + + Each operation carries a different part of the query: + + - `"fetch"` carries everything: sort keys, paging, joins and how the + results are built. + - `"count"` ignores sort keys, paging and any projection, and keeps the + conditions and joins that decide which rows match. + - `"update"` and `"delete"` write one table: paging, relation paths and + `include()` are refused, and the payload leaves out the paging keys. + + Args: + query: The built query. + verb: The operation: `"fetch"`, `"count"`, `"update"` or `"delete"`. + assignments: The literal column values of a keyword `update()`. + Defaults to `None`. + recipe: The column values an `update()` recipe returned. Defaults to + `None`. + + Returns: + The payload, its JSON and the related model classes. Raises: - ValueError: From the mutate guardrails. + ValueError: An update or delete has paging, a relation path or an + `include()`, or a recipe value is not allowed. + TypeError: Both `assignments` and `recipe` are given, or a value + cannot be assigned. """ identity = model_identity(query.model_cls) where = tuple(node.to_ir_dict() for node in query.where_clause) @@ -1103,7 +1243,15 @@ def compile_query( def to_wire_json(payload: QueryIrPayload) -> str: - """Serialize a payload into the versioned IR envelope JSON string.""" + """Serialize a payload as the versioned JSON document sent to the core. + + Args: + payload: The payload to serialize. + + Returns: + The JSON text, with the payload under `"payload"` beside its kind and + version. + """ return json.dumps( { "ir_kind": "query", diff --git a/src/ferro/raw.py b/src/ferro/raw.py index 92d57e93..9177523d 100644 --- a/src/ferro/raw.py +++ b/src/ferro/raw.py @@ -1,22 +1,14 @@ -"""Raw SQL escape hatch — ``execute``, ``fetch_all``, ``fetch_one``. +"""Raw SQL: `execute`, `fetch_all`, `fetch_one` and the `Transaction` handle. -Raw SQL is an escape hatch. Bind values cross the FFI as wire-close primitives, -and rows come back as ``dict[str, str | int | float | bool | bytes | None]``. -UUID / datetime / JSON columns are returned as strings. **If you want typed -rows, use the ORM.** +Bound values are sent as plain primitives, and rows come back as +`dict[str, str | int | float | bool | bytes | None]`: UUID, datetime and JSON +columns come back as strings. For typed rows, use the ORM. -Two interchangeable surfaces, both backed by the same Rust functions: - -* :class:`Transaction` — handle yielded by ``async with transaction() as tx``. - Hard-to-misuse path; ``tx.execute(...)`` cannot be spelled outside the tx - block. -* Top-level :func:`execute` / :func:`fetch_all` / :func:`fetch_one` — auto-pick - up the active tx via the same ``_CURRENT_TRANSACTION`` ContextVar that - ``Model.create()`` uses. Outside any tx, they run on a one-off pool - connection. - -See ``docs/api/raw-sql.md`` for the full bind type table and the Postgres cast -cheat-sheet. +The top-level functions run in the current session, or inside the enclosing +`transaction()` block; the `Transaction` handle a block yields has the same +three methods, bound to its connection. The Raw SQL guide +(`docs/pages/guide/raw-sql.md`) has the bind type table and the Postgres +casts. """ from __future__ import annotations @@ -82,36 +74,54 @@ async def execute( session: Any | None = None, autocommit: bool = False, ) -> int: - """Run a raw SQL statement, returning rows affected. + """Run one SQL statement and return the number of rows it changed. + + It runs inside the enclosing `transaction()` block, if any. Outside one, + two calls may run on different pooled connections; wrap statements that + need one connection (`SET LOCAL`, an advisory lock) in `transaction()`. + See [Raw SQL](../guide/raw-sql.md). - Honors the active ``transaction()`` block via the ``_CURRENT_TRANSACTION`` - ContextVar. Outside any transaction, runs on a one-off pool connection. - Two consecutive top-level ``execute`` calls outside a transaction may use - different pool connections — wrap in ``transaction()`` if you need - connection affinity (e.g. ``SET LOCAL``, advisory locks, ``LISTEN/NOTIFY``). + !!! warning "`autocommit=True` drops the session's settings" + Under the default settings delivery, an `autocommit` statement + carries none of the session's settings, so a row-security policy + sees no tenant while it runs. Use it for maintenance statements, + never for statements that read or write rows a policy protects. Args: - autocommit: Run the statement outside any transaction Ferro would open - for it. A few Postgres statements refuse to run inside a - transaction block — ``CREATE INDEX CONCURRENTLY``, ``VACUUM``, - ``ALTER TYPE ... ADD VALUE`` before PG 12 — and in a session that - carries settings, Ferro otherwise wraps every operation in one, so - they would fail with ``25001``. Set this for those statements:: - - async with engines.session(settings={"myapp.tenant_id": "acme"}): - await execute( - "CREATE INDEX CONCURRENTLY idx_invoice_status " - "ON invoice (status)", - autocommit=True, - ) - - Plainly: an ``autocommit`` statement is **not** tenant-scoped. It - carries none of the session's settings, so a row-security policy - reading ``current_setting(...)`` sees nothing during it. That is the - right trade for maintenance DDL, which is not tenant data; do not - use it for statements that read or write rows a policy governs. - Inside an explicit ``transaction()`` block it changes nothing — you - asked for a transaction, and Postgres will reject the statement. + sql: One SQL statement, with `?` placeholders on SQLite and `$1`, + `$2`, ... on Postgres. + *args: The values bound to the placeholders, in order: `str`, `int`, + `float`, `bool`, `bytes`, `None`, `UUID`, `datetime`, `date`, + `time`, `Decimal`, an `Enum` member, or a `dict` or `list` sent + as JSON text. + using: The connection to run on, outside a session. Defaults to + `None`, the current session's connection. + session: The session to run in. Defaults to `None`, the one opened + by the enclosing `async with engines.session():`. + autocommit: Run the statement outside the transaction Ferro opens for + each operation in a session with settings, for the Postgres + statements that refuse one (`CREATE INDEX CONCURRENTLY`, + `VACUUM`). Defaults to `False`. Inside an explicit + `transaction()` block it changes nothing. See + [Statements that refuse a transaction](../guide/raw-sql.md#statements-that-refuse-a-transaction). + + Returns: + The number of rows the statement changed. + + Raises: + ValueError: `sql` is empty, or `using` names another connection than + the session's or the transaction's. + TypeError: A value in `args` has a type that cannot be bound. + RuntimeError: No session is open and neither `using` nor `session` + was given. + + Examples: + ```python {test=run prelude=reference} + from ferro import execute + + changed = await execute('UPDATE "post" SET "views" = "views" + 1 WHERE "published" = ?', True) + assert changed == 2 + ``` """ _check_sql(sql) marshalled = [_marshal(a) for a in args] @@ -124,10 +134,43 @@ async def execute( async def fetch_all( sql: str, *args: Any, using: str | None = None, session: Any | None = None ) -> list[dict[str, Any]]: - """Run a raw SQL query and return all rows as a list of dicts. + """Run one SQL query and return every row as a dict. - Values are wire-close primitives. UUID/datetime/JSON columns come back as - strings. If you want typed rows, use the ORM. + It runs inside the enclosing `transaction()` block, if any. Values come + back as plain primitives: UUID, datetime and JSON columns as strings. + + Args: + sql: One SQL statement, with `?` placeholders on SQLite and `$1`, + `$2`, ... on Postgres. + *args: The values bound to the placeholders, in order: `str`, `int`, + `float`, `bool`, `bytes`, `None`, `UUID`, `datetime`, `date`, + `time`, `Decimal`, an `Enum` member, or a `dict` or `list` sent + as JSON text. + using: The connection to run on, outside a session. Defaults to + `None`, the current session's connection. + session: The session to run in. Defaults to `None`, the one opened + by the enclosing `async with engines.session():`. + + Returns: + One dict per row, keyed by column name, in the query's order. + + Raises: + ValueError: `sql` is empty, or `using` names another connection than + the session's or the transaction's. + TypeError: A value in `args` has a type that cannot be bound. + RuntimeError: No session is open and neither `using` nor `session` + was given. + + Examples: + ```python {test=run prelude=reference} + from ferro import fetch_all + + rows = await fetch_all('SELECT "title", "views" FROM "post" WHERE "views" > ? ORDER BY "views"', 900) + assert rows == [ + {"title": "The Education of a Computer", "views": 952}, + {"title": "Notes on the Analytical Engine", "views": 1843}, + ] + ``` """ _check_sql(sql) marshalled = [_marshal(a) for a in args] @@ -140,9 +183,42 @@ async def fetch_all( async def fetch_one( sql: str, *args: Any, using: str | None = None, session: Any | None = None ) -> dict[str, Any] | None: - """Run a raw SQL query and return the first row as a dict, or ``None``. + """Run one SQL query and return its first row as a dict, or `None`. - Callers should ``LIMIT 1`` if the query may return more than one row. + It runs inside the enclosing `transaction()` block, if any. Ferro does + not add `LIMIT 1`; write it when more than one row can match, so the + database stops at the first. + + Args: + sql: One SQL statement, with `?` placeholders on SQLite and `$1`, + `$2`, ... on Postgres. + *args: The values bound to the placeholders, in order: `str`, `int`, + `float`, `bool`, `bytes`, `None`, `UUID`, `datetime`, `date`, + `time`, `Decimal`, an `Enum` member, or a `dict` or `list` sent + as JSON text. + using: The connection to run on, outside a session. Defaults to + `None`, the current session's connection. + session: The session to run in. Defaults to `None`, the one opened + by the enclosing `async with engines.session():`. + + Returns: + The first row, keyed by column name, or `None` when no row matched. + + Raises: + ValueError: `sql` is empty, or `using` names another connection than + the session's or the transaction's. + TypeError: A value in `args` has a type that cannot be bound. + RuntimeError: No session is open and neither `using` nor `session` + was given. + + Examples: + ```python {test=run prelude=reference} + from ferro import fetch_one + + row = await fetch_one('SELECT "name" FROM "author" WHERE "email" = ?', "ada@example.com") + assert row == {"name": "Ada Lovelace"} + assert await fetch_one('SELECT "name" FROM "author" WHERE "id" = ?', 99) is None + ``` """ _check_sql(sql) marshalled = [_marshal(a) for a in args] @@ -153,17 +229,22 @@ async def fetch_one( class Transaction: - """Handle for a live transaction. - - Obtained via ``async with transaction() as tx``. Methods delegate to the - top-level :func:`execute` / :func:`fetch_all` / :func:`fetch_one` with this - transaction's route (a :class:`RouteHandle` built by - ``models.transaction()``) passed explicitly, so they don't depend on the - ContextVar state. This makes the connection-affinity invariant - structurally impossible to violate from inside the ``async with`` block. - - The handle becomes invalid once the ``async with`` block exits — any - subsequent call raises :class:`RuntimeError`. + """The handle `async with transaction() as tx` yields, for raw SQL on its connection. + + Its methods take the same `sql` and `*args` as the top-level + [`execute`][ferro.execute], [`fetch_all`][ferro.fetch_all] and + [`fetch_one`][ferro.fetch_one], and always run on the transaction's + connection. After the block exits, every call raises `RuntimeError`. + + Examples: + ```python {test=run prelude=reference} + from ferro import transaction + + async with transaction() as tx: + await tx.execute('UPDATE "post" SET "views" = 0 WHERE "id" = ?', 1) + row = await tx.fetch_one('SELECT "views" FROM "post" WHERE "id" = ?', 1) + assert row == {"views": 0} + ``` """ __slots__ = ("_route",) @@ -172,6 +253,21 @@ def __init__(self, route: RouteHandle) -> None: self._route = route async def execute(self, sql: str, *args: Any) -> int: + """Run one SQL statement on this transaction's connection. + + Args: + sql: One SQL statement, with `?` placeholders on SQLite and `$1`, + `$2`, ... on Postgres. + *args: The values bound to the placeholders, in order. + + Returns: + The number of rows the statement changed. + + Raises: + ValueError: `sql` is empty. + TypeError: A value in `args` has a type that cannot be bound. + RuntimeError: The transaction's block has exited. + """ _check_sql(sql) marshalled = [_marshal(a) for a in args] return await _raw_execute( @@ -179,6 +275,21 @@ async def execute(self, sql: str, *args: Any) -> int: ) async def fetch_all(self, sql: str, *args: Any) -> list[dict[str, Any]]: + """Run one SQL query on this transaction's connection and return every row. + + Args: + sql: One SQL query, with `?` placeholders on SQLite and `$1`, + `$2`, ... on Postgres. + *args: The values bound to the placeholders, in order. + + Returns: + One dict per row, keyed by column name. + + Raises: + ValueError: `sql` is empty. + TypeError: A value in `args` has a type that cannot be bound. + RuntimeError: The transaction's block has exited. + """ _check_sql(sql) marshalled = [_marshal(a) for a in args] return await _raw_fetch_all( @@ -186,6 +297,21 @@ async def fetch_all(self, sql: str, *args: Any) -> list[dict[str, Any]]: ) async def fetch_one(self, sql: str, *args: Any) -> dict[str, Any] | None: + """Run one SQL query on this transaction's connection and return its first row. + + Args: + sql: One SQL query, with `?` placeholders on SQLite and `$1`, + `$2`, ... on Postgres. + *args: The values bound to the placeholders, in order. + + Returns: + The first row, keyed by column name, or `None` when no row matched. + + Raises: + ValueError: `sql` is empty. + TypeError: A value in `args` has a type that cannot be bound. + RuntimeError: The transaction's block has exited. + """ _check_sql(sql) marshalled = [_marshal(a) for a in args] return await _raw_fetch_one( diff --git a/src/ferro/registry.py b/src/ferro/registry.py index bfd4c557..91413887 100644 --- a/src/ferro/registry.py +++ b/src/ferro/registry.py @@ -1,31 +1,32 @@ """The registry: single owner of Ferro's Python-side registration state. -One :class:`Registry` instance (:data:`REGISTRY` — process-global by nature: +One `Registry` instance (`REGISTRY` — process-global by nature: the metaclass writes to it at import time) owns the model registry, per-model SchemaIR envelope + fingerprint caches, join-table bundles, pending relations, the modelset artifact, and the provisional/resolved generation -counters (GLOSSARY.md "Provisional registration" / "Resolved registration"). +counters (see "Provisional registration" and "Resolved registration" in +`GLOSSARY.md`). The stores are private; every reader and writer goes through the interface, so their agreement invariants have exactly one home: - A stored fingerprint is always a pure function of the envelope it - accompanies (:meth:`Registry.persist_envelope` / :meth:`Registry.set_modelset` + accompanies (`Registry.persist_envelope` / `Registry.set_modelset` compute it — no caller can persist a mismatched pair). - Evicting join tables evicts their envelopes with them - (:meth:`Registry.clear_join_tables` — previously a hand-written loop in + (`Registry.clear_join_tables` — previously a hand-written loop in ``clear_registry`` guarded by a warning comment). - A test reset or snapshot/restore covers every store and counter in one - call (:meth:`Registry.reset_for_test`, :meth:`Registry.snapshot`). + call (`Registry.reset_for_test`, `Registry.snapshot`). - A modelset that must never mix with today's (a data step's historical - models) is built against an empty registry (:meth:`Registry.capture`) and - installed for a block (:meth:`Registry.swap`), which restores today's on + models) is built against an empty registry (`Registry.capture`) and + installed for a block (`Registry.swap`), which restores today's on exit, error and cancellation. Ordering caveat (unchanged from the pre-Registry design, closed by a future atomic register→persist step): defining model classes concurrently with ``connect``/``create_tables``/``migrate`` is unsupported — a registry entry -becomes visible on :meth:`register` before its envelope is persisted, so an +becomes visible on `register` before its envelope is persisted, so an unlocked reader can observe a transient missing-envelope state. """ @@ -42,12 +43,12 @@ class FerroModelClass(Protocol): """Structural type for a registrable Ferro model class. - The registry keys by ``__ferro_identity__`` (what :meth:`Registry.register` + The registry keys by ``__ferro_identity__`` (what `Registry.register` derives its key from) and resolves bare references against ``__name__`` / ``__qualname__``. Typed as a ``Protocol`` rather than ``type[Model]`` so ``registry`` — which ``models`` and ``metaclass`` import — does not import back into them (circular import); a non-Ferro class handed to - :meth:`Registry.register` is then a type error at the call site instead of + `Registry.register` is then a type error at the call site instead of a raw ``AttributeError`` at runtime. """ @@ -60,38 +61,60 @@ class FerroModelClass(Protocol): def canonical_ir_json(value: dict[str, Any]) -> str: - """Serialize an IR artifact with deterministic key ordering.""" + """Serialize a schema artifact as JSON with a deterministic key order. + + Args: + value: The schema artifact. + + Returns: + Compact JSON with sorted keys, ASCII only. + """ return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=True) def ir_fingerprint(value: dict[str, Any]) -> str: - """Return the canonical SHA-256 fingerprint for an IR artifact. + """Return the canonical SHA-256 fingerprint of a schema artifact. + + The single source of the fingerprint function, so a stored fingerprint + is always a pure function of the envelope it accompanies: the + fingerprint gate reads these, and a mismatched pair would serve a stale + schema silently. - Single source of the fingerprint function so a stored fingerprint is - always a pure function of the envelope it accompanies (the ADR - fingerprint gate reads these; a mismatched pair would serve stale schema - silently). + Args: + value: The schema artifact. + + Returns: + The hex SHA-256 of its canonical JSON. """ return hashlib.sha256(canonical_ir_json(value).encode("utf-8")).hexdigest() @dataclass(frozen=True) class RegistrySnapshot: - """Opaque copy of every Registry store and counter (test fixtures).""" + """An opaque copy of every `Registry` store and counter, for test fixtures.""" models: dict[str, FerroModelClass] + """The registered models, keyed by canonical identity.""" envelopes: dict[str, dict[str, Any]] + """The compiled schema envelopes, keyed by model identity or join-table name.""" fingerprints: dict[str, str] + """The fingerprint of each envelope, under the same keys.""" join_tables: dict[str, Any] + """The join-table bundles, keyed by table name.""" pending_relations: list[Any] = field(repr=False) + """The relation declarations waiting for resolution.""" modelset: dict[str, Any] | None + """The assembled modelset envelope, or `None` if never built.""" modelset_fingerprint: str | None + """The modelset's fingerprint, or `None` if never built.""" registration_generation: int + """The registration generation counter.""" resolved_generation: int + """The generation last cleared by relationship resolution.""" class SwappableModelset(Protocol): - """What :meth:`Registry.swap` installs (``ferro.migrations.context.HistoricalModels``).""" + """What `Registry.swap` installs: a data step's historical models.""" @property def registry_state(self) -> RegistrySnapshot: @@ -99,14 +122,22 @@ def registry_state(self) -> RegistrySnapshot: ... def unreachable(self, cls: FerroModelClass) -> str: - """Why today's ``cls`` cannot be queried while the modelset is installed.""" + """Return why today's `cls` cannot be queried while the modelset is installed. + + Args: + cls: A model class of today's codebase. + + Returns: + The error text its entry points raise. + """ ... class SwappedOutModelError(RuntimeError): - """A model class from the codebase was queried while :meth:`Registry.swap` - had a different modelset installed (inside a data step: use - ``ctx.models.<Name>``).""" + """A model class from the codebase was queried while another modelset was installed. + + Inside a data step, query `ctx.models.<Name>` instead. + """ class _SwappedOut: @@ -137,7 +168,7 @@ def _entry_points() -> list[str]: def _guard(cls: FerroModelClass, text: str) -> list[tuple[FerroModelClass, str, Any]]: - """Shadow every entry point of ``cls`` with a :class:`_SwappedOut`; + """Shadow every entry point of ``cls`` with a `_SwappedOut`; return what to put back.""" placed: list[tuple[FerroModelClass, str, Any]] = [] for name in _entry_points(): @@ -158,7 +189,7 @@ def _install_in_rust(state: RegistrySnapshot) -> None: class Registry: - """Single owner of the registration stores — see module docstring.""" + """The single owner of the registration stores; see the module docstring.""" def __init__(self) -> None: self._models: dict[str, FerroModelClass] = {} @@ -179,10 +210,13 @@ def register(self, cls: FerroModelClass) -> None: """Record a model class (provisional registration). The key is the model's canonical identity (``__ferro_identity__``), - derived from ``cls`` here rather than caller-supplied (#249): the - registry key is a pure function of the model, so every emitter — the - Rust runtime registry, the raw-FFI operations — keys the same model - by the same name. Idempotent by construction. + derived from ``cls`` here rather than caller-supplied: the registry + key is a pure function of the model, so every emitter (the Rust + runtime registry, the raw operations) keys the same model by the same + name. Idempotent by construction. + + Args: + cls: The model class. """ self._models[cls.__ferro_identity__] = cls self._registration_generation += 1 @@ -191,8 +225,11 @@ def deregister(self, name: str) -> None: """Evict one model from every per-model store (registry + envelopes). Takes a ``name`` (the canonical identity) rather than a class, unlike - :meth:`register` — teardown paths iterate registry keys and hold the + `register` — teardown paths iterate registry keys and hold the identity string, not the class object. Idempotent. + + Args: + name: The model's canonical identity. """ removed = name in self._models self._models.pop(name, None) @@ -201,23 +238,37 @@ def deregister(self, name: str) -> None: self._registration_generation += 1 def models(self) -> dict[str, FerroModelClass]: - """The registered models, keyed by canonical identity. + """Return the registered models, keyed by canonical identity. - A read view: callers iterate, look up, and pass it on (e.g. shadow-FK - reconciliation) — they never mutate it. Mutation goes through - :meth:`register` / :meth:`deregister`, which own the generation + A read view: callers iterate, look up, and pass it on (shadow + foreign-key reconciliation, say) and never mutate it. Mutation goes + through `register` and `deregister`, which own the generation counter bump. + + Returns: + The live mapping of identity to model class. """ return self._models def resolve_reference(self, ref: str, *, default: Any = _UNSET) -> Any: - """Resolve a model reference to a registered model class (FF-E E1). + """Resolve a model reference to a registered model class. + + A qualified reference is a direct hit; a bare name matching exactly + one registered model resolves to it. + + Args: + ref: A qualified identity (`module.QualName`) or a bare class + name. + default: What to return when nothing matches. When not given, + no match raises. - Accepts a qualified identity (``module.QualName``) or a bare class - name. A qualified reference is a direct hit; a bare name matching - exactly one registered model resolves to it; several matches raise - with the qualified candidates listed; no match raises (or returns - ``default`` when given). Ambiguity always raises. + Returns: + The model class, or `default` when nothing matches. + + Raises: + RuntimeError: A bare name matches several models (always, with + the qualified candidates listed), or nothing matches and no + `default` was given. """ model = self._models.get(ref) if model is not None: @@ -242,34 +293,63 @@ def resolve_reference(self, ref: str, *, default: Any = _UNSET) -> Any: # -- per-model envelopes ----------------------------------------------- def persist_envelope(self, name: str, envelope: dict[str, Any]) -> None: - """Store one model's (or join table's) compiled SchemaIR envelope. + """Store one model's (or join table's) compiled schema envelope. + + The fingerprint is computed here from the envelope, a pure function + of it, so no caller can persist a stale envelope and fingerprint + pair. - The fingerprint is computed here from the envelope — a pure function - of it, so no caller can persist a stale envelope/fingerprint pair. - Keyed by the same ``name`` the IR compiler uses (a model's - ``__ferro_identity__`` or a join-table name). + Args: + name: The key the schema compiler uses: a model's + `__ferro_identity__` or a join table's name. + envelope: The compiled envelope. """ self._envelopes[name] = envelope self._fingerprints[name] = ir_fingerprint(envelope) def evict_envelope(self, name: str) -> None: - """Evict one entry from the envelope + fingerprint caches only.""" + """Evict one entry from the envelope and fingerprint caches only. + + Args: + name: The model identity or join-table name. + """ self._envelopes.pop(name, None) self._fingerprints.pop(name, None) def envelope(self, name: str) -> dict[str, Any] | None: - """One compiled envelope, or None — a live reference, not a copy - (the modelset assembler reads it in place).""" + """Return one compiled envelope, as a live reference, not a copy. + + The modelset assembler reads it in place. + + Args: + name: The model identity or join-table name. + + Returns: + The envelope, or `None` when none is stored. + """ return self._envelopes.get(name) def fingerprint(self, name: str) -> str | None: - """The stored fingerprint paired with :meth:`envelope`.""" + """Return the stored fingerprint paired with `envelope`. + + Args: + name: The model identity or join-table name. + + Returns: + The fingerprint, or `None` when none is stored. + """ return self._fingerprints.get(name) def missing_envelope_names(self) -> list[str]: - """Registered names whose envelope is absent — the store-agreement - arithmetic (registry keys and dict-bundle join tables minus envelope - keys). Non-empty means a cold-rehydrate must recompile first.""" + """Return the registered names whose envelope is absent. + + It is the store-agreement arithmetic: registry keys and dict-bundle + join tables, minus envelope keys. A non-empty result means a cold + rehydrate must recompile first. + + Returns: + The model identities and join-table names with no envelope. + """ missing = [name for name in self._models if name not in self._envelopes] missing.extend( table_name @@ -281,11 +361,20 @@ def missing_envelope_names(self) -> list[str]: # -- join tables --------------------------------------------------------- def add_join_table(self, name: str, bundle: Any) -> None: - """Record one auto-generated join table's bundle.""" + """Record one generated join table's bundle. + + Args: + name: The join table's name. + bundle: What the schema compiler needs to build its envelope. + """ self._join_tables[name] = bundle def join_tables(self) -> dict[str, Any]: - """The join-table bundles, keyed by table name (read view).""" + """Return the join-table bundles, keyed by table name, as a read view. + + Returns: + The live mapping of table name to bundle. + """ return self._join_tables def clear_join_tables(self) -> None: @@ -293,8 +382,7 @@ def clear_join_tables(self) -> None: Join tables never live in the model registry, so this touches nothing a declared model needs to cold-rehydrate; leaving their envelopes - behind would resurrect stale join-table DDL on the next assemble - (the #153 guard depends on these stores agreeing). + behind would resurrect stale join-table DDL on the next assemble. """ for name in list(self._join_tables): self.evict_envelope(name) @@ -303,11 +391,24 @@ def clear_join_tables(self) -> None: # -- pending relations --------------------------------------------------- def defer_relation(self, identity: str, field_name: str, metadata: Any) -> None: - """Queue one relationship declaration for resolution (metaclass).""" + """Queue one relation declaration for resolution; the metaclass calls it. + + Args: + identity: The declaring model's canonical identity. + field_name: The relation field's name. + metadata: The relation's declaration (`ForeignKey`, + `ManyToManyRelation` or `"BackRef"`). + """ self._pending_relations.append((identity, field_name, metadata)) def drain_pending_relations(self) -> list[Any]: - """Return-and-clear the pending relation queue (resolution pass).""" + """Return the pending relation declarations and clear the queue. + + Relationship resolution calls it. + + Returns: + The queued `(identity, field_name, metadata)` entries, in order. + """ drained = list(self._pending_relations) self._pending_relations.clear() return drained @@ -315,30 +416,48 @@ def drain_pending_relations(self) -> list[Any]: # -- modelset artifact ----------------------------------------------------- def set_modelset(self, envelope: dict[str, Any]) -> None: - """Cache the assembled modelset envelope; fingerprint computed here.""" + """Cache the assembled modelset envelope and compute its fingerprint. + + Args: + envelope: The assembled modelset. + """ self._modelset = envelope self._modelset_fingerprint = ir_fingerprint(envelope) def modelset(self) -> tuple[dict[str, Any], str] | None: - """The cached (envelope, fingerprint) pair, or None if never built.""" + """Return the cached modelset and its fingerprint. + + Returns: + The `(envelope, fingerprint)` pair, or `None` if never built. + """ if self._modelset is None or self._modelset_fingerprint is None: return None return self._modelset, self._modelset_fingerprint def clear_modelset(self) -> None: - """Drop the cached modelset pair, forcing the next assemble to - rebuild it (per-model envelopes are untouched).""" + """Drop the cached modelset, so the next assemble rebuilds it. + + Per-model envelopes are untouched. + """ self._modelset = None self._modelset_fingerprint = None # -- generations ----------------------------------------------------------- def registration_generation(self) -> int: - """Current registration generation counter (test/diagnostic).""" + """Return the registration generation counter, for tests and diagnostics. + + Returns: + The counter; each `register` and `deregister` bumps it. + """ return self._registration_generation def resolved_generation(self) -> int: - """The generation last cleared by relationship resolution.""" + """Return the generation last cleared by relationship resolution. + + Returns: + The registration generation at the last resolution. + """ return self._resolved_generation def mark_resolved(self) -> None: @@ -346,8 +465,12 @@ def mark_resolved(self) -> None: self._resolved_generation = self._registration_generation def is_dirty(self) -> bool: - """True when the registry changed since last resolve or relations - are pending.""" + """Return whether the registry needs resolving again. + + Returns: + `True` when a model registered or deregistered since the last + resolution, or relations are pending. + """ return self._registration_generation != self._resolved_generation or bool( self._pending_relations ) @@ -355,14 +478,19 @@ def is_dirty(self) -> bool: # -- scoped modelsets ----------------------------------------------------------- def capture[T](self, define: Callable[[], T]) -> tuple[T, RegistrySnapshot]: - """Run ``define`` against an empty registry and return its result with - the registry state it left, restoring this registry's own state - whatever happens. - - How a modelset that must never mix with today's is built: classes - ``define`` creates register (through the metaclass) into the empty - registry, and the returned :class:`RegistrySnapshot` holds exactly - them, ready for :meth:`swap`. + """Run `define` against an empty registry and capture what it registers. + + This is how a modelset that must never mix with today's is built: + the classes `define` creates register, through the metaclass, into + the empty registry. This registry's own state is restored whatever + happens. + + Args: + define: A callable that declares the modelset's classes. + + Returns: + What `define` returned, and a `RegistrySnapshot` holding exactly + the classes it registered, ready for `swap`. """ today = self.snapshot() try: @@ -374,16 +502,24 @@ def capture[T](self, define: Callable[[], T]) -> tuple[T, RegistrySnapshot]: @contextmanager def swap(self, modelset: SwappableModelset) -> Iterator[None]: - """Install ``modelset`` as the registry, in Python and in the Rust - runtime, for the block; restore today's on exit, error and - cancellation. + """Install `modelset` as the registry for the block, restoring today's after. + + It is installed in Python and in the Rust runtime, and today's comes + back on exit, error and cancellation. While it is installed, every query and persistence entry point of today's model classes (``Author.where``, ``author.save``, ...) raises - :class:`SwappedOutModelError` with ``modelset.unreachable(cls)``, so + `SwappedOutModelError` with ``modelset.unreachable(cls)``, so a data step can never query today's table shape by mistake. The Rust side is installed through the fingerprint gate, once on entry and once on exit. + + Args: + modelset: The modelset to install, a data step's historical + models. + + Yields: + Nothing; the modelset is installed while the block runs. """ today = self.snapshot() guarded: list[tuple[FerroModelClass, str, Any]] = [] @@ -405,7 +541,11 @@ def swap(self, modelset: SwappableModelset) -> Iterator[None]: # -- test surface ------------------------------------------------------------ def snapshot(self) -> RegistrySnapshot: - """Copy every store and counter — pair with :meth:`restore`.""" + """Copy every store and counter, for `restore` to put back. + + Returns: + The copy. + """ return RegistrySnapshot( models=dict(self._models), envelopes=dict(self._envelopes), @@ -419,7 +559,11 @@ def snapshot(self) -> RegistrySnapshot: ) def restore(self, snap: RegistrySnapshot) -> None: - """Replace every store and counter with the snapshot's contents.""" + """Replace every store and counter with the snapshot's contents. + + Args: + snap: A copy from `snapshot` or `capture`. + """ self._models.clear() self._models.update(snap.models) self._envelopes.clear() @@ -449,3 +593,4 @@ def reset_for_test(self) -> None: REGISTRY = Registry() +"""The process-wide registry the metaclass registers every model class in.""" diff --git a/src/ferro/relations/__init__.py b/src/ferro/relations/__init__.py index b78b3eee..8d94f411 100644 --- a/src/ferro/relations/__init__.py +++ b/src/ferro/relations/__init__.py @@ -1,3 +1,5 @@ +"""Relation resolution: binding each declared relation to its target model.""" + from typing import ForwardRef from .._shadow_fk_types import pk_python_type_for_model, reconcile_shadow_fk_types @@ -44,12 +46,19 @@ def _join_table_rename_hints( def resolve_relationships(): - """Finalize all pending relationships and cross-validate. - - After binding each ``ForeignKey.to`` to a concrete model, upgrades shadow - ``{name}_id`` Pydantic annotations from the forward-ref fallback to the related - model's PK type where applicable, then ``model_rebuild``s affected classes before - the schema re-registration pass. + """Bind every declared relation to its target model and check the pair. + + Ferro calls this on `connect()` and before creating tables. Each + foreign key's `<field>_id` column takes the type of the target's primary + key, and the affected models are rebuilt and registered again. + + Raises: + RuntimeError: A relation names a model that is not defined, the + target has no field named by `related_name`, a many-to-many join + table's name is another model's table, or a model's schema fails + to rebuild. + TypeError: A model's declaration is invalid once its relations are + known. """ # Drain (copy-and-clear) so that we don't process the same relations # multiple times if resolve_relationships is called again (e.g. in tests) diff --git a/src/ferro/relations/descriptors.py b/src/ferro/relations/descriptors.py index 80784724..ee9c3fe5 100644 --- a/src/ferro/relations/descriptors.py +++ b/src/ferro/relations/descriptors.py @@ -1,3 +1,5 @@ +"""The attributes that read a relation from a model instance.""" + from __future__ import annotations from typing import TYPE_CHECKING @@ -20,15 +22,28 @@ def _instance_origin_outside_transaction(instance: object) -> str | None: class RelationshipDescriptor(BaseModel): - """Descriptor that returns either a Query object or a single object (for 1:1).""" + """The attribute behind a reverse (`BackRef`) or many-to-many relation. + + On an instance, a to-many relation reads as a + [`Relation`][ferro.Relation] query over the related rows, and a + one-to-one reverse relation as an awaitable of the related instance or + `None`. See [Relationships](../guide/relationships.md). + """ target_model_name: str + """The related model's name.""" field_name: str + """The name of the related model's foreign key field that points back.""" is_one_to_one: bool = False + """Whether at most one related row exists.""" is_m2m: bool = False + """Whether the relation goes through a join table.""" join_table: str | None = None + """The join table, for a many-to-many relation.""" source_col: str | None = None + """The join table column that points at this model.""" target_col: str | None = None + """The join table column that points at the related model.""" _target_model: Model | None = None def __get__(self, instance, owner): @@ -60,7 +75,7 @@ def __get__(self, instance, owner): fk_field = f"{self.field_name}_id" if self.is_one_to_one: return self._target_model.where( - lambda t, f=fk_field, v=pk_val: getattr(t, f) == v + lambda related, f=fk_field, v=pk_val: getattr(related, f) == v ).first() from ..query.builder import Relation @@ -68,14 +83,21 @@ def __get__(self, instance, owner): return Relation( self._target_model, using=_instance_origin_outside_transaction(instance), - ).where(lambda t, f=fk_field, v=pk_val: getattr(t, f) == v) + ).where(lambda related, f=fk_field, v=pk_val: getattr(related, f) == v) class ForwardDescriptor(BaseModel): - """Descriptor that handles lazy loading of a related object.""" + """The attribute behind a foreign key relation, loaded when awaited. + + On an instance, the relation reads as an awaitable of the related + instance, or `None` when the foreign key is empty. A query with + `include()` fills the relation in instead, as a plain attribute. + """ target_model_name: str + """The related model's name.""" field_name: str + """The relation's field name; its foreign key column is `<field_name>_id`.""" _target_model: Model | None = None def __get__(self, instance, owner): diff --git a/src/ferro/rowsecurity.py b/src/ferro/rowsecurity.py index e3719e71..12ea49f4 100644 --- a/src/ferro/rowsecurity.py +++ b/src/ferro/rowsecurity.py @@ -1,34 +1,37 @@ -"""Row security declared on Ferro models (PRD #406). +"""Row security declared on Ferro models. -A model says once who is allowed to see its rows:: +A model says once who is allowed to see its rows: - class Invoice(Model): - id: int | None = Field(default=None, primary_key=True) - tenant_id: UUID - amount: float +```python {test=compile reason="a model fragment; its imports are left out"} +class Invoice(Model): + id: int | None = Field(default=None, primary_key=True) + tenant_id: UUID + amount: float - __ferro_rls__: ClassVar = RowSecurity( - RowPolicy(column="tenant_id", setting="app.tenant_id") - ) - -and ferro creates the table with row-level security switched on and the -matching policy:: - - ALTER TABLE "invoice" ENABLE ROW LEVEL SECURITY - ALTER TABLE "invoice" FORCE ROW LEVEL SECURITY - CREATE POLICY "rls_invoice_tenant_id" ON "invoice" FOR ALL - USING ("tenant_id" = NULLIF(current_setting('app.tenant_id', true), '')::uuid) - WITH CHECK ("tenant_id" = NULLIF(current_setting('app.tenant_id', true), '')::uuid) - -From then on the database itself decides which rows a query can see: a -connection whose ``app.tenant_id`` setting is unset sees **no** rows, and one -that carries a tenant id sees only that tenant's rows — a forgotten ``where`` -filter is no longer a data leak. - -This module owns the declaration surface and the validation that runs at class -definition. Every SQL decision — the policy name, the ``NULLIF`` expression, -the ``CREATE POLICY`` statement — lives in ``crates/ferro-ddl-lowering`` so -each emitter renders the same bytes (AGENTS.md § I-1). + __ferro_rls__: ClassVar = RowSecurity( + RowPolicy(column="tenant_id", setting="app.tenant_id") + ) +``` + +and Ferro creates the table with row-level security switched on and the +matching policy: + +```sql +ALTER TABLE "invoice" ENABLE ROW LEVEL SECURITY +ALTER TABLE "invoice" FORCE ROW LEVEL SECURITY +CREATE POLICY "rls_invoice_tenant_id" ON "invoice" FOR ALL + USING ("tenant_id" = NULLIF(current_setting('app.tenant_id', true), '')::uuid) + WITH CHECK ("tenant_id" = NULLIF(current_setting('app.tenant_id', true), '')::uuid) +``` + +From then on the database decides which rows a query can see: a connection +whose `app.tenant_id` setting is unset sees no rows, and one that carries a +tenant id sees only that tenant's rows. + +This module owns the declaration surface and the validation that runs at +class definition. Every SQL decision (the policy name, the `NULLIF` +expression, the `CREATE POLICY` statement) is made in the Rust core, so +every emitter renders the same bytes (AGENTS.md I-1). """ from __future__ import annotations @@ -42,6 +45,7 @@ class Invoice(Model): from ._core import _ddl_row_policy_name, _rls_command_matrix, _rls_shorthand_cast FERRO_RLS = "__ferro_rls__" +"""The class attribute a model declares its row security in.""" #: Which clauses Postgres accepts per command, read once from the shared #: lowering layer. There is no second copy of this table here on purpose: the @@ -51,8 +55,8 @@ class Invoice(Model): row["command"]: row for row in json.loads(_rls_command_matrix()) } -#: The commands a policy may be scoped to (`FOR <command>`), in Rust's order. COMMANDS = tuple(_COMMAND_CLAUSES) +"""The commands a policy may cover (`FOR <command>`).""" #: A custom Postgres setting key: dotted identifiers, nothing else. Ferro's #: renderer already doubles single quotes when it inlines the key, but that @@ -70,55 +74,75 @@ class Invoice(Model): @dataclass(frozen=True) class RowPolicy: - """One row policy: which rows a command may touch. + """One Postgres row policy: which rows a command may read or write. - Two forms. The **shorthand** compares a column to a session setting:: + It comes in two forms. The shorthand, `RowPolicy(column=..., setting=...)`, + compares a column with a session setting and renders, for both `USING` + and `WITH CHECK`: - RowPolicy(column="tenant_id", setting="app.tenant_id") + ```sql + "tenant_id" = NULLIF(current_setting('app.tenant_id', true), '')::uuid + ``` + + The cast follows the column's type: `uuid`, `text`, `varchar` and the + integer types are supported, and any other type is refused when the + class is defined. The `NULLIF` makes an unset or reset setting match no + rows instead of raising a cast error; see + [Policy expressions](../concepts/row-level-security.md#policy-expressions). + The raw form, `RowPolicy(name=..., using=...)`, takes the SQL expressions + as written and needs a name. - which renders ``"tenant_id" = NULLIF(current_setting('app.tenant_id', - true), '')::uuid`` for both ``USING`` and ``WITH CHECK``. The cast comes - from the column's own storage type; ``uuid``, ``text``/``varchar`` and the - integer families are supported, and anything else is a class-definition - error pointing here, at the raw form. The ``NULLIF`` is what makes an unset - (or reset) setting mean *no rows* instead of a cast error. + Args: + name: The end of the policy's name, `rls_<table>_<name>`: a lowercase + identifier. Defaults to `None`, the column's name in the + shorthand; required in the raw form. + command: The command the policy covers: `"all"` (the default), + `"select"`, `"insert"`, `"update"` or `"delete"`. + restrictive: Make the policy `AS RESTRICTIVE`, so it must hold + alongside every other policy instead of being one alternative. + Defaults to `False`. + column: Shorthand: the model column holding the scope value. + setting: Shorthand: the dotted Postgres setting to compare it with, + such as `"app.tenant_id"`. + using: Raw: the `USING` expression, which decides the rows a command + can see. + with_check: Raw: the `WITH CHECK` expression, which decides the rows + a write may leave behind. - The **raw** form takes SQL and requires a name:: + Raises: + TypeError: The two forms are mixed or neither is given, a clause is + one the command does not accept, or a name, command or flag has + the wrong shape. + ValueError: `setting` is not a dotted custom setting name. - RowPolicy( + Examples: + ```python {test=run prelude=reference} + from ferro import RowPolicy + + tenant = RowPolicy(column="tenant_id", setting="app.tenant_id") + invitee = RowPolicy( name="invitee_read", command="select", - using="id IN (SELECT doc_id FROM membership WHERE ...)", + using="id IN (SELECT document_id FROM membership)", ) - - Args: - name: Policy name suffix; the live policy is ``rls_<table>_<name>``. - Defaults to the column name in the shorthand form; required in the - raw form. - command: ``"all"`` (default), ``"select"``, ``"insert"``, ``"update"`` - or ``"delete"``. - restrictive: ``True`` makes the policy AND-compose with every other - policy (``AS RESTRICTIVE``) instead of OR-composing. - column: Shorthand — the model column carrying the scope value. - setting: Shorthand — the Postgres setting (GUC) key to compare it to. - using: Raw — the ``USING`` expression, deciding which rows are visible. - with_check: Raw — the ``WITH CHECK`` expression, deciding which rows - may be written. - - Examples: - >>> RowPolicy(column="tenant_id", setting="app.tenant_id") - RowPolicy(name='tenant_id', command='all') - >>> RowPolicy(name="owner_all", command="update", using="true") - RowPolicy(name='owner_all', command='update') + assert (tenant.resolved_name, invitee.resolved_name) == ("tenant_id", "invitee_read") + ``` """ name: str | None = None + """The policy name's suffix, or `None` for the shorthand's column name.""" command: str = "all" + """The command the policy covers.""" restrictive: bool = False + """Whether the policy is `AS RESTRICTIVE`.""" column: str | None = None + """Shorthand: the column holding the scope value.""" setting: str | None = None + """Shorthand: the Postgres setting the column is compared with.""" using: str | None = None + """Raw: the `USING` expression.""" with_check: str | None = None + """Raw: the `WITH CHECK` expression.""" def __post_init__(self) -> None: self._validate_command() @@ -239,7 +263,7 @@ def _validate_name(self) -> None: @property def resolved_name(self) -> str: - """The policy's name suffix — ``name``, or the column in the shorthand.""" + """The policy name's suffix: `name`, or the column in the shorthand.""" if self.name is not None: return self.name return self.column # type: ignore[return-value] @@ -250,20 +274,58 @@ def __repr__(self) -> str: @dataclass(frozen=True) class RowSecurity: - """A model's row security declaration: its policies plus the table flags. + """A model's row security: its policies and the table's flags. + + Assign it to the model's `__ferro_rls__`. On Postgres the table gets + `ENABLE ROW LEVEL SECURITY` and one `CREATE POLICY` per policy; on + SQLite, which has no row security, Ferro warns and creates the table + without them. See + [Scope rows by tenant](../guide/row-level-security.md) for the setup. - ``force=True`` (the default) emits ``FORCE ROW LEVEL SECURITY`` as well as - ``ENABLE``, so the table's owner is filtered too. Without it a deployment - that connects as the table owner — the common single-role setup — gets - policies that are never consulted. + Args: + *policies: The table's policies. + force: Also emit `FORCE ROW LEVEL SECURITY`, so the policies apply to + the table's owner too. Defaults to `True`; without it, an + application that connects as the owner is never filtered. See + [`FORCE` and `BYPASSRLS`](../concepts/row-level-security.md#force-and-bypassrls). + + Raises: + TypeError: A policy is not a [`RowPolicy`][ferro.RowPolicy], or + `force` is not a `bool`. Examples: - >>> RowSecurity(RowPolicy(column="tenant_id", setting="app.tenant_id")) - RowSecurity(policies=1, force=True) + ```python {test=run prelude=reference} + from typing import ClassVar + from uuid import UUID + + from ferro import Field, Model, RowPolicy, RowSecurity + + + class Invoice(Model): + id: int | None = Field(default=None, primary_key=True) + tenant_id: UUID + total: int + + __ferro_rls__: ClassVar = RowSecurity( + RowPolicy(column="tenant_id", setting="app.tenant_id") + ) + ``` + + On Postgres, `Invoice`'s table is created with: + + ```sql + ALTER TABLE "invoice" ENABLE ROW LEVEL SECURITY + ALTER TABLE "invoice" FORCE ROW LEVEL SECURITY + CREATE POLICY "rls_invoice_tenant_id" ON "invoice" FOR ALL + USING ("tenant_id" = NULLIF(current_setting('app.tenant_id', true), '')::uuid) + WITH CHECK ("tenant_id" = NULLIF(current_setting('app.tenant_id', true), '')::uuid) + ``` """ policies: tuple[RowPolicy, ...] + """The table's policies, in declaration order.""" force: bool = True + """Whether the table also gets `FORCE ROW LEVEL SECURITY`.""" def __init__(self, *policies: RowPolicy, force: bool = True) -> None: object.__setattr__(self, "policies", tuple(policies)) diff --git a/src/ferro/session.py b/src/ferro/session.py index d174702f..62c2759f 100644 --- a/src/ferro/session.py +++ b/src/ferro/session.py @@ -1,4 +1,4 @@ -"""Session-scoped runtime state for Ferro operations.""" +"""Sessions: `Session`, `engines` and `current_session`.""" from __future__ import annotations @@ -6,13 +6,17 @@ import contextvars from collections.abc import Mapping from dataclasses import dataclass, field -from typing import Any +from typing import TYPE_CHECKING, Any from ._core import close_session as _core_close_session from ._core import open_session as _core_open_session from ._core import set_session_config as _core_set_session_config from .state import _CURRENT_SESSION +if TYPE_CHECKING: + from .models import Model + from .query.builder import Query + _SESSION_CLOSE_AMBIENT_MISMATCH = ( "Session close failed: ambient session does not match the closing session. " "This usually indicates session lifecycle misuse in the current asyncio context." @@ -72,35 +76,39 @@ def _validated_settings(settings: Mapping[str, str] | None) -> dict[str, str]: @dataclass(slots=True) class Session: - """A unit of work: one connection scope, one identity map, one settings set. - - `settings` are *session settings* — Postgres settings (GUCs) that Ferro - applies to every operation this session runs, so queries are scoped without - a per-query `where`: - - async with engines.session(settings={"myapp.tenant_id": "acme"}): - # Postgres sees myapp.tenant_id = 'acme' for every statement this - # session sends, which is what a row-level-security policy reads - # back — inside a transaction() block or, as here, outside one. - invoices = await Invoice.where(lambda invoice: invoice.paid).all() - - They are validated when the `Session` is constructed and again when it is - entered, which is when the effective set — this session's settings merged - over anything it inherits from an enclosing session — is snapshotted. See - `EngineManager.session` for the full contract, including what an operation - outside a transaction sends and how inherited settings behave off Postgres. - - A value not known until partway through the session's life — resolved - from an auth chain, say — is set with `set_config` rather than at open; - see there for the mid-transaction and identity-map-eviction contract. + """A unit of work on one connection, with its own identity map and settings. + + Open one with `async with engines.session():`. Every ORM call and raw + query inside the block runs in it, and code called from the block finds + it with [`current_session`][ferro.current_session]. Leaving the block + closes the session. See [Sessions](../guide/connections.md#sessions). + + `settings` are Postgres settings that every operation in the session + sends, which is what row-level security policies read; see + [Session settings](../concepts/row-level-security.md#session-settings). + A session opened inside another starts from the outer one's settings. + + Examples: + ```python {test=run prelude=reference} + from ferro import engines + + async with engines.session() as session: + authors = await Author.where(lambda author: author.id <= 2).all() + assert [author.name for author in authors] == ["Ada Lovelace", "Grace Hopper"] + assert session.connection_name == "default" + ``` """ connection_name: str | None = None + """The connection the session runs on, or `None` for the default until it opens.""" session_id: str | None = None + """The open session's id, or `None` before it opens and after it closes.""" settings: Mapping[str, str] | None = None + """The settings this session declares, as given.""" effective_settings: dict[str, str] = field( default_factory=dict, init=False, repr=False, compare=False ) + """The settings the session runs with: its own over the ones it inherits.""" _token: Any = field(default=None, repr=False, compare=False) _enter_context: contextvars.Context | None = field( default=None, repr=False, compare=False @@ -146,26 +154,20 @@ async def __aexit__(self, exc_type, exc, tb) -> None: raise close_exc async def close(self) -> None: - """Close this session and release its runtime state. - - Under ``settings_delivery="connection"`` this is also where the - connection the session pinned goes back to the pool — after Ferro - resets **exactly the settings keys this session set on it**, and - nothing else. (Never ``RESET ALL``: that would also wipe the - ``search_path`` the pool installs on every connection.) A session - under the default ``transaction`` delivery has nothing to release and - this costs no round trip. + """Close the session and release what it holds. - Safe to call from a different asyncio context than ``__aenter__``. - Repeated calls are no-ops. + Leaving the `async with` block calls it; call it yourself for a + session you entered by hand. Under `settings_delivery="connection"`, + the session first resets exactly the setting keys it set on its + connection, then returns the connection to the pool. A second call + does nothing. Raises: - RuntimeError: If the ambient session in this asyncio context does not - match this handle (same-context lifecycle misuse), or if - session-scoped transactions are still open, or if resetting a - pinned connection failed — in which case the session is closed - regardless and the connection was discarded rather than - returned to the pool. + RuntimeError: A `transaction()` block of the session is still + open, another session became current in this task without + closing first, or resetting the held connection failed. In + the last case the session is closed anyway and the + connection is discarded. """ if self._close_lock is None: self._close_lock = asyncio.Lock() @@ -267,74 +269,35 @@ def _merge_ambient_settings(self, declared: dict[str, str]) -> dict[str, str]: return {**inherited, **declared} async def set_config(self, key: str, value: str) -> None: - """Mutate this session's settings while it is open. - - Everything you can `settings=` at open, you can also set later, for the - case where the value isn't known until partway through a request — an - auth chain that resolves the tenant only after checking a token, say: - - async with engines.session() as session: # no settings yet - tenant = await resolve_tenant_from_auth_header(request) - await session.set_config("myapp.tenant_id", tenant) - # every query from here on is scoped to `tenant` - invoices = await Invoice.where(lambda invoice: invoice.paid).all() - - Deep in a call stack, reach the session through `ferro.current_session()` - rather than threading it through every function signature: - - async def resolve_tenant_and_scope(request) -> None: - tenant = await resolve_tenant_from_auth_header(request) - await ferro.current_session().set_config("myapp.tenant_id", tenant) - - **When the new value takes effect, precisely.** Any operation or - `transaction()` STARTED after `set_config` returns sees the new value - — this holds for any task, not just the one that called `set_config`, - because the change is committed before `set_config`'s `await` - returns. An operation or transaction already in flight *when* - `set_config` runs keeps running under the scope it started with — - that is the existing per-operation-atomicity guarantee, working as - intended, not something `set_config` reaches back into. The one place - that needs (and gets) special handling is a `transaction()` block - already open in the SAME task that calls `set_config` — there, the - very next statement in that same transaction sees the new value - immediately, rather than waiting for the transaction to end and a new - one to begin (there is no race to resolve here: the two are - sequential code in one task, by construction): - - async with engines.session() as session: - async with transaction(): - await session.set_config("myapp.tenant_id", "acme") - # this SELECT, in the SAME transaction, already sees it - rows = await Invoice.where(lambda invoice: invoice.paid).all() - - `set_config` also evicts this session's identity map on a real change: - an instance `get()`-ed under the old scope is never handed back under - the new one — the next `get()` for that primary key refetches instead. - A call that doesn't actually change the value (the key already holds - it) is a true no-op: no database round trip, no identity-map eviction. + """Set one session setting while the session is open. - Validation matches `engines.session(settings=...)` exactly (see there - for the full rationale): `key` and `value` must be `str`, and `key` must - be a dotted custom setting name. + Use it when the value is known only partway through the session, + such as a tenant resolved from an auth token. An operation or + `transaction()` block that starts after it returns sees the new + value, and so does a `transaction()` block of this session already + open in any task. It also empties the session's identity map. Setting + a key to the value it holds sends nothing. See + [Visibility of a new value](../concepts/row-level-security.md#visibility-of-a-new-value). Args: - key: Dotted custom Postgres setting name (e.g. `"myapp.tenant_id"`). + key: A dotted custom Postgres setting name, such as + `"myapp.tenant_id"`. value: The setting's new value. Raises: - RuntimeError: This session is not open (`set_config` mutates a live - session, so there has to be one), or this session's connection - is not Postgres — `set_config` is a declaration, exactly like - `settings=` at open, so it has to be honourable rather than a - silent no-op on a backend with no GUCs. On the rare failure - while delivering the change into an already-open transaction, - this session's recorded settings and identity map have - already committed the change regardless (see the Ferro source - for the exact sequencing) — the affected transaction cannot go - on to leak an old-scope read, because Postgres aborts a - transaction after a failed statement until it is rolled back. + RuntimeError: The session is not open, its connection is not + Postgres, or delivering the value to an open transaction + failed. The new value is recorded even then. TypeError: `key` or `value` is not a `str`. ValueError: `key` is not a dotted custom setting name. + + Examples: + ```python {test=run prelude=invoices backend=postgres} + async with engines.session() as session: + tenant = await resolve_tenant(request) + await session.set_config("myapp.tenant_id", tenant) + invoices = await Invoice.where(lambda invoice: invoice.total > 100).all() + ``` """ if self.session_id is None: raise RuntimeError(_SET_CONFIG_NOT_OPEN_MESSAGE) @@ -365,121 +328,104 @@ async def resolve_tenant_and_scope(request) -> None: ) self.effective_settings = dict(committed) - def query(self, model_cls): + def query[M: Model](self, model_cls: type[M]) -> Query[M]: + """Start a query of a model's records in this session. + + Args: + model_cls: The model to query. + + Returns: + A [`Query`][ferro.query.builder.Query] over every row of the + model's table, run in this session. + + Examples: + ```python {test=run prelude=reference} + from ferro import current_session + + posts = await current_session().query(Post).where(lambda post: post.views > 1000).all() + assert [post.title for post in posts] == ["Notes on the Analytical Engine"] + ``` + """ from .query import Query return Query(model_cls, session=self) def current_session() -> "Session | None": - """Return the session open in the current asyncio task, or `None`. - - A `Session` is ambient: once you `async with engines.session(...)`, code - called from inside that block doesn't need the session passed down through - every function signature — it can just ask for it. This is what makes the - deferred-resolution pattern possible: open a session before you know the - tenant, resolve it partway through the request, and hand it to the session - from wherever that resolution happens: + """Return the session open in the current task, or `None`. - async with engines.session(): # tenant not known yet - await handle_request(request) # ... called deep inside ... + Code called from inside `async with engines.session():` reaches the + session through it instead of taking it as an argument. Inside nested + sessions it returns the innermost one. - async def handle_request(request) -> None: - tenant = await resolve_tenant_from_auth_header(request) - await ferro.current_session().set_config("myapp.tenant_id", tenant) - # every query for the rest of this request is scoped to `tenant` + Returns: + The current session, or `None` outside every session. - Nested sessions each become the ambient one for the code inside their own - `async with` block; `current_session()` always returns the innermost one, - and `None` outside every session. + Examples: + ```python {test=run prelude=reference} + from ferro import current_session, engines - Returns: - Session | None: The ambient session, or `None` if none is open. + async with engines.session() as session: + assert current_session() is session + ``` """ return _CURRENT_SESSION.get() # type: ignore[return-value] # ty: ignore[invalid-return-type] class EngineManager: + """The registry of connections that sessions open on; use the `engines` instance.""" + def session( self, name: str | None = None, *, settings: Mapping[str, str] | None = None, ) -> Session: - """Open a session on `name` (or the default connection). + """Open a session on a connection, for use with `async with`. - Pass `settings` to give every operation in the session a set of Postgres - settings (GUCs) — the tenancy scope row-level-security policies read: - - async with engines.session(settings={"myapp.tenant_id": "acme"}): - open_invoices = await Invoice.where( - lambda invoice: invoice.status == "open" - ).all() - - Ferro sends them as one parameter-bound statement right after `BEGIN`, - before any statement of yours: - - BEGIN - SELECT set_config($1, $2, true) -- 'myapp.tenant_id', 'acme' - SELECT "invoice".* FROM "invoice" WHERE "status" = $1 - COMMIT - - `true` is Postgres' `is_local` flag, so the value dies with the - transaction and a recycled pool connection can never leak one request's - scope into the next. Nested savepoints inherit it; keys and values are - always bound parameters, never pasted into SQL. - - Every operation in the session is scoped, whether or not you opened a - transaction. Inside a `transaction()` block the settings ride that - block's `BEGIN`. Outside one, the operation opens an implicit - transaction of its own — as above — so a plain `where().all()`, - `save()`, `get()` or raw query is scoped too. The wrap is per - *operation*, not per statement, so an operation that issues several - statements (a chunked `bulk_create`) sends all of them inside one - transaction and stays all-or-nothing, exactly as it already did. - - The cost, stated plainly: about two extra round-trips per *operation* - that is not already inside a transaction — where one operation is one - call into Ferro's core, not one line of your code. A flow that saves a - row and then links it is two operations and pays twice; put such a flow - in a `transaction()` block and it pays once, for the block. - `ferro.raw.execute(..., autocommit=True)` opts a statement out of the - wrap entirely, for the few Postgres statements that cannot run inside a - transaction — and out of the scoping with it. - - A session without `settings` sends nothing extra at all — same - statements, same connections, no wrap. - - Settings you declare here have to be honourable, so opening this session - on a non-Postgres connection raises. Settings it merely *inherits* from - an enclosing session are different: they ride along, so a Postgres - session nested deeper still gets them, but they lie inert on a - non-Postgres connection — which has no row security to scope anyway. - A tenant-scoped request can therefore still reach an auxiliary SQLite - database without ceremony: - - async with engines.session("pg", settings={"myapp.tenant_id": "acme"}): - async with engines.session("local_cache"): # SQLite: fine - stale = await Lookup.where(lambda row: row.stale).all() + With `settings`, every operation in the session sends those Postgres + settings first, so row-level security policies see them. Inside a + `transaction()` block they follow its `BEGIN` once; outside one, + each operation wraps itself in a transaction to carry them, which + costs three extra round trips. A session without settings sends + nothing extra. See + [Round trips per operation](../concepts/row-level-security.md#round-trips-per-operation). Args: - name: Registered connection name; defaults to the default connection. - settings: Session settings as `{dotted.key: str value}`. + name: The connection to open on. Defaults to `None`, the default + connection. + settings: Postgres settings for every operation, as dotted + custom names mapped to `str` values. Defaults to `None`, no + settings beyond the ones an enclosing session passes on. Returns: - Session: An unopened session handle; `async with` it. + A [`Session`][ferro.Session], not yet open: enter it with + `async with`. Raises: TypeError: A settings key or value is not a `str`. - ValueError: A settings key is not dotted (built-in Postgres settings - are deliberately not settable this way). - RuntimeError: On enter, when a session that *declares* settings - resolves to a non-Postgres connection — settings are Postgres - GUCs, so scoping fails loudly rather than silently doing nothing. + ValueError: A settings key is not a dotted custom setting name. + RuntimeError: On entering, a session that declares settings is + on a connection that is not Postgres. + + Examples: + ```python {test=run prelude=reference} + from ferro import engines + + async with engines.session(): + post = await Post.where(lambda post: post.views > 1000).first() + assert post.title == "Notes on the Analytical Engine" + ``` + + With settings, on Postgres: + + ```python {test=run prelude=invoices backend=postgres} + async with engines.session(settings={"myapp.tenant_id": "acme"}): + invoices = await Invoice.where(lambda invoice: invoice.total > 100).all() + ``` """ return Session(connection_name=name, settings=settings) engines = EngineManager() -"""The connections ``connect()`` registered, by name: -``ferro.engines.session(name)`` opens a ``Session`` routed to one of them.""" +"""The connections [`connect`][ferro.connect] registered: `engines.session(name)` opens a session on one.""" diff --git a/src/ferro/settings.py b/src/ferro/settings.py index f467da48..ad1919ed 100644 --- a/src/ferro/settings.py +++ b/src/ferro/settings.py @@ -1,6 +1,6 @@ """Project configuration: the one way every ferro tool reads a project's databases. -A project commits one file that names its databases (ADR-0036, ADR-0039): +A project commits one file that names its databases: ```toml # ferro.toml — or the same keys under [tool.ferro] in pyproject.toml @@ -8,14 +8,14 @@ dialects = ["postgres", "sqlite"] ``` -```python +```python {test=compile reason="needs a project with a ferro.toml and its database URL set"} from ferro import FerroSettings settings = FerroSettings() db = settings.database() # implied: there is one -db.directory # <config dir>/migrations -db.url_for() # $DATABASE_URL, never the file -db.import_models() # imports myapp.models, returns its models +directory = db.directory # <config dir>/migrations +url = db.url_for() # $DATABASE_URL, never the file +models = db.import_models() # imports myapp.models, returns its models ``` The rules, each enforced here with a refusal that names its fix: @@ -79,13 +79,18 @@ __all__ = ["DatabaseSettings", "FerroSettings", "SettingsError"] Dialect = Literal["postgres", "sqlite"] +"""A dialect a database's migrations are rendered for.""" FERRO_TOML = "ferro.toml" +"""The name of the dedicated config file.""" PYPROJECT_TOML = "pyproject.toml" +"""The name of the project file whose `[tool.ferro]` table can hold the config.""" CONFIG_ENV = "FERRO_CONFIG" +"""The environment variable that names the config file to read.""" DEFAULT_DATABASE = "default" """The name of the one database a config declares without a databases table.""" DEFAULT_DIRECTORY = "migrations" +"""The migrations directory of a database that sets no `directory`.""" _DATABASE_KEYS = ( "models", @@ -124,8 +129,18 @@ class SettingsError(FerroError): def parse_ddl_lock_timeout(value: str) -> timedelta: - """Parse a ``ddl_lock_timeout`` duration (``"500ms"``, ``"5s"``, ``"1m"``), - or ``"0"``, which turns the timeout and its retries off (ADR-0044).""" + """Parse a `ddl_lock_timeout` value into a duration. + + Args: + value: A positive duration written as `"500ms"`, `"5s"` or `"1m"`, + or `"0"`, which turns the timeout and its retries off. + + Returns: + The duration; zero for `"0"`. + + Raises: + ValueError: `value` is not one of those forms. + """ if value == "0": return timedelta(0) match = _DURATION.match(value) if isinstance(value, str) else None @@ -138,19 +153,26 @@ def parse_ddl_lock_timeout(value: str) -> timedelta: MAX_LOCK_TIMEOUT_S = 60.0 * 60 * 24 * 365 -"""The longest lock timeout accepted: one year, "until it is free" while -staying well within the runtime's range. A configured ``lock_timeout = "0"`` -waits this long.""" +"""The longest lock timeout accepted, in seconds: one year. + +A configured `lock_timeout = "0"` waits this long, which stands for "until +the lock is free" while staying within the runtime's range. +""" def parse_lock_timeout(value: str | float) -> float: - """Seconds from ``"30s"``, ``"500ms"``, ``"1m"`` or a plain number of - seconds, as ``--lock-timeout`` takes them (``0`` tries the lock once, - :data:`MAX_LOCK_TIMEOUT_S` is the most). + """Parse a lock timeout as `--lock-timeout` takes it, into seconds. + + Args: + value: `"30s"`, `"500ms"`, `"1m"`, or a number of seconds. `0` tries + the lock once. + + Returns: + The timeout in seconds. Raises: - SettingsError: ``value`` is not a duration, is negative, or is longer - than :data:`MAX_LOCK_TIMEOUT_S`. + SettingsError: `value` is not a duration, is negative, or is longer + than one year. """ if isinstance(value, int | float) and not isinstance(value, bool): seconds = float(value) @@ -182,13 +204,20 @@ def parse_lock_timeout(value: str | float) -> float: def lock_timeout_setting_seconds(value: str) -> float: - """A configured ``lock_timeout`` in seconds: the forms ``--lock-timeout`` - takes, except that ``"0"`` waits without a limit - (:data:`MAX_LOCK_TIMEOUT_S`): a setting that refused every contended - run would be no wait at all. + """Parse a configured `lock_timeout` into seconds. + + It takes the forms `--lock-timeout` takes, except that `"0"` waits + without a limit: a setting that refused every contended run would be no + wait at all. + + Args: + value: `"30s"`, `"500ms"`, `"1m"`, a number of seconds, or `"0"`. + + Returns: + The timeout in seconds; one year for `"0"`. Raises: - ValueError: ``value`` is not one of those forms. + ValueError: `value` is not one of those forms. """ try: seconds = parse_lock_timeout(value) @@ -214,23 +243,32 @@ class _Project: class DatabaseSettings(BaseModel): - """One configured *database*: a named set of models with one migration lineage. + """One configured database: a named set of models with one line of migrations. - Built by :class:`FerroSettings` from ``[tool.ferro]`` (one database, named - ``"default"``) or ``[tool.ferro.databases.<name>]`` (several). Paths are - already resolved against the config file's directory. + [`FerroSettings`][ferro.FerroSettings] builds it from `[tool.ferro]` (one + database, named `"default"`) or from `[tool.ferro.databases.<name>]` + (several). Paths are already resolved against the config file's + directory. See [Keys](../reference/configuration.md#keys) for each key. """ model_config = ConfigDict(extra="forbid", frozen=True) name: str = Field(min_length=1) + """The database's name: `"default"` for a single database.""" models: list[str] = Field(min_length=1) + """The dotted modules that define the database's models.""" dialects: list[Dialect] = Field(min_length=1) + """The dialects its migrations are rendered for: `"postgres"`, `"sqlite"` or both.""" url_env: str = Field(default="DATABASE_URL", min_length=1) + """The environment variable holding the database URL. Defaults to `"DATABASE_URL"`.""" directory: Path + """The migrations directory, resolved against the config file's directory.""" tracking_schema: str | None = Field(default=None, min_length=1) + """Postgres only: the schema of the tracking tables. Defaults to `None`, the current schema.""" ddl_lock_timeout: str = "5s" + """How long a DDL statement on Postgres waits for its table lock. Defaults to `"5s"`.""" lock_timeout: str = "30s" + """How long a run waits for the run lock. Defaults to `"30s"`; `"0"` waits without a limit.""" _project: _Project | None = PrivateAttr(default=None) @@ -258,34 +296,51 @@ def _tracking_schema_needs_postgres(self) -> DatabaseSettings: @property def ddl_lock_timeout_seconds(self) -> float: - """``ddl_lock_timeout`` in seconds; ``0.0`` when it is off.""" + """`ddl_lock_timeout` in seconds; `0.0` when it is off.""" return parse_ddl_lock_timeout(self.ddl_lock_timeout).total_seconds() @property def lock_timeout_seconds(self) -> float: - """``lock_timeout`` in seconds: how long a run (``ferro migrate up``, - ``down``, ``baseline``, ``rerecord``) or an auto-migrate pass waits - for the run lock another one holds; ``"0"`` is - :data:`MAX_LOCK_TIMEOUT_S`, no limit.""" + """`lock_timeout` in seconds; one year, no limit, for `"0"`. + + It is how long a run (`ferro migrate up`, `down`, `baseline`, + `rerecord`) or an auto-migrate pass waits for the run lock another + one holds. + """ return lock_timeout_setting_seconds(self.lock_timeout) def lock_wait(self, override: str | float | None = None) -> float: - """Seconds a run on this database waits for the run lock: ``override`` - (``--lock-timeout``, or a call's ``lock_timeout=``) when given, else - the configured ``lock_timeout``. + """Return how many seconds a run on this database waits for the run lock. + + Args: + override: `--lock-timeout`, or a call's `lock_timeout=`. Defaults + to `None`, the configured `lock_timeout`. + + Returns: + The wait in seconds. Raises: - SettingsError: ``override`` is not a lock timeout. + SettingsError: `override` is not a lock timeout. """ if override is None: return self.lock_timeout_seconds return parse_lock_timeout(override) def url_for(self, override: str | None = None) -> str: - """The database URL: ``override`` (``--url``), else ``$<url_env>``. + """Return the database URL: `override`, else the `url_env` variable. - The file never holds a URL; an unset or empty variable is refused by - name. + The config file never holds a URL. + + Args: + override: A URL given on the command line (`--url`). Defaults to + `None`, which reads the variable `url_env` names. + + Returns: + The URL. + + Raises: + SettingsError: No `override` was given and the variable is unset + or empty. """ if override: return override @@ -298,20 +353,24 @@ def url_for(self, override: str | None = None) -> str: return url def import_models(self) -> list[type]: - """Import this database's ``models`` modules and return its models. - - The config file's directory goes first on ``sys.path``, then each - ``python_path`` entry, and each module is imported as written. Those - ``sys.path`` entries stay for the rest of the process, since an - imported module may import its siblings later. The result is every - registered model this database claims, in registration order, after - checking that every registered model has a database and that none of - this database's models has a foreign key into another. A foreign key - naming its target by string is resolved against every configured - database's modules, importing them if needed. - - An empty result is refused: no models is never read as "drop every - table" (ADR-0036). + """Import this database's `models` modules and return its models. + + The config file's directory goes first on `sys.path`, then each + `python_path` entry, and the entries stay for the rest of the + process, since an imported module may import its siblings later. A + foreign key naming its target by string is resolved against every + configured database's modules, importing them if needed. + + Returns: + Every registered model this database claims, in registration + order. + + Raises: + SettingsError: The modules registered no model (no models is + never read as "drop every table"), a registered model belongs + to no database, one of this database's models has a foreign + key into another database, or the settings were built by hand + rather than read from a config file. """ project = self._require_project() _import_database_modules(project, self) @@ -336,31 +395,53 @@ def _require_project(self) -> _Project: class FerroSettings(BaseSettings): - """A project's ferro configuration, read from its one config file. - - ``FerroSettings()`` finds the file (see the module docstring for the - lookup); ``FerroSettings(config=path)`` names it. With no file found the - object is empty: ``databases == {}`` and ``searched`` lists every - directory examined. - - The lookup is ferro's; reading the file is pydantic-settings': - ``__init__`` hands the located path to ``settings_customise_sources`` - through the init source, which then reads it with one - ``TomlConfigSettingsSource`` and nothing else. The validators below shape - the file's keys (one database or a ``databases`` table) into - ``databases`` and refuse what does not belong, naming where it goes. + """A project's Ferro configuration, read from its one config file. + + `FerroSettings()` looks for the file in this order: the `FERRO_CONFIG` + environment variable, then the nearest directory, walking up from the + working directory, that holds a `ferro.toml` or a `pyproject.toml` with + a `[tool.ferro]` table. The file found is used alone: no environment + variable overrides a key. With no file found the object is empty + (`databases == {}`, and `searched` lists every directory examined); it + refuses only when a database is asked for. Each `FerroSettings()` reads + the file again. See [Configuration](../reference/configuration.md). + + Args: + config: The config file to read, skipping the lookup. Defaults to + `None`. + + Raises: + SettingsError: The file is malformed or contradicts itself, a key is + unknown or misplaced, or one directory holds both a `ferro.toml` + and a `[tool.ferro]` table. + + Examples: + ```python {test=run prelude=reference} + import tempfile + from pathlib import Path + + from ferro import FerroSettings + + config = Path(tempfile.mkdtemp()) / "ferro.toml" + config.write_text('models = ["blog.models"]\ndialects = ["postgres", "sqlite"]\n') + database = FerroSettings(config=config).database() + assert database.name == "default" + assert database.directory == config.resolve().parent / "migrations" + ``` """ model_config = SettingsConfigDict(extra="forbid", frozen=True, case_sensitive=True) config_path: Path | None = None - """The config file read, or ``None`` when none was found.""" + """The config file read, or `None` when none was found.""" config_table: tuple[str, ...] = () - """Where the keys sit in it: ``("tool", "ferro")``, or ``()`` for the top level.""" + """Where the keys sit in it: `("tool", "ferro")`, or `()` for the top level.""" searched: list[Path] = Field(default_factory=list) """Every directory the walk-up examined without finding a config.""" python_path: list[Path] = Field(default_factory=list) + """Directories put on `sys.path` before importing models, resolved against the config file.""" databases: dict[str, DatabaseSettings] = Field(default_factory=dict) + """The configured databases, by name; empty when no file was found.""" def __init__(self, config: Path | str | None = None) -> None: path, searched = _locate(config) @@ -382,12 +463,27 @@ def settings_customise_sources( dotenv_settings: PydanticBaseSettingsSource, file_secret_settings: PydanticBaseSettingsSource, ) -> tuple[PydanticBaseSettingsSource, ...]: - """The located config file, read by ``TomlConfigSettingsSource``. + """Read the located config file and nothing else. + + Pydantic-settings calls it while building the object. No environment + variable, `.env` file or secrets directory sets a field, since these + keys decide what a migration contains. + + Args: + settings_cls: The settings class being built. + init_settings: What the constructor located: the path, its table + header and the directories searched. + env_settings: Unused: no environment variable sets a key. + dotenv_settings: Unused: no `.env` file is read. + file_secret_settings: Unused: no secrets directory is read. + + Returns: + The constructor's source, and the config file's when one was + found. - ``init_settings`` carries only what ``__init__`` located (the path, - its table header, the directories searched). No environment - variable, ``.env`` file or secrets directory ever sets a field - (ADR-0036: these keys decide what a migration contains). + Raises: + SettingsError: The file is not valid TOML, or a key the + constructor sets appears in it. """ located = ( init_settings.init_kwargs @@ -472,7 +568,20 @@ def _bind_databases(self) -> FerroSettings: return self def database(self, name: str | None = None) -> DatabaseSettings: - """The database called ``name``; implied when exactly one is configured.""" + """Return the database called `name`. + + Args: + name: The database's name. Defaults to `None`, which is allowed + when exactly one database is configured. + + Returns: + That database's settings. + + Raises: + SettingsError: No config file was found, no database has that + name, or `name` is `None` and several databases are + configured. + """ if not self.databases: raise SettingsError(self._no_config_message()) if name is None: @@ -492,11 +601,21 @@ def database(self, name: str | None = None) -> DatabaseSettings: ) from None def database_for(self, model: type) -> DatabaseSettings: - """The database that owns ``model``, decided by its defining module. + """Return the database that owns `model`, decided by its defining module. + + A model belongs to each database whose `models` list holds its + module or a parent of it. + + Args: + model: A model class. - Refused when no database claims it, when several do (it then has no - single owner; ask for a database by name), or when one of its foreign - keys points at a model its database does not claim. + Returns: + The settings of the one database that claims the model. + + Raises: + SettingsError: `model` is not a model class, no config file was + found, no database or several claim the model, or one of its + foreign keys points at a model its database does not claim. """ if not isinstance(model, type) or not hasattr(model, "__ferro_identity__"): raise SettingsError( diff --git a/src/ferro/state.py b/src/ferro/state.py index 2c1ca2dd..d19cab56 100644 --- a/src/ferro/state.py +++ b/src/ferro/state.py @@ -1,8 +1,8 @@ """Operation routing: sessions, transactions, and RouteHandle construction. The single resolution site for "which connection does this operation hit" -(FF-D D3/D4) and the only module allowed to construct ``RouteHandle``. -Registration state lives in :mod:`ferro.registry`. +and the only module allowed to construct ``RouteHandle``. +Registration state lives in `ferro.registry`. """ from contextvars import ContextVar @@ -22,11 +22,18 @@ class SessionLike(Protocol): + """What routing reads from a session: its id, connection and settings.""" + session_id: str + """The open session's id.""" connection_name: str - # Settings this session runs with, snapshotted at enter; empty for the - # sessions that declare none. Read by a nested session to inherit from. + """The connection the session runs on.""" effective_settings: dict[str, str] + """The settings the session runs with, snapshotted when it opened. + + Empty for a session that declares none; a nested session inherits from + it. + """ _CURRENT_SESSION: ContextVar[SessionLike | None] = ContextVar( @@ -47,8 +54,8 @@ def _ensure_active_session(session: SessionLike | None) -> None: _NO_ROUTE_MESSAGE = ( "No database route for this operation. Open a session " "(`async with ferro.engines.session(...)`) or pass `using=`/`session=` " - "explicitly. Implicit default-connection routing was removed in v0.13 " - "(deprecated since v0.12); see the sessions migration guide." + "explicitly. Implicit default-connection routing was removed in v0.14 " + "(deprecated since v0.12); see the upgrade guide." ) @@ -65,14 +72,25 @@ def resolve_operation_scope( using: str | None, session: SessionLike | None, ) -> RouteHandle: - """Resolve the route for ORM/raw operations. - - Exactly one resolution site per operation (FF-D D3/D4): a session - (ambient or explicit), an explicit `using=`, or an active transaction. - No route is an error; `using=` conflicting with the ambient session is - an error (the silent sessionless bypass was removed in v0.13). The - returned `RouteHandle` is resolved once and threaded by value through - every FFI operation — Rust never re-derives it. + """Resolve the route of an ORM or raw SQL operation. + + The one resolution site per operation: a session (current or explicit), + an explicit `using=`, or an active transaction. The returned + `RouteHandle` is resolved once and passed by value through every core + operation; the Rust core never derives it again. + + Args: + using: An explicit connection name, or `None`. + session: An explicit session, or `None` for the current one. + + Returns: + The route the operation runs on. + + Raises: + ValueError: `using` names another connection than the session's or + the transaction's, or an explicit session and `using` disagree. + RuntimeError: There is no route (no session, no `using`), or the + session is closed. """ tx_id = _CURRENT_TRANSACTION.get() tx_connection = _CURRENT_TRANSACTION_CONNECTION.get() @@ -124,11 +142,21 @@ def resolve_compile_connection_name( using: str | None, session: SessionLike | None, ) -> str | None: - """Connection name execute would use, or ``None`` when there is no route. + """Return the connection an operation would run on, or `None` when there is no route. + + Compile-time dialect gates (`.merge()`) must inspect the same connection + as `resolve_operation_scope`, not only `Query._using`. Schema goldens + compile without an engine; that case stays `None`. - Compile-time dialect gates (``.merge()``) must inspect the same - connection as ``resolve_operation_scope``, not only ``Query._using``. - IR goldens compile without an engine; that case stays ``None``. + Args: + using: An explicit connection name, or `None`. + session: An explicit session, or `None` for the current one. + + Returns: + The connection name, or `None` when there is no route. + + Raises: + ValueError: `using` conflicts with the session or the transaction. """ try: return resolve_operation_scope(using=using, session=session).connection_name @@ -143,10 +171,22 @@ def resolve_transaction_scope( using: str | None, session: SessionLike | None, ) -> RouteHandle: - """Resolve the route for `transaction()` — same rules as - `resolve_operation_scope`, but nested transactions always inherit the - parent transaction's route. The returned handle's `tx_id` is the - *parent* transaction (or `None` for a root transaction). + """Resolve the route of a `transaction()` block. + + The rules are `resolve_operation_scope`'s, except that a nested + transaction always inherits its parent's route. + + Args: + using: An explicit connection name, or `None`. + session: An explicit session, or `None` for the current one. + + Returns: + The route, whose `tx_id` is the parent transaction's, or `None` for + a root transaction. + + Raises: + ValueError: `using` conflicts with the session's connection. + RuntimeError: There is no route, or the session is closed. """ parent_tx_id = _CURRENT_TRANSACTION.get() tx_connection = _CURRENT_TRANSACTION_CONNECTION.get() @@ -187,13 +227,19 @@ def resolve_transaction_scope( def route_for_transaction( connection_name: str, tx_id: str, session_id: str | None ) -> RouteHandle: - """Build the child route for the `Transaction` handle yielded by - `transaction()` (FF-D D3). + """Build the route of the `Transaction` handle a `transaction()` block yields. + + It lives here, in the only module allowed to construct `RouteHandle`, + so that invariant holds for every route, including the one raw SQL + uses inside a transaction block. + + Args: + connection_name: The transaction's connection. + tx_id: The transaction's id. + session_id: The session's id, or `None` outside a session. - Kept in `state.py` — the only module allowed to construct - `RouteHandle` — rather than inline in `models.py`, so the single - construction-site invariant holds for every route, including the one - handed to raw SQL inside a transaction block. + Returns: + The route. """ return RouteHandle( connection_name=connection_name, tx_id=tx_id, session_id=session_id diff --git a/tests/_docs_checks.py b/tests/_docs_checks.py new file mode 100644 index 00000000..5e7234bd --- /dev/null +++ b/tests/_docs_checks.py @@ -0,0 +1,244 @@ +"""Checks over docs examples: query lambdas and error checks. + +Two mistakes compile and run but teach the wrong thing: + +- ``Post.where(lambda p: p.published)``: a query lambda with a one-letter + parameter. The docs name it after the model, ``lambda post: post.published``. +- A ``try`` whose ``except`` asserts on the error but has no ``else`` that + raises. When the error stops happening, the block still passes. + +Both checks read Python files (with the fences in their docstrings) and the +Python fences of Markdown pages. Run them from the command line: + + python -m tests._docs_checks docs/examples docs/pages src/ferro + +Each violation prints as ``path:line: message``; any violation exits 1. +""" + +from __future__ import annotations + +import ast +import inspect +import sys +import typing +from collections.abc import Iterator +from dataclasses import dataclass +from functools import cache +from pathlib import Path + +from tests._docs_harness import fenced_examples + +# Query methods the docs call with a lambda whose signature does not say so +# (aggregate projections, keyset paging). The signatures add the rest. +_NAMED_QUERY_METHODS = frozenset( + { + "where", + "select", + "order_by", + "include", + "exists", + "count", + "aggregate", + "group_by", + "after", + "before", + } +) +_CALLABLE_ANNOTATIONS = ("Callable", "Predicate", "RowSelector") + + +@dataclass(frozen=True) +class Violation: + """One finding: where it is and what to change.""" + + path: Path + line: int + message: str + + def __str__(self) -> str: + return f"{self.path}:{self.line}: {self.message}" + + +@cache +def query_methods_taking_callables() -> frozenset[str]: + """Name every public ``Query`` or ``Model`` method that takes a callable. + + Read from the signatures (overloads included), plus the names the docs + use on aggregate projections and keyset paging. + """ + from ferro.models import Model, ModelConnection + from ferro.query import ProjectedQuery, Query, Relation + + names = set(_NAMED_QUERY_METHODS) + for cls in (Query, ProjectedQuery, Relation, Model, ModelConnection): + for name, member in vars(cls).items(): + if name.startswith("_"): + continue + function = getattr(member, "__func__", member) + if not callable(function): + continue + for signature_of in (function, *typing.get_overloads(function)): + try: + signature = inspect.signature(signature_of) + except (TypeError, ValueError): + continue + if any( + marker in str(parameter.annotation) + for parameter in signature.parameters.values() + for marker in _CALLABLE_ANNOTATIONS + ): + names.add(name) + return frozenset(names) + + +def _sources(path: Path) -> Iterator[tuple[str, int]]: + """Yield each Python source in ``path`` with the file line it starts on.""" + text = path.read_text(encoding="utf-8") + if path.suffix == ".md": + for example in fenced_examples(path, text): + # Snippet directives are expanded by the docs build; the file + # they include is checked on its own. + if "--8<--" not in example.source: + yield example.source, example.start_line + 1 + return + yield text, 1 + lines = text.splitlines(keepends=True) + for node in ast.walk(_parse(text, path, 1)): + if ( + isinstance(node, ast.Expr) + and isinstance(node.value, ast.Constant) + and isinstance(node.value.value, str) + ): + docstring = "".join(lines[node.lineno - 1 : node.end_lineno]) + for example in fenced_examples( + path, docstring, line_offset=node.lineno - 1 + ): + yield example.source, example.start_line + 1 + + +def _parse(source: str, path: Path, first_line: int) -> ast.AST: + # Padding keeps every node's line number the file's own. + return compile( + "\n" * (first_line - 1) + source, + str(path), + "exec", + flags=ast.PyCF_ONLY_AST | ast.PyCF_ALLOW_TOP_LEVEL_AWAIT, + ) + + +def _trees(path: Path) -> Iterator[ast.AST]: + for source, first_line in _sources(path): + yield _parse(source, path, first_line) + + +def _supplied_parameters(function: ast.Lambda) -> list[ast.arg]: + """The parameters the query fills: positional ones without a default. + + ``lambda post, f=field: getattr(post, f)`` captures ``f`` by default; the + query only ever passes ``post``. + """ + positional = [*function.args.posonlyargs, *function.args.args] + return positional[: len(positional) - len(function.args.defaults)] + + +def lambda_param_violations(path: Path) -> list[Violation]: + """Report each query lambda in ``path`` with a one-letter parameter.""" + methods = query_methods_taking_callables() + violations = [] + for tree in _trees(path): + for call in ast.walk(tree): + if not ( + isinstance(call, ast.Call) + and isinstance(call.func, ast.Attribute) + and call.func.attr in methods + ): + continue + arguments = [*call.args, *(keyword.value for keyword in call.keywords)] + for argument in arguments: + if not isinstance(argument, ast.Lambda): + continue + for param in _supplied_parameters(argument): + if len(param.arg) < 2: + violations.append( + Violation( + path, + argument.lineno, + f"lambda passed to {call.func.attr}() takes " + f"one-letter parameter {param.arg!r}; name it " + "after the model in lowercase singular", + ) + ) + return sorted(violations, key=lambda violation: violation.line) + + +def _asserts(handler: ast.ExceptHandler) -> bool: + return any( + isinstance(node, ast.Assert) for stmt in handler.body for node in ast.walk(stmt) + ) + + +def _expected(handler: ast.ExceptHandler) -> str: + if handler.type is None: + return "an error" + return ast.unparse(handler.type) + + +def unfailing_error_checks(path: Path) -> list[Violation]: + """Report each ``try`` whose ``except`` asserts but whose ``else`` never raises.""" + violations = [] + for tree in _trees(path): + for node in ast.walk(tree): + if not isinstance(node, ast.Try | ast.TryStar): + continue + checking = [handler for handler in node.handlers if _asserts(handler)] + if not checking: + continue + if any(isinstance(stmt, ast.Raise) for stmt in node.orelse): + continue + violations.append( + Violation( + path, + node.lineno, + "try/except asserts on the error but has no else that raises, " + "so it passes when the error stops happening; add " + f'else: raise AssertionError("expected {_expected(checking[0])}")', + ) + ) + return sorted(violations, key=lambda violation: violation.line) + + +def _files(paths: list[str]) -> Iterator[Path]: + for name in paths: + path = Path(name) + if path.is_dir(): + yield from sorted( + file + for file in path.rglob("*") + if file.suffix in {".py", ".md"} and file.is_file() + ) + elif path.is_file(): + yield path + else: + raise SystemExit(f"not a file or directory: {name}") + + +def main(argv: list[str]) -> int: + if not argv: + print("usage: python -m tests._docs_checks PATH...", file=sys.stderr) + return 2 + violations: list[Violation] = [] + for path in _files(argv): + try: + violations += lambda_param_violations(path) + violations += unfailing_error_checks(path) + except SyntaxError as error: + violations.append( + Violation(path, error.lineno or 0, f"does not parse: {error.msg}") + ) + for violation in violations: + print(violation) + return 1 if violations else 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) diff --git a/tests/_docs_harness.py b/tests/_docs_harness.py new file mode 100644 index 00000000..6d147100 --- /dev/null +++ b/tests/_docs_harness.py @@ -0,0 +1,520 @@ +"""Shared pieces of the docs examples harness. + +A Python fence in a docs page or a docstring can carry settings after its +language: + + ```python {test=run prelude=reference} + print(await Author.where(lambda author: author.id == 1).first()) + ``` + +``test`` says how the block is tested, and each mode takes one setting: + +- ``{test=run prelude=<name>}`` runs the block in a fresh interpreter after + ``docs/examples/_preludes/<name>.py``, which declares the models, connects + and opens a session; top-level ``await`` works there. +- ``{test=compile reason="..."}`` only compiles the block and says why it + cannot run (it needs Postgres, or it is a generated file). +- ``{test=pytest path="tests/<file>.py"}`` is a test module the testing + how-to hands readers; ``tests/test_docs_testing_fixtures.py`` writes it at + that path in a scratch project and runs pytest there. + +``backend=postgres`` on a ``test=run`` fence runs it against PostgreSQL: +every ``connect()`` the prelude or the block makes is pointed at the test +database, whatever URL it names. Such a fence runs only under the +``postgres_only`` marker, so a run without Postgres skips it, saying so. + +``lint=check`` also lints the block. Every Python fence, on a docs page or in +a docstring, carries a ``test`` mode (``fence_mode``). Any other setting, or a +mode without its setting, is refused with the marker to write. + +One reader finds every fenced block (``fenced_blocks``), and one runner starts +every subprocess the docs tests start (``run_python``). +""" + +from __future__ import annotations + +import ast +import os +import re +import subprocess +import sys +from collections.abc import Iterator, Sequence +from dataclasses import dataclass +from pathlib import Path + +import griffe +from pytest_examples import CodeExample, EvalExample +from pytest_examples.find_examples import remove_indent + +# Re-exported: the region reader and the connect redirect are the harness's, +# kept stdlib-only in their own module for the scripts the harness starts. +from tests._docs_shared import ( # noqa: F401 + SnippetRegion, + point_connect_at, + redirected_url, + snippet_regions, +) + +REPO_ROOT = Path(__file__).resolve().parents[1] +DOCS_PAGES = REPO_ROOT / "docs" / "pages" +PRELUDES = REPO_ROOT / "docs" / "examples" / "_preludes" + +_SETTING = re.compile(r"""([A-Za-z_][\w-]*)=(?:"([^"]*)"|'([^']*)'|([^\s{}"']+))""") + + +_TEST_MODES = ("run", "compile", "pytest") +# The setting each test mode needs, and the only mode that takes it. +_MODE_SETTING = {"run": "prelude", "compile": "reason", "pytest": "path"} +_KNOWN_SETTINGS = frozenset({"test", "lint", "backend", *_MODE_SETTING.values()}) +_BACKENDS = ("postgres",) +"""The databases a ``test=run`` fence can ask for besides SQLite.""" +_MODE_EXAMPLE = { + "run": "prelude=<name>", + "compile": 'reason="<why it cannot run>"', + "pytest": 'path="tests/<file>.py"', +} + + +# -- the one runner ---------------------------------------------------------------- + + +def run_python( + args: Sequence[str], + *, + timeout: float = 180, + env: dict[str, str] | None = None, + cwd: Path = REPO_ROOT, + input: str | None = None, # noqa: A002 - subprocess.run's own name + clean_env: bool = False, +) -> subprocess.CompletedProcess[str]: + """Run this interpreter with ``args`` and return the finished process. + + Args: + args: The arguments after the interpreter (a script and its arguments, + ``-m module`` or ``-c code``). + timeout: Seconds before the run is killed. Defaults to 180. + env: Variables to set for the run, over this process's environment. + cwd: The working directory. Defaults to the repository root. + input: Text piped to the run's stdin. + clean_env: Start from an empty environment instead of this process's, + so only ``env`` reaches the run. + + Returns: + The process, its ``stdout`` and ``stderr`` as text. + + Raises: + AssertionError: The run exited non-zero. The message names the command, + the exit code and both streams. + """ + command = [sys.executable, *args] + result = subprocess.run( + command, + input=input, + capture_output=True, + text=True, + timeout=timeout, + cwd=cwd, + env={**({} if clean_env else os.environ), **(env or {})}, + ) + if result.returncode != 0: + raise AssertionError( + f"{' '.join(command)} exited {result.returncode}\n" + f"stdout:\n{result.stdout}\nstderr:\n{result.stderr}" + ) + return result + + +# -- the one fence reader ---------------------------------------------------------- + + +@dataclass(frozen=True) +class Fence: + """One fenced block of a Markdown file or a docstring.""" + + path: Path + """The file the block is in.""" + line: int + """The line of the block's opening fence.""" + language: str + """The first word of the info string (``python``, ``sql``, ``text``), or ``""``.""" + info: str + """Everything after the opening backticks, stripped.""" + body: str + """Everything between the fences, the fence's own indentation removed.""" + indent: str + """The indentation of the opening fence.""" + note: tuple[str, str] | None + """The ``<!-- <kind>: <value> -->`` comment on the line above the fence, at + the same indentation, as ``(kind, value)``.""" + + @property + def where(self) -> str: + """The block's file and opening line, as ``<path>:<line>``.""" + return f"{self.path}:{self.line}" + + +_OPENING = re.compile(r"^(?P<indent>[ \t]*)(?P<ticks>`{3,}|~{3,})(?P<info>[^`\n]*)$") +_NOTE = re.compile(r"^<!-- (?P<kind>[a-z][a-z-]*): (?P<value>.+?) -->$") + + +def _language(info: str) -> str: + word = info.lstrip("{").split(" ", 1)[0].split("{", 1)[0] + return word.lstrip(".").lower() + + +def fenced_blocks( + path: Path, text: str, language: str | None = None, *, line_offset: int = 0 +) -> list[Fence]: + """Read every fenced block of ``text``, numbered as lines of ``path``. + + A fence opens with three or more backticks (or tildes) and closes on a line + holding only the same run at the same indentation, so a longer fence can + quote a shorter one. Each block's ``note`` is the HTML comment directly + above it, the way pages mark where a block comes from + (``<!-- prints: predicates.py:first-query -->``). + + Args: + path: The file ``text`` comes from, named in each block and error. + text: The Markdown, or a docstring's lines as they sit in the file. + language: Keep only the blocks in this language. Defaults to every block. + line_offset: The number of lines of ``path`` before ``text``. + + Returns: + The blocks in file order. + + Raises: + ValueError: A fence is never closed; the message names its line. + """ + lines = text.splitlines(keepends=True) + blocks: list[Fence] = [] + index = 0 + while index < len(lines): + opening = _OPENING.match(lines[index].rstrip("\n")) + if opening is None: + index += 1 + continue + indent, ticks = opening["indent"], opening["ticks"] + info = opening["info"].strip() + closing = re.compile( + rf"^{re.escape(indent)}{re.escape(ticks[0])}{{{len(ticks)},}}[ \t]*$" + ) + end = next( + (n for n in range(index + 1, len(lines)) if closing.match(lines[n])), + None, + ) + if end is None: + raise ValueError( + f"{path}:{line_offset + index + 1}: the {ticks}{_language(info)} " + "fence is never closed" + ) + note = None + if index > 0: + above = lines[index - 1].rstrip("\n") + if above.startswith(indent): + marked = _NOTE.match(above[len(indent) :]) + if marked is not None: + note = (marked["kind"], marked["value"]) + body = "".join(line.removeprefix(indent) for line in lines[index + 1 : end]) + fence = Fence( + path, line_offset + index + 1, _language(info), info, body, indent, note + ) + if language is None or fence.language == language: + blocks.append(fence) + index = end + 1 + return blocks + + +def page_fences(language: str | None = None) -> list[Fence]: + """Read every fenced block a docs page shows, numbered as lines of its file. + + A page's own blocks come first, then the blocks of every Markdown section a + page pulls in with ``--8<-- "<file>.md:<section>"`` (``CONTRIBUTING.md``, + ``README.md``, ``docs/includes``), each named by its own file and line, so + a block reaches the reader only through a reader that sees it. + """ + blocks: list[Fence] = [] + included: set[tuple[Path, str]] = set() + for page in sorted(DOCS_PAGES.rglob("*.md")): + text = page.read_text(encoding="utf-8") + blocks += fenced_blocks(page, text, language) + for match in _INCLUDE.finditer(text): + target = REPO_ROOT / match["file"] + if target.name == "CHANGELOG.md": + continue # Release tooling writes it; it holds no examples. + included.add((target, match["section"] or "")) + for target, section in sorted(included): + text, offset = _included(target, section) + blocks += fenced_blocks(target, text, language, line_offset=offset) + return blocks + + +_INCLUDE = re.compile(r'--8<-- "(?P<file>[^":]+\.md)(?::(?P<section>[A-Za-z_-]+))?"') + + +def _included(path: Path, section: str) -> tuple[str, int]: + """The text of ``path``'s region ``section`` (all of it when ``section`` is + empty), and the number of lines before it.""" + text = path.read_text(encoding="utf-8") + if not section: + return text, 0 + region = snippet_regions(text, where=str(path)).get(section) + if region is None: + raise ValueError(f"{path} has no section {section!r}") + return region.body, region.start + + +def fenced_examples( + path: Path, text: str, *, line_offset: int = 0 +) -> list[CodeExample]: + """Find the Python fences in ``text``, numbered as lines of ``path``. + + ``line_offset`` is the number of lines of ``path`` before ``text``. Each + example's ``start_line`` is its opening fence's line, as pytest-examples + numbers them, so its source starts on the next line. + """ + return [ + _example(fence) + for fence in fenced_blocks(path, text, line_offset=line_offset) + if fence.language in ("py", "python") + ] + + +def _example(fence: Fence) -> CodeExample: + source, indent = remove_indent(fence.body) + return CodeExample( + source=source, + path=fence.path, + start_line=fence.line, + end_line=fence.line + fence.body.count("\n") + 1, + start_index=0, + end_index=0, + prefix=fence.info.lower(), + indent=len(fence.indent) + indent, + ) + + +def page_examples() -> list[CodeExample]: + """Every Python fence a docs page shows (``page_fences``), as an example. + + A fence holding a ``--8<--`` line embeds an example script, which + ``tests/test_docs_examples.py`` runs as a whole, and is left out. + """ + return [ + _example(fence) + for fence in page_fences() + if fence.language in ("py", "python") and "--8<--" not in fence.body + ] + + +# -- fence settings --------------------------------------------------------------- + + +def preludes() -> list[str]: + """Name every prelude a fence can ask for.""" + return sorted(p.stem for p in PRELUDES.glob("*.py") if not p.stem.startswith("_")) + + +def fence_settings(example: CodeExample) -> dict[str, str]: + """Read the ``key=value`` settings after a fence's language. + + Values may be bare (``test=run``) or quoted (``lint="check"``). Any + setting the harness would silently ignore is refused, with the marker to + write instead: an unknown key (such as ``compile-only=``), a ``test`` + mode other than ``run``, ``compile`` or ``pytest``, a mode without the + setting it needs, and ``prelude``, ``reason`` or ``path`` without its mode. + """ + settings = { + match.group(1): next(g for g in match.groups()[1:] if g is not None) + for match in _SETTING.finditer(example.prefix) + } + unknown = sorted(set(settings) - _KNOWN_SETTINGS) + if unknown: + raise ValueError( + f"{example}: unknown setting {', '.join(unknown)}; a block that runs " + "is marked {test=run prelude=<name>}, one that cannot says why with " + '{test=compile reason="..."}' + ) + mode = settings.get("test") + if mode is not None and mode not in _TEST_MODES: + raise ValueError( + f"{example}: test={mode} is not run, compile or pytest; mark the fence " + '{test=run prelude=<name>} or {test=compile reason="..."}' + ) + for needed_by, setting in _MODE_SETTING.items(): + if mode == needed_by and not settings.get(setting): + hint = _MODE_EXAMPLE[needed_by] + if needed_by == "run": + hint += f" (one of: {', '.join(preludes())})" + raise ValueError(f"{example}: test={needed_by} needs {hint}") + if setting in settings and mode != needed_by: + raise ValueError( + f"{example}: {setting}={settings[setting]} needs test={needed_by}; " + f"mark the fence " + f"{{test={needed_by} {setting}={settings[setting]}}}" + ) + if "lint" in settings and settings["lint"] != "check": + raise ValueError(f"{example}: lint={settings['lint']} is not lint=check") + backend = settings.get("backend") + if backend is not None and backend not in _BACKENDS: + raise ValueError(f"{example}: backend={backend} is not backend=postgres") + if backend is not None and mode != "run": + raise ValueError( + f"{example}: backend={backend} needs test=run; mark the fence " + f"{{test=run prelude=<name> backend={backend}}}" + ) + return settings + + +def fence_mode(example: CodeExample) -> str: + """Return how a Python fence is tested: run, compile or pytest. + + A fence on a docs page or in a docstring must say: one that runs is marked + ``{test=run prelude=<name>}``, one that cannot run says why with + ``{test=compile reason="..."}``. An unmarked fence is refused, so no block + is compile-only by accident. + """ + mode = fence_settings(example).get("test") + if mode is None: + raise ValueError( + f"{example}: the fence says neither test=run nor test=compile; mark it " + '{test=run prelude=<name>}, or {test=compile reason="<why it cannot ' + 'run>"} when it cannot run' + ) + return mode + + +_LINT_IGNORES = ["D", "T", "B", "C4", "E711", "E712", "E721"] +"""Ruff rules a docs block may break: blocks print, and show partial code.""" + + +def check_fence( + example: CodeExample, + *, + eval_example: EvalExample | None = None, + database_url: str | None = None, +) -> str | None: + """Hold one Python fence to its contract and return what it printed. + + The fence compiles (top-level ``await`` allowed), says how it is tested + (``fence_mode``), is linted when marked ``lint=check``, and runs when + marked ``test=run``. A ``test=pytest`` fence is run by the testing + how-to's own test. A ``backend=postgres`` fence runs only when given + ``database_url``. + + Args: + example: The fence, from a docs page or a docstring. + eval_example: pytest-examples' fixture, which lints a ``lint=check`` + fence. A fence marked ``lint=check`` without one is refused. + database_url: The PostgreSQL database a ``backend=postgres`` fence runs + against. Without it such a fence is only compiled and checked. + + Returns: + What a ``test=run`` fence printed, or ``None`` for one that only + compiles. + + Raises: + ValueError: The fence is unmarked or mis-marked. + AssertionError: The fence failed to run. + """ + compile( + example.source, str(example.path), "exec", flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT + ) + mode = fence_mode(example) + if fence_settings(example).get("lint") == "check": + if eval_example is None: + raise ValueError(f"{example}: lint=check needs the eval_example fixture") + eval_example.set_config( + ruff_ignore=_LINT_IGNORES, line_length=100, target_version="py313" + ) + eval_example.lint(example) + if mode != "run": + return None + if fence_settings(example).get("backend") == "postgres" and database_url is None: + return None + return run_fence(example, database_url=database_url) + + +# -- docstrings ------------------------------------------------------------------- + + +def _documented(obj: griffe.Object) -> Iterator[griffe.Object]: + yield obj + for member in obj.members.values(): + if not member.is_alias: + yield from _documented(member) + + +def docstring_examples(package: str, search_path: Path) -> list[CodeExample]: + """Collect the Python fences in the docstrings of every module of ``package``. + + griffe finds each docstring and its lines; the fences are read from the + file's own lines, so every example carries its real file and line. + """ + # Static only: once ``ferro`` is imported, griffe would otherwise inspect + # the compiled ``_core`` module and hand back its ``.so`` as a source file. + root = griffe.load(package, search_paths=[str(search_path)], allow_inspection=False) + examples: list[CodeExample] = [] + seen: set[tuple[Path, int]] = set() + for obj in _documented(root): + docstring = obj.docstring + if docstring is None or docstring.lineno is None: + continue + path = Path(obj.filepath) if not isinstance(obj.filepath, list) else None + if path is None or path.suffix not in {".py", ".pyi"}: + continue + if (path, docstring.lineno) in seen: + continue + seen.add((path, docstring.lineno)) + lines = path.read_text(encoding="utf-8").splitlines(keepends=True) + text = "".join(lines[docstring.lineno - 1 : docstring.endlineno]) + examples += fenced_examples(path, text, line_offset=docstring.lineno - 1) + return sorted(examples, key=lambda example: (str(example.path), example.start_line)) + + +def run_fence(example: CodeExample, *, database_url: str | None = None) -> str: + """Run a ``{test=run prelude=<name>}`` example and return what it printed. + + The example runs in a subprocess, so models and engines never leak + between examples. A failure raises ``AssertionError`` carrying the + traceback, whose frames name the example's own file and line. + + Args: + example: The fence. + database_url: The PostgreSQL database a ``backend=postgres`` fence runs + against; every ``connect()`` in the run is pointed at it. + + Raises: + ValueError: The fence is not marked ``test=run``, or is marked + ``backend=postgres`` and no ``database_url`` was given. + """ + settings = fence_settings(example) + if settings.get("test") != "run": + raise ValueError( + f"{example}: only a fence marked {{test=run prelude=<name>}} runs" + ) + prelude = settings["prelude"] + env = None + if settings.get("backend") == "postgres": + if database_url is None: + raise ValueError( + f"{example}: backend=postgres needs a database_url; it runs " + "under the postgres_only marker" + ) + env = {"FERRO_DOCS_DATABASE_URL": database_url} + try: + result = run_python( + [ + str(PRELUDES / "__init__.py"), + prelude, + str(example.path), + str(example.start_line + 1), + ], + input=example.source, + timeout=120, + env=env, + ) + except AssertionError as failure: + raise AssertionError( + f"{example} failed after prelude {prelude!r}\n{failure}" + ) from None + return result.stdout diff --git a/tests/_docs_shared.py b/tests/_docs_shared.py new file mode 100644 index 00000000..7abda701 --- /dev/null +++ b/tests/_docs_shared.py @@ -0,0 +1,182 @@ +"""The two pieces the docs harness shares with the scripts it runs. + +- ``snippet_regions`` reads the snippet-region grammar + (``--8<-- [start:<name>]`` … ``--8<-- [end:<name>]``) of an example script + or a Markdown file. The harness (``_docs_harness``, ``_docs_sql``), the + docs tests and ``docs/examples/_migrations_project.py`` all read regions + through it. +- ``point_connect_at`` points every ``ferro.connect`` call at the test + database. The SQL and output capture (``python -m tests._docs_sql``) and + the prelude runner (``docs/examples/_preludes/__init__.py``) both install it. + +It imports nothing beyond the standard library at import time, so a script +the harness starts (an example helper, the prelude runner) imports it without +loading the harness's own dependencies. ``tests._docs_harness`` re-exports both. +""" + +from __future__ import annotations + +import atexit +import re +from collections.abc import Callable +from dataclasses import dataclass +from typing import Any +from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit + +# -- snippet regions ------------------------------------------------------------------ + +SNIPPET = re.compile(r"--8<--\s*\[(?P<edge>start|end):(?P<name>[A-Za-z0-9_-]+)\]") +"""A snippet region's start or end marker: ``# --8<-- [start:<name>]`` in a +script, ``<!-- --8<-- [start:<name>] -->`` in Markdown.""" + + +@dataclass(frozen=True) +class SnippetRegion: + """One snippet region of a file.""" + + start: int + """The line of its ``[start:<name>]`` marker, counting from 1.""" + end: int + """The line of its ``[end:<name>]`` marker.""" + body: str + """The lines between the two markers, as written, each with its line end.""" + + +def snippet_regions(text: str, *, where: str = "<text>") -> dict[str, SnippetRegion]: + """Read every snippet region of ``text``, by name. + + A region runs from the line holding ``--8<-- [start:<name>]`` to the line + holding ``--8<-- [end:<name>]``; regions may nest or overlap. One marker per + line is read. + + Args: + text: An example script or a Markdown file. + where: The file's name, for the error message. + + Returns: + Each region's marker lines and body, in the order the regions start. + + Raises: + ValueError: A name starts twice, ends without starting, or never + ends. The message names the file, the line and the region. + """ + lines = text.splitlines(keepends=True) + open_at: dict[str, int] = {} + regions: dict[str, SnippetRegion] = {} + order: list[str] = [] + for number, line in enumerate(lines, start=1): + marker = SNIPPET.search(line) + if marker is None: + continue + name = marker["name"] + if marker["edge"] == "start": + if name in open_at or name in regions: + raise ValueError( + f"{where}:{number}: the snippet region {name!r} starts a " + "second time; give each region its own name" + ) + open_at[name] = number + order.append(name) + continue + if name not in open_at: + raise ValueError( + f"{where}:{number}: the snippet region {name!r} ends without " + f"starting; add '--8<-- [start:{name}]' above it" + ) + start = open_at.pop(name) + regions[name] = SnippetRegion(start, number, "".join(lines[start : number - 1])) + if open_at: + name, number = next(iter(open_at.items())) + raise ValueError( + f"{where}:{number}: the snippet region {name!r} never ends; add " + f"'--8<-- [end:{name}]' below it" + ) + return {name: regions[name] for name in order} + + +# -- the connect redirect ------------------------------------------------------------- + + +def redirected_url(named: str, target: str) -> str: + """Return the URL a ``connect(named)`` reaches when pointed at ``target``. + + The database is ``target``'s. When both name PostgreSQL, the query options + ``named`` carries are set on it too, over ``target``'s own, so a block + that connects with ``?ferro_search_path=tenant_a`` runs with that option. + Options of another database's URL (SQLite's ``mode=rwc``) mean nothing to + ``target`` and are left behind. + """ + if not (_is_postgres(named) and _is_postgres(target)): + return target + shown = dict(parse_qsl(urlsplit(named).query, keep_blank_values=True)) + if not shown: + return target + parts = urlsplit(target) + options = dict(parse_qsl(parts.query, keep_blank_values=True)) | shown + return urlunsplit(parts._replace(query=urlencode(options))) + + +def _is_postgres(url: str) -> bool: + return urlsplit(url).scheme in ("postgres", "postgresql") + + +def point_connect_at( + url: str | None, *, before: Callable[[], object] | None = None +) -> None: + """Send every ``ferro.connect`` call to ``url``, whatever URL it names. + + The URL is ``redirected_url(<the one named>, url)``. A + ``ferro_search_path`` it carries names a schema the test database may + lack, so the schema is created before the connect and dropped when the + process exits. Install it before anything imports ``connect`` by name. + + Args: + url: The database to connect to. ``None`` keeps the URL each call + names, so only ``before`` is added. + before: Called with no arguments before each connect. + """ + import ferro + + connect = ferro.connect + + async def connect_to(named: str, *args: Any, **kwargs: Any) -> Any: + if before is not None: + before() + reached = named if url is None else redirected_url(named, url) + if url is not None and reached != url: + _create_search_path_schema(reached) + return await connect(reached, *args, **kwargs) + + ferro.connect = connect_to # type: ignore[assignment] + + +_created_schemas: set[tuple[str, str]] = set() + + +def _create_search_path_schema(url: str) -> None: + parts = urlsplit(url) + options = dict(parse_qsl(parts.query, keep_blank_values=True)) + schema = options.pop("ferro_search_path", None) + if schema is None or not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", schema): + return # connect() itself refuses a value that is not one identifier + server = urlunsplit(parts._replace(query=urlencode(options))) + if (server, schema) in _created_schemas: + return + import psycopg + + with psycopg.connect(server, autocommit=True) as admin: + exists = admin.execute( + "SELECT 1 FROM pg_namespace WHERE nspname = %s", (schema,) + ).fetchone() + if exists: + return # the test's own schema, or one this run does not own + admin.execute(f'CREATE SCHEMA "{schema}"') + _created_schemas.add((server, schema)) + atexit.register(_drop_schema, server, schema) + + +def _drop_schema(server: str, schema: str) -> None: + import psycopg + + with psycopg.connect(server, autocommit=True) as admin: + admin.execute(f'DROP SCHEMA IF EXISTS "{schema}" CASCADE') diff --git a/tests/_docs_sql.py b/tests/_docs_sql.py new file mode 100644 index 00000000..dce5a2b0 --- /dev/null +++ b/tests/_docs_sql.py @@ -0,0 +1,669 @@ +"""What a docs example sends and prints, region by region, and the blocks that show it. + +A page shows two things an example script does: the SQL a snippet region +sends (a ```sql block opening with ``-- <example>.py:<snippet> (<dialect>)``) +and what it prints (a ```text block under ``<!-- prints: <example>.py:<snippet> -->``). +Both are regenerated, never typed: ``captured`` runs the script once per +dialect in a subprocess, records each statement and each printed line at the +example line that caused it, and groups them by snippet region; ``check_block`` +and ``check_printed`` compare a page's block with that record byte for byte. + +A capture runs ``python -m tests._docs_sql <example> <dialect> <out.json> <url|-> [args...]``: +the script runs as ``__main__`` with ``args`` as its arguments and +``ferro.connect`` pointed at ``url`` unless it is ``-``, and the record is +written to ``out.json``. + +What a block shows: + +- A query terminal's statement as its ``.sql()`` renders it, and any other + statement as the statement log records it. Schema statements, the Postgres + catalog probe, session settings and a ``transaction()`` block's own + ``BEGIN`` / ``COMMIT`` are left out (``HIDDEN_OPERATIONS``). A region that + sends nothing but prints statements (a ``create_tables()`` report) shows + what it prints. +- Each statement laid out one clause per line (``layout``), ending in ``;``, + with a blank line between two. +- A printed line belongs to the innermost line of the example inside a region + on the call stack, so a helper defined outside the region that prints for it + counts for the region. +""" + +from __future__ import annotations + +import io +import json +import logging +import re +import sys +import tempfile +from collections.abc import Awaitable, Callable +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +from tests._docs_harness import ( + DOCS_PAGES, + REPO_ROOT, + Fence, + fence_settings, + fenced_examples, + point_connect_at, + run_fence, + run_python, + snippet_regions, +) + +DOCS_EXAMPLES = REPO_ROOT / "docs" / "examples" + +MARKER = re.compile( + r"^-- (?P<example>[A-Za-z0-9_]+\.py):(?P<snippet>[A-Za-z0-9_-]+) " + r"\((?P<dialect>sqlite|postgres)\)$" +) +"""The first line of a ```sql block: ``-- <example>.py:<snippet> (<dialect>)``.""" + +PRINTS = re.compile( + r"^(?P<example>[A-Za-z0-9_]+\.py):(?P<snippet>[A-Za-z0-9_-]+)" + r"(?: \((?P<dialect>postgres)\))?$" +) +"""The value of a ```text block's ``<!-- prints: <example>.py:<snippet> -->`` +note, with `` (postgres)`` after it for what the region prints on PostgreSQL.""" + +HIDDEN_OPERATIONS = frozenset({"migrate", "catalog", "settings", "transaction"}) +"""Statement-log operations a block does not show: schema statements, the +Postgres catalog probe, session settings and a transaction block's own +statements.""" + +CLAUSES = ( + "FROM", + "INNER JOIN", + "LEFT JOIN", + "WHERE", + "GROUP BY", + "HAVING", + "ORDER BY", + "LIMIT", + "OFFSET", + "SET", + "VALUES", + "ON CONFLICT", + "RETURNING", +) +"""The keywords that start a new line of a laid-out statement, at the top +level of the statement only (never inside parentheses or quotes).""" + +PLACEHOLDER = re.compile(r"\$\d+|\?") +"""A bind placeholder: SQLite's ``?`` or PostgreSQL's ``$1``.""" + + +# -- the layout ------------------------------------------------------------------ + + +def layout(sql: str) -> str: + """Lay ``sql`` out one clause per line. + + A line break replaces the single space before each top-level clause + keyword (``CLAUSES``). A ``CREATE TABLE`` statement also puts each column + and constraint of its parenthesised body on its own line, indented two + spaces. Removing each line's indentation and joining the lines with + single spaces gives back ``sql``. + """ + table = sql.startswith("CREATE TABLE") + breaks: dict[int, str] = {} + depth = 0 + quote: str | None = None + for index, char in enumerate(sql): + if quote is not None: + if char == quote: + quote = None + continue + if char in "\"'": + quote = char + elif char == "(": + depth += 1 + elif char == ")": + depth -= 1 + elif char != " ": + continue + elif depth == 0: + rest = sql[index + 1 :] + if any( + rest.startswith(keyword) + and (len(rest) == len(keyword) or rest[len(keyword)] in " (") + for keyword in CLAUSES + ): + breaks[index] = "\n" + elif table and depth == 1: + # At depth 1 the body's own "(" or a "," between its items comes + # before the space; its closing ")" comes after it. + if sql[index - 1] in "(,": + breaks[index] = "\n " + elif sql[index + 1 : index + 2] == ")": + breaks[index] = "\n" + return "".join(breaks.get(index, char) for index, char in enumerate(sql)) + + +def unlaid(laid: str) -> str: + """Return the statement ``layout`` laid out: its lines unindented and joined.""" + return " ".join(line.lstrip(" ") for line in laid.splitlines()) + + +def statements(sqls: list[str]) -> str: + """Return the body a ```sql block holds for ``sqls``. + + Each statement is laid out and ends in ``;``, with a blank line between two. + """ + return "\n".join(f"{layout(sql)};\n" for sql in sqls) + + +def region_lines(source: str, *, where: str = "<source>") -> dict[str, tuple[int, int]]: + """Return each snippet region of an example's ``source`` as its marker lines.""" + return { + name: (region.start, region.end) + for name, region in snippet_regions(source, where=where).items() + } + + +# -- what a run records ---------------------------------------------------------- + + +@dataclass +class Region: + """What one snippet region of an example did during a run.""" + + sent: list[str] = field(default_factory=list) + """The statements it sent, in order.""" + printed: list[str] = field(default_factory=list) + """The lines it printed, in order, without their line ends.""" + + @property + def shown_sql(self) -> list[str]: + """The statements a ```sql block shows: what it sent, or, for a region + that sends nothing, the statements it prints.""" + return self.sent or self.printed + + @property + def shown_output(self) -> str: + """The text a ```text block under ``<!-- prints: ... -->`` shows.""" + return "".join(f"{line}\n" for line in self.printed) + + +@dataclass +class Run: + """Everything one run of an example script did.""" + + regions: dict[str, Region] + """Each snippet region's statements and printed lines.""" + stdout: str + """Everything the script printed.""" + schemas: list[Any] + """The registered models' SchemaIR each time the script connected or + created tables, and once at its end, consecutive repeats dropped.""" + + +class Capture: + """What one example run sends and prints, attributed to the example lines.""" + + def __init__(self, path: Path, dialect: str) -> None: + self.path = str(path) + self.dialect = dialect + self.root = str(path.parent) + self.ranges = region_lines(path.read_text(encoding="utf-8"), where=self.path) + self.where: tuple[str, int] | None = None + """The docs example file and line running now.""" + self.depth = 0 + """How many terminals are running; their own statements are rendered.""" + self.sent: list[tuple[int, str]] = [] + self.printed: list[tuple[int, str]] = [] + self.stdout = "" + self.pending = "" + self.schemas: list[Any] = [] + + def on_line(self, code: Any, line: int) -> Any: + """Note the example line running now (a ``sys.monitoring`` LINE callback).""" + if code.co_filename.startswith(self.root): + self.where = (code.co_filename, line) + return None + return sys.monitoring.DISABLE + + def record(self, sql: str) -> None: + if self.where is not None and self.where[0] == self.path: + self.sent.append((self.where[1], sql)) + + @property + def line(self) -> str: + """The example line running now, as ``<example>:<line>``.""" + if self.where is None: + return f"{Path(self.path).name}:?" + return f"{Path(self.where[0]).name}:{self.where[1]}" + + async def run_terminal( + self, render: Callable[[], Awaitable[Any]], call: Awaitable[Any] + ) -> Any: + """Record what a terminal sends, rendered by its ``.sql()``, then run it. + + A terminal another terminal runs is not rendered again: the outer + rendering already holds its statement. Its own statements reach the + statement log while ``depth`` is up, and are not recorded twice. + + When ``.sql()`` raises, the call must raise the same exception type, + having sent nothing; a call that then succeeds, or fails otherwise, + would send a statement no block could show, so it fails the capture + naming the example line. + """ + if self.depth: + return await call + self.depth += 1 + try: + line = self.line + try: + rendered = await render() + except Exception as refused: + try: + result = await call + except type(refused): + raise + except Exception as other: + raise AssertionError( + f"{line}: .sql() raised {refused!r} but the call raised " + f"{other!r}; a terminal must refuse the same way in both" + ) from other + raise AssertionError( + f"{line}: .sql() raised {refused!r} but the call succeeded " + f"(returned {result!r}), so what it sent cannot be shown" + ) from refused + assert rendered.dialect == self.dialect, ( + f"{line}: rendered for {rendered.dialect}, ran on {self.dialect}" + ) + self.record(str(rendered)) + return await call + finally: + self.depth -= 1 + + def _printing_line(self) -> int | None: + """The example line a print belongs to: the innermost one on the stack + inside a region, else the line running now.""" + frame = sys._getframe(1) + while frame is not None: + if frame.f_code.co_filename == self.path and any( + start < frame.f_lineno < end for start, end in self.ranges.values() + ): + return frame.f_lineno + frame = frame.f_back + if self.where is not None and self.where[0] == self.path: + return self.where[1] + return None + + def write(self, text: str) -> None: + """Record each line the example prints, at the example line printing it.""" + self.stdout += text + self.pending += text + *lines, self.pending = self.pending.split("\n") + if lines: + line = self._printing_line() + if line is not None: + self.printed.extend((line, printed) for printed in lines) + + def on_statement(self, record: logging.LogRecord) -> None: + operation = getattr(record, "operation", None) + if self.depth == 0 and operation not in HIDDEN_OPERATIONS: + assert record.dialect == self.dialect, record.dialect # type: ignore[attr-defined] + self.record(record.sql) # type: ignore[attr-defined] + + def snapshot(self) -> None: + """Record the registered models' SchemaIR, unless it is the last one.""" + from ferro.ir.compiler import compile_registry_schema_ir + + try: + ir = compile_registry_schema_ir() + except TypeError as refused: + # A model the script declared and Ferro refused (two primary keys) + # stays registered; the refusal is the schema the script ends with. + ir = {"refused": str(refused)} + schema = json.loads(json.dumps(ir, default=str)) + if not self.schemas or self.schemas[-1] != schema: + self.schemas.append(schema) + + def by_region(self) -> dict[str, Region]: + """Return what each snippet region sent and printed.""" + return { + name: Region( + sent=[sql for line, sql in self.sent if start < line < end], + printed=[text for line, text in self.printed if start < line < end], + ) + for name, (start, end) in self.ranges.items() + } + + def run(self) -> Run: + """Return the record of the run so far.""" + return Run(self.by_region(), self.stdout, self.schemas) + + +def _run_capture(path: Path, dialect: str, url: str | None, args: list[str]) -> Run: + """Run the example at ``path`` here, as ``__main__``, and return its record.""" + import runpy + + import ferro + from ferro.query.rendered import Terminal + + capture = Capture(path, dialect) + + class _Handler(logging.Handler): + def emit(self, record: logging.LogRecord) -> None: + capture.on_statement(record) + + logger = logging.getLogger("ferro.sql") + logger.addHandler(_Handler()) + logger.setLevel(logging.DEBUG) + + call = Terminal.__call__ + + def render_then_call(self: Terminal, *args: Any, **kwargs: Any) -> Any: + # Called first, so a terminal that refuses its query before returning + # an awaitable raises here, where the example expects it. + awaitable = call(self, *args, **kwargs) + + return capture.run_terminal(lambda: self.sql(*args, **kwargs), awaitable) + + Terminal.__call__ = render_then_call # type: ignore[method-assign] + + create_tables = ferro.create_tables + + async def create_then_snapshot(*args: Any, **kwargs: Any) -> Any: + capture.snapshot() + return await create_tables(*args, **kwargs) + + point_connect_at(url, before=capture.snapshot) + ferro.create_tables = create_then_snapshot # type: ignore[assignment] + + tool = 4 + sys.monitoring.use_tool_id(tool, "docs-capture") + sys.monitoring.register_callback(tool, sys.monitoring.events.LINE, capture.on_line) + sys.monitoring.set_events(tool, sys.monitoring.events.LINE) + sys.path.insert(0, str(path.parent)) + sys.argv = [str(path), *args] + stdout = sys.stdout + + class _Tee(io.TextIOBase): + def write(self, text: str) -> int: + capture.write(text) + return stdout.write(text) + + sys.stdout = _Tee() + try: + runpy.run_path(str(path), run_name="__main__") + finally: + sys.stdout = stdout + sys.monitoring.set_events(tool, 0) + sys.monitoring.free_tool_id(tool) + capture.snapshot() + return capture.run() + + +_captured: dict[tuple[str, str, str | None, tuple[str, ...]], Run] = {} +"""Each example's run per dialect and database, made once per test process.""" + + +def captured( + example: str | Path, + dialect: str = "sqlite", + url: str | None = None, + args: tuple[str, ...] = (), +) -> Run: + """Run an example script once in a subprocess and return what it did. + + Args: + example: A script name in ``docs/examples`` (``predicates.py``), or a + path to a script elsewhere. + dialect: The dialect the script runs on, ``sqlite`` or ``postgres``. + url: The database ``ferro.connect`` is pointed at. Defaults to the URL + the script passes. + args: The script's command-line arguments (``sys.argv[1:]``). + + Returns: + The run's record, cached for the rest of the test process. + + Raises: + AssertionError: The script failed; the message carries its output. + """ + path = example if isinstance(example, Path) else DOCS_EXAMPLES / example + key = (str(path), dialect, url, args) + if key not in _captured: + with tempfile.TemporaryDirectory() as scratch: + out = Path(scratch) / "run.json" + command = ["-m", "tests._docs_sql", str(path), dialect, str(out)] + command += [url or "-", *args] + run_python(command) + record = json.loads(out.read_text(encoding="utf-8")) + _captured[key] = Run( + regions={ + name: Region(**region) for name, region in record["regions"].items() + }, + stdout=record["stdout"], + schemas=record["schemas"], + ) + return _captured[key] + + +# -- the pages' blocks --------------------------------------------------------------- + + +def page_name(fence: Fence) -> str: + """The block's page relative to ``docs/pages``, as ``<page>:<line>``.""" + try: + page = Path(fence.path).relative_to(DOCS_PAGES).as_posix() + except ValueError: + page = str(fence.path) + return f"{page}:{fence.line}" + + +def sql_marker(block: Fence) -> re.Match[str] | None: + """The ``-- <example>.py:<snippet> (<dialect>)`` first line of a ```sql block.""" + return MARKER.match(block.body.partition("\n")[0]) + + +def _named_region(where: str, example: str, snippet: str, named: str) -> None: + path = DOCS_EXAMPLES / example + assert path.is_file(), f"{named}: docs/examples/{example} does not exist" + ranges = snippet_regions(path.read_text(encoding="utf-8"), where=str(path)) + assert snippet in ranges, f"{named}: {example} has no snippet region {snippet!r}" + + +def check_block(block: Fence, run: Callable[[str, str], Run] = captured) -> None: + """Assert a ```sql block is marked and holds what its example's region sends. + + Args: + block: The page's block. + run: Given an example script's name and a dialect, its run's record. + Defaults to ``captured`` on the dialect's own database. + + Raises: + AssertionError: The block has no marker, names an example or region + that does not exist, or differs from what the region sends. The + message names the page, the line and the example. + """ + where = page_name(block) + marker = sql_marker(block) + assert marker is not None, ( + f"{where}: a ```sql block must open with a marker line " + f"'-- <example>.py:<snippet> (<dialect>)' naming where its SQL comes " + f"from; this one opens with {block.body.partition(chr(10))[0]!r}" + ) + example, snippet, dialect = marker["example"], marker["snippet"], marker["dialect"] + named = f"{where} ({example}:{snippet}, {dialect})" + _named_region(where, example, snippet, named) + region = run(example, dialect).regions.get(snippet, Region()) + sent = region.shown_sql + assert sent, f"{named}: the region {snippet!r} of {example} sends nothing" + expected = statements(sent) + shown = block.body.partition("\n")[2] + assert shown == expected, ( + f"{named}: the block differs from what {example}:{snippet} sends on " + f"{dialect}. It sends:\n{expected}" + ) + + +def check_same_on_postgres(block: Fence, run: Callable[[str, str], Run]) -> None: + """Assert a SQLite ```sql block stands for PostgreSQL too. + + The region must send the same statements there once each placeholder is + read as ``?``. A region whose SQL differs in more than its placeholders + shows a ``(postgres)`` block of its own, and this check does not apply to it. + + Raises: + AssertionError: PostgreSQL's statements differ in more than their + placeholders. The message names the page, the line and the example. + """ + where = page_name(block) + marker = sql_marker(block) + assert marker is not None, f"{where}: a ```sql block has no marker line" + example, snippet = marker["example"], marker["snippet"] + sent = run(example, "postgres").regions.get(snippet, Region()).shown_sql + expected = PLACEHOLDER.sub("?", statements(sent)) + shown = block.body.partition("\n")[2] + assert PLACEHOLDER.sub("?", shown) == expected, ( + f"{where} ({example}:{snippet}, sqlite): the block stands for both " + f"databases, but on postgres the region sends more than other " + f"placeholders; show a (postgres) block beside it. It sends:\n" + f"{statements(sent)}" + ) + + +NOT_PRINTED = "shows" +"""The note kind of a ```text block holding text Ferro does not print (a +sketch, a file tree, another tool's shell session, a prompt to paste), which +says what it shows instead: ``<!-- shows: the directory tree -->``. Whatever +Ferro prints, raises or warns is a ``prints:`` block.""" + +TEXT_NOTES = { + "prints": ( + "what an example region, or the run fence above, prints; regenerated by " + "tests/test_docs_output.py" + ), + "output": "what ferro migrate prints, regenerated by tests/test_docs_migrations.py", + "refusal": "a refusal ferro prints, regenerated by tests/test_docs_migrations.py", + NOT_PRINTED: "text Ferro does not print; the note says what it shows", +} +"""Every note a ```text block can carry, and what checks it.""" + + +def check_printed( + block: Fence, + run: Callable[[str, str], Run] = captured, + *, + dialect: str = "sqlite", + database_url: str | None = None, +) -> None: + """Assert a ```text block says where it comes from, and holds what it says. + + A block under ``<!-- prints: <example>.py:<snippet> -->`` must equal what + that region prints on SQLite, byte for byte, and one under + ``<!-- prints: <example>.py:<snippet> (postgres) -->`` what it prints on + PostgreSQL. One under ``<!-- prints: above -->`` must equal what the + nearest Python fence above it prints, a fence marked + ``{test=run prelude=<name>}``. A block under another note + (``TEXT_NOTES``) is checked elsewhere, or says what it shows instead. + + Args: + block: The page's ```text block. + run: Given an example script's name and a dialect, its run's record. + Defaults to ``captured``. + dialect: The database this check runs against. A ``prints:`` block for + the other one only has its example and region checked, or its + fence found. + database_url: The PostgreSQL database a ``prints: above`` block's + fence runs against when that fence is marked ``backend=postgres``. + + Raises: + AssertionError: The block has no note, an unknown one, names an + example, region or fence that does not exist, or differs from what + it prints. The message names the page and the line. + """ + where = page_name(block) + notes = ", ".join(f"<!-- {kind}: ... -->" for kind in TEXT_NOTES) + assert block.note is not None, ( + f"{where}: a ```text block must sit under a note saying where it comes " + f"from: <!-- prints: <example>.py:<snippet> --> for what an example " + f"region prints, <!-- prints: above --> for what the run fence above " + f"prints, or <!-- shows: <what it is> --> for text no script prints " + f"(one of {notes})" + ) + kind, value = block.note + assert kind in TEXT_NOTES, f"{where}: unknown note {kind!r}; use one of {notes}" + if kind != "prints": + return + if value == "above": + _check_printed_above(block, where, dialect, database_url) + return + marker = PRINTS.match(value) + assert marker is not None, ( + f"{where}: <!-- prints: {value} --> must name <example>.py:<snippet>, " + "or say above" + ) + example, snippet = marker["example"], marker["snippet"] + named = f"{where} ({example}:{snippet})" + _named_region(where, example, snippet, named) + if (marker["dialect"] or "sqlite") != dialect: + return + printed = run(example, dialect).regions.get(snippet, Region()).shown_output + assert printed, f"{named}: the region {snippet!r} of {example} prints nothing" + assert block.body == printed, ( + f"{named}: the block differs from what {example}:{snippet} prints. " + f"It prints:\n{printed}" + ) + + +def printing_fence(block: Fence) -> Any: + """The Python fence a ``<!-- prints: above -->`` block shows the output of: + the nearest one above it, or ``None``.""" + path = Path(block.path) + fences = [ + example + for example in fenced_examples(path, path.read_text(encoding="utf-8")) + if example.start_line < block.line + ] + return fences[-1] if fences else None + + +def on_postgres(fence: Any) -> bool: + """Whether a Python fence is marked ``backend=postgres``.""" + return fence_settings(fence).get("backend") == "postgres" + + +def _check_printed_above( + block: Fence, where: str, dialect: str, database_url: str | None +) -> None: + above = printing_fence(block) + assert above is not None, ( + f"{where}: <!-- prints: above --> has no Python fence above it" + ) + assert fence_settings(above).get("test") == "run", ( + f"{where}: <!-- prints: above --> needs the fence above " + f"(line {above.start_line}) marked {{test=run prelude=<name>}}" + ) + if dialect != ("postgres" if on_postgres(above) else "sqlite"): + return + printed = run_fence(above, database_url=database_url) + assert block.body == printed, ( + f"{where}: the block differs from what the fence at line " + f"{above.start_line} prints. It prints:\n{printed}" + ) + + +if __name__ == "__main__": + script, dialect, out, url, *args = sys.argv[1:] + result = _run_capture( + Path(script).resolve(), dialect, None if url == "-" else url, args + ) + Path(out).write_text( + json.dumps( + { + "regions": { + name: {"sent": region.sent, "printed": region.printed} + for name, region in result.regions.items() + }, + "stdout": result.stdout, + "schemas": result.schemas, + } + ), + encoding="utf-8", + ) diff --git a/tests/_docs_twins.py b/tests/_docs_twins.py new file mode 100644 index 00000000..47d0738d --- /dev/null +++ b/tests/_docs_twins.py @@ -0,0 +1,308 @@ +"""The Annotated twin of a docs example script, derived from its Assignment form. + +AGENTS.md I-7 shows every field-declaring example in two tabs, and the two +differ only in where the field options sit. A twin is therefore not written by +hand: ``annotated_twin`` rewrites each ``name: T = Field(...)`` of the +Assignment script into the one Annotated idiom of +``docs/solutions/patterns/docs-page-conventions.md``: + +- options move inside ``Annotated[T, Field(...)]``; +- a ``default`` stays the assigned value (``= None``), where type checkers + see it; +- a ``default_factory`` stays an assigned ``Field(default_factory=...)``, for + the same reason; +- a comment quoting a declaration (``# before: bio: str = Field(index=True)``) + quotes it in the same idiom; +- everything else in the script (data, queries, prints, asserts) is the same + text, so both twins print the same thing. + +Run ``python -m tests._docs_twins docs/examples/<name>.py`` to write +``<name>_annotated.py`` after changing ``<name>.py``. +""" + +from __future__ import annotations + +import ast +import io +import re +import subprocess +import sys +import tokenize +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +DOCS_EXAMPLES = REPO_ROOT / "docs" / "examples" + +_FIELD_CALLS = frozenset({"Field", "FerroField"}) +_ASSIGNED = frozenset({"default", "default_factory"}) + + +def twin_pairs() -> list[tuple[Path, Path]]: + """Every ``docs/examples/<name>.py`` that has a ``<name>_annotated.py``.""" + return [ + (path.with_name(path.name.removesuffix("_annotated.py") + ".py"), path) + for path in sorted(DOCS_EXAMPLES.glob("*_annotated.py")) + ] + + +def _segment(source: str, node: ast.AST) -> str: + text = ast.get_source_segment(source, node) + assert text is not None + return text + + +def _declaration(source: str, node: ast.AnnAssign) -> str | None: + """The Annotated form of one ``name: T = Field(...)``, or ``None`` when the + statement is not one.""" + call = node.value + if not ( + isinstance(call, ast.Call) + and isinstance(call.func, ast.Name) + and call.func.id in _FIELD_CALLS + ): + return None + if any(keyword.arg is None for keyword in call.keywords): + raise ValueError( + f"line {node.lineno}: a **kwargs Field call has no Annotated twin; " + "spell the options out" + ) + default: str | None = None + if call.args: + if len(call.args) > 1: + raise ValueError(f"line {node.lineno}: Field takes one positional default") + default = _segment(source, call.args[0]) + factory: str | None = None + options: list[str] = [] + for keyword in call.keywords: + value = _segment(source, keyword.value) + if keyword.arg == "default": + default = value + elif keyword.arg == "default_factory": + factory = value + else: + options.append(f"{keyword.arg}={value}") + target = _segment(source, node.target) + annotation = _segment(source, node.annotation) + if options: + metadata = f"{call.func.id}({', '.join(options)})" + if ( + isinstance(node.annotation, ast.Subscript) + and isinstance(node.annotation.value, ast.Name) + and node.annotation.value.id == "Annotated" + ): + annotation = f"{annotation[:-1]}, {metadata}]" + else: + annotation = f"Annotated[{annotation}, {metadata}]" + declaration = f"{target}: {annotation}" + if factory is not None: + declaration += f" = {call.func.id}(default_factory={factory})" + elif default is not None: + declaration += f" = {default}" + return declaration + + +def _module_name(node: ast.Import | ast.ImportFrom) -> str: + if isinstance(node, ast.ImportFrom): + return node.module or "" + return node.names[0].name + + +def _sorted_names(names: list[str]) -> list[str]: + """``names`` in ruff's order: constants, then classes, then the rest.""" + return sorted( + names, + key=lambda name: ( + 0 if name.isupper() and len(name) > 1 else 1 if name[0].isupper() else 2, + name, + ), + ) + + +def _import_groups(tree: ast.Module) -> list[tuple[list[ast.stmt], int]]: + """Each run of top-level imports, with the line where the code it serves ends. + + An example whose regions each stand alone imports again at the top of each + region, so each run serves the code up to the next run. + """ + groups: list[list[ast.stmt]] = [] + previous_was_import = False + for node in tree.body: + is_import = isinstance(node, ast.Import | ast.ImportFrom) + if is_import and not previous_was_import: + groups.append([]) + if is_import: + groups[-1].append(node) + previous_was_import = is_import + ends = [group[0].lineno - 1 for group in groups[1:]] + [10**9] + return list(zip(groups, ends, strict=True)) + + +def _shared_comment(lines: list[str], group: list[ast.stmt]) -> str: + """The trailing comment every import of ``group`` carries (``# noqa: E402`` + on imports placed after code), to carry onto the one added beside them.""" + comments = { + lines[(node.end_lineno or node.lineno) - 1].partition(" #")[2].rstrip("\n") + for node in group + } + [comment] = comments if len(comments) == 1 else [""] + return f" #{comment}" if comment else "" + + +def _with_annotated_import(source: str) -> str: + """``source`` importing ``Annotated`` from ``typing`` in every run of imports + whose code uses it, where ruff's import order puts it.""" + tree = ast.parse(source) + used = sorted( + node.lineno + for node in ast.walk(tree) + if isinstance(node, ast.Name) and node.id == "Annotated" + ) + lines = source.splitlines(keepends=True) + edits: list[tuple[int, int, list[str]]] = [] + for group, ends_at in _import_groups(tree): + starts_at = group[-1].end_lineno or group[-1].lineno + if not any(starts_at < line <= ends_at for line in used): + continue + typing = next( + ( + node + for node in group + if isinstance(node, ast.ImportFrom) and node.module == "typing" + ), + None, + ) + if typing is not None: + names = [alias.name for alias in typing.names] + if "Annotated" not in names: + names = _sorted_names([*names, "Annotated"]) + edits.append( + ( + typing.lineno - 1, + typing.end_lineno or typing.lineno, + [ + f"from typing import {', '.join(names)}" + f"{_shared_comment(lines, group)}\n" + ], + ) + ) + continue + stdlib = [ + node + for node in group + if _module_name(node).split(".")[0] in sys.stdlib_module_names + and _module_name(node) != "__future__" + ] + line = f"from typing import Annotated{_shared_comment(lines, group)}\n" + if stdlib: + after = [ + node + for node in stdlib + if isinstance(node, ast.ImportFrom) and _module_name(node) > "typing" + ] + at = after[0].lineno - 1 if after else stdlib[-1].end_lineno or 0 + edits.append((at, at, [line])) + else: + first = next(node for node in group if _module_name(node) != "__future__") + edits.append((first.lineno - 1, first.lineno - 1, [line, "\n"])) + for start, stop, text in sorted(edits, reverse=True): + lines[start:stop] = text + return "".join(lines) + + +_QUOTED = re.compile(r"^# (?P<lead>[a-z ]+: )(?P<code>\w+: .+)$") + + +def _quoted_declaration(comment: str) -> str | None: + """A whole-line comment quoting a declaration (``# before: bio: str = + Field(index=True)``), with the declaration in the Annotated idiom, or + ``None`` when the comment quotes none.""" + match = _QUOTED.match(comment) + if match is None: + return None + try: + [node] = ast.parse(match["code"]).body + except SyntaxError: + return None + if not isinstance(node, ast.AnnAssign) or node.value is None: + return None + declaration = _declaration(match["code"], node) + if declaration is None: + return None + return f"# {match['lead']}{declaration}" + + +def _formatted(source: str) -> str: + result = subprocess.run( + [sys.executable, "-m", "ruff", "format", "--stdin-filename", "twin.py", "-"], + input=source, + capture_output=True, + text=True, + cwd=REPO_ROOT, + check=False, + ) + if result.returncode != 0: + raise ValueError(f"ruff format refused the twin:\n{result.stderr}") + return result.stdout + + +def annotated_twin(source: str) -> str: + """Rewrite an Assignment-style example script into its Annotated twin. + + Args: + source: The Assignment script, formatted by ruff. + + Returns: + The twin, formatted by ruff: every ``Field(...)`` declaration in the + Annotated idiom, ``Annotated`` imported, every other line unchanged. + + Raises: + ValueError: A declaration has no Annotated twin (``Field(**options)``), + or a comment sits inside a declaration the rewrite would drop. + """ + tree = ast.parse(source) + lines = source.splitlines(keepends=True) + comments = { + token.start[0]: token + for token in tokenize.generate_tokens(io.StringIO(source).readline) + if token.type == tokenize.COMMENT + } + rewrites: list[tuple[int, int, str]] = [] + for node in ast.walk(tree): + if not isinstance(node, ast.AnnAssign) or node.value is None: + continue + declaration = _declaration(source, node) + if declaration is None: + continue + end = node.end_lineno or node.lineno + inside = [line for line in range(node.lineno, end) if line in comments] + if inside: + raise ValueError( + f"line {inside[0]}: a comment inside a multi-line Field " + "declaration would be lost; move it above the field" + ) + trailing = f" {comments[end].string}" if end in comments else "" + indent = lines[node.lineno - 1][: node.col_offset] + rewrites.append((node.lineno, end, f"{indent}{declaration}{trailing}\n")) + for line, token in comments.items(): + quoted = _quoted_declaration(token.string) + if quoted is not None and not any(s <= line <= e for s, e, _ in rewrites): + start = token.start[1] + text = lines[line - 1] + rewrites.append((line, line, f"{text[:start]}{quoted}\n")) + if not rewrites: + return source + for start, end, text in sorted(rewrites, reverse=True): + lines[start - 1 : end] = [text] + return _formatted(_with_annotated_import("".join(lines))) + + +def main(argv: list[str]) -> None: + for name in argv: + path = Path(name) + twin = path.with_name(f"{path.stem}_annotated.py") + twin.write_text(annotated_twin(path.read_text(encoding="utf-8")), "utf-8") + print(f"wrote {twin}") + + +if __name__ == "__main__": + main(sys.argv[1:]) diff --git a/tests/test_chunked_steps.py b/tests/test_chunked_steps.py index 84a6ca60..0b409692 100644 --- a/tests/test_chunked_steps.py +++ b/tests/test_chunked_steps.py @@ -40,7 +40,7 @@ async def up(ctx, batch): import ferro from ferro.migrations import runner from ferro.migrations.report import RunRefused -from ferro.migrations.chunked import decode_cursor, encode_cursor +from ferro.migrations._chunk_loop import decode_cursor, encode_cursor from ferro.settings import FerroSettings from tests.test_migrate_new import ( # noqa: F401 - fixtures pkg, diff --git a/tests/test_cli_help.py b/tests/test_cli_help.py new file mode 100644 index 00000000..443508ef --- /dev/null +++ b/tests/test_cli_help.py @@ -0,0 +1,119 @@ +"""``--help`` prints every ``<placeholder>`` written in a command's help source. + +Help rendered as Markdown swallows ``<migration>:<step>`` as an HTML tag, so +``ferro migrate rerecord --help`` used to read "The step whose edit to +accept: :, as 0007:01." Help is plain text; this pins it for every command. +""" + +import inspect +import re +from collections.abc import Iterator + +import pytest + +from ferro.cli import app + +PLACEHOLDER = re.compile(r"<[A-Za-z_][^<>\s]*>") + + +def _commands( + root, path: tuple[str, ...] = () +) -> Iterator[tuple[tuple[str, ...], object]]: + yield path, root + for name, sub in root._commands.items(): + if name.startswith("-"): + continue + yield from _commands(sub, (*path, name)) + + +def _help_source(command) -> str: + """Every help string written for the command: summary, docstring, parameters.""" + parts = [] + if command.help: + parts.append(command.help) + func = command.default_command + if func is not None: + parts.append(inspect.getdoc(func) or "") + for argument in command.assemble_argument_collection(): + parts.append(argument.parameter.help or "") + return "\n".join(parts) + + +CASES = [ + pytest.param(path, command, id=" ".join(("ferro", *path))) + for path, command in _commands(app._meta_parent) +] + + +@pytest.mark.parametrize(("path", "command"), CASES) +def test_help_keeps_every_placeholder(path, command, capsys) -> None: + placeholders = set(PLACEHOLDER.findall(_help_source(command))) + try: + app([*path, "--help"]) + except SystemExit: + pass + out = capsys.readouterr().out + # The help table wraps long lines; compare with whitespace folded away. + folded = re.sub(r"\s+", "", out) + missing = sorted(p for p in placeholders if re.sub(r"\s+", "", p) not in folded) + assert not missing, f"--help dropped {missing}:\n{out}" + + +def test_the_scan_sees_the_placeholders_the_ticket_names() -> None: + found = { + " ".join(path): set(PLACEHOLDER.findall(_help_source(command))) + for path, command in _commands(app._meta_parent) + } + assert {"<migration>", "<step>"} <= found["migrate rerecord"] + assert found["migrate new"] + assert found["migrate squash"] + + +MARKDOWN = re.compile(r"`|\*\*|\[[^\]]+\]\([^)]+\)") + + +@pytest.mark.parametrize(("path", "command"), CASES) +def test_help_carries_no_markdown(path, command) -> None: + # Help prints as plain text, so Markdown shows up literally: a `new` in + # backticks prints with its backticks. + found = MARKDOWN.findall(_help_source(command)) + assert not found, f"{' '.join(('ferro', *path))} help carries Markdown {found}" + + +@pytest.mark.parametrize(("path", "command"), CASES) +def test_help_summary_is_one_line(path, command) -> None: + func = command.default_command + doc = inspect.getdoc(func) if func is not None else command.help + summary = (doc or "").split("\n\n")[0] + assert summary[:1].isupper() and "\n" not in summary and summary.endswith("."), ( + f"{' '.join(('ferro', *path))}: summary {summary!r} is not one capitalised line ending " + "with a period" + ) + + +@pytest.mark.parametrize( + ("flag", "parameter", "value"), + [ + ("--steps", "steps", True), + ("--no-steps", "steps", False), + ("--json", "json_", True), + ("--no-json", "json_", False), + ], +) +def test_status_accepts_both_spellings_of_its_flags(flag, parameter, value) -> None: + # `--no-steps` and `--no-json` shipped with `ferro migrate status`; a + # script that passes them must keep parsing. + _, bound, _ = app["migrate"].parse_args( + ["status", flag], exit_on_error=False, print_error=False + ) + assert bound.arguments[parameter] is value + + +def test_status_help_lists_the_negative_flags(capsys) -> None: + try: + app(["migrate", "status", "--help"]) + except SystemExit: + pass + out = capsys.readouterr().out + assert "--no-steps" in out + assert "--no-json" in out diff --git a/tests/test_connection.py b/tests/test_connection.py index b5db02e2..0df598d2 100644 --- a/tests/test_connection.py +++ b/tests/test_connection.py @@ -492,6 +492,12 @@ async def test_pool_config_rejects_min_connections_above_max(tmp_path): ferro.PoolConfig(max_connections=1, min_connections=2) +def test_pool_config_validate_returns_the_instance(): + """`PoolConfig.validate` is the public bounds check, not pydantic's classmethod.""" + pool = ferro.PoolConfig(max_connections=3, min_connections=1) + assert pool.validate() is pool + + @pytest.mark.asyncio async def test_connect_passes_pool_config_to_core(monkeypatch): """Validated pool configuration should cross the Python/Rust API boundary.""" diff --git a/tests/test_docs_checks.py b/tests/test_docs_checks.py new file mode 100644 index 00000000..7721d55b --- /dev/null +++ b/tests/test_docs_checks.py @@ -0,0 +1,238 @@ +"""The docs checks: one-letter query lambdas and error checks that cannot fail.""" + +from __future__ import annotations + +import subprocess +import sys +import textwrap +from pathlib import Path + +from tests._docs_checks import ( + Violation, + lambda_param_violations, + query_methods_taking_callables, + unfailing_error_checks, +) + +REPO_ROOT = Path(__file__).resolve().parents[1] + + +def _write(tmp_path: Path, name: str, text: str) -> Path: + path = tmp_path / name + path.write_text(textwrap.dedent(text).lstrip("\n"), encoding="utf-8") + return path + + +def test_two_letter_lambda_parameter_passes(tmp_path: Path) -> None: + path = _write( + tmp_path, + "ok.py", + """ + authors = Author.where(lambda author: author.id == 1) + """, + ) + assert lambda_param_violations(path) == [] + + +def test_one_letter_lambda_parameter_fails_with_its_line(tmp_path: Path) -> None: + path = _write( + tmp_path, + "bad.py", + """ + import ferro + + authors = Author.where(lambda a: a.id == 1) + """, + ) + assert lambda_param_violations(path) == [ + Violation( + path, + 3, + "lambda passed to where() takes one-letter parameter 'a'; " + "name it after the model in lowercase singular", + ) + ] + + +def test_lambda_passed_to_sorted_is_ignored(tmp_path: Path) -> None: + path = _write( + tmp_path, + "sorted.py", + """ + names = sorted(rows, key=lambda r: r.name) + rows.sort(key=lambda r: r.name) + """, + ) + assert lambda_param_violations(path) == [] + + +def test_parameters_captured_by_default_are_not_query_parameters( + tmp_path: Path, +) -> None: + path = _write( + tmp_path, + "capture.py", + """ + Post.where(lambda post, f=field, v=value: getattr(post, f) == v) + Post.where(lambda p, f=field: getattr(p, f)) + """, + ) + assert [(v.line, "'p'" in v.message) for v in lambda_param_violations(path)] == [ + (2, True) + ] + + +def test_every_named_query_method_is_checked(tmp_path: Path) -> None: + methods = query_methods_taking_callables() + # Derived from the Query and Model signatures, plus the names the + # docs use on aggregate projections. + assert { + "where", + "select", + "order_by", + "include", + "join", + "left_join", + "update", + "exists", + "count", + "aggregate", + "group_by", + "after", + "before", + } <= methods + lines = "\n".join(f"q.{name}(lambda x: x.id)" for name in sorted(methods)) + path = _write(tmp_path, "all.py", lines + "\n") + assert [v.line for v in lambda_param_violations(path)] == list( + range(1, len(methods) + 1) + ) + + +def test_markdown_fences_are_checked_at_their_page_line(tmp_path: Path) -> None: + path = _write( + tmp_path, + "page.md", + """ + # Page + + === "Tab" + + ```python + posts = await Post.where(lambda p: p.published).all() + ``` + + ```text + Post.where(lambda p: p.published) + ``` + """, + ) + assert [(v.line, "'p'" in v.message) for v in lambda_param_violations(path)] == [ + (6, True) + ] + + +def test_docstring_fences_are_checked_at_their_file_line(tmp_path: Path) -> None: + path = _write( + tmp_path, + "module.py", + ''' + """A module. + + ```python + posts = await Post.where(lambda post: post.published).all() + ``` + """ + + + def helper(): + """Return the published posts. + + Examples: + ```python + posts = await Post.where(lambda p: p.published).all() + ``` + """ + ''', + ) + assert [v.line for v in lambda_param_violations(path)] == [14] + + +def test_except_that_asserts_without_else_is_reported(tmp_path: Path) -> None: + path = _write( + tmp_path, + "errors.py", + """ + try: + Post.where(lambda post: post.views.sum() > 100) + except TypeError as exc: + assert "having()" in str(exc) + """, + ) + assert unfailing_error_checks(path) == [ + Violation( + path, + 1, + "try/except asserts on the error but has no else that raises, " + "so it passes when the error stops happening; add " + 'else: raise AssertionError("expected TypeError")', + ) + ] + + +def test_except_that_asserts_with_raising_else_is_not_reported( + tmp_path: Path, +) -> None: + path = _write( + tmp_path, + "errors.py", + """ + try: + Post.where(lambda post: post.views.sum() > 100) + except TypeError as exc: + assert "having()" in str(exc) + else: + raise AssertionError("expected TypeError") + """, + ) + assert unfailing_error_checks(path) == [] + + +def test_except_without_assert_is_not_an_error_check(tmp_path: Path) -> None: + path = _write( + tmp_path, + "cleanup.py", + """ + try: + import tomllib + except ImportError: + tomllib = None + """, + ) + assert unfailing_error_checks(path) == [] + + +def test_cli_lists_violations_and_exits_1(tmp_path: Path) -> None: + _write(tmp_path, "good.py", "Post.where(lambda post: post.id)\n") + bad = _write(tmp_path, "bad.py", "Post.where(lambda p: p.id)\n") + result = subprocess.run( + [sys.executable, "-m", "tests._docs_checks", str(tmp_path)], + capture_output=True, + text=True, + cwd=REPO_ROOT, + ) + assert result.returncode == 1, result.stderr + assert result.stdout.splitlines() == [ + f"{bad}:1: lambda passed to where() takes one-letter parameter 'p'; " + "name it after the model in lowercase singular" + ] + + +def test_cli_exits_0_when_clean(tmp_path: Path) -> None: + good = _write(tmp_path, "good.py", "Post.where(lambda post: post.id)\n") + result = subprocess.run( + [sys.executable, "-m", "tests._docs_checks", str(good)], + capture_output=True, + text=True, + cwd=REPO_ROOT, + ) + assert (result.returncode, result.stdout) == (0, ""), result.stderr diff --git a/tests/test_docs_cli_reference.py b/tests/test_docs_cli_reference.py new file mode 100644 index 00000000..a07b0d02 --- /dev/null +++ b/tests/test_docs_cli_reference.py @@ -0,0 +1,91 @@ +"""The CLI reference lists every flag each ``ferro migrate`` verb accepts. + +For each verb, every ``--flag`` its ``--help`` prints, bar the global options +(``--config``, ``--database``, ``--url``, ``--help``, ``--version``), appears in +the verb's ``## `ferro migrate <verb> ...``` section of +``docs/pages/reference/cli.md``. The global options appear in its +``## Global options`` section. A flag added to the CLI and not to the page +fails here, naming the verb and the flag. +""" + +from __future__ import annotations + +import re + +import pytest + +from ferro.cli import app +from tests._docs_harness import DOCS_PAGES + +CLI_REFERENCE = DOCS_PAGES / "reference" / "cli.md" +GLOBAL_OPTIONS = ("--config", "--database", "--url", "--help", "--version") +FLAG = re.compile(r"(?<![\w-])--[a-z][a-z0-9-]*") + + +def _verbs() -> list[str]: + migrate = app._meta_parent._commands["migrate"] + return sorted(name for name in migrate._commands if not name.startswith("-")) + + +POSITIONAL = re.compile( + r"(?<![\w-])(?P<name>[A-Z][A-Z_]*) +(?P<flag>--[a-z][a-z0-9-]*)" +) +"""A positional argument as ``--help`` lists it, with its keyword spelling: +``NAME --name``.""" + + +def _help_flags(verb: str, capsys: pytest.CaptureFixture[str]) -> set[str]: + """The verb's flags as ``--help`` prints them, bar the global options and + the keyword spellings of its positional arguments (the heading names + those, as ``ferro migrate new NAME``).""" + try: + app(["migrate", verb, "--help"]) + except SystemExit: + pass + printed = capsys.readouterr().out + positionals = {match["flag"] for match in POSITIONAL.finditer(printed)} + return set(FLAG.findall(printed)) - set(GLOBAL_OPTIONS) - positionals + + +def _section(heading: re.Pattern[str]) -> str: + """The text of the cli.md section whose ``##`` heading matches ``heading``.""" + text = CLI_REFERENCE.read_text(encoding="utf-8") + sections = re.split(r"(?m)^(?=## )", text) + found = [section for section in sections if heading.match(section)] + assert len(found) == 1, f"cli.md has {len(found)} sections matching {heading}" + return found[0] + + +@pytest.mark.parametrize("verb", _verbs()) +def test_the_reference_lists_every_flag_of_the_verb( + verb: str, capsys: pytest.CaptureFixture[str] +) -> None: + flags = _help_flags(verb, capsys) + section = _section(re.compile(rf"## `ferro migrate {verb}[` ]")) + listed = set(FLAG.findall(section)) + missing = sorted(flags - listed) + assert not missing, ( + f"ferro migrate {verb} --help prints {missing}, which its section of " + f"docs/pages/reference/cli.md does not list" + ) + + +def test_the_reference_lists_every_global_option() -> None: + section = _section(re.compile(r"## Global options")) + missing = [flag for flag in GLOBAL_OPTIONS if flag not in section] + assert not missing, f"cli.md's Global options section omits {missing}" + + +def test_every_verb_has_a_section() -> None: + assert _verbs() == [ + "baseline", + "check", + "down", + "drift", + "init", + "new", + "rerecord", + "squash", + "status", + "up", + ] diff --git a/tests/test_docs_examples.py b/tests/test_docs_examples.py index a101651a..47ccea87 100644 --- a/tests/test_docs_examples.py +++ b/tests/test_docs_examples.py @@ -1,61 +1,84 @@ from __future__ import annotations -import ast -import subprocess -import sys from pathlib import Path import pytest -from pytest_examples import CodeExample, EvalExample, find_examples +from pytest_examples import CodeExample, EvalExample + +from tests._docs_harness import ( + check_fence, + fence_mode, + fence_settings, + fenced_blocks, + fenced_examples, + page_examples, + run_fence, + run_python, + snippet_regions, +) REPO_ROOT = Path(__file__).resolve().parents[1] DOCS_PAGES = REPO_ROOT / "docs" / "pages" DOCS_EXAMPLES = REPO_ROOT / "docs" / "examples" -def _inline_examples() -> list[CodeExample]: - # Snippet directives (--8<--) are expanded by the docs build, not by - # pytest-examples, so blocks containing them are not valid Python here. +@pytest.mark.parametrize("example", page_examples(), ids=str) +def test_docs_examples(example: CodeExample, eval_example: EvalExample) -> None: + """Every inline page block compiles; one marked to run, runs. + + The blocks are every Python fence a page shows, including those of the + Markdown a page pulls in (``CONTRIBUTING.md``). Each says how it is tested + (``tests._docs_harness.fence_mode``): ``{test=run prelude=<name>}`` runs it + after that prelude, ``{test=compile reason="..."}`` compiles it and says + why it cannot run, ``{test=pytest path="tests/<file>.py"}`` runs it as a + test module (``tests/test_docs_testing_fixtures.py``). ``lint=check`` also + lints it. + """ + check_fence(example, eval_example=eval_example) + + +def _on_postgres(examples: list[CodeExample]) -> list[CodeExample]: return [ example - for example in find_examples(str(DOCS_PAGES)) - if "--8<--" not in example.source + for example in examples + if fence_settings(example).get("backend") == "postgres" ] -@pytest.mark.parametrize("example", _inline_examples(), ids=str) -def test_docs_examples(example: CodeExample, eval_example: EvalExample) -> None: - """Validate docs snippets, with opt-in linting/execution.""" - # Baseline for all snippets: parse + compile as valid Python syntax. - compile( - example.source, - str(example.path), - "exec", - flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT, - ) +@pytest.mark.postgres_only +@pytest.mark.parametrize("example", _on_postgres(page_examples()), ids=str) +def test_docs_examples_on_postgres(example: CodeExample, db_url: str) -> None: + """A page fence marked ``backend=postgres`` runs against PostgreSQL, every + ``connect()`` in it pointed at the test's database.""" + check_fence(example, database_url=db_url) - settings = example.prefix_settings() - - # Opt in for stricter linting using pytest-examples prefix settings: - # ```python {lint=check} - # ... - # ``` - if settings.get("lint") == "check": - eval_example.set_config( - # Docs snippets often intentionally use print statements or partial examples. - ruff_ignore=["D", "T", "B", "C4", "E721"], - line_length=100, - target_version="py313", - ) - eval_example.lint(example) - - # Opt in for runnable snippets using pytest-examples prefix settings: - # ```python {test=run} - # ... - # ``` - if settings.get("test") == "run": - eval_example.run_print_check(example) - eval_example.run(example) + +def test_a_postgres_fence_without_a_database_is_refused_by_run_fence() -> None: + example = _fence("python {test=run prelude=reference backend=postgres}") + with pytest.raises(ValueError, match="backend=postgres needs a database_url"): + run_fence(example) + + +def test_backend_needs_test_run() -> None: + with pytest.raises(ValueError, match=r"backend=postgres needs test=run"): + fence_settings(_fence('python {test=compile reason="x" backend=postgres}')) + + +def test_only_postgres_is_a_backend() -> None: + with pytest.raises(ValueError, match=r"backend=mysql is not backend=postgres"): + fence_settings(_fence("python {test=run prelude=reference backend=mysql}")) + + +@pytest.mark.postgres_only +def test_a_postgres_fence_runs_on_postgres(tmp_path: Path, db_url: str) -> None: + example = _page( + tmp_path, + "python {test=run prelude=reference backend=postgres}", + "from ferro import fetch_one\n" + 'row = await fetch_one("SELECT version() AS v")\nprint(row["v"].split()[0])\n' + "print((await Author.get(1)).name)", + ) + assert run_fence(example, database_url=db_url) == "PostgreSQL\nAda Lovelace\n" @pytest.mark.parametrize( @@ -75,13 +98,167 @@ def test_docs_example_scripts(script: Path) -> None: examples build their project in a temporary directory and drive ``ferro migrate`` against SQLite there. """ - result = subprocess.run( - [sys.executable, str(script)], - capture_output=True, - text=True, - timeout=120, - cwd=REPO_ROOT, + run_python([str(script)]) + + +@pytest.mark.postgres_only +@pytest.mark.parametrize( + "script", ["row_level_security.py", "row_level_security_annotated.py"] +) +def test_row_level_security_example_runs_on_postgres( + script: str, db_url: str, pg_role +) -> None: + """The row-level security pages' demo, on Postgres. + + The script migrates as the test's (superuser) role, creates the + application's NOSUPERUSER login role and asserts what each session sees: + a tenant-scoped session reads its own two invoices, a session without + settings reads none, the migrating role reads all four, and a write into + another tenant is refused. Without arguments the script runs on SQLite + (``test_docs_example_scripts``), where no policy exists to check. + """ + run_python([str(DOCS_EXAMPLES / script), db_url, pg_role("app")]) + + +def test_every_prelude_is_cited_by_a_fence() -> None: + """A prelude no page or docstring fence asks for is dead weight; remove it.""" + from tests._docs_harness import REPO_ROOT, docstring_examples, preludes + + cited = { + fence_settings(example).get("prelude") + for example in [ + *page_examples(), + *docstring_examples("ferro", REPO_ROOT / "src"), + ] + } + unused = sorted(set(preludes()) - cited) + assert not unused, f"no fence uses the preludes {unused}; delete them" + + +def _page(tmp_path: Path, prefix: str, body: str) -> CodeExample: + page = tmp_path / "page.md" + page.write_text(f"# Page\n\n```{prefix}\n{body}\n```\n", encoding="utf-8") + [example] = fenced_examples(page, page.read_text(encoding="utf-8")) + return example + + +def test_prelude_fence_runs_after_its_prelude(tmp_path: Path) -> None: + example = _page( + tmp_path, + "python {test=run prelude=reference}", + "print(await Author.where(lambda author: author.id == 1).first())", + ) + assert fence_settings(example) == {"test": "run", "prelude": "reference"} + output = run_fence(example) + assert "Ada Lovelace" in output + + +def test_prelude_fence_failure_names_the_page_line(tmp_path: Path) -> None: + example = _page( + tmp_path, + "python {test=run prelude=reference}", + 'author = await Author.get(1)\nraise RuntimeError("boom")', ) - assert result.returncode == 0, ( - f"{script.name} failed\nstdout:\n{result.stdout}\nstderr:\n{result.stderr}" + with pytest.raises(AssertionError) as failure: + run_fence(example) + message = str(failure.value) + assert f'File "{example.path}", line 5' in message + assert "RuntimeError: boom" in message + + +def test_unknown_prelude_fails_loudly(tmp_path: Path) -> None: + example = _page(tmp_path, "python {test=run prelude=nope}", "pass") + with pytest.raises(AssertionError, match="no prelude named 'nope'"): + run_fence(example) + + +def test_prelude_without_test_run_is_refused(tmp_path: Path) -> None: + example = _page(tmp_path, "python {prelude=reference}", "pass") + with pytest.raises(ValueError, match=r"prelude=reference needs test=run"): + fence_settings(example) + + +def test_fence_settings_read_bare_and_quoted_values() -> None: + example = CodeExample.create( + "pass", + prefix='python {test=run prelude=reference lint="check"}', + path=Path("p.md"), ) + assert fence_settings(example) == { + "test": "run", + "prelude": "reference", + "lint": "check", + } + + +def _fence(prefix: str) -> CodeExample: + return CodeExample.create("pass", prefix=prefix, path=Path("page.md")) + + +def test_run_without_prelude_names_the_preludes() -> None: + with pytest.raises(ValueError, match=r"test=run needs prelude=<name>.*reference"): + fence_settings(_fence("python {test=run}")) + + +def test_compile_without_reason_is_refused() -> None: + with pytest.raises(ValueError, match=r'test=compile needs reason="'): + fence_settings(_fence("python {test=compile}")) + + +def test_pytest_without_path_is_refused() -> None: + with pytest.raises(ValueError, match=r'test=pytest needs path="tests/'): + fence_settings(_fence("python {test=pytest}")) + + +def test_unknown_test_mode_is_refused() -> None: + with pytest.raises(ValueError, match=r"test=skip is not run, compile or pytest"): + fence_settings(_fence("python {test=skip}")) + + +def test_other_compile_only_spelling_is_refused() -> None: + with pytest.raises( + ValueError, match=r'unknown setting compile-only.*test=compile reason="' + ): + fence_settings(_fence('python {compile-only="needs Postgres"}')) + + +def test_reason_without_compile_is_refused() -> None: + with pytest.raises(ValueError, match=r"reason=x needs test=compile"): + fence_settings(_fence('python {test=run prelude=reference reason="x"}')) + + +def test_unmarked_page_fence_must_say_how_it_is_tested() -> None: + with pytest.raises( + ValueError, match=r'says neither test=run.*test=compile reason="' + ): + fence_mode(_fence("python")) + + +def test_compile_only_page_fence_states_its_reason() -> None: + assert ( + fence_mode(_fence('python {test=compile reason="needs Postgres"}')) == "compile" + ) + + +README = REPO_ROOT / "README.md" + + +def _readme_quick_start() -> str: + [fence] = fenced_blocks(README, README.read_text(encoding="utf-8"), "python") + return fence.body + + +def test_readme_quick_start_runs_and_prints_what_it_claims(tmp_path: Path) -> None: + """The README's Python block runs as written and prints its stated output.""" + script = tmp_path / "readme_quick_start.py" + script.write_text(_readme_quick_start(), encoding="utf-8") + result = run_python([str(script)], cwd=tmp_path) + assert "Found 1 active users." in README.read_text(encoding="utf-8") + assert result.stdout.strip() == "Found 1 active users." + + +def test_readme_model_is_the_home_page_example() -> None: + """The README's model is the one the home page embeds in both tabs.""" + script = DOCS_EXAMPLES / "readme_quickstart.py" + models = snippet_regions(script.read_text(encoding="utf-8"))["models"].body + assert models.strip() in _readme_quick_start() diff --git a/tests/test_docs_field_type_table.py b/tests/test_docs_field_type_table.py new file mode 100644 index 00000000..5c7ce54f --- /dev/null +++ b/tests/test_docs_field_type_table.py @@ -0,0 +1,221 @@ +"""The field type tables in ``docs/includes/field-types.md`` are generated. + +Each cell is what the create pass writes for one column on one dialect: the +type ``ferro_ddl_lowering::resolve_column_storage`` decides (read here over +FFI as ``_core._resolve_storage_type``, AGENTS.md I-1 item 3), rendered by +the same create pass ``connect(auto_migrate=True)`` runs. A hand edit to the +include, or a change to the storage decision, fails +``test_field_type_tables_are_current`` until the include is regenerated: + + python -m tests.test_docs_field_type_table +""" + +from __future__ import annotations + +import datetime as dt +import json +import re +import sys +from collections.abc import Iterator +from contextlib import contextmanager +from decimal import Decimal +from enum import IntEnum, StrEnum +from pathlib import Path +from typing import Any, Literal +from uuid import UUID + +from pydantic import BaseModel + +from ferro import Field, Model, clear_registry, reset_engine, varchar +from ferro._core import _render_create_table_sql_for_test, _resolve_storage_type +from ferro.columns import build_column_specs +from ferro.ir.compiler import compile_schema_ir_payload + +INCLUDE = Path(__file__).resolve().parents[1] / "docs" / "includes" / "field-types.md" +DIALECTS = (("sqlite", "SQLite"), ("postgres", "PostgreSQL")) + + +class Status(StrEnum): + DRAFT = "draft" + ACTIVE = "active" + RETIRED = "retired" + + +class Priority(IntEnum): + LOW = 1 + NORMAL = 2 + HIGH = 3 + + +class Dimensions(BaseModel): + width_mm: int + height_mm: int + + +# (column name, annotation as the table prints it, annotation, Field() options) +ANNOTATION_ROWS: list[tuple[str, str, Any, dict[str, Any]]] = [ + ("id", "`int` primary key", int | None, {"default": None, "primary_key": True}), + ("quantity", "`int`", int, {}), + ("name", "`str`", str, {}), + ("in_stock", "`bool`", bool, {}), + ("weight", "`float`", float, {}), + ("price", "`decimal.Decimal`", Decimal, {}), + ("thumbnail", "`bytes`", bytes, {}), + ("created_at", "`datetime.datetime`", dt.datetime, {}), + ("released_on", "`datetime.date`", dt.date, {}), + ("opens_at", "`datetime.time`", dt.time, {}), + ("token", "`uuid.UUID`", UUID, {}), + ("status", "`Status`, a `StrEnum` (`draft`, `active`, `retired`)", Status, {}), + ("priority", "`Priority`, an `IntEnum` (`1`, `2`, `3`)", Priority, {}), + ("size", '`Literal["S", "M", "L"]`', Literal["S", "M", "L"], {}), + ("attributes", "`dict[str, str]`", dict[str, str], {}), + ("tags", "`list[str]`", list[str], {}), + ("dimensions", "`Dimensions`, a Pydantic `BaseModel`", Dimensions, {}), +] + +# (db_type token as the table prints it, annotation, the token) +TOKEN_ROWS: list[tuple[str, Any, str]] = [ + ('`"text"`', str, "text"), + ("`varchar(255)`", str, varchar(255)), + ('`"smallint"`', int, "smallint"), + ('`"int"`', int, "int"), + ('`"bigint"`', int, "bigint"), + ('`"uuid"`', UUID, "uuid"), + ('`"timestamp"`', dt.datetime, "timestamp"), + ('`"timestamptz"`', dt.datetime, "timestamptz"), + ('`"date"`', dt.date, "date"), + ('`"time"`', dt.time, "time"), + ('`"json"`', dict[str, str], "json"), + ('`"jsonb"`', dict[str, str], "jsonb"), +] + + +@contextmanager +def _empty_registry() -> Iterator[None]: + from ferro.registry import REGISTRY + + def reset() -> None: + REGISTRY.reset_for_test() + reset_engine() + clear_registry() + + reset() + try: + yield + finally: + reset() + + +def _column_definitions( + name: str, columns: list[tuple[str, Any, dict[str, Any]]] +) -> dict[str, dict[str, str]]: + """Each column's definition after its name, per dialect, ``NOT NULL`` left out. + + The model is compiled to SchemaIR and rendered by the create pass, the + one each dialect's ``connect(auto_migrate=True)`` runs. A native enum + column is named by its type, as ``_resolve_storage_type`` decides it. + """ + annotations = {column: annotation for column, annotation, _ in columns} + namespace: dict[str, Any] = { + "__annotations__": annotations, + "__module__": __name__, + **{column: Field(**options) for column, _, options in columns if options}, + } + with _empty_registry(): + model = type(name, (Model,), namespace) + specs = build_column_specs(model) + payload = compile_schema_ir_payload( + name, list(specs.values()), table_name=name.lower() + ) + [ir_model] = payload["models"] + definitions: dict[str, dict[str, str]] = {column: {} for column in annotations} + for dialect, _ in DIALECTS: + create, _, _ = _render_create_table_sql_for_test( + name, json.dumps(payload), dialect + ) + for column in ir_model["columns"]: + storage = json.loads(_resolve_storage_type(json.dumps(column), dialect)) + if storage["kind"] == "pg_enum": + cell = f"native enum type `{storage['name']}`" + else: + cell = f"`{_definition(create, column['name'])}`" + definitions[column["name"]][dialect] = cell + return definitions + + +def _definition(create: str, column: str) -> str: + match = re.search(rf'"{column}" ([^,]+?)\s*(?:,|\s\)$)', create) + if match is None: + raise AssertionError(f"column {column!r} not found in {create}") + return re.sub(r"\s*\bNOT NULL\b", "", match.group(1)).strip() + + +def _table(header: str, rows: list[tuple[str, dict[str, str]]]) -> list[str]: + lines = [ + f"| {header} | {' | '.join(label for _, label in DIALECTS)} |", + "| :--- | :--- | :--- |", + ] + for label, cells in rows: + row = " | ".join(cells[dialect] for dialect, _ in DIALECTS) + lines.append(f"| {label} | {row} |") + return lines + + +def render_include() -> str: + """The include's full text, regenerated from the create pass.""" + annotation_cells = _column_definitions( + "FieldTypes", + [ + (column, annotation, options) + for column, _, annotation, options in ANNOTATION_ROWS + ], + ) + token_columns = [f"c{index}" for index in range(len(TOKEN_ROWS))] + token_cells = _column_definitions( + "DbTypeTokens", + [ + (column, annotation, {"db_type": token}) + for column, (_, annotation, token) in zip(token_columns, TOKEN_ROWS) + ], + ) + lines = [ + "<!-- Generated by `python -m tests.test_docs_field_type_table`; do not edit. -->", + "", + "<!-- --8<-- [start:annotations] -->", + *_table( + "Annotation", + [ + (label, annotation_cells[column]) + for column, label, _, _ in ANNOTATION_ROWS + ], + ), + "<!-- --8<-- [end:annotations] -->", + "", + "<!-- --8<-- [start:db-type-tokens] -->", + *_table( + "`db_type`", + [ + (label, token_cells[column]) + for column, (label, _, _) in zip(token_columns, TOKEN_ROWS) + ], + ), + "<!-- --8<-- [end:db-type-tokens] -->", + ] + return "\n".join(lines) + "\n" + + +def test_field_type_tables_are_current() -> None: + assert INCLUDE.read_text(encoding="utf-8") == render_include(), ( + f"{INCLUDE.relative_to(INCLUDE.parents[2])} is out of date; regenerate it " + "with `python -m tests.test_docs_field_type_table`" + ) + + +def test_a_cell_is_the_create_pass_column_type() -> None: + cells = _column_definitions("Probe", [("name", str, {})]) + assert cells["name"] == {"sqlite": "`varchar`", "postgres": "`varchar`"} + + +if __name__ == "__main__": + INCLUDE.write_text(render_include(), encoding="utf-8") + print(f"wrote {INCLUDE}", file=sys.stderr) diff --git a/tests/test_docs_harness.py b/tests/test_docs_harness.py new file mode 100644 index 00000000..69d7337e --- /dev/null +++ b/tests/test_docs_harness.py @@ -0,0 +1,99 @@ +"""The docs harness's shared pieces: the one fence reader and the one runner.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +import pytest + +from tests._docs_harness import Fence, fenced_blocks, fenced_examples, run_python + +PAGE = """\ +# Page + +```python {test=run prelude=reference} +print(1) +``` + +!!! note + + <!-- prints: predicates.py:first-query --> + ```text + Rent -1450 + ``` + +````markdown +```python +not a fence of its own +``` +```` + +```sql +-- predicates.py:first-query (sqlite) +SELECT 1; +``` +""" + + +def test_fenced_blocks_reads_every_fence_with_its_line_and_note() -> None: + blocks = fenced_blocks(Path("page.md"), PAGE) + assert [(block.language, block.line) for block in blocks] == [ + ("python", 3), + ("text", 10), + ("markdown", 14), + ("sql", 20), + ] + python, text, markdown, sql = blocks + assert python.info == "python {test=run prelude=reference}" + assert python.body == "print(1)\n" + assert text.body == "Rent -1450\n" + assert text.indent == " " + assert text.note == ("prints", "predicates.py:first-query") + assert python.note is None + assert markdown.body == "```python\nnot a fence of its own\n```\n" + assert sql.where == "page.md:20" + + +def test_fenced_blocks_filters_by_language() -> None: + [sql] = fenced_blocks(Path("page.md"), PAGE, language="sql") + assert sql == Fence( + path=Path("page.md"), + line=20, + language="sql", + info="sql", + body="-- predicates.py:first-query (sqlite)\nSELECT 1;\n", + indent="", + note=None, + ) + + +def test_an_unclosed_fence_is_refused_naming_its_line() -> None: + with pytest.raises( + ValueError, match=r"page\.md:3: the ```python fence is never closed" + ): + fenced_blocks(Path("page.md"), "# Page\n\n```python\nprint(1)\n") + + +def test_fenced_examples_are_the_python_fences() -> None: + [example] = fenced_examples(Path("page.md"), PAGE) + assert (example.start_line, example.prefix, example.source) == ( + 3, + "python {test=run prelude=reference}", + "print(1)\n", + ) + + +def test_run_python_returns_what_the_script_printed() -> None: + result = run_python( + ["-c", "import os; print(os.environ['DOCS_X'])"], env={"DOCS_X": "1"} + ) + assert result.stdout == "1\n" + + +def test_run_python_raises_with_both_streams() -> None: + with pytest.raises(AssertionError) as failure: + run_python(["-c", "print('out'); import sys; sys.exit('err')"]) + message = str(failure.value) + assert message.startswith(f"{sys.executable} -c ") + assert "exited 1\nstdout:\nout\n\nstderr:\nerr\n" in message diff --git a/tests/test_docs_lint.py b/tests/test_docs_lint.py new file mode 100644 index 00000000..43265470 --- /dev/null +++ b/tests/test_docs_lint.py @@ -0,0 +1,46 @@ +"""The docs lint (``tests._docs_checks``) as a gate over the whole tree. + +``python -m tests._docs_checks docs/pages docs/examples src/ferro`` must find +nothing: no query lambda with a one-letter parameter and no error check that +cannot fail, on any docs page, in any example script or in any docstring. +``tests/test_docs_checks.py`` covers each check case by case. +""" + +from __future__ import annotations + +import subprocess +import sys +from pathlib import Path + +REPO_ROOT = Path(__file__).resolve().parents[1] +LINTED = ("docs/pages", "docs/examples", "src/ferro") + + +def _lint(*paths: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [sys.executable, "-m", "tests._docs_checks", *paths], + cwd=REPO_ROOT, + capture_output=True, + text=True, + timeout=120, + ) + + +def test_docs_pages_examples_and_docstrings_pass_the_lint() -> None: + result = _lint(*LINTED) + assert result.returncode == 0, result.stdout + result.stderr + assert result.stdout == "" + + +def test_one_letter_lambda_on_a_page_fails_the_lint(tmp_path: Path) -> None: + page = tmp_path / "page.md" + page.write_text( + "# Page\n\nThe first post:\n\n```python\n" + "post = await Post.where(lambda t: t.id == 1).first()\n```\n", + encoding="utf-8", + ) + result = _lint(str(page)) + assert result.returncode == 1 + assert f"{page}:6: lambda passed to where() takes one-letter parameter 't'" in ( + result.stdout + ) diff --git a/tests/test_docs_migrations.py b/tests/test_docs_migrations.py index 15519c55..82fd7348 100644 --- a/tests/test_docs_migrations.py +++ b/tests/test_docs_migrations.py @@ -1,14 +1,23 @@ """The Schema Management docs say what the code does (#539). -Five promises the pages make, each checked against the implementation: +The promises the pages make, each checked against the implementation: - every ``ferro migrate`` output a page quotes (a ```text block marked ``<!-- output: <id> -->``) is what the verb prints with no terminal, on a scratch SQLite project, once the parts that vary are normalised; +- every step file a page quotes (a block headed ``-- NNNN_<name>/<file>``, or a + Python block headed ``# migrations/NNNN_<name>/<file>``) is, byte for byte, + what ``ferro migrate new`` writes from the page's example models; +- every Alembic revision the bridge page quotes (``<!-- revision: <id> -->``) + is what ``alembic revision --autogenerate`` writes for that change; - the two declaration styles of every migrations example generate the same migration, byte for byte (the ``_annotated`` twin's ``migrations/`` equals the assignment twin's); +- every example of the renames and retirements page starts from the one + running example the page declares; +- the migrations reference lists every ``PassReport`` statement role and every + warning kind the core raises, in the core's order; - every refusal the CLI reference quotes is the text ``ferro`` prints, for the refusals that need no database; - the Migrations API page documents exactly ``ferro.migrations.__all__``, and @@ -32,6 +41,7 @@ import pytest import ferro.migrations +from tests._docs_harness import Fence, fenced_blocks, page_fences, run_python REPO_ROOT = Path(__file__).resolve().parents[1] DOCS_PAGES = REPO_ROOT / "docs" / "pages" @@ -62,16 +72,7 @@ def _files(root: Path) -> dict[str, bytes]: def _run_example(name: str, project: Path) -> None: - result = subprocess.run( - [sys.executable, str(DOCS_EXAMPLES / f"{name}.py"), str(project)], - capture_output=True, - text=True, - timeout=180, - cwd=REPO_ROOT, - ) - assert result.returncode == 0, ( - f"{name} failed\nstdout:\n{result.stdout}\nstderr:\n{result.stderr}" - ) + run_python([str(DOCS_EXAMPLES / f"{name}.py"), str(project)]) @pytest.mark.parametrize("name", MIGRATION_EXAMPLES) @@ -87,6 +88,35 @@ def test_both_declaration_styles_generate_the_same_migrations( assert _files(annotated / "migrations") == written +RENAMES_PAGE_EXAMPLES = [ + "migrations_rename", + "migrations_retire", + "migrations_retire_label", +] +"""The examples of the renames and retirements page, each one change to its running +example, which ``migrations_rename``'s ``base`` region declares.""" + + +def _examples_project() -> ModuleType: + """``docs/examples/_migrations_project.py``, which reads an example's regions.""" + sys.path.insert(0, str(DOCS_EXAMPLES)) + try: + return importlib.import_module("_migrations_project") + finally: + sys.path.remove(str(DOCS_EXAMPLES)) + + +@pytest.mark.parametrize("style", ["", "_annotated"]) +@pytest.mark.parametrize("name", RENAMES_PAGE_EXAMPLES) +def test_each_renames_page_example_starts_from_the_page_s_running_example( + name: str, style: str +) -> None: + project = _examples_project() + change = project._region(DOCS_EXAMPLES / f"{name}{style}.py", "models") + base = project._region(DOCS_EXAMPLES / f"migrations_rename{style}.py", "base") + assert project._before(change) == base + + # -- the refusals the CLI reference quotes ------------------------------------------- MODELS = """\ @@ -197,17 +227,67 @@ def _empty_squash(project: Path) -> None: } +def _marked(kind: str, blocks: list[Fence] | None = None) -> dict[str, str]: + """Each ```text block marked ``<!-- <kind>: <id> -->``, by id. + + ``blocks`` defaults to every block a docs page shows, so a marked block + on any page is compared: there is no page where the note goes unchecked. + + Raises: + AssertionError: Two blocks carry one id. + """ + found: dict[str, str] = {} + for fence in page_fences("text") if blocks is None else blocks: + if fence.note is None or fence.note[0] != kind: + continue + block_id = fence.note[1] + assert block_id not in found, ( + f"{fence.where}: {kind} id {block_id!r} marks two blocks" + ) + found[block_id] = fence.body + return found + + +def _registry_gaps( + kind: str, registry: dict, blocks: list[Fence] | None = None +) -> list[str]: + """What keeps the marked blocks and ``registry`` from naming the same ids.""" + marked = _marked(kind, blocks) + return [ + *( + f"<!-- {kind}: {block_id} --> has no entry in the registry" + for block_id in sorted(set(marked) - set(registry)) + ), + *( + f"the registry's {block_id!r} marks no block" + for block_id in sorted(set(registry) - set(marked)) + ), + ] + + def _quoted_refusals() -> dict[str, str]: - """Each ```text block the CLI reference marks ``<!-- refusal: <id> -->``.""" - page = CLI_REFERENCE.read_text() - found = re.findall( - r"<!-- refusal: ([a-z-]+) -->\s*```text\n(.*?)```", page, flags=re.DOTALL - ) - return {key: text for key, text in found} + return _marked("refusal") -def test_the_cli_reference_quotes_each_database_free_refusal() -> None: - assert sorted(_quoted_refusals()) == sorted(REFUSALS) +def test_the_pages_quote_each_database_free_refusal() -> None: + assert _registry_gaps("refusal", REFUSALS) == [] + + +@pytest.mark.parametrize("kind", ["output", "refusal"]) +def test_a_marked_block_on_any_page_is_compared(kind: str) -> None: + """A typed block under ``output:`` or ``refusal:`` on a page outside the + migrations guides still has to name a registered scenario.""" + page = DOCS_PAGES / "howto" / "testing.md" + fabricated = fenced_blocks( + page, + f"# Testing\n\n<!-- {kind}: made-up -->\n```text\nferro migrate: typed\n```\n", + "text", + ) + registry = OUTPUTS if kind == "output" else REFUSALS + blocks = [*page_fences("text"), *fabricated] + assert _registry_gaps(kind, registry, blocks) == [ + f"<!-- {kind}: made-up --> has no entry in the registry" + ] @pytest.mark.parametrize("refusal", sorted(REFUSALS)) @@ -234,10 +314,11 @@ def test_the_api_page_documents_every_public_name_of_ferro_migrations() -> None: targets = re.findall( r"^::: (ferro\.migrations\.[\w.]+)", API_PAGE.read_text(), re.M ) + # Only `ferro.migrations.<name>`: the testing harness and the reports + # documented under their defining module (`ferro.migrations.runner.RunReport`) + # are not in `ferro.migrations.__all__`. documented = sorted( - target.rsplit(".", 1)[1] - for target in targets - if not target.startswith("ferro.migrations.testing.") + target.rsplit(".", 1)[1] for target in targets if target.count(".") == 2 ) assert documented == sorted(ferro.migrations.__all__) @@ -302,6 +383,43 @@ def test_the_api_page_documents_the_test_harness() -> None: assert f"::: ferro.migrations.testing.{name}\n" in page, name +# -- the PassReport the migrations reference describes ------------------------------ + +MIGRATIONS_REFERENCE = DOCS_PAGES / "reference" / "migrations.md" +REPORT_KINDS_SOURCE = REPO_ROOT / "crates" / "ferro-ddl-lowering" / "src" / "lib.rs" + + +def _first_column(page: Path, header: str) -> list[str]: + """The backticked first cell of every row of the table whose header row + starts with ``header``.""" + lines = page.read_text().splitlines() + start = next(number for number, line in enumerate(lines) if line.startswith(header)) + cells = [] + for line in lines[start + 2 :]: + if not line.startswith("|"): + break + cells.append(line.split("|")[1].strip().strip("`").strip('"')) + return cells + + +def test_the_reference_lists_every_report_kind_the_core_raises() -> None: + source = REPORT_KINDS_SOURCE.read_text() + body = source[source.index("pub enum ReportKind {") :] + body = body[: body.index("\n}\n")] + kinds = re.findall(r"^ ([A-Z]\w+)\b", body, flags=re.MULTILINE) + assert kinds, "no ReportKind variant found" + assert _first_column(MIGRATIONS_REFERENCE, "| `kind` |") == kinds + + +def test_the_reference_lists_every_executed_statement_role() -> None: + from typing import get_args + + from ferro.pass_report import Role + + roles = _first_column(MIGRATIONS_REFERENCE, "| `role` |") + assert roles == list(get_args(Role)) + + # -- the doors' names --------------------------------------------------------------- @@ -327,9 +445,11 @@ def test_no_page_names_a_door_the_old_way(phrase: str) -> None: OUTPUT_PAGES = [ CLI_REFERENCE, DOCS_PAGES / "guide" / "schema" / "migrations.md", + DOCS_PAGES / "guide" / "schema" / "renames-and-retirements.md", + DOCS_PAGES / "guide" / "schema" / "squash.md", DOCS_PAGES / "guide" / "schema" / "data-steps.md", DOCS_PAGES / "guide" / "schema" / "deploying.md", - DOCS_PAGES / "howto" / "adopting-migrations.md", + DOCS_PAGES / "guide" / "schema" / "adopting-migrations.md", ] HEAD = "from ferro import Field, Model\n\n\n" @@ -346,7 +466,6 @@ class Author(Model): AUTHOR_NICKNAME = ( AUTHOR + " nickname: str | None = Field(default=None, index=True)\n" ) -AUTHOR_HANDLE = AUTHOR + " handle: str | None = Field(default=None, index=True)\n" NAMED_AUTHOR = ( HEAD @@ -359,15 +478,7 @@ class Author(Model): NAMED_AUTHOR_SLUG = NAMED_AUTHOR + " slug: str\n" NAMED_AUTHOR_OPTIONAL_SLUG = NAMED_AUTHOR + " slug: str | None = None\n" -POST = ( - HEAD - + """\ -class Post(Model): - id: int | None = Field(default=None, primary_key=True) - title: str | None = None -""" -) -POST_TITLE_REQUIRED = POST.replace("title: str | None = None", "title: str") +AUTHOR_NICKNAME_REQUIRED = AUTHOR + " nickname: str = Field(index=True)\n" def _write_models(project: Path, source: str) -> None: @@ -480,54 +591,81 @@ def _sql_step_added(project: Path) -> None: _run_ok(project, "migrate", "new", "seed", "--sql-step", "seed_admins") +# The renames page: its running example, and nickname renamed with no hint. + +RENAMES_BASE = ( + "from enum import StrEnum\n\n" + + HEAD + + """\ +class Status(StrEnum): + ACTIVE = "active" + PAUSED = "paused" + CANCELED = "canceled" + + +class Author(Model): + id: int | None = Field(default=None, primary_key=True) + name: str + email: str = Field(unique=True) + nickname: str | None = Field(default=None, index=True) + bio: str = Field(index=True) + status: Status = Status.ACTIVE +""" +) + + def _before_rename(project: Path) -> None: _configured(project) - _write_models(project, AUTHOR_NICKNAME) + _write_models(project, RENAMES_BASE) _run_ok(project, "migrate", "new", "create_author") _run_ok(project, "migrate", "up") - _write_models(project, AUTHOR_HANDLE) + _write_models(project, RENAMES_BASE.replace("nickname:", "handle:")) def _rebuild_blocked(project: Path) -> None: - _configured(project) - _write_models(project, POST) - _run_ok(project, "migrate", "new", "create_post") - _run_ok(project, "migrate", "up") + """The loop's database, with an index and a trigger ferro never declared, meets + a migration that makes ``nickname`` required: a SQLite rebuild.""" + _loop_applied(project) _database( project, - "CREATE INDEX my_idx ON post(title);" - " CREATE TRIGGER t AFTER INSERT ON post BEGIN SELECT 1; END;", + "CREATE INDEX my_idx ON author(name);" + " CREATE TRIGGER t AFTER INSERT ON author BEGIN SELECT 1; END;", ) - _write_models(project, POST_TITLE_REQUIRED) + _write_models(project, AUTHOR_NICKNAME_REQUIRED) _run_ok( - project, "migrate", "new", "post_title_required", "--no-backfill", "post.title" + project, + "migrate", + "new", + "nickname_required", + "--no-backfill", + "author.nickname", ) -# Two branches that both generated migration 0002. +# Two branches that both generated migration 0002: a teammate's, the loop's +# ``0002_author_nickname``, and yours, ``0002_author_bio``. -AUTHOR_NICK = NAMED_AUTHOR + " nick: str | None = None\n" -AUTHOR_BIO = NAMED_AUTHOR + " bio: str | None = None\n" -AUTHOR_BIO_AND_NICK = AUTHOR_NICK + " bio: str | None = None\n" +AUTHOR_BIO = AUTHOR + " bio: str | None = None\n" +AUTHOR_NICKNAME_AND_BIO = AUTHOR_NICKNAME + " bio: str | None = None\n" def _two_branches(project: Path, *, applied: bool) -> None: import shutil _configured(project) - _write_models(project, NAMED_AUTHOR) + _write_models(project, AUTHOR) _run_ok(project, "migrate", "new", "create_author") _run_ok(project, "migrate", "up") - _write_models(project, AUTHOR_NICK) - _run_ok(project, "migrate", "new", "add_y") - teammate = project / "migrations" / "0002_add_y" + _write_models(project, AUTHOR_NICKNAME) + _run_ok(project, "migrate", "new", "author_nickname") + teammate = project / "migrations" / "0002_author_nickname" shutil.move(teammate, project / "teammate") _write_models(project, AUTHOR_BIO) - _run_ok(project, "migrate", "new", "add_x", "--sql-step", "fix_rows") + _run_ok(project, "migrate", "new", "author_bio", "--sql-step", "fix_rows") if applied: _run_ok(project, "migrate", "up") shutil.move(project / "teammate", teammate) - _write_models(project, AUTHOR_BIO_AND_NICK) + _write_models(project, AUTHOR_NICKNAME_AND_BIO) def _two_branches_unapplied(project: Path) -> None: @@ -536,10 +674,10 @@ def _two_branches_unapplied(project: Path) -> None: def _two_branches_replaced(project: Path) -> None: _two_branches(project, applied=True) - _run_ok(project, "migrate", "new", "add_x", "--replace", "0002_add_x") + _run_ok(project, "migrate", "new", "author_bio", "--replace", "0002_author_bio") -# An epic's branch, one migration per task, squashed before it merges. +# A feature branch, one migration per task, squashed before it merges. TEAM = ( "\n\nclass Team(Model):\n" @@ -550,27 +688,21 @@ def _two_branches_replaced(project: Path) -> None: TEAM_SLUG = " slug: str\n" -def _teams_epic(project: Path) -> None: - """Main holds ``0001``..``0004``; the epic's tasks add ``0005_add_teams`` (with - a ``tmp_code`` column), ``0006_team_slug`` (a required slug and its written - backfill), ``0007_drop_tmp`` and ``0008_fix_rows`` (a hand-written SQL step). - The developer's database applied every one.""" +def _teams_branch(project: Path) -> None: + """Main holds ``0001_create_author``, the loop's ``Author``; the branch's tasks + add ``0002_add_teams`` (with a ``tmp_code`` column), ``0003_team_slug`` (a + required slug and its written backfill), ``0004_drop_tmp`` and + ``0005_fix_rows`` (a hand-written SQL step). The developer's database applied + every one.""" _configured(project) - models = NAMED_AUTHOR - for name, column in ( - ("create_author", ""), - ("author_email", " email: str | None = None\n"), - ("author_bio", " bio: str | None = None\n"), - ("author_nick", " nick: str | None = None\n"), - ): - models += column - _write_models(project, models) - _run_ok(project, "migrate", "new", name) + models = AUTHOR_NICKNAME + _write_models(project, models) + _run_ok(project, "migrate", "new", "create_author") _write_models(project, models + TEAM + TEAM_TMP_CODE) _run_ok(project, "migrate", "new", "add_teams") _write_models(project, models + TEAM + TEAM_TMP_CODE + TEAM_SLUG) _run_ok(project, "migrate", "new", "team_slug") - backfill = project / "migrations" / "0006_team_slug" / "02_backfill_team.py" + backfill = project / "migrations" / "0003_team_slug" / "02_backfill_team.py" backfill.write_text( backfill.read_text().replace( 'todo("the slug for an existing team")', "team.name.lower()" @@ -579,7 +711,7 @@ def _teams_epic(project: Path) -> None: _write_models(project, models + TEAM + TEAM_SLUG) _run_ok(project, "migrate", "new", "drop_tmp") _run_ok(project, "migrate", "new", "fix_rows", "--sql-step", "fix_rows") - fix_rows = project / "migrations" / "0008_fix_rows" + fix_rows = project / "migrations" / "0005_fix_rows" (fix_rows / "01_fix_rows.up.sql").write_text("UPDATE team SET name = trim(name);\n") (fix_rows / "01_fix_rows.down.sql").write_text( "-- ferro: nothing-to-reverse the names stay trimmed\n" @@ -587,26 +719,26 @@ def _teams_epic(project: Path) -> None: _run_ok(project, "migrate", "up") -def _teams_epic_squashed(project: Path) -> None: - _teams_epic(project) - _run_ok(project, "migrate", "squash", "teams_epic", "--from", "0005") +def _teams_branch_squashed(project: Path) -> None: + _teams_branch(project) + _run_ok(project, "migrate", "squash", "teams", "--from", "0002") -def _teams_epic_dead_name(project: Path) -> None: - """``0006_team_slug``'s backfill, a chunked step, selects its rows by +def _teams_branch_dead_name(project: Path) -> None: + """``0003_team_slug``'s backfill, a chunked step, selects its rows by ``tmp_code``, a column that exists only inside the range.""" - _teams_epic(project) - backfill = project / "migrations" / "0006_team_slug" / "02_backfill_team.py" + _teams_branch(project) + backfill = project / "migrations" / "0003_team_slug" / "02_backfill_team.py" backfill.write_text( backfill.read_text().replace("team.slug == None", "team.tmp_code == None") ) -def _teams_epic_partly_applied(project: Path) -> None: - """A teammate's database: it applied ``0005``..``0007`` and not ``0008``.""" - _teams_epic(project) - _run_ok(project, "migrate", "down", "--to", "0007", "--yes") - _run_ok(project, "migrate", "squash", "teams_epic", "--from", "0005") +def _teams_branch_partly_applied(project: Path) -> None: + """A teammate's database: it applied ``0002``..``0004`` and not ``0005``.""" + _teams_branch(project) + _run_ok(project, "migrate", "down", "--to", "0004", "--yes") + _run_ok(project, "migrate", "squash", "teams", "--from", "0002") # A required column, a backfill, and the rows the backfill is for. @@ -663,6 +795,33 @@ def _slug_late_row_reverted(project: Path) -> None: step.write_text(NOT_APPLICABLE) +# The Deploying guide's two-release chain, as docs/examples/migration_errors.py +# builds it: 0002_author_slug_nullable (one expand step), then +# 0003_author_slug_required (backfill, add-constraint, contract), so the page +# names one migration per number. + +REQUIRED = "0003_author_slug_required" + + +def _chain_written(project: Path) -> None: + _slug_edited(project) + _write_models(project, NAMED_AUTHOR_OPTIONAL_SLUG) + _run_ok(project, "migrate", "new", "author_slug_nullable") + _write_models(project, NAMED_AUTHOR_SLUG) + _run_ok(project, "migrate", "new", "author_slug_required") + backfill = project / "migrations" / REQUIRED / "01_backfill_author.py" + backfill.write_text(backfill.read_text().replace(TODO, WRITTEN)) + + +def _chain_late_row_failed(project: Path) -> None: + """A writer inserts a row without a slug after the backfill, so the + contract step fails.""" + _chain_written(project) + step = project / "migrations" / REQUIRED / "02_add_constraint.up.sqlite.sql" + step.write_text(LATE_ROW) + assert _ferro(project, "migrate", "up", scratch=True).returncode == 1 + + # A database built by auto-migrate, about to be baselined. @@ -677,10 +836,11 @@ def _adopt_base(project: Path) -> None: def _adopt_initial(project: Path) -> None: - """Models described by ``0001_initial``; the database was built by the old door.""" + """Models described by ``0001_create_author``; the database was built by the + old door.""" _initialised(project) - _run_ok(project, "migrate", "new", "initial") - first = project / "migrations" / "0001_initial" / "01_schema.up.sqlite.sql" + _run_ok(project, "migrate", "new", "create_author") + first = project / "migrations" / "0001_create_author" / "01_schema.up.sqlite.sql" _database(project, first.read_text()) @@ -690,9 +850,22 @@ def _adopt_ahead(project: Path) -> None: _database(project, "ALTER TABLE author ADD COLUMN slug varchar") -def _adopt_staged(project: Path) -> None: +STAGING = ("--url", "sqlite:staging.db") +"""A second database beside ``app.db``: the staging copy the adopting how-to +baselines with ``--url "$STAGING_URL"``.""" + + +def _adopt_staging(project: Path) -> None: + """A staging copy restored from a dump taken at ``0001``.""" + import shutil + _adopt_base(project) - _run_ok(project, "migrate", "baseline", "0001") + shutil.copy(project / "app.db", project / "staging.db") + + +def _adopt_staged(project: Path) -> None: + _adopt_staging(project) + _run_ok(project, "migrate", "baseline", "0001", *STAGING) def _adopt_baselined(project: Path) -> None: @@ -762,7 +935,7 @@ def _ddl_lock_retry(project: Path) -> None: _run_ok(project, "migrate", "new", "create_author") _run_ok(project, "migrate", "up") _write_models(project, NAMED_AUTHOR_OPTIONAL_SLUG) - _run_ok(project, "migrate", "new", "author_slug") + _run_ok(project, "migrate", "new", "author_slug_nullable") @contextmanager @@ -825,7 +998,7 @@ def _drop_postgres_schemas(templates: list[Path]) -> None: ), ), "new-small": (_loop_edited, ("migrate", "new", "author_nickname")), - "squash": (_feature_branch, ("migrate", "squash", "epic", "--from", "0002")), + "squash": (_feature_branch, ("migrate", "squash", "teams", "--from", "0002")), "check-ok": (_loop_generated, ("migrate", "check")), "check-problem": (_loop_edited, ("migrate", "check")), "up": (_loop_generated, ("migrate", "up")), @@ -841,7 +1014,6 @@ def _drop_postgres_schemas(templates: list[Path]) -> None: "loop-check": (_loop_edited, ("migrate", "check")), "loop-new": (_loop_edited, ("migrate", "new", "author_nickname")), "loop-up": (_loop_generated, ("migrate", "up")), - "rename-hint": (_before_rename, ("migrate", "new", "rename_writer")), "sql-step": ( _created_and_applied, ("migrate", "new", "seed", "--sql-step", "seed_admins"), @@ -856,21 +1028,21 @@ def _drop_postgres_schemas(templates: list[Path]) -> None: "duplicate-number": (_two_branches_unapplied, ("migrate", "check")), "replace": ( _two_branches_unapplied, - ("migrate", "new", "add_x", "--replace", "0002_add_x"), + ("migrate", "new", "author_bio", "--replace", "0002_author_bio"), ), "replaced-up-refused": (_two_branches_replaced, ("migrate", "up")), - "squash-epic": ( - _teams_epic, - ("migrate", "squash", "teams_epic", "--from", "0005"), - ), + "rekey-refused": (_primary_key_change, ("migrate", "new", "rekey")), + # renames and retirements + "rename-hint": (_before_rename, ("migrate", "new", "rename_handle")), + # squashing + "squash-teams": (_teams_branch, ("migrate", "squash", "teams", "--from", "0002")), "squash-dead-name": ( - _teams_epic_dead_name, - ("migrate", "squash", "teams_epic", "--from", "0005"), + _teams_branch_dead_name, + ("migrate", "squash", "teams", "--from", "0002"), ), - "squash-status-inheritable": (_teams_epic_squashed, ("migrate", "status")), - "squash-up-inherits": (_teams_epic_squashed, ("migrate", "up")), - "squash-partial-witness": (_teams_epic_partly_applied, ("migrate", "up")), - "rekey-refused": (_primary_key_change, ("migrate", "new", "rekey")), + "squash-status-inheritable": (_teams_branch_squashed, ("migrate", "status")), + "squash-up-inherits": (_teams_branch_squashed, ("migrate", "up")), + "squash-partial-witness": (_teams_branch_partly_applied, ("migrate", "up")), # data steps "backfill-new": (_slug_edited, ("migrate", "new", "author_slug")), "backfill-unwritten": (_slug_unwritten, ("migrate", "up")), @@ -884,19 +1056,19 @@ def _drop_postgres_schemas(templates: list[Path]) -> None: # deploying "run-lock-wait": (_lock_wait, ("migrate", "up", "--lock-timeout", "1s")), "ddl-lock-retry": (_ddl_lock_retry, ("migrate", "up")), - "ci-check": (_slug_written, ("migrate", "check")), - "server-up": (_slug_written, ("migrate", "up")), - "status-steps": (_slug_late_row_failed, ("migrate", "status", "--steps")), + "ci-check": (_chain_written, ("migrate", "check")), + "server-up": (_chain_written, ("migrate", "up")), + "status-steps": (_chain_late_row_failed, ("migrate", "status", "--steps")), "edited-up": (_loop_edited_step, ("migrate", "up")), "edited-rerecord": (_loop_edited_step, ("migrate", "rerecord", "0001:01")), # adopting - "adopt-new-initial": (_initialised, ("migrate", "new", "initial")), + "adopt-new-initial": (_initialised, ("migrate", "new", "create_author")), "adopt-baseline": (_adopt_initial, ("migrate", "baseline")), "adopt-up-refused": (_adopt_base, ("migrate", "up")), "baseline-drift": (_adopt_ahead, ("migrate", "baseline", "0001")), "baseline-foreign-default": (_adopt_foreign_default, ("migrate", "baseline")), - "staging-baseline": (_adopt_base, ("migrate", "baseline", "0001")), - "staging-up": (_adopt_staged, ("migrate", "up")), + "staging-baseline": (_adopt_staging, ("migrate", "baseline", "0001", *STAGING)), + "staging-up": (_adopt_staged, ("migrate", "up", *STAGING)), "status-baselined": (_adopt_baselined, ("migrate", "status")), "baseline-remove": (_adopt_baselined, ("migrate", "baseline", "--remove")), "down-baselined-refused": (_adopt_baselined, ("migrate", "down", "--yes")), @@ -907,34 +1079,22 @@ def _quoted_blocks(page: Path) -> list[tuple[str | None, str | None, str, str]]: """Every ```text block of ``page`` as (kind, id, first line, body), the kind and id from the ``<!-- output|refusal: <id> -->`` marker above it, if any.""" blocks = [] - pattern = re.compile( - r"^(?P<indent>[ \t]*)(?:<!-- (?P<kind>output|refusal): (?P<id>[a-z0-9-]+) -->\n" - r"(?P=indent))?```text\n(?P<body>.*?)^(?P=indent)```", - flags=re.DOTALL | re.MULTILINE, - ) - for match in pattern.finditer(page.read_text()): - indent = match["indent"] - body = "".join( - line.removeprefix(indent) - for line in match["body"].splitlines(keepends=True) - ) - blocks.append((match["kind"], match["id"], body.split("\n", 1)[0], body)) + for fence in fenced_blocks(page, page.read_text(), "text"): + kind, block_id = fence.note or (None, None) + if kind not in ("output", "refusal"): + kind = block_id = None + blocks.append((kind, block_id, fence.body.split("\n", 1)[0], fence.body)) return blocks def _quoted_outputs() -> dict[str, str]: - return { - block_id: body - for page in OUTPUT_PAGES - for kind, block_id, _first, body in _quoted_blocks(page) - if kind == "output" - } + return _marked("output") def _normalise(text: str) -> str: """The parts of ferro's output that differ from run to run, made equal.""" text = re.sub( - r"sqlite:(?://)?\S*?app\.db(?:\?mode=rwc)?", "sqlite:///./app.db", text + r"sqlite:(?://)?\S*?(\w+)\.db(?:\?mode=rwc)?", r"sqlite:///./\1.db", text ) text = re.sub( r"postgres(?:ql)?://[^:\s/]+:\*+@\S+", @@ -948,14 +1108,7 @@ def _normalise(text: str) -> str: def test_every_output_block_has_a_registry_entry_and_the_reverse() -> None: - marked = [ - block_id - for page in OUTPUT_PAGES - for kind, block_id, _first, _body in _quoted_blocks(page) - if kind == "output" - ] - assert len(marked) == len(set(marked)), "an output id marks two blocks" - assert sorted(marked) == sorted(OUTPUTS) + assert _registry_gaps("output", OUTPUTS) == [] CLI_LINE = re.compile( @@ -1018,3 +1171,336 @@ def test_a_quoted_output_is_what_ferro_prints_without_a_terminal( printed = _normalise(result.stdout + result.stderr) assert printed == _normalise(_quoted_outputs()[output]) + + +# -- the step files the guides quote -------------------------------------------------- +# +# A fenced block whose first line is ``-- NNNN_<name>/<file>`` quotes a step file +# ``ferro migrate new`` wrote; a block may quote several, each under its own +# header line. A Python block whose first line is ``# migrations/NNNN_<name>/<file>`` +# quotes one generated data step. Each quoted file must be, byte for byte, what the +# page's example (or a scenario of this module) writes when it runs: the step file is +# regenerated from the models the page shows, never kept by hand. + +STEP_FILE_HEADER = re.compile(r"^-- (?P<migration>\d{4}_[a-z0-9_]+)/(?P<file>\S+)$") +DATA_STEP_HEADER = re.compile( + r"^# migrations/(?P<migration>\d{4}_[a-z0-9_]+)/(?P<file>\S+\.py)$" +) + + +def _slug_guarded(project: Path) -> None: + _slug_edited(project) + _run_ok(project, "migrate", "new", "author_slug", "--no-backfill", "author.slug") + + +ORDER = ( + "from enum import StrEnum\n\n" + + HEAD + + """\ +class Status(StrEnum): + PAID = "paid" + CANCELED = "canceled" + + +class Order(Model): + id: int | None = Field(default=None, primary_key=True) + status: Status = Status.PAID +""" +) + + +def _label_removed(project: Path) -> None: + """``Status`` loses ``canceled``, a label rows may still hold.""" + _configured(project) + _write_models(project, ORDER) + _run_ok(project, "migrate", "new", "create_order") + _write_models(project, ORDER.replace(' CANCELED = "canceled"\n', "")) + _run_ok(project, "migrate", "new", "drop_canceled") + + +INVOICE = """\ +import uuid +from typing import ClassVar + +from ferro import Field, Model, RowPolicy, RowSecurity + + +class Invoice(Model): + id: int | None = Field(default=None, primary_key=True) + tenant_id: uuid.UUID + customer: str + total: int +""" +INVOICE_RLS = ( + INVOICE + + """ + __ferro_rls__: ClassVar = RowSecurity( + RowPolicy(column="tenant_id", setting="app.tenant_id") + ) +""" +) + + +def _invoice_rls(project: Path) -> None: + """The row-level security guide's ``Invoice``, which exists, gains its policy.""" + _configured(project) + _write_models(project, INVOICE) + _run_ok(project, "migrate", "new", "create_invoice") + _write_models(project, INVOICE_RLS) + _run_ok(project, "migrate", "new", "invoice_rls") + + +STEP_FILE_SOURCES: dict[tuple[str, ...], str | Callable[[Path], None]] = { + ("guide/schema/migrations.md", "0001_create_author"): "migrations_quickstart", + ("guide/schema/migrations.md", "0002_author_nickname"): "migrations_quickstart", + ( + "guide/schema/renames-and-retirements.md", + "0002_rename_writer", + ): "migrations_rename", + ( + "guide/schema/renames-and-retirements.md", + "0002_retire_bio", + ): "migrations_retire", + ("guide/schema/squash.md", "0002_teams"): "migrations_squash", + ("reference/migrations.md", "0002_author_nickname"): "migrations_quickstart", + ("guide/schema/data-steps.md", "0002_author_slug"): "migrations_backfill", + ( + "guide/schema/data-steps.md", + "0002_author_slug", + "02_backfill_author.py", + ): _slug_unwritten, + ( + "guide/schema/data-steps.md", + "0002_author_slug", + "02_guard_author.py", + ): _slug_guarded, + ("guide/schema/data-steps.md", "0002_drop_canceled"): _label_removed, + ("guide/row-level-security.md", "0002_invoice_rls"): _invoice_rls, + ( + "guide/schema/migrations.md", + "0002_add_writer", + "03_backfill_writer.py", + ): None, +} +"""What writes the migration a page quotes: an example's name, or a scenario. Keyed by +(page, migration), or by (page, migration, file) where one file of a migration comes +from elsewhere (an unwritten backfill, a guard step in place of it). ``None`` marks a +data step the page shows already written by hand, which nothing generates.""" + + +def _step_files_in(body: str) -> list[tuple[str, str, str]]: + """The step files one block quotes, as (migration, file, text): the text runs + from under its header line to the next header or the block's end, blank lines + around it dropped.""" + first, _, rest = body.partition("\n") + data_step = DATA_STEP_HEADER.match(first) + if data_step: + return [(data_step["migration"], data_step["file"], rest)] + if not STEP_FILE_HEADER.match(first): + return [] + quoted = [] + header: re.Match[str] | None = None + rows: list[str] = [] + for row in [*body.splitlines(), None]: + match = None if row is None else STEP_FILE_HEADER.match(row) + if row is not None and match is None: + rows.append(row) + continue + if header is not None: + text = "\n".join(rows).strip("\n") + "\n" + quoted.append((header["migration"], header["file"], text)) + header, rows = match, [] + return quoted + + +def _quoted_step_files() -> list[tuple[str, int, str, str, str]]: + """Every step file a page quotes, as (page, block line, migration, file, text).""" + return [ + (page.relative_to(DOCS_PAGES).as_posix(), line, migration, file, text) + for page in sorted(DOCS_PAGES.rglob("*.md")) + for line, body in ( + (f.line, f.body) for f in fenced_blocks(page, page.read_text()) + ) + for migration, file, text in _step_files_in(body) + ] + + +def _step_file_source(page: str, migration: str, file: str) -> tuple[str, ...] | None: + for key in ((page, migration, file), (page, migration)): + if key in STEP_FILE_SOURCES: + return key + return None + + +def test_every_quoted_step_file_has_a_source_and_the_reverse() -> None: + used = { + _step_file_source(page, migration, file) + for page, _line, migration, file, _text in _quoted_step_files() + } + assert None not in used, "a quoted step file has no source in STEP_FILE_SOURCES" + assert used == set(STEP_FILE_SOURCES) + + +@pytest.fixture(scope="module") +def example_projects(tmp_path_factory: pytest.TempPathFactory): + """Run each example once, into a project it leaves behind.""" + built: dict[str, Path] = {} + + def make(name: str) -> Path: + if name not in built: + project = tmp_path_factory.mktemp(name) / "app" + _run_example(name, project) + built[name] = project + return built[name] + + return make + + +def _step_file_id(value: object) -> str: + """The page, block line, migration and file name a test is about; not the text.""" + if isinstance(value, int): + return str(value) + return value if isinstance(value, str) and "\n" not in value else "" + + +def _generated_step_files() -> list[tuple[str, int, str, str, str]]: + """The quoted step files to regenerate: every one but those written by hand + (one with no source stays, and fails naming itself).""" + generated = [] + for page, line, migration, file, text in _quoted_step_files(): + key = _step_file_source(page, migration, file) + if key is None or STEP_FILE_SOURCES[key] is not None: + generated.append((page, line, migration, file, text)) + return generated + + +@pytest.mark.parametrize( + ("page", "line", "migration", "file", "text"), + _generated_step_files(), + ids=_step_file_id, +) +def test_a_quoted_step_file_is_what_the_generator_writes( + page: str, + line: int, + migration: str, + file: str, + text: str, + example_projects, + scenarios, +) -> None: + where = f"{page}:{line} quotes {migration}/{file}" + key = _step_file_source(page, migration, file) + assert key, f"{where}, a migration STEP_FILE_SOURCES names no source for" + source = STEP_FILE_SOURCES[key] + if isinstance(source, str): + project, name = example_projects(source), source + else: + project, name = scenarios(source), source.__name__ + written = project / "migrations" / migration / file + assert written.is_file(), f"{where}, which {name} never writes" + assert text == written.read_text(), f"{where}, which {name} writes otherwise" + + +# -- the Alembic revisions the bridge page quotes ----------------------------------- +# +# A ```python block marked ``<!-- revision: <id> -->`` is the ``upgrade()`` and +# ``downgrade()`` that ``alembic revision --autogenerate`` writes for a change, on a +# SQLite database the before models built, without Alembic's ``# ###`` banner lines. + +ALEMBIC_PAGE = DOCS_PAGES / "guide" / "schema" / "alembic.md" + +REVISIONS = { + "nickname": (AUTHOR, AUTHOR_NICKNAME), + "drop-bio": (AUTHOR + " bio: str | None = None\n", AUTHOR), +} +"""Each quoted revision's models: (before, after).""" + +AUTOGENERATE = """\ +import asyncio +import sys + +import sqlalchemy as sa +from alembic.autogenerate import produce_migrations, render_python_code +from alembic.migration import MigrationContext + +import ferro +import blog.models # noqa: F401 +from ferro.migrations import ferro_options, get_metadata + + +async def main(step): + if step == "create": + await ferro.connect("sqlite:app.db?mode=rwc", auto_migrate=True) + return + engine = sa.create_engine("sqlite:///app.db") + options = ferro_options() + with engine.connect() as connection: + context = MigrationContext.configure( + connection, + opts={"compare_type": True, "compare_server_default": True, **options}, + ) + script = produce_migrations(context, get_metadata()) + engine.dispose() + for name, ops in (("upgrade", script.upgrade_ops), ("downgrade", script.downgrade_ops)): + print(f"def {name}() -> None:") + for line in render_python_code(ops, render_item=options["render_item"]).splitlines(): + if line.strip() and not line.strip().startswith("# ###"): + print(" " + line.strip()) + if name == "upgrade": + print("\\n") + + +asyncio.run(main(sys.argv[1])) +""" +"""Builds the database from the before models (``create``), then, run again over +the after models, prints the revision as the page quotes it.""" + + +def _quoted_revisions() -> dict[str, str]: + found = re.findall( + r"<!-- revision: ([a-z0-9-]+) -->\n```python[^\n]*\n(.*?)```", + ALEMBIC_PAGE.read_text(), + flags=re.DOTALL, + ) + return dict(found) + + +def test_every_quoted_revision_has_models_and_the_reverse() -> None: + assert sorted(_quoted_revisions()) == sorted(REVISIONS) + + +@pytest.mark.parametrize("revision", sorted(REVISIONS)) +def test_a_quoted_revision_is_what_autogenerate_writes( + revision: str, tmp_path: Path +) -> None: + pytest.importorskip("alembic", reason="the alembic extra is not installed") + before, after = REVISIONS[revision] + project = tmp_path / "app" + (project / "blog").mkdir(parents=True) + (project / "blog" / "__init__.py").write_text("") + (project / "autogenerate.py").write_text(AUTOGENERATE) + printed = "" + for models, step in ((before, "create"), (after, "render")): + _write_models(project, models) + result = run_python( + ["autogenerate.py", step], + timeout=120, + cwd=project, + env={"PYTHONPATH": str(project)}, + ) + printed = result.stdout + + assert printed == _quoted_revisions().get(revision), ( + f"guide/schema/alembic.md quotes revision {revision!r}; autogenerate writes:\n" + f"{printed}" + ) + + +def test_the_deploying_guide_names_one_migration_per_number() -> None: + """Every block and sentence of the Deploying guide follows one project, so + a number never names two migrations there.""" + page = (DOCS_PAGES / "guide" / "schema" / "deploying.md").read_text() + names: dict[str, set[str]] = {} + for name in re.findall(r"\b(\d{4})_([a-z0-9_]+)", page): + names.setdefault(name[0], set()).add(name[1]) + assert {number: sorted(n) for number, n in names.items() if len(n) > 1} == {} diff --git a/tests/test_docs_output.py b/tests/test_docs_output.py new file mode 100644 index 00000000..dbe9c618 --- /dev/null +++ b/tests/test_docs_output.py @@ -0,0 +1,450 @@ +"""The output a page shows is what its example prints. + +Every ```text block on a docs page sits under a note saying where its text +comes from (``tests._docs_sql.TEXT_NOTES``): + +- ``<!-- prints: <example>.py:<snippet> -->``: what that snippet region of + ``docs/examples/<example>.py`` prints. This test runs the script once on + SQLite and compares the block with the region's printed lines byte for byte. +- ``<!-- prints: <example>.py:<snippet> (postgres) -->``: what the region + prints on PostgreSQL, where the script runs as + ``<example>.py <database-url> <role>`` (a fresh schema and a role name the + test owns), under the ``postgres_only`` marker. +- ``<!-- prints: above -->``: what the nearest Python fence above it prints, + a fence marked ``{test=run prelude=<name>}``, run the way + ``tests/test_docs_examples.py`` runs it. +- ``<!-- output: <id> -->`` and ``<!-- refusal: <id> -->``: what + ``ferro migrate`` prints, regenerated by ``tests/test_docs_migrations.py``. +- ``<!-- shows: <what it is> -->``: text Ferro does not print (a sketch, a + file tree, another tool's shell session, a prompt to paste), skipped on + purpose and saying so. Ferro's own errors and warnings are ``prints:`` blocks, + and a ``shows:`` block that reads like one is refused. + +An unmarked block is refused, so typed output cannot slip onto a page. +""" + +from __future__ import annotations + +from pathlib import Path + +import re + +import pytest + +from tests._docs_harness import ( + REPO_ROOT, + Fence, + fenced_blocks, + page_fences, + snippet_regions, +) +from tests._docs_sql import ( + PRINTS, + Region, + Run, + printing_fence, + captured, + on_postgres, + check_printed, + page_name, +) + + +@pytest.mark.parametrize("block", page_fences("text"), ids=page_name) +def test_a_text_block_is_what_it_says_it_is(block: Fence) -> None: + check_printed(block) + + +def _on_postgres(block: Fence) -> bool: + """Whether a ``prints:`` block shows what something prints on PostgreSQL: + an example region marked ``(postgres)``, or a ``backend=postgres`` fence.""" + if not block.note or block.note[0] != "prints": + return False + if block.note[1] == "above": + fence = printing_fence(block) + return fence is not None and on_postgres(fence) + marker = PRINTS.match(block.note[1]) + return marker is not None and marker["dialect"] == "postgres" + + +def _postgres_blocks() -> list[Fence]: + return [block for block in page_fences("text") if _on_postgres(block)] + + +@pytest.mark.postgres_only +@pytest.mark.parametrize("block", _postgres_blocks(), ids=page_name) +def test_a_postgres_text_block_is_what_its_example_prints_there( + block: Fence, db_url: str, pg_role +) -> None: + args = (db_url, pg_role("app")) + check_printed( + block, + lambda example, dialect: captured(example, dialect, args=args), + dialect="postgres", + database_url=db_url, + ) + + +def _defined_exception_classes() -> set[type]: + """Every exception class Ferro defines, internal ones included.""" + import ferro.migrations + import ferro.migrations._chunk_loop # noqa: F401 - defines BatchFailed + + found: set[type] = set() + pending: list[type] = [BaseException] + while pending: + cls = pending.pop() + pending.extend(cls.__subclasses__()) + if cls.__module__.split(".")[0] == "ferro": + found.add(cls) + return found + + +def _public_exception_names() -> set[str]: + """The exception classes a reader can import by the name a page prints: + those in ``ferro.__all__`` or ``ferro.migrations.__all__``, and the public + classes ``ferro.exceptions`` defines.""" + import ferro + import ferro.exceptions + import ferro.migrations + + exported = { + exported + for module in (ferro, ferro.migrations) + for name in module.__all__ + if isinstance(exported := vars(module).get(name), type) + } + return { + cls.__name__ + for cls in _defined_exception_classes() + if cls in exported + or (cls.__module__ == "ferro.exceptions" and not cls.__name__.startswith("_")) + } + + +def _ferro_exception_names() -> list[str]: + """The name of every public exception class Ferro defines, longest first.""" + return sorted(_public_exception_names(), key=lambda name: (-len(name), name)) + + +def _internal_exception_names() -> set[str]: + """The exception classes Ferro defines that no public path exports.""" + return {cls.__name__ for cls in _defined_exception_classes()} - set( + _public_exception_names() + ) + + +_EXCEPTION_LINE = re.compile(r"(?m)^\s*(?:[^\s:]+:\d+: )?(?:[\w.]*\.)?(\w+): ") +"""A line opening with an exception's name, as ``print`` of a raised error or +a traceback's last line shows it.""" + + +def internal_exception_names_in(body: str) -> list[str]: + """The internal Ferro exception classes a block prints by name; a reader + who writes ``except`` with one of them gets a ``NameError``.""" + internal = _internal_exception_names() + return [name for name in _EXCEPTION_LINE.findall(body) if name in internal] + + +FERRO_OUTPUT = re.compile( + r"(?m)^\s*(?:[^\s:]+:\d+: )?" # a warning's or a traceback's `path:N: ` + r"(?:ferro\." + r"|ferro (?:auto-)?migrate(?: [a-z]+)?: " + rf"|(?:[\w.]*\.)?(?:{'|'.join(_ferro_exception_names())}):" + r"|[\w.]*(?:Error|Refused|Warning|Exception):" + r"|Traceback \(most recent call last\)" + r'|File "[^"]+", line \d+, in ' + r"|\d+ validation errors? for " + r"|connect\(" + r"|No database route " + r"|Cannot add NOT NULL column " + r"|\S+ \u00b7 \S+$" # the `<database> · <target>` line every verb but init opens with + r"|Wrote (?:ferro\.toml|\[tool\.ferro\]) " + r"|\S+\.toml: " + r"|\S+ holds two ferro configs" + r"|squash --from \S+: " + r"|changing the primary key of \S+ is not generated" + r")" +) +"""A line that reads like something Ferro prints, raises or warns: an +exception or warning line (with its ``path:N:`` prefix, if any), a +``ferro migrate`` output or refusal, or one of Ferro's own messages.""" + + +VARIES = { + "guide/connections.md:333": "the statement log's lines end with how long " + "each statement took, which differs on every run", +} +"""The ``shows:`` blocks holding Ferro's output that no run reproduces byte +for byte, by page and line, and why.""" + + +_PRINTS_A_MESSAGE = re.compile( + r"print\((?:f\"\{type\((\w+)\)\.__name__\}: \{\1\}\"|\w+\.message)\)" +) +"""A ``print`` of what was raised (``f"{type(error).__name__}: {error}"``) or +warned (``warning.message``), the forms the conventions give.""" + + +def _shows_a_message(block: Fence) -> bool: + """Whether a block shows Ferro's own words: ``ferro migrate``'s output or + a refusal, or a ``prints:`` block whose source prints what Ferro raised or + warned.""" + kind, value = block.note or ("", "") + if kind in ("output", "refusal"): + return True + if kind != "prints": + return False + if value == "above": + fence = printing_fence(block) + source = fence.source if fence is not None else "" + else: + marker = PRINTS.match(value) + assert marker is not None, page_name(block) + script = REPO_ROOT / "docs" / "examples" / marker["example"] + regions = snippet_regions(script.read_text(encoding="utf-8")) + source = regions[marker["snippet"]].body + return _PRINTS_A_MESSAGE.search(source) is not None + + +def _ferro_output_blocks() -> list[Fence]: + """Every block the pages show as Ferro's own words.""" + return [block for block in page_fences("text") if _shows_a_message(block)] + + +@pytest.mark.parametrize( + "line", + [ + "No database route for this operation. Open a session", + "Cannot add NOT NULL column 'author.bio' to an existing table", + "ferro.exceptions.ModelDoesNotExist: Author with pk 7 does not exist", + ' File "app.py", line 3, in <module>', + "ferro.migrations.BatchFailed: batch 3 failed", + "app.py:12: UserWarning: Check constraint 'ck_author_kind' is declared", + "Traceback (most recent call last):", + "ferro migrate: this database has applied 0002_author_slug", + "ferro migrate squash: nothing to squash", + ], +) +def test_the_recogniser_knows_each_kind_of_ferro_output(line: str) -> None: + assert FERRO_OUTPUT.search(line), line + + +@pytest.mark.parametrize("block", _ferro_output_blocks(), ids=page_name) +def test_the_recogniser_matches_every_block_of_ferro_output(block: Fence) -> None: + """The ``shows:`` guard is only as good as this pattern: it must match every + block the pages already show as Ferro's output.""" + assert FERRO_OUTPUT.search(block.body), ( + f"{page_name(block)}: FERRO_OUTPUT does not recognise this block of " + f"Ferro's output, so a <!-- shows: --> block holding it would pass:\n" + f"{block.body}" + ) + + +@pytest.mark.parametrize("block", page_fences("text"), ids=page_name) +def test_a_printed_exception_name_is_one_a_reader_can_import(block: Fence) -> None: + internal = internal_exception_names_in(block.body) + assert not internal, ( + f"{page_name(block)}: the block prints {internal}, which neither " + "ferro nor ferro.migrations exports; export and document the class, " + "or print its nearest public base" + ) + + +def test_a_block_naming_an_internal_exception_class_fails() -> None: + assert "StepRefused" in _internal_exception_names() + assert internal_exception_names_in( + "Step refused:\nStepRefused: the step needs a reason\n" + ) == ["StepRefused"] + assert internal_exception_names_in( + "ferro.migrations.steps.StepRefused: the step needs a reason\n" + ) == ["StepRefused"] + assert internal_exception_names_in("MigrationRefused: refused\n") == [] + + +def test_the_recogniser_names_only_public_exception_classes() -> None: + names = set(_ferro_exception_names()) + assert "MigrationRefused" in names + assert not names & _internal_exception_names() + + +def test_each_varies_entry_names_a_shows_block() -> None: + shows = { + page_name(block) + for block in page_fences("text") + if block.note and block.note[0] == "shows" + } + stale = sorted(set(VARIES) - shows) + assert not stale, f"VARIES names no shows: block at {stale}; update the line" + + +@pytest.mark.parametrize( + "block", + [block for block in page_fences("text") if block.note and block.note[0] == "shows"], + ids=page_name, +) +def test_a_shows_block_holds_nothing_ferro_prints(block: Fence) -> None: + if page_name(block) in VARIES: + return + line = FERRO_OUTPUT.search(block.body) + assert line is None, ( + f"{page_name(block)}: a <!-- shows: --> block quotes what Ferro prints " + f"({line.group(0)!r}...); make it a <!-- prints: --> block backed by an " + "example region that triggers it" + ) + + +def _fake(example: str, dialect: str) -> Run: + regions = {"first-query": Region(printed=["Rent -1450", "Groceries -82.40"])} + return Run(regions if example == "predicates.py" else {}, "", []) + + +def _block(note: str, body: str) -> Fence: + [block] = fenced_blocks( + Path("docs/pages/guide/queries.md"), + f"# Queries\n\nIt prints:\n\n{note}\n```text\n{body}```\n", + "text", + ) + return block + + +def test_a_block_holding_what_its_region_prints_passes() -> None: + check_printed( + _block( + "<!-- prints: predicates.py:first-query -->", + "Rent -1450\nGroceries -82.40\n", + ), + _fake, + ) + + +def test_a_block_that_says_what_it_shows_is_not_compared() -> None: + check_printed(_block("<!-- shows: a traceback -->", "Traceback\n"), _fake) + + +def _page_below(tmp_path: Path, fence: str, body: str) -> Fence: + page = tmp_path / "page.md" + page.write_text( + f"# Page\n\n{fence}\n\nIt prints:\n\n<!-- prints: above -->\n```text\n" + f"{body}```\n", + encoding="utf-8", + ) + [block] = fenced_blocks(page, page.read_text(encoding="utf-8"), "text") + return block + + +RUN_FENCE = ( + "```python {test=run prelude=reference}\n" + "print((await Author.where(lambda author: author.id == 1).first()).name)\n```" +) + + +def test_a_block_under_a_run_fence_holds_what_the_fence_prints( + tmp_path: Path, +) -> None: + check_printed(_page_below(tmp_path, RUN_FENCE, "Ada Lovelace\n")) + with pytest.raises(AssertionError) as raised: + check_printed(_page_below(tmp_path, RUN_FENCE, "Grace Hopper\n")) + assert "the block differs from what the fence at line 3 prints" in str(raised.value) + + +def test_a_block_under_a_fence_that_does_not_run_is_refused(tmp_path: Path) -> None: + fence = '```python {test=compile reason="needs Postgres"}\nprint(1)\n```' + with pytest.raises(AssertionError, match=r"needs the fence above \(line 3\)"): + check_printed(_page_below(tmp_path, fence, "1\n")) + + +@pytest.mark.parametrize( + ("note", "body", "message"), + [ + ( + "<!-- prints: predicates.py:first-query -->", + "Rent -1450\n", + "docs/pages/guide/queries.md:6 (predicates.py:first-query): the block " + "differs from what predicates.py:first-query prints", + ), + ( + "", + "Rent -1450\nGroceries -82.40\n", + "docs/pages/guide/queries.md:6: a ```text block must sit under a note", + ), + ( + "<!-- printed: predicates.py:first-query -->", + "Rent -1450\n", + "docs/pages/guide/queries.md:6: unknown note 'printed'", + ), + ( + "<!-- prints: predicates.py -->", + "Rent -1450\n", + "docs/pages/guide/queries.md:6: <!-- prints: predicates.py --> must name", + ), + ( + "<!-- prints: predicates.py:no-such-region -->", + "Rent -1450\n", + "docs/pages/guide/queries.md:6 (predicates.py:no-such-region): " + "predicates.py has no snippet region 'no-such-region'", + ), + ], + ids=["edited", "unmarked", "unknown-note", "no-region", "unknown-region"], +) +def test_a_block_that_is_not_what_it_says_fails_naming_it( + note: str, body: str, message: str +) -> None: + with pytest.raises(AssertionError) as raised: + check_printed(_block(note, body), _fake) + assert str(raised.value).startswith(message), str(raised.value) + + +PG_FENCE = ( + "```python {test=run prelude=reference backend=postgres}\n" + "from ferro import fetch_one\n\n" + 'row = await fetch_one("SELECT version() AS v")\nprint(row["v"].split()[0])\n```' +) + + +def test_a_block_under_a_postgres_fence_is_compared_only_on_postgres( + tmp_path: Path, +) -> None: + """On SQLite the fence above is only found and its marker checked; the + PostgreSQL job runs it and compares.""" + block = _page_below(tmp_path, PG_FENCE, "anything\n") + check_printed(block) + assert printing_fence(block) is not None + + +@pytest.mark.postgres_only +@pytest.mark.parametrize("shown", ["PostgreSQL\n", "SQLite\n"], ids=["same", "edited"]) +def test_a_block_under_a_postgres_fence_holds_what_it_prints_there( + tmp_path: Path, db_url: str, shown: str +) -> None: + block = _page_below(tmp_path, PG_FENCE, shown) + if shown == "PostgreSQL\n": + check_printed(block, dialect="postgres", database_url=db_url) + return + with pytest.raises(AssertionError, match="differs from what the fence at line 3"): + check_printed(block, dialect="postgres", database_url=db_url) + + +def test_the_pages_say_only_some_refusals_carry_a_kind() -> None: + """``RunRefused.kind`` is ``None`` on many refusals (the tracked-database + refusal of ``connect(auto_migrate=True)`` among them); no page or + docstring may say it always names the refusal.""" + import ferro.migrations + + pages = REPO_ROOT / "docs" / "pages" + overview = (pages / "guide/schema/overview.md").read_text(encoding="utf-8") + tree = (pages / "api/exceptions.md").read_text(encoding="utf-8") + [run_refused] = [line for line in tree.splitlines() if "RunRefused " in line] + import inspect + + source = inspect.getsource(ferro.migrations.RunRefused.__init__) + kind_doc = source.split("self.kind = kind", 1)[1].split('"""', 2)[1] + for text in (overview, run_refused, kind_doc): + assert "names why" not in text + assert "names its refusal in `kind`" not in text + assert "connect(auto_migrate=True)" in run_refused + assert "Only some refusals carry a kind" in overview + assert "its `kind` is `None`" in overview + assert "Only some refusals carry a kind" in kind_doc + assert "connect(auto_migrate=True)" in kind_doc + assert "MigrationRefused" in kind_doc diff --git a/tests/test_docs_shared.py b/tests/test_docs_shared.py new file mode 100644 index 00000000..28714547 --- /dev/null +++ b/tests/test_docs_shared.py @@ -0,0 +1,147 @@ +"""The snippet-region reader and the connect redirect the docs harness shares.""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from tests import _docs_harness, _docs_sql +from tests._docs_shared import ( + SnippetRegion, + redirected_url, + snippet_regions, +) + +REPO_ROOT = Path(__file__).resolve().parents[1] + +SCRIPT = """\ +from ferro import Model +# --8<-- [start:models] +class Author(Model): + # --8<-- [start:field] + name: str + # --8<-- [end:field] +# --8<-- [end:models] +print("done") +""" + + +def test_a_region_is_its_marker_lines_and_the_lines_between() -> None: + assert snippet_regions(SCRIPT) == { + "models": SnippetRegion( + start=2, + end=7, + body=( + "class Author(Model):\n" + " # --8<-- [start:field]\n" + " name: str\n" + " # --8<-- [end:field]\n" + ), + ), + "field": SnippetRegion(start=4, end=6, body=" name: str\n"), + } + + +def test_markdown_regions_read_the_same_way() -> None: + text = "intro\n<!-- --8<-- [start:intro] -->\nHello.\n<!-- --8<-- [end:intro] -->\n" + assert snippet_regions(text) == {"intro": SnippetRegion(2, 4, "Hello.\n")} + + +@pytest.mark.parametrize( + ("text", "message"), + [ + ( + "# --8<-- [start:a]\n# --8<-- [end:a]\n# --8<-- [start:a]\n# --8<-- [end:a]\n", + "ex.py:3: the snippet region 'a' starts a second time", + ), + ( + "x\n# --8<-- [end:a]\n", + "ex.py:2: the snippet region 'a' ends without starting", + ), + ("# --8<-- [start:a]\nx\n", "ex.py:1: the snippet region 'a' never ends"), + ], + ids=["twice", "end-only", "unclosed"], +) +def test_a_malformed_region_is_refused_naming_its_line(text: str, message: str) -> None: + with pytest.raises(ValueError) as raised: + snippet_regions(text, where="ex.py") + assert str(raised.value).startswith(message), str(raised.value) + + +def _region_files() -> list[Path]: + scripts = [ + path + for path in (REPO_ROOT / "docs" / "examples").glob("*.py") + if not path.name.startswith("_") + ] + markdown = [ + REPO_ROOT / "README.md", + REPO_ROOT / "CONTRIBUTING.md", + *(REPO_ROOT / "docs" / "includes").glob("*.md"), + ] + return sorted([*scripts, *markdown]) + + +@pytest.mark.parametrize("path", _region_files(), ids=lambda p: p.name) +def test_every_file_with_regions_reads_cleanly(path: Path) -> None: + snippet_regions(path.read_text(encoding="utf-8"), where=str(path)) + + +def test_the_four_region_readers_are_one() -> None: + """The capture, the include reader, the README check and the migrations + examples' helper all read regions with ``snippet_regions``.""" + import importlib + import sys + + from tests import test_docs_examples + + assert _docs_harness.snippet_regions is snippet_regions + assert not hasattr(_docs_sql, "snippet_ranges") + assert not hasattr(_docs_harness, "_section") + assert not hasattr(test_docs_examples, "_MODELS_SECTION") + sys.path.insert(0, str(REPO_ROOT / "docs" / "examples")) + try: + helper = importlib.import_module("_migrations_project") + finally: + sys.path.remove(str(REPO_ROOT / "docs" / "examples")) + assert helper.snippet_regions is snippet_regions + + +PG = "postgresql://postgres:password@localhost:5432/postgres?ferro_search_path=ferro_ab12" + + +@pytest.mark.parametrize( + ("named", "reached"), + [ + ("postgresql://localhost/app", PG), + ( + "postgresql://user:password@localhost:5432/app?ferro_search_path=tenant_a", + "postgresql://postgres:password@localhost:5432/postgres" + "?ferro_search_path=tenant_a", + ), + ( + "postgresql://user:password@localhost:5432/app?sslmode=require", + "postgresql://postgres:password@localhost:5432/postgres" + "?ferro_search_path=ferro_ab12&sslmode=require", + ), + ("sqlite:dev.db?mode=rwc", PG), + ("sqlite::memory:", PG), + ], + ids=["no-options", "search-path", "other-option", "sqlite-options", "sqlite"], +) +def test_a_redirect_keeps_the_postgres_options_the_block_names( + named: str, reached: str +) -> None: + assert redirected_url(named, PG) == reached + + +def test_the_capture_and_the_prelude_runner_install_the_one_redirect() -> None: + """Neither patches ``ferro.connect`` itself.""" + for path in ( + REPO_ROOT / "tests" / "_docs_sql.py", + REPO_ROOT / "docs" / "examples" / "_preludes" / "__init__.py", + ): + source = path.read_text(encoding="utf-8") + assert "ferro.connect =" not in source, path + assert "point_connect_at(" in source, path diff --git a/tests/test_docs_site.py b/tests/test_docs_site.py new file mode 100644 index 00000000..9f058dee --- /dev/null +++ b/tests/test_docs_site.py @@ -0,0 +1,944 @@ +"""The built documentation site renders what the configuration promises. + +The site is built once per session with ``zensical build --strict`` into a +temporary directory, so a warning anywhere fails the session, and every test +below reads the HTML a reader would get: + +- a code block embedded from ``docs/examples`` starts at column 0, so a + pasted block runs (``pymdownx.snippets.dedent_subsections``), and no code + block shows a ``# noqa`` comment; +- Assignment/Annotated tabs carry word anchors and switch together site-wide; +- acronyms such as DDL carry a hover definition from the glossary; +- ``#NNN`` references link to GitHub; +- the old migrations URL and its anchors redirect to the sections that + replaced them; +- the API reference hides source blocks, dunder and ``__ferro_*__`` members, + links the types in its signatures, renders ``Field`` once and gives + ``Relation.all()`` / ``.first()`` their inherited docstrings; +- every exported name has one API entry under its public path, every public + object and member has a docstring, no entry renders empty, each API page is + titled after its objects, and every link from an API page lands on an + anchor that exists; +- the docs build in CI and in the publish workflow is strict, and CI's docs + job runs every docs test file and the docs lint; +- the nav has six tabs holding every page once, each label equal to its + page's title; every page has a one-sentence description and new pages are + marked new; every moved page, and every anchor ``MOVED_ANCHORS`` lists, + redirects to the section that replaced it; +- no page renders a literal Sphinx role, an unresolved cross-reference, a + list merged into a paragraph, a docstring section as prose or a section + table row that stops mid-sentence, and griffe reads every docstring without + a warning; a second build, of a page and a module seeded with each defect, + shows every check failing on it. +""" + +from __future__ import annotations + +import html +import json +import re +import shutil +import subprocess +import sys +import tomllib +from collections.abc import Iterator +from pathlib import Path +from urllib.parse import urljoin + +import pytest + +pytest.importorskip("zensical", reason="the docs dependency group is not installed") +yaml = pytest.importorskip("yaml", reason="the docs dependency group is not installed") + +REPO_ROOT = Path(__file__).resolve().parents[1] +WORKFLOWS = REPO_ROOT / ".github" / "workflows" + +COPIED = ("zensical.toml", "docs/pages", "docs/includes") +"""Paths copied into the scratch project: Zensical refuses a ``docs_dir`` +that resolves outside the project root, so the pages are copied.""" + +LINKED = ("docs/examples", "src", "README.md", "CHANGELOG.md", "CONTRIBUTING.md") +"""Paths the pages embed or document, linked into the scratch project.""" + +_CODE_BLOCK = re.compile( + r'<div class="language-[\w-]+ highlight">(?:<span class="filename">.*?</span>)?' + r"<pre><span></span><code>(.*?)</code></pre>", + re.DOTALL, +) +_TAG = re.compile(r"<[^>]+>") + + +BUILD_LOG = "build.log" +"""The build's output, written next to ``site/`` in the scratch project.""" + + +def _scratch_project(project: Path) -> Path: + for name in COPIED: + source, target = REPO_ROOT / name, project / name + target.parent.mkdir(parents=True, exist_ok=True) + if source.is_dir(): + shutil.copytree(source, target) + else: + shutil.copy2(source, target) + for name in LINKED: + target = project / name + target.parent.mkdir(parents=True, exist_ok=True) + target.symlink_to(REPO_ROOT / name) + return project + + +def _build(project: Path, *flags: str) -> subprocess.CompletedProcess[str]: + result = subprocess.run( + [sys.executable, "-m", "zensical", "build", "--clean", *flags], + cwd=project, + capture_output=True, + text=True, + timeout=600, + ) + (project / BUILD_LOG).write_text(result.stdout + result.stderr, encoding="utf-8") + return result + + +@pytest.fixture(scope="session") +def built_site(tmp_path_factory: pytest.TempPathFactory) -> Path: + """Build the docs site with ``zensical build --strict``; return its root.""" + project = _scratch_project(tmp_path_factory.mktemp("docs-site")) + result = _build(project, "--strict") + assert result.returncode == 0, ( + f"zensical build --strict failed\nstdout:\n{result.stdout}\n" + f"stderr:\n{result.stderr}" + ) + return project / "site" + + +def rendered_pages(built_site: Path) -> Iterator[tuple[str, str]]: + """Yield ``(url_path, html)`` for every HTML page of the built site.""" + for page in sorted(built_site.rglob("*.html")): + relative = page.relative_to(built_site).as_posix() + url_path = "/" + relative.removesuffix("index.html") + yield url_path, page.read_text(encoding="utf-8") + + +def _page(built_site: Path, url_path: str) -> str: + page = built_site / url_path.strip("/") / "index.html" + assert page.is_file(), f"{url_path} was not built" + return page.read_text(encoding="utf-8") + + +def _code_lines(block: str) -> list[str]: + text = html.unescape(_TAG.sub("", block)) + return [line for line in text.splitlines() if line.strip()] + + +def _indented_blocks(page_html: str) -> list[str]: + """Return the code blocks whose every line shares leading whitespace.""" + indented = [] + for match in _CODE_BLOCK.finditer(page_html): + lines = _code_lines(match.group(1)) + if lines and all(line[0] in " \t" for line in lines): + indented.append("\n".join(lines)) + return indented + + +def test_embedded_code_starts_at_column_zero(built_site: Path) -> None: + offenders = { + url: blocks[0] + for url, page_html in rendered_pages(built_site) + if (blocks := _indented_blocks(page_html)) + } + assert not offenders, "code blocks render indented:\n" + "\n\n".join( + f"{url}:\n{block}" for url, block in offenders.items() + ) + + +def test_no_code_block_shows_a_noqa_comment(built_site: Path) -> None: + """Lint exceptions for the examples live in ``pyproject.toml``, not on pages.""" + offenders = [ + f"{url}: {line.strip()}" + for url, page_html in rendered_pages(built_site) + for match in _CODE_BLOCK.finditer(page_html) + for line in _code_lines(match.group(1)) + if "# noqa" in line + ] + assert not offenders, "code blocks show # noqa:\n" + "\n".join(offenders) + + +def test_quickstart_has_code_blocks_to_check(built_site: Path) -> None: + assert len(_CODE_BLOCK.findall(_page(built_site, "getting-started/quickstart"))) > 3 + + +def test_declaration_tabs_have_word_anchors_and_switch_together( + built_site: Path, +) -> None: + page = _page(built_site, "guide/models-and-fields") + assert page.count('class="tabbed-set') >= 2 + labels = re.findall(r'<label for="([^"]+)">(Assignment|Annotated)<', page) + assert labels, "no Assignment/Annotated tabs on the models page" + for anchor, label in labels: + assert not anchor.startswith("__tabbed_"), anchor + assert anchor.endswith(f"-{label.lower()}"), anchor + assert "content.tabs.link" in _theme_features(page) + + +def _theme_features(page_html: str) -> list[str]: + match = re.search( + r'<script id="__config" type="application/json">(.*?)</script>', + page_html, + re.DOTALL, + ) + assert match, "page carries no theme config" + return json.loads(match.group(1))["features"] + + +def test_acronyms_carry_their_glossary_definition(built_site: Path) -> None: + page = _page(built_site, "concepts/architecture") + match = re.search(r'<abbr title="([^"]+)">DDL</abbr>', page) + assert match, "DDL has no <abbr> on the architecture page" + assert "CREATE" in match.group(1) + + +def test_issue_references_link_to_github(built_site: Path) -> None: + page = _page(built_site, "changelog") + assert re.search( + r'<a [^>]*href="https://github\.com/syn54x/ferro-orm/issues/\d+"', page + ) + + +OLD_MIGRATIONS_ANCHORS = { + "": "guide/schema/overview/", + "#three-ways-to-manage-schema": "guide/schema/overview/#the-ladder", + "#auto-migration": "guide/schema/auto-migrate/", + "#creating-tables-with-auto_migratetrue": "guide/schema/auto-migrate/#creating-tables-with-auto_migratetrue", + "#applying-column-changes-with-migrate_updates": "guide/schema/auto-migrate/#applying-column-changes-with-migrate_updates", + "#evolving-enums-label-addition": "guide/schema/auto-migrate/#evolving-enums-label-addition", + "#destructive-drops-with-migrate_destructive": "guide/schema/auto-migrate/#destructive-drops-with-migrate_destructive", + "#on-demand-migrate": "guide/schema/auto-migrate/#on-demand-migrate", + "#safety-guidance": "guide/schema/auto-migrate/#safety-guidance", + "#alembic-for-production": "guide/schema/alembic/", + "#install": "guide/schema/alembic/#install", + "#initialize": "guide/schema/alembic/#initialize", + "#configure-envpy": "guide/schema/alembic/#configure-envpy", + "#autogenerate": "guide/schema/alembic/#one-decider", + "#review-apply": "guide/schema/alembic/#one-decider", + "#choosing-a-workflow": "guide/schema/overview/#choosing", +} +"""Every anchor of the old ``guide/migrations/`` page and the section that +replaced it, as the deleted hand-written stub mapped them.""" + + +def test_old_migrations_page_is_gone_from_the_source() -> None: + assert not (REPO_ROOT / "docs/pages/guide/migrations.md").exists() + + +def test_old_migrations_url_redirects_each_anchor(built_site: Path) -> None: + old_url = "https://ferro-orm.x54.sh/guide/migrations/" + targets = _redirect_targets(_page(built_site, "guide/migrations")) + for anchor, section in OLD_MIGRATIONS_ANCHORS.items(): + assert anchor in targets, f"old anchor {anchor or '(none)'} is not redirected" + resolved = urljoin(old_url, targets[anchor]) + assert resolved == urljoin(old_url, "/" + section), ( + f"{anchor or '(none)'} redirects to {resolved}, expected {section}" + ) + path, _, fragment = section.partition("#") + target_page = _page(built_site, path) + if fragment: + assert f'id="{fragment}"' in target_page, f"{section} has no such anchor" + + +def _redirect_targets(page_html: str) -> dict[str, str]: + """Read the anchor map of a generated redirect page.""" + targets: dict[str, str] = {} + refresh = re.search(r'http-equiv="refresh" content="0; url=([^"]+)"', page_html) + assert refresh, "the old migrations URL is not a redirect page" + targets[""] = html.unescape(refresh.group(1)) + for script in re.findall(r"<script[^>]*>(.*?)</script>", page_html, re.DOTALL): + for anchor, target in re.findall(r'"(#[\w-]+)"\s*:\s*"([^"]+)"', script): + targets[anchor] = target + return targets + + +def test_old_migrations_url_is_not_in_the_sitemap(built_site: Path) -> None: + sitemap = (built_site / "sitemap.xml").read_text(encoding="utf-8") + assert "guide/migrations/" not in sitemap + assert "guide/schema/overview/" in sitemap + + +def test_model_api_page_shows_the_public_surface_only(built_site: Path) -> None: + page = _page(built_site, "api/model") + assert "Source code in" not in page + ids = re.findall(r'id="(ferro\.[\w.]+)"', page) + assert ids, "the Model API page documents nothing" + dunders = [anchor for anchor in ids if re.search(r"\.__\w+__$", anchor)] + assert not dunders, f"dunder members rendered: {dunders}" + signatures = re.findall( + r'<div class="language-python doc-signature highlight">.*?</code>', + page, + re.DOTALL, + ) + assert signatures, "no separate signatures on the Model API page" + assert any('<a class="autorefs' in signature for signature in signatures), ( + "no linked type annotation in any Model signature" + ) + + +def test_field_renders_once_without_its_overloads(built_site: Path) -> None: + page = _page(built_site, "api/fields") + assert page.count('id="ferro.Field"') == 1 + assert "doc-overloads" not in page + assert 'id="ferro.Field(index)"' in page + + +def test_relation_query_methods_inherit_their_docstrings(built_site: Path) -> None: + page = _page(built_site, "api/relationships") + for method in ("all", "first"): + section = re.search( + rf'id="ferro\.Relation\.{method}" class="doc doc-heading">(.*?)(?:class="doc doc-heading"|</article>)', + page, + re.DOTALL, + ) + assert section, f"Relation.{method} is not documented" + text = html.unescape(_TAG.sub("", section.group(1))) + assert "Returns:" in text, f"Relation.{method} renders without a docstring" + assert "Examples:" in text, f"Relation.{method} renders without a docstring" + + +@pytest.mark.parametrize("workflow", ["publish-docs.yml", "ci.yml"]) +def test_docs_build_in_workflows_is_strict(workflow: str) -> None: + text = (WORKFLOWS / workflow).read_text(encoding="utf-8") + builds = re.findall(r"zensical build[^\n]*", text) + assert builds, f"{workflow} does not build the docs" + for build in builds: + assert "--strict" in build, f"{workflow}: {build}" + + +def test_ci_docs_job_runs_every_docs_test_and_the_lint() -> None: + jobs = yaml.safe_load((WORKFLOWS / "ci.yml").read_text(encoding="utf-8"))["jobs"] + """The job runs every module matching ``tests/test_doc*.py``, by glob, and + every test module that uses the docs harness is named to match it.""" + commands = "\n".join(step.get("run", "") for step in jobs["docs"]["steps"]) + assert re.search( + r"pytest [^\n]*tests/test_doc\*\.py", commands.replace("\\\n", " ") + ), "the CI docs job must run pytest over tests/test_doc*.py" + helpers = ("tests._docs_harness", "tests._docs_sql", "tests._docs_twins") + misnamed = sorted( + path.name + for path in (REPO_ROOT / "tests").glob("test_*.py") + if not path.name.startswith("test_doc") + and any(helper in path.read_text(encoding="utf-8") for helper in helpers) + ) + assert not misnamed, ( + f"{misnamed} use the docs harness but the CI docs job's glob, " + "tests/test_doc*.py, does not reach them; rename them test_docs_*.py" + ) + assert "python -m tests._docs_checks docs/pages docs/examples src/ferro" in commands + + +RETURNED_TYPES = ( + "ferro.models.ModelConnection", + "ferro.session.EngineManager", + "ferro.session.EngineManager.session", + "ferro.migrations.runner.RunReport", + "ferro.migrations.report.StatusReport", + "ferro.migrations.generate.CheckReport", + "ferro.pass_report.Subject", +) +"""Objects no ``__all__`` exports that a public call returns or is called on: +``Model.using()``, ``engines.session()``, ``up()``, ``status()``, +``check()`` and ``Report.subject``. They are documented under the module that +defines them.""" + +_DOC_HEADING_ID = re.compile(r'id="([\w.]+)" class="doc doc-heading"') + + +def _public_paths() -> set[str]: + """Return the public path of every exported name, and the returned types.""" + import ferro + import ferro.migrations + import ferro.migrations.testing + import ferro.query + + paths = {f"ferro.{name}" for name in ferro.__all__} + paths |= { + f"ferro.query.{name}" + for name in ferro.query.__all__ + if name not in ferro.__all__ + } + paths |= {f"ferro.migrations.{name}" for name in ferro.migrations.__all__} + paths |= { + f"ferro.migrations.testing.{name}" for name in ferro.migrations.testing.__all__ + } + return paths | set(RETURNED_TYPES) + + +def _api_entries(built_site: Path) -> dict[str, list[str]]: + """Map each object the API pages document to the pages that render it.""" + entries: dict[str, list[str]] = {} + for url, page_html in rendered_pages(built_site): + if url.startswith("/api/"): + for anchor in _DOC_HEADING_ID.findall(page_html): + entries.setdefault(anchor, []).append(url) + return entries + + +def test_every_export_has_an_api_entry(built_site: Path) -> None: + entries = _api_entries(built_site) + public = _public_paths() + missing = sorted(path for path in public if path not in entries) + assert not missing, f"exported names with no API entry: {missing}" + repeated = {path: entries[path] for path in public if len(entries[path]) > 1} + assert not repeated, f"exported names with more than one API entry: {repeated}" + + +def test_api_entries_use_public_paths(built_site: Path) -> None: + public = _public_paths() + private = sorted( + path + for path in _api_entries(built_site) + if path not in public and not any(path.startswith(f"{p}.") for p in public) + ) + assert not private, f"API entries under a non-public path: {private}" + + +def test_api_pages_are_titled_after_their_objects(built_site: Path) -> None: + entries = _api_entries(built_site) + for url, page_html in rendered_pages(built_site): + if not url.startswith("/api/"): + continue + title = re.search(r"<h1[^>]*>(.*?)</h1>", page_html, re.DOTALL) + assert title, f"{url} has no title" + words = re.findall(r"[\w.]+", html.unescape(_TAG.sub("", title.group(1)))) + named = set() + for path, pages in entries.items(): + if url in pages: + parts = path.split(".") + named |= {".".join(parts[:i]) for i in range(2, len(parts) + 1)} + named.add(parts[-1]) + assert named & set(words), ( + f"{url} is titled {words}, naming none of its objects" + ) + + +def test_no_api_entry_renders_empty(built_site: Path) -> None: + config = (REPO_ROOT / "zensical.toml").read_text(encoding="utf-8") + assert re.search(r"^show_if_no_docstring = false$", config, re.MULTILINE) + empty = [] + for url, page_html in rendered_pages(built_site): + if not url.startswith("/api/"): + continue + for section in re.split( + r'(?=<h\d id="[\w.]+" class="doc doc-heading")', page_html + )[1:]: + anchor = _DOC_HEADING_ID.search(section) + contents = section.split('class="doc doc-contents', 1)[-1] + contents = re.sub( + r'<div class="[^"]*doc-signature.*?</div>', + "", + contents, + flags=re.DOTALL, + ) + if not re.search(r"<p>", contents.split("doc doc-heading", 1)[0]): + empty.append(f"{url} {anchor.group(1) if anchor else '?'}") + assert not empty, "API entries with no text:\n" + "\n".join(empty) + + +def test_every_public_object_and_member_has_a_docstring() -> None: + """``show_if_no_docstring`` is off, so an undocumented member would vanish.""" + import griffe + + package = griffe.load( + "ferro", + search_paths=[str(REPO_ROOT / "src")], + extensions=griffe.load_extensions("griffe_inherited_docstrings"), + resolve_aliases=True, + resolve_external=False, + ) + undocumented = [] + for path in sorted(_public_paths()): + obj = package[path.removeprefix("ferro.")] + if not obj.docstring: + undocumented.append(path) + if obj.is_class: + undocumented += [ + f"{path}.{name}" + for name, member in obj.members.items() + if not name.startswith("_") + and name != "model_config" + and not member.docstring + ] + assert not undocumented, f"public objects with no docstring: {undocumented}" + + +_HREF = re.compile(r'<a\b[^>]*\bhref="([^"]+)"') + + +def test_links_from_api_pages_land_on_existing_anchors(built_site: Path) -> None: + dead = [] + for url, page_html in rendered_pages(built_site): + if not url.startswith("/api/"): + continue + for href in sorted({html.unescape(href) for href in _HREF.findall(page_html)}): + if re.match(r"[a-z]+:", href): + continue + path, _, fragment = urljoin(url, href).partition("#") + if not fragment: + continue + target = built_site / path.strip("/") / "index.html" + if not target.is_file() or f'id="{fragment}"' not in target.read_text( + encoding="utf-8" + ): + dead.append(f"{url} -> {href}") + assert not dead, "links to anchors that do not exist:\n" + "\n".join(dead) + + +# --- Navigation, front matter and moved pages (#723) --- + +DOCS_PAGES = REPO_ROOT / "docs" / "pages" + +NAV_TABS = ["Get started", "Guide", "How-to", "Concepts", "Reference", "Project"] + +NEW_PAGES = { + "concepts/row-level-security.md", + "guide/schema/renames-and-retirements.md", + "guide/schema/squash.md", + "reference/migrations.md", + "reference/row-level-security.md", +} +"""Pages created by the docs rewrite (#698); they carry ``status: new``.""" + +MOVED_PAGES = { + "getting-started/next-steps.md": "getting-started/quickstart.md#next-steps", + "howto/adopting-migrations.md": "guide/schema/adopting-migrations.md", + "howto/upgrade-guide.md": "upgrading.md", +} +"""Every page moved or deleted by the nav rework, and where its URL now goes.""" + +_NEXT_STEPS = "getting-started/quickstart.md#next-steps" + +MOVED_ANCHORS = { + # The migrations guide split (#714): sections moved to their own pages, + # or renamed on the page they stayed on. + "guide/schema/migrations.md#renames": "guide/schema/renames-and-retirements.md#rename-hints", + "guide/schema/migrations.md#retiring-a-column": "guide/schema/renames-and-retirements.md#retiring-a-column", + "guide/schema/migrations.md#retiring-an-enum-label": "guide/schema/renames-and-retirements.md#retiring-an-enum-label", + "guide/schema/migrations.md#headers": "reference/migrations.md#step-file-headers", + "guide/schema/migrations.md#changes-existing-rows-need-a-value-for": "guide/schema/migrations.md#backfills", + "guide/schema/migrations.md#checking": "guide/schema/migrations.md#check-status-and-drift", + "guide/schema/migrations.md#going-back": "guide/schema/migrations.md#reverting-with-down", + "guide/schema/migrations.md#editing-a-migration": "guide/schema/migrations.md#edited-migrations", + "guide/schema/migrations.md#two-branches-one-number": "guide/schema/migrations.md#duplicate-migration-numbers-and-replace", + "guide/schema/migrations.md#squashing-an-epics-migrations": "guide/schema/squash.md", + "guide/schema/migrations.md#a-rename-inside-the-range": "guide/schema/squash.md#renames-inside-the-range", + "guide/schema/migrations.md#every-database-that-applied-the-originals-inherits-the-squash": "guide/schema/squash.md#inheritance-on-databases-that-ran-the-originals", + "guide/schema/migrations.md#a-squash-is-pre-release": "guide/schema/squash.md#squash-timing", + "guide/schema/migrations.md#squash-or---replace": "guide/schema/squash.md#squash-vs-replace", + # The row-level security split (#715): each section of the old guide and + # the page that holds it now. + "guide/row-level-security.md#the-pooler-constraint": "concepts/row-level-security.md#the-pooler-constraint", + "guide/row-level-security.md#deploy-ordering": "guide/row-level-security.md#deploy-order", + "guide/row-level-security.md#declaring-row-security": "guide/row-level-security.md#the-tenant-policy", + "guide/row-level-security.md#the-shorthand": "reference/row-level-security.md#the-shorthand", + "guide/row-level-security.md#multiple-policies-composing-permissive-and-restrictive": "reference/row-level-security.md#composing-policies", + "guide/row-level-security.md#the-raw-escape-hatch": "reference/row-level-security.md#the-sql-form", + "guide/row-level-security.md#session-settings": "concepts/row-level-security.md#session-settings", + "guide/row-level-security.md#opening-a-session-with-settings": "guide/row-level-security.md#tenant-scoped-sessions", + "guide/row-level-security.md#deferred-resolution-current_sessionset_config": "guide/row-level-security.md#a-tenant-known-after-the-session-opens", + "guide/row-level-security.md#nesting-and-inheritance": "concepts/row-level-security.md#nested-sessions", + "guide/row-level-security.md#scope-stability-mid-operation": "concepts/row-level-security.md#visibility-of-a-new-value", + "guide/row-level-security.md#the-nullif-contract": "concepts/row-level-security.md#policy-expressions", + "guide/row-level-security.md#bypassrls": "concepts/row-level-security.md#force-and-bypassrls", + "guide/row-level-security.md#the-json-claims-recipe": "guide/row-level-security.md#json-claims", + "guide/row-level-security.md#framework-integration": "guide/row-level-security.md#web-framework-middleware", + "guide/row-level-security.md#outside-the-request-context": "guide/row-level-security.md#background-jobs", + "guide/row-level-security.md#connection-delivery-mode": "concepts/row-level-security.md#connection-delivery", + "guide/row-level-security.md#raw-policies-and-drift": "concepts/row-level-security.md#drift-and-ownership", + "guide/row-level-security.md#migrations-and-alembic": "guide/row-level-security.md#deploy-order", + # Sections renamed on pages the rewrite kept, the ones linked from outside + # the docs (the README, issues, the old upgrade guide). + "guide/mutations.md#get-or-create": "guide/mutations.md#get_or_create-and-update_or_create", + "guide/transactions.md#nesting-behavior": "guide/transactions.md#nested-transactions", + "guide/connections.md#sessions-recommended": "guide/connections.md#sessions", + "faq.md#what-is-ferro": "faq.md#what-ferro-is", + "howto/testing.md#writing-your-first-test": "howto/testing.md#a-first-test", + "howto/upgrade-guide.md#save-no-longer-upserts": "upgrading.md#save-and-upserts", + # The deleted Next steps page: every section lands on the quickstart's. + "getting-started/next-steps.md#learn-by-use-case": _NEXT_STEPS, + "getting-started/next-steps.md#building-an-api": _NEXT_STEPS, + "getting-started/next-steps.md#data-processing": _NEXT_STEPS, + "getting-started/next-steps.md#production-deployment": _NEXT_STEPS, + "getting-started/next-steps.md#common-patterns": _NEXT_STEPS, + "getting-started/next-steps.md#reference-material": _NEXT_STEPS, + "getting-started/next-steps.md#get-help": _NEXT_STEPS, +} +"""The anchors the docs rewrite redirects, and the section that replaced each. + +They are every section of the split pages (the migrations guide and the +row-level security guide) and of the deleted Next steps page, and the renamed +sections that outside links point at. The rewrite renamed most other sections +of the pages it rewrote without a redirect: an old link to one lands at the +top of its page, which still exists (``MOVED_PAGES`` redirects each page that +does not).""" + + +def _site_config() -> dict: + with (REPO_ROOT / "zensical.toml").open("rb") as config: + return tomllib.load(config)["project"] + + +def _nav_pages( + entries: list, trail: tuple[str, ...] = () +) -> Iterator[tuple[tuple[str, ...], str, str]]: + """Yield ``(sections, label, page)`` for every page in a nav subtree.""" + for entry in entries: + ((label, target),) = entry.items() + if isinstance(target, list): + yield from _nav_pages(target, (*trail, label)) + else: + yield trail, label, target + + +def _front_matter(page: Path) -> dict: + text = page.read_text(encoding="utf-8") + match = re.match(r"---\n(.*?)\n---\n", text, re.DOTALL) + return (yaml.safe_load(match.group(1)) or {}) if match else {} + + +def _url_path(page: str) -> str: + """The URL path of a page, without slashes: ``guide/queries.md`` → ``guide/queries``.""" + return "" if page == "index.md" else page.removesuffix(".md") + + +def _source_pages() -> list[str]: + return sorted( + page.relative_to(DOCS_PAGES).as_posix() for page in DOCS_PAGES.rglob("*.md") + ) + + +def test_nav_has_the_six_tabs() -> None: + tabs = [next(iter(entry)) for entry in _site_config()["nav"]] + assert tabs == NAV_TABS + + +def test_the_row_level_security_split_sits_in_its_three_tabs() -> None: + """The RLS guide split into a how-to, an explanation and a reference; each + part sits in the tab of its kind.""" + tabs = {page: trail[0] for trail, _, page in _nav_pages(_site_config()["nav"])} + assert tabs["guide/row-level-security.md"] == "How-to" + assert tabs["concepts/row-level-security.md"] == "Concepts" + assert tabs["reference/row-level-security.md"] == "Reference" + + +def test_every_page_is_in_the_nav_once() -> None: + in_nav = [page for _, _, page in _nav_pages(_site_config()["nav"])] + duplicated = sorted({page for page in in_nav if in_nav.count(page) > 1}) + assert not duplicated, f"pages listed twice in the nav: {duplicated}" + missing = sorted(set(_source_pages()) - set(in_nav)) + assert not missing, f"pages missing from the nav: {missing}" + + +def test_nav_labels_are_unique() -> None: + labels = [label for _, label, _ in _nav_pages(_site_config()["nav"])] + assert sorted({label for label in labels if labels.count(label) > 1}) == [] + + +def _h1(page_html: str) -> str: + match = re.search(r"<h1\b[^>]*>(.*?)</h1>", page_html, re.DOTALL) + assert match, "page has no <h1>" + heading = re.sub(r'<a class="headerlink".*?</a>', "", match.group(1)) + return html.unescape(_TAG.sub("", heading)).strip() + + +def test_every_nav_label_equals_its_page_h1(built_site: Path) -> None: + mismatched = [ + f"{page}: nav {label!r}, page {heading!r}" + for _, label, page in _nav_pages(_site_config()["nav"]) + if (heading := _h1(_page(built_site, _url_path(page)))) != label + ] + assert not mismatched, "nav labels differ from page titles:\n" + "\n".join( + mismatched + ) + + +def test_every_page_has_a_one_sentence_description() -> None: + lacking = [] + for page in _source_pages(): + description = _front_matter(DOCS_PAGES / page).get("description") + if ( + not isinstance(description, str) + or not description.endswith(".") + or ". " in description + ): + lacking.append(page) + assert not lacking, f"pages without a one-sentence description: {lacking}" + + +def test_page_descriptions_reach_the_built_pages(built_site: Path) -> None: + for page in _source_pages(): + description = _front_matter(DOCS_PAGES / page)["description"] + meta = re.search( + r'<meta name="description" content="([^"]*)"', + _page(built_site, _url_path(page)), + ) + assert meta, f"{page} has no description meta tag" + assert html.unescape(meta.group(1)) == description, page + + +def test_pages_new_in_the_rewrite_are_marked_new() -> None: + marked = { + page + for page in _source_pages() + if _front_matter(DOCS_PAGES / page).get("status") == "new" + } + assert marked == NEW_PAGES + assert "new" in _site_config()["extra"]["status"] + + +@pytest.mark.parametrize("old", sorted(MOVED_PAGES)) +def test_moved_page_is_gone_and_its_url_redirects(built_site: Path, old: str) -> None: + assert not (DOCS_PAGES / old).exists(), f"{old} is still in the source" + old_url = f"https://ferro-orm.x54.sh/{_url_path(old)}/" + targets = _redirect_targets(_page(built_site, _url_path(old))) + path, _, fragment = MOVED_PAGES[old].partition("#") + expected = f"/{_url_path(path)}/" + (f"#{fragment}" if fragment else "") + assert urljoin(old_url, targets[""]) == urljoin(old_url, expected) + sitemap = (built_site / "sitemap.xml").read_text(encoding="utf-8") + assert f"/{_url_path(old)}/<" not in sitemap + + +def test_moved_anchors_redirect_to_existing_sections(built_site: Path) -> None: + redirects = json.loads((built_site / "redirect.json").read_text(encoding="utf-8")) + for old, new in MOVED_ANCHORS.items(): + old_page, _, old_fragment = old.partition("#") + new_page, _, new_fragment = new.partition("#") + key = f"{_url_path(old_page)}/#{old_fragment}" + expected = f"{_url_path(new_page)}/" + ( + f"#{new_fragment}" if new_fragment else "" + ) + assert redirects.get(key) == expected, ( + f"{old} redirects to {redirects.get(key)}" + ) + if new_fragment: + target = _page(built_site, _url_path(new_page)) + assert f'id="{new_fragment}"' in target, f"{new} has no such anchor" + if (DOCS_PAGES / old_page).exists(): + assert f'id="{old_fragment}"' not in _page( + built_site, _url_path(old_page) + ), f"{old} is a live section; redirecting it would hide it" + + +# --- Markup that renders wrong --- + +_PRE = re.compile(r"<pre\b.*?</pre>", re.DOTALL) +_INLINE_CODE = re.compile(r"<code\b[^>]*>.*?</code>", re.DOTALL) +_PARAGRAPH = re.compile(r"<p>(.*?)</p>", re.DOTALL) +_SPHINX_ROLE = re.compile(r":(?:py:)?(?:class|meth|func|attr|data|mod|exc|obj):(?=\S)") +_LEFTOVER_REFERENCE = re.compile(r"\[[^\[\]]*\]\[[\w.:-]*\]") +_LIST_ITEM_LINE = re.compile(r"^\s*(?:[-*+]|\d+\.)\s") +_SECTION_HEADER_LINE = re.compile( + r"^\s*(?:Args|Arguments|Parameters|Attributes|Returns|Raises|Yields|Examples):\s*$" +) +_SECTION_TABLE = re.compile( + r'<span class="doc-section-title">(\w+):</span></p>\s*<table>(.*?)</table>', + re.DOTALL, +) +_SECTION_ROW = re.compile(r'<tr class="doc-section-item">(.*?)</tr>', re.DOTALL) +_CELL = re.compile(r"<td[^>]*>(.*?)</td>", re.DOTALL) +_SENTENCE_END = re.compile(r"[.?!]\)?$") +_GRIFFE_WARNING = re.compile(r"^\s*griffe: .*$", re.MULTILINE) + + +def _text(fragment: str) -> str: + return html.unescape(_TAG.sub("", fragment)) + + +def markup_defects(page_html: str) -> list[str]: + """Name each place a page renders Markdown or a docstring wrongly. + + Code blocks are skipped and inline code is blanked, so an example that + shows ``x[0][1]`` or a Sphinx role on purpose is not a defect. Found: + + - a literal Sphinx role (``:class:`Model```), which MkDocs never renders; + - a ``[text][target]`` left as text: a cross-reference nothing resolved; + - a paragraph holding a line that starts ``- `` or ``1. ``: a list with + no blank line before it, merged into the paragraph; + - a paragraph holding an ``Args:`` or ``Attributes:`` line: a docstring + section the parser did not recognise, rendered as prose; + - a ``Returns:``/``Attributes:``/... table row whose description stops + mid-sentence: a continuation line not indented, split into its own row. + """ + defects = [] + prose = _INLINE_CODE.sub("<code>C</code>", _PRE.sub("", page_html)) + text = _text(prose) + defects += [f"Sphinx role {m.group()}" for m in _SPHINX_ROLE.finditer(text)] + defects += [ + f"unresolved cross-reference {m.group()}" + for m in _LEFTOVER_REFERENCE.finditer(text) + ] + for paragraph in _PARAGRAPH.findall(prose): + if paragraph.startswith('<span class="doc-section-title">'): + continue + lines = _text(paragraph).splitlines() + if any(_LIST_ITEM_LINE.match(line) for line in lines[1:]): + defects.append(f"list merged into a paragraph: {lines[0]!r}") + defects += [ + f"docstring section rendered as prose: {line.strip()!r}" + for line in lines + if _SECTION_HEADER_LINE.match(line) + ] + for title, table in _SECTION_TABLE.findall(page_html): + headers = re.findall(r"<th>(.*?)</th>", table) + if "Description" not in headers: + continue + column = headers.index("Description") + for row in _SECTION_ROW.findall(table): + description = " ".join(_text(_CELL.findall(row)[column]).split()) + if not _SENTENCE_END.search(description): + defects.append(f"{title} row is a sentence fragment: {description!r}") + return defects + + +def docstring_warnings(build_log: str) -> list[str]: + """Return griffe's warnings from a build's output. + + A docstring item griffe cannot read (an ``Args:`` entry with no + ``name: description`` pair) is dropped from the page with only a + warning, and ``--strict`` does not fail on it. + """ + return [match.group().strip() for match in _GRIFFE_WARNING.finditer(build_log)] + + +def test_no_page_renders_markup_defects(built_site: Path) -> None: + defects = [ + f"{url}: {defect}" + for url, page_html in rendered_pages(built_site) + for defect in markup_defects(page_html) + ] + assert not defects, "pages render markup wrongly:\n" + "\n".join(defects) + + +def test_every_docstring_parses_without_a_warning(built_site: Path) -> None: + log = (built_site.parent / BUILD_LOG).read_text(encoding="utf-8") + assert docstring_warnings(log) == [] + + +SEEDED_MODULE = '''\ +"""A module seeded with one docstring defect per check.""" + + +def sphinx_role(value): + """Return a :class:`ferro.Model` for ``value``.""" + + +def unparsed_args(value): + """Return ``value``. + + Args: + value the value, with no colon and no indent + """ + + +def unreadable_args(value): + """Return ``value``. + + Args: + value the value, with no colon + """ + + +def split_returns(): + """Return a value. + + Returns: + A value whose description continues + on a second line that is not indented. + """ + + +class SplitAttributes: + """A class whose attribute description splits into two rows. + + Attributes: + name: The name, whose description runs + on: to a second row. + """ + + +class DroppedAttributes: + """A class whose attributes section is not indented. + + Attributes: + name the name + """ + + +def unresolved(): + """Point at [`Nope`][ferro.Nope].""" +''' + +SEEDED_PAGE = """\ +--- +description: "A page seeded with one Markdown defect per check." +--- + +# Seeded + +A dash list with no blank line before it: +- one +- two + +A numbered list with no blank line before it: +1. first +2. second + +::: seeded +""" + + +@pytest.fixture(scope="session") +def seeded_site(tmp_path_factory: pytest.TempPathFactory) -> tuple[str, str]: + """Build the site with a seeded page and module; return its HTML and log. + + The build is not strict: a strict build stops at the unresolved + reference before the other defects reach a page. + """ + project = _scratch_project(tmp_path_factory.mktemp("seeded-site")) + (project / "seeded_src").mkdir() + (project / "seeded_src" / "seeded.py").write_text(SEEDED_MODULE, encoding="utf-8") + (project / "docs" / "pages" / "seeded.md").write_text(SEEDED_PAGE, encoding="utf-8") + config = project / "zensical.toml" + text = config.read_text(encoding="utf-8") + assert '\npaths = ["src"]\n' in text + config.write_text( + text.replace('\npaths = ["src"]\n', '\npaths = ["src", "seeded_src"]\n'), + encoding="utf-8", + ) + result = _build(project) + assert result.returncode == 0, result.stdout + result.stderr + page = _page(project / "site", "seeded") + return page, (project / BUILD_LOG).read_text(encoding="utf-8") + + +@pytest.mark.parametrize( + "expected", + [ + "Sphinx role :class:", + "unresolved cross-reference [C][ferro.Nope]", + "list merged into a paragraph: 'A dash list with no blank line before it:'", + "list merged into a paragraph: 'A numbered list with no blank line before it:'", + "docstring section rendered as prose: 'Args:'", + "docstring section rendered as prose: 'Attributes:'", + "Returns row is a sentence fragment: 'A value whose description continues'", + "Attributes row is a sentence fragment: 'The name, whose description runs'", + ], +) +def test_seeded_markup_defect_is_caught( + seeded_site: tuple[str, str], expected: str +) -> None: + page, _ = seeded_site + assert expected in markup_defects(page) + + +def test_seeded_unreadable_docstring_item_is_caught( + seeded_site: tuple[str, str], +) -> None: + _, log = seeded_site + assert any("seeded.py:" in warning for warning in docstring_warnings(log)), log diff --git a/tests/test_docs_sql.py b/tests/test_docs_sql.py new file mode 100644 index 00000000..88c86cb1 --- /dev/null +++ b/tests/test_docs_sql.py @@ -0,0 +1,405 @@ +"""The SQL the query and mutation guides show is the SQL Ferro sends. + +A page never carries SQL a person typed. Every ```sql block on the Queries, +Aggregations, Relationships and Mutations guides opens with a marker line +naming where its SQL comes from:: + + -- predicates.py:first-query (sqlite) + +that is, an example script in ``docs/examples/``, one of its snippet regions +(``# --8<-- [start:first-query]`` ... ``[end:first-query]``) and the dialect. +The block holds every statement that region sends, in order: a query +terminal's statement as its ``.sql()`` renders it, and any other statement +(``save()``, ``create()``, ``bulk_create()``, ``delete()`` on an instance, a +many-to-many ``add()``) as the statement log records it. Schema statements +(``migrate``), Postgres catalog probes (``catalog``), session settings and a +``transaction()`` block's own ``BEGIN`` / ``COMMIT`` are not shown. + +Each statement is laid out with one clause per line (``layout``), ends with +``;`` and is set off from the next by a blank line. Joining a statement's +lines with single spaces gives back the rendered SQL byte for byte, so the +layout adds line breaks and nothing else. + +The capture, the layout and the block checks live in ``tests/_docs_sql.py``; +this module is the gate over the pages in ``PAGES``. Each named example runs +once per dialect in a subprocess, and each marked block is compared with what +its region sent. A block marked ``(postgres)`` runs under the +``postgres_only`` marker, against the database ``--db-backends`` selects. +""" + +from __future__ import annotations + +import asyncio +from pathlib import Path +from types import SimpleNamespace +from typing import Any + +import pytest + +from tests._docs_harness import DOCS_PAGES, Fence, fenced_blocks +from tests._docs_sql import ( + Capture, + Region, + Run, + captured, + check_block, + check_same_on_postgres, + layout, + sql_marker, + unlaid, +) + +PAGES = [ + "guide/queries.md", + "guide/aggregations.md", + "guide/relationships.md", + "guide/mutations.md", +] +"""The pages whose every ```sql block is regenerated by this test.""" + + +def _sql_blocks(page: str, text: str | None = None) -> list[Fence]: + """Every ```sql block of ``page`` (or of ``text``, read as that page).""" + path = DOCS_PAGES / page + if text is None: + text = path.read_text(encoding="utf-8") + return fenced_blocks(path, text, "sql") + + +def _all_blocks() -> list[Fence]: + return [block for page in PAGES for block in _sql_blocks(page)] + + +def _where(block: Fence) -> str: + return f"{block.path.relative_to(DOCS_PAGES).as_posix()}:{block.line}" + + +# -- the layout ------------------------------------------------------------------- + + +def test_layout_only_adds_line_breaks() -> None: + sql = ( + 'SELECT "t"."id" FROM "t" INNER JOIN "a" AS "x" ON "x"."id" = "t"."a_id" ' + 'WHERE EXISTS (SELECT 1 FROM "s" WHERE "s"."t_id" = "t"."id") ' + 'AND "t"."memo" = \'FROM x\' ORDER BY "t"."id" ASC LIMIT ? OFFSET ?' + ) + laid = layout(sql) + assert laid.splitlines() == [ + 'SELECT "t"."id"', + 'FROM "t"', + 'INNER JOIN "a" AS "x" ON "x"."id" = "t"."a_id"', + 'WHERE EXISTS (SELECT 1 FROM "s" WHERE "s"."t_id" = "t"."id") ' + 'AND "t"."memo" = \'FROM x\'', + 'ORDER BY "t"."id" ASC', + "LIMIT ?", + "OFFSET ?", + ] + assert unlaid(laid) == sql + + +def test_layout_puts_each_column_of_a_create_table_on_its_own_line() -> None: + sql = ( + 'CREATE TABLE IF NOT EXISTS "t" ( "id" integer NOT NULL, "a_id" integer, ' + 'CONSTRAINT "fk_t_a_id_a" FOREIGN KEY ("a_id") REFERENCES "a" ("id") ' + "ON DELETE CASCADE )" + ) + laid = layout(sql) + assert laid.splitlines() == [ + 'CREATE TABLE IF NOT EXISTS "t" (', + ' "id" integer NOT NULL,', + ' "a_id" integer,', + ' CONSTRAINT "fk_t_a_id_a" FOREIGN KEY ("a_id") REFERENCES "a" ("id") ' + "ON DELETE CASCADE", + ")", + ] + assert unlaid(laid) == sql + + +# -- the block check -------------------------------------------------------------- + + +def _fake(regions: dict[str, Region]) -> Any: + """A run of ``predicates.py`` whose regions did ``regions``.""" + return lambda example, dialect: Run( + regions if example == "predicates.py" else {}, "", [] + ) + + +SENT = _fake( + {"first-query": Region(sent=['SELECT "id" FROM "transaction" WHERE "amount" <= ?'])} +) + + +def _page_with(body: str) -> Fence: + [block] = _sql_blocks( + "guide/queries.md", f"# Queries\n\nIt sends:\n\n```sql\n{body}```\n" + ) + return block + + +def test_a_block_holding_what_its_region_sends_passes() -> None: + check_block( + _page_with( + "-- predicates.py:first-query (sqlite)\n" + 'SELECT "id"\nFROM "transaction"\nWHERE "amount" <= ?;\n' + ), + SENT, + ) + + +@pytest.mark.parametrize( + ("body", "message"), + [ + ( + "-- predicates.py:first-query (sqlite)\n" + 'SELECT "id"\nFROM "transaction"\nWHERE "amount" < ?;\n', + "guide/queries.md:5 (predicates.py:first-query, sqlite): the block differs", + ), + ( + 'SELECT "id"\nFROM "transaction"\nWHERE "amount" <= ?;\n', + "guide/queries.md:5: a ```sql block must open with a marker line", + ), + ( + "-- predicate.py:first-query (sqlite)\nSELECT 1;\n", + "guide/queries.md:5 (predicate.py:first-query, sqlite): " + "docs/examples/predicate.py does not exist", + ), + ( + "-- predicates.py:first-queries (sqlite)\nSELECT 1;\n", + "guide/queries.md:5 (predicates.py:first-queries, sqlite): " + "predicates.py has no snippet region 'first-queries'", + ), + ], + ids=["edited", "unmarked", "unknown-example", "unknown-region"], +) +def test_a_block_that_is_not_what_its_region_sends_fails_naming_it( + body: str, message: str +) -> None: + with pytest.raises(AssertionError) as raised: + check_block(_page_with(body), SENT) + assert str(raised.value).startswith(message), str(raised.value) + + +def test_a_block_for_a_region_that_sends_nothing_fails_naming_it() -> None: + block = _page_with("-- predicates.py:first-query (sqlite)\nSELECT 1;\n") + with pytest.raises(AssertionError) as raised: + check_block(block, _fake({})) + assert str(raised.value).startswith( + "guide/queries.md:5 (predicates.py:first-query, sqlite): " + "the region 'first-query' of predicates.py sends nothing" + ) + + +def test_every_page_shows_sql() -> None: + """Each of the four pages shows the SQL of its queries; none is left without.""" + for page in PAGES: + assert _sql_blocks(page), f"{page} shows no SQL" + + +def test_a_block_differing_on_postgres_only_in_placeholders_stands_for_both() -> None: + block = _page_with( + "-- predicates.py:first-query (sqlite)\n" + 'SELECT "id"\nFROM "transaction"\nWHERE "amount" <= ?;\n' + ) + numbered = 'SELECT "id" FROM "transaction" WHERE "amount" <= $1' + check_same_on_postgres(block, _fake({"first-query": Region(sent=[numbered])})) + + cast = 'SELECT "id" FROM "transaction" WHERE "amount" <= CAST($1 AS numeric)' + with pytest.raises(AssertionError) as raised: + check_same_on_postgres(block, _fake({"first-query": Region(sent=[cast])})) + assert str(raised.value).startswith( + "guide/queries.md:5 (predicates.py:first-query, sqlite): the block stands " + "for both databases" + ) + + +# -- the gate --------------------------------------------------------------------- + + +def _on(url: str | None) -> Any: + return lambda example, dialect: captured(example, dialect, url) + + +def _dialect(block: Fence) -> str: + """The dialect a block is marked for; an unmarked block counts as SQLite's, + so it fails there naming itself.""" + marker = sql_marker(block) + return marker["dialect"] if marker else "sqlite" + + +def _key(block: Fence) -> tuple[Path, str, str] | None: + marker = sql_marker(block) + return (block.path, marker["example"], marker["snippet"]) if marker else None + + +def _postgres_checked() -> list[Fence]: + """Every block but a SQLite one whose region has a ``(postgres)`` twin.""" + twins = {_key(block) for block in _all_blocks() if _dialect(block) == "postgres"} + return [ + block + for block in _all_blocks() + if not ( + _dialect(block) == "sqlite" and sql_marker(block) and _key(block) in twins + ) + ] + + +@pytest.mark.parametrize( + "block", + [block for block in _all_blocks() if _dialect(block) == "sqlite"], + ids=_where, +) +def test_a_sqlite_block_is_what_its_example_sends(block: Fence) -> None: + check_block(block, _on(None)) + + +@pytest.mark.postgres_only +@pytest.mark.parametrize("block", _postgres_checked(), ids=_where) +def test_a_block_is_what_its_example_sends_on_postgres( + block: Fence, db_url: str +) -> None: + """A ``(postgres)`` block is what its region sends on PostgreSQL; a SQLite + block with no ``(postgres)`` twin is too, once placeholders are read alike.""" + if _dialect(block) == "postgres": + check_block(block, _on(db_url)) + return + check_same_on_postgres(block, _on(db_url)) + + +# -- the capture, through what it reports ----------------------------------------- + + +class _Rendered: + def __init__(self, sql: str, dialect: str = "sqlite") -> None: + self.sql, self.dialect = sql, dialect + + def __str__(self) -> str: + return self.sql + + +SCRATCH = """\ +async def main(): + # --8<-- [start:query] + rows = await query() + # --8<-- [end:query] +""" + + +def _capture_at(tmp_path: Path, line: int) -> Capture: + """A capture of a scratch example whose line ``line`` is running now.""" + path = tmp_path / "scratch.py" + path.write_text(SCRATCH, encoding="utf-8") + capture = Capture(path, "sqlite") + capture.on_line(SimpleNamespace(co_filename=str(path)), line) + return capture + + +async def _returns(value: Any) -> Any: + return value + + +async def _raises(error: Exception) -> Any: + raise error + + +def test_a_terminal_whose_sql_raises_but_whose_call_succeeds_fails_the_capture( + tmp_path: Path, +) -> None: + capture = _capture_at(tmp_path, 3) + with pytest.raises(AssertionError) as raised: + asyncio.run( + capture.run_terminal( + lambda: _raises(ValueError("cannot render")), _returns([1, 2]) + ) + ) + assert str(raised.value).startswith( + "scratch.py:3: .sql() raised ValueError('cannot render') but the call succeeded" + ), str(raised.value) + + +def test_a_terminal_whose_sql_and_call_refuse_differently_fails_the_capture( + tmp_path: Path, +) -> None: + capture = _capture_at(tmp_path, 3) + with pytest.raises(AssertionError, match=r"^scratch\.py:3: \.sql\(\) raised"): + asyncio.run( + capture.run_terminal( + lambda: _raises(ValueError("no")), _raises(TypeError("no")) + ) + ) + + +def test_a_terminal_that_refuses_in_both_raises_its_error_and_records_nothing( + tmp_path: Path, +) -> None: + capture = _capture_at(tmp_path, 3) + with pytest.raises(ValueError, match="refused"): + asyncio.run( + capture.run_terminal( + lambda: _raises(ValueError("cannot")), _raises(ValueError("refused")) + ) + ) + assert capture.by_region() == {"query": Region()} + + +def test_a_terminal_records_its_rendering_once_and_returns_its_result( + tmp_path: Path, +) -> None: + capture = _capture_at(tmp_path, 3) + + async def outer() -> Any: + # A terminal another runs is rendered by the outer one only. + inner = capture.run_terminal( + lambda: _returns(_Rendered("SELECT 2")), _returns(2) + ) + return await capture.run_terminal( + lambda: _returns(_Rendered("SELECT 1")), inner + ) + + assert asyncio.run(outer()) == 2 + assert capture.by_region() == {"query": Region(sent=["SELECT 1"])} + + +EXAMPLE = """\ +import asyncio + +from ferro import Field, Model, connect, engines + + +class Note(Model): + id: int | None = Field(default=None, primary_key=True) + text: str + + +def show(notes): + print([note.text for note in notes]) + + +async def main(): + await connect("sqlite::memory:", auto_migrate=True) + async with engines.session(): + # --8<-- [start:create] + await Note.create(text="first") + print("created") + # --8<-- [end:create] + # --8<-- [start:list] + show(await Note.where(lambda note: note.id >= 1).all()) + # --8<-- [end:list] + + +asyncio.run(main()) +""" + + +def test_a_run_reports_what_each_region_sends_and_prints(tmp_path: Path) -> None: + script = tmp_path / "notes.py" + script.write_text(EXAMPLE, encoding="utf-8") + run = captured(script) + assert run.regions["create"].printed == ["created"] + assert [sql.split(" ")[0] for sql in run.regions["create"].sent] == ["INSERT"] + # The helper prints outside any region; the line inside "list" that + # called it owns the output. + assert run.regions["list"].shown_output == "['first']\n" + assert run.regions["list"].sent[0].startswith("SELECT ") + assert run.stdout == "created\n['first']\n" + assert run.schemas, "the run records the models' schema" diff --git a/tests/test_docs_testing_fixtures.py b/tests/test_docs_testing_fixtures.py new file mode 100644 index 00000000..ce96066c --- /dev/null +++ b/tests/test_docs_testing_fixtures.py @@ -0,0 +1,192 @@ +"""The testing how-to's fixtures, run the way the page tells readers to. + +``docs/pages/howto/testing.md`` hands readers a ``tests/conftest.py`` +(``docs/examples/testing_conftest.py``), a pytest configuration and a few +test modules. Each test here builds that project in a temporary directory: + +- ``myapp/models.py`` from ``docs/examples/testing_models.py``, +- ``tests/conftest.py`` from ``docs/examples/testing_conftest.py``, +- ``pyproject.toml`` from the page's ``[tool.pytest.ini_options]`` block, +- every page fence marked ``{test=pytest path="tests/<file>.py"}`` at its path, + +then runs pytest there in a subprocess, so the fixtures run under the +pytest-asyncio mode the page configures, not this suite's. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +import pytest + +from tests._docs_harness import ( + fence_settings, + fenced_blocks, + fenced_examples, + run_python, +) + +REPO_ROOT = Path(__file__).resolve().parents[1] +PAGE = REPO_ROOT / "docs" / "pages" / "howto" / "testing.md" +EXAMPLES = REPO_ROOT / "docs" / "examples" + +_PYTEST_OUTPUT = re.compile( + r"^```text\n\$ pytest -q (?P<target>\S+)\n.*?^(?P<passed>\d+) passed", + flags=re.M | re.S, +) + + +def _page_test_modules() -> dict[str, str]: + """The page's pytest modules: path in the project -> source.""" + text = PAGE.read_text(encoding="utf-8") + modules = {} + for example in fenced_examples(PAGE, text): + settings = fence_settings(example) + if settings.get("test") == "pytest": + modules[settings["path"]] = example.source + return modules + + +def _write_project(root: Path, extra: dict[str, str] | None = None) -> None: + text = PAGE.read_text(encoding="utf-8") + [pyproject] = [ + fence.body + for fence in fenced_blocks(PAGE, text, "toml") + if "[tool.pytest.ini_options]" in fence.body + ] + (root / "pyproject.toml").write_text(pyproject, encoding="utf-8") + (root / "myapp").mkdir() + (root / "myapp" / "__init__.py").write_text("", encoding="utf-8") + (root / "myapp" / "models.py").write_text( + (EXAMPLES / "testing_models.py").read_text(encoding="utf-8"), encoding="utf-8" + ) + (root / "tests").mkdir() + (root / "tests" / "conftest.py").write_text( + (EXAMPLES / "testing_conftest.py").read_text(encoding="utf-8"), encoding="utf-8" + ) + for path, source in {**_page_test_modules(), **(extra or {})}.items(): + target = root / path + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(source, encoding="utf-8") + + +def _pytest(root: Path, *args: str, env: dict[str, str] | None = None) -> str: + result = run_python( + ["-m", "pytest", "-q", "-p", "no:cacheprovider", *args], + timeout=120, + cwd=root, + env=env, + ) + return result.stdout + + +def test_page_has_its_pytest_modules() -> None: + assert set(_page_test_modules()) == { + "tests/test_users.py", + "tests/test_raw_sql.py", + "tests/test_posts.py", + } + + +def test_page_tests_pass_with_the_page_fixtures(tmp_path: Path) -> None: + _write_project(tmp_path) + + stdout = _pytest(tmp_path) + + assert "4 passed" in stdout, stdout + + +def test_page_shows_what_pytest_prints(tmp_path: Path) -> None: + _write_project(tmp_path) + shown = _PYTEST_OUTPUT.search(PAGE.read_text(encoding="utf-8")) + assert shown is not None, "the page shows no `$ pytest -q <file>` output" + + stdout = _pytest(tmp_path, shown["target"]) + + assert f"{shown['passed']} passed" in stdout, stdout + + +ROLLBACK_PROBE = ''' +import pytest + +from myapp.models import User + + +@pytest.fixture +async def users_after_test(db): + """Torn down after db_transaction and before db: counts what was kept.""" + yield + remaining = await User.select().count() + assert remaining == 0, f"db_transaction kept {remaining} row(s)" + + +async def test_row_created_in_the_transaction(users_after_test, db_transaction): + await User.create(username="ada", email="ada@example.com") + row = await db_transaction.fetch_one('SELECT count(*) AS total FROM "user"') + assert row == {"total": 1} +''' + +ISOLATION_PROBE = """ +from myapp.models import User + + +async def test_first_ada(db): + await User.create(username="ada", email="ada@example.com") + assert await User.select().count() == 1 + + +async def test_second_ada(db): + await User.create(username="ada", email="ada@example.com") + assert await User.select().count() == 1 +""" + + +def test_db_transaction_rolls_the_row_back(tmp_path: Path) -> None: + _write_project(tmp_path, {"tests/test_rollback_probe.py": ROLLBACK_PROBE}) + + stdout = _pytest(tmp_path, "tests/test_rollback_probe.py") + + assert "1 passed" in stdout, stdout + + +def test_db_gives_each_test_an_empty_database(tmp_path: Path) -> None: + _write_project(tmp_path, {"tests/test_isolation_probe.py": ISOLATION_PROBE}) + + stdout = _pytest(tmp_path, "tests/test_isolation_probe.py") + + assert "2 passed" in stdout, stdout + + +POSTGRES_PROBE = """ +from myapp.models import User + + +async def test_user_in_its_own_schema(pg_db): + await User.create(username="ada", email="ada@example.com") + assert await User.select().count() == 1 +""" + + +@pytest.mark.postgres_only +def test_pg_db_isolates_each_test_in_a_schema( + tmp_path: Path, postgres_base_url: str | None +) -> None: + if postgres_base_url is None: + pytest.skip("No Postgres provider is configured.") + _write_project(tmp_path) + pg_dir = tmp_path / "tests" / "postgres" + pg_dir.mkdir() + (pg_dir / "conftest.py").write_text( + (EXAMPLES / "testing_conftest_postgres.py").read_text(encoding="utf-8"), + encoding="utf-8", + ) + (pg_dir / "test_pg.py").write_text(POSTGRES_PROBE, encoding="utf-8") + + stdout = _pytest( + tmp_path, + "tests/postgres", + env={"TEST_POSTGRES_URL": postgres_base_url}, + ) + + assert "1 passed" in stdout, stdout diff --git a/tests/test_docs_twins.py b/tests/test_docs_twins.py new file mode 100644 index 00000000..3d6119f5 --- /dev/null +++ b/tests/test_docs_twins.py @@ -0,0 +1,177 @@ +"""Each example's Annotated twin is the same program in the other declaration style. + +AGENTS.md I-7 shows every field-declaring example in an Assignment tab and an +Annotated tab. For every ``docs/examples/<name>.py`` with a +``<name>_annotated.py``: + +- the twin is the Assignment script with each ``Field(...)`` declaration + rewritten into the Annotated idiom (``tests._docs_twins.annotated_twin``), + and nothing else changed, so the tabs differ only in field syntax; +- both declare the same schema (the registered models' SchemaIR each time the + script connects or creates tables) and print the same output. + +After editing an Assignment script, run +``python -m tests._docs_twins docs/examples/<name>.py`` to rewrite its twin. +""" + +from __future__ import annotations + +import textwrap +from pathlib import Path + +import pytest + +from tests._docs_sql import captured +from tests._docs_twins import annotated_twin, twin_pairs + + +def _twin(body: str) -> str: + return annotated_twin(textwrap.dedent(body).lstrip("\n")) + + +def test_options_move_inside_annotated_and_the_default_stays_assigned() -> None: + assert _twin( + """ + from ferro import Field, Model + + + class User(Model): + id: int | None = Field(default=None, primary_key=True) + email: str = Field(unique=True) + nickname: str | None = Field(None, index=True) + name: str + """ + ) == textwrap.dedent( + """\ + from typing import Annotated + + from ferro import Field, Model + + + class User(Model): + id: Annotated[int | None, Field(primary_key=True)] = None + email: Annotated[str, Field(unique=True)] + nickname: Annotated[str | None, Field(index=True)] = None + name: str + """ + ) + + +def test_a_default_factory_stays_an_assigned_field() -> None: + """Type checkers do not see a default inside Annotated, so the factory stays + the assigned value; only the other options move.""" + assert _twin( + """ + import uuid + from datetime import UTC, datetime + + from ferro import Field, Model + + + class Event(Model): + id: uuid.UUID = Field(default_factory=uuid.uuid4, primary_key=True) + at: datetime = Field(default_factory=lambda: datetime.now(UTC)) + """ + ) == textwrap.dedent( + """\ + import uuid + from datetime import UTC, datetime + from typing import Annotated + + from ferro import Field, Model + + + class Event(Model): + id: Annotated[uuid.UUID, Field(primary_key=True)] = Field( + default_factory=uuid.uuid4 + ) + at: datetime = Field(default_factory=lambda: datetime.now(UTC)) + """ + ) + + +def test_existing_annotated_metadata_and_typing_imports_are_kept() -> None: + assert _twin( + """ + from typing import TYPE_CHECKING, Annotated + + from ferro import Field, ForeignKey, Model + + + class Post(Model): + author: Annotated["Author", ForeignKey(related_name="posts")] + slug: Annotated[str, "slug"] = Field(unique=True) # a URL part + """ + ) == textwrap.dedent( + """\ + from typing import TYPE_CHECKING, Annotated + + from ferro import Field, ForeignKey, Model + + + class Post(Model): + author: Annotated["Author", ForeignKey(related_name="posts")] + slug: Annotated[str, "slug", Field(unique=True)] # a URL part + """ + ) + + +def test_a_comment_quoting_a_declaration_quotes_it_in_the_same_idiom() -> None: + assert _twin( + """ + from typing import Annotated + + from ferro import Field, Model + + + class Author(Model): + # before: nickname: str | None = Field(default=None, index=True) + handle: str | None = Field(default=None, renamed_from="nickname") + """ + ).splitlines()[6:8] == [ + " # before: nickname: Annotated[str | None, Field(index=True)] = None", + ' handle: Annotated[str | None, Field(renamed_from="nickname")] = None', + ] + + +def test_a_comment_inside_a_multi_line_declaration_is_refused() -> None: + with pytest.raises(ValueError, match="line 5: a comment inside a multi-line"): + _twin( + """ + from ferro import Field, Model + + class User(Model): + email: str = Field( + unique=True, # one account per address + ) + """ + ) + + +@pytest.mark.parametrize( + ("assignment", "annotated"), twin_pairs(), ids=lambda path: path.name +) +def test_a_twin_is_its_assignment_script_rewritten( + assignment: Path, annotated: Path +) -> None: + expected = annotated_twin(assignment.read_text(encoding="utf-8")) + assert annotated.read_text(encoding="utf-8") == expected, ( + f"docs/examples/{annotated.name} is not docs/examples/{assignment.name} " + f"in the Annotated idiom; run python -m tests._docs_twins " + f"docs/examples/{assignment.name} to rewrite it" + ) + + +@pytest.mark.parametrize( + ("assignment", "annotated"), twin_pairs(), ids=lambda path: path.name +) +def test_twins_declare_the_same_schema_and_print_the_same( + assignment: Path, annotated: Path +) -> None: + left, right = captured(assignment.name), captured(annotated.name) + assert right.schemas == left.schemas, ( + f"{annotated.name} declares a different schema from {assignment.name}" + ) + assert right.stdout == left.stdout, ( + f"{annotated.name} prints something other than {assignment.name}" + ) diff --git a/tests/test_docstring_examples.py b/tests/test_docstring_examples.py new file mode 100644 index 00000000..934b9200 --- /dev/null +++ b/tests/test_docstring_examples.py @@ -0,0 +1,253 @@ +"""Docstring examples in ``src/ferro``: every one is marked, compiles, and runs +when marked to. + +A fenced Python example in a docstring holds to the contract a docs page fence +does (``tests._docs_harness.check_fence``): it says how it is tested, compiles, +and one marked ``{test=run prelude=<name>}`` runs in a subprocess after +``docs/examples/_preludes/<name>.py``, inside its session. An unmarked fence is +refused. The entry points the reference leads with each keep an example that +runs (``RUNNING_ENTRY_POINTS``). +""" + +from __future__ import annotations + +import textwrap +from pathlib import Path + +import griffe +import pytest +from pytest_examples import CodeExample + +from tests._docs_harness import ( + check_fence, + docstring_examples, + fence_settings, + run_fence, +) + +REPO_ROOT = Path(__file__).resolve().parents[1] + + +@pytest.mark.parametrize( + "example", docstring_examples("ferro", REPO_ROOT / "src"), ids=str +) +def test_docstring_example(example: CodeExample) -> None: + check_fence(example) + + +@pytest.mark.postgres_only +@pytest.mark.parametrize( + "example", + [ + example + for example in docstring_examples("ferro", REPO_ROOT / "src") + if fence_settings(example).get("backend") == "postgres" + ], + ids=str, +) +def test_docstring_example_on_postgres(example: CodeExample, db_url: str) -> None: + """A docstring fence marked ``backend=postgres`` runs against PostgreSQL.""" + check_fence(example, database_url=db_url) + + +RUNNING_ENTRY_POINTS = [ + "models.Model.create", + "models.Model.get", + "models.Model.where", + "models.Model.save", + "models.Model.delete", + "query.builder.Query.after", + "query.builder.Query.before", + "query.builder.Query.position_of", + "migrate", +] +"""The entry points whose docstrings must each keep an ``Examples:`` fence +marked ``{test=run prelude=<name>}``, as paths inside the ``ferro`` package.""" + + +def entry_points_without_a_running_example( + package: str, search_path: Path, paths: list[str] +) -> list[str]: + """Name each of ``paths`` whose docstring holds no fence marked ``test=run``. + + Args: + package: The package to load, such as ``ferro``. + search_path: The directory that holds the package. + paths: Objects inside the package, such as ``models.Model.create``. + + Returns: + The paths with no running example, or with no docstring at all. + """ + root = griffe.load(package, search_paths=[str(search_path)], allow_inspection=False) + running: dict[Path, list[int]] = {} + for example in docstring_examples(package, search_path): + if fence_settings(example).get("test") == "run": + running.setdefault(Path(example.path), []).append(example.start_line) + missing = [] + for path in paths: + obj = root[path] + docstring = obj.docstring + lines = running.get(Path(str(obj.filepath)), []) + if docstring is None or not any( + docstring.lineno <= line <= (docstring.endlineno or 0) for line in lines + ): + missing.append(path) + return missing + + +def test_each_entry_point_keeps_a_running_example() -> None: + assert ( + entry_points_without_a_running_example( + "ferro", REPO_ROOT / "src", RUNNING_ENTRY_POINTS + ) + == [] + ) + + +def test_an_entry_point_whose_example_only_compiles_is_named(tmp_path: Path) -> None: + _scratch_package( + tmp_path, + ''' + def first_author(): + """Return the first author. + + Examples: + ```python {test=compile reason="shows a call"} + first_author() + ``` + """ + + + def last_author(): + """Return the last author. + + Examples: + ```python {test=run prelude=reference} + print(1) + ``` + """ + + + def any_author(): + """Return an author.""" + ''', + ) + assert entry_points_without_a_running_example( + "scratch", + tmp_path, + ["library.first_author", "library.last_author", "library.any_author"], + ) == ["library.first_author", "library.any_author"] + + +def test_an_unmarked_docstring_fence_is_refused(tmp_path: Path) -> None: + _scratch_package( + tmp_path, + ''' + def first_author(): + """Return the first author. + + ```python + first_author() + ``` + """ + ''', + ) + [example] = docstring_examples("scratch", tmp_path) + with pytest.raises(ValueError, match=r"says neither test=run nor test=compile"): + check_fence(example) + + +def _scratch_package(tmp_path: Path, body: str) -> Path: + package = tmp_path / "scratch" + package.mkdir() + (package / "__init__.py").write_text('"""Scratch package."""\n', "utf-8") + (package / "library.py").write_text( + textwrap.dedent(body).lstrip("\n"), encoding="utf-8" + ) + return package / "library.py" + + +def test_marked_docstring_example_is_collected_and_run(tmp_path: Path) -> None: + module = _scratch_package( + tmp_path, + ''' + def first_author(): + """Return the first author. + + Examples: + ```python {test=run prelude=reference} + author = await Author.where(lambda author: author.id == 1).first() + print(author.name) + ``` + + ```python + first_author() + ``` + """ + ''', + ) + examples = docstring_examples("scratch", tmp_path) + assert [(ex.path, ex.start_line) for ex in examples] == [ + (module, 5), + (module, 10), + ] + marked, unmarked = examples + assert fence_settings(marked) == {"test": "run", "prelude": "reference"} + assert fence_settings(unmarked) == {} + assert run_fence(marked).strip() == "Ada Lovelace" + + +def test_failing_docstring_example_names_its_file_line(tmp_path: Path) -> None: + module = _scratch_package( + tmp_path, + ''' + class Library: + """A library. + + ```python {test=run prelude=reference} + raise RuntimeError("boom") + ``` + """ + ''', + ) + [example] = docstring_examples("scratch", tmp_path) + with pytest.raises(AssertionError) as failure: + run_fence(example) + assert f'File "{module}", line 5' in str(failure.value) + + +def test_docstring_example_can_declare_related_models(tmp_path: Path) -> None: + _scratch_package( + tmp_path, + ''' + class Library: + """A library. + + ```python {test=run prelude=reference} + from typing import Annotated + + from ferro import BackRef, FerroField, ForeignKey, Model, Relation, create_tables + + + class Shelf(Model): + id: Annotated[int | None, FerroField(primary_key=True)] = None + label: str + books: Relation[list["Book"]] = BackRef() + + + class Book(Model): + id: Annotated[int | None, FerroField(primary_key=True)] = None + title: str + shelf: Annotated[Shelf, ForeignKey(related_name="books")] + + + await create_tables() + shelf = await Shelf.create(label="History") + await Book.create(title="Middlemarch", shelf=shelf) + print([book.title for book in await shelf.books.all()]) + ``` + """ + ''', + ) + [example] = docstring_examples("scratch", tmp_path) + assert run_fence(example).strip() == "['Middlemarch']" diff --git a/tests/test_documentation_features.py b/tests/test_documentation_features.py index 3f719c08..a969898c 100644 --- a/tests/test_documentation_features.py +++ b/tests/test_documentation_features.py @@ -1,458 +1,48 @@ -""" -Comprehensive integration tests for documented Ferro features. +"""Documented behaviour that no docs example exercises. -This test suite validates that all features documented in the user guide -work as expected. Each test corresponds to a specific documented capability. +Every other documented feature runs in the example scripts under +``docs/examples`` (``tests/test_docs_examples.py``) or in a page block marked +``{test=run prelude=<name>}``. What remains here is documented but shown by +no runnable example: an ``|`` predicate, and two tasks committing their own +transactions at once. Both run on every backend. """ import asyncio -from datetime import date, datetime -from decimal import Decimal from enum import Enum from typing import Annotated import pytest -from ferro import ( - BackRef, - FerroField, - Field, - ForeignKey, - ManyToMany, - Model, - Relation, - connect, - create_tables, - engines, - reset_engine, - transaction, -) +from ferro import FerroField, Model, connect, engines, transaction pytestmark = pytest.mark.backend_matrix -# Test Models class UserRole(Enum): - """Test enum for role field""" - USER = "user" ADMIN = "admin" MODERATOR = "moderator" class DocUser(Model): - """DocUser model for testing""" - id: Annotated[int | None, FerroField(primary_key=True)] = None username: Annotated[str, FerroField(unique=True)] email: Annotated[str, FerroField(unique=True, index=True)] - is_active: bool = True role: UserRole = UserRole.USER - posts: Relation[list["DocPost"]] = BackRef() - comments: Relation[list["Comment"]] = BackRef() - - -class DocPost(Model): - """DocPost model for testing""" - - id: Annotated[int | None, FerroField(primary_key=True)] = None - title: str - content: str - published: bool = False - created_at: datetime = Field(default_factory=datetime.now) - author: Annotated[DocUser, ForeignKey(related_name="posts")] - comments: Relation[list["Comment"]] = BackRef() - tags: Relation[list["Tag"]] = ManyToMany(related_name="posts") - - -class Comment(Model): - """Comment model for testing""" - - id: Annotated[int | None, FerroField(primary_key=True)] = None - text: str - created_at: datetime = Field(default_factory=datetime.now) - author: Annotated[DocUser, ForeignKey(related_name="comments")] - post: Annotated[DocPost, ForeignKey(related_name="comments")] -class Tag(Model): - """Tag model for testing many-to-many""" - - id: Annotated[int | None, FerroField(primary_key=True)] = None - name: Annotated[str, FerroField(unique=True)] - posts: Relation[list["DocPost"]] = BackRef() - - -class DocProduct(Model): - """DocProduct model for testing field types""" - - sku: str = Field(primary_key=True) - name: str - price: Decimal = Field(ge=0, decimal_places=2) - stock: int = 0 - created_date: date = Field(default_factory=date.today) - metadata_json: dict | None = None - - -# Re-register models before each test so they are in the Rust and Python -# registries even if another test's fixture cleared them (e.g. alembic/schema). @pytest.fixture(autouse=True) def _ensure_models_registered(): - from ferro.base import ForeignKey, ManyToManyRelation - from ferro.relations import resolve_relationships + """Re-register the model: another test's fixture may clear the registry.""" from ferro.registry import REGISTRY - for model_cls in (DocUser, DocPost, Comment, Tag, DocProduct): - REGISTRY.register(model_cls) - - # Some tests clear private relationship state to model fresh imports. These - # module-level models must restore both schemas and descriptors afterward. - for model_cls in (DocUser, DocPost, Comment, Tag, DocProduct): - for field_name, relation in model_cls.ferro_relations.items(): - if isinstance(relation, (ForeignKey, ManyToManyRelation)): - REGISTRY.defer_relation( - model_cls.__ferro_identity__, field_name, relation - ) - - resolve_relationships() + REGISTRY.register(DocUser) yield -# ============================================================================ -# MODELS & FIELDS TESTS (docs/guide/models-and-fields.md) -# ============================================================================ - - -@pytest.mark.asyncio -async def test_basic_model_definition(db_url): - """Test basic model definition from docs""" - await connect(db_url, auto_migrate=True) - - class SimpleUser(Model): - id: int - username: str - is_active: bool = True - - reset_engine() - await connect(db_url, auto_migrate=True) - async with engines.session(): - await create_tables() - user = SimpleUser(id=1, username="alice") - assert user.username == "alice" - assert user.is_active is True - - -@pytest.mark.asyncio -async def test_field_types(db_url): - """Test all documented field types work correctly""" - await connect(db_url, auto_migrate=True) - async with engines.session(): - product = await DocProduct.create( - sku="PROD-001", - name="Test Product", - price=Decimal("19.99"), - stock=100, - metadata_json={"color": "blue", "size": "large"}, - ) - - assert product.sku == "PROD-001" - assert product.price == Decimal("19.99") - assert product.stock == 100 - assert product.metadata_json["color"] == "blue" - assert isinstance(product.created_date, date) - - -@pytest.mark.asyncio -async def test_enum_field_type(db_url): - """Test enum field type works as documented""" - await connect(db_url, auto_migrate=True) - async with engines.session(): - user = await DocUser.create( - username="admin_user", email="admin@example.com", role=UserRole.ADMIN - ) - - fetched = await DocUser.get(user.id) - assert fetched.role == UserRole.ADMIN - assert isinstance(fetched.role, UserRole) - - -@pytest.mark.asyncio -async def test_field_constraints_pydantic_style(db_url): - """Test Field() constraint syntax""" - await connect(db_url, auto_migrate=True) - async with engines.session(): - product = await DocProduct.create( - sku="TEST-001", name="Test", price=Decimal("10.00") - ) - assert product.sku == "TEST-001" - - -@pytest.mark.asyncio -async def test_field_constraints_annotated_style(db_url): - """Test FerroField() annotated syntax""" - await connect(db_url, auto_migrate=True) - async with engines.session(): - user = await DocUser.create(username="test", email="test@example.com") - assert user.username == "test" - - # Verify unique constraint - with pytest.raises(Exception): # Should raise integrity error - await DocUser.create(username="test", email="other@example.com") - - -# ============================================================================ -# CRUD OPERATIONS TESTS (docs/guide/mutations.md) -# ============================================================================ - - -@pytest.mark.asyncio -async def test_create_method(db_url): - """Test Model.create() as documented""" - await connect(db_url, auto_migrate=True) - async with engines.session(): - user = await DocUser.create( - username="alice", email="alice@example.com", is_active=True - ) - - assert user.id is not None - assert user.username == "alice" - assert user.email == "alice@example.com" - - -@pytest.mark.asyncio -async def test_get_method(db_url): - """Test Model.get() as documented""" - await connect(db_url, auto_migrate=True) - async with engines.session(): - user = await DocUser.create(username="bob", email="bob@example.com") - - fetched = await DocUser.get(user.id) - assert fetched is not None - assert fetched.id == user.id - assert fetched.username == "bob" - - -@pytest.mark.asyncio -async def test_all_method(db_url): - """Test Model.all() as documented""" - await connect(db_url, auto_migrate=True) - async with engines.session(): - await DocUser.create(username="user1", email="user1@example.com") - await DocUser.create(username="user2", email="user2@example.com") - - all_users = await DocUser.all() - assert len(all_users) == 2 - - -@pytest.mark.asyncio -async def test_save_method(db_url): - """Test instance.save() as documented""" - await connect(db_url, auto_migrate=True) - async with engines.session(): - user = await DocUser.create(username="alice", email="alice@example.com") - - user.email = "alice.new@example.com" - user.is_active = False - await user.save() - - fetched = await DocUser.get(user.id) - assert fetched.email == "alice.new@example.com" - assert fetched.is_active is False - - -@pytest.mark.asyncio -async def test_delete_method(db_url): - """Test instance.delete() as documented""" - await connect(db_url, auto_migrate=True) - async with engines.session(): - user = await DocUser.create(username="alice", email="alice@example.com") - user_id = user.id - - await user.delete() - - assert await DocUser.get_or_none(user_id) is None - - -@pytest.mark.asyncio -async def test_refresh_method(db_url): - """Test instance.refresh() as documented""" - await connect(db_url, auto_migrate=True) - async with engines.session(): - user = await DocUser.create(username="alice", email="alice@example.com") - - # Simulate external update - await DocUser.where(lambda t: t.id == user.id).update( - email="updated@example.com" - ) - - # Refresh instance - await user.refresh() - assert user.email == "updated@example.com" - - -@pytest.mark.asyncio -async def test_bulk_create(db_url): - """Test Model.bulk_create() as documented""" - await connect(db_url, auto_migrate=True) - async with engines.session(): - users = [ - DocUser(username=f"user_{i}", email=f"user{i}@example.com") - for i in range(100) - ] - - count = await DocUser.bulk_create(users) - assert count == 100 - - all_users = await DocUser.all() - assert len(all_users) == 100 - - -@pytest.mark.asyncio -async def test_get_or_create(db_url): - """Test Model.get_or_create() as documented""" - await connect(db_url, auto_migrate=True) - async with engines.session(): - # First call creates - user1, created1 = await DocUser.get_or_create( - email="test@example.com", defaults={"username": "testuser"} - ) - assert created1 is True - assert user1.username == "testuser" - - # Second call retrieves - user2, created2 = await DocUser.get_or_create( - email="test@example.com", defaults={"username": "different"} - ) - assert created2 is False - assert user2.id == user1.id - assert user2.username == "testuser" # Defaults not applied - - -@pytest.mark.asyncio -async def test_update_or_create(db_url): - """Test Model.update_or_create() as documented""" - await connect(db_url, auto_migrate=True) - async with engines.session(): - # First call creates - user1, created1 = await DocUser.update_or_create( - email="test@example.com", defaults={"username": "testuser"} - ) - assert created1 is True - - # Second call updates - user2, created2 = await DocUser.update_or_create( - email="test@example.com", defaults={"username": "updated"} - ) - assert created2 is False - assert user2.id == user1.id - assert user2.username == "updated" - - -# ============================================================================ -# QUERY OPERATIONS TESTS (docs/guide/queries.md) -# ============================================================================ - - -@pytest.mark.asyncio -async def test_where_equality(db_url): - """Test .where() with equality operator""" - await connect(db_url, auto_migrate=True) - async with engines.session(): - await DocUser.create( - username="alice", email="alice@example.com", is_active=True - ) - await DocUser.create(username="bob", email="bob@example.com", is_active=False) - - active_users = await DocUser.where(lambda t: t.is_active == True).all() # noqa: E712 - assert len(active_users) == 1 - assert active_users[0].username == "alice" - - -@pytest.mark.asyncio -async def test_where_comparison_operators(db_url): - """Test comparison operators in queries""" - await connect(db_url, auto_migrate=True) - async with engines.session(): - for i in range(5): - await DocProduct.create( - sku=f"PROD-{i}", name=f"Product {i}", price=Decimal(str(i * 10)) - ) - - # Greater than - expensive = await DocProduct.where(lambda t: t.price > Decimal("20")).all() - assert len(expensive) == 2 # 30 and 40 - - # Less than or equal - cheap = await DocProduct.where(lambda t: t.price <= Decimal("20")).all() - assert len(cheap) == 3 # 0, 10, 20 - - -@pytest.mark.asyncio -async def test_where_like_operator(db_url): - """Test .like() operator as documented""" - await connect(db_url, auto_migrate=True) - async with engines.session(): - await DocUser.create(username="alice", email="alice@gmail.com") - await DocUser.create(username="bob", email="bob@yahoo.com") - await DocUser.create(username="charlie", email="charlie@gmail.com") - - gmail_users = await DocUser.where(lambda t: t.email.like("%gmail.com")).all() - assert len(gmail_users) == 2 - - -@pytest.mark.asyncio -async def test_where_in_operator(db_url): - """Test .in_() operator as documented""" - await connect(db_url, auto_migrate=True) - async with engines.session(): - await DocUser.create( - username="alice", email="alice@example.com", role=UserRole.ADMIN - ) - await DocUser.create( - username="bob", email="bob@example.com", role=UserRole.MODERATOR - ) - await DocUser.create( - username="charlie", email="charlie@example.com", role=UserRole.USER - ) - - # Use enum values instead of enum instances - staff = await DocUser.where( - lambda t: t.role.in_([UserRole.ADMIN.value, UserRole.MODERATOR.value]) - ).all() - assert len(staff) == 2 - - -@pytest.mark.asyncio -async def test_logical_and_operator(db_url): - """Test & (AND) operator in queries""" - await connect(db_url, auto_migrate=True) - async with engines.session(): - await DocUser.create( - username="alice", - email="alice@example.com", - is_active=True, - role=UserRole.ADMIN, - ) - await DocUser.create( - username="bob", email="bob@example.com", is_active=True, role=UserRole.USER - ) - await DocUser.create( - username="charlie", - email="charlie@example.com", - is_active=False, - role=UserRole.ADMIN, - ) - - active_admins = await DocUser.where( - lambda t: (t.is_active == True) & (t.role == UserRole.ADMIN.value) # noqa: E712 - ).all() - assert len(active_admins) == 1 - assert active_admins[0].username == "alice" - - @pytest.mark.asyncio async def test_logical_or_operator(db_url): - """Test | (OR) operator in queries""" + """An ``|`` predicate matches rows that meet either side.""" await connect(db_url, auto_migrate=True) async with engines.session(): await DocUser.create( @@ -466,356 +56,17 @@ async def test_logical_or_operator(db_url): ) staff = await DocUser.where( - lambda t: ( - (t.role == UserRole.ADMIN.value) | (t.role == UserRole.MODERATOR.value) + lambda doc_user: ( + (doc_user.role == UserRole.ADMIN.value) + | (doc_user.role == UserRole.MODERATOR.value) ) ).all() - assert len(staff) == 2 - - -@pytest.mark.asyncio -async def test_order_by(db_url): - """Test .order_by() as documented""" - await connect(db_url, auto_migrate=True) - async with engines.session(): - await DocUser.create(username="charlie", email="charlie@example.com") - await DocUser.create(username="alice", email="alice@example.com") - await DocUser.create(username="bob", email="bob@example.com") - - # Ascending - users_asc = await DocUser.select().order_by("username", "asc").all() - assert users_asc[0].username == "alice" - assert users_asc[1].username == "bob" - assert users_asc[2].username == "charlie" - - # Descending - users_desc = await DocUser.select().order_by("username", "desc").all() - assert users_desc[0].username == "charlie" - - -@pytest.mark.asyncio -async def test_limit_and_offset(db_url): - """Test .limit() and .offset() as documented""" - await connect(db_url, auto_migrate=True) - async with engines.session(): - for i in range(10): - await DocUser.create(username=f"user_{i}", email=f"user{i}@example.com") - - # Limit - first_5 = await DocUser.select().order_by("id").limit(5).all() - assert len(first_5) == 5 - - # Offset - skip_5 = await DocUser.select().order_by("id").offset(5).limit(5).all() - assert len(skip_5) == 5 - assert skip_5[0].id != first_5[0].id - - -@pytest.mark.asyncio -async def test_query_first(db_url): - """Test .first() as documented""" - await connect(db_url, auto_migrate=True) - async with engines.session(): - await DocUser.create(username="alice", email="alice@example.com") - - user = await DocUser.where(lambda t: t.username == "alice").first() - assert user is not None - assert user.username == "alice" - - none_user = await DocUser.where(lambda t: t.username == "nonexistent").first() - assert none_user is None - - -@pytest.mark.asyncio -async def test_query_count(db_url): - """Test .count() as documented""" - await connect(db_url, auto_migrate=True) - async with engines.session(): - await DocUser.create( - username="alice", email="alice@example.com", is_active=True - ) - await DocUser.create(username="bob", email="bob@example.com", is_active=True) - await DocUser.create( - username="charlie", email="charlie@example.com", is_active=False - ) - - active_count = await DocUser.where(lambda t: t.is_active == True).count() # noqa: E712 - assert active_count == 2 - - -@pytest.mark.asyncio -async def test_query_exists(db_url): - """Test .exists() as documented""" - await connect(db_url, auto_migrate=True) - async with engines.session(): - await DocUser.create( - username="alice", email="alice@example.com", role=UserRole.ADMIN - ) - - has_admin = await DocUser.where( - lambda t: t.role == UserRole.ADMIN.value - ).exists() - assert has_admin is True - - has_moderator = await DocUser.where( - lambda t: t.role == UserRole.MODERATOR.value - ).exists() - assert has_moderator is False - - -@pytest.mark.asyncio -async def test_query_update(db_url): - """Test .update() as documented""" - await connect(db_url, auto_migrate=True) - async with engines.session(): - await DocUser.create( - username="alice", email="alice@example.com", is_active=True - ) - await DocUser.create(username="bob", email="bob@example.com", is_active=True) - - count = await DocUser.where(lambda t: t.is_active == True).update( # noqa: E712 - is_active=False - ) - assert count == 2 - - active_users = await DocUser.where(lambda t: t.is_active == True).all() # noqa: E712 - assert len(active_users) == 0 - - -@pytest.mark.asyncio -async def test_query_delete(db_url): - """Test .delete() on query as documented""" - await connect(db_url, auto_migrate=True) - async with engines.session(): - await DocUser.create( - username="alice", email="alice@example.com", is_active=True - ) - await DocUser.create(username="bob", email="bob@example.com", is_active=False) - - count = await DocUser.where(lambda t: t.is_active == False).delete() # noqa: E712 - assert count == 1 - - remaining = await DocUser.all() - assert len(remaining) == 1 - assert remaining[0].username == "alice" - - -# ============================================================================ -# RELATIONSHIPS TESTS (docs/guide/relationships.md) -# ============================================================================ - - -@pytest.mark.asyncio -async def test_foreign_key_creation(db_url): - """Test creating records with ForeignKey relationships""" - await connect(db_url, auto_migrate=True) - async with engines.session(): - author = await DocUser.create(username="alice", email="alice@example.com") - - # Pass model instance - post = await DocPost.create( - title="My Post", content="Content here", author=author - ) - - assert post.author_id == author.id - - -@pytest.mark.asyncio -async def test_foreign_key_forward_relation(db_url): - """Test accessing forward relation (ForeignKey)""" - await connect(db_url, auto_migrate=True) - async with engines.session(): - author = await DocUser.create(username="alice", email="alice@example.com") - post = await DocPost.create(title="Test", content="Content", author=author) - - # Access forward relation - post_author = await post.author - assert post_author.id == author.id - assert post_author.username == "alice" - - -@pytest.mark.asyncio -async def test_foreign_key_reverse_relation(db_url): - """Test accessing reverse relation (BackRef)""" - await connect(db_url, auto_migrate=True) - async with engines.session(): - author = await DocUser.create(username="alice", email="alice@example.com") - - await DocPost.create(title="DocPost 1", content="Content 1", author=author) - await DocPost.create(title="DocPost 2", content="Content 2", author=author) - - # Access reverse relation - author_posts = await author.posts.all() - assert len(author_posts) == 2 - - -@pytest.mark.asyncio -async def test_reverse_relation_filtering(db_url): - """Test filtering on reverse relations""" - await connect(db_url, auto_migrate=True) - async with engines.session(): - author = await DocUser.create(username="alice", email="alice@example.com") - - await DocPost.create( - title="Published", content="Content", author=author, published=True - ) - await DocPost.create( - title="Draft", content="Content", author=author, published=False - ) - - published = await author.posts.where(lambda t: t.published == True).all() # noqa: E712 - assert len(published) == 1 - assert published[0].title == "Published" - - -@pytest.mark.asyncio -async def test_shadow_field_access(db_url): - """Test accessing shadow fields (author_id) as documented""" - await connect(db_url, auto_migrate=True) - async with engines.session(): - author = await DocUser.create(username="alice", email="alice@example.com") - post = await DocPost.create(title="Test", content="Content", author=author) - - # Access shadow field - assert post.author_id == author.id - - # Query by shadow field - posts = await DocPost.where(lambda t: t.author_id == author.id).all() - assert len(posts) == 1 - - -@pytest.mark.skip( - reason="Many-to-many join tables not automatically created - see coming-soon.md" -) -@pytest.mark.asyncio -async def test_many_to_many_add(db_url): - """Test .add() for many-to-many relationships""" - await connect(db_url, auto_migrate=True) - post = await DocPost.create( - title="Test Post", - content="Content", - author=await DocUser.create(username="alice", email="alice@example.com"), - ) - - tag1 = await Tag.create(name="python") - tag2 = await Tag.create(name="rust") - - await post.tags.add(tag1, tag2) - - post_tags = await post.tags.all() - assert len(post_tags) == 2 - - -@pytest.mark.skip( - reason="Many-to-many join tables not automatically created - see coming-soon.md" -) -@pytest.mark.asyncio -async def test_many_to_many_remove(db_url): - """Test .remove() for many-to-many relationships""" - await connect(db_url, auto_migrate=True) - post = await DocPost.create( - title="Test Post", - content="Content", - author=await DocUser.create(username="alice", email="alice@example.com"), - ) - - tag1 = await Tag.create(name="python") - tag2 = await Tag.create(name="rust") - - await post.tags.add(tag1, tag2) - await post.tags.remove(tag1) - - post_tags = await post.tags.all() - assert len(post_tags) == 1 - assert post_tags[0].name == "rust" - - -@pytest.mark.skip( - reason="Many-to-many join tables not automatically created - see coming-soon.md" -) -@pytest.mark.asyncio -async def test_many_to_many_clear(db_url): - """Test .clear() for many-to-many relationships""" - await connect(db_url, auto_migrate=True) - post = await DocPost.create( - title="Test Post", - content="Content", - author=await DocUser.create(username="alice", email="alice@example.com"), - ) - - tag1 = await Tag.create(name="python") - tag2 = await Tag.create(name="rust") - - await post.tags.add(tag1, tag2) - await post.tags.clear() - - post_tags = await post.tags.all() - assert len(post_tags) == 0 - - -@pytest.mark.skip( - reason="Many-to-many join tables not automatically created - see coming-soon.md" -) -@pytest.mark.asyncio -async def test_many_to_many_reverse(db_url): - """Test many-to-many from both sides""" - await connect(db_url, auto_migrate=True) - post = await DocPost.create( - title="Test Post", - content="Content", - author=await DocUser.create(username="alice", email="alice@example.com"), - ) - tag = await Tag.create(name="python") - - await post.tags.add(tag) - - # Access from reverse side - tag_posts = await tag.posts.all() - assert len(tag_posts) == 1 - assert tag_posts[0].id == post.id - - -# ============================================================================ -# TRANSACTIONS TESTS (docs/guide/transactions.md) -# ============================================================================ - - -@pytest.mark.asyncio -async def test_transaction_commit(db_url): - """Test transaction commits on success""" - await connect(db_url, auto_migrate=True) - async with engines.session(): - async with transaction(): - user = await DocUser.create(username="alice", email="alice@example.com") - await DocPost.create(title="Test", content="Content", author=user) - - # Verify data persisted - users = await DocUser.all() - posts = await DocPost.all() - assert len(users) == 1 - assert len(posts) == 1 - - -@pytest.mark.asyncio -async def test_transaction_rollback(db_url): - """Test transaction rolls back on exception""" - await connect(db_url, auto_migrate=True) - async with engines.session(): - try: - async with transaction(): - await DocUser.create(username="alice", email="alice@example.com") - raise ValueError("Test error") - except ValueError: - pass - - # Verify data was rolled back - users = await DocUser.all() - assert len(users) == 0 + assert sorted(user.username for user in staff) == ["alice", "bob"] @pytest.mark.asyncio -async def test_transaction_isolation(db_url): - """Test transaction isolation between concurrent tasks""" +async def test_concurrent_transactions_both_commit(db_url): + """Two tasks in one session each commit their own transaction.""" await connect(db_url, auto_migrate=True) async with engines.session(): @@ -831,78 +82,7 @@ async def task_b(): await asyncio.gather(task_a(), task_b()) users = await DocUser.all() - assert len(users) == 2 - - -# ============================================================================ -# TUTORIAL EXAMPLES TESTS (docs/getting-started/tutorial.md) -# ============================================================================ - - -@pytest.mark.asyncio -async def test_tutorial_blog_example(db_url): - """Test the complete tutorial blog example""" - await connect(db_url, auto_migrate=True) - async with engines.session(): - # Create users - alice = await DocUser.create(username="alice", email="alice@example.com") - bob = await DocUser.create(username="bob", email="bob@example.com") - - # Create posts - post1 = await DocPost.create( - title="Why Ferro is Fast", - content="Ferro uses a Rust engine...", - published=True, - author=alice, - ) - - await DocPost.create( - title="Getting Started with Async Python", - content="Async programming can be tricky...", - published=True, - author=alice, - ) - - draft = await DocPost.create( - title="Draft Post", - content="This is not published yet", - published=False, - author=bob, - ) - - # Create comments - await Comment.create(text="Great article!", author=bob, post=post1) - - await Comment.create(text="Thanks for sharing", author=alice, post=post1) - - # Query: Find all published posts - published = await DocPost.where(lambda t: t.published == True).all() # noqa: E712 - assert len(published) == 2 - - # Query: Find posts by author - alice_posts = await DocPost.where(lambda t: t.author_id == alice.id).all() - assert len(alice_posts) == 2 - - # Query: Get post with pattern matching - post = await DocPost.where(lambda t: t.title.like("%Fast%")).first() - assert post is not None - assert post.title == "Why Ferro is Fast" - - # Query: Access forward relation - post_author = await post.author - assert post_author.username == "alice" - - # Query: Access reverse relation - post_comments = await post.comments.all() - assert len(post_comments) == 2 - - # Update: Publish draft - draft.published = True - await draft.save() - - published_after = await DocPost.where(lambda t: t.published == True).all() # noqa: E712 - assert len(published_after) == 3 - - -if __name__ == "__main__": - pytest.main([__file__, "-v"]) + assert sorted(user.username for user in users) == [ + "task_a_user", + "task_b_user", + ] diff --git a/tests/test_migrate_rerecord.py b/tests/test_migrate_rerecord.py index ad0502a3..79af1585 100644 --- a/tests/test_migrate_rerecord.py +++ b/tests/test_migrate_rerecord.py @@ -549,7 +549,7 @@ def test_status_shows_the_edited_chunked_refusal_with_its_doors( def test_the_cursor_names_its_order_keys(): - from ferro.migrations.chunked import encode_cursor + from ferro.migrations._chunk_loop import encode_cursor assert json.loads(encode_cursor((7,), 7, order_by=["author.id"])) == { "keys": [7], diff --git a/tests/test_routing_errors.py b/tests/test_routing_errors.py index fff3f73c..e68d7914 100644 --- a/tests/test_routing_errors.py +++ b/tests/test_routing_errors.py @@ -4,6 +4,8 @@ using=. The silent using-bypasses-session path is a ValueError. """ +from pathlib import Path + import pytest from ferro import Field, Model, connect, engines, execute @@ -20,6 +22,39 @@ class RtA(Model): await RtA.all() +NO_ROUTE_MESSAGE = ( + "No database route for this operation. Open a session " + "(`async with ferro.engines.session(...)`) or pass `using=`/`session=` " + "explicitly. Implicit default-connection routing was removed in v0.14 " + "(deprecated since v0.12); see the upgrade guide." +) + + +@pytest.mark.asyncio +async def test_no_route_message_is_the_current_wording(db_url): + # Pins today's wording (reworded since 0.25.0, which said "removed in + # v0.13 ... see the sessions migration guide"); the upgrade guide's 0.26 + # section names the change for callers that match on the text. + await connect(db_url, auto_migrate=True) + with pytest.raises(RuntimeError) as raised: + await execute("SELECT 1") + assert str(raised.value) == NO_ROUTE_MESSAGE + + +def test_the_upgrade_guide_names_the_reworded_no_route_message(): + guide = ( + Path(__file__).resolve().parents[1] / "docs/pages/upgrading.md" + ).read_text() + section = guide.split("## Upgrading to 0.26", 1)[1].split("\n## ", 1)[0] + assert "asks nothing" not in section + new_tail = NO_ROUTE_MESSAGE.split("routing was ", 1)[1] + assert new_tail in section, new_tail + assert ( + "removed in v0.13 (deprecated since v0.12); see the sessions migration guide." + in section + ) + + @pytest.mark.asyncio async def test_raw_with_no_route_raises(db_url): await connect(db_url, auto_migrate=True) diff --git a/tests/test_save_column_order.py b/tests/test_save_column_order.py new file mode 100644 index 00000000..0cd41a06 --- /dev/null +++ b/tests/test_save_column_order.py @@ -0,0 +1,259 @@ +"""A write lists its columns in the model's field order (#776). + +```python +class Customer(Model): + id: int | None = Field(default=None, primary_key=True) + name: str = "" + updated_at: datetime | None = None + +await customer.save(only={"updated_at", "name"}) +# UPDATE "customer" SET "name" = ?, "updated_at" = ? WHERE "id" = ? +``` + +``only=`` and ``exclude=`` take a set, whose iteration order changes with +the process's hash seed, and a fetched instance holds its values in the +order the ``SELECT`` listed them. Neither may reach the statement: every +write lists its columns in the order the model declares its fields, so the +statement log and ``.sql()`` show one statement for one call. A bytes or +secret column, bound raw rather than through the JSON dump, keeps its +declared place too. +""" + +import logging +import re +from pathlib import Path +from typing import Annotated + +import pytest +import pytest_asyncio +from pydantic import SecretStr + +import ferro +from ferro import Model, engines +from ferro.base import FerroField + +pytestmark = pytest.mark.backend_matrix + +# Declared out of alphabetical order, which is the order a fetch hydrates in. +FIELDS = ["h", "b", "blob", "f", "a", "token", "d", "c", "g", "e"] + + +class ColOrder(Model): + id: Annotated[int | None, FerroField(primary_key=True)] = None + h: str = "h" + b: str = "b" + blob: bytes = b"\x00" + f: int = 0 + a: str = "a" + token: SecretStr = SecretStr("s") + d: int = 0 + c: str = "c" + g: str = "g" + e: str = "e" + + +class PlainOrder(Model): + """No bytes or secret column: every value goes through the JSON dump.""" + + id: Annotated[int | None, FerroField(primary_key=True)] = None + zeta: str = "z" + mid: int = 0 + alpha: str = "a" + + +SET_COLUMN = re.compile(r'"(\w+)" = ') + + +def _set_columns(sql: str) -> list[str]: + """The columns of an ``UPDATE``'s ``SET`` clause, in the order it lists them.""" + head, _, _ = sql.partition(" WHERE ") + _, _, assignments = head.partition(" SET ") + return SET_COLUMN.findall(assignments) + + +def _insert_columns(sql: str) -> list[str]: + columns = sql.partition("(")[2].partition(")")[0] + return [c for c in re.findall(r'"(\w+)"', columns) if c != "id"] + + +def _sent(records: list[logging.LogRecord], prefix: str) -> list[str]: + return [ + r.sql + for r in records + if r.operation not in ("migrate", "catalog") and r.sql.startswith(prefix) + ] + + +def _in_field_order(columns: set[str]) -> list[str]: + return [name for name in FIELDS if name in columns] + + +@pytest_asyncio.fixture +async def saved_row(db_url): + await ferro.connect(db_url, auto_migrate=True) + async with engines.session(): + await ColOrder(id=1).save() + await PlainOrder(id=1).save() + yield + ferro.reset_engine() + + +ONLY = {"h", "b", "token", "f", "a", "blob", "d"} +EXCLUDE = {"c", "g"} + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("saved_row") +async def test_save_only_lists_columns_in_field_order(statement_records): + async with engines.session(): + row = await ColOrder.get(1) + await row.save(only=ONLY) + + (update,) = _sent(statement_records, "UPDATE") + assert _set_columns(update) == _in_field_order(ONLY) + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("saved_row") +async def test_save_only_two_columns_in_field_order(statement_records): + async with engines.session(): + row = await ColOrder.get(1) + await row.save(only={"a", "b"}) + + (update,) = _sent(statement_records, "UPDATE") + assert _set_columns(update) == ["b", "a"] + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("saved_row") +async def test_save_exclude_lists_columns_in_field_order(statement_records): + async with engines.session(): + row = await ColOrder.get(1) + await row.save(exclude=EXCLUDE) + + (update,) = _sent(statement_records, "UPDATE") + assert _set_columns(update) == [n for n in FIELDS if n not in EXCLUDE] + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("saved_row") +async def test_a_full_save_lists_columns_in_field_order(statement_records): + async with engines.session(): + row = await ColOrder.get(1) + await row.save() + await ColOrder(id=2).save() + + (update,) = _sent(statement_records, "UPDATE") + assert _set_columns(update) == FIELDS + (insert,) = _sent(statement_records, "INSERT") + assert _insert_columns(insert) == FIELDS + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("saved_row") +async def test_a_fetched_instance_without_raw_columns_writes_in_field_order( + statement_records, +): + async with engines.session(): + row = await PlainOrder.get(1) + await row.save() + await row.save(only={"alpha", "zeta"}) + await row.save(exclude={"mid"}) + + updates = _sent(statement_records, "UPDATE") + assert [_set_columns(update) for update in updates] == [ + ["zeta", "mid", "alpha"], + ["zeta", "alpha"], + ["zeta", "alpha"], + ] + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("saved_row") +async def test_bulk_create_lists_columns_in_field_order(statement_records): + async with engines.session(): + await ColOrder.bulk_create([ColOrder(id=3), ColOrder(id=4)]) + + (insert,) = _sent(statement_records, "INSERT") + assert _insert_columns(insert) == FIELDS + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("saved_row") +async def test_query_update_lists_columns_in_keyword_order(statement_records): + async with engines.session(): + await ColOrder.where(lambda col_order: col_order.id == 1).update( + e="x", a="y", d=3 + ) + + (update,) = _sent(statement_records, "UPDATE") + assert _set_columns(update) == ["e", "a", "d"] + + +@pytest.mark.asyncio +@pytest.mark.usefixtures("saved_row") +async def test_query_update_keeps_a_raw_column_where_the_call_puts_it( + statement_records, +): + async with engines.session(): + await ColOrder.where(lambda col_order: col_order.id == 1).update( + e="x", blob=b"y", a="z" + ) + + (update,) = _sent(statement_records, "UPDATE") + assert _set_columns(update) == ["e", "blob", "a"] + + +def _upgrade_bullet() -> str: + guide = ( + Path(__file__).resolve().parents[1] / "docs/pages/upgrading.md" + ).read_text() + section = guide.split("## Upgrading to 0.26", 1)[1].split("\n## ", 1)[0] + [bullet] = [line for line in section.splitlines() if "field order" in line] + return bullet + + +def test_the_upgrade_guide_states_each_write_door_s_column_order(): + """The 0.26 note says what the tests above pin: the save doors and + ``bulk_create`` write in field order, ``update()`` in the call's order.""" + bullet = _upgrade_bullet() + field_order, _, rest = bullet.partition("field order") + for door in ( + "`save()`", + "`save(only=...)`", + "`save(exclude=...)`", + "`bulk_create()`", + ): + assert door in field_order, door + assert "update()" not in field_order + assert ( + "`update()` is unchanged: it lists the columns in the order the call passes them" + in rest + ) + + +def _upgrade_section() -> str: + guide = ( + Path(__file__).resolve().parents[1] / "docs/pages/upgrading.md" + ).read_text() + return guide.split("## Upgrading to 0.26", 1)[1].split("\n## ", 1)[0] + + +def test_the_upgrade_guide_states_the_secret_column_change(): + """0.25 wrote a secret column as the mask, in field order, and made a + ``SecretBytes`` column ``varchar``; the 0.26 note says so and says what + a 0.25 user must do.""" + section = _upgrade_section() + assert "secret column came last" not in _upgrade_bullet() + [bullet] = [line for line in section.splitlines() if "`SecretStr`" in line] + for fact in ( + "`SecretBytes`", + "`**********`", + "cannot be recovered", + "`varchar`", + "`BYTEA`", + "`BLOB`", + "migration", + ): + assert fact in bullet, fact + assert section.split(":", 1)[0].strip().startswith("Five changes") diff --git a/tests/test_step_class.py b/tests/test_step_class.py index 2f856f11..ed40434d 100644 --- a/tests/test_step_class.py +++ b/tests/test_step_class.py @@ -457,17 +457,25 @@ def test_generated_files_carry_no_class_header(project, pkg, db): DEPLOYING = Path(__file__).resolve().parents[1] / "docs/pages/guide/schema/deploying.md" -def test_the_deploying_guide_quotes_the_refusal_expand_only_raises(project, pkg, db): - """The rolling-deploy recipe is ``"expand-only"``, and the refusal it - quotes is the text ferro raises (the guide's migration is ``0008``).""" +def test_expand_only_refuses_a_contract_step_as_the_deploying_guide_shows( + project, pkg, db +): + """The rolling-deploy recipe is ``"expand-only"``, and it refuses a + database ahead by a contract step, naming the step. + + The refusal the guide quotes is regenerated from + ``docs/examples/migration_errors.py`` (``tests/test_docs_output.py``), so + the page half here checks only that the guide shows the recipe and that + its refusal is that regenerated block, never a typed copy. + """ page = DEPLOYING.read_text() recipe = 'await ferro.migrations.require_applied(allow_ahead="expand-only")' - assert f"```python\n{recipe}\n```" in page - quoted = re.search( - r"```text\n(ferro\.migrations\.DatabaseAheadError: [^`]*?0008_drop_bio:01[^`]*)```", + assert re.search(rf"```python[^\n]*\n{re.escape(recipe)}\n```", page) + assert re.search( + r"<!-- prints: migration_errors\.py:ahead-by-contract -->\n```text\n" + r"DatabaseAheadError: ", page, ) - assert quoted is not None configure(project, pkg, db.backend) write_models(project, pkg, BIO) @@ -479,7 +487,4 @@ def test_the_deploying_guide_quotes_the_refusal_expand_only_raises(project, pkg, with pytest.raises(DatabaseAheadError) as raised: require_applied(db, allow_ahead="expand-only") assert raised.value.contract_steps == ["0002_drop_bio:01"] - raised_text = f"ferro.migrations.DatabaseAheadError: {raised.value}\n".replace( - "0002_drop_bio", "0008_drop_bio" - ) - assert quoted.group(1) == raised_text + assert "starting with 0002_drop_bio:01 (contract" in str(raised.value) diff --git a/uv.lock b/uv.lock index 748a4835..d4dd22ca 100644 --- a/uv.lock +++ b/uv.lock @@ -604,6 +604,7 @@ dev = [ { name = "commitizen" }, { name = "cyclopts" }, { name = "greenlet" }, + { name = "griffe-inherited-docstrings" }, { name = "jupyter" }, { name = "maturin" }, { name = "mkdocstrings-python" }, @@ -623,6 +624,7 @@ dev = [ { name = "zensical" }, ] docs = [ + { name = "griffe-inherited-docstrings" }, { name = "mkdocstrings-python" }, { name = "pymdown-extensions" }, { name = "zensical" }, @@ -663,6 +665,7 @@ dev = [ { name = "commitizen", specifier = ">=3.13.0" }, { name = "cyclopts", specifier = ">=5,<6" }, { name = "greenlet", specifier = ">=3.3.1" }, + { name = "griffe-inherited-docstrings", specifier = ">=1.1.4" }, { name = "jupyter", specifier = ">=1.1.1" }, { name = "maturin", specifier = ">=1.11.5" }, { name = "mkdocstrings-python", specifier = ">=2.0.9" }, @@ -682,6 +685,7 @@ dev = [ { name = "zensical", specifier = ">=0.0.71" }, ] docs = [ + { name = "griffe-inherited-docstrings", specifier = ">=1.1.4" }, { name = "mkdocstrings-python", specifier = ">=2.0.9" }, { name = "pymdown-extensions", specifier = ">=12.3" }, { name = "zensical", specifier = ">=0.0.71" }, @@ -770,6 +774,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e1/2b/98c7f93e6db9977aaee07eb1e51ca63bd5f779b900d362791d3252e60558/greenlet-3.3.1-cp314-cp314t-win_amd64.whl", hash = "sha256:301860987846c24cb8964bdec0e31a96ad4a2a801b41b4ef40963c1b44f33451", size = 233181, upload-time = "2026-01-23T15:33:00.29Z" }, ] +[[package]] +name = "griffe-inherited-docstrings" +version = "1.1.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "griffelib" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/32/61/1dd4a7c59618c5e9ee0f61eb9fb934ba11163fa66932c8e2cc99d74f3d7b/griffe_inherited_docstrings-1.1.4.tar.gz", hash = "sha256:ac98890a4505d986254302325959de6c82716968ec499aca009cb91c1d75eecb", size = 32495, upload-time = "2026-10-06T09:07:03.147Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/34/35/364ed24b4cba404a6f02998b381c872f99480a26f0c57687537578d4b5ad/griffe_inherited_docstrings-1.1.4-py3-none-any.whl", hash = "sha256:6c813c567f6f7396d6e1a78575fb7ad0d5b472fc02c1b782f830fcf26ab7254d", size = 10888, upload-time = "2026-10-06T09:07:02.095Z" }, +] + [[package]] name = "griffelib" version = "2.3.2" diff --git a/zensical.toml b/zensical.toml index 188debe8..125c70a0 100644 --- a/zensical.toml +++ b/zensical.toml @@ -12,89 +12,106 @@ watch = ["src", "docs/examples", "README.md", "CHANGELOG.md", "CONTRIBUTING.md"] extra_css = ["stylesheets/extra.css"] nav = [ - { "Home" = "index.md" }, - { "Why Ferro" = "why-ferro.md" }, - { "Getting Started" = [ + { "Get started" = [ + { "Ferro ORM" = "index.md" }, + { "Why Ferro" = "why-ferro.md" }, { "Installation" = "getting-started/installation.md" }, - { "Quickstart Tutorial" = "getting-started/quickstart.md" }, - { "Next Steps" = "getting-started/next-steps.md" }, + { "Quickstart" = "getting-started/quickstart.md" }, ] }, { "Guide" = [ - { "Models & Fields" = "guide/models-and-fields.md" }, + { "Models and fields" = "guide/models-and-fields.md" }, { "Queries" = "guide/queries.md" }, - { "Aggregations" = "guide/aggregations.md" }, + { "Aggregations and grouped queries" = "guide/aggregations.md" }, { "Mutations" = "guide/mutations.md" }, { "Relationships" = "guide/relationships.md" }, { "Transactions" = "guide/transactions.md" }, { "Raw SQL" = "guide/raw-sql.md" }, - { "Connections & Databases" = "guide/connections.md" }, - { "Schema Management" = [ - { "Overview" = "guide/schema/overview.md" }, + { "Connections" = "guide/connections.md" }, + { "Schema management" = [ + { "Overview of schema management" = "guide/schema/overview.md" }, { "Auto-migrate" = "guide/schema/auto-migrate.md" }, { "Migrations" = "guide/schema/migrations.md" }, - { "Data Steps and Backfills" = "guide/schema/data-steps.md" }, - { "Deploying Migrations" = "guide/schema/deploying.md" }, - { "Testing Migrations" = "guide/schema/testing.md" }, - { "Alembic" = "guide/schema/alembic.md" }, + { "Renaming and retiring columns and labels" = "guide/schema/renames-and-retirements.md" }, + { "Data steps and backfills" = "guide/schema/data-steps.md" }, + { "Squashing a feature branch's migrations" = "guide/schema/squash.md" }, + { "Deploying migrations" = "guide/schema/deploying.md" }, + { "Testing migrations" = "guide/schema/testing.md" }, + { "Adopting migrations on an existing database" = "guide/schema/adopting-migrations.md" }, + { "Alembic bridge" = "guide/schema/alembic.md" }, ] }, - { "Using with Postgres RLS" = "guide/row-level-security.md" }, ] }, - { "How-To" = [ - { "Upgrade Guide" = "howto/upgrade-guide.md" }, - { "Testing" = "howto/testing.md" }, - { "Pagination" = "howto/pagination.md" }, - { "Timestamps" = "howto/timestamps.md" }, - { "Soft Deletes" = "howto/soft-deletes.md" }, - { "Multiple Databases" = "howto/multiple-databases.md" }, - { "Adopting Migrations on an Existing Database" = "howto/adopting-migrations.md" }, - { "Migrating from SQLAlchemy" = "howto/migrate-from-sqlalchemy.md" }, + { "How-to" = [ + { "How to test a Ferro app with pytest" = "howto/testing.md" }, + { "How to paginate query results" = "howto/pagination.md" }, + { "How to record created and updated times" = "howto/timestamps.md" }, + { "How to soft-delete rows" = "howto/soft-deletes.md" }, + { "How to use several databases in one app" = "howto/multiple-databases.md" }, + { "Scope rows by tenant with Postgres row-level security" = "guide/row-level-security.md" }, + { "Migrate from SQLAlchemy" = "howto/migrate-from-sqlalchemy.md" }, ] }, { "Concepts" = [ { "Architecture" = "concepts/architecture.md" }, - { "Identity Map" = "concepts/identity-map.md" }, - { "Type Safety" = "concepts/type-safety.md" }, - { "Typed Query Predicates" = "concepts/query-typing.md" }, - { "Database Backends" = "concepts/backends.md" }, + { "Identity map" = "concepts/identity-map.md" }, + { "Type safety" = "concepts/type-safety.md" }, + { "Typed query predicates" = "concepts/query-typing.md" }, + { "Database backends" = "concepts/backends.md" }, + { "Row-level security and session settings" = "concepts/row-level-security.md" }, { "Performance" = "concepts/performance.md" }, ] }, { "Reference" = [ + { "Python API" = [ + { "Model" = "api/model.md" }, + { "Field, FerroField and DbType" = "api/fields.md" }, + { "Query" = "api/queries.md" }, + { "ForeignKey, BackRef, ManyToMany and Relation" = "api/relationships.md" }, + { "transaction and Transaction" = "api/transactions.md" }, + { "execute, fetch_all and fetch_one" = "api/raw-sql.md" }, + { "connect, Session and the registry" = "api/connection.md" }, + { "ferro.migrations" = "api/migrations.md" }, + ] }, { "CLI" = "reference/cli.md" }, { "Configuration" = "reference/configuration.md" }, + { "FerroError and its subclasses" = "api/exceptions.md" }, + { "Migrations reference" = "reference/migrations.md" }, + { "Row-level security" = "reference/row-level-security.md" }, ] }, - { "API Reference" = [ - { "Model" = "api/model.md" }, - { "Fields & Types" = "api/fields.md" }, - { "Queries" = "api/queries.md" }, - { "Relationships" = "api/relationships.md" }, - { "Transactions" = "api/transactions.md" }, - { "Raw SQL" = "api/raw-sql.md" }, - { "Connection & Registry" = "api/connection.md" }, - { "Migrations" = "api/migrations.md" }, - { "Exceptions" = "api/exceptions.md" }, - ] }, - { "Community" = [ + { "Project" = [ { "FAQ" = "faq.md" }, { "Roadmap" = "roadmap.md" }, - { "Contributing" = "contributing.md" }, + { "Upgrading between versions" = "upgrading.md" }, { "Changelog" = "changelog.md" }, + { "Contributing" = "contributing.md" }, ] }, ] [project.theme] features = [ - "navigation.tabs", - "navigation.sections", + "content.action.copy", + "content.action.edit", + "content.action.view", + "content.code.annotate", + "content.code.copy", + "content.code.select", + "content.tabs.link", + "content.tooltips", "navigation.expand", + "navigation.footer", + "navigation.instant", + "navigation.instant.progress", + "navigation.path", + "navigation.sections", + "navigation.tabs", "navigation.top", - "toc.follow", - "search.suggest", + "navigation.tracking", "search.highlight", - "content.code.copy", - "content.code.annotate", - "content.action.edit", - "content.action.view", + "toc.follow", ] +[[project.theme.palette]] +media = "(prefers-color-scheme)" +toggle.icon = "lucide/sun-moon" +toggle.name = "Switch to light mode" + [[project.theme.palette]] media = "(prefers-color-scheme: light)" scheme = "light-rust" @@ -109,21 +126,151 @@ scheme = "slate" primary = "custom" accent = "amber" toggle.icon = "lucide/moon" -toggle.name = "Switch to light mode" +toggle.name = "Switch to system preference" + +[project.extra] +status.new = "Recently added" +status.deprecated = "Deprecated" + +[[project.extra.social]] +icon = "fontawesome/brands/github" +link = "https://github.com/syn54x/ferro-orm" +name = "Ferro on GitHub" + +[[project.extra.social]] +icon = "fontawesome/brands/python" +link = "https://pypi.org/project/ferro-orm/" +name = "Ferro on PyPI" + +[project.plugins.social] +cards_layout_options.background_color = "#B7410E" + +[project.plugins.llmstxt] +markdown_description = "Ferro is an async Python ORM: Pydantic models, a Rust core, SQLite and Postgres." +full_output = "llms-full.txt" + +[project.plugins.llmstxt.sections] +"Get started" = [ + "index.md", + "why-ferro.md", + "getting-started/installation.md", + "getting-started/quickstart.md", +] +"Guide" = ["guide/*.md", "guide/schema/*.md"] +"How-to" = ["howto/*.md"] +"Concepts" = ["concepts/*.md"] +"Reference" = ["reference/*.md", "api/*.md"] +"Project" = ["faq.md", "roadmap.md", "upgrading.md", "changelog.md", "contributing.md"] + +# Moved pages. Each old URL, and each anchor it had, points at the section +# that replaced it. +[project.plugins.redirects.redirect_maps] +"guide/migrations.md" = "guide/schema/overview.md" +"guide/migrations.md#three-ways-to-manage-schema" = "guide/schema/overview.md#the-ladder" +"guide/migrations.md#auto-migration" = "guide/schema/auto-migrate.md" +"guide/migrations.md#creating-tables-with-auto_migratetrue" = "guide/schema/auto-migrate.md#creating-tables-with-auto_migratetrue" +"guide/migrations.md#applying-column-changes-with-migrate_updates" = "guide/schema/auto-migrate.md#applying-column-changes-with-migrate_updates" +"guide/migrations.md#evolving-enums-label-addition" = "guide/schema/auto-migrate.md#evolving-enums-label-addition" +"guide/migrations.md#destructive-drops-with-migrate_destructive" = "guide/schema/auto-migrate.md#destructive-drops-with-migrate_destructive" +"guide/migrations.md#on-demand-migrate" = "guide/schema/auto-migrate.md#on-demand-migrate" +"guide/migrations.md#safety-guidance" = "guide/schema/auto-migrate.md#safety-guidance" +"guide/migrations.md#alembic-for-production" = "guide/schema/alembic.md" +"guide/migrations.md#install" = "guide/schema/alembic.md#install" +"guide/migrations.md#initialize" = "guide/schema/alembic.md#initialize" +"guide/migrations.md#configure-envpy" = "guide/schema/alembic.md#configure-envpy" +"guide/migrations.md#autogenerate" = "guide/schema/alembic.md#one-decider" +"guide/migrations.md#review-apply" = "guide/schema/alembic.md#one-decider" +"guide/migrations.md#choosing-a-workflow" = "guide/schema/overview.md#choosing" +"getting-started/next-steps.md" = "getting-started/quickstart.md#next-steps" +"getting-started/next-steps.md#learn-by-use-case" = "getting-started/quickstart.md#next-steps" +"getting-started/next-steps.md#building-an-api" = "getting-started/quickstart.md#next-steps" +"getting-started/next-steps.md#data-processing" = "getting-started/quickstart.md#next-steps" +"getting-started/next-steps.md#production-deployment" = "getting-started/quickstart.md#next-steps" +"getting-started/next-steps.md#common-patterns" = "getting-started/quickstart.md#next-steps" +"getting-started/next-steps.md#reference-material" = "getting-started/quickstart.md#next-steps" +"getting-started/next-steps.md#get-help" = "getting-started/quickstart.md#next-steps" +"howto/adopting-migrations.md" = "guide/schema/adopting-migrations.md" +"howto/upgrade-guide.md" = "upgrading.md" +# Sections renamed on pages the rewrite kept, linked from outside the docs. +"guide/mutations.md#get-or-create" = "guide/mutations.md#get_or_create-and-update_or_create" +"guide/transactions.md#nesting-behavior" = "guide/transactions.md#nested-transactions" +"guide/connections.md#sessions-recommended" = "guide/connections.md#sessions" +"faq.md#what-is-ferro" = "faq.md#what-ferro-is" +"howto/testing.md#writing-your-first-test" = "howto/testing.md#a-first-test" +"howto/upgrade-guide.md#save-no-longer-upserts" = "upgrading.md#save-and-upserts" +# Sections the migrations guide split (#714) moved to other pages, or renamed. +"guide/schema/migrations.md#renames" = "guide/schema/renames-and-retirements.md#rename-hints" +"guide/schema/migrations.md#retiring-a-column" = "guide/schema/renames-and-retirements.md#retiring-a-column" +"guide/schema/migrations.md#retiring-an-enum-label" = "guide/schema/renames-and-retirements.md#retiring-an-enum-label" +"guide/schema/migrations.md#headers" = "reference/migrations.md#step-file-headers" +"guide/schema/migrations.md#changes-existing-rows-need-a-value-for" = "guide/schema/migrations.md#backfills" +"guide/schema/migrations.md#checking" = "guide/schema/migrations.md#check-status-and-drift" +"guide/schema/migrations.md#going-back" = "guide/schema/migrations.md#reverting-with-down" +"guide/schema/migrations.md#editing-a-migration" = "guide/schema/migrations.md#edited-migrations" +"guide/schema/migrations.md#two-branches-one-number" = "guide/schema/migrations.md#duplicate-migration-numbers-and-replace" +"guide/schema/migrations.md#squashing-an-epics-migrations" = "guide/schema/squash.md" +"guide/schema/migrations.md#a-rename-inside-the-range" = "guide/schema/squash.md#renames-inside-the-range" +"guide/schema/migrations.md#every-database-that-applied-the-originals-inherits-the-squash" = "guide/schema/squash.md#inheritance-on-databases-that-ran-the-originals" +"guide/schema/migrations.md#a-squash-is-pre-release" = "guide/schema/squash.md#squash-timing" +"guide/schema/migrations.md#squash-or---replace" = "guide/schema/squash.md#squash-vs-replace" +# Sections of the old row-level security guide, split three ways (#715). +"guide/row-level-security.md#the-pooler-constraint" = "concepts/row-level-security.md#the-pooler-constraint" +"guide/row-level-security.md#deploy-ordering" = "guide/row-level-security.md#deploy-order" +"guide/row-level-security.md#declaring-row-security" = "guide/row-level-security.md#the-tenant-policy" +"guide/row-level-security.md#the-shorthand" = "reference/row-level-security.md#the-shorthand" +"guide/row-level-security.md#multiple-policies-composing-permissive-and-restrictive" = "reference/row-level-security.md#composing-policies" +"guide/row-level-security.md#the-raw-escape-hatch" = "reference/row-level-security.md#the-sql-form" +"guide/row-level-security.md#session-settings" = "concepts/row-level-security.md#session-settings" +"guide/row-level-security.md#opening-a-session-with-settings" = "guide/row-level-security.md#tenant-scoped-sessions" +"guide/row-level-security.md#deferred-resolution-current_sessionset_config" = "guide/row-level-security.md#a-tenant-known-after-the-session-opens" +"guide/row-level-security.md#nesting-and-inheritance" = "concepts/row-level-security.md#nested-sessions" +"guide/row-level-security.md#scope-stability-mid-operation" = "concepts/row-level-security.md#visibility-of-a-new-value" +"guide/row-level-security.md#the-nullif-contract" = "concepts/row-level-security.md#policy-expressions" +"guide/row-level-security.md#bypassrls" = "concepts/row-level-security.md#force-and-bypassrls" +"guide/row-level-security.md#the-json-claims-recipe" = "guide/row-level-security.md#json-claims" +"guide/row-level-security.md#framework-integration" = "guide/row-level-security.md#web-framework-middleware" +"guide/row-level-security.md#outside-the-request-context" = "guide/row-level-security.md#background-jobs" +"guide/row-level-security.md#connection-delivery-mode" = "concepts/row-level-security.md#connection-delivery" +"guide/row-level-security.md#raw-policies-and-drift" = "concepts/row-level-security.md#drift-and-ownership" +"guide/row-level-security.md#migrations-and-alembic" = "guide/row-level-security.md#deploy-order" [project.plugins.mkdocstrings.handlers.python] paths = ["src"] -inventories = ["https://docs.python.org/3/objects.inv"] +inventories = [ + "https://docs.python.org/3/objects.inv", + "https://docs.pydantic.dev/latest/objects.inv", +] [project.plugins.mkdocstrings.handlers.python.options] docstring_style = "google" -show_source = true +docstring_options = { ignore_init_summary = true } +merge_init_into_class = true +show_source = false show_root_heading = true show_root_full_path = false -show_category_heading = true -show_if_no_docstring = true +show_category_heading = false +# A member with no docstring is not rendered; every public object has one. +show_if_no_docstring = false members_order = "source" +# Not "public": griffe counts every dunder as public, so "public" would render +# __repr__, __post_init__ and the __ferro_*__ class attributes. A protocol a +# reader uses (async with, len(), comparisons) belongs in the class docstring. +# `model_config` is Pydantic's configuration, inherited with Pydantic's own +# docstring; it is no part of a Ferro model's API. +filters = ["!^_", "!^model_config$"] +separate_signature = true +show_signature_annotations = true +signature_crossrefs = true +show_symbol_type_heading = true +show_symbol_type_toc = true +scoped_crossrefs = true +relative_crossrefs = true +summary = true +backlinks = "tree" +extensions = ["griffe_inherited_docstrings"] +# --- Python-Markdown core --- +[project.markdown_extensions.abbr] [project.markdown_extensions.attr_list] [project.markdown_extensions.md_in_html] [project.markdown_extensions.admonition] @@ -131,13 +278,23 @@ members_order = "source" [project.markdown_extensions.toc] permalink = true +permalink_title = "Anchor link to this section" +# --- pymdown-extensions --- [project.markdown_extensions.pymdownx.details] [project.markdown_extensions.pymdownx.inlinehilite] [project.markdown_extensions.pymdownx.keys] [project.markdown_extensions.pymdownx.mark] [project.markdown_extensions.pymdownx.smartsymbols] +[project.markdown_extensions.pymdownx.magiclink] +provider = "github" +user = "syn54x" +repo = "ferro-orm" +repo_url_shorthand = true +repo_url_shortener = true +normalize_issue_symbols = true + [project.markdown_extensions.pymdownx.superfences] custom_fences = [ { name = "mermaid", class = "mermaid", format = "pymdownx.superfences.fence_code_format" }, @@ -146,6 +303,11 @@ custom_fences = [ [project.markdown_extensions.pymdownx.snippets] base_path = ["."] check_paths = true +# Example sections sit inside `async def main():`; strip that indentation so +# an embedded block starts at column 0 and runs when pasted. +dedent_subsections = true +# The glossary lives outside docs_dir so it is not built as a page. +auto_append = ["docs/includes/abbreviations.md"] [project.markdown_extensions.pymdownx.highlight] anchor_linenums = true @@ -154,6 +316,8 @@ pygments_lang_class = true [project.markdown_extensions.pymdownx.tabbed] alternate_style = true +combine_header_slug = true +slugify = { object = "pymdownx.slugs.slugify", kwds = { case = "lower" } } [project.markdown_extensions.pymdownx.emoji] emoji_index = "zensical.extensions.emoji.twemoji"