diff --git a/.github/workflows/python-test.yml b/.github/workflows/python-test.yml index d402d5f..e10c7d3 100644 --- a/.github/workflows/python-test.yml +++ b/.github/workflows/python-test.yml @@ -14,16 +14,18 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.12"] + python-version: ["3.14"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: submodules: "recursive" - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} + - name: Ensure podman is installed + run: command -v podman || (sudo apt-get update && sudo apt-get install -y podman) - name: Install uv run: curl -LsSf https://astral.sh/uv/install.sh | sh - name: Install project dependencies diff --git a/.gitignore b/.gitignore index ecfbe1c..47b8173 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,6 @@ docs/superpowers/ # Local migration checklist (not shipped with the package) /MIGRATION_NOTES.md + +# Local follow-up tracking (not shipped with the package) +/FOLLOWUP.md diff --git a/docs/database-layout.md b/docs/database-layout.md new file mode 100644 index 0000000..68d2887 --- /dev/null +++ b/docs/database-layout.md @@ -0,0 +1,145 @@ +# `database/` folder layout + +The Postgres schema is versioned as plain `.sql` files under a `database/` +folder in the repo — that folder *is* the source of truth for the schema. +`pgdb testdb` (see [`pgdevkit/testdb/schema.py`](../pgdevkit/testdb/schema.py)) +applies it to a local test database; a human applies the same files to +production. + +This is the single source for the convention — don't duplicate this table or +these rules elsewhere; link here instead. + +--- + +## Layer directories + +Top-level directories group tables/objects by conceptual layer — one +directory per Postgres schema, or per logical grouping within one schema. +Names and count are entirely per-project; a generic example: + +``` +database/ +├── schema.sql # CREATE SCHEMA statements +├── 0_public/ # shared types/functions usable from anywhere +├── 1_reference_data/ # dimension / reference tables +├── 2_transactional/ # fact / transactional tables +├── 3_app/ # user-editable, app-owned tables +├── 4_reporting/ # aggregated / statistics tables +├── permissions.sql # grants +``` + +The leading number is a **display/sort aid only** — it groups related +folders together in a file listing so a human can scan them top-to-bottom in +a sensible order. It does not control apply order: `pgdb testdb` applies +files by object-type priority (below) and resolves cross-file dependencies +itself, regardless of which numbered folder a file sits in. Feel free to +renumber layers for readability without worrying about breaking anything. + +--- + +## Object-type subfolders, in apply order + +Within a layer directory, group files by object type. This is what actually +controls apply order, across all layer directories — cross-file dependencies +between objects of the same type (e.g. one view selecting from another) are +resolved automatically; this table only fixes the order *between* types. +This table must match `_TYPE_ORDER` / `_SCHEMA_QUALIFIED_TYPES` in +[`pgdevkit/testdb/schema.py`](../pgdevkit/testdb/schema.py) exactly — update +both together. + +| Priority | Directory | Object type | +|---|---|---| +| 1 | `schema` | `CREATE SCHEMA` | +| 2 | `types` | Custom types / enums | +| 3 | `tables` | Tables | +| 4 | `scalar_functions` | Scalar functions | +| 5 | `functions` | Functions | +| 6 | `views` | Views | +| 7 | `table_functions` | Table functions | +| 8 | `procedures` | Procedures | +| 100 | `permissions` | Grants | +| 101 | `indexes` | Indexes | + +One object per file: `tables/user.sql`, `views/all_edits.sql`, +`types/measurement_unit.sql`. + +--- + +## File-naming conventions + +| Suffix | Meaning | +|---|---| +| `.sql` | The object's live definition (`CREATE TABLE`, `CREATE OR REPLACE VIEW`, ...) | +| `.test_data.json` | Seed rows for a table — a JSON array of row objects, loaded after the table is created | +| `.init.sql` | One-time setup for an object (e.g. a backfill), run once, kept separate from the reusable definition | +| `.prod.sql` / `.prod` anywhere in the name | Production-only (real permission grants, real user accounts) — skipped by `pgdb testdb` | +| `all.sql` | Generated concatenation of the whole tree — not hand-edited, not committed | + +--- + +## Migrations + +One-off, non-idempotent changes (rename/drop column, backfill, data fix) go +in a `migrations/` (or `_migration_scripts/`) folder — one file per change, +named by date: + +``` +database/migrations/2026-07-10_customer_geocode.sql +``` + +Rules: + +- Skipped by `pgdb testdb` — it only applies the layer directories above. +- Update the corresponding table/view `.sql` file in the same change so its + definition already reflects the new shape — the migration and the + source-of-truth file must never drift apart. +- Applied to production manually, once, by a human, after being verified + locally. +- Never edited after being applied — a further change gets a new dated file. + +--- + +## `COMMENT ON` — document schema in the object's own file + +Add a `COMMENT ON` for every table, and for any column whose purpose isn't +obvious from its name and type (flags, status codes, denormalized fields, +units). Put it directly in the table's `.sql` file, right after the +`CREATE TABLE` — not in a migration, wiki, or README. It lives with the +definition it describes and survives `\d+` / `pg_catalog` inspection. + +```sql +-- database/1_reference_data/tables/user.sql +create table dim.user ( + id bigint generated always as identity primary key, + email text not null, + is_active boolean not null default true +); + +comment on table dim.user is 'End-user accounts; one row per registered person.'; +comment on column dim.user.is_active is 'False once a user is soft-deleted; keep for audit trail.'; +``` + +--- + +## Backfilling untracked objects + +If a table, scalar function, or table function was created directly on the +database and never got a `.sql` file, use `pgdb fetch-missing` to find it and +generate one — see `pgdb fetch-missing --help`. It connects to Postgres, +diffs the live schema against what's already tracked under `database/`, and +for each object you select, reverse-engineers its DDL into the matching +layer folder's `tables/`, `views/`, `scalar_functions/`, or +`table_functions/` subfolder. The layer folder is matched by schema name +against the existing top-level directories under `database/`, ignoring their +leading sort number. + +--- + +## Quick checklist + +- [ ] New table/view/function/type gets its own `.sql` file under the right layer + object-type folder +- [ ] Object-type folder (`tables`, `views`, ...) matches the apply-order table above — that's what governs ordering, not the layer's leading number +- [ ] One-off changes go in `migrations/`, dated, never edited after applying +- [ ] The live `.sql` file is updated in the same change as any migration touching that object +- [ ] `.prod` files are production-only and skipped by `pgdb testdb` +- [ ] Every table (and non-obvious column) has a `COMMENT ON`, placed in the object's own `.sql` file diff --git a/pgdevkit/cli.py b/pgdevkit/cli.py index 2879978..d5baa8e 100644 --- a/pgdevkit/cli.py +++ b/pgdevkit/cli.py @@ -3,6 +3,7 @@ import os from pathlib import Path +import psycopg import typer from rich.console import Console from rich.table import Table @@ -11,6 +12,7 @@ from . import testdb from .connection import build_conninfo from .diff import DiffKind, compute_diff +from .fetch_missing import SUBFOLDER, find_missing_objects, layer_folder_for, reconstruct_ddl from .introspect import introspect_db from .parser import parse_directory @@ -67,6 +69,67 @@ def compare( raise typer.Exit(1) +@app.command("fetch-missing") +def fetch_missing( + scripts_dir: Path = typer.Argument(..., help="The database/ folder to compare against and write into"), + url: str = typer.Option(..., "--url", help="PostgreSQL DSN (postgresql://user:pass@host:port/db)"), + entra_user: str | None = typer.Option(None, "--entra-user", help="Azure Entra user (triggers token auth)"), + write: bool = typer.Option(False, "--write", help="Write the reconstructed .sql files (default: dry run)"), + only: list[str] = typer.Option([], "--only", help="Only fetch schema.name (repeatable); default is everything"), +) -> None: + """Find tables/views/functions that exist in the database but aren't + tracked under scripts_dir, and reverse-engineer their DDL into new files.""" + if not scripts_dir.is_dir(): + err_console.print(f"[red]Error:[/red] {scripts_dir} is not a directory") + raise typer.Exit(2) + + conninfo = build_conninfo(url, entra_user) + + with console.status("Comparing database/ against the live schema..."): + missing = find_missing_objects(scripts_dir, conninfo) + + if only: + wanted = set(only) + missing = [m for m in missing if m.qualified_name in wanted] + + if not missing: + console.print("[green]No missing objects.[/green]") + return + + table = Table(box=box.SIMPLE, show_header=True, header_style="bold") + table.add_column("Type", style="magenta") + table.add_column("Object") + table.add_column("Destination", style="dim") + for m in missing: + dest = layer_folder_for(scripts_dir, m.schema) / SUBFOLDER[m.object_type] / f"{m.name}.sql" + table.add_row(m.object_type, m.qualified_name, str(dest)) + console.print(table) + + if not write: + console.print("\n[yellow]Dry run[/yellow] — pass --write to create these files.") + return + + written = 0 + with psycopg.connect(conninfo) as conn: + for m in missing: + dest_dir = layer_folder_for(scripts_dir, m.schema) / SUBFOLDER[m.object_type] + dest = dest_dir / f"{m.name}.sql" + if dest.exists(): + console.print(f" [yellow]SKIP[/yellow] {dest} (already exists)") + continue + try: + ddl = reconstruct_ddl(conn, m) + except Exception as e: # noqa: BLE001 + err_console.print(f"[red]Error[/red] reconstructing {m.qualified_name}: {e}") + continue + dest_dir.mkdir(parents=True, exist_ok=True) + dest.write_text(ddl, encoding="utf-8") + console.print(f" [green]WROTE[/green] {dest}") + written += 1 + + console.print(f"\nWrote {written} file(s).") + + @testdb_app.command("up") def testdb_up() -> None: """Ensure the container is running, the workspace DB exists, and schema is applied.""" @@ -104,7 +167,7 @@ def testdb_run_sql( console.print(f"OK — {len(rows)} row(s)") return if not rows: - console.print("(0 rows)") + console.print("(0 row(s))") return table = Table(box=box.SIMPLE, show_header=True, header_style="bold") for col in rows[0]: diff --git a/pgdevkit/db/__init__.py b/pgdevkit/db/__init__.py new file mode 100644 index 0000000..d66a75d --- /dev/null +++ b/pgdevkit/db/__init__.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +from .connection import PgPool +from .crud import ( + pg_delete, + pg_delete_dict, + pg_insert, + pg_insert_many, + pg_retrieve, + pg_retrieve_many, + pg_update, + pg_update_dict, + pg_upsert, + pg_upsert_dict, + pg_upsert_many, + pg_upsert_many_dict, +) +from .loader import SqlLoader +from .model import PostgresTableModel + +__all__ = [ + "PgPool", + "PostgresTableModel", + "SqlLoader", + "pg_delete", + "pg_delete_dict", + "pg_insert", + "pg_insert_many", + "pg_retrieve", + "pg_retrieve_many", + "pg_update", + "pg_update_dict", + "pg_upsert", + "pg_upsert_dict", + "pg_upsert_many", + "pg_upsert_many_dict", +] diff --git a/pgdevkit/db/connection.py b/pgdevkit/db/connection.py new file mode 100644 index 0000000..c0d63a6 --- /dev/null +++ b/pgdevkit/db/connection.py @@ -0,0 +1,46 @@ +from __future__ import annotations + +import os + +from psycopg_pool import AsyncConnectionPool + + +class PgPool: + """A connection pool keyed off `{env_prefix}HOST/PORT/DB/USER/PASSWORD` + environment variables (e.g. env_prefix="MDM_POSTGRES_").""" + + def __init__(self, env_prefix: str = "POSTGRES_", max_size: int = 40) -> None: + self._env_prefix = env_prefix + self._max_size = max_size + self._pool: AsyncConnectionPool | None = None + + def _dsn(self) -> str: + p = self._env_prefix + return ( + f"host={os.environ[p + 'HOST']} " + f"port={os.environ[p + 'PORT']} " + f"dbname={os.environ[p + 'DB']} " + f"user={os.environ[p + 'USER']} " + f"password={os.environ[p + 'PASSWORD']}" + ) + + async def open(self) -> None: + if self._pool is None: + self._pool = AsyncConnectionPool( + conninfo=self._dsn, + open=False, + max_size=self._max_size, + check=AsyncConnectionPool.check_connection, + ) + if not self._pool._opened: + await self._pool.open() + + async def close(self) -> None: + if self._pool is not None: + await self._pool.close() + + def connection(self): + """Return an async connection context manager from the pool.""" + if self._pool is None: + raise RuntimeError("Call open() first (e.g. in app startup).") + return self._pool.connection() diff --git a/pgdevkit/db/crud.py b/pgdevkit/db/crud.py new file mode 100644 index 0000000..87ebc40 --- /dev/null +++ b/pgdevkit/db/crud.py @@ -0,0 +1,184 @@ +from __future__ import annotations + +from typing import Any, Callable, Mapping, Optional, Sequence, Type, TypeVar + +from psycopg.connection_async import AsyncConnection +from psycopg.rows import dict_row +from psycopg.sql import SQL, Identifier, Placeholder + +from .model import PostgresTableModel + +T = TypeVar("T", bound=PostgresTableModel) + + +async def pg_retrieve(con: AsyncConnection, data_type: Type[T], pks: dict) -> T | None: + """Fetch a single row by primary key(s).""" + async with con.cursor(row_factory=dict_row) as cur: + schema, table = data_type.get_table_name() + query = SQL("SELECT * FROM {tbl} WHERE {where}").format( + tbl=Identifier(schema, table), + where=SQL(" AND ").join(SQL("{col} = {val}").format(col=Identifier(pk), val=Placeholder(pk)) for pk in pks), + ) + await cur.execute(query, pks) + row = await cur.fetchone() + return data_type(**row) if row else None + + +async def pg_retrieve_many( + con: AsyncConnection, + data_type: Type[T], + filters: dict, + *, + from_dict: Optional[Callable[[Mapping], T]] = None, +) -> Sequence[T]: + """Fetch multiple rows matching all filter key=value pairs.""" + async with con.cursor(row_factory=dict_row) as cur: + schema, table = data_type.get_table_name() + if filters: + query = SQL("SELECT * FROM {tbl} WHERE {where}").format( + tbl=Identifier(schema, table), + where=SQL(" AND ").join( + SQL("{col} = {val}").format(col=Identifier(k), val=Placeholder(k)) for k in filters + ), + ) + else: + query = SQL("SELECT * FROM {tbl}").format(tbl=Identifier(schema, table)) + await cur.execute(query, filters) + rows = await cur.fetchall() + fn = from_dict or (lambda d: data_type(**d)) + return [fn(r) for r in rows] + + +async def pg_insert(con: AsyncConnection, table_name: tuple[str, str], data: dict) -> dict[str, Any]: + """Insert one row and return the full row (RETURNING *).""" + query = SQL("INSERT INTO {tbl} ({cols}) VALUES ({vals}) RETURNING *").format( + tbl=Identifier(*table_name), + cols=SQL(", ").join(Identifier(k) for k in data), + vals=SQL(", ").join(Placeholder(k) for k in data), + ) + async with con.cursor(row_factory=dict_row) as cur: + await cur.execute(query, data) + row = await cur.fetchone() + assert row is not None + return row + + +async def pg_update_dict( + con: AsyncConnection, + table_name: tuple[str, str], + data: dict, + primary_keys: Sequence[str], +) -> Any | None: + """Update a row identified by primary_keys. Returns the raw row tuple.""" + set_parts = [ + SQL("{col} = {val}").format(col=Identifier(k), val=Placeholder(k)) for k in data if k not in primary_keys + ] + where_parts = [SQL("{col} = {val}").format(col=Identifier(pk), val=Placeholder(pk)) for pk in primary_keys] + query = SQL("UPDATE {tbl} SET {sets} WHERE {where} RETURNING *").format( + tbl=Identifier(*table_name), + sets=SQL(", ").join(set_parts), + where=SQL(" AND ").join(where_parts), + ) + async with con.cursor() as cur: + await cur.execute(query, data) + return await cur.fetchone() + + +async def pg_update(con: AsyncConnection, data: T, data_type: type[T]) -> Any | None: + """Update a typed model instance.""" + return await pg_update_dict(con, data_type.get_table_name(), data.model_dump(), data_type.get_primary_key()) + + +async def pg_upsert_dict( + con: AsyncConnection, + table_name: tuple[str, str], + data: dict, + primary_keys: Sequence[str], +) -> dict: + """INSERT ... ON CONFLICT ... DO UPDATE, returns the row as a dict.""" + fields = list(data) + updates = [SQL("{col} = EXCLUDED.{col}").format(col=Identifier(k)) for k in fields] + query = SQL( + "INSERT INTO {tbl} ({cols}) VALUES ({vals}) ON CONFLICT ({pks}) DO UPDATE SET {updates} RETURNING *" + ).format( + tbl=Identifier(*table_name), + cols=SQL(", ").join(Identifier(k) for k in fields), + vals=SQL(", ").join(Placeholder(k) for k in fields), + pks=SQL(", ").join(Identifier(pk) for pk in primary_keys), + updates=SQL(", ").join(updates), + ) + async with con.cursor(row_factory=dict_row) as cur: + await cur.execute(query, data) + row = await cur.fetchone() + assert row is not None + return row + + +async def pg_upsert(con: AsyncConnection, data: T, data_type: type[T]) -> dict: + """Upsert a typed model instance.""" + return await pg_upsert_dict(con, data_type.get_table_name(), data.model_dump(), data_type.get_primary_key()) + + +async def pg_upsert_many_dict( + con: AsyncConnection, + table_name: tuple[str, str], + data: Sequence[dict], + primary_keys: Sequence[str], +) -> None: + """Batch upsert — one round-trip via executemany.""" + if not data: + return + fields = list(data[0]) + updates = [SQL("{col} = EXCLUDED.{col}").format(col=Identifier(k)) for k in fields if k not in primary_keys] + query = SQL("INSERT INTO {tbl} ({cols}) VALUES ({vals}) ON CONFLICT ({pks}) DO UPDATE SET {updates}").format( + tbl=Identifier(*table_name), + cols=SQL(", ").join(Identifier(k) for k in fields), + vals=SQL(", ").join(Placeholder(k) for k in fields), + pks=SQL(", ").join(Identifier(pk) for pk in primary_keys), + updates=SQL(", ").join(updates), + ) + async with con.cursor() as cur: + await cur.executemany(query, data) + + +async def pg_upsert_many(con: AsyncConnection, data: Sequence[T], data_type: type[T]) -> None: + await pg_upsert_many_dict( + con, data_type.get_table_name(), [d.model_dump() for d in data], data_type.get_primary_key() + ) + + +async def pg_insert_many( + con: AsyncConnection, + table_name: tuple[str, str], + data: Sequence[dict], +) -> None: + """Batch insert — no RETURNING, one round-trip via executemany.""" + if not data: + return + fields = list(data[0]) + query = SQL("INSERT INTO {tbl} ({cols}) VALUES ({vals})").format( + tbl=Identifier(*table_name), + cols=SQL(", ").join(Identifier(k) for k in fields), + vals=SQL(", ").join(Placeholder(k) for k in fields), + ) + async with con.cursor() as cur: + await cur.executemany(query, data) + + +async def pg_delete_dict(con: AsyncConnection, table_name: tuple[str, str], data: dict) -> dict | None: + """Delete by arbitrary key dict, returns the deleted row.""" + where_parts = [SQL("{col} = {val}").format(col=Identifier(k), val=Placeholder(k)) for k in data] + query = SQL("DELETE FROM {tbl} WHERE {where} RETURNING *").format( + tbl=Identifier(*table_name), + where=SQL(" AND ").join(where_parts), + ) + async with con.cursor(row_factory=dict_row) as cur: + await cur.execute(query, data) + return await cur.fetchone() + + +async def pg_delete(con: AsyncConnection, data: T, data_type: type[T]) -> T | None: + """Delete a typed model instance by its primary key(s).""" + pk_dict = {pk: getattr(data, pk) for pk in data_type.get_primary_key()} + row = await pg_delete_dict(con, data_type.get_table_name(), pk_dict) + return data_type.model_validate(row) if row else None diff --git a/pgdevkit/db/loader.py b/pgdevkit/db/loader.py new file mode 100644 index 0000000..0ae69a9 --- /dev/null +++ b/pgdevkit/db/loader.py @@ -0,0 +1,19 @@ +from __future__ import annotations + +from functools import lru_cache +from pathlib import Path +from typing import LiteralString, cast + + +class SqlLoader: + """Loads and caches SQL text from `{root}//.sql`.""" + + def __init__(self, root: Path) -> None: + self._root = root + self._load = lru_cache(maxsize=None)(self._read) + + def _read(self, topic: str, name: str) -> LiteralString: + return cast(LiteralString, (self._root / topic / f"{name}.sql").read_text(encoding="utf-8")) + + def load_sql(self, topic: str, name: str) -> LiteralString: + return self._load(topic, name) diff --git a/pgdevkit/db/model.py b/pgdevkit/db/model.py new file mode 100644 index 0000000..c04e16d --- /dev/null +++ b/pgdevkit/db/model.py @@ -0,0 +1,23 @@ +from __future__ import annotations + +from abc import ABC, abstractmethod +from typing import Sequence + +from pydantic import BaseModel + + +class PostgresTableModel(BaseModel, ABC): + """Base class for models that map 1:1 to a database table/row. + + Models representing partial results (joins, aggregations, projections) + should extend `pydantic.BaseModel` directly instead.""" + + @staticmethod + @abstractmethod + def get_table_name() -> tuple[str, str]: + """Return (schema, table), e.g. ('public', 'users').""" + + @staticmethod + @abstractmethod + def get_primary_key() -> Sequence[str]: + """Return the primary key column name(s).""" diff --git a/pgdevkit/fetch_missing.py b/pgdevkit/fetch_missing.py new file mode 100644 index 0000000..bde49c3 --- /dev/null +++ b/pgdevkit/fetch_missing.py @@ -0,0 +1,229 @@ +from __future__ import annotations + +import re +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import psycopg + +from .diff import DiffKind, compute_diff +from .introspect import introspect_db +from .parser import parse_directory + +_LAYER_PREFIX_RE = re.compile(r"^\d+_?") + +# Matches the `tables`/`views`/`scalar_functions`/`table_functions` object-type +# subfolders documented in docs/database-layout.md. +SUBFOLDER = { + "table": "tables", + "view": "views", + "scalar_function": "scalar_functions", + "table_function": "table_functions", +} + + +@dataclass +class MissingObject: + object_type: str # "table" | "view" | "scalar_function" | "table_function" + schema: str + name: str + + @property + def qualified_name(self) -> str: + return f"{self.schema}.{self.name}" + + +def layer_folder_for(scripts_dir: Path, schema: str) -> Path: + """Map a schema name to its existing layer directory under scripts_dir, + matched by stripping each top-level folder's leading sort number (e.g. + "1_reference_data" -> "reference_data"). Falls back to scripts_dir/schema + if no existing folder matches.""" + if scripts_dir.is_dir(): + for entry in scripts_dir.iterdir(): + if not entry.is_dir() or entry.name in ("migrations", "_migration_scripts"): + continue + if _LAYER_PREFIX_RE.sub("", entry.name).lower() == schema.lower(): + return entry + return scripts_dir / schema + + +def find_missing_objects(scripts_dir: Path, conninfo: str) -> list[MissingObject]: + """Tables, views, and functions that exist in the live database but + aren't tracked as .sql files under scripts_dir.""" + scripts = parse_directory(scripts_dir) + db = introspect_db(conninfo) + diffs = compute_diff(scripts, db, report_extra_db=True) + + missing: list[MissingObject] = [] + with psycopg.connect(conninfo) as conn: + for d in diffs: + if d.kind != DiffKind.MISSING_IN_SCRIPTS or d.object_type not in ("table", "view", "function"): + continue + schema, name = d.object_name.split(".", 1) + object_type = d.object_type + if object_type == "function": + object_type = "table_function" if _returns_set(conn, schema, name) else "scalar_function" + missing.append(MissingObject(object_type, schema, name)) + return missing + + +def _returns_set(conn: Any, schema: str, name: str) -> bool: + with conn.cursor() as cur: + cur.execute( + """ + SELECT p.proretset + FROM pg_catalog.pg_proc p + JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace + WHERE n.nspname = %s AND p.proname = %s AND p.prokind = 'f' + ORDER BY p.oid LIMIT 1 + """, + (schema, name), + ) + row = cur.fetchone() + return bool(row and row[0]) + + +def reconstruct_ddl(conn: Any, obj: MissingObject) -> str: + """Reverse-engineer the DDL for a missing object from pg_catalog.""" + if obj.object_type == "table": + return _table_ddl(conn, obj.schema, obj.name) + if obj.object_type == "view": + return _view_ddl(conn, obj.schema, obj.name) + return _function_ddl(conn, obj.schema, obj.name) + + +def _table_ddl(conn: Any, schema: str, table: str) -> str: + with conn.cursor() as cur: + cur.execute( + """ + SELECT c.oid + FROM pg_catalog.pg_class c + JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = %s AND c.relname = %s AND c.relkind = 'r' + """, + (schema, table), + ) + row = cur.fetchone() + if not row: + raise ValueError(f"Table {schema}.{table} not found in pg_catalog") + oid = row[0] + + cur.execute( + """ + SELECT + a.attname, + pg_catalog.format_type(a.atttypid, a.atttypmod), + a.attnotnull, + a.attidentity, + a.attgenerated, + CASE + WHEN a.attidentity = 'a' THEN 'GENERATED ALWAYS AS IDENTITY' + WHEN a.attidentity = 'd' THEN 'GENERATED BY DEFAULT AS IDENTITY' + WHEN a.attgenerated = 's' THEN + 'GENERATED ALWAYS AS (' || pg_catalog.pg_get_expr(ad.adbin, ad.adrelid) || ') STORED' + WHEN ad.adbin IS NOT NULL THEN + 'DEFAULT ' || pg_catalog.pg_get_expr(ad.adbin, ad.adrelid) + ELSE NULL + END + FROM pg_catalog.pg_attribute a + LEFT JOIN pg_catalog.pg_attrdef ad + ON ad.adrelid = a.attrelid AND ad.adnum = a.attnum + WHERE a.attrelid = %s AND a.attnum > 0 AND NOT a.attisdropped + ORDER BY a.attnum + """, + (oid,), + ) + columns = cur.fetchall() + + cur.execute( + """ + SELECT conname, pg_catalog.pg_get_constraintdef(oid, true) + FROM pg_catalog.pg_constraint + WHERE conrelid = %s + ORDER BY contype, conname + """, + (oid,), + ) + constraints = cur.fetchall() + + cur.execute( + """ + SELECT indexname, indexdef + FROM pg_indexes + WHERE schemaname = %s AND tablename = %s + AND indexname NOT IN ( + SELECT conname FROM pg_catalog.pg_constraint WHERE conrelid = %s + ) + ORDER BY indexname + """, + (schema, table, oid), + ) + indexes = cur.fetchall() + + col_defs: list[str] = [] + for attname, data_type, not_null, identity, _generated, extra in columns: + col_def = f"{attname} {data_type}" + if not_null and not identity: + col_def += " NOT NULL" + if extra: + col_def += f" {extra}" + col_defs.append(col_def) + + for conname, condef in constraints: + col_defs.append(f"CONSTRAINT {conname} {condef}") + + ddl = f"CREATE TABLE IF NOT EXISTS {schema}.{table} (\n" + ddl += ",\n".join(f" {c}" for c in col_defs) + ddl += "\n);\n" + + for _, indexdef in indexes: + indexdef = re.sub(r"^CREATE INDEX\b", "CREATE INDEX IF NOT EXISTS", indexdef) + indexdef = re.sub(r"^CREATE UNIQUE INDEX\b", "CREATE UNIQUE INDEX IF NOT EXISTS", indexdef) + ddl += f"\n{indexdef};\n" + + return ddl + + +def _view_ddl(conn: Any, schema: str, name: str) -> str: + with conn.cursor() as cur: + cur.execute( + """ + SELECT pg_get_viewdef(c.oid, true) + FROM pg_catalog.pg_class c + JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace + WHERE n.nspname = %s AND c.relname = %s AND c.relkind = 'v' + """, + (schema, name), + ) + row = cur.fetchone() + if not row or row[0] is None: + raise ValueError(f"View {schema}.{name} not found in pg_catalog") + return f"CREATE OR REPLACE VIEW {schema}.{name} AS\n{row[0].rstrip()};\n" + + +def _function_ddl(conn: Any, schema: str, name: str) -> str: + """Fetch a function's DDL via pg_get_functiondef. + + If the function is overloaded, this uses the lowest-oid (oldest) match — + good enough to seed a file; resolve manually if there's more than one.""" + with conn.cursor() as cur: + cur.execute( + """ + SELECT p.oid + FROM pg_catalog.pg_proc p + JOIN pg_catalog.pg_namespace n ON n.oid = p.pronamespace + WHERE n.nspname = %s AND p.proname = %s AND p.prokind = 'f' + ORDER BY p.oid LIMIT 1 + """, + (schema, name), + ) + row = cur.fetchone() + if not row: + raise ValueError(f"Function {schema}.{name} not found in pg_catalog") + + cur.execute("SELECT pg_catalog.pg_get_functiondef(%s)", (row[0],)) + ddl_row = cur.fetchone() + if not ddl_row or ddl_row[0] is None: + raise ValueError(f"Could not reconstruct DDL for function {schema}.{name}") + return ddl_row[0].rstrip() + ";\n" diff --git a/pgdevkit/testdb/config.py b/pgdevkit/testdb/config.py index 79ace7c..e8f0e1c 100644 --- a/pgdevkit/testdb/config.py +++ b/pgdevkit/testdb/config.py @@ -36,10 +36,16 @@ def load_config(start: Path | None = None) -> ProjectConfig: data = tomllib.loads(pyproject.read_text(encoding="utf-8")) section = data.get("tool", {}).get("pgdevkit", {}) + extensions = section.get("extensions", []) + if not isinstance(extensions, list): + raise TypeError( + f"[tool.pgdevkit].extensions in {pyproject} must be a list, got {type(extensions).__name__}" + ) + return ProjectConfig( name=section.get("name") or root.name, database_dir=section.get("database_dir", "database"), env_prefix=section.get("env_prefix", ""), - extensions=tuple(section.get("extensions", [])), + extensions=tuple(extensions), root=root, ) diff --git a/pgdevkit/testdb/query.py b/pgdevkit/testdb/query.py index bf82f3d..7fb7a75 100644 --- a/pgdevkit/testdb/query.py +++ b/pgdevkit/testdb/query.py @@ -1,15 +1,58 @@ from __future__ import annotations +import re from typing import LiteralString, cast import psycopg from psycopg.rows import dict_row +_DOLLAR_TAG = re.compile(r"\$[A-Za-z_]*\$") + + +def _split_statements(sql: str) -> list[str]: + """Split on ';', but treat single/double-quoted strings and dollar-quoted + bodies (e.g. a plpgsql function's $$ ... $$) as opaque so an embedded ';' + inside them doesn't split the statement in two.""" + statements: list[str] = [] + buf: list[str] = [] + i, n = 0, len(sql) + while i < n: + ch = sql[i] + if ch == "$" and (m := _DOLLAR_TAG.match(sql, i)): + tag = m.group() + end = sql.find(tag, m.end()) + end = n if end == -1 else end + len(tag) + buf.append(sql[i:end]) + i = end + continue + if ch in ("'", '"'): + end = i + 1 + while end < n: + if sql[end] == ch: + end += 1 + if sql[end : end + 1] == ch: # doubled quote = escaped literal quote + end += 1 + continue + break + end += 1 + buf.append(sql[i:end]) + i = end + continue + if ch == ";": + statements.append("".join(buf)) + buf = [] + i += 1 + continue + buf.append(ch) + i += 1 + statements.append("".join(buf)) + return [s.strip() for s in statements if s.strip()] + async def execute(dsn: str, sql: str) -> list[dict] | None: - """Run one or more ';'-separated statements against dsn. Returns the - rows of the final statement if it produced any, else None.""" - statements = [s.strip() for s in sql.split(";") if s.strip()] + """Run one or more statements against dsn. Returns the rows of the final + statement if it produced any, else None.""" + statements = _split_statements(sql) last_rows: list[dict] | None = None async with await psycopg.AsyncConnection.connect(dsn, autocommit=True) as con: for stmt in statements: diff --git a/pgdevkit/testdb/schema.py b/pgdevkit/testdb/schema.py index a6fc571..77fddd8 100644 --- a/pgdevkit/testdb/schema.py +++ b/pgdevkit/testdb/schema.py @@ -16,6 +16,9 @@ logger = logging.getLogger(__name__) logging.getLogger("sqlglot").setLevel(logging.ERROR) +# The `database/` folder convention (layer dirs, object-type subfolders and +# their apply order, file-naming rules) is documented in +# docs/database-layout.md — keep that table in sync with this dict. _TYPE_ORDER = { "schema": 1, "types": 2, @@ -47,20 +50,27 @@ def _strip_layer_prefix(schema_name: str) -> str: return schema_name -def _get_sql_deps(sql: str) -> tuple[set[str], set[str]]: +_SCHEMA_QUALIFIED_TYPES = { + "types", + "tables", + "scalar_functions", + "functions", + "views", + "table_functions", + "procedures", +} + + +def _get_sql_deps(sql: str) -> set[str]: exprs = sqlglot.parse(sql, dialect="postgres") deps: set[str] = set() - declares: set[str] = set() for e in exprs: if e is None: continue - for t in e.find_all(exp.Create): - if t.args.get("this") is not None and t.args.get("db") is not None: - declares.add(str(t)) for t in e.find_all(exp.Table): if t.args.get("this") is not None and t.args.get("db") is not None: deps.add(str(t)) - return declares, deps + return deps def _iter_sql_files(database_dir: Path): @@ -75,26 +85,25 @@ def _iter_sql_files(database_dir: Path): if file.endswith(".sql") and ".prod" not in file: files.append(Path(root) / file) - delivered_tables: set[str] = set() + delivered: set[str] = set() delayed: list[tuple[str | None, Path, str]] = [] all_declared: set[str] = set() for file in sorted(files, key=lambda p: (_get_type_order(p), p.name)): content = file.read_text(encoding="utf-8") - declares, deps = _get_sql_deps(content) - if file.parent.name == "tables": + deps = _get_sql_deps(content) + if file.parent.name in _SCHEMA_QUALIFIED_TYPES: schema = _strip_layer_prefix(file.parent.parent.name) full_name = f"{schema}.{file.stem}" - deps.discard(full_name) # the file's own CREATE TABLE target is not a real dependency - declares.add(full_name) - all_declared.update(declares) - if not deps or all(d in delivered_tables for d in deps): - delivered_tables.add(full_name) + deps.discard(full_name) # the file's own CREATE target is not a real dependency + all_declared.add(full_name) + if not deps or all(d in delivered for d in deps): + delivered.add(full_name) yield file, content else: delayed.append((full_name, file, content)) continue - if not deps or all(d in delivered_tables for d in deps): + if not deps or all(d in delivered for d in deps): yield file, content else: delayed.append((None, file, content)) @@ -102,13 +111,13 @@ def _iter_sql_files(database_dir: Path): while delayed: progressed = False for i in range(len(delayed) - 1, -1, -1): - tbl_name, file, content = delayed[i] - _, deps = _get_sql_deps(content) - if tbl_name: - deps.discard(tbl_name) - if all(d in delivered_tables or d not in all_declared for d in deps): - if tbl_name: - delivered_tables.add(tbl_name) + declared_name, file, content = delayed[i] + deps = _get_sql_deps(content) + if declared_name: + deps.discard(declared_name) + if all(d in delivered or d not in all_declared for d in deps): + if declared_name: + delivered.add(declared_name) yield file, content delayed.pop(i) progressed = True @@ -160,17 +169,20 @@ async def apply_schema( for extension in extensions: await con.execute(SQL("CREATE EXTENSION IF NOT EXISTS {e}").format(e=Identifier(extension))) + async def _apply(file: Path, sql: str) -> None: + await con.execute(cast(Any, sql)) + json_file = file.with_suffix(".test_data.json") + if json_file.exists(): + schema_name = _strip_layer_prefix(file.parent.parent.name) + await _insert_test_data(json_file, f"{schema_name}.{file.stem}", force_reset, con) + failures: list[tuple[Path, str]] = [] for file, sql in _iter_sql_files(database_dir): try: - await con.execute(cast(Any, sql)) - json_file = file.with_suffix(".test_data.json") - if json_file.exists(): - schema_name = _strip_layer_prefix(file.parent.parent.name) - await _insert_test_data(json_file, f"{schema_name}.{file.stem}", force_reset, con) + await _apply(file, sql) except Exception as e: # noqa: BLE001 logger.warning("Error executing %s (will retry): %s", file, e) failures.append((file, sql)) for file, sql in failures: - await con.execute(cast(Any, sql)) + await _apply(file, sql) diff --git a/pyproject.toml b/pyproject.toml index abf8600..00dfd0c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,6 +22,10 @@ cli = [ "rich>=13.0.0", "typer>=0.26.7", ] +db = [ + "psycopg-pool>=3.2.0", + "pydantic>=2.0", +] [project.scripts] pgdb = "pgdevkit.cli:app" diff --git a/skills/pgdevkit/SKILL.md b/skills/pgdevkit/SKILL.md new file mode 100644 index 0000000..b8fc97f --- /dev/null +++ b/skills/pgdevkit/SKILL.md @@ -0,0 +1,283 @@ +--- +name: pgdevkit +plugin: coding +description: > + Use pgdevkit for any PostgreSQL work in a Python project: a local + Docker/Podman test database (`pgdb testdb`), importable ORM-free CRUD + helpers (`pgdevkit.db`), and the `database/`-folder schema-as-code + convention. Supersedes the old postgres-test-setup, postgres-best-practices, + and database-in-source skills — pgdevkit is a real dependency now, not + copy-pasted reference files. Use whenever the user wants to set up a local + test Postgres, write or review psycopg code, add a table/view/function to + a `database/` folder, or asks things like "add a database query", "create + a repository", "set up a test database", "reset the database", "add a + table", "migration script", "backfill missing objects", or mentions + psycopg/psycopg2/asyncpg/SQLAlchemy where the user seems open to a + different approach. +--- + +# pgdevkit — Postgres for Python projects + +One dependency covers three things that used to be three separate skills +with copy-pasted reference files: + +| Old skill | Replaced by | +|---|---| +| `postgres-test-setup` | `pgdb testdb` (container + schema apply) | +| `postgres-best-practices` | `pgdevkit.db` (importable CRUD helpers) | +| `database-in-source` | The `database/` folder convention — see [docs/database-layout.md](../../docs/database-layout.md) | + +Core rules for every piece of database code in a project using pgdevkit: + +- **No ORM** — use [psycopg](https://www.psycopg.org/psycopg3/) directly, via `pgdevkit.db`'s helpers or hand-written queries. +- **Inline SQL** — trivial queries of **4 lines or fewer** may be written inline in Python. Anything with JOINs, subqueries, CTEs, aggregations, or multiple conditions lives in its own `.sql` file. +- **Named parameters** — always `%(name)s` style, never positional `%s`. +- **The `database/` folder is the source of truth for the schema** — see [docs/database-layout.md](../../docs/database-layout.md) for the layer/object-type/file-naming conventions. +- **Result mapping** — every query result maps to a Pydantic model; table-mapped models extend `pgdevkit.db.PostgresTableModel`. + +--- + +## Install + +```bash +uv add pgdevkit[cli,db] +``` + +`cli` pulls in `typer`/`rich` for the `pgdb` command; `db` pulls in `pydantic`/`psycopg-pool` for the importable CRUD helpers. Skip either extra if the project doesn't need it (e.g. a project using only `pgdb testdb` doesn't need `db`). + +--- + +## Local test database — `pgdb testdb` + +Add to `pyproject.toml`: + +```toml +[tool.pgdevkit] +database_dir = "database" # optional, defaults to "database" +env_prefix = "MDM_" # optional, defaults to "{name.upper()}_" +extensions = ["vector"] # optional, CREATE EXTENSION IF NOT EXISTS +``` + +`name` is optional too — it falls back to the repo directory name. + +In `tests/conftest.py`: + +```python +import pytest +from pgdevkit.testdb import ensure_testdb + +@pytest.fixture(scope="session", autouse=True) +def _testdb_env(): + env = ensure_testdb() + for key, value in env.items(): + os.environ[key] = value +``` + +`ensure_testdb()` starts the shared `pgdevkit-postgres` container if needed (via `podman`), creates a database scoped to this project+branch (so different worktrees/branches never collide), and applies every `.sql` file under `database_dir` in dependency order, seeding any `.test_data.json` sidecar files. + +| What do you need? | Command | +|---|---| +| First-time setup / apply new files | `pgdb testdb up` | +| Breaking change (rename/drop column) | `pgdb testdb reset` | +| Inspect test DB data | `pgdb testdb run-sql --sql "SELECT ..." --results` | +| Re-apply one file (e.g. a function/view) | `pgdb testdb run-sql database/path/to/file.sql` | +| Drop this workspace's database | `pgdb testdb clean` | +| Drop every database for this project (all branches) | `pgdb testdb clean --all` | + +### CI + +Podman needs to be available on the runner (`apt-get install -y podman` on `ubuntu-latest` if not preinstalled) — `ensure_testdb()`/`ensure_container()` shell out to it directly. There's no `PGDEVKIT_SKIP_CONTAINER` + service-container escape hatch wired through every fixture yet — if a project's CI can't run podman, it needs its own workaround for now. + +--- + +## Application-side DB code — `pgdevkit.db` + +``` +app/ +├── db/ +│ ├── queries/ +│ │ ├── users/ +│ │ │ ├── get_user_by_id.sql +│ │ │ └── list_active_users.sql +│ └── repositories/ +│ └── user_repository.py +├── models/ +│ └── user_models.py # Pydantic models for the user domain +``` + +SQL files live under `db/queries//`. Every custom query gets its own file — no multi-statement files that lump unrelated queries together. + +### Connection pool + +```python +# db/connection.py +from pgdevkit.db import PgPool + +pool = PgPool(env_prefix="APP_POSTGRES_") # matches ensure_testdb()'s {env_prefix}POSTGRES_* vars + +async def startup(): + await pool.open() +``` + +### Models + +```python +# models/user_models.py +from __future__ import annotations +from datetime import datetime +from pydantic import BaseModel, ConfigDict +from pgdevkit.db import PostgresTableModel + +class UserRow(PostgresTableModel): + model_config = ConfigDict(from_attributes=True) + id: int + email: str + display_name: str + created_at: datetime + + @staticmethod + def get_table_name() -> tuple[str, str]: + return ("public", "users") + + @staticmethod + def get_primary_key() -> list[str]: + return ["id"] + +class UserSummary(BaseModel): + model_config = ConfigDict(from_attributes=True) + id: int + display_name: str +``` + +Models that represent partial results (joins, aggregations, partial selects) extend `BaseModel` directly instead of `PostgresTableModel`. + +### CRUD helpers + +```python +from pgdevkit.db import pg_retrieve, pg_insert, pg_upsert, pg_delete +``` + +| Helper | Purpose | +|--------|---------| +| `pg_retrieve` | Fetch single row by PK | +| `pg_retrieve_many` | Fetch rows matching filter dict | +| `pg_insert` | Insert one row, `RETURNING *` | +| `pg_update` / `pg_update_dict` | Update by PK | +| `pg_upsert` / `pg_upsert_dict` | `INSERT ... ON CONFLICT ... DO UPDATE` | +| `pg_upsert_many` / `pg_upsert_many_dict` | Batch upsert via `executemany` | +| `pg_insert_many` | Batch insert via `executemany` | +| `pg_delete` / `pg_delete_dict` | Delete by PK, returns deleted row | + +Use these for simple CRUD. For custom `WHERE` clauses, joins, aggregations, or ordering, write a dedicated `.sql` file and a repository method. + +### Loading `.sql` files + +```python +# db/loader.py +from pathlib import Path +from pgdevkit.db import SqlLoader + +sql = SqlLoader(Path(__file__).parent / "queries") +``` + +```python +# db/repositories/user_repository.py +from psycopg.rows import dict_row +from pgdevkit.db import pg_retrieve, pg_delete +from db.connection import pool +from db.loader import sql +from models.user_models import UserRow, UserSummary + +class UserRepository: + async def get_by_id(self, user_id: int) -> UserRow | None: + async with pool.connection() as conn: + return await pg_retrieve(conn, UserRow, {"id": user_id}) + + async def list_active(self, limit: int = 100) -> list[UserSummary]: + async with pool.connection() as conn: + async with conn.cursor(row_factory=dict_row) as cur: + await cur.execute(sql.load_sql("users", "list_active_users"), {"limit": limit}) + rows = await cur.fetchall() + return [UserSummary.model_validate(r) for r in rows] + + async def delete(self, user: UserRow) -> UserRow | None: + async with pool.connection() as conn: + return await pg_delete(conn, user, UserRow) +``` + +Named parameters, `%(name)s` style, dict argument — never positional `%s`, never f-strings or `str.format()` for SQL text. + +### Dynamic SQL + +Avoid it whenever possible — a static `.sql` file is always clearer. See [`references/dynamic-sql.md`](references/dynamic-sql.md) for t-string templates (3.14+) and `psycopg.sql` (< 3.14) when column/table names genuinely vary at runtime. + +### Avoid `LATERAL JOIN` — use a CTE instead + +```sql +with latest_order as ( + select + o.user_id, + o.total, + row_number() over (partition by o.user_id order by o.created_at desc) as rn + from orders as o +) +select u.id, u.email, lo.total +from users as u +join latest_order as lo on lo.user_id = u.id and lo.rn = 1 +``` + +### Temporal tables + +See [`references/temporal-tables.md`](references/temporal-tables.md) for row-level history via `nearform/temporal_tables`. + +### Custom Postgres types in test data + +`pgdevkit.testdb.schema`'s test-data seeding handles plain columns and JSONB, but not composite types or enums directly. See [`references/complex_helper.py`](references/complex_helper.py) for a psycopg adapter (`ComplexHelper`) if a project needs that. + +### SQL formatting + +```bash +uv add --dev shandy-sqlfmt[jinjafmt] +sqlfmt db/queries/ # format +sqlfmt --check db/queries/ # CI check +``` + +--- + +## The `database/` folder & backfilling untracked objects + +See [docs/database-layout.md](../../docs/database-layout.md) for the full convention: layer directories, object-type subfolders and their apply order, file-naming rules (`.test_data.json`, `.init.sql`, `.prod`), and how migrations are organised. + +If a table, view, or function was created directly on the database and never got a `.sql` file: + +```bash +pgdb fetch-missing database/ --url postgresql://... # dry run, lists what's missing +pgdb fetch-missing database/ --url postgresql://... --write +``` + +It diffs the live schema against `database/`, reverse-engineers DDL for anything untracked, and writes it into the matching layer folder's `tables/`, `views/`, `scalar_functions/`, or `table_functions/` subfolder (matched by schema name against existing top-level directories, ignoring their leading sort number). + +--- + +## Comparing scripts to a live database + +```bash +pgdb compare --url postgresql://... database/ +``` + +Reports drift between the `database/` `.sql` files and the actual schema — tables, views, functions, enums, composite types, and indexes. Pass `--report-extra-db` to also flag objects that exist in the database but aren't tracked (this is what `pgdb fetch-missing` uses internally). + +--- + +## Quick checklist + +- [ ] `[tool.pgdevkit]` configured in `pyproject.toml`; `tests/conftest.py` calls `ensure_testdb()` +- [ ] New table/view/function/type gets its own `.sql` file under the right layer + object-type folder (see [docs/database-layout.md](../../docs/database-layout.md)) +- [ ] Simple CRUD uses `pgdevkit.db`'s `pg_*` helpers; custom queries use `.sql` files loaded via `SqlLoader` +- [ ] Inline SQL only for trivial queries ≤ 4 lines; anything with JOINs/CTEs/aggregations/subqueries uses a `.sql` file +- [ ] All parameters use `%(name)s` style with a dict argument +- [ ] Results mapped to a Pydantic model; table-mapped models extend `PostgresTableModel` +- [ ] No `LATERAL JOIN` — use a CTE that groups/aggregates first, then joins it +- [ ] `.prod` files are production-only and skipped by `pgdb testdb` +- [ ] Every table (and non-obvious column) has a `COMMENT ON`, placed in the object's own `.sql` file +- [ ] Untracked DB objects backfilled via `pgdb fetch-missing`, not left undocumented diff --git a/skills/pgdevkit/references/complex_helper.py b/skills/pgdevkit/references/complex_helper.py new file mode 100644 index 0000000..9fe1a39 --- /dev/null +++ b/skills/pgdevkit/references/complex_helper.py @@ -0,0 +1,234 @@ +""" +ComplexHelper — psycopg adapter for PostgreSQL composite types, enums, and JSONB. + +Use this when your project has custom PostgreSQL types (composite types, enums) +that need to be registered with psycopg before inserting test data. Plain columns +and JSONB are handled automatically by pgdevkit.testdb.schema's _insert_test_data; +you only need this class for USER-DEFINED types, which that function does not +yet support natively — wrap the connection it's given before the INSERT: + + from pgdevkit.testdb import schema # for reference, not a public extension point yet + from your_project.pg_complex_helper import ComplexHelper + + async def insert_test_data_with_complex_types(json_file, table, force_reset, con): + helper = ComplexHelper(con) + schema_name, table_name = table.split(".") + complex_types = await helper.load_all_complex_types((schema_name, table_name)) + rows = json.loads(json_file.read_text(encoding="utf-8")) + for row in rows: + for col, info in complex_types.items(): + if col in row: + row[col] = await helper.recursive_convert(row[col], info, con) + # then insert `rows` the same way schema.py's _insert_test_data does +""" + +from psycopg import AsyncConnection +from psycopg.rows import dict_row +from typing import Any +from psycopg.sql import Identifier +from psycopg.types.composite import CompositeInfo, register_composite +from psycopg.types.json import Jsonb +from psycopg.types.enum import EnumInfo, register_enum + + +class ComplexHelper: + complex_types: dict[tuple[str, str], CompositeInfo | EnumInfo] = {} + + def __init__(self, con: AsyncConnection): + self.con = con + self.system_complex_type_dict = None + + self.registered: set[CompositeInfo | EnumInfo] = set() + + async def load_complex_type_dict(self): + async with self.con.cursor(row_factory=dict_row) as cur: + await cur.execute(""" + SELECT t.oid, + pg_catalog.format_type ( t.oid, NULL ) AS obj_name, + t.typtype + FROM pg_catalog.pg_type t + JOIN pg_catalog.pg_namespace n + ON n.oid = t.typnamespace + WHERE ( t.typrelid = 0 + OR ( SELECT c.relkind = 'c' + FROM pg_catalog.pg_class c + WHERE c.oid = t.typrelid ) ) + AND n.nspname <> 'pg_catalog' + AND n.nspname <> 'information_schema' + AND n.nspname !~ '^pg_toast'""") + system_complex_types = await cur.fetchall() + self.system_complex_type_dict = { + r["oid"]: (r["obj_name"], r["typtype"]) for r in system_complex_types + } + + async def _load_complex_type_from_colinfos( + self, res: dict[str, Any] | None + ) -> CompositeInfo | EnumInfo | type[Jsonb] | None: + if not res: + return None + if res["data_type"].lower() == "jsonb": + return Jsonb + if res["data_type"].upper() == "ARRAY" and res["udt_name"] == "_jsonb": + return Jsonb + if ( + not res["is_enum"] + and not res["is_user_defined"] + and not (res["data_type"] == "ARRAY" and res["udt_schema"] != "pg_catalog") + ): + return None + udt_schema: str = res["udt_schema"] + udt_name: str = res["udt_name"] + c = await self._get_complex_type( + f"{udt_schema}.{udt_name}", res["is_enum"], self.con + ) + await self._recurse_register(c, self.con) + return c + + async def load_all_complex_types( + self, table_name: tuple[str, str], include_generated: bool = False + ) -> dict[str, CompositeInfo | type[Jsonb] | EnumInfo | None]: + if self.system_complex_type_dict is None: + await self.load_complex_type_dict() + colquery = """ + with enum_types as ( + select n.nspname as enum_schema, t.typname as enum_name from pg_type t + inner join pg_namespace n on n.oid=t.typnamespace + where typtype='e' + ) + select column_name, data_type, + data_type='USER-DEFINED' as is_user_defined, + udt_schema, udt_name, + e.enum_name is not null as is_enum + from information_schema.columns c + left join enum_types e on e.enum_schema=c.udt_schema and e.enum_name=c.udt_name + where table_schema=%(schema)s and table_name = %(tbl)s and (is_generated <> 'ALWAYS' or %(include_generated)s)""" + async with self.con.cursor(row_factory=dict_row) as cur: + await cur.execute( + colquery, + { + "schema": table_name[0], + "tbl": table_name[1], + "include_generated": include_generated, + }, + ) + res = await cur.fetchall() + return { + r["column_name"]: await self._load_complex_type_from_colinfos(r) + for r in res + } + + async def load_complex_type( + self, table_name: tuple[str, str], col_name: str + ) -> CompositeInfo | type[Jsonb] | EnumInfo | None: + if self.system_complex_type_dict is None: + await self.load_complex_type_dict() + colquery = """ + with enum_types as ( + select n.nspname as enum_schema, t.typname as enum_name from pg_type t + inner join pg_namespace n on n.oid=t.typnamespace + where typtype='e' + ) + select column_name, data_type, + data_type='USER-DEFINED' as is_user_defined, + udt_schema, udt_name, + e.enum_name is not null as is_enum + from information_schema.columns c + left join enum_types e on e.enum_schema=c.udt_schema and e.enum_name=c.udt_name + where table_schema=%(schema)s and table_name = %(tbl)s + and column_name = %(col)s""" + async with self.con.cursor(row_factory=dict_row) as cur: + await cur.execute( + colquery, + {"schema": table_name[0], "tbl": table_name[1], "col": col_name}, + ) + res = await cur.fetchone() + + return await self._load_complex_type_from_colinfos(res) + + async def _get_complex_type( + self, name: str, is_enum: bool, con: AsyncConnection + ) -> CompositeInfo | EnumInfo: + if name.endswith("[]"): + name = name[:-2] + schema, type_name = name.split(".") + if type_name.startswith( + "_" + ): # the array type in PostgreSQL starts with an underscore + type_name = type_name[1:] + if is_enum: + ci = await EnumInfo.fetch(con, Identifier(schema, type_name)) + assert ci is not None, f"Enum type {name} not found in database" + self.complex_types[(schema, type_name)] = ci + if (schema, type_name) not in self.complex_types: + ci = await CompositeInfo.fetch(con, Identifier(schema, type_name)) + assert ci is not None, f"Complex type {name} not found in database" + self.complex_types[(schema, type_name)] = ci + return self.complex_types[(schema, type_name)] + + async def _recurse_register( + self, info: CompositeInfo | EnumInfo, con: AsyncConnection + ): + assert self.system_complex_type_dict is not None, ( + "System complex type dictionary not loaded" + ) + if info not in self.registered: + if isinstance(info, EnumInfo): + register_enum(info, con) + else: + register_composite(info, con) + self.registered.add(info) + if isinstance(info, EnumInfo): + return + for t in info.field_types: + if t in self.system_complex_type_dict: + name, typtype = self.system_complex_type_dict[t] + ci = await self._get_complex_type(name, typtype == "e", con) + await self._recurse_register(ci, con) + + async def recursive_convert( + self, + value: Any, + info: CompositeInfo | EnumInfo | type[Jsonb] | None, + con: AsyncConnection, + ) -> Any: + if info is None: + return value + if value is None: + return None + if self.system_complex_type_dict is None: + await self.load_complex_type_dict() + if isinstance(value, list): + return [await self.recursive_convert(item, info, con) for item in value] + prms = {} + if info == Jsonb: + return Jsonb(value) + if isinstance(value, str): + assert isinstance(info, EnumInfo), f"Expected EnumInfo, got {type(info)}" + return getattr(info.enum, value) # Enum + assert isinstance(info, CompositeInfo), ( + f"Expected CompositeInfo, got {type(info)}" + ) + assert self.system_complex_type_dict is not None, ( + "System complex type dictionary not loaded" + ) + for k, v in value.items(): + if v is None: + prms[k] = None + continue + fi = info.field_names.index(k) + type_oid = info.field_types[fi] + if type_oid in self.system_complex_type_dict: + name, typtype = self.system_complex_type_dict[type_oid] + ci = await self._get_complex_type(name, typtype == "e", con) + if name.endswith("[]"): + prms[k] = [ + await self.recursive_convert(item, ci, con) for item in v + ] + else: + prms[k] = await self.recursive_convert(v, ci, con) + else: + prms[k] = v + assert info.python_type is not None, ( + f"Python type for {info.name} is null, maybe an array?" + ) + return info.python_type(**prms) if prms else None diff --git a/skills/pgdevkit/references/dynamic-sql.md b/skills/pgdevkit/references/dynamic-sql.md new file mode 100644 index 0000000..36ab414 --- /dev/null +++ b/skills/pgdevkit/references/dynamic-sql.md @@ -0,0 +1,92 @@ +# Dynamic SQL + +Avoid dynamic SQL whenever possible — a static `.sql` file is always clearer. When column or table names genuinely vary at runtime, check `pyproject.toml` (or `[project] requires-python`) to pick an approach at authoring time, not with a runtime `sys.version_info` check. + +--- + +## Python 3.14+ — t-string templates + +T-strings look like f-strings but are evaluated by psycopg — values are always sent as bound parameters, never interpolated. + +**Format specifiers:** + +| Specifier | Meaning | +|-----------|---------| +| `{val}` / `{val:s}` | Bound parameter, automatic format (default) | +| `{val:b}` | Bound parameter, binary format | +| `{val:t}` | Bound parameter, text format | +| `{name:i}` | SQL identifier (table/column name) — double-quoted | +| `{val:l}` | Literal value merged client-side (use sparingly) | +| `{snippet:q}` | SQL snippet — another t-string or `sql.SQL`/`Composed` instance | + +**Basic parameter:** + +```python +await cur.execute(t"SELECT * FROM users WHERE id = {user_id}") +``` + +**Dynamic identifier (`:i`):** + +```python +column = "email" +await cur.execute(t"SELECT {column:i} FROM users WHERE active = {active}") +``` + +**NOTIFY — requires client-side composition (`:i` and `:l`):** + +```python +def send_notify(conn: Connection, channel: str, payload: str) -> None: + conn.execute(t"NOTIFY {channel:i}, {payload:l}") +``` + +**Nested templates with `:q` (dynamic WHERE clause):** + +```python +from psycopg import sql + +def search_users( + conn: Connection, + ids: Sequence[int] | None = None, + name_pattern: str | None = None, +) -> list[UserRow]: + filters = [] + if ids is not None: + filters.append(t"u.id = ANY({list(ids)})") + if name_pattern is not None: + filters.append(t"u.name ~* {name_pattern}") + if not filters: + raise TypeError("at least one filter required") + joined = sql.SQL(" AND ").join(filters) + cur = conn.cursor(row_factory=class_row(UserRow)) + cur.execute(t"SELECT * FROM users AS u WHERE {joined:q}") + return cur.fetchall() +``` + +**Inspect composed SQL without executing:** + +```python +from psycopg import sql + +name = "O'Reilly" +dob = datetime.date(1970, 1, 1) +print(sql.as_string(t"INSERT INTO tbl VALUES ({name}, {dob})")) +# INSERT INTO tbl VALUES ('O''Reilly', '1970-01-01'::date) +``` + +--- + +## Python < 3.14 — `psycopg.sql` + +```python +from psycopg import sql + +column = "email" +query = sql.SQL("SELECT {col} FROM users WHERE active = %(active)s").format( + col=sql.Identifier(column), +) +await cur.execute(query, {"active": True}) +``` + +`sql.Identifier` quotes the identifier at the driver level — SQL injection via column/table names is impossible. Values always stay as bound parameters. + +**Never** use f-strings or string concatenation for identifiers or values. diff --git a/skills/pgdevkit/references/temporal-tables.md b/skills/pgdevkit/references/temporal-tables.md new file mode 100644 index 0000000..5176622 --- /dev/null +++ b/skills/pgdevkit/references/temporal-tables.md @@ -0,0 +1,33 @@ +# Temporal Tables (row-level history) + +Use [nearform/temporal_tables](https://github.com/nearform/temporal_tables) — a pure PL/pgSQL trigger that archives every changed row to a parallel history table. Copy `versioning_function.sql` and `system_time_function.sql` from the repo into `db/migrations/` and apply them as a baseline migration. + +For each versioned table, add `sys_period`, create the history table with `LIKE`, and attach the trigger: + +```sql +alter table users + add column sys_period tstzrange not null default tstzrange(current_timestamp, null); + +create table users_history (like users); + +create index on users_history using gist (id, sys_period); + +create trigger users_versioning +before insert or update or delete on users +for each row execute procedure versioning( + 'sys_period', -- system-period column + 'users_history', -- history table name + true, -- conflict mitigation + false -- skip entry when no values changed +); +``` + +`CREATE TABLE … LIKE` copies all columns and their types. Do not add a PK to the history table — the same `id` appears once per version. + +Query point-in-time with `@>`: + +```sql +select * from users_history where id = %(id)s and sys_period @> %(as_of)s::timestamptz +``` + +`set_system_time('2023-01-01+00')` / `set_system_time(null)` backdates operations (useful for migrations). diff --git a/tests/db/__init__.py b/tests/db/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/db/test_crud.py b/tests/db/test_crud.py new file mode 100644 index 0000000..1d1cab1 --- /dev/null +++ b/tests/db/test_crud.py @@ -0,0 +1,108 @@ +from __future__ import annotations + +from typing import Sequence + +import psycopg +import pytest +from pydantic import ConfigDict + +from pgdevkit.db import ( + PgPool, + PostgresTableModel, + pg_delete, + pg_insert, + pg_retrieve, + pg_retrieve_many, + pg_update, + pg_upsert, + pg_upsert_many, +) +from pgdevkit.testdb import constants +from pgdevkit.testdb.container import ensure_container +from tests.testdb.conftest import RUN_SUFFIX, requires_podman + +TEST_DB = f"pgdevkit_db_selftest_{RUN_SUFFIX}" +ENV_PREFIX = "PGDEVKIT_DB_SELFTEST_" + + +def _admin_dsn() -> str: + return f"postgresql://{constants.USER}:{constants.PASSWORD}@{constants.HOST}:{constants.PORT}/postgres" + + +class Widget(PostgresTableModel): + model_config = ConfigDict(from_attributes=True) + id: int + name: str + + @staticmethod + def get_table_name() -> tuple[str, str]: + return ("public", "widget") + + @staticmethod + def get_primary_key() -> Sequence[str]: + return ["id"] + + +@pytest.fixture +async def pool(monkeypatch: pytest.MonkeyPatch): + ensure_container() + with psycopg.connect(_admin_dsn(), autocommit=True) as con: + con.execute(f'DROP DATABASE IF EXISTS "{TEST_DB}"') + con.execute(f'CREATE DATABASE "{TEST_DB}"') + with psycopg.connect(f"{_admin_dsn().rsplit('/', 1)[0]}/{TEST_DB}", autocommit=True) as con: + con.execute("CREATE TABLE widget (id serial PRIMARY KEY, name text NOT NULL)") + + monkeypatch.setenv(f"{ENV_PREFIX}HOST", constants.HOST) + monkeypatch.setenv(f"{ENV_PREFIX}PORT", str(constants.PORT)) + monkeypatch.setenv(f"{ENV_PREFIX}DB", TEST_DB) + monkeypatch.setenv(f"{ENV_PREFIX}USER", constants.USER) + monkeypatch.setenv(f"{ENV_PREFIX}PASSWORD", constants.PASSWORD) + + p = PgPool(env_prefix=ENV_PREFIX) + await p.open() + yield p + await p.close() + with psycopg.connect(_admin_dsn(), autocommit=True) as con: + con.execute(f'DROP DATABASE IF EXISTS "{TEST_DB}"') + + +@requires_podman +async def test_insert_retrieve_update_upsert_delete_roundtrip(pool: PgPool): + async with pool.connection() as con: + inserted = await pg_insert(con, ("public", "widget"), {"name": "sprocket"}) + assert inserted["name"] == "sprocket" + widget_id = inserted["id"] + + fetched = await pg_retrieve(con, Widget, {"id": widget_id}) + assert fetched is not None + assert fetched.name == "sprocket" + + fetched.name = "gadget" + await pg_update(con, fetched, Widget) + refetched = await pg_retrieve(con, Widget, {"id": widget_id}) + assert refetched is not None + assert refetched.name == "gadget" + + many = await pg_retrieve_many(con, Widget, {}) + assert len(many) == 1 + + deleted = await pg_delete(con, refetched, Widget) + assert deleted is not None + assert await pg_retrieve(con, Widget, {"id": widget_id}) is None + + +@requires_podman +async def test_upsert_and_upsert_many(pool: PgPool): + async with pool.connection() as con: + inserted = await pg_insert(con, ("public", "widget"), {"name": "sprocket"}) + widget = Widget(id=inserted["id"], name="sprocket") + + widget.name = "renamed" + upserted = await pg_upsert(con, widget, Widget) + assert upserted["name"] == "renamed" + + widget2 = Widget(id=widget.id + 1000, name="new-widget") + await pg_upsert_many(con, [widget2], Widget) + fetched = await pg_retrieve(con, Widget, {"id": widget2.id}) + assert fetched is not None + assert fetched.name == "new-widget" diff --git a/tests/db/test_loader.py b/tests/db/test_loader.py new file mode 100644 index 0000000..b896890 --- /dev/null +++ b/tests/db/test_loader.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +from pathlib import Path + +from pgdevkit.db.loader import SqlLoader + + +def test_load_sql_reads_topic_name_file(tmp_path: Path): + (tmp_path / "users").mkdir() + (tmp_path / "users" / "get_by_id.sql").write_text("SELECT * FROM users WHERE id = %(id)s", encoding="utf-8") + + loader = SqlLoader(tmp_path) + assert loader.load_sql("users", "get_by_id") == "SELECT * FROM users WHERE id = %(id)s" + + +def test_load_sql_is_cached(tmp_path: Path): + (tmp_path / "users").mkdir() + sql_file = tmp_path / "users" / "get_by_id.sql" + sql_file.write_text("original", encoding="utf-8") + + loader = SqlLoader(tmp_path) + assert loader.load_sql("users", "get_by_id") == "original" + + sql_file.write_text("changed", encoding="utf-8") + assert loader.load_sql("users", "get_by_id") == "original" # cached, not re-read diff --git a/tests/test_fetch_missing.py b/tests/test_fetch_missing.py new file mode 100644 index 0000000..e666230 --- /dev/null +++ b/tests/test_fetch_missing.py @@ -0,0 +1,82 @@ +from __future__ import annotations + +from pathlib import Path + +import psycopg +import pytest + +from pgdevkit.fetch_missing import find_missing_objects, layer_folder_for, reconstruct_ddl +from pgdevkit.testdb import constants +from pgdevkit.testdb.container import ensure_container +from tests.testdb.conftest import RUN_SUFFIX, requires_podman + +TEST_DB = f"pgdevkit_fetchmissing_selftest_{RUN_SUFFIX}" + + +def _admin_dsn() -> str: + return f"postgresql://{constants.USER}:{constants.PASSWORD}@{constants.HOST}:{constants.PORT}/postgres" + + +def _db_dsn() -> str: + return f"postgresql://{constants.USER}:{constants.PASSWORD}@{constants.HOST}:{constants.PORT}/{TEST_DB}" + + +def test_layer_folder_for_matches_stripped_sort_prefix(tmp_path: Path): + (tmp_path / "1_reference_data").mkdir() + (tmp_path / "2_app").mkdir() + assert layer_folder_for(tmp_path, "reference_data") == tmp_path / "1_reference_data" + assert layer_folder_for(tmp_path, "app") == tmp_path / "2_app" + + +def test_layer_folder_for_falls_back_to_schema_name(tmp_path: Path): + assert layer_folder_for(tmp_path, "unmatched") == tmp_path / "unmatched" + + +@pytest.fixture +def fetchmissing_db(tmp_path: Path): + ensure_container() + with psycopg.connect(_admin_dsn(), autocommit=True) as con: + con.execute(f'DROP DATABASE IF EXISTS "{TEST_DB}"') + con.execute(f'CREATE DATABASE "{TEST_DB}"') + with psycopg.connect(_db_dsn(), autocommit=True) as con: + con.execute("CREATE SCHEMA app") + con.execute("CREATE TABLE app.tracked (id serial PRIMARY KEY)") + con.execute("CREATE TABLE app.untracked (id serial PRIMARY KEY, name text NOT NULL)") + con.execute("CREATE VIEW app.untracked_view AS SELECT id, name FROM app.untracked") + con.execute("CREATE FUNCTION app.double_it(x int) RETURNS int AS 'SELECT x * 2' LANGUAGE sql") + con.execute( + "CREATE FUNCTION app.list_names() RETURNS TABLE(name text) AS " + "'SELECT name FROM app.untracked' LANGUAGE sql" + ) + + scripts_dir = tmp_path / "database" + (scripts_dir / "app" / "tables").mkdir(parents=True) + (scripts_dir / "app" / "tables" / "tracked.sql").write_text( + "CREATE TABLE IF NOT EXISTS app.tracked (id serial PRIMARY KEY);", encoding="utf-8" + ) + yield scripts_dir + with psycopg.connect(_admin_dsn(), autocommit=True) as con: + con.execute(f'DROP DATABASE IF EXISTS "{TEST_DB}"') + + +@requires_podman +def test_find_missing_objects_reports_untracked_table_view_and_functions(fetchmissing_db: Path): + missing = find_missing_objects(fetchmissing_db, _db_dsn()) + by_name = {m.qualified_name: m.object_type for m in missing} + + assert by_name["app.untracked"] == "table" + assert by_name["app.untracked_view"] == "view" + assert by_name["app.double_it"] == "scalar_function" + assert by_name["app.list_names"] == "table_function" + assert "app.tracked" not in by_name + + +@requires_podman +def test_reconstruct_ddl_round_trips_for_each_object_type(fetchmissing_db: Path): + missing = find_missing_objects(fetchmissing_db, _db_dsn()) + by_name = {m.qualified_name: m for m in missing} + + with psycopg.connect(_db_dsn(), autocommit=True) as conn: + for name in ("app.untracked", "app.untracked_view", "app.double_it", "app.list_names"): + ddl = reconstruct_ddl(conn, by_name[name]) + conn.execute(ddl) # must not raise — DDL is valid and re-appliable (CREATE OR REPLACE / IF NOT EXISTS) diff --git a/tests/test_fetch_missing_cli.py b/tests/test_fetch_missing_cli.py new file mode 100644 index 0000000..0dd77c3 --- /dev/null +++ b/tests/test_fetch_missing_cli.py @@ -0,0 +1,52 @@ +from __future__ import annotations + +from pathlib import Path + +import psycopg +from typer.testing import CliRunner + +from pgdevkit.cli import app +from pgdevkit.testdb import constants +from pgdevkit.testdb.container import ensure_container +from tests.testdb.conftest import RUN_SUFFIX, requires_podman + +runner = CliRunner() + +TEST_DB = f"pgdevkit_fetchmissing_cli_selftest_{RUN_SUFFIX}" + + +def _admin_dsn() -> str: + return f"postgresql://{constants.USER}:{constants.PASSWORD}@{constants.HOST}:{constants.PORT}/postgres" + + +def _db_dsn() -> str: + return f"postgresql://{constants.USER}:{constants.PASSWORD}@{constants.HOST}:{constants.PORT}/{TEST_DB}" + + +@requires_podman +def test_fetch_missing_dry_run_then_write(tmp_path: Path): + ensure_container() + with psycopg.connect(_admin_dsn(), autocommit=True) as con: + con.execute(f'DROP DATABASE IF EXISTS "{TEST_DB}"') + con.execute(f'CREATE DATABASE "{TEST_DB}"') + with psycopg.connect(_db_dsn(), autocommit=True) as con: + con.execute("CREATE TABLE public.orphan (id serial PRIMARY KEY, name text NOT NULL)") + + scripts_dir = tmp_path / "database" + scripts_dir.mkdir() + + try: + dry = runner.invoke(app, ["fetch-missing", str(scripts_dir), "--url", _db_dsn()]) + assert dry.exit_code == 0, dry.output + assert "public.orphan" in dry.output + assert "Dry run" in dry.output + assert not any(scripts_dir.rglob("*.sql")) + + written = runner.invoke(app, ["fetch-missing", str(scripts_dir), "--url", _db_dsn(), "--write"]) + assert written.exit_code == 0, written.output + dest = scripts_dir / "public" / "tables" / "orphan.sql" + assert dest.exists() + assert "orphan" in dest.read_text(encoding="utf-8") + finally: + with psycopg.connect(_admin_dsn(), autocommit=True) as con: + con.execute(f'DROP DATABASE IF EXISTS "{TEST_DB}"') diff --git a/tests/testdb/conftest.py b/tests/testdb/conftest.py index e7f118a..c8c6260 100644 --- a/tests/testdb/conftest.py +++ b/tests/testdb/conftest.py @@ -1,5 +1,6 @@ from __future__ import annotations +import os import shutil import subprocess from pathlib import Path @@ -13,13 +14,20 @@ FIXTURES = Path(__file__).parent / "fixtures" / "database" +# Appended to every test project's [tool.pgdevkit].name so that two pytest +# processes (e.g. from separate git worktrees) running against the shared +# pgdevkit-postgres container at the same time get distinct database names +# instead of dropping each other's throwaway databases mid-run. +RUN_SUFFIX = f"pid{os.getpid()}" + def _make_project(base: Path, name: str, branch: str) -> Path: project = base / f"{name}-{branch}" project.mkdir() (project / "database").symlink_to(FIXTURES) (project / "pyproject.toml").write_text( - f'[tool.pgdevkit]\nname = "{name}"\n', encoding="utf-8" + f'[tool.pgdevkit]\nname = "{name}_{RUN_SUFFIX}"\nenv_prefix = "{name.upper()}_"\n', + encoding="utf-8", ) for cmd in ( ["git", "init", "-q"], diff --git a/tests/testdb/fixtures/database/app/views/a_wrapper_view.sql b/tests/testdb/fixtures/database/app/views/a_wrapper_view.sql new file mode 100644 index 0000000..442a128 --- /dev/null +++ b/tests/testdb/fixtures/database/app/views/a_wrapper_view.sql @@ -0,0 +1,2 @@ +CREATE OR REPLACE VIEW app.a_wrapper_view AS +SELECT id, name FROM app.b_base_view; diff --git a/tests/testdb/fixtures/database/app/views/b_base_view.sql b/tests/testdb/fixtures/database/app/views/b_base_view.sql new file mode 100644 index 0000000..c4b2e20 --- /dev/null +++ b/tests/testdb/fixtures/database/app/views/b_base_view.sql @@ -0,0 +1,2 @@ +CREATE OR REPLACE VIEW app.b_base_view AS +SELECT id, name FROM app.widget; diff --git a/tests/testdb/test_api.py b/tests/testdb/test_api.py index c06dd87..25a34e5 100644 --- a/tests/testdb/test_api.py +++ b/tests/testdb/test_api.py @@ -8,6 +8,8 @@ from pgdevkit.testdb import constants from pgdevkit.testdb.api import clean_testdb, ensure_testdb, reset_testdb, status +from pgdevkit.testdb.config import load_config +from pgdevkit.testdb.naming import slugify from tests.testdb.conftest import requires_podman @@ -39,9 +41,10 @@ def test_clean_all_removes_every_branch_database(project_factory: Callable[[str, clean_testdb(project_a, all=True) + prefix = slugify(load_config(project_a).name) with psycopg.connect(_admin_dsn()) as con: with con.cursor() as cur: - cur.execute("SELECT count(*) FROM pg_database WHERE datname LIKE 'apitest2_%'") + cur.execute("SELECT count(*) FROM pg_database WHERE datname LIKE %s", (f"{prefix}_%",)) (count,) = cur.fetchone() assert count == 0 diff --git a/tests/testdb/test_cli.py b/tests/testdb/test_cli.py index 67bd12b..8ad607a 100644 --- a/tests/testdb/test_cli.py +++ b/tests/testdb/test_cli.py @@ -42,6 +42,35 @@ def test_testdb_run_sql_inline_with_results(project_factory: Callable[[str, str] clean_testdb(project) +@requires_podman +def test_testdb_run_sql_file_with_dollar_quoted_semicolon( + project_factory: Callable[[str, str], Path], monkeypatch +): + # A naive ';'-split would cut this function body in half at the embedded + # semicolon inside the dollar-quoted body, breaking both statements. + project = project_factory("clitest6", "main") + monkeypatch.chdir(project) + sql_file = project / "adhoc.sql" + sql_file.write_text( + """ + CREATE OR REPLACE FUNCTION app.answer() RETURNS int AS $$ + BEGIN + RETURN 42; + END; + $$ LANGUAGE plpgsql; + SELECT app.answer() AS answer; + """, + encoding="utf-8", + ) + try: + runner.invoke(app, ["testdb", "up"]) + result = runner.invoke(app, ["testdb", "run-sql", str(sql_file), "--results"]) + assert result.exit_code == 0, result.output + assert "42" in result.output + finally: + clean_testdb(project) + + @requires_podman def test_testdb_run_sql_with_results_and_zero_rows(project_factory: Callable[[str, str], Path], monkeypatch): project = project_factory("clitest5", "main") diff --git a/tests/testdb/test_config.py b/tests/testdb/test_config.py index b865fde..d9904a9 100644 --- a/tests/testdb/test_config.py +++ b/tests/testdb/test_config.py @@ -2,6 +2,8 @@ from pathlib import Path +import pytest + from pgdevkit.testdb.config import load_config @@ -51,3 +53,19 @@ def test_searches_upward_for_pyproject(tmp_path: Path): config = load_config(subdir) assert config.name == "root_project" assert config.root == tmp_path + + +def test_explicit_empty_name_falls_back_to_root_name(tmp_path: Path): + project = tmp_path / "myproj" + project.mkdir() + (project / "pyproject.toml").write_text('[tool.pgdevkit]\nname = ""\n', encoding="utf-8") + config = load_config(project) + assert config.name == "myproj" + + +def test_extensions_must_be_a_list(tmp_path: Path): + (tmp_path / "pyproject.toml").write_text( + '[tool.pgdevkit]\nname = "x"\nextensions = "vector"\n', encoding="utf-8" + ) + with pytest.raises(TypeError, match="extensions"): + load_config(tmp_path) diff --git a/tests/testdb/test_container.py b/tests/testdb/test_container.py index e4b90a4..cc09196 100644 --- a/tests/testdb/test_container.py +++ b/tests/testdb/test_container.py @@ -3,7 +3,7 @@ import psycopg from pgdevkit.testdb import constants -from pgdevkit.testdb.container import ensure_container +from pgdevkit.testdb.container import _create_container, ensure_container from tests.testdb.conftest import requires_podman @@ -33,3 +33,15 @@ def test_ensure_container_is_idempotent(): with con.cursor() as cur: cur.execute("SELECT 1") assert cur.fetchone() == (1,) + + +@requires_podman +def test_create_container_falls_back_to_start_when_name_in_use(): + ensure_container() # container already exists under constants.CONTAINER_NAME + + _create_container() # "podman run" will fail with "already in use"; must fall back to "podman start" + + with psycopg.connect(_admin_dsn()) as con: + with con.cursor() as cur: + cur.execute("SELECT 1") + assert cur.fetchone() == (1,) diff --git a/tests/testdb/test_query.py b/tests/testdb/test_query.py index 9c6b3db..17fcfe8 100644 --- a/tests/testdb/test_query.py +++ b/tests/testdb/test_query.py @@ -5,9 +5,35 @@ from pgdevkit.testdb import constants, query from pgdevkit.testdb.container import ensure_container -from tests.testdb.conftest import requires_podman +from pgdevkit.testdb.query import _split_statements +from tests.testdb.conftest import RUN_SUFFIX, requires_podman -TEST_DB = "pgdevkit_query_selftest" + +def test_split_statements_ignores_semicolon_inside_dollar_quote(): + sql = """ + CREATE FUNCTION f() RETURNS int AS $$ + BEGIN + RETURN 1; + END; + $$ LANGUAGE plpgsql; + SELECT f(); + """ + statements = _split_statements(sql) + assert len(statements) == 2 + assert "RETURN 1;" in statements[0] + assert statements[1] == "SELECT f();" or statements[1].startswith("SELECT f()") + + +def test_split_statements_ignores_semicolon_inside_string_literal(): + statements = _split_statements("INSERT INTO t (s) VALUES ('a;b'); SELECT 1") + assert statements == ["INSERT INTO t (s) VALUES ('a;b')", "SELECT 1"] + + +def test_split_statements_handles_tagged_dollar_quotes(): + statements = _split_statements("SELECT $tag$a;b$tag$ AS x; SELECT 2") + assert statements == ["SELECT $tag$a;b$tag$ AS x", "SELECT 2"] + +TEST_DB = f"pgdevkit_query_selftest_{RUN_SUFFIX}" def _admin_dsn() -> str: diff --git a/tests/testdb/test_schema.py b/tests/testdb/test_schema.py index 4f00ed0..edf70d0 100644 --- a/tests/testdb/test_schema.py +++ b/tests/testdb/test_schema.py @@ -8,10 +8,10 @@ from pgdevkit.testdb import constants from pgdevkit.testdb.container import ensure_container from pgdevkit.testdb.schema import apply_schema -from tests.testdb.conftest import requires_podman +from tests.testdb.conftest import RUN_SUFFIX, requires_podman FIXTURES = Path(__file__).parent / "fixtures" / "database" -TEST_DB = "pgdevkit_schema_selftest" +TEST_DB = f"pgdevkit_schema_selftest_{RUN_SUFFIX}" def _admin_dsn() -> str: @@ -81,3 +81,17 @@ async def test_apply_schema_resolves_multi_level_fk_dependency(schema_test_db): ) (detail_id,) = await cur.fetchone() assert detail_id == 1 + + +@requires_podman +async def test_apply_schema_resolves_view_to_view_dependency(schema_test_db): + # a_wrapper_view selects from b_base_view, but the filenames sort in the + # opposite order — this only passes if cross-view dependency tracking + # (not just table FK tracking) delays a_wrapper_view until its dependency + # exists. + async with await psycopg.AsyncConnection.connect(_db_dsn(), autocommit=True) as con: + await apply_schema(con, FIXTURES) # must not raise + async with con.cursor() as cur: + await cur.execute("SELECT id, name FROM app.a_wrapper_view ORDER BY id") + rows = await cur.fetchall() + assert rows == [(1, "sprocket")] diff --git a/uv.lock b/uv.lock index 7121098..4e240dc 100644 --- a/uv.lock +++ b/uv.lock @@ -11,6 +11,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/1e/d3/26bf1008eb3d2daa8ef4cacc7f3bfdc11818d111f7e2d0201bc6e3b49d45/annotated_doc-0.0.4-py3-none-any.whl", hash = "sha256:571ac1dc6991c450b25a9c2d84a3705e2ae7a53467b5d111c24fa8baabbed320", size = 5303, upload-time = "2025-11-10T22:07:40.673Z" }, ] +[[package]] +name = "annotated-types" +version = "0.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ee/67/531ea369ba64dcff5ec9c3402f9f51bf748cec26dde048a2f973a4eea7f5/annotated_types-0.7.0.tar.gz", hash = "sha256:aff07c09a53a08bc8cfccb9c85b05f1aa9a2a6f23728d790723543408344ce89", size = 16081, upload-time = "2024-05-20T21:33:25.928Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/78/b6/6307fbef88d9b5ee7421e68d78a9f162e0da4900bc5f5793f6d3d0e34fb8/annotated_types-0.7.0-py3-none-any.whl", hash = "sha256:1f02e8b43a8fbbc3f3e0d4f0f4bfc8131bcb4eebe8849b8e5c773f3a1c582a53", size = 13643, upload-time = "2024-05-20T21:33:24.1Z" }, +] + [[package]] name = "azure-core" version = "1.41.0" @@ -273,6 +282,10 @@ cli = [ { name = "rich" }, { name = "typer" }, ] +db = [ + { name = "psycopg-pool" }, + { name = "pydantic" }, +] [package.dev-dependencies] dev = [ @@ -289,11 +302,13 @@ test = [ requires-dist = [ { name = "azure-identity", marker = "extra == 'azure'", specifier = ">=1.19.0" }, { name = "psycopg", extras = ["binary"], specifier = ">=3.2.0" }, + { name = "psycopg-pool", marker = "extra == 'db'", specifier = ">=3.2.0" }, + { name = "pydantic", marker = "extra == 'db'", specifier = ">=2.0" }, { name = "rich", marker = "extra == 'cli'", specifier = ">=13.0.0" }, { name = "sqlglot", extras = ["c"], specifier = ">=30.11.0" }, { name = "typer", marker = "extra == 'cli'", specifier = ">=0.26.7" }, ] -provides-extras = ["azure", "cli"] +provides-extras = ["azure", "cli", "db"] [package.metadata.requires-dev] dev = [{ name = "ty", specifier = ">=0.0.59" }] @@ -348,6 +363,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/eb/e6/5fff07a70d1f945ed90ae131c3bd76cab32beff7c58c6db15ad5820b6d1f/psycopg_binary-3.3.4-cp314-cp314-win_amd64.whl", hash = "sha256:c37e024c07308cd06cf3ec51bfd0e7f6157585a4d84d1bce4a7f5f7913719bf8", size = 3666849, upload-time = "2026-05-01T23:31:51.165Z" }, ] +[[package]] +name = "psycopg-pool" +version = "3.3.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/90/82/7a23d26039827ecd4ebe93905651029ddd307c5182ad59296dfb6f67b528/psycopg_pool-3.3.1.tar.gz", hash = "sha256:b10b10b7a175d5cc1592147dc5b7eec8a9e0834eb3ed2c4a92c858e2f51eb63c", size = 31661, upload-time = "2026-05-01T23:31:59.809Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/37/ed/89c2c620af0e1660354cd8aabf9f5b21f911597ce22acb37c805d6c86bc8/psycopg_pool-3.3.1-py3-none-any.whl", hash = "sha256:2af5b432941c4c9ad5c87b3fa410aec910ec8f7c122855897983a06c45f2e4b5", size = 40023, upload-time = "2026-05-01T23:31:53.136Z" }, +] + [[package]] name = "pycparser" version = "3.0" @@ -357,6 +384,62 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0c/c3/44f3fbbfa403ea2a7c779186dc20772604442dde72947e7d01069cbe98e3/pycparser-3.0-py3-none-any.whl", hash = "sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992", size = 48172, upload-time = "2026-01-21T14:26:50.693Z" }, ] +[[package]] +name = "pydantic" +version = "2.13.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-types" }, + { name = "pydantic-core" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/18/a5/b60d21ac674192f8ab0ba4e9fd860690f9b4a6e51ca5df118733b487d8d6/pydantic-2.13.4.tar.gz", hash = "sha256:c40756b57adaa8b1efeeced5c196f3f3b7c435f90e84ea7f443901bec8099ef6", size = 844775, upload-time = "2026-05-06T13:43:05.343Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fd/7b/122376b1fd3c62c1ed9dc80c931ace4844b3c55407b6fb2d199377c9736f/pydantic-2.13.4-py3-none-any.whl", hash = "sha256:45a282cde31d808236fd7ea9d919b128653c8b38b393d1c4ab335c62924d9aba", size = 472262, upload-time = "2026-05-06T13:43:02.641Z" }, +] + +[[package]] +name = "pydantic-core" +version = "2.46.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9d/56/921726b776ace8d8f5db44c4ef961006580d91dc52b803c489fafd1aa249/pydantic_core-2.46.4.tar.gz", hash = "sha256:62f875393d7f270851f20523dd2e29f082bcc82292d66db2b64ea71f64b6e1c1", size = 471464, upload-time = "2026-05-06T13:37:06.98Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/8d/74/228a26ddad29c6672b805d9fd78e8d251cd04004fa7eed0e622096cd0250/pydantic_core-2.46.4-cp314-cp314-macosx_10_12_x86_64.whl", hash = "sha256:428e04521a40150c85216fc8b85e8d39fece235a9cf5e383761238c7fa9b96fb", size = 2102079, upload-time = "2026-05-06T13:38:41.019Z" }, + { url = "https://files.pythonhosted.org/packages/ad/1f/8970b150a4b4365623ae00fc88603491f763c627311ae8031e3111356d6e/pydantic_core-2.46.4-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:23ace664830ee0bfe014a0c7bc248b1f7f25ed7ad103852c317624a1083af462", size = 1952179, upload-time = "2026-05-06T13:36:59.812Z" }, + { url = "https://files.pythonhosted.org/packages/95/30/5211a831ae054928054b2f79731661087a2bc5c01e825c672b3a4a8f1b3e/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce5c1d2a8b27468f433ca974829c44060b8097eedc39933e3c206a90ee49c4a9", size = 1978926, upload-time = "2026-05-06T13:37:39.933Z" }, + { url = "https://files.pythonhosted.org/packages/57/e9/689668733b1eb67adeef047db3c2e8788fcf65a7fd9c9e2b46b7744fe245/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:7283d57845ecf5a163403eb0702dfc220cc4fbdd18919cb5ccea4f95ee1cdab4", size = 2046785, upload-time = "2026-05-06T13:38:01.995Z" }, + { url = "https://files.pythonhosted.org/packages/60/d9/6715260422ff50a2109878fd24d948a6c3446bb2664f34ee78cd972b3acd/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8daafc69c93ee8a0204506a3b6b30f586ef54028f52aeeeb5c4cfc5184fd5914", size = 2228733, upload-time = "2026-05-06T13:40:50.371Z" }, + { url = "https://files.pythonhosted.org/packages/18/ae/fdb2f64316afca925640f8e70bb1a564b0ec2721c1389e25b8eb4bf9a299/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd2213145bcc2ba85884d0ac63d222fece9209678f77b9b4d76f054c561adb28", size = 2307534, upload-time = "2026-05-06T13:37:21.531Z" }, + { url = "https://files.pythonhosted.org/packages/89/1d/8eff589b45bb8190a9d12c49cfad0f176a5cbd1534908a6b5125e2886239/pydantic_core-2.46.4-cp314-cp314-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7a5f930472650a82629163023e630d160863fce524c616f4e5186e5de9d9a49b", size = 2099732, upload-time = "2026-05-06T13:39:31.942Z" }, + { url = "https://files.pythonhosted.org/packages/06/d5/ee5a3366637fee41dee51a1fc91562dcf12ddbc68fda34e6b253da2324bb/pydantic_core-2.46.4-cp314-cp314-manylinux_2_31_riscv64.whl", hash = "sha256:c1b3f518abeca3aa13c712fd202306e145abf59a18b094a6bafb2d2bbf59192c", size = 2129627, upload-time = "2026-05-06T13:37:25.033Z" }, + { url = "https://files.pythonhosted.org/packages/94/33/2414be571d2c6a6c4d08be21f9292b6d3fdb08949a97b6dfe985017821db/pydantic_core-2.46.4-cp314-cp314-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1a7dd0b3ee80d90150e3495a3a13ac34dbcbfd4f012996a6a1d8900e91b5c0fb", size = 2179141, upload-time = "2026-05-06T13:37:14.046Z" }, + { url = "https://files.pythonhosted.org/packages/7b/79/7daa95be995be0eecc4cf75064cb33f9bbbfe3fe0158caf2f0d4a996a5c7/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_aarch64.whl", hash = "sha256:3fb702cd90b0446a3a1c5e470bfa0dd23c0233b676a9099ddcc964fa6ca13898", size = 2184325, upload-time = "2026-05-06T13:36:53.615Z" }, + { url = "https://files.pythonhosted.org/packages/9f/cb/d0a382f5c0de8a222dc61c65348e0ce831b1f68e0a018450d31c2cace3a5/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_armv7l.whl", hash = "sha256:b8458003118a712e66286df6a707db01c52c0f52f7db8e4a38f0da1d3b94fc4e", size = 2323990, upload-time = "2026-05-06T13:40:29.971Z" }, + { url = "https://files.pythonhosted.org/packages/05/db/d9ba624cc4a5aced1598e88c04fdbd8310c8a69b9d38b9a3d39ce3a61ed7/pydantic_core-2.46.4-cp314-cp314-musllinux_1_1_x86_64.whl", hash = "sha256:372429a130e469c9cd698925ce5fc50940b7a1336b0d82038e63d5bbc4edc519", size = 2369978, upload-time = "2026-05-06T13:37:23.027Z" }, + { url = "https://files.pythonhosted.org/packages/f2/20/d15df15ba918c423461905802bfd2981c3af0bfa0e40d05e13edbfa48bc3/pydantic_core-2.46.4-cp314-cp314-win32.whl", hash = "sha256:85bb3611ff1802f3ee7fdd7dbff26b56f343fb432d57a4728fdd49b6ef35e2f4", size = 1966354, upload-time = "2026-05-06T13:38:03.499Z" }, + { url = "https://files.pythonhosted.org/packages/fc/b6/6b8de4c0a7d7ab3004c439c80c5c1e0a3e8d78bbae19379b01960383d9e5/pydantic_core-2.46.4-cp314-cp314-win_amd64.whl", hash = "sha256:811ff8e9c313ab425368bcbb36e5c4ebd7108c2bbf4e4089cfbb0b01eff63fac", size = 2072238, upload-time = "2026-05-06T13:39:40.807Z" }, + { url = "https://files.pythonhosted.org/packages/32/36/51eb763beec1f4cf59b1db243a7dcc39cbb41230f050a09b9d69faaf0a48/pydantic_core-2.46.4-cp314-cp314-win_arm64.whl", hash = "sha256:bfec22eab3c8cc2ceec0248aec886624116dc079afa027ecc8ad4a7e62010f8a", size = 2018251, upload-time = "2026-05-06T13:37:26.72Z" }, + { url = "https://files.pythonhosted.org/packages/e8/91/855af51d625b23aa987116a19e231d2aaef9c4a415273ddc189b79a45fee/pydantic_core-2.46.4-cp314-cp314t-macosx_10_12_x86_64.whl", hash = "sha256:af8244b2bef6aaad6d92cda81372de7f8c8d36c9f0c3ea36e827c60e7d9467a0", size = 2099593, upload-time = "2026-05-06T13:39:47.682Z" }, + { url = "https://files.pythonhosted.org/packages/fb/1b/8784a54c65edb5f49f0a14d6977cf1b209bba85a4c77445b255c2de58ab3/pydantic_core-2.46.4-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5a4330cdbc57162e4b3aa303f588ba752257694c9c9be3e7ebb11b4aca659b5d", size = 1935226, upload-time = "2026-05-06T13:40:40.428Z" }, + { url = "https://files.pythonhosted.org/packages/e8/e7/1955d28d1afc56dd4b3ad7cc0cf39df1b9852964cf16e5d13912756d6d6b/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:29c61fc04a3d840155ff08e475a04809278972fe6aef51e2720554e96367e34b", size = 1974605, upload-time = "2026-05-06T13:37:32.029Z" }, + { url = "https://files.pythonhosted.org/packages/93/e2/3fedbf0ba7a22850e6e9fd78117f1c0f10f950182344d8a6c535d468fdd8/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c50f2528cf200c5eed56faf3f4e22fcd5f38c157a8b78576e6ba3168ec35f000", size = 2030777, upload-time = "2026-05-06T13:38:55.239Z" }, + { url = "https://files.pythonhosted.org/packages/f8/61/46be275fcaaba0b4f5b9669dd852267ce1ff616592dccf7a7845588df091/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:0cbe8b01f948de4286c74cdd6c667aceb38f5c1e26f0693b3983d9d74887c65e", size = 2236641, upload-time = "2026-05-06T13:37:08.096Z" }, + { url = "https://files.pythonhosted.org/packages/60/db/12e93e46a8bac9988be3c016860f83293daea8c716c029c9ace279036f2f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:617d7e2ca7dcb8c5cf6bcb8c59b8832c94b36196bbf1cbd1bfb56ed341905edd", size = 2286404, upload-time = "2026-05-06T13:40:20.221Z" }, + { url = "https://files.pythonhosted.org/packages/e2/4a/4d8b19008f38d31c53b8219cfedc2e3d5de5fe99d90076b7e767de29274f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7027560ee92211647d0d34e3f7cd6f50da56399d26a9c8ad0da286d3869a53f3", size = 2109219, upload-time = "2026-05-06T13:38:12.153Z" }, + { url = "https://files.pythonhosted.org/packages/88/70/3cbc40978fefb7bb09c6708d40d4ad1a5d70fd7213c3d17f971de868ec1f/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_31_riscv64.whl", hash = "sha256:f99626688942fb746e545232e7726926f3be91b5975f8b55327665fafda991c7", size = 2110594, upload-time = "2026-05-06T13:40:02.971Z" }, + { url = "https://files.pythonhosted.org/packages/9d/20/b8d36736216e29491125531685b2f9e61aa5b4b2599893f8268551da3338/pydantic_core-2.46.4-cp314-cp314t-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:fc3e9034a63de20e15e8ade85358bc6efc614008cab72898b4b4952bea0509ff", size = 2159542, upload-time = "2026-05-06T13:39:27.506Z" }, + { url = "https://files.pythonhosted.org/packages/1d/a2/367df868eb584dacf6bf82a389272406d7178e301c4ac82545ab98bc2dd9/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_aarch64.whl", hash = "sha256:97e7cf2be5c77b7d1a9713a05605d49460d02c6078d38d8bef3cbe323c548424", size = 2168146, upload-time = "2026-05-06T13:38:31.93Z" }, + { url = "https://files.pythonhosted.org/packages/c1/b8/4460f77f7e201893f649a29ab355dddd3beee8a97bcb1a320db414f9a06e/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_armv7l.whl", hash = "sha256:3bf92c5d0e00fefaab325a4d27828fe6b6e2a21848686b5b60d2d9eeb09d76c6", size = 2306309, upload-time = "2026-05-06T13:37:44.717Z" }, + { url = "https://files.pythonhosted.org/packages/64/c4/be2639293acd87dc8ddbcec41a73cee9b2ebf996fe6d892a1a74e88ad3f7/pydantic_core-2.46.4-cp314-cp314t-musllinux_1_1_x86_64.whl", hash = "sha256:3ecbc122d18468d06ca279dc26a8c2e2d5acb10943bb35e36ae92096dc3b5565", size = 2369736, upload-time = "2026-05-06T13:37:05.645Z" }, + { url = "https://files.pythonhosted.org/packages/30/a6/9f9f380dbb301f67023bf8f707aaa75daadf84f7152d95c410fd7e81d994/pydantic_core-2.46.4-cp314-cp314t-win32.whl", hash = "sha256:e846ae7835bf0703ae43f534ab79a867146dadd59dc9ca5c8b53d5c8f7c9ef02", size = 1955575, upload-time = "2026-05-06T13:38:51.116Z" }, + { url = "https://files.pythonhosted.org/packages/40/1f/f1eb9eb350e795d1af8586289746f5c5677d16043040d63710e22abc43c9/pydantic_core-2.46.4-cp314-cp314t-win_amd64.whl", hash = "sha256:2108ba5c1c1eca18030634489dc544844144ee36357f2f9f780b93e7ddbb44b5", size = 2051624, upload-time = "2026-05-06T13:38:21.672Z" }, + { url = "https://files.pythonhosted.org/packages/f6/d2/42dd53d0a85c27606f316d3aa5d2869c4e8470a5ed6dec30e4a1abe19192/pydantic_core-2.46.4-cp314-cp314t-win_arm64.whl", hash = "sha256:4fcbe087dbc2068af7eda3aa87634eba216dbda64d1ae73c8684b621d33f6596", size = 2017325, upload-time = "2026-05-06T13:40:52.723Z" }, +] + [[package]] name = "pygments" version = "2.20.0" @@ -545,6 +628,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/18/67/36e9267722cc04a6b9f15c7f3441c2363321a3ea07da7ae0c0707beb2a9c/typing_extensions-4.15.0-py3-none-any.whl", hash = "sha256:f0fa19c6845758ab08074a0cfa8b7aecb71c999ca73d62883bc25cc018c4e548", size = 44614, upload-time = "2025-08-25T13:49:24.86Z" }, ] +[[package]] +name = "typing-inspection" +version = "0.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/55/e3/70399cb7dd41c10ac53367ae42139cf4b1ca5f36bb3dc6c9d33acdb43655/typing_inspection-0.4.2.tar.gz", hash = "sha256:ba561c48a67c5958007083d386c3295464928b01faa735ab8547c5692e87f464", size = 75949, upload-time = "2025-10-01T02:14:41.687Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", size = 14611, upload-time = "2025-10-01T02:14:40.154Z" }, +] + [[package]] name = "tzdata" version = "2026.2"