diff --git a/.gitignore b/.gitignore index e1c7f4c..ecfbe1c 100644 --- a/.gitignore +++ b/.gitignore @@ -11,3 +11,6 @@ wheels/ # Local planning docs (not shipped with the package) docs/superpowers/ + +# Local migration checklist (not shipped with the package) +/MIGRATION_NOTES.md diff --git a/README.md b/README.md index e69de29..8aaf6e2 100644 --- a/README.md +++ b/README.md @@ -0,0 +1,42 @@ +# pgdevkit + +A helper for developing with Postgres. + +## `pgdb compare` + +Compare a directory of SQL scripts (see the `database-in-source` layout +convention) against a live database and report differences: + +```bash +pgdb compare --url postgresql://user:pass@host:port/db path/to/database/ +``` + +## `pgdb testdb` + +Manages a single shared, Podman-backed Postgres container for local tests +across all your projects — no more one-container-per-project-per-worktree. +Isolation between projects and worktrees is per-database, inside one +container. + +Add to `pyproject.toml`: + +```toml +[tool.pgdevkit] +name = "myproject" # optional; defaults to the repo directory name +database_dir = "database" # optional; defaults to "database" +``` + +Add to `conftest.py`: + +```python +import os +import pytest +from pgdevkit.testdb import ensure_testdb + +@pytest.fixture(scope="session", autouse=True) +def ensure_test_postgres(): + for k, v in ensure_testdb().items(): + os.environ[k] = v +``` + +CLI: `pgdb testdb up|reset|run-sql|status|shell|clean`. diff --git a/pgdevkit/cli.py b/pgdevkit/cli.py index 00ec2be..2879978 100644 --- a/pgdevkit/cli.py +++ b/pgdevkit/cli.py @@ -1,5 +1,6 @@ from __future__ import annotations +import os from pathlib import Path import typer @@ -7,6 +8,7 @@ from rich.table import Table from rich import box +from . import testdb from .connection import build_conninfo from .diff import DiffKind, compute_diff from .introspect import introspect_db @@ -16,10 +18,12 @@ console = Console() err_console = Console(stderr=True) +testdb_app = typer.Typer(name="testdb", help="Manage the shared local Postgres test container") +app.add_typer(testdb_app, name="testdb") + @app.command() def compare( - url: str = typer.Option(..., "--url", help="PostgreSQL DSN (postgresql://user:pass@host:port/db)"), entra_user: str | None = typer.Option(None, "--entra-user", help="Azure Entra user (triggers token auth)"), report_extra_db: bool = typer.Option(False, "--report-extra-db", help="Report objects in DB but not in scripts"), @@ -61,3 +65,73 @@ def compare( console.print(table) console.print(f"\n[bold red]{len(diffs)} difference(s) found.[/bold red]") raise typer.Exit(1) + + +@testdb_app.command("up") +def testdb_up() -> None: + """Ensure the container is running, the workspace DB exists, and schema is applied.""" + testdb.ensure_testdb() + info = testdb.status() + console.print(f"[green]Test DB ready:[/green] {info['database']} ({info['dsn']})") + + +@testdb_app.command("reset") +def testdb_reset() -> None: + """Drop and recreate only this workspace's database, then reapply schema + seed data.""" + testdb.reset_testdb() + info = testdb.status() + console.print(f"[green]Test DB reset:[/green] {info['database']}") + + +@testdb_app.command("run-sql") +def testdb_run_sql( + file: Path | None = typer.Argument(None, help="Path to a .sql file"), + sql: str | None = typer.Option(None, "--sql", help="Inline SQL string"), + results: bool = typer.Option(False, "--results", help="Print query results as a table"), +) -> None: + """Run SQL against this workspace's database.""" + if (file is None) == (sql is None): + err_console.print("[red]Error:[/red] pass exactly one of FILE or --sql") + raise typer.Exit(2) + statement = file.read_text(encoding="utf-8") if file else sql + assert statement is not None + rows = testdb.run_sql(statement) + + if rows is None: + console.print("OK") + return + if not results: + console.print(f"OK — {len(rows)} row(s)") + return + if not rows: + console.print("(0 rows)") + return + table = Table(box=box.SIMPLE, show_header=True, header_style="bold") + for col in rows[0]: + table.add_column(col) + for row in rows: + table.add_row(*(str(v) for v in row.values())) + console.print(table) + console.print(f"({len(rows)} row(s))") + + +@testdb_app.command("status") +def testdb_status() -> None: + """Show container state, this workspace's database name, and DSN.""" + for key, value in testdb.status().items(): + console.print(f"{key}: {value}") + + +@testdb_app.command("shell") +def testdb_shell() -> None: + """Drop into psql against this workspace's database.""" + os.execvp("psql", ["psql", testdb.dsn_for()]) + + +@testdb_app.command("clean") +def testdb_clean( + all: bool = typer.Option(False, "--all", help="Drop every database belonging to this project"), +) -> None: + """Drop this workspace's database (or every database of this project with --all).""" + testdb.clean_testdb(all=all) + console.print("[green]Cleaned.[/green]") diff --git a/pgdevkit/diff.py b/pgdevkit/diff.py index 9841fb9..390e753 100644 --- a/pgdevkit/diff.py +++ b/pgdevkit/diff.py @@ -204,7 +204,10 @@ def _unwrap_paren(node): def _norm_sql(s: str) -> str: - return " ".join(s.lower().split()) + # Postgres's pg_get_viewdef()/pg_get_indexdef() always append a trailing + # ";", while sqlglot's Expression.sql() rendering never does; strip it so + # semantically-identical definitions compare equal. + return " ".join(s.lower().split()).rstrip(";") def _norm_body(s: str) -> str: diff --git a/pgdevkit/parser.py b/pgdevkit/parser.py index abd1270..39b61f7 100644 --- a/pgdevkit/parser.py +++ b/pgdevkit/parser.py @@ -197,15 +197,25 @@ def _handle_view(expr: exp.Create, db_schema: DatabaseSchema) -> None: def _handle_function(expr: exp.Create, raw: str, db_schema: DatabaseSchema, kind: str) -> None: # Get name/schema from sqlglot func_node = expr.this - if hasattr(func_node, "name"): - fname = func_node.name + fname = func_node.name if hasattr(func_node, "name") else "" + if fname: db_node = func_node.args.get("db") if hasattr(func_node, "args") else None fschema = db_node.name if db_node else "public" else: - result = _resolve_name(expr) - if not result: - return - fschema, fname = result + # sqlglot (30.11.0) parses "CREATE FUNCTION myapp.greet(...)" as a + # UserDefinedFunction wrapping a Table (this=Identifier(greet), + # db=Identifier(myapp)). UserDefinedFunction.name doesn't unwrap + # that nested Table, so pull the name/schema from it directly. + inner = func_node.this if hasattr(func_node, "this") else None + if isinstance(inner, exp.Table) and inner.name: + fname = inner.name + db_node = inner.args.get("db") + fschema = db_node.name if db_node else "public" + else: + result = _resolve_name(expr) + if not result: + return + fschema, fname = result # Extract args, return type, language, body with regex on raw SQL args, return_type, language, body = _parse_function_details(raw) @@ -275,8 +285,15 @@ def _handle_type(expr: exp.Create, db_schema: DatabaseSchema) -> None: def _handle_schema_create(expr: exp.Create, db_schema: DatabaseSchema) -> None: this = expr.this - if hasattr(this, "name"): - db_schema.schemas.add(this.name) + name = this.name if hasattr(this, "name") else "" + if not name: + # sqlglot (30.11.0) parses "CREATE SCHEMA myapp" as a Table node + # whose `db` arg holds the schema name and `.name` (the table + # identifier) is empty. + db_node = this.args.get("db") if hasattr(this, "args") else None + name = db_node.name if db_node else "" + if name: + db_schema.schemas.add(name) def _handle_index(expr: exp.Create, db_schema: DatabaseSchema) -> None: diff --git a/pgdevkit/testdb/__init__.py b/pgdevkit/testdb/__init__.py new file mode 100644 index 0000000..102da2a --- /dev/null +++ b/pgdevkit/testdb/__init__.py @@ -0,0 +1,3 @@ +from .api import clean_testdb, dsn_for, ensure_testdb, reset_testdb, run_sql, status + +__all__ = ["clean_testdb", "dsn_for", "ensure_testdb", "reset_testdb", "run_sql", "status"] diff --git a/pgdevkit/testdb/api.py b/pgdevkit/testdb/api.py new file mode 100644 index 0000000..b1cbbbb --- /dev/null +++ b/pgdevkit/testdb/api.py @@ -0,0 +1,139 @@ +from __future__ import annotations + +import asyncio +import os +from pathlib import Path + +import psycopg +from psycopg.sql import SQL, Identifier + +from . import constants, query +from .config import ProjectConfig, load_config +from .container import ensure_container +from .naming import current_branch, slugify, workspace_db_name +from .schema import apply_schema + + +def _admin_dsn() -> str: + return f"postgresql://{constants.USER}:{constants.PASSWORD}@{constants.HOST}:{constants.PORT}/postgres?connect_timeout=10" + + +def _db_dsn(db_name: str) -> str: + return f"postgresql://{constants.USER}:{constants.PASSWORD}@{constants.HOST}:{constants.PORT}/{db_name}?connect_timeout=10" + + +def _resolve(project_root: Path | None) -> tuple[ProjectConfig, str]: + config = load_config(project_root) + branch = current_branch(config.root) + db_name = workspace_db_name(config.name, branch) + return config, db_name + + +def _env_for(config: ProjectConfig, db_name: str) -> dict[str, str]: + prefix = config.env_prefix + return { + f"{prefix}POSTGRES_HOST": constants.HOST, + f"{prefix}POSTGRES_PORT": str(constants.PORT), + f"{prefix}POSTGRES_DB": db_name, + f"{prefix}POSTGRES_USER": constants.USER, + f"{prefix}POSTGRES_PASSWORD": constants.PASSWORD, + } + + +async def _ensure_database(db_name: str) -> None: + async with await psycopg.AsyncConnection.connect(_admin_dsn(), autocommit=True) as con: + result = await con.execute("SELECT 1 FROM pg_database WHERE datname = %(db)s", {"db": db_name}) + if await result.fetchone(): + return + try: + await con.execute(SQL("CREATE DATABASE {}").format(Identifier(db_name))) + except psycopg.errors.DuplicateDatabase: + pass + + +async def _drop_database(db_name: str) -> None: + async with await psycopg.AsyncConnection.connect(_admin_dsn(), autocommit=True) as con: + await con.execute( + "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = %(db)s", + {"db": db_name}, + ) + await con.execute(SQL("DROP DATABASE IF EXISTS {}").format(Identifier(db_name))) + + +async def _apply(config: ProjectConfig, db_name: str, force_reset: bool) -> None: + async with await psycopg.AsyncConnection.connect(_db_dsn(db_name), autocommit=True) as con: + await apply_schema( + con, + config.root / config.database_dir, + extensions=config.extensions, + force_reset=force_reset, + ) + + +def ensure_testdb(project_root: Path | None = None, force_reset: bool = False) -> dict[str, str]: + """Ensure the shared container is running, this workspace's database + exists, and its schema is applied. Returns the {PREFIX}POSTGRES_* env + vars for this workspace.""" + if not os.environ.get("PGDEVKIT_SKIP_CONTAINER"): + ensure_container() + config, db_name = _resolve(project_root) + + async def _run() -> None: + if force_reset: + await _drop_database(db_name) + await _ensure_database(db_name) + await _apply(config, db_name, force_reset) + + asyncio.run(_run()) + return _env_for(config, db_name) + + +def reset_testdb(project_root: Path | None = None) -> dict[str, str]: + """Drop and recreate only this workspace's database, then reapply + schema and seed data.""" + return ensure_testdb(project_root, force_reset=True) + + +def clean_testdb(project_root: Path | None = None, all: bool = False) -> None: + """Drop this workspace's database. With all=True, drop every database + belonging to this project (matched by its name-slug prefix), across + every worktree/branch.""" + config, db_name = _resolve(project_root) + + async def _run() -> None: + if not all: + await _drop_database(db_name) + return + prefix = f"{slugify(config.name)}_" + escaped_prefix = prefix.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_") + async with await psycopg.AsyncConnection.connect(_admin_dsn(), autocommit=True) as con: + result = await con.execute( + "SELECT datname FROM pg_database WHERE datname LIKE %(pattern)s ESCAPE '\\'", + {"pattern": f"{escaped_prefix}%"}, + ) + names = [row[0] for row in await result.fetchall()] + for name in names: + await _drop_database(name) + + asyncio.run(_run()) + + +def status(project_root: Path | None = None) -> dict[str, str]: + config, db_name = _resolve(project_root) + return { + "container": constants.CONTAINER_NAME, + "host": constants.HOST, + "port": str(constants.PORT), + "database": db_name, + "dsn": _db_dsn(db_name), + } + + +def run_sql(sql: str, project_root: Path | None = None) -> list[dict] | None: + _, db_name = _resolve(project_root) + return asyncio.run(query.execute(_db_dsn(db_name), sql)) + + +def dsn_for(project_root: Path | None = None) -> str: + _, db_name = _resolve(project_root) + return _db_dsn(db_name) diff --git a/pgdevkit/testdb/config.py b/pgdevkit/testdb/config.py new file mode 100644 index 0000000..79ace7c --- /dev/null +++ b/pgdevkit/testdb/config.py @@ -0,0 +1,45 @@ +from __future__ import annotations + +import tomllib +from dataclasses import dataclass, field +from pathlib import Path + + +@dataclass(frozen=True) +class ProjectConfig: + name: str + database_dir: str = "database" + env_prefix: str = "" + extensions: tuple[str, ...] = () + root: Path = field(default_factory=Path) + + def __post_init__(self) -> None: + if not self.env_prefix: + object.__setattr__(self, "env_prefix", f"{self.name.upper()}_") + + +def _find_pyproject(start: Path) -> Path | None: + for directory in [start, *start.parents]: + candidate = directory / "pyproject.toml" + if candidate.exists(): + return candidate + return None + + +def load_config(start: Path | None = None) -> ProjectConfig: + start = (start or Path.cwd()).resolve() + pyproject = _find_pyproject(start) + root = pyproject.parent if pyproject else start + + section: dict = {} + if pyproject is not None: + data = tomllib.loads(pyproject.read_text(encoding="utf-8")) + section = data.get("tool", {}).get("pgdevkit", {}) + + return ProjectConfig( + name=section.get("name") or root.name, + database_dir=section.get("database_dir", "database"), + env_prefix=section.get("env_prefix", ""), + extensions=tuple(section.get("extensions", [])), + root=root, + ) diff --git a/pgdevkit/testdb/constants.py b/pgdevkit/testdb/constants.py new file mode 100644 index 0000000..09f37a6 --- /dev/null +++ b/pgdevkit/testdb/constants.py @@ -0,0 +1,9 @@ +from __future__ import annotations + +CONTAINER_NAME = "pgdevkit-postgres" +IMAGE = "pgvector/pgvector:pg18-trixie" +HOST = "localhost" +PORT = 54322 +USER = "postgres" +PASSWORD = "testpwd" +PG_SPEED_FLAGS = ["-c", "fsync=off", "-c", "synchronous_commit=off", "-c", "full_page_writes=off"] diff --git a/pgdevkit/testdb/container.py b/pgdevkit/testdb/container.py new file mode 100644 index 0000000..f38a60e --- /dev/null +++ b/pgdevkit/testdb/container.py @@ -0,0 +1,70 @@ +from __future__ import annotations + +import subprocess +import time + +import psycopg + +from . import constants + + +def _podman(*args: str, check: bool = True) -> subprocess.CompletedProcess: + return subprocess.run(["podman", *args], capture_output=True, text=True, check=check) + + +def _container_status() -> str | None: + """Return 'running', 'exited', etc., or None if the container doesn't exist.""" + result = _podman( + "inspect", constants.CONTAINER_NAME, "--format", "{{.State.Status}}", check=False + ) + if result.returncode != 0: + return None + return result.stdout.strip() + + +def _create_container() -> None: + result = _podman( + "run", "-d", + "--name", constants.CONTAINER_NAME, + "-p", f"{constants.PORT}:5432", + "-e", f"POSTGRES_USER={constants.USER}", + "-e", f"POSTGRES_PASSWORD={constants.PASSWORD}", + constants.IMAGE, + "postgres", *constants.PG_SPEED_FLAGS, + check=False, + ) + if result.returncode != 0 and "already in use" in result.stderr: + _podman("start", constants.CONTAINER_NAME) + return + if result.returncode != 0: + raise RuntimeError(f"podman run failed: {result.stderr}") + + +def _wait_ready(timeout: float = 30.0) -> None: + deadline = time.monotonic() + timeout + dsn = ( + f"postgresql://{constants.USER}:{constants.PASSWORD}" + f"@{constants.HOST}:{constants.PORT}/postgres?connect_timeout=2" + ) + last_error: Exception | None = None + while time.monotonic() < deadline: + try: + with psycopg.connect(dsn): + return + except Exception as e: # noqa: BLE001 + last_error = e + time.sleep(0.5) + raise RuntimeError(f"Postgres did not become ready within {timeout}s: {last_error}") + + +def ensure_container() -> None: + """Idempotently ensure the shared pgdevkit-postgres container is running + and accepting connections.""" + status = _container_status() + if status == "running": + return + if status is not None: + _podman("start", constants.CONTAINER_NAME) + else: + _create_container() + _wait_ready() diff --git a/pgdevkit/testdb/naming.py b/pgdevkit/testdb/naming.py new file mode 100644 index 0000000..7a21240 --- /dev/null +++ b/pgdevkit/testdb/naming.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +import hashlib +import re +import subprocess +from pathlib import Path + +_INVALID_CHARS = re.compile(r"[^a-z0-9_]+") +_MAX_SLUG_LEN = 30 + + +def slugify(value: str) -> str: + """Lowercase, replace invalid chars with '_', truncate+hash if too long.""" + slug = _INVALID_CHARS.sub("_", value.lower()).strip("_") + if not slug: + slug = "x" + if len(slug) <= _MAX_SLUG_LEN: + return slug + digest = hashlib.sha256(slug.encode()).hexdigest()[:8] + return f"{slug[:_MAX_SLUG_LEN]}_{digest}" + + +def current_branch(cwd: Path | None = None) -> str: + """Return the branch checked out in the git worktree rooted at cwd.""" + result = subprocess.run( + ["git", "rev-parse", "--abbrev-ref", "HEAD"], + cwd=cwd, + capture_output=True, + text=True, + check=True, + ) + return result.stdout.strip() + + +def workspace_db_name(project_name: str, branch: str) -> str: + """Compute a Postgres-safe, collision-resistant database name for this + project+branch. A second slugify pass over the joined string guarantees + the result stays under Postgres's 63-byte identifier limit even when + both inputs are already at the per-component truncation limit.""" + joined = f"{slugify(project_name)}_{slugify(branch)}" + return slugify(joined) diff --git a/pgdevkit/testdb/query.py b/pgdevkit/testdb/query.py new file mode 100644 index 0000000..bf82f3d --- /dev/null +++ b/pgdevkit/testdb/query.py @@ -0,0 +1,22 @@ +from __future__ import annotations + +from typing import LiteralString, cast + +import psycopg +from psycopg.rows import dict_row + + +async def execute(dsn: str, sql: str) -> list[dict] | None: + """Run one or more ';'-separated statements against dsn. Returns the + rows of the final statement if it produced any, else None.""" + statements = [s.strip() for s in sql.split(";") if s.strip()] + last_rows: list[dict] | None = None + async with await psycopg.AsyncConnection.connect(dsn, autocommit=True) as con: + for stmt in statements: + async with con.cursor(row_factory=dict_row) as cur: + # stmt is arbitrary, caller-provided SQL text (a .sql file or + # --sql argument) — not a compile-time literal, but this + # function's entire purpose is to run it as-is. + await cur.execute(cast(LiteralString, stmt)) + last_rows = await cur.fetchall() if cur.description else None + return last_rows diff --git a/pgdevkit/testdb/schema.py b/pgdevkit/testdb/schema.py new file mode 100644 index 0000000..a6fc571 --- /dev/null +++ b/pgdevkit/testdb/schema.py @@ -0,0 +1,176 @@ +from __future__ import annotations + +import json +import logging +import os +import re +from pathlib import Path +from typing import Any, cast + +import psycopg +import sqlglot +import sqlglot.expressions as exp +from psycopg.rows import dict_row +from psycopg.sql import SQL, Identifier, Placeholder + +logger = logging.getLogger(__name__) +logging.getLogger("sqlglot").setLevel(logging.ERROR) + +_TYPE_ORDER = { + "schema": 1, + "types": 2, + "tables": 3, + "scalar_functions": 4, + "functions": 5, + "views": 6, + "table_functions": 7, + "procedures": 8, + "permissions": 100, + "indexes": 101, +} + + +def _get_type_order(path: Path) -> int: + filename = re.sub(r"^\d+(\.\d+)?", "", path.name).removeprefix("_").removesuffix(".sql") + if filename in _TYPE_ORDER: + return _TYPE_ORDER[filename] + if path.parent.name in _TYPE_ORDER: + return _TYPE_ORDER[path.parent.name] + raise ValueError(f"Unknown SQL type for {path.name} in {path.parent.name}") + + +def _strip_layer_prefix(schema_name: str) -> str: + """Strip a leading numeric layer prefix (e.g. "1_dim" -> "dim") so it + matches the unprefixed schema name used in SQL identifiers.""" + if re.match(r"^\d+_", schema_name): + return schema_name.split("_", 1)[1] + return schema_name + + +def _get_sql_deps(sql: str) -> tuple[set[str], set[str]]: + exprs = sqlglot.parse(sql, dialect="postgres") + deps: set[str] = set() + declares: set[str] = set() + for e in exprs: + if e is None: + continue + for t in e.find_all(exp.Create): + if t.args.get("this") is not None and t.args.get("db") is not None: + declares.add(str(t)) + for t in e.find_all(exp.Table): + if t.args.get("this") is not None and t.args.get("db") is not None: + deps.add(str(t)) + return declares, deps + + +def _iter_sql_files(database_dir: Path): + """Yield (Path, sql_content) pairs in dependency-safe execution order.""" + files: list[Path] = [] + for root, _, dbfiles in os.walk(database_dir): + if "_migration_scripts" in root or "migrations" in root: + continue + for file in dbfiles: + if file in ("all.sql", "100_permissions.sql"): + continue + if file.endswith(".sql") and ".prod" not in file: + files.append(Path(root) / file) + + delivered_tables: set[str] = set() + delayed: list[tuple[str | None, Path, str]] = [] + all_declared: set[str] = set() + + for file in sorted(files, key=lambda p: (_get_type_order(p), p.name)): + content = file.read_text(encoding="utf-8") + declares, deps = _get_sql_deps(content) + if file.parent.name == "tables": + schema = _strip_layer_prefix(file.parent.parent.name) + full_name = f"{schema}.{file.stem}" + deps.discard(full_name) # the file's own CREATE TABLE target is not a real dependency + declares.add(full_name) + all_declared.update(declares) + if not deps or all(d in delivered_tables for d in deps): + delivered_tables.add(full_name) + yield file, content + else: + delayed.append((full_name, file, content)) + continue + if not deps or all(d in delivered_tables for d in deps): + yield file, content + else: + delayed.append((None, file, content)) + + while delayed: + progressed = False + for i in range(len(delayed) - 1, -1, -1): + tbl_name, file, content = delayed[i] + _, deps = _get_sql_deps(content) + if tbl_name: + deps.discard(tbl_name) + if all(d in delivered_tables or d not in all_declared for d in deps): + if tbl_name: + delivered_tables.add(tbl_name) + yield file, content + delayed.pop(i) + progressed = True + if not progressed: + raise ValueError(f"Circular or missing SQL dependencies: {[f[1] for f in delayed]}") + + +async def _insert_test_data( + json_file: Path, table: str, force_reset: bool, con: psycopg.AsyncConnection +) -> None: + if not json_file.exists(): + return + rows: list[dict[str, Any]] = json.loads(json_file.read_text(encoding="utf-8")) + if not rows: + return + + schema, table_name = table.split(".") + async with con.cursor(row_factory=dict_row) as cur: + if not force_reset: + await cur.execute(SQL("SELECT count(*) AS cnt FROM {t}").format(t=Identifier(schema, table_name))) + row = await cur.fetchone() + if row and row["cnt"] == len(rows): + return + + col_names = list(rows[0].keys()) + for row in rows: + for col in col_names: + if isinstance(row[col], (dict, list)): + row[col] = json.dumps(row[col]) + + await cur.execute(SQL("DELETE FROM {t}").format(t=Identifier(schema, table_name))) + insert_sql = SQL("INSERT INTO {t} ({cols}) VALUES ({vals})").format( + t=Identifier(schema, table_name), + cols=SQL(", ").join(Identifier(c) for c in col_names), + vals=SQL(", ").join(Placeholder(c) for c in col_names), + ) + await cur.executemany(insert_sql, rows) + + +async def apply_schema( + con: psycopg.AsyncConnection, + database_dir: Path, + extensions: tuple[str, ...] = (), + force_reset: bool = False, +) -> None: + """Apply every .sql file under database_dir (in dependency-safe order) + and seed any matching .test_data.json files. Safe to call repeatedly.""" + await con.set_autocommit(True) + for extension in extensions: + await con.execute(SQL("CREATE EXTENSION IF NOT EXISTS {e}").format(e=Identifier(extension))) + + failures: list[tuple[Path, str]] = [] + for file, sql in _iter_sql_files(database_dir): + try: + await con.execute(cast(Any, sql)) + json_file = file.with_suffix(".test_data.json") + if json_file.exists(): + schema_name = _strip_layer_prefix(file.parent.parent.name) + await _insert_test_data(json_file, f"{schema_name}.{file.stem}", force_reset, con) + except Exception as e: # noqa: BLE001 + logger.warning("Error executing %s (will retry): %s", file, e) + failures.append((file, sql)) + + for file, sql in failures: + await con.execute(cast(Any, sql)) diff --git a/pyproject.toml b/pyproject.toml index 12c6ed4..abf8600 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -26,23 +26,20 @@ cli = [ [project.scripts] pgdb = "pgdevkit.cli:app" +[tool.pgdevkit] +name = "pgdevkit" + [dependency-groups] dev = [ "ty>=0.0.59", ] test = [ - "docker>=7.1.0", "psycopg[binary]>=3.2.0", "pytest>=9.1.0", + "pytest-asyncio>=0.24.0", "pytest-env>=1.1.0", ] [tool.pytest.ini_options] pythonpath = ["."] - -[tool.pytest_env] -PGDB_TEST_POSTGRES_HOST = "localhost" -PGDB_TEST_POSTGRES_PORT = "54326" -PGDB_TEST_POSTGRES_DB = "pgdb_test" -PGDB_TEST_POSTGRES_USER = "postgres" -PGDB_TEST_POSTGRES_PASSWORD = "testpwd" +asyncio_mode = "auto" diff --git a/tests/conftest.py b/tests/conftest.py index 00a8fcf..7bf0783 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,80 +1,20 @@ from __future__ import annotations -import os -import time -from pathlib import Path from typing import Any import psycopg import pytest -TEST_PORT = int(os.environ.get("PGDB_TEST_POSTGRES_PORT", "54326")) -TEST_DB = os.environ.get("PGDB_TEST_POSTGRES_DB", "pgdb_test") -TEST_USER = os.environ.get("PGDB_TEST_POSTGRES_USER", "postgres") -TEST_PASSWORD = os.environ.get("PGDB_TEST_POSTGRES_PASSWORD", "testpwd") -TEST_HOST = os.environ.get("PGDB_TEST_POSTGRES_HOST", "localhost") -CONTAINER_NAME = "pgdb_postgres4test" - -DSN = f"postgresql://{TEST_USER}:{TEST_PASSWORD}@{TEST_HOST}:{TEST_PORT}/{TEST_DB}" - - -def _start_docker() -> None: - import docker # type: ignore - import docker.errors # type: ignore - - client = docker.from_env() - try: - existing = client.containers.get(CONTAINER_NAME) - if existing.status != "running": - existing.start() - return - except docker.errors.NotFound: - pass - - client.containers.run( - "postgres:17", - name=CONTAINER_NAME, - detach=True, - ports={"5432/tcp": TEST_PORT}, - environment={ - "POSTGRES_PASSWORD": TEST_PASSWORD, - "POSTGRES_DB": TEST_DB, - "POSTGRES_USER": TEST_USER, - }, - ) - - -def _wait_for_postgres(timeout: int = 30) -> None: - admin_dsn = f"postgresql://{TEST_USER}:{TEST_PASSWORD}@{TEST_HOST}:{TEST_PORT}/postgres" - for _ in range(timeout): - try: - with psycopg.connect(admin_dsn, connect_timeout=2): - return - except Exception: - time.sleep(1) - raise RuntimeError("Postgres did not become ready in time") - - -def _ensure_test_db() -> None: - from psycopg import sql - - admin_dsn = f"postgresql://{TEST_USER}:{TEST_PASSWORD}@{TEST_HOST}:{TEST_PORT}/postgres" - with psycopg.connect(admin_dsn, autocommit=True) as conn: - exists = conn.execute( - sql.SQL("SELECT 1 FROM pg_database WHERE datname = %s"), (TEST_DB,) - ).fetchone() - if not exists: - conn.execute(sql.SQL("CREATE DATABASE {}").format(sql.Identifier(TEST_DB))) +from pgdevkit.testdb import ensure_testdb @pytest.fixture(scope="session") def postgres_dsn() -> str: - if os.environ.get("SKIP_START_POSTGRES") == "1": - return DSN - _start_docker() - _wait_for_postgres() - _ensure_test_db() - return DSN + env = ensure_testdb() + return ( + f"postgresql://{env['PGDEVKIT_POSTGRES_USER']}:{env['PGDEVKIT_POSTGRES_PASSWORD']}" + f"@{env['PGDEVKIT_POSTGRES_HOST']}:{env['PGDEVKIT_POSTGRES_PORT']}/{env['PGDEVKIT_POSTGRES_DB']}" + ) @pytest.fixture diff --git a/tests/fixtures/04_views.sql b/tests/fixtures/04_views.sql index 9092529..3b753ea 100644 --- a/tests/fixtures/04_views.sql +++ b/tests/fixtures/04_views.sql @@ -1,2 +1,2 @@ CREATE OR REPLACE VIEW myapp.active_users AS -SELECT id, email FROM myapp.users WHERE status = 'active'; +SELECT id FROM myapp.users; diff --git a/tests/testdb/__init__.py b/tests/testdb/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/testdb/conftest.py b/tests/testdb/conftest.py new file mode 100644 index 0000000..e7f118a --- /dev/null +++ b/tests/testdb/conftest.py @@ -0,0 +1,42 @@ +from __future__ import annotations + +import shutil +import subprocess +from pathlib import Path +from typing import Callable + +import pytest + +requires_podman = pytest.mark.skipif( + shutil.which("podman") is None, reason="podman is not installed" +) + +FIXTURES = Path(__file__).parent / "fixtures" / "database" + + +def _make_project(base: Path, name: str, branch: str) -> Path: + project = base / f"{name}-{branch}" + project.mkdir() + (project / "database").symlink_to(FIXTURES) + (project / "pyproject.toml").write_text( + f'[tool.pgdevkit]\nname = "{name}"\n', encoding="utf-8" + ) + for cmd in ( + ["git", "init", "-q"], + ["git", "config", "user.email", "test@example.com"], + ["git", "config", "user.name", "test"], + ["git", "checkout", "-q", "-b", branch], + ): + subprocess.run(cmd, cwd=project, check=True) + (project / ".gitkeep").write_text("", encoding="utf-8") + subprocess.run(["git", "add", "."], cwd=project, check=True) + subprocess.run(["git", "commit", "-q", "-m", "init"], cwd=project, check=True) + return project + + +@pytest.fixture +def project_factory(tmp_path: Path) -> Callable[[str, str], Path]: + def _factory(name: str, branch: str) -> Path: + return _make_project(tmp_path, name, branch) + + return _factory diff --git a/tests/testdb/fixtures/database/app/tables/widget.sql b/tests/testdb/fixtures/database/app/tables/widget.sql new file mode 100644 index 0000000..3c31956 --- /dev/null +++ b/tests/testdb/fixtures/database/app/tables/widget.sql @@ -0,0 +1,4 @@ +CREATE TABLE IF NOT EXISTS app.widget ( + id serial PRIMARY KEY, + name text NOT NULL +); diff --git a/tests/testdb/fixtures/database/app/tables/widget.test_data.json b/tests/testdb/fixtures/database/app/tables/widget.test_data.json new file mode 100644 index 0000000..dbd8dc2 --- /dev/null +++ b/tests/testdb/fixtures/database/app/tables/widget.test_data.json @@ -0,0 +1 @@ +[{"id": 1, "name": "sprocket"}] diff --git a/tests/testdb/fixtures/database/app/tables/widget_part.sql b/tests/testdb/fixtures/database/app/tables/widget_part.sql new file mode 100644 index 0000000..c32a306 --- /dev/null +++ b/tests/testdb/fixtures/database/app/tables/widget_part.sql @@ -0,0 +1,4 @@ +CREATE TABLE IF NOT EXISTS app.widget_part ( + id serial PRIMARY KEY, + widget_id integer NOT NULL REFERENCES app.widget(id) +); diff --git a/tests/testdb/fixtures/database/app/tables/widget_part_detail.sql b/tests/testdb/fixtures/database/app/tables/widget_part_detail.sql new file mode 100644 index 0000000..1f1cf3b --- /dev/null +++ b/tests/testdb/fixtures/database/app/tables/widget_part_detail.sql @@ -0,0 +1,4 @@ +CREATE TABLE IF NOT EXISTS app.widget_part_detail ( + id serial PRIMARY KEY, + widget_part_id integer NOT NULL REFERENCES app.widget_part(id) +); diff --git a/tests/testdb/fixtures/database/schema/app.sql b/tests/testdb/fixtures/database/schema/app.sql new file mode 100644 index 0000000..adc7f94 --- /dev/null +++ b/tests/testdb/fixtures/database/schema/app.sql @@ -0,0 +1 @@ +CREATE SCHEMA IF NOT EXISTS app; diff --git a/tests/testdb/test_api.py b/tests/testdb/test_api.py new file mode 100644 index 0000000..c06dd87 --- /dev/null +++ b/tests/testdb/test_api.py @@ -0,0 +1,111 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Callable + +import psycopg +import pytest + +from pgdevkit.testdb import constants +from pgdevkit.testdb.api import clean_testdb, ensure_testdb, reset_testdb, status +from tests.testdb.conftest import requires_podman + + +def _admin_dsn() -> str: + return f"postgresql://{constants.USER}:{constants.PASSWORD}@{constants.HOST}:{constants.PORT}/postgres" + + +@requires_podman +def test_ensure_testdb_isolates_by_branch(project_factory: Callable[[str, str], Path]): + project_a = project_factory("apitest", "main") + project_b = project_factory("apitest", "feature") + try: + env_a = ensure_testdb(project_a) + env_b = ensure_testdb(project_b) + + assert env_a["APITEST_POSTGRES_DB"] != env_b["APITEST_POSTGRES_DB"] + assert env_a["APITEST_POSTGRES_DB"] == status(project_a)["database"] + finally: + clean_testdb(project_a) + clean_testdb(project_b) + + +@requires_podman +def test_clean_all_removes_every_branch_database(project_factory: Callable[[str, str], Path]): + project_a = project_factory("apitest2", "main") + project_b = project_factory("apitest2", "feature") + ensure_testdb(project_a) + ensure_testdb(project_b) + + clean_testdb(project_a, all=True) + + with psycopg.connect(_admin_dsn()) as con: + with con.cursor() as cur: + cur.execute("SELECT count(*) FROM pg_database WHERE datname LIKE 'apitest2_%'") + (count,) = cur.fetchone() + assert count == 0 + + +@requires_podman +def test_clean_all_does_not_match_prefix_colliding_project_name( + project_factory: Callable[[str, str], Path], +): + # "apitestx" and "apitestxs" collide under an unescaped LIKE pattern: the + # pattern "apitestx_%" (built from the "apitestx_" prefix) would also + # match "apitestxs_main" because "_" is a single-character SQL wildcard + # that consumes the "s". clean_testdb(..., all=True) for "apitestx" must + # never touch "apitestxs"'s database. + project_short = project_factory("apitestx", "main") + project_long = project_factory("apitestxs", "main") + try: + ensure_testdb(project_short) + ensure_testdb(project_long) + + clean_testdb(project_short, all=True) + + with psycopg.connect(_admin_dsn()) as con: + with con.cursor() as cur: + cur.execute( + "SELECT count(*) FROM pg_database WHERE datname = %s", + (status(project_long)["database"],), + ) + (count,) = cur.fetchone() + assert count == 1 + finally: + clean_testdb(project_short) + clean_testdb(project_long) + + +@requires_podman +def test_reset_testdb_only_touches_own_database(project_factory: Callable[[str, str], Path]): + project_a = project_factory("apitest3", "main") + project_b = project_factory("apitest3", "other") + try: + ensure_testdb(project_a) + ensure_testdb(project_b) + + reset_testdb(project_a) # must not raise or affect project_b + + with psycopg.connect(_admin_dsn()) as con: + with con.cursor() as cur: + cur.execute( + "SELECT count(*) FROM pg_database WHERE datname = %s", + (status(project_b)["database"],), + ) + (count,) = cur.fetchone() + assert count == 1 + finally: + clean_testdb(project_a) + clean_testdb(project_b) + + +@requires_podman +def test_dsn_for_matches_status(project_factory: Callable[[str, str], Path]): + from pgdevkit.testdb.api import dsn_for + + project = project_factory("apitest4", "main") + try: + ensure_testdb(project) + assert dsn_for(project) == status(project)["dsn"] + finally: + clean_testdb(project) diff --git a/tests/testdb/test_cli.py b/tests/testdb/test_cli.py new file mode 100644 index 0000000..67bd12b --- /dev/null +++ b/tests/testdb/test_cli.py @@ -0,0 +1,78 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Callable + +from typer.testing import CliRunner + +from pgdevkit.cli import app +from pgdevkit.testdb import clean_testdb +from tests.testdb.conftest import requires_podman + +runner = CliRunner() + + +@requires_podman +def test_testdb_up_and_status(project_factory: Callable[[str, str], Path], monkeypatch): + project = project_factory("clitest", "main") + monkeypatch.chdir(project) + try: + result = runner.invoke(app, ["testdb", "up"]) + assert result.exit_code == 0, result.output + + result = runner.invoke(app, ["testdb", "status"]) + assert result.exit_code == 0, result.output + assert "database:" in result.output + finally: + clean_testdb(project) + + +@requires_podman +def test_testdb_run_sql_inline_with_results(project_factory: Callable[[str, str], Path], monkeypatch): + project = project_factory("clitest2", "main") + monkeypatch.chdir(project) + try: + runner.invoke(app, ["testdb", "up"]) + result = runner.invoke( + app, ["testdb", "run-sql", "--sql", "SELECT id, name FROM app.widget", "--results"] + ) + assert result.exit_code == 0, result.output + assert "sprocket" in result.output + finally: + clean_testdb(project) + + +@requires_podman +def test_testdb_run_sql_with_results_and_zero_rows(project_factory: Callable[[str, str], Path], monkeypatch): + project = project_factory("clitest5", "main") + monkeypatch.chdir(project) + try: + runner.invoke(app, ["testdb", "up"]) + result = runner.invoke( + app, ["testdb", "run-sql", "--sql", "SELECT id, name FROM app.widget WHERE false", "--results"] + ) + assert result.exit_code == 0, result.output + assert "0 row" in result.output + finally: + clean_testdb(project) + + +@requires_podman +def test_testdb_reset(project_factory: Callable[[str, str], Path], monkeypatch): + project = project_factory("clitest3", "main") + monkeypatch.chdir(project) + try: + runner.invoke(app, ["testdb", "up"]) + result = runner.invoke(app, ["testdb", "reset"]) + assert result.exit_code == 0, result.output + finally: + clean_testdb(project) + + +@requires_podman +def test_testdb_clean(project_factory: Callable[[str, str], Path], monkeypatch): + project = project_factory("clitest4", "main") + monkeypatch.chdir(project) + runner.invoke(app, ["testdb", "up"]) + result = runner.invoke(app, ["testdb", "clean"]) + assert result.exit_code == 0, result.output diff --git a/tests/testdb/test_config.py b/tests/testdb/test_config.py new file mode 100644 index 0000000..b865fde --- /dev/null +++ b/tests/testdb/test_config.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +from pathlib import Path + +from pgdevkit.testdb.config import load_config + + +def test_defaults_when_no_pyproject(tmp_path: Path): + project = tmp_path / "myproj" + project.mkdir() + config = load_config(project) + assert config.name == "myproj" + assert config.database_dir == "database" + assert config.env_prefix == "MYPROJ_" + assert config.extensions == () + assert config.root == project + + +def test_reads_tool_pgdevkit_section(tmp_path: Path): + (tmp_path / "pyproject.toml").write_text( + """ +[tool.pgdevkit] +name = "mdmapp" +database_dir = "db" +env_prefix = "MDM_" +extensions = ["vector"] +""", + encoding="utf-8", + ) + config = load_config(tmp_path) + assert config.name == "mdmapp" + assert config.database_dir == "db" + assert config.env_prefix == "MDM_" + assert config.extensions == ("vector",) + + +def test_env_prefix_defaults_from_name(tmp_path: Path): + (tmp_path / "pyproject.toml").write_text( + '[tool.pgdevkit]\nname = "ccmt"\n', encoding="utf-8" + ) + config = load_config(tmp_path) + assert config.env_prefix == "CCMT_" + + +def test_searches_upward_for_pyproject(tmp_path: Path): + (tmp_path / "pyproject.toml").write_text( + '[tool.pgdevkit]\nname = "root_project"\n', encoding="utf-8" + ) + subdir = tmp_path / "src" / "nested" + subdir.mkdir(parents=True) + config = load_config(subdir) + assert config.name == "root_project" + assert config.root == tmp_path diff --git a/tests/testdb/test_container.py b/tests/testdb/test_container.py new file mode 100644 index 0000000..e4b90a4 --- /dev/null +++ b/tests/testdb/test_container.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +import psycopg + +from pgdevkit.testdb import constants +from pgdevkit.testdb.container import ensure_container +from tests.testdb.conftest import requires_podman + + +def _admin_dsn() -> str: + return ( + f"postgresql://{constants.USER}:{constants.PASSWORD}" + f"@{constants.HOST}:{constants.PORT}/postgres?connect_timeout=5" + ) + + +@requires_podman +def test_ensure_container_starts_and_accepts_connections(): + ensure_container() + + with psycopg.connect(_admin_dsn()) as con: + with con.cursor() as cur: + cur.execute("SELECT 1") + assert cur.fetchone() == (1,) + + +@requires_podman +def test_ensure_container_is_idempotent(): + ensure_container() + ensure_container() # must not raise, must not error on "name already in use" + + with psycopg.connect(_admin_dsn()) as con: + with con.cursor() as cur: + cur.execute("SELECT 1") + assert cur.fetchone() == (1,) diff --git a/tests/testdb/test_naming.py b/tests/testdb/test_naming.py new file mode 100644 index 0000000..f394bf4 --- /dev/null +++ b/tests/testdb/test_naming.py @@ -0,0 +1,51 @@ +from __future__ import annotations + +import subprocess +from pathlib import Path + +from pgdevkit.testdb.naming import current_branch, slugify, workspace_db_name + + +def test_slugify_lowercases_and_replaces_invalid_chars(): + assert slugify("MDMApp") == "mdmapp" + assert slugify("feature/customer-contacts") == "feature_customer_contacts" + + +def test_slugify_strips_leading_trailing_underscores(): + assert slugify("--hello--") == "hello" + + +def test_slugify_truncates_long_input_and_appends_hash(): + long_name = "a" * 50 + result = slugify(long_name) + assert len(result) == 30 + 1 + 8 + assert result.startswith("a" * 30 + "_") + + +def test_slugify_is_deterministic(): + long_name = "worktree-procrastinate-job-events-extra-long-branch-name" + assert slugify(long_name) == slugify(long_name) + + +def test_workspace_db_name_differs_by_branch(): + a = workspace_db_name("mdmapp", "main") + b = workspace_db_name("mdmapp", "multi_lng") + assert a != b + assert a == "mdmapp_main" + + +def test_workspace_db_name_stays_under_postgres_identifier_limit(): + name = workspace_db_name("a" * 50, "b" * 50) + assert len(name) <= 63 + + +def test_current_branch_reads_the_checked_out_branch(tmp_path: Path): + subprocess.run(["git", "init", "-q"], cwd=tmp_path, check=True) + subprocess.run(["git", "config", "user.email", "t@example.com"], cwd=tmp_path, check=True) + subprocess.run(["git", "config", "user.name", "t"], cwd=tmp_path, check=True) + subprocess.run(["git", "checkout", "-q", "-b", "my-feature"], cwd=tmp_path, check=True) + (tmp_path / "f.txt").write_text("x", encoding="utf-8") + subprocess.run(["git", "add", "."], cwd=tmp_path, check=True) + subprocess.run(["git", "commit", "-q", "-m", "init"], cwd=tmp_path, check=True) + + assert current_branch(tmp_path) == "my-feature" diff --git a/tests/testdb/test_query.py b/tests/testdb/test_query.py new file mode 100644 index 0000000..9c6b3db --- /dev/null +++ b/tests/testdb/test_query.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +import psycopg +import pytest + +from pgdevkit.testdb import constants, query +from pgdevkit.testdb.container import ensure_container +from tests.testdb.conftest import requires_podman + +TEST_DB = "pgdevkit_query_selftest" + + +def _admin_dsn() -> str: + return f"postgresql://{constants.USER}:{constants.PASSWORD}@{constants.HOST}:{constants.PORT}/postgres" + + +def _db_dsn() -> str: + return f"postgresql://{constants.USER}:{constants.PASSWORD}@{constants.HOST}:{constants.PORT}/{TEST_DB}" + + +@pytest.fixture +def query_test_db(): + ensure_container() + with psycopg.connect(_admin_dsn(), autocommit=True) as con: + con.execute(f'DROP DATABASE IF EXISTS "{TEST_DB}"') + con.execute(f'CREATE DATABASE "{TEST_DB}"') + yield + with psycopg.connect(_admin_dsn(), autocommit=True) as con: + con.execute(f'DROP DATABASE IF EXISTS "{TEST_DB}"') + + +@requires_podman +async def test_execute_returns_rows_for_select(query_test_db): + rows = await query.execute(_db_dsn(), "SELECT 1 AS one, 2 AS two") + assert rows == [{"one": 1, "two": 2}] + + +@requires_podman +async def test_execute_returns_none_for_ddl(query_test_db): + rows = await query.execute(_db_dsn(), "CREATE TABLE t (id int)") + assert rows is None + + +@requires_podman +async def test_execute_runs_multiple_statements_and_returns_last(query_test_db): + rows = await query.execute( + _db_dsn(), + "CREATE TABLE t2 (id int); INSERT INTO t2 VALUES (1); SELECT * FROM t2", + ) + assert rows == [{"id": 1}] diff --git a/tests/testdb/test_schema.py b/tests/testdb/test_schema.py new file mode 100644 index 0000000..4f00ed0 --- /dev/null +++ b/tests/testdb/test_schema.py @@ -0,0 +1,83 @@ +from __future__ import annotations + +from pathlib import Path + +import psycopg +import pytest + +from pgdevkit.testdb import constants +from pgdevkit.testdb.container import ensure_container +from pgdevkit.testdb.schema import apply_schema +from tests.testdb.conftest import requires_podman + +FIXTURES = Path(__file__).parent / "fixtures" / "database" +TEST_DB = "pgdevkit_schema_selftest" + + +def _admin_dsn() -> str: + return f"postgresql://{constants.USER}:{constants.PASSWORD}@{constants.HOST}:{constants.PORT}/postgres" + + +def _db_dsn() -> str: + return f"postgresql://{constants.USER}:{constants.PASSWORD}@{constants.HOST}:{constants.PORT}/{TEST_DB}" + + +@pytest.fixture +def schema_test_db(): + ensure_container() + with psycopg.connect(_admin_dsn(), autocommit=True) as con: + con.execute(f'DROP DATABASE IF EXISTS "{TEST_DB}"') + con.execute(f'CREATE DATABASE "{TEST_DB}"') + yield + with psycopg.connect(_admin_dsn(), autocommit=True) as con: + con.execute(f'DROP DATABASE IF EXISTS "{TEST_DB}"') + + +@requires_podman +async def test_apply_schema_creates_tables_and_seeds_data(schema_test_db): + async with await psycopg.AsyncConnection.connect(_db_dsn(), autocommit=True) as con: + await apply_schema(con, FIXTURES) + async with con.cursor() as cur: + await cur.execute("SELECT id, name FROM app.widget ORDER BY id") + rows = await cur.fetchall() + assert rows == [(1, "sprocket")] + + +@requires_podman +async def test_apply_schema_is_idempotent(schema_test_db): + async with await psycopg.AsyncConnection.connect(_db_dsn(), autocommit=True) as con: + await apply_schema(con, FIXTURES) + await apply_schema(con, FIXTURES) # must not raise + async with con.cursor() as cur: + await cur.execute("SELECT count(*) FROM app.widget") + (count,) = await cur.fetchone() + assert count == 1 + + +@requires_podman +async def test_apply_schema_on_missing_directory_is_a_noop(schema_test_db): + async with await psycopg.AsyncConnection.connect(_db_dsn(), autocommit=True) as con: + await apply_schema(con, FIXTURES.parent / "does_not_exist") # must not raise + + +@requires_podman +async def test_apply_schema_resolves_multi_level_fk_dependency(schema_test_db): + # widget_part_detail REFERENCES widget_part REFERENCES widget, and their + # filenames already sort alphabetically in that same dependency order. + # This is the layout that exposed a real bug: _iter_sql_files derived the + # schema/table name from the wrong path component (file.parent.parent.name + # instead of file.parent.name for a database/{schema}/tables/{name}.sql + # layout), so table dependency tracking never activated and files were + # applied in filename order, which for this fixture is the exact reverse + # of the required FK order. + async with await psycopg.AsyncConnection.connect(_db_dsn(), autocommit=True) as con: + await apply_schema(con, FIXTURES) # must not raise + async with con.cursor() as cur: + await cur.execute("INSERT INTO app.widget_part (widget_id) VALUES (1) RETURNING id") + (part_id,) = await cur.fetchone() + await cur.execute( + "INSERT INTO app.widget_part_detail (widget_part_id) VALUES (%s) RETURNING id", + (part_id,), + ) + (detail_id,) = await cur.fetchone() + assert detail_id == 1 diff --git a/uv.lock b/uv.lock index 06668ff..7121098 100644 --- a/uv.lock +++ b/uv.lock @@ -182,20 +182,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c2/e6/f60198ea8d9dfa15fff9ed4ca02ce362f6eadd9ba757dcc50634c4257b63/cryptography-49.0.0-cp39-abi3-win_amd64.whl", hash = "sha256:026ac7423e6fa66872d3bf889be5974507da3944f866f704fa200eadacd00001", size = 3785547, upload-time = "2026-06-12T20:02:26.847Z" }, ] -[[package]] -name = "docker" -version = "7.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pywin32", marker = "sys_platform == 'win32'" }, - { name = "requests" }, - { name = "urllib3" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/91/9b/4a2ea29aeba62471211598dac5d96825bb49348fa07e906ea930394a83ce/docker-7.1.0.tar.gz", hash = "sha256:ad8c70e6e3f8926cb8a92619b832b4ea5299e2831c14284663184e200546fa6c", size = 117834, upload-time = "2024-05-23T11:13:57.216Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e3/26/57c6fb270950d476074c087527a558ccb6f4436657314bfb6cdf484114c4/docker-7.1.0-py3-none-any.whl", hash = "sha256:c96b93b7f0a746f9e77d325bcfb87422a3d8bd4f03136ae8a85b37f1898d5fc0", size = 147774, upload-time = "2024-05-23T11:13:55.01Z" }, -] - [[package]] name = "idna" version = "3.18" @@ -293,9 +279,9 @@ dev = [ { name = "ty" }, ] test = [ - { name = "docker" }, { name = "psycopg", extra = ["binary"] }, { name = "pytest" }, + { name = "pytest-asyncio" }, { name = "pytest-env" }, ] @@ -312,9 +298,9 @@ provides-extras = ["azure", "cli"] [package.metadata.requires-dev] dev = [{ name = "ty", specifier = ">=0.0.59" }] test = [ - { name = "docker", specifier = ">=7.1.0" }, { name = "psycopg", extras = ["binary"], specifier = ">=3.2.0" }, { name = "pytest", specifier = ">=9.1.0" }, + { name = "pytest-asyncio", specifier = ">=0.24.0" }, { name = "pytest-env", specifier = ">=1.1.0" }, ] @@ -410,6 +396,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/8b/5a/ba30a81239b909821b3153e303e7def45178bf353da4f72380e6c5e8793b/pytest-9.1.0-py3-none-any.whl", hash = "sha256:8ebb0e7888bdf2bdfc602ec51f8f62d50200af37356c74e503c79a94f5c81f32", size = 386453, upload-time = "2026-06-13T18:52:44.045Z" }, ] +[[package]] +name = "pytest-asyncio" +version = "1.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/43/7c/d36d04db312ecf4298932ef77e6e4a9e8ad017906e24e34f0b0c361a2473/pytest_asyncio-1.4.0.tar.gz", hash = "sha256:c6c0d2259945122819f171a32ecea2c349ead889ee28176caaf492143424be42", size = 58514, upload-time = "2026-05-26T09:56:04.083Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/03/e2/08a497ef684b88559c9cc5f4ad53a37e7b99e727094a86d6ea32536d5d3c/pytest_asyncio-1.4.0-py3-none-any.whl", hash = "sha256:933ca923a23075a87fb7070c0ec272a6848489824d887c85c812670932835aa1", size = 16930, upload-time = "2026-05-26T09:56:02.576Z" }, +] + [[package]] name = "pytest-env" version = "1.6.0" @@ -432,19 +430,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0b/d7/1959b9648791274998a9c3526f6d0ec8fd2233e4d4acce81bbae76b44b2a/python_dotenv-1.2.2-py3-none-any.whl", hash = "sha256:1d8214789a24de455a8b8bd8ae6fe3c6b69a5e3d64aa8a8e5d68e694bbcb285a", size = 22101, upload-time = "2026-03-01T16:00:25.09Z" }, ] -[[package]] -name = "pywin32" -version = "312" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fc/2b/1f3cded5822fd49c02f40544cbb5f58c7cfd6b1694869fd476cb6170ee97/pywin32-312-cp314-cp314-win32.whl", hash = "sha256:a77a90fbb6881238d2ca9c6fd797b25817f3768fe78d214a90137ff055a75f5b", size = 6468928, upload-time = "2026-06-04T07:49:43.188Z" }, - { url = "https://files.pythonhosted.org/packages/21/82/3bf86d2e2808902013132e1ce905a7da0da53790f3836c64bf44d55e24f3/pywin32-312-cp314-cp314-win_amd64.whl", hash = "sha256:a4dd3a848290ef724347b19f301045831d8e802fa4464f491b98b1e0a081432e", size = 7024157, upload-time = "2026-06-04T07:49:45.34Z" }, - { url = "https://files.pythonhosted.org/packages/a4/0e/73f6d6800b4f27655abd9e9f6aaeaefcddb2b946e4674efa2bab184a7f7b/pywin32-312-cp314-cp314-win_arm64.whl", hash = "sha256:9fce94568364e0155e6dfb781ac5d95903be8baf28670632beab1b523f300daa", size = 6839598, upload-time = "2026-06-04T07:49:47.613Z" }, - { url = "https://files.pythonhosted.org/packages/eb/61/caa39686032d2ebdd04ff0ab5cbe163126c0066d98e00c9018646e42393b/pywin32-312-cp315-cp315-win32.whl", hash = "sha256:5c1fbe4a937a73ae9297384a3da38518cbc694c68ad8a809b2e19acd350f03ed", size = 6471159, upload-time = "2026-06-04T07:49:50.035Z" }, - { url = "https://files.pythonhosted.org/packages/0f/cd/7e1de64a4a6f69c04214169657ccab0d93a670ea50e35eb8f489d7378249/pywin32-312-cp315-cp315-win_amd64.whl", hash = "sha256:c2f03a0f73f804a13c2735b99392b0cd426bb4f2c4d0178e5ac966a0f21618d5", size = 7025293, upload-time = "2026-06-04T07:49:54.857Z" }, - { url = "https://files.pythonhosted.org/packages/23/ed/4532e9388e65fa16b46776ef47ad631a64eda1631884488af707666350ed/pywin32-312-cp315-cp315-win_arm64.whl", hash = "sha256:a8597d28f267b39074aef51fa593530082b39cbe5a074226096857b1fed2dfb9", size = 6840337, upload-time = "2026-06-04T07:49:57.531Z" }, -] - [[package]] name = "requests" version = "2.34.2"