Use the Docker API instead of shelling out to podman; fix schema-appl…

.github/workflows/python-test.yml

@@ -24,8 +24,6 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
-      - name: Ensure podman is installed
-        run: command -v podman || (sudo apt-get update && sudo apt-get install -y podman)
       - name: Install uv
         run: curl -LsSf https://astral.sh/uv/install.sh | sh
       - name: Install project dependencies

README.md

@@ -70,15 +70,20 @@ CLI: `pgdb testdb up|reset|run-sql|status|shell|clean`.
 
 Container connection defaults (`localhost:54322`, `postgres`/`testpwd`) can
 be overridden with `PGDEVKIT_TESTDB_HOST`, `PGDEVKIT_TESTDB_PORT`,
-`PGDEVKIT_TESTDB_USER`, `PGDEVKIT_TESTDB_PASSWORD`. Before touching
-podman/docker, pgdevkit first checks (with a short timeout) whether Postgres
+`PGDEVKIT_TESTDB_USER`, `PGDEVKIT_TESTDB_PASSWORD`. Before touching the
+Docker API, pgdevkit first checks (with a short timeout) whether Postgres
 is already reachable at that address and skips container management if so.
 Set `PGDEVKIT_SKIP_CONTAINER=1` to always assume it's already there and skip
 that check too.
 
-To point at a local Postgres install instead of the podman container —
-useful when podman isn't available, or you'd rather use peer authentication
-as the current OS user — set `PGDEVKIT_TESTDB_HOST` to the unix socket
+Container management goes through the Docker API (the `docker` package,
+`docker.from_env()`, falling back to Podman's rootful/rootless socket) — it
+works against a real Docker daemon or Podman transparently, no CLI binary
+required either way.
+
+To point at a local Postgres install instead of the container — useful when
+neither is available, or you'd rather use peer authentication as the
+current OS user — set `PGDEVKIT_TESTDB_HOST` to the unix socket
 directory (e.g. `/var/run/postgresql`) and `PGDEVKIT_TESTDB_PASSWORD=""`.
 The role named by `PGDEVKIT_TESTDB_USER` must exist and match your OS user
 (`CREATE ROLE <user> SUPERUSER LOGIN;`) and `pg_hba.conf` must allow `peer`

pgdevkit/testdb/container.py

@@ -1,23 +1,53 @@
 from __future__ import annotations
 
 import os
-import subprocess
 import time
 
+import docker
+import docker.errors
 import psycopg
 
 from . import constants
 
+# Candidate Docker-API-compatible socket URLs tried after plain
+# docker.from_env() (which only looks at DOCKER_HOST / the default Docker
+# socket) fails to connect -- covers rootful and rootless Podman, which
+# speaks the same API but doesn't always advertise itself via DOCKER_HOST.
+_FALLBACK_SOCKET_URLS = [
+    f"unix://{os.environ['XDG_RUNTIME_DIR']}/podman/podman.sock" if os.environ.get("XDG_RUNTIME_DIR") else None,
+    "unix:///run/podman/podman.sock",
+]
 
-def _podman(*args: str, check: bool = True) -> subprocess.CompletedProcess:
-    return subprocess.run(["podman", *args], capture_output=True, text=True, check=check)
+
+def _client() -> docker.DockerClient:
+    """A Docker-API client, working against a real Docker daemon or a
+    Podman one (Podman exposes the same API over its own socket) -- callers
+    never need to know or care which one is actually running."""
+    try:
+        client = docker.from_env()
+        client.ping()
+        return client
+    except Exception:  # noqa: BLE001
+        pass
+    for base_url in _FALLBACK_SOCKET_URLS:
+        if base_url is None:
+            continue
+        try:
+            client = docker.DockerClient(base_url=base_url)
+            client.ping()
+            return client
+        except Exception:  # noqa: BLE001
+            continue
+    raise RuntimeError(
+        "Could not reach a Docker-compatible API. Set DOCKER_HOST, or make sure "
+        "Docker or Podman's API socket is running."
+    )
 
 
 def _available(timeout: float = 3.0) -> bool:
     """Quick check (short timeout) for whether Postgres is already reachable
     at HOST:PORT, so a database started outside pgdevkit's control (or the
-    container from a previous run) doesn't trigger another podman/docker
-    lifecycle call."""
+    container from a previous run) doesn't trigger another Docker API call."""
     # libpq's connect_timeout is whole seconds; anything below 1 means "wait
     # indefinitely" instead of a short timeout, so it's clamped up to 1.
     connect_timeout = max(1, round(timeout))
@@ -29,32 +59,24 @@ def _available(timeout: float = 3.0) -> bool:
         return False
 
 
-def _container_status() -> str | None:
-    """Return 'running', 'exited', etc., or None if the container doesn't exist."""
-    result = _podman(
-        "inspect", constants.CONTAINER_NAME, "--format", "{{.State.Status}}", check=False
-    )
-    if result.returncode != 0:
-        return None
-    return result.stdout.strip()
-
-
-def _create_container() -> None:
-    result = _podman(
-        "run", "-d",
-        "--name", constants.CONTAINER_NAME,
-        "-p", f"{constants.PORT}:5432",
-        "-e", f"POSTGRES_USER={constants.USER}",
-        "-e", f"POSTGRES_PASSWORD={constants.PASSWORD}",
-        constants.IMAGE,
-        "postgres", *constants.PG_SPEED_FLAGS,
-        check=False,
-    )
-    if result.returncode != 0 and "already in use" in result.stderr:
-        _podman("start", constants.CONTAINER_NAME)
-        return
-    if result.returncode != 0:
-        raise RuntimeError(f"podman run failed: {result.stderr}")
+def _create_container(client: docker.DockerClient) -> None:
+    try:
+        client.containers.run(
+            constants.IMAGE,
+            name=constants.CONTAINER_NAME,
+            detach=True,
+            ports={"5432/tcp": constants.PORT},
+            environment={
+                "POSTGRES_USER": constants.USER,
+                "POSTGRES_PASSWORD": constants.PASSWORD,
+            },
+            command=["postgres", *constants.PG_SPEED_FLAGS],
+        )
+    except docker.errors.APIError as e:
+        if getattr(e, "status_code", None) == 409 or "already in use" in str(e):
+            client.containers.get(constants.CONTAINER_NAME).start()
+            return
+        raise RuntimeError(f"Starting the {constants.CONTAINER_NAME} container failed: {e}") from e
 
 
 def _wait_ready(timeout: float = 30.0) -> None:
@@ -73,17 +95,20 @@ def _wait_ready(timeout: float = 30.0) -> None:
 
 def ensure_container() -> None:
     """Idempotently ensure the shared pgdevkit-postgres container is running
-    and accepting connections. Never touches podman/docker if Postgres is
+    and accepting connections. Never touches the Docker API if Postgres is
     already reachable, or if PGDEVKIT_SKIP_CONTAINER says to assume it is."""
     if os.environ.get("PGDEVKIT_SKIP_CONTAINER"):
         return
     if _available():
         return
-    status = _container_status()
-    if status == "running":
-        return
-    if status is not None:
-        _podman("start", constants.CONTAINER_NAME)
+    client = _client()
+    try:
+        container = client.containers.get(constants.CONTAINER_NAME)
+    except docker.errors.NotFound:
+        container = None
+    if container is not None:
+        if container.status != "running":
+            container.start()
     else:
-        _create_container()
+        _create_container(client)
     _wait_ready()

pgdevkit/testdb/schema.py

@@ -63,8 +63,31 @@ def _strip_layer_prefix(schema_name: str) -> str:
 }
 
 
+_DECLARE_RE = re.compile(
+    r"CREATE\s+(?:OR\s+REPLACE\s+)?(?:TABLE|VIEW|FUNCTION|PROCEDURE|TYPE|SCHEMA)\s+(\w+\.\w+)", re.IGNORECASE
+)
+_DEPEND_RE = re.compile(r"(?:FROM|JOIN|INTO|UPDATE|TABLE|ON)\s+(\w+\.\w+)", re.IGNORECASE)
+
+
+def _get_sql_deps_regex_fallback(sql: str) -> set[str]:
+    """Crude regex scan used when sqlglot can't parse a statement even with
+    error_level=IGNORE. This only feeds dependency *ordering* (which file to
+    apply first), not execution, so an imprecise-but-safe approximation here
+    is fine."""
+    declares = set(_DECLARE_RE.findall(sql))
+    deps = set(_DEPEND_RE.findall(sql))
+    return deps - declares
+
+
 def _get_sql_deps(sql: str) -> set[str]:
-    exprs = sqlglot.parse(sql, dialect="postgres")
+    try:
+        # error_level=IGNORE lets sqlglot recover from statements it can't
+        # fully parse (e.g. a schema-qualified `DROP TRIGGER ... ON
+        # schema.table`) and keep going, instead of raising and losing every
+        # other statement's dependency info in the same file.
+        exprs = sqlglot.parse(sql, dialect="postgres", error_level=sqlglot.ErrorLevel.IGNORE)
+    except Exception:  # noqa: BLE001
+        return _get_sql_deps_regex_fallback(sql)
     deps: set[str] = set()
     for e in exprs:
         if e is None:
@@ -206,5 +229,21 @@ async def _apply(file: Path, sql: str) -> None:
             logger.warning("Error executing %s (will retry): %s", file, e)
             failures.append((file, sql))
 
-    for file, sql in failures:
-        await _apply(file, sql)
+    # Retry the whole failed set, not just once: a single extra pass in
+    # original order can still raise on an item whose dependency is later
+    # in the same list and hasn't had its own retry yet. Every CREATE here
+    # is idempotent, so looping until a full pass makes no progress
+    # converges on any resolvable ordering without re-doing finished work.
+    while failures:
+        still_failing: list[tuple[Path, str]] = []
+        last_error: Exception | None = None
+        for file, sql in failures:
+            try:
+                await _apply(file, sql)
+            except Exception as e:  # noqa: BLE001
+                still_failing.append((file, sql))
+                last_error = e
+        if len(still_failing) == len(failures):
+            assert last_error is not None
+            raise last_error
+        failures = still_failing

pyproject.toml

@@ -16,6 +16,7 @@ description = "A helper for developing with Postgres"
 readme = "README.md"
 requires-python = ">=3.14"
 dependencies = [
+    "docker>=7.1.0",
     "psycopg[binary]>=3.2.0",
     "sqlglot>=30.11.0",
 ]

skills/pgdevkit/SKILL.md

@@ -73,7 +73,7 @@ def _testdb_env():
         os.environ[key] = value
 ```
 
-`ensure_testdb()` starts the shared `pgdevkit-postgres` container if needed (via `podman`), creates a database scoped to this project+branch (so different worktrees/branches never collide), and applies every `.sql` file under `database_dir` in dependency order, seeding any `.test_data.json` sidecar files.
+`ensure_testdb()` starts the shared `pgdevkit-postgres` container if needed (via the Docker API — works against a real Docker daemon or Podman's socket, no CLI binary required), creates a database scoped to this project+branch (so different worktrees/branches never collide), and applies every `.sql` file under `database_dir` in dependency order, seeding any `.test_data.json` sidecar files.
 
 **`migrations/` is never applied here, on purpose.** If the test schema is missing something, that's a sign the base `tables/`/`views`/... file has drifted behind a migration that was only ever run manually against a real database — fix the base file, don't add migration-replay to `apply_schema()` (tried once, reverted: a migration can't be judged "safe to re-run" from its SQL text alone — see `docs/database-layout.md`'s Migrations section).
 
@@ -88,7 +88,7 @@ def _testdb_env():
 
 ### CI
 
-Podman needs to be available on the runner (`apt-get install -y podman` on `ubuntu-latest` if not preinstalled) — `ensure_testdb()`/`ensure_container()` shell out to it directly. There's no `PGDEVKIT_SKIP_CONTAINER` + service-container escape hatch wired through every fixture yet — if a project's CI can't run podman, it needs its own workaround for now.
+`ensure_testdb()`/`ensure_container()` talk to whatever Docker-compatible API is reachable (`DOCKER_HOST`, the default Docker socket, or Podman's socket as a fallback) — `ubuntu-latest`'s preinstalled Docker daemon just works, no setup step needed. There's no `PGDEVKIT_SKIP_CONTAINER` + service-container escape hatch wired through every fixture yet — if a project's CI has neither Docker nor Podman reachable, it needs its own workaround for now.
 
 ---
 

tests/testdb/conftest.py

@@ -1,15 +1,25 @@
 from __future__ import annotations
 
 import os
-import shutil
 import subprocess
 from pathlib import Path
 from typing import Callable
 
 import pytest
 
+
+def _has_container_runtime() -> bool:
+    from pgdevkit.testdb.container import _client
+
+    try:
+        _client()
+        return True
+    except Exception:  # noqa: BLE001
+        return False
+
+
 requires_podman = pytest.mark.skipif(
-    shutil.which("podman") is None, reason="podman is not installed"
+    not _has_container_runtime(), reason="no Docker-compatible API reachable"
 )
 
 FIXTURES = Path(__file__).parent / "fixtures" / "database"

tests/testdb/test_container.py

@@ -32,10 +32,10 @@ def test_ensure_container_skips_podman_when_already_available(monkeypatch):
     monkeypatch.setattr(container, "_available", lambda timeout=1.0: True)
 
     def _fail(*a, **k):
-        raise AssertionError("must not shell out to podman when already available")
+        raise AssertionError("must not touch the Docker API when already available")
 
-    monkeypatch.setattr(container, "_podman", _fail)
-    ensure_container()  # must return without calling podman
+    monkeypatch.setattr(container, "_client", _fail)
+    ensure_container()  # must return without calling the Docker API
 
 
 def test_ensure_container_skips_everything_when_skip_env_set(monkeypatch):
@@ -45,8 +45,8 @@ def _fail(*a, **k):
         raise AssertionError("must not run when PGDEVKIT_SKIP_CONTAINER is set")
 
     monkeypatch.setattr(container, "_available", _fail)
-    monkeypatch.setattr(container, "_podman", _fail)
-    ensure_container()  # must return without checking availability or calling podman
+    monkeypatch.setattr(container, "_client", _fail)
+    ensure_container()  # must return without checking availability or calling the Docker API
 
 
 def _admin_dsn() -> str:
@@ -78,7 +78,7 @@ def test_ensure_container_is_idempotent():
 def test_create_container_falls_back_to_start_when_name_in_use():
     ensure_container()  # container already exists under constants.CONTAINER_NAME
 
-    _create_container()  # "podman run" will fail with "already in use"; must fall back to "podman start"
+    _create_container(container._client())  # run will conflict; must fall back to start()
 
     with psycopg.connect(_admin_dsn()) as con:
         with con.cursor() as cur: