From c65b1efce3c79a66e0ed9364bdbb41893782fce4 Mon Sep 17 00:00:00 2001 From: Lachlan Walker Date: Mon, 8 Jun 2026 15:38:44 +0800 Subject: [PATCH 1/4] add testcontainers for DB setup --- .github/workflows/tests.yaml | 62 +--------- requirements/test-requirements.txt | 1 + tests/conftest.py | 191 +++++++++++++++++++++++++++-- 3 files changed, 186 insertions(+), 68 deletions(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 07491e85e..6d2ff2f40 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -42,18 +42,6 @@ jobs: # version. python-version: ["3.13"] postgres-version: [17] - services: - postgres: - image: postgres:${{ matrix.postgres-version }} - env: - POSTGRES_PASSWORD: postgres - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - - 5432:5432 steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -66,19 +54,10 @@ jobs: pip install -r requirements/requirements.txt pip install -r requirements/test-requirements.txt pip install -r requirements/extras/postgres.txt - - name: Setup postgres - run: | - export PGPASSWORD=postgres - psql -h localhost -c 'CREATE DATABASE piccolo;' -U postgres - psql -h localhost -c "CREATE USER piccolo PASSWORD 'piccolo';" -U postgres - psql -h localhost -c "GRANT ALL PRIVILEGES ON DATABASE piccolo TO piccolo;" -U postgres - psql -h localhost -c "CREATE EXTENSION IF NOT EXISTS \"uuid-ossp\";" -d piccolo -U postgres - name: Run integration tests run: ./scripts/test-integration.sh env: - PG_HOST: localhost - PG_DATABASE: piccolo - PG_PASSWORD: postgres + TC_POSTGRES_VERSION: ${{ matrix.postgres-version }} postgres: runs-on: ubuntu-latest @@ -88,24 +67,6 @@ jobs: python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"] postgres-version: [13, 14, 15, 16, 17, 18] - # Service containers to run with `container-job` - services: - # Label used to access the service container - postgres: - # Docker Hub image - image: postgres:${{ matrix.postgres-version }} - # Provide the password for postgres - env: - POSTGRES_PASSWORD: postgres - # Set health checks to wait until postgres has started - options: >- - --health-cmd pg_isready - --health-interval 10s - --health-timeout 5s - --health-retries 5 - ports: - - 5432:5432 - steps: - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} @@ -118,20 +79,10 @@ jobs: pip install -r requirements/requirements.txt pip install -r requirements/test-requirements.txt pip install -r requirements/extras/postgres.txt - - name: Setup postgres - run: | - export PGPASSWORD=postgres - psql -h localhost -c 'CREATE DATABASE piccolo;' -U postgres - psql -h localhost -c "CREATE USER piccolo PASSWORD 'piccolo';" -U postgres - psql -h localhost -c "GRANT ALL PRIVILEGES ON DATABASE piccolo TO piccolo;" -U postgres - psql -h localhost -c "CREATE EXTENSION IF NOT EXISTS \"uuid-ossp\";" -d piccolo -U postgres - - name: Test with pytest, Postgres run: ./scripts/test-postgres.sh env: - PG_HOST: localhost - PG_DATABASE: piccolo - PG_PASSWORD: postgres + TC_POSTGRES_VERSION: ${{ matrix.postgres-version }} - name: Upload coverage uses: codecov/codecov-action@v1 if: matrix.python-version == '3.13' @@ -155,17 +106,10 @@ jobs: pip install -r requirements/requirements.txt pip install -r requirements/test-requirements.txt pip install -r requirements/extras/postgres.txt - - name: Setup CockroachDB - run: | - wget -qO- https://binaries.cockroachdb.com/cockroach-${{ matrix.cockroachdb-version }}.linux-amd64.tgz | tar xz - ./cockroach-${{ matrix.cockroachdb-version }}.linux-amd64/cockroach start-single-node --insecure --background - ./cockroach-${{ matrix.cockroachdb-version }}.linux-amd64/cockroach sql --insecure -e 'create database piccolo;' - - name: Test with pytest, CockroachDB run: ./scripts/test-cockroach.sh env: - PG_HOST: localhost - PG_DATABASE: piccolo + TC_COCKROACH_VERSION: ${{ matrix.cockroachdb-version }} - name: Upload coverage uses: codecov/codecov-action@v1 if: matrix.python-version == '3.13' diff --git a/requirements/test-requirements.txt b/requirements/test-requirements.txt index a094ddd10..423946348 100644 --- a/requirements/test-requirements.txt +++ b/requirements/test-requirements.txt @@ -3,4 +3,5 @@ httpx==0.28.1 pytest-cov==7.1.0 pytest==9.0.3 python-dateutil==2.9.0 +testcontainers>=4.0.0 typing-extensions>=4.15.0 diff --git a/tests/conftest.py b/tests/conftest.py index 3a349d23a..bc5cc08bb 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,11 +1,145 @@ import asyncio +import os +import typing +import pytest from piccolo.engine.finder import engine_finder -ENGINE = engine_finder() +# Set in pytest_configure and cleared in pytest_unconfigure. +_postgres_container: typing.Any = None +_cockroach_container: typing.Any = None -async def drop_tables(): +def pytest_configure(config: typing.Any) -> None: + """Start the appropriate testcontainer based on PICCOLO_CONF. + + Runs before collection and before any test-module imports, so PG_* env vars + are in place when *_conf.py is first imported inside pytest_sessionstart. + + Set TESTCONTAINERS=false to skip and use an external DB via PG_* env vars. + """ + global _postgres_container, _cockroach_container + + piccolo_conf = os.environ.get("PICCOLO_CONF", "") + if os.environ.get("TESTCONTAINERS", "true").lower() in ("false", "0", "no"): + return + + if "cockroach" in piccolo_conf: + _cockroach_container = _start_cockroach() + elif "postgres" in piccolo_conf: + _postgres_container = _start_postgres() + # No container needed for SQLite + + +def pytest_unconfigure(config: typing.Any) -> None: + global _postgres_container, _cockroach_container + if _postgres_container is not None: + _postgres_container.stop() + _postgres_container = None + if _cockroach_container is not None: + _cockroach_container.stop() + _cockroach_container = None + + +def _start_postgres() -> typing.Any: + from testcontainers.core.container import DockerContainer + from testcontainers.core.wait_strategies import ExecWaitStrategy + + pg_version = os.environ.get("TC_POSTGRES_VERSION", "17") + container = ( + DockerContainer(f"postgres:{pg_version}") + .with_env("POSTGRES_USER", "piccolo") + .with_env("POSTGRES_PASSWORD", "piccolo") + .with_env("POSTGRES_DB", "piccolo") + .with_exposed_ports(5432) + .waiting_for( + ExecWaitStrategy(["pg_isready", "-U", "piccolo", "-d", "piccolo"]) + ) + ) + try: + container.start() + except Exception as exc: # pragma: no cover + raise RuntimeError( + "testcontainers: failed to start Postgres. " + "Ensure Docker is running, or set TESTCONTAINERS=false." + ) from exc + + host = container.get_container_host_ip() + port = int(container.get_exposed_port(5432)) + + os.environ["PG_HOST"] = host + os.environ["PG_PORT"] = str(port) + os.environ["PG_USER"] = "piccolo" + os.environ["PG_PASSWORD"] = "piccolo" + os.environ["PG_DATABASE"] = "piccolo" + + asyncio.run(_exec_sql( + host=host, port=port, + user="piccolo", password="piccolo", database="piccolo", + sql='CREATE EXTENSION IF NOT EXISTS "uuid-ossp"', + )) + return container + + +def _start_cockroach() -> typing.Any: + from testcontainers.core.container import DockerContainer + from testcontainers.core.wait_strategies import HealthcheckWaitStrategy + + crdb_version = os.environ.get("TC_COCKROACH_VERSION", "v25.4.3") + container = ( + DockerContainer(f"cockroachdb/cockroach:{crdb_version}") + .with_command("start-single-node --insecure") + .with_exposed_ports(26257) + .waiting_for(HealthcheckWaitStrategy()) + ) + try: + container.start() + except Exception as exc: # pragma: no cover + raise RuntimeError( + "testcontainers: failed to start CockroachDB. " + "Ensure Docker is running, or set TESTCONTAINERS=false." + ) from exc + + host = container.get_container_host_ip() + port = int(container.get_exposed_port(26257)) + + os.environ["PG_HOST"] = host + os.environ["PG_PORT"] = str(port) + os.environ["PG_USER"] = "root" + os.environ["PG_PASSWORD"] = "" + os.environ["PG_DATABASE"] = "piccolo" + + # CockroachDB starts with 'defaultdb'. Create the piccolo database. + asyncio.run(_exec_sql( + host=host, port=port, + user="root", password=None, database="defaultdb", + sql="CREATE DATABASE IF NOT EXISTS piccolo", + )) + return container + + +async def _exec_sql( + host: str, + port: int, + user: str, + password: typing.Optional[str], + database: str, + sql: str, +) -> None: + import asyncpg + conn = await asyncpg.connect( + host=host, port=port, user=user, password=password, database=database, + ) + try: + await conn.execute(sql) + finally: + await conn.close() + + +async def drop_tables() -> None: + engine = engine_finder() + assert engine is not None + tables = [ "ticket", "concert", @@ -24,22 +158,20 @@ async def drop_tables(): "mega_table", "small_table", ] - assert ENGINE is not None - if ENGINE.engine_type == "sqlite": - # SQLite doesn't allow us to drop more than one table at a time. + if engine.engine_type == "sqlite": for table in tables: - await ENGINE._run_in_new_connection( + await engine._run_in_new_connection( f"DROP TABLE IF EXISTS {table}" ) else: table_str = ", ".join(tables) - await ENGINE._run_in_new_connection( + await engine._run_in_new_connection( f"DROP TABLE IF EXISTS {table_str} CASCADE" ) -def pytest_sessionstart(session): +def pytest_sessionstart(session: typing.Any) -> None: """ Make sure all the tables have been dropped, just in case a previous test run was aborted part of the way through. @@ -50,8 +182,49 @@ def pytest_sessionstart(session): asyncio.run(drop_tables()) -def pytest_sessionfinish(session, exitstatus): +def pytest_sessionfinish(session: typing.Any, exitstatus: typing.Any) -> None: """ https://docs.pytest.org/en/latest/reference.html#_pytest.hookspec.pytest_sessionfinish """ print("Session finishing") + + +TABLESPACE_NAME = "piccolo_test_space" +TABLESPACE_DIR = "/var/lib/postgresql/testspace" + + +@pytest.fixture(scope="session") +def custom_tablespace() -> typing.Optional[str]: + """Provisions a custom Postgres tablespace inside the testcontainer. + + Returns the tablespace name when running against testcontainers Postgres; + returns None for SQLite, CockroachDB, or external Postgres. + + Tests should skip when None: + + def test_something(custom_tablespace): + if custom_tablespace is None: + pytest.skip("requires testcontainers Postgres") + """ + if _postgres_container is None: + return None + + # Create directory inside the container. + # Works whether exec runs as root or postgres: + # - root: mkdir (owned root) → chown fixes it + # - postgres: mkdir (owned postgres) → chown fails silently (already correct) + exit_code, output = _postgres_container.exec([ + "sh", "-c", + f"mkdir -p {TABLESPACE_DIR} && chown postgres:postgres {TABLESPACE_DIR} 2>/dev/null || true", + ]) + if exit_code != 0: + raise RuntimeError(f"tablespace dir setup failed: {output.decode()}") + + host = _postgres_container.get_container_host_ip() + port = int(_postgres_container.get_exposed_port(5432)) + asyncio.run(_exec_sql( + host=host, port=port, + user="piccolo", password="piccolo", database="piccolo", + sql=f"CREATE TABLESPACE {TABLESPACE_NAME} LOCATION '{TABLESPACE_DIR}'", + )) + return TABLESPACE_NAME From 8419a832f6dd6a85892964fa955a7b5fe5de8c03 Mon Sep 17 00:00:00 2001 From: Lachlan Walker Date: Wed, 22 Jul 2026 11:04:02 +0800 Subject: [PATCH 2/4] update docs --- docs/src/piccolo/contributing/index.rst | 21 +-------------------- pyproject.toml | 3 ++- tests/README.md | 12 +++++------- 3 files changed, 8 insertions(+), 28 deletions(-) diff --git a/docs/src/piccolo/contributing/index.rst b/docs/src/piccolo/contributing/index.rst index de947e5fd..57c8c8a60 100644 --- a/docs/src/piccolo/contributing/index.rst +++ b/docs/src/piccolo/contributing/index.rst @@ -8,25 +8,6 @@ instructions. ------------------------------------------------------------------------------- -Running Cockroach ------------------ - -To get a local Cockroach instance running, you can use: - -.. code-block:: console - - cockroach start-single-node --insecure --store=type=mem,size=2GiB - -Make sure the test database exists: - -.. code-block:: console - - cockroach sql --insecure - >>> create database piccolo; - >>> use piccolo; - -------------------------------------------------------------------------------- - Get the tests running --------------------- @@ -37,7 +18,7 @@ Get the tests running * Install development dependencies: ``pip install -r requirements/dev-requirements.txt`` * Install test dependencies: ``pip install -r requirements/test-requirements.txt`` * Install database drivers: ``pip install -r requirements/extras/postgres.txt -r requirements/extras/sqlite.txt`` -* Setup Postgres, and make sure a database called ``piccolo`` exists (see ``tests/postgres_conf.py``). +* Make sure Docker is running - Postgres and Cockroach are started automatically via `testcontainers `_. To use an external database instead, set ``TESTCONTAINERS=false`` and configure it via the ``PG_*`` env vars (see ``tests/postgres_conf.py``). * Run the automated code linting/formatting tools: ``./scripts/lint.sh`` * Run the test suite with Postgres: ``./scripts/test-postgres.sh`` * Run the test suite with Cockroach: ``./scripts/test-cockroach.sh`` diff --git a/pyproject.toml b/pyproject.toml index 0c94768d4..dc43de848 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -17,7 +17,8 @@ module = [ "jinja2", "orjson", "aiosqlite", - "uvicorn" + "uvicorn", + "testcontainers.*" ] ignore_missing_imports = true diff --git a/tests/README.md b/tests/README.md index 04b57823d..2986fcec1 100644 --- a/tests/README.md +++ b/tests/README.md @@ -2,11 +2,9 @@ ## Database -A local database needs to be running with the following credentials: +Postgres and Cockroach are started automatically via testcontainers, so +Docker needs to be running. -``` -host = localhost:5432 -database = piccolo -user = piccolo -password = piccolo -``` +To use an external database instead, set `TESTCONTAINERS=false` and point +`PG_HOST` / `PG_PORT` / `PG_DATABASE` / `PG_USER` / `PG_PASSWORD` at it (see +`postgres_conf.py`). From 48c1bb5483ca53a69b2ae7eba620b640e278f137 Mon Sep 17 00:00:00 2001 From: Lachlan Walker Date: Wed, 22 Jul 2026 11:04:41 +0800 Subject: [PATCH 3/4] fix env vars --- tests/apps/sql_shell/commands/test_run.py | 7 +- tests/conftest.py | 94 ++++++++++++++++------- 2 files changed, 72 insertions(+), 29 deletions(-) diff --git a/tests/apps/sql_shell/commands/test_run.py b/tests/apps/sql_shell/commands/test_run.py index 8d0c5689c..f798af193 100644 --- a/tests/apps/sql_shell/commands/test_run.py +++ b/tests/apps/sql_shell/commands/test_run.py @@ -1,3 +1,4 @@ +import os from unittest import TestCase from unittest.mock import MagicMock, patch @@ -18,11 +19,11 @@ def test_psql(self, subprocess: MagicMock): assert subprocess.run.call_args.args[0] == [ "psql", "-U", - "postgres", + os.environ.get("PG_USER", "postgres"), "-h", - "localhost", + os.environ.get("PG_HOST", "localhost"), "-p", - "5432", + os.environ.get("PG_PORT", "5432"), "piccolo", ] diff --git a/tests/conftest.py b/tests/conftest.py index bc5cc08bb..2ce2ac8eb 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,10 +1,19 @@ import asyncio import os +import time import typing import pytest + from piccolo.engine.finder import engine_finder +# Force UTC regardless of the host's local timezone. Without this, naive +# datetimes get encoded using the host's local offset, causing timezone +# arithmetic tests to fail on any machine that isn't already running UTC. +os.environ["TZ"] = "UTC" +if hasattr(time, "tzset"): + time.tzset() + # Set in pytest_configure and cleared in pytest_unconfigure. _postgres_container: typing.Any = None _cockroach_container: typing.Any = None @@ -21,7 +30,11 @@ def pytest_configure(config: typing.Any) -> None: global _postgres_container, _cockroach_container piccolo_conf = os.environ.get("PICCOLO_CONF", "") - if os.environ.get("TESTCONTAINERS", "true").lower() in ("false", "0", "no"): + if os.environ.get("TESTCONTAINERS", "true").lower() in ( + "false", + "0", + "no", + ): return if "cockroach" in piccolo_conf: @@ -73,24 +86,33 @@ def _start_postgres() -> typing.Any: os.environ["PG_PASSWORD"] = "piccolo" os.environ["PG_DATABASE"] = "piccolo" - asyncio.run(_exec_sql( - host=host, port=port, - user="piccolo", password="piccolo", database="piccolo", - sql='CREATE EXTENSION IF NOT EXISTS "uuid-ossp"', - )) + asyncio.run( + _exec_sql( + host=host, + port=port, + user="piccolo", + password="piccolo", + database="piccolo", + sql='CREATE EXTENSION IF NOT EXISTS "uuid-ossp"', + ) + ) return container def _start_cockroach() -> typing.Any: from testcontainers.core.container import DockerContainer - from testcontainers.core.wait_strategies import HealthcheckWaitStrategy + from testcontainers.core.wait_strategies import ExecWaitStrategy crdb_version = os.environ.get("TC_COCKROACH_VERSION", "v25.4.3") container = ( DockerContainer(f"cockroachdb/cockroach:{crdb_version}") .with_command("start-single-node --insecure") .with_exposed_ports(26257) - .waiting_for(HealthcheckWaitStrategy()) + .waiting_for( + ExecWaitStrategy( + ["cockroach", "sql", "--insecure", "-e", "SELECT 1"] + ) + ) ) try: container.start() @@ -110,11 +132,16 @@ def _start_cockroach() -> typing.Any: os.environ["PG_DATABASE"] = "piccolo" # CockroachDB starts with 'defaultdb'. Create the piccolo database. - asyncio.run(_exec_sql( - host=host, port=port, - user="root", password=None, database="defaultdb", - sql="CREATE DATABASE IF NOT EXISTS piccolo", - )) + asyncio.run( + _exec_sql( + host=host, + port=port, + user="root", + password=None, + database="defaultdb", + sql="CREATE DATABASE IF NOT EXISTS piccolo", + ) + ) return container @@ -127,8 +154,13 @@ async def _exec_sql( sql: str, ) -> None: import asyncpg + conn = await asyncpg.connect( - host=host, port=port, user=user, password=password, database=database, + host=host, + port=port, + user=user, + password=password, + database=database, ) try: await conn.execute(sql) @@ -161,12 +193,12 @@ async def drop_tables() -> None: if engine.engine_type == "sqlite": for table in tables: - await engine._run_in_new_connection( + await engine._run_in_new_connection( # type: ignore[attr-defined] f"DROP TABLE IF EXISTS {table}" ) else: table_str = ", ".join(tables) - await engine._run_in_new_connection( + await engine._run_in_new_connection( # type: ignore[attr-defined] f"DROP TABLE IF EXISTS {table_str} CASCADE" ) @@ -212,19 +244,29 @@ def test_something(custom_tablespace): # Create directory inside the container. # Works whether exec runs as root or postgres: # - root: mkdir (owned root) → chown fixes it - # - postgres: mkdir (owned postgres) → chown fails silently (already correct) - exit_code, output = _postgres_container.exec([ - "sh", "-c", - f"mkdir -p {TABLESPACE_DIR} && chown postgres:postgres {TABLESPACE_DIR} 2>/dev/null || true", - ]) + # - postgres: mkdir (owned postgres) → chown fails silently (already + # correct) + mkdir_cmd = ( + f"mkdir -p {TABLESPACE_DIR} && " + f"chown postgres:postgres {TABLESPACE_DIR} 2>/dev/null || true" + ) + exit_code, output = _postgres_container.exec(["sh", "-c", mkdir_cmd]) if exit_code != 0: raise RuntimeError(f"tablespace dir setup failed: {output.decode()}") host = _postgres_container.get_container_host_ip() port = int(_postgres_container.get_exposed_port(5432)) - asyncio.run(_exec_sql( - host=host, port=port, - user="piccolo", password="piccolo", database="piccolo", - sql=f"CREATE TABLESPACE {TABLESPACE_NAME} LOCATION '{TABLESPACE_DIR}'", - )) + asyncio.run( + _exec_sql( + host=host, + port=port, + user="piccolo", + password="piccolo", + database="piccolo", + sql=( + f"CREATE TABLESPACE {TABLESPACE_NAME} " + f"LOCATION '{TABLESPACE_DIR}'" + ), + ) + ) return TABLESPACE_NAME From 224ed22435c648b9bbbf1cf6c7a0f3453a82db30 Mon Sep 17 00:00:00 2001 From: Lachlan Walker Date: Thu, 23 Jul 2026 08:06:20 +0800 Subject: [PATCH 4/4] improve container readiness check --- tests/conftest.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tests/conftest.py b/tests/conftest.py index 2ce2ac8eb..f84a5a7ed 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -66,7 +66,17 @@ def _start_postgres() -> typing.Any: .with_env("POSTGRES_DB", "piccolo") .with_exposed_ports(5432) .waiting_for( - ExecWaitStrategy(["pg_isready", "-U", "piccolo", "-d", "piccolo"]) + ExecWaitStrategy( + [ + "pg_isready", + "-h", + "127.0.0.1", + "-U", + "piccolo", + "-d", + "piccolo", + ] + ) ) ) try: