diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 115da7b0..442a3de9 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -2,9 +2,9 @@ name: Build on: push: - branches: [main] + branches: [main, postgres] pull_request: - branches: [main] + branches: [main, postgres] permissions: contents: read @@ -13,6 +13,20 @@ jobs: build: runs-on: ubuntu-24.04 timeout-minutes: 30 + services: + # Backend tests that take the `postgres` fixture run here (src/database/testing.py). + postgres: + image: pgvector/pgvector:pg17 + env: + POSTGRES_PASSWORD: test + ports: + - 5432:5432 + options: >- + --tmpfs /var/lib/postgresql/data + --health-cmd "pg_isready -h 127.0.0.1 -U postgres" + --health-interval 2s + --health-timeout 3s + --health-retries 30 steps: - uses: actions/checkout@v7 with: @@ -45,6 +59,8 @@ jobs: - name: Test backend if: ${{ !cancelled() }} working-directory: backend + env: + TEST_DATABASE_URL: postgresql://postgres:test@127.0.0.1:5432/postgres run: python -m pytest -q --junitxml=../test-results/backend.xml - name: Upload test reports if: ${{ !cancelled() && hashFiles('test-results/*.xml') != '' }} diff --git a/README.md b/README.md index 2994ea6e..82c7b7c0 100644 --- a/README.md +++ b/README.md @@ -38,6 +38,17 @@ and run the frontend Vitest and backend pytest suites with the same JUnit output (cd backend && python -m pip install -r requirements.txt -r requirements-dev.txt && python -m pytest -q --junitxml=../test-results/backend.xml) ``` +Backend tests that take the `postgres` fixture (`backend/src/database/testing.py`) +need a Postgres server and a superuser URL in `TEST_DATABASE_URL`; each run creates +and drops its own database. Without it they skip locally. CI runs them against a +`pgvector/pgvector:pg17` service. Locally, for example: + +```sh +docker run --rm -d --name ps-test-pg -p 55432:5432 -e POSTGRES_PASSWORD=test \ + --tmpfs /var/lib/postgresql/data pgvector/pgvector:pg17 -c fsync=off +export TEST_DATABASE_URL=postgresql://postgres:test@127.0.0.1:55432/postgres +``` + CI also lints the frontend with oxlint and checks its formatting with oxfmt. From `frontend/`, `npm run lint` and `npm run format:check` run the same checks; `npm run format` rewrites files in place. diff --git a/backend/conftest.py b/backend/conftest.py index e53d3dd5..aef7c435 100644 --- a/backend/conftest.py +++ b/backend/conftest.py @@ -10,6 +10,8 @@ import src.skins as skins from src.skins import db +# Postgres fixtures (`postgres`, `scratch_database`); see src/database/testing.py. +from src.database.testing import postgres, postgres_session, scratch_database # noqa: F401 # Fixtures replace db paths with temporary directories. All imported connect # functions must retain that same module's globals throughout test collection. diff --git a/backend/requirements.txt b/backend/requirements.txt index e7f67321..5af5ba85 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -10,6 +10,7 @@ pyyaml httpx anthropic psycopg2-binary -# The database studio's Postgres reads and edits: one statement per request, enforced by Postgres. -psycopg[binary] +# Site data on Postgres (src/database) and the database studio's Postgres reads and edits. +psycopg[binary]==3.3.6 +psycopg-pool==3.3.3 pgvector diff --git a/backend/server.py b/backend/server.py index 0f15c3ac..0e3a4dd6 100644 --- a/backend/server.py +++ b/backend/server.py @@ -1,5 +1,6 @@ import logging import asyncio +import os from contextlib import asynccontextmanager from pathlib import Path @@ -52,6 +53,12 @@ async def lifespan(app: FastAPI): from src.skins.db import migrate migrate() + from src.database import close_pool + from src.database.migrate import migrate as migrate_postgres + + # The site's Postgres; previews and local runs without one skip it. + if os.environ.get("DATABASE_URL", "").strip(): + migrate_postgres(seed=True) from src.patreon.config import Config from src.patreon.service import sync_loop @@ -71,6 +78,7 @@ async def lifespan(app: FastAPI): if names_task is not None: names_stop.set() await names_task + close_pool() app = FastAPI(lifespan=lifespan) diff --git a/backend/src/database/__init__.py b/backend/src/database/__init__.py new file mode 100644 index 00000000..770c5d1e --- /dev/null +++ b/backend/src/database/__init__.py @@ -0,0 +1,59 @@ +"""Province site data on Postgres (psycopg 3). + + from src import database + + with database.connect() as conn: # one transaction, rows are dicts + row = conn.execute("SELECT * FROM users WHERE id = %s", (user_id,)).fetchone() + +Schema changes are numbered files in `migrations/`, applied by `migrate`. +""" + +from psycopg import DatabaseError, Error, IntegrityError, OperationalError +from psycopg.errors import ( + CheckViolation, + DeadlockDetected, + ForeignKeyViolation, + LockNotAvailable, + QueryCanceled, + SerializationFailure, + UniqueViolation, +) +from psycopg.types.json import Jsonb + +from .connection import ( + Connection, + NotConfiguredError, + advisory_lock, + close_pool, + connect, + database_url, + get_pool, + open_pool, + serializable, + transaction, +) + +__all__ = [ + "CheckViolation", + "Connection", + "DatabaseError", + "DeadlockDetected", + "Error", + "ForeignKeyViolation", + "IntegrityError", + "Jsonb", + "LockNotAvailable", + "NotConfiguredError", + "OperationalError", + "QueryCanceled", + "SerializationFailure", + "UniqueViolation", + "advisory_lock", + "close_pool", + "connect", + "database_url", + "get_pool", + "open_pool", + "serializable", + "transaction", +] diff --git a/backend/src/database/connection.py b/backend/src/database/connection.py new file mode 100644 index 00000000..b2ae4228 --- /dev/null +++ b/backend/src/database/connection.py @@ -0,0 +1,224 @@ +"""The process-wide connection pool and the transaction and locking helpers. + +Every block borrows a pooled connection and is one transaction: it commits when +the block ends normally and rolls back on an exception. Rows are dicts. +""" + +from __future__ import annotations + +import os +import random +import threading +import time +from collections.abc import Callable, Iterator +from contextlib import contextmanager +from typing import Any, TypeVar + +import psycopg +from psycopg import errors +from psycopg.rows import dict_row +from psycopg_pool import ConnectionPool + +Connection = psycopg.Connection[dict[str, Any]] +T = TypeVar("T") + +# Set at connection start-up, so a reset (RESET ALL) returns to them. The +# database is shared with Precedent and patch notes: names resolve in public only. +# Timestamps come back as aware UTC datetimes; a stuck lock or query fails +# instead of holding a worker. +SESSION_SETTINGS = { + "search_path": "public", + "TimeZone": "UTC", + "lock_timeout": "5s", + "statement_timeout": "30s", + "idle_in_transaction_session_timeout": "60s", +} +# How long advisory_lock() waits: longer than the 5 s row-lock bound, because an +# advisory holder may run a whole recompute (Patreon, a LuckPerms snapshot), and +# shorter than statement_timeout, so a waiter fails with LockNotAvailable rather +# than a cancelled statement. +ADVISORY_LOCK_TIMEOUT = "20s" +POOL_TIMEOUT_SECONDS = 10.0 +SERIALIZABLE_ATTEMPTS = 5 +_ISOLATION = {"read committed", "repeatable read", "serializable"} + +_pool: ConnectionPool | None = None +_pool_lock = threading.Lock() + + +class NotConfiguredError(RuntimeError): + """DATABASE_URL is not set.""" + + +def database_url() -> str: + url = os.environ.get("DATABASE_URL", "").strip() + if not url: + raise NotConfiguredError("DATABASE_URL is not set") + return url + + +def session_options() -> str: + return " ".join(f"-c {name}={value}" for name, value in SESSION_SETTINGS.items()) + + +def _size(name: str, default: int) -> int: + raw = os.environ.get(name, "").strip() + return int(raw) if raw else default + + +# DISCARD ALL without DEALLOCATE ALL. psycopg notices a DISCARD ALL only the first +# time a connection runs it, then keeps naming prepared statements the server has +# dropped; psycopg's own prepared statements are safe to keep. +_DISCARD = ( + "CLOSE ALL; SET SESSION AUTHORIZATION DEFAULT; RESET ALL; UNLISTEN *; " + "SELECT pg_advisory_unlock_all(); DISCARD PLANS; DISCARD SEQUENCES; DISCARD TEMP" +) + + +def _reset(conn: Connection) -> None: + """Return a connection to the pool as it started. + + The pool has already rolled back any open transaction. Settings go back to + their start-up values; cursors, temp tables, LISTENs and session advisory + locks go; the connection attributes go back to their defaults. + """ + conn.autocommit = True + conn.execute(_DISCARD) + conn.autocommit = False + conn.isolation_level = None + conn.read_only = None + conn.deferrable = None + conn.row_factory = dict_row + + +def _new_pool(url: str, min_size: int | None, max_size: int | None) -> ConnectionPool: + return ConnectionPool( + url, + min_size=_size("DATABASE_POOL_MIN_SIZE", 1) if min_size is None else min_size, + max_size=_size("DATABASE_POOL_MAX_SIZE", 10) if max_size is None else max_size, + timeout=POOL_TIMEOUT_SECONDS, + kwargs={"row_factory": dict_row, "options": session_options(), "application_name": "provincesystem"}, + # One round trip per borrow, so a restarted Postgres costs nobody a failed request. + check=ConnectionPool.check_connection, + reset=_reset, + name="province", + open=True, + ) + + +def open_pool(url: str | None = None, *, min_size: int | None = None, max_size: int | None = None) -> ConnectionPool: + """Open the pool for `url` (default DATABASE_URL), closing any pool already open. + + Sizes default to DATABASE_POOL_MIN_SIZE (1) and DATABASE_POOL_MAX_SIZE (10). + """ + global _pool + pool = _new_pool(url or database_url(), min_size, max_size) + with _pool_lock: + old, _pool = _pool, pool + if old is not None: + old.close() + return pool + + +def close_pool() -> None: + global _pool + with _pool_lock: + pool, _pool = _pool, None + if pool is not None: + pool.close() + + +def get_pool() -> ConnectionPool: + """The open pool, opened from DATABASE_URL on first use.""" + global _pool + pool = _pool + if pool is None: + with _pool_lock: + if _pool is None: + _pool = _new_pool(database_url(), None, None) + pool = _pool + return pool + + +@contextmanager +def connect(*, timeout: float | None = None) -> Iterator[Connection]: + """Borrow a pooled connection for one transaction. + + The first statement begins the transaction. The block commits when it ends + normally and rolls back on an exception, then the connection goes back to + the pool. Use `conn.transaction()` inside for a savepoint that can fail on + its own (for example an INSERT that may hit a unique constraint). + """ + with get_pool().connection(timeout=timeout) as conn: + yield conn + + +@contextmanager +def transaction( + *, isolation: str | None = None, read_only: bool = False, lock: str | None = None +) -> Iterator[Connection]: + """`connect()` with the transaction's settings applied before any other statement. + + `isolation` is "read committed" (the default), "repeatable read" or + "serializable". `lock` names an advisory lock taken first (see `advisory_lock`). + """ + modes = [] + if isolation is not None: + level = isolation.lower() + if level not in _ISOLATION: + raise ValueError(f"unknown isolation level: {isolation}") + modes.append(f"ISOLATION LEVEL {level.upper()}") + if read_only: + modes.append("READ ONLY") + with connect() as conn: + if modes: + conn.execute(f"SET TRANSACTION {', '.join(modes)}") + if lock is not None: + advisory_lock(conn, lock) + yield conn + + +def advisory_lock(conn: Connection, name: str, *, wait: bool = True) -> bool: + """Take the transaction-scoped advisory lock `name`; commit or rollback releases it. + + For checks with no row to lock: a cooldown over several rows, or "no row + exists yet". Name it `:[:]`, for example + `codes:mint:`. Every path that changes what the check reads + must take the same lock. Returns False instead of waiting when `wait` is + False and another transaction holds it. + + Waits up to ADVISORY_LOCK_TIMEOUT, then raises LockNotAvailable; the + transaction's own lock_timeout applies again once the lock is held. + """ + if conn.autocommit: + raise RuntimeError("advisory_lock needs a transaction; this connection is in autocommit") + if wait: + # The CTE reads the old value before set_config changes it. + previous = conn.execute( + "WITH old AS MATERIALIZED (SELECT current_setting('lock_timeout') AS value) " + "SELECT old.value, set_config('lock_timeout', %s, true) FROM old", + (ADVISORY_LOCK_TIMEOUT,), + ).fetchone()["value"] + conn.execute("SELECT pg_advisory_xact_lock(hashtextextended(%s, 0))", (name,)) + conn.execute("SELECT set_config('lock_timeout', %s, true)", (previous,)) + return True + row = conn.execute("SELECT pg_try_advisory_xact_lock(hashtextextended(%s, 0)) AS locked", (name,)).fetchone() + return bool(row["locked"]) + + +def serializable(work: Callable[[Connection], T], *, attempts: int = SERIALIZABLE_ATTEMPTS) -> T: + """Run `work(conn)` in a SERIALIZABLE transaction and return its result. + + Retries the whole transaction when Postgres reports a serialization failure + or a deadlock, with a short random backoff, up to `attempts` times. `work` + may run more than once, so it must not do anything outside the database. + """ + for attempt in range(1, attempts + 1): + try: + with transaction(isolation="serializable") as conn: + return work(conn) + except (errors.SerializationFailure, errors.DeadlockDetected): + if attempt == attempts: + raise + time.sleep(random.uniform(0, 0.01 * 2**attempt)) + raise AssertionError("unreachable") diff --git a/backend/src/database/migrate.py b/backend/src/database/migrate.py new file mode 100644 index 00000000..a5df439d --- /dev/null +++ b/backend/src/database/migrate.py @@ -0,0 +1,162 @@ +"""Apply the numbered SQL files in `migrations/` that a database has not had yet. + +Files are `NNNN_name.sql`, applied in order, each in its own transaction with +its row in `schema_migrations`. A session advisory lock lets only one runner +work at a time, so backends starting together never apply a file twice. An +applied file must not change, every applied version must still have its file, +and a new file must be numbered above the newest applied one. + +`seeds.sql` holds rows the site needs (idempotent inserts). The backend applies +migrations and seeds at start-up; the copy from SQLite migrates without seeding, +so the copied rows take their place. + +By hand, from `backend/`: + + python -m src.database.migrate # apply pending files + python -m src.database.migrate --seed # and the seed rows + python -m src.database.migrate --status # list files and when each was applied +""" + +from __future__ import annotations + +import argparse +import hashlib +import re +import sys +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path + +import psycopg + +from .connection import database_url + +MIGRATIONS_DIR = Path(__file__).resolve().with_name("migrations") +SEEDS_PATH = Path(__file__).resolve().with_name("seeds.sql") +LOCK_NAME = "provincesystem:migrations" +_FILE = re.compile(r"^(\d{4})_([a-z0-9_]+)\.sql$") + +_CREATE_TABLE = """ +CREATE TABLE IF NOT EXISTS schema_migrations ( + version text COLLATE "C" PRIMARY KEY, + name text COLLATE "C" NOT NULL, + checksum text COLLATE "C" NOT NULL, + applied_at timestamptz NOT NULL DEFAULT now() +) +""" + + +class MigrationError(RuntimeError): + pass + + +@dataclass(frozen=True) +class Migration: + version: str + name: str + sql: str + + @property + def checksum(self) -> str: + return hashlib.sha256(self.sql.encode("utf-8")).hexdigest() + + +def discover(directory: Path | None = None) -> list[Migration]: + directory = directory or MIGRATIONS_DIR + migrations = [] + for path in sorted(directory.iterdir()): + if path.suffix != ".sql": + continue + match = _FILE.match(path.name) + if not match: + raise MigrationError(f"migration file name must be NNNN_name.sql: {path.name}") + migrations.append(Migration(match[1], match[2], path.read_text(encoding="utf-8"))) + versions = [m.version for m in migrations] + if len(set(versions)) != len(versions): + raise MigrationError(f"two migration files share a number in {directory}") + return migrations + + +def _connect(url: str | None) -> psycopg.Connection: + return psycopg.connect(url or database_url(), autocommit=True, + options="-c search_path=public -c TimeZone=UTC", + application_name="provincesystem-migrate") + + +def _applied(conn: psycopg.Connection) -> dict[str, tuple[str, datetime]]: + rows = conn.execute("SELECT version, checksum, applied_at FROM schema_migrations").fetchall() + return {version: (checksum, applied_at) for version, checksum, applied_at in rows} + + +def _check_order(migrations: list[Migration], applied: dict) -> None: + files = {m.version: m for m in migrations} + unknown = sorted(set(applied) - set(files)) + if unknown: + raise MigrationError(f"the database has applied migrations with no file: {', '.join(unknown)}") + for version, (checksum, _) in applied.items(): + if files[version].checksum != checksum: + raise MigrationError(f"migration {version}_{files[version].name} changed after it was applied") + newest = max(applied, default=None) + late = [m for m in migrations if m.version not in applied and newest is not None and m.version < newest] + if late: + raise MigrationError( + f"migration {late[0].version}_{late[0].name} is numbered below the newest applied one ({newest})") + + +def migrate(url: str | None = None, *, directory: Path | None = None, seed: bool = False) -> list[str]: + """Apply pending migrations to `url` (default DATABASE_URL), then `seeds.sql` if `seed`. + + Returns the versions applied. Refuses to run, changing nothing, when the + files and the database's history disagree. + """ + migrations = discover(directory) + done = [] + with _connect(url) as conn: + conn.execute("SELECT pg_advisory_lock(hashtextextended(%s, 0))", (LOCK_NAME,)) + try: + conn.execute(_CREATE_TABLE) + applied = _applied(conn) + _check_order(migrations, applied) + for migration in migrations: + if migration.version in applied: + continue + with conn.transaction(): + conn.execute(migration.sql) + conn.execute( + "INSERT INTO schema_migrations (version, name, checksum) VALUES (%s, %s, %s)", + (migration.version, migration.name, migration.checksum), + ) + done.append(migration.version) + if seed: + with conn.transaction(): + conn.execute(SEEDS_PATH.read_text(encoding="utf-8")) + finally: + conn.execute("SELECT pg_advisory_unlock(hashtextextended(%s, 0))", (LOCK_NAME,)) + return done + + +def status(url: str | None = None, *, directory: Path | None = None) -> list[tuple[str, str, datetime | None]]: + """(version, name, applied_at or None) for every migration file.""" + with _connect(url) as conn: + exists = conn.execute("SELECT to_regclass('schema_migrations') IS NOT NULL").fetchone()[0] + applied = _applied(conn) if exists else {} + return [(m.version, m.name, applied.get(m.version, (None, None))[1]) for m in discover(directory)] + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(prog="python -m src.database.migrate", description=__doc__.split("\n\n")[0]) + parser.add_argument("--database-url", help="defaults to DATABASE_URL") + parser.add_argument("--status", action="store_true", help="list migrations instead of applying them") + parser.add_argument("--seed", action="store_true", help="also insert the seed rows (seeds.sql)") + args = parser.parse_args(argv) + if args.status: + for version, name, applied_at in status(args.database_url): + print(f"{version}_{name}\t{applied_at.isoformat() if applied_at else 'pending'}") + return 0 + applied = migrate(args.database_url, seed=args.seed) + print(f"applied {', '.join(applied)}" if applied else "up to date") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/backend/src/database/migrations/0001_baseline.sql b/backend/src/database/migrations/0001_baseline.sql new file mode 100644 index 00000000..b149b296 --- /dev/null +++ b/backend/src/database/migrations/0001_baseline.sql @@ -0,0 +1,945 @@ +-- Province site data: every table that was in province.db, at its final shape. +-- +-- Types: identity columns are bigint GENERATED BY DEFAULT AS IDENTITY, so the copy +-- from SQLite can insert the existing ids (it must setval each sequence after). +-- Times are timestamptz, 0/1 flags boolean, JSON jsonb, other integers bigint and +-- REAL double precision. Day strings ('YYYY-MM-DD'), ledger founded_at (it feeds +-- faction_key) and character_roster.created_at (RPCharacters' epoch-seconds string) +-- stay text. +-- +-- Rows the site needs at start (patreon_sync_state) are in ../seeds.sql, so +-- the copy from SQLite starts from empty tables. +-- +-- Collation: every text column is COLLATE "C", whatever the database default, so +-- comparison and ORDER BY follow code-point order as SQLite's BINARY did, and lower() +-- folds ASCII only, as SQLite's did. + +CREATE TABLE codes ( + id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + code_hash text COLLATE "C" NOT NULL UNIQUE, + code_plaintext text COLLATE "C", + player_uuid text COLLATE "C" NOT NULL, + scope text COLLATE "C" NOT NULL DEFAULT 'skin', + realm_id text COLLATE "C" NOT NULL DEFAULT 'main', + created_at timestamptz NOT NULL, + expires_at timestamptz NOT NULL, + redeemed_at timestamptz, + revoked boolean NOT NULL DEFAULT false, + minted_via text COLLATE "C" +); + +CREATE TABLE submissions ( + id text COLLATE "C" PRIMARY KEY, + player_uuid text COLLATE "C" NOT NULL, + code_id bigint NOT NULL, + kind text COLLATE "C" NOT NULL, + slug text COLLATE "C" NOT NULL, + display_name text COLLATE "C" NOT NULL, + grip_preset text COLLATE "C", + base_set text COLLATE "C", + tiers jsonb, + tier_aliases jsonb, + helmet_3d_tiers jsonb, + texture_hash text COLLATE "C", + add_name boolean NOT NULL DEFAULT false, + name_colours jsonb, + name_styles jsonb, + status text COLLATE "C" NOT NULL, + deny_reason text COLLATE "C", + dir_path text COLLATE "C" NOT NULL, + created_at timestamptz NOT NULL, + reviewed_at timestamptz, + applied_at timestamptz, + discord_message_id text COLLATE "C", + discord_user_id text COLLATE "C", + staff boolean NOT NULL DEFAULT false, + category text COLLATE "C", + scroll text COLLATE "C", + tier_scrolls jsonb, + realm_id text COLLATE "C" NOT NULL DEFAULT 'main', + tier_sets jsonb, + FOREIGN KEY (code_id) REFERENCES codes(id) +); + +CREATE TABLE sessions ( + id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + token_hash text COLLATE "C" NOT NULL UNIQUE, + code_id bigint NOT NULL, + player_uuid text COLLATE "C" NOT NULL, + expires_at timestamptz NOT NULL, + created_at timestamptz NOT NULL, + FOREIGN KEY (code_id) REFERENCES codes(id) +); + +CREATE TABLE discord_links ( + player_uuid text COLLATE "C" PRIMARY KEY, + discord_user_id text COLLATE "C" NOT NULL UNIQUE, + minecraft_name text COLLATE "C", + discord_username text COLLATE "C", + linked_at timestamptz NOT NULL, + left_guild_at timestamptz, + grace_until timestamptz, + -- Their nickname in the TFMC Discord server, if any; discord_username is the account handle. + discord_nickname text COLLATE "C", + -- How the link was proved: 'code' (in-game code) or 'microsoft'. NULL for links made before this was kept. + link_method text COLLATE "C" +); + +CREATE TABLE discord_link_codes ( + id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + code_hash text COLLATE "C" NOT NULL UNIQUE, + player_uuid text COLLATE "C" NOT NULL, + minecraft_name text COLLATE "C", + created_at timestamptz NOT NULL, + expires_at timestamptz NOT NULL, + used_at timestamptz +); + +-- One-time war declare codes. Staff mint one in Discord bound to an attacker, +-- a defender and a war goal; the plugin validates it, then redeems it only once +-- the war actually exists. No plaintext is stored: staff see the code once. +CREATE TABLE war_declare_codes ( + id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + code_hash text COLLATE "C" NOT NULL UNIQUE, + realm_id text COLLATE "C" NOT NULL DEFAULT 'main', + attacker_faction_id text COLLATE "C" NOT NULL, + defender_faction_id text COLLATE "C" NOT NULL, + goal text COLLATE "C" NOT NULL, + created_by_discord_id text COLLATE "C", + created_at timestamptz NOT NULL, + expires_at timestamptz NOT NULL, + redeemed_at timestamptz, + redeemed_war_id text COLLATE "C", + revoked boolean NOT NULL DEFAULT false +); + +CREATE TABLE skin_notifications ( + id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + type text COLLATE "C" NOT NULL, + submission_id text COLLATE "C" NOT NULL, + discord_user_id text COLLATE "C" NOT NULL, + payload jsonb NOT NULL, + created_at timestamptz NOT NULL, + delivered_at timestamptz +); + +CREATE TABLE plugin_notices ( + id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + type text COLLATE "C" NOT NULL, + player_uuid text COLLATE "C" NOT NULL, + payload jsonb NOT NULL, + created_at timestamptz NOT NULL, + delivered_at timestamptz +); + +CREATE INDEX idx_submissions_status ON submissions(status); + +CREATE INDEX idx_submissions_slug ON submissions(slug); + +CREATE INDEX idx_war_declare_codes_outstanding + ON war_declare_codes(realm_id, redeemed_at, revoked); + +CREATE INDEX idx_skin_notifications_undelivered + ON skin_notifications(delivered_at, created_at); + +CREATE INDEX idx_plugin_notices_undelivered + ON plugin_notices(delivered_at, created_at); + +CREATE TABLE player_warnings ( + id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + player_uuid text COLLATE "C" NOT NULL, + staff_uuid text COLLATE "C", + staff_name text COLLATE "C", + reason text COLLATE "C" NOT NULL, + created_at timestamptz NOT NULL +); + +CREATE TABLE moderation_notifications ( + id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + type text COLLATE "C" NOT NULL, + discord_user_id text COLLATE "C" NOT NULL, + player_uuid text COLLATE "C", + payload jsonb NOT NULL, + created_at timestamptz NOT NULL, + delivered_at timestamptz +); + +CREATE INDEX idx_player_warnings_uuid + ON player_warnings(player_uuid, created_at); + +CREATE INDEX idx_moderation_notifications_undelivered + ON moderation_notifications(delivered_at, created_at); + +CREATE TABLE armourshop_catalog ( + id integer PRIMARY KEY CHECK (id = 1), + payload jsonb NOT NULL, + updated_at timestamptz NOT NULL +); + +CREATE TABLE lore_item_customisations ( + player_uuid text COLLATE "C" NOT NULL, + character_id text COLLATE "C" NOT NULL, + kit_key text COLLATE "C" NOT NULL, + kit_id text COLLATE "C", + name_colours jsonb, + name_styles jsonb, + display_name text COLLATE "C" NOT NULL DEFAULT '', + lore_json jsonb NOT NULL DEFAULT '[]', + existing_skin_id text COLLATE "C", + submission_id text COLLATE "C", + state text COLLATE "C" NOT NULL DEFAULT 'draft', + skin_slug text COLLATE "C", + ready_at timestamptz, + applied_at timestamptz, + deny_reason text COLLATE "C", + updated_at timestamptz NOT NULL, + realm_id text COLLATE "C" NOT NULL DEFAULT 'main', + PRIMARY KEY (player_uuid, character_id, kit_key) +); + +CREATE TABLE character_wardrobe_slots ( + player_uuid text COLLATE "C" NOT NULL, + character_id text COLLATE "C" NOT NULL, + slot text COLLATE "C" NOT NULL, + png_relpath text COLLATE "C", + texture_value text COLLATE "C", + texture_signature text COLLATE "C", + model text COLLATE "C", + display_name text COLLATE "C", + apply_pending boolean NOT NULL DEFAULT false, + updated_at timestamptz NOT NULL, + PRIMARY KEY (player_uuid, character_id, slot) +); + +CREATE TABLE character_create_wardrobe ( + create_id text COLLATE "C" NOT NULL, + slot text COLLATE "C" NOT NULL, + png_relpath text COLLATE "C", + texture_value text COLLATE "C", + texture_signature text COLLATE "C", + model text COLLATE "C", + display_name text COLLATE "C", + updated_at timestamptz NOT NULL, + PRIMARY KEY (create_id, slot) +); + +CREATE TABLE drink_submissions ( + id text COLLATE "C" PRIMARY KEY, + player_uuid text COLLATE "C" NOT NULL, + code_id bigint NOT NULL, + slug text COLLATE "C" NOT NULL, + display_name text COLLATE "C" NOT NULL, + recipe_json jsonb NOT NULL, + status text COLLATE "C" NOT NULL, + deny_reason text COLLATE "C", + texture_id text COLLATE "C", + new_texture boolean NOT NULL DEFAULT false, + dir_path text COLLATE "C" NOT NULL, + discord_user_id text COLLATE "C", + created_at timestamptz NOT NULL, + reviewed_at timestamptz, + applied_at timestamptz, + realm_id text COLLATE "C" NOT NULL DEFAULT 'main', + FOREIGN KEY (code_id) REFERENCES codes(id) +); + +CREATE TABLE drink_textures ( + id text COLLATE "C" PRIMARY KEY, + owner_uuid text COLLATE "C" NOT NULL, + cmd bigint, + ia_item_id text COLLATE "C", + png_path text COLLATE "C" NOT NULL, + refcount bigint NOT NULL DEFAULT 0, + created_at timestamptz NOT NULL +); + +CREATE TABLE drink_catalog ( + id integer PRIMARY KEY CHECK (id = 1), + payload jsonb NOT NULL, + updated_at timestamptz NOT NULL +); + +CREATE TABLE rpc_player_meta ( + player_uuid text COLLATE "C" NOT NULL, + realm_id text COLLATE "C" NOT NULL DEFAULT 'main', + name_colour_stops bigint NOT NULL DEFAULT 0, + allow_drink_texture boolean NOT NULL DEFAULT false, + allow_drink_message boolean NOT NULL DEFAULT false, + max_alive_characters bigint, + wardrobe_skin_slots bigint NOT NULL DEFAULT 1, + max_3d_pair_bytes bigint NOT NULL DEFAULT 0, + skin_token_cooldown_days bigint NOT NULL DEFAULT -1, + skin_kinds_json jsonb NOT NULL DEFAULT '[]', + allow_armor_3d_helmet boolean NOT NULL DEFAULT false, + permission_flags_json jsonb NOT NULL DEFAULT '{}', + updated_at timestamptz NOT NULL, + PRIMARY KEY (player_uuid, realm_id) +); + +CREATE TABLE drink_notifications ( + id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + type text COLLATE "C" NOT NULL, + submission_id text COLLATE "C" NOT NULL, + discord_user_id text COLLATE "C" NOT NULL, + payload jsonb NOT NULL, + created_at timestamptz NOT NULL, + delivered_at timestamptz +); + +CREATE TABLE cosmetic_mint_resets ( + id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + player_uuid text COLLATE "C" NOT NULL, + reset_at timestamptz NOT NULL, + staff_uuid text COLLATE "C" +); + +CREATE INDEX idx_drink_submissions_status + ON drink_submissions(status); + +CREATE INDEX idx_drink_submissions_slug + ON drink_submissions(slug); + +CREATE INDEX idx_drink_textures_owner + ON drink_textures(owner_uuid); + +CREATE INDEX idx_drink_notifications_undelivered + ON drink_notifications(delivered_at, created_at); + +CREATE INDEX idx_cosmetic_mint_resets_player + ON cosmetic_mint_resets(player_uuid); + +CREATE TABLE map_chronicle_snapshots ( + map_id text COLLATE "C" NOT NULL, + day text COLLATE "C" NOT NULL, + realm_id text COLLATE "C" NOT NULL DEFAULT 'main', + captured_at bigint NOT NULL, + bytes bigint NOT NULL, + geometry_version text COLLATE "C", + manifest jsonb NOT NULL, + PRIMARY KEY (map_id, day) +); + +CREATE TABLE map_chronicle_snapshots_archive ( + map_id text COLLATE "C" NOT NULL, + day text COLLATE "C" NOT NULL, + realm_id text COLLATE "C" NOT NULL DEFAULT 'main', + captured_at bigint NOT NULL, + bytes bigint NOT NULL, + geometry_version text COLLATE "C", + manifest jsonb NOT NULL, + archived_at bigint NOT NULL, + PRIMARY KEY (map_id, day, archived_at) +); + +CREATE INDEX idx_map_chronicle_snapshots_archive_map + ON map_chronicle_snapshots_archive(map_id, archived_at); + +-- Audit trail for staff-triggered chronicle wipes (and their restores). +-- `wiped_at` is the archive stamp the wipe used: it is both the audit clock and +-- the key that ties this row to its `map_chronicle_snapshots_archive` rows and +-- to the `chronicle.bak.` directory, so a restore can find both. +CREATE TABLE map_chronicle_wipes ( + id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + map_id text COLLATE "C" NOT NULL, + wiped_at bigint NOT NULL, + wiped_by text COLLATE "C" NOT NULL, + day_count bigint NOT NULL DEFAULT 0, + -- NULL when the wipe found index rows but no directory to set aside. + backup_path text COLLATE "C", + reason text COLLATE "C" NOT NULL, + restored_at bigint, + restored_by text COLLATE "C" +); + +CREATE INDEX idx_map_chronicle_wipes_map + ON map_chronicle_wipes(map_id, wiped_at); + +-- Economy ledger (SimpleFactions `chronicle` upload mode). Partitioned on the +-- captured_at UTC date; `server_day` is the in-game clock and drifts, so it is +-- carried alongside but never used as a key. +CREATE TABLE map_ledger_days ( + map_id text COLLATE "C" NOT NULL, + day text COLLATE "C" NOT NULL, + captured_at timestamptz NOT NULL, + captured_at_ts bigint NOT NULL, + server_day bigint, + day_progress_seconds bigint, + -- False when this day's canonical row is only the latest partial snapshot. + complete boolean NOT NULL DEFAULT false, + schema_version bigint NOT NULL DEFAULT 0, + faction_count bigint NOT NULL DEFAULT 0, + guild_count bigint NOT NULL DEFAULT 0, + globals jsonb NOT NULL, + PRIMARY KEY (map_id, day) +); + +-- Faction identity is (id, founded_at) hashed into faction_key: ids derive from +-- the faction name and are reused after deletion. +CREATE TABLE map_ledger_factions ( + map_id text COLLATE "C" NOT NULL, + faction_key text COLLATE "C" NOT NULL, + faction_id text COLLATE "C" NOT NULL, + founded_at text COLLATE "C" NOT NULL, + first_seen_day text COLLATE "C" NOT NULL, + first_seen_at timestamptz NOT NULL, + last_seen_day text COLLATE "C" NOT NULL, + last_seen_at timestamptz NOT NULL, + last_name text COLLATE "C", + last_rgb text COLLATE "C", + deleted_day text COLLATE "C", + deleted_at timestamptz, + PRIMARY KEY (map_id, faction_key) +); + +CREATE TABLE map_ledger_faction_days ( + map_id text COLLATE "C" NOT NULL, + day text COLLATE "C" NOT NULL, + faction_key text COLLATE "C" NOT NULL, + faction_id text COLLATE "C" NOT NULL, + founded_at text COLLATE "C" NOT NULL, + name text COLLATE "C", + rgb text COLLATE "C", + overlord text COLLATE "C", + wealth double precision, + bank double precision, + vassal_wealth double precision, + net_income double precision, + inflation_delta double precision, + trade_power double precision, + prestige double precision, + "rank" text COLLATE "C", + rank_level bigint, + rank_up_at double precision, + rank_down_at double precision, + prestige_position bigint, + wealth_position bigint, + provinces bigint, + realm_size bigint, + tier text COLLATE "C", + tier_index bigint, + highest_title text COLLATE "C", + members bigint, + members_with_vassals bigint, + settlements bigint, + population bigint, + installations bigint, + forts bigint, + wealth_breakdown jsonb NOT NULL DEFAULT '{}', + prestige_breakdown jsonb NOT NULL DEFAULT '{}', + subjects jsonb NOT NULL DEFAULT '[]', + wars jsonb NOT NULL DEFAULT '[]', + PRIMARY KEY (map_id, day, faction_key) +); + +-- No founded_at in the guild payload, so guild identity is weak: a deleted and +-- recreated guild reads as continuous. Documented limitation, no guild registry. +CREATE TABLE map_ledger_guild_days ( + map_id text COLLATE "C" NOT NULL, + day text COLLATE "C" NOT NULL, + guild_id text COLLATE "C" NOT NULL, + faction_id text COLLATE "C", + name text COLLATE "C", + type text COLLATE "C", + wealth double precision, + bank double precision, + expansions bigint, + trade_power double precision, + credit_score double precision, + size bigint, + PRIMARY KEY (map_id, day, guild_id) +); + +CREATE INDEX idx_map_ledger_factions_id + ON map_ledger_factions(map_id, faction_id); + +CREATE INDEX idx_map_ledger_faction_days_key + ON map_ledger_faction_days(map_id, faction_key, day); + +-- Patreon lives beside discord_links; link tombstones retain relink history. +CREATE TABLE patreon_members ( + member_id text COLLATE "C" PRIMARY KEY, + patreon_user_id text COLLATE "C" NOT NULL UNIQUE, + data_json jsonb NOT NULL, + last_tier text COLLATE "C", + declined_since timestamptz, + updated_at timestamptz NOT NULL +); + +CREATE TABLE patreon_links ( + patreon_user_id text COLLATE "C" PRIMARY KEY, + discord_user_id text COLLATE "C", + player_uuid text COLLATE "C", + method text COLLATE "C" NOT NULL, + active boolean NOT NULL DEFAULT true, + linked_at timestamptz NOT NULL, + person_changed_at timestamptz NOT NULL +); + +CREATE UNIQUE INDEX idx_patreon_link_discord + ON patreon_links(discord_user_id) WHERE active; + +CREATE UNIQUE INDEX idx_patreon_link_uuid + ON patreon_links(player_uuid) WHERE active; + +CREATE TABLE patreon_tokens ( + id integer PRIMARY KEY CHECK (id = 1), + access_token text COLLATE "C" NOT NULL, + refresh_token text COLLATE "C" NOT NULL, + expires_at timestamptz NOT NULL +); + +CREATE TABLE patreon_leases ( + name text COLLATE "C" PRIMARY KEY, + owner text COLLATE "C" NOT NULL, + expires_at timestamptz NOT NULL +); + +CREATE TABLE patreon_sync_state ( + id integer PRIMARY KEY CHECK (id = 1), + last_sync_at timestamptz, + last_sync_ok boolean, + consecutive_failures bigint NOT NULL DEFAULT 0, + brake_held boolean NOT NULL DEFAULT false +); + +CREATE TABLE patreon_applied ( + target text COLLATE "C" NOT NULL, + subject text COLLATE "C" NOT NULL, + tiers_json jsonb NOT NULL DEFAULT '[]', + ever_granted boolean NOT NULL DEFAULT false, + dm_generation bigint NOT NULL DEFAULT 0, + PRIMARY KEY (target, subject) +); + +CREATE TABLE patreon_desired ( + target text COLLATE "C" NOT NULL, + subject text COLLATE "C" NOT NULL, + tier_key text COLLATE "C", + grace_until timestamptz, + dm text COLLATE "C", + generation bigint NOT NULL DEFAULT 1, + link_event text COLLATE "C", + PRIMARY KEY (target, subject) +); + +CREATE TABLE patreon_changes ( + id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + target text COLLATE "C" NOT NULL, + subject text COLLATE "C" NOT NULL, + add_tier text COLLATE "C", + remove_json jsonb NOT NULL, + dm text COLLATE "C", + grace_until timestamptz, + generation bigint NOT NULL, + created_at timestamptz NOT NULL, + dispatched_at timestamptz, + acked_at timestamptz, + cancelled_at timestamptz, + dm_suppressed boolean NOT NULL DEFAULT false +); + +CREATE INDEX idx_patreon_changes_pending + ON patreon_changes(target, acked_at, cancelled_at, subject); + +CREATE TABLE patreon_alerts ( + id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + kind text COLLATE "C" NOT NULL, + message text COLLATE "C" NOT NULL, + created_at timestamptz NOT NULL, + acked_at timestamptz +); +CREATE UNIQUE INDEX idx_patreon_alerts_open + ON patreon_alerts(kind) WHERE acked_at IS NULL; + +CREATE TABLE patreon_brake_approvals ( + target text COLLATE "C" NOT NULL, + subject text COLLATE "C" NOT NULL, + generation bigint NOT NULL, + PRIMARY KEY (target, subject) +); + +-- OAuth state is stored hashed, single-use, and bound to the subject that started it. +CREATE TABLE patreon_oauth_states ( + state_hash text COLLATE "C" PRIMARY KEY, + discord_user_id text COLLATE "C", + player_uuid text COLLATE "C", + target_kind text COLLATE "C", + target_name text COLLATE "C", + expires_at timestamptz NOT NULL, + used_at timestamptz, + CHECK ((discord_user_id IS NULL) <> (player_uuid IS NULL)) +); + +-- Consent alone grants nothing; the patron must explicitly confirm this target. +CREATE TABLE patreon_pending_links ( + token_hash text COLLATE "C" PRIMARY KEY, + patreon_user_id text COLLATE "C" NOT NULL, + patreon_name text COLLATE "C" NOT NULL, + discord_user_id text COLLATE "C", + player_uuid text COLLATE "C", + target_kind text COLLATE "C" NOT NULL CHECK (target_kind IN ('discord', 'minecraft')), + target_name text COLLATE "C" NOT NULL, + expires_at timestamptz NOT NULL, + used_at timestamptz, + CHECK ((discord_user_id IS NULL) <> (player_uuid IS NULL)) +); + +-- Webhook member ids waiting for an out-of-band refresh (kept if the sync lease is busy). +CREATE TABLE patreon_webhook_members ( + member_id text COLLATE "C" PRIMARY KEY, + recorded_at timestamptz NOT NULL +); + +CREATE TABLE creation_catalog ( + id integer PRIMARY KEY CHECK (id = 1), + payload jsonb NOT NULL, + updated_at timestamptz NOT NULL +); + +CREATE TABLE character_creates ( + id text COLLATE "C" PRIMARY KEY, + player_uuid text COLLATE "C" NOT NULL, + client_request_id text COLLATE "C", + payload jsonb NOT NULL, + status text COLLATE "C" NOT NULL, + character_id text COLLATE "C", + error text COLLATE "C", + created_at timestamptz NOT NULL, + applied_at timestamptz, + realm_id text COLLATE "C" NOT NULL DEFAULT 'main', + wardrobe_active_slot text COLLATE "C", + held_until timestamptz +); + +CREATE TABLE character_roster ( + player_uuid text COLLATE "C" NOT NULL, + realm_id text COLLATE "C" NOT NULL DEFAULT 'main', + character_id text COLLATE "C" NOT NULL, + name text COLLATE "C" NOT NULL, + status text COLLATE "C" NOT NULL, + race text COLLATE "C", + class text COLLATE "C", + created_at text COLLATE "C", + updated_at timestamptz NOT NULL, + kit_status text COLLATE "C", + kit_statuses_json jsonb, + sheet_json jsonb, + wardrobe_active_slot text COLLATE "C", + died_at timestamptz, + death_id text COLLATE "C", + will_heir_id text COLLATE "C", + will_heir_name text COLLATE "C", + will_revision bigint, + PRIMARY KEY (player_uuid, realm_id, character_id) +); + +-- One row per death, from RPCharacters' roster pushes. A push never deletes these. +CREATE TABLE character_estates ( + realm_id text COLLATE "C" NOT NULL, + death_id text COLLATE "C" NOT NULL, + player_uuid text COLLATE "C" NOT NULL, + character_id text COLLATE "C" NOT NULL, + status text COLLATE "C" NOT NULL, + revision bigint NOT NULL, + source_updated_at timestamptz, + payload jsonb NOT NULL, + received_at timestamptz NOT NULL, + PRIMARY KEY (realm_id, death_id) +); + +CREATE INDEX idx_character_estates_character + ON character_estates(realm_id, character_id); + +-- Will changes made on Profile, pulled and acknowledged by RPCharacters. +CREATE TABLE will_requests ( + id text COLLATE "C" PRIMARY KEY, + realm_id text COLLATE "C" NOT NULL, + character_id text COLLATE "C" NOT NULL, + account_uuid text COLLATE "C" NOT NULL, + heir_character_id text COLLATE "C", + base_revision bigint NOT NULL, + created_at timestamptz NOT NULL, + status text COLLATE "C" NOT NULL CHECK (status IN ('pending', 'acked', 'replaced')), + result text COLLATE "C", + acked_at timestamptz +); + +CREATE INDEX idx_will_requests_pending + ON will_requests(realm_id, status, created_at); + +-- At most one pending request per character. +CREATE UNIQUE INDEX idx_will_requests_one_pending + ON will_requests(realm_id, character_id) WHERE status = 'pending'; + +CREATE INDEX idx_will_requests_character + ON will_requests(realm_id, character_id, created_at); + +CREATE TABLE character_player_meta ( + player_uuid text COLLATE "C" PRIMARY KEY, + -- Age-only upserts need to leave the character limit unset. + max_alive_characters bigint, + eighteen boolean, + real_age_set boolean NOT NULL DEFAULT false, + account_created_at_epoch bigint, + name_colour_stops bigint, + updated_at timestamptz NOT NULL, + kit_cooldown_seconds_remaining bigint, + kit_cooldown_hours bigint, + kit_cooldowns_json jsonb, + wardrobe_skin_slots bigint +); + +CREATE INDEX idx_discord_links_grace + ON discord_links(grace_until); + +CREATE INDEX idx_submissions_texture_hash + ON submissions(player_uuid, texture_hash); + +CREATE INDEX idx_character_creates_realm_status + ON character_creates(realm_id, status); + +CREATE UNIQUE INDEX idx_character_creates_client_req + ON character_creates(player_uuid, client_request_id) + WHERE client_request_id IS NOT NULL AND TRIM(client_request_id) != ''; + +CREATE INDEX idx_character_creates_pending + ON character_creates(status, created_at); + +CREATE INDEX idx_lore_item_customisations_realm_state + ON lore_item_customisations(realm_id, state); + +CREATE INDEX idx_submissions_realm_apply + ON submissions(realm_id, status, applied_at); + +CREATE INDEX idx_drink_submissions_realm_apply + ON drink_submissions(realm_id, status, applied_at); + +-- Website accounts. Discord is the sign-in provider; the Minecraft account is +-- the discord_links row for the same discord_user_id, so links made through +-- the Discord bot appear on the website without another step. +CREATE TABLE users ( + id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + discord_user_id text COLLATE "C" NOT NULL UNIQUE, + discord_username text COLLATE "C", + discord_global_name text COLLATE "C", + discord_avatar text COLLATE "C", + created_at timestamptz NOT NULL, + updated_at timestamptz NOT NULL, + last_login_at timestamptz NOT NULL, + -- Website staff role. + role text COLLATE "C" NOT NULL DEFAULT 'player' CHECK (role IN ('player', 'mod', 'admin', 'root')), + -- True for an account made so an admin can view the site as a player who has + -- never signed in (src/auth/view_as.py). Deleted once its views end; + -- signing in makes it a real account. + placeholder boolean NOT NULL DEFAULT false +); + +CREATE INDEX idx_users_role ON users(role); + +-- Cookie sessions for users. Separate from `sessions`, which belong to +-- in-game codes and carry a scope and realm. guild_member records the TFMC +-- Discord membership seen at sign-in; linking requires a recent check. +-- acting_user_id is the admin viewing the site as this user (view only, +-- src/auth/view_as.py). +CREATE TABLE user_sessions ( + id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + token_hash text COLLATE "C" NOT NULL UNIQUE, + user_id bigint NOT NULL REFERENCES users(id) ON DELETE CASCADE, + guild_member boolean NOT NULL DEFAULT false, + guild_checked_at timestamptz, + created_at timestamptz NOT NULL, + expires_at timestamptz NOT NULL, + acting_user_id bigint REFERENCES users(id) ON DELETE CASCADE +); + +CREATE INDEX idx_user_sessions_user ON user_sessions(user_id); +CREATE INDEX idx_user_sessions_acting ON user_sessions(acting_user_id); + +CREATE INDEX idx_user_sessions_expiry ON user_sessions(expires_at); + +-- Single-use Discord OAuth states. The browser also holds the state in a +-- cookie, so a callback only completes in the browser that started it. +CREATE TABLE discord_oauth_states ( + state_hash text COLLATE "C" PRIMARY KEY, + return_to text COLLATE "C" NOT NULL, + expires_at timestamptz NOT NULL +); + +CREATE INDEX idx_discord_link_codes_player ON discord_link_codes(player_uuid); + +-- Single-use tickets that carry a dev sign-in to a branch preview, which +-- redeems one with dev within two minutes (src/auth/preview_sign_in.py). +CREATE TABLE preview_sign_in_tickets ( + ticket_hash text COLLATE "C" PRIMARY KEY, + site text COLLATE "C" NOT NULL, + user_id bigint NOT NULL REFERENCES users(id) ON DELETE CASCADE, + guild_member boolean NOT NULL, + guild_checked_at timestamptz, + expires_at timestamptz NOT NULL +); + +-- Single-use Microsoft link attempts, tied to the site session that started +-- them. The callback marks a row used and deletes it once done; unlinking +-- deletes the user's rows, which cancels any attempt still in flight. A new +-- attempt replaces the user's last one (INSERT ... ON CONFLICT (user_id)). +CREATE TABLE microsoft_link_states ( + state_hash text COLLATE "C" PRIMARY KEY, + session_id bigint NOT NULL, + user_id bigint NOT NULL UNIQUE, + code_verifier text COLLATE "C" NOT NULL, + expires_at timestamptz NOT NULL, + used_at timestamptz +); + +-- Append-only record of staff actions on website accounts, including refused +-- attempts; the triggers below refuse UPDATE, DELETE and TRUNCATE. No foreign +-- keys: rows outlive the accounts they mention, so each row keeps the names it +-- was written with. +CREATE TABLE admin_audit ( + id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + created_at timestamptz NOT NULL, + actor_type text COLLATE "C" NOT NULL CHECK (actor_type IN ('user', 'system')), + actor_user_id bigint, + actor_discord_id text COLLATE "C", + actor_name text COLLATE "C", + actor_role text COLLATE "C", + action text COLLATE "C" NOT NULL, + outcome text COLLATE "C" NOT NULL, + target_user_id bigint, + target_discord_id text COLLATE "C", + target_name text COLLATE "C", + reason text COLLATE "C", + detail_json jsonb NOT NULL DEFAULT '{}' +); + +CREATE INDEX idx_admin_audit_target ON admin_audit(target_user_id, id); + +CREATE INDEX idx_admin_audit_actor ON admin_audit(actor_user_id, id); + +CREATE FUNCTION admin_audit_append_only() RETURNS trigger LANGUAGE plpgsql AS $$ +BEGIN + RAISE EXCEPTION 'admin_audit is append-only'; +END; +$$; + +-- An upsert that meets an existing id fires the UPDATE trigger too. +CREATE TRIGGER admin_audit_no_change BEFORE UPDATE OR DELETE ON admin_audit + FOR EACH ROW EXECUTE FUNCTION admin_audit_append_only(); + +-- Row triggers do not fire on TRUNCATE. +CREATE TRIGGER admin_audit_no_truncate BEFORE TRUNCATE ON admin_audit + FOR EACH STATEMENT EXECUTE FUNCTION admin_audit_append_only(); + +-- LuckPerms mirror for the staff panel. The TFMCWeb bridge publishes a whole +-- snapshot, which replaces these tables in one transaction; see +-- src/luckperms/README.md. Contexts are canonical JSON ({} when global) and +-- expiry is unix seconds, 0 when permanent. +CREATE TABLE lp_state ( + id integer PRIMARY KEY CHECK (id = 1), + server text COLLATE "C", + hash text COLLATE "C", + generated_at bigint, + -- Newest bridge revision applied: a snapshot's, or a change result's. + revision bigint, + received_at timestamptz, + -- Last time the bridge confirmed the snapshot is still current. + checked_at timestamptz, + -- Last time an applying bridge asked for changes. + polled_at timestamptz +); + +CREATE TABLE lp_groups ( + name text COLLATE "C" PRIMARY KEY, + display_name text COLLATE "C", + weight bigint +); + +CREATE TABLE lp_group_nodes ( + group_name text COLLATE "C" NOT NULL, + key text COLLATE "C" NOT NULL, + value boolean NOT NULL, + contexts jsonb NOT NULL, + expiry bigint NOT NULL +); + +CREATE INDEX idx_lp_group_nodes_group ON lp_group_nodes(group_name); + +CREATE TABLE lp_tracks ( + name text COLLATE "C" PRIMARY KEY, + groups_json jsonb NOT NULL +); + +CREATE TABLE lp_users ( + uuid text COLLATE "C" PRIMARY KEY, + name text COLLATE "C" +); + +CREATE INDEX idx_lp_users_name ON lp_users(lower(name)); + +CREATE TABLE lp_user_nodes ( + uuid text COLLATE "C" NOT NULL, + key text COLLATE "C" NOT NULL, + value boolean NOT NULL, + contexts jsonb NOT NULL, + expiry bigint NOT NULL +); + +CREATE INDEX idx_lp_user_nodes_uuid ON lp_user_nodes(uuid); + +CREATE INDEX idx_lp_user_nodes_key ON lp_user_nodes(key); + +-- The server's commands and each group's access, as TFMCWeb last published +-- them for the Commands tab; see src/luckperms/commands.py. +CREATE TABLE lp_commands ( + id integer PRIMARY KEY CHECK (id = 1), + server text COLLATE "C", + hash text COLLATE "C", + generated_at bigint, + received_at timestamptz, + payload jsonb NOT NULL +); + +-- Changes staff queued for the bridge. Rows are kept as the change history. +CREATE TABLE lp_changes ( + id bigint GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY, + created_at timestamptz NOT NULL, + actor_user_id bigint NOT NULL, + actor_name text COLLATE "C", + actor_role text COLLATE "C", + actor_minecraft_uuid text COLLATE "C", + target_type text COLLATE "C" NOT NULL CHECK (target_type IN ('user', 'group', 'track')), + target text COLLATE "C" NOT NULL, + target_name text COLLATE "C", + description text COLLATE "C" NOT NULL, + ops_json jsonb NOT NULL, + reason text COLLATE "C" NOT NULL, + status text COLLATE "C" NOT NULL DEFAULT 'pending' + CHECK (status IN ('pending', 'sent', 'applied', 'failed', 'expired', 'unknown')), + attempts bigint NOT NULL DEFAULT 0, + sent_at timestamptz, + finished_at timestamptz, + error text COLLATE "C" +); + +CREATE INDEX idx_lp_changes_status ON lp_changes(status, id); + +CREATE INDEX idx_lp_changes_target ON lp_changes(target_type, target, id); + +-- At most one open change per target. +CREATE UNIQUE INDEX idx_lp_changes_one_open + ON lp_changes(target_type, target) WHERE status IN ('pending', 'sent'); + +-- Reports players send from /diagnostic, looked up by code; kept 30 days. +CREATE TABLE diagnostic_reports ( + code text COLLATE "C" PRIMARY KEY, + created_at timestamptz NOT NULL, + user_id bigint, + discord_name text COLLATE "C", + minecraft_name text COLLATE "C", + country text COLLATE "C", + worst text COLLATE "C" NOT NULL CHECK (worst IN ('fail', 'warn', 'info', 'ok')), + summary text COLLATE "C" NOT NULL, + report_json jsonb NOT NULL +); + +CREATE INDEX idx_diagnostic_reports_created ON diagnostic_reports(created_at); diff --git a/backend/src/database/seeds.sql b/backend/src/database/seeds.sql new file mode 100644 index 00000000..6ea20915 --- /dev/null +++ b/backend/src/database/seeds.sql @@ -0,0 +1,4 @@ +-- Rows the site needs, inserted at start-up after the migrations. Every +-- statement must be safe to run again and must not overwrite existing rows. + +INSERT INTO patreon_sync_state (id) VALUES (1) ON CONFLICT (id) DO NOTHING; diff --git a/backend/src/database/test_connection.py b/backend/src/database/test_connection.py new file mode 100644 index 00000000..9baa04ff --- /dev/null +++ b/backend/src/database/test_connection.py @@ -0,0 +1,291 @@ +"""The pool, transactions and locking helpers against a real Postgres.""" + +import threading +from datetime import datetime, timezone + +import pytest +from psycopg.rows import tuple_row + +from src import database +from src.database import connection + +pytestmark = pytest.mark.usefixtures("postgres") + + +def _warn(conn, player="p", reason="r"): + conn.execute( + "INSERT INTO player_warnings (player_uuid, reason, created_at) VALUES (%s, %s, now())", (player, reason)) + + +def _warnings(player="p") -> int: + with database.connect() as conn: + return conn.execute("SELECT count(*) AS n FROM player_warnings WHERE player_uuid = %s", (player,)).fetchone()["n"] + + +def test_block_commits_and_rows_are_dicts(): + with database.connect() as conn: + row = conn.execute( + "INSERT INTO player_warnings (player_uuid, reason, created_at) VALUES ('p', 'r', %s) " + "RETURNING id, reason, created_at", + ("2026-10-16T10:00:00Z",), + ).fetchone() + assert row == {"id": 1, "reason": "r", "created_at": datetime(2026, 10, 16, 10, tzinfo=timezone.utc)} + assert _warnings() == 1 + + +def test_exception_rolls_back_and_the_connection_is_reusable(): + with pytest.raises(RuntimeError): + with database.connect() as conn: + _warn(conn) + raise RuntimeError + assert _warnings() == 0 + with database.connect() as conn: + assert conn.info.transaction_status.name == "IDLE" + + +def test_savepoint_survives_a_unique_violation(): + with database.connect() as conn: + conn.execute("INSERT INTO patreon_leases (name, owner, expires_at) VALUES ('sync', 'a', now())") + with pytest.raises(database.UniqueViolation): + with conn.transaction(): + conn.execute("INSERT INTO patreon_leases (name, owner, expires_at) VALUES ('sync', 'b', now())") + owner = conn.execute("SELECT owner FROM patreon_leases WHERE name = 'sync'").fetchone()["owner"] + assert owner == "a" + + +def test_session_settings(): + with database.connect() as conn: + settings = {name: next(iter(conn.execute(f"SHOW {name}").fetchone().values())) + for name in ("TimeZone", "lock_timeout", "statement_timeout", + "idle_in_transaction_session_timeout")} + assert conn.execute("SHOW application_name").fetchone()["application_name"] == "provincesystem" + search_path = conn.execute("SHOW search_path").fetchone()["search_path"] + assert settings == {"TimeZone": "UTC", "lock_timeout": "5s", "statement_timeout": "30s", + "idle_in_transaction_session_timeout": "1min"} + assert search_path == "public" + + +@pytest.mark.parametrize(("isolation", "expected"), [ + (None, "read committed"), + ("repeatable read", "repeatable read"), + ("SERIALIZABLE", "serializable"), +]) +def test_transaction_isolation(isolation, expected): + with database.transaction(isolation=isolation) as conn: + assert conn.execute("SHOW transaction_isolation").fetchone()["transaction_isolation"] == expected + # The level applied to that transaction only, not to the pooled connection. + with database.connect() as conn: + assert conn.execute("SHOW transaction_isolation").fetchone()["transaction_isolation"] == "read committed" + + +def test_transaction_read_only_refuses_writes(): + with pytest.raises(database.DatabaseError, match="read-only"): + with database.transaction(read_only=True) as conn: + _warn(conn) + + +def test_transaction_rejects_an_unknown_isolation_level(): + with pytest.raises(ValueError): + with database.transaction(isolation="chaos"): + pass + + +def test_transaction_lock_serialises_holders(): + holding, release = threading.Event(), threading.Event() + order = [] + + def first(): + with database.transaction(lock="test:lock") as conn: + holding.set() + release.wait(5) + _warn(conn, reason="first") + order.append("first") + + thread = threading.Thread(target=first) + thread.start() + assert holding.wait(5) + waiter = threading.Thread(target=lambda: _second(order)) + waiter.start() + waiter.join(0.3) + assert waiter.is_alive(), "the second holder must wait for the first to commit" + release.set() + thread.join(5) + waiter.join(5) + assert order == ["first", "second saw 1"] + + +def _second(order): + with database.transaction(lock="test:lock") as conn: + n = conn.execute("SELECT count(*) AS n FROM player_warnings").fetchone()["n"] + order.append(f"second saw {n}") + + +def test_advisory_lock_without_waiting(): + holding, release = threading.Event(), threading.Event() + + def hold(): + with database.connect() as conn: + assert database.advisory_lock(conn, "test:try") + holding.set() + release.wait(5) + + thread = threading.Thread(target=hold) + thread.start() + assert holding.wait(5) + try: + with database.connect() as conn: + assert database.advisory_lock(conn, "test:try", wait=False) is False + assert database.advisory_lock(conn, "test:other", wait=False) is True + finally: + release.set() + thread.join(5) + with database.connect() as conn: + assert database.advisory_lock(conn, "test:try", wait=False) is True + + +def test_advisory_lock_refuses_autocommit(): + with database.connect() as conn: + conn.autocommit = True + try: + with pytest.raises(RuntimeError, match="autocommit"): + database.advisory_lock(conn, "test:auto") + finally: + conn.autocommit = False + + +def test_serializable_retries_write_skew(): + # Both transactions read "no warning yet" before either inserts; SERIALIZABLE + # aborts one, and its retry sees the other's row. + barrier = threading.Barrier(2, timeout=5) + attempts = [] + + def work(conn): + attempts.append(1) + n = conn.execute("SELECT count(*) AS n FROM player_warnings WHERE player_uuid = 'p'").fetchone()["n"] + if len(attempts) <= 2: + barrier.wait() + if n == 0: + _warn(conn) + return "inserted" + return "present" + + results = [] + threads = [threading.Thread(target=lambda: results.append(database.serializable(work))) for _ in range(2)] + for thread in threads: + thread.start() + for thread in threads: + thread.join(10) + assert sorted(results) == ["inserted", "present"] + assert len(attempts) == 3 + assert _warnings() == 1 + + +def test_serializable_gives_up_after_its_attempts(): + calls = [] + + def work(conn): + calls.append(1) + raise database.SerializationFailure("simulated") + + with pytest.raises(database.SerializationFailure): + database.serializable(work, attempts=3) + assert len(calls) == 3 + + +def test_pool_opens_from_database_url_on_first_use(postgres, monkeypatch): + connection.close_pool() + monkeypatch.delenv("DATABASE_URL") + with pytest.raises(database.NotConfiguredError): + database.connect().__enter__() + monkeypatch.setenv("DATABASE_URL", postgres) + with database.connect() as conn: + assert conn.execute("SELECT current_database() AS db").fetchone()["db"].startswith("ps_test_") + assert database.get_pool() is database.get_pool() + + +def test_pool_sizes_come_from_the_environment(postgres, monkeypatch): + monkeypatch.setenv("DATABASE_POOL_MIN_SIZE", "0") + monkeypatch.setenv("DATABASE_POOL_MAX_SIZE", "3") + pool = database.open_pool(postgres) + assert (pool.min_size, pool.max_size) == (0, 3) + assert database.get_pool() is pool + + +def test_a_returned_connection_leaks_nothing(postgres): + database.open_pool(postgres, min_size=1, max_size=1) + pids = set() + for _ in range(2): # the same connection, reset twice + with database.connect() as conn: + pids.add(conn.info.backend_pid) + conn.execute("SET TimeZone = 'Europe/London'") + conn.execute("SET statement_timeout = '1s'") + conn.execute("SET search_path = pg_catalog") + conn.execute("CREATE TEMP TABLE leak (x int)") + conn.execute("LISTEN leak") + conn.execute("SELECT pg_advisory_lock(42)") + for _ in range(6): # past psycopg's prepare threshold + conn.execute("SELECT 1 AS one WHERE %s", (True,)) + conn.commit() + conn.autocommit = True + conn.read_only = True + conn.row_factory = tuple_row + with database.connect() as conn: + assert (conn.autocommit, conn.read_only, conn.isolation_level) == (False, None, None) + settings = {name: next(iter(conn.execute(f"SHOW {name}").fetchone().values())) + for name in ("TimeZone", "statement_timeout", "search_path")} + assert settings == {"TimeZone": "UTC", "statement_timeout": "30s", "search_path": "public"} + assert conn.execute("SELECT to_regclass('pg_temp.leak') AS t").fetchone() == {"t": None} + assert conn.execute("SELECT count(*) AS n FROM pg_listening_channels()").fetchone()["n"] == 0 + assert conn.execute("SELECT count(*) AS n FROM pg_locks WHERE locktype = 'advisory' " + "AND pid = pg_backend_pid()").fetchone()["n"] == 0 + assert conn.execute("SELECT 1 AS one WHERE %s", (True,)).fetchone() == {"one": 1} + assert len(pids) == 1 + + +def test_a_leaked_autocommit_cannot_commit_a_failed_block(postgres): + database.open_pool(postgres, min_size=1, max_size=1) + with database.connect() as conn: + conn.autocommit = True + with pytest.raises(RuntimeError): + with database.connect() as conn: + _warn(conn) + raise RuntimeError + assert _warnings() == 0 + + +def _hold(name, holding, release): + with database.connect() as conn: + database.advisory_lock(conn, name) + holding.set() + release.wait(5) + + +def test_advisory_lock_waits_past_lock_timeout(): + holding, release = threading.Event(), threading.Event() + thread = threading.Thread(target=_hold, args=("test:slow", holding, release)) + thread.start() + assert holding.wait(5) + threading.Timer(1.0, release.set).start() + try: + with database.connect() as conn: + conn.execute("SET LOCAL lock_timeout = '300ms'") + assert database.advisory_lock(conn, "test:slow") + assert conn.execute("SHOW lock_timeout").fetchone()["lock_timeout"] == "300ms" + finally: + release.set() + thread.join(5) + + +def test_advisory_lock_gives_up_after_its_own_timeout(monkeypatch): + monkeypatch.setattr(connection, "ADVISORY_LOCK_TIMEOUT", "300ms") + holding, release = threading.Event(), threading.Event() + thread = threading.Thread(target=_hold, args=("test:stuck", holding, release)) + thread.start() + assert holding.wait(5) + try: + with pytest.raises(database.LockNotAvailable): + with database.transaction(lock="test:stuck"): + pass + finally: + release.set() + thread.join(5) diff --git a/backend/src/database/test_migrate.py b/backend/src/database/test_migrate.py new file mode 100644 index 00000000..24d6e8cb --- /dev/null +++ b/backend/src/database/test_migrate.py @@ -0,0 +1,122 @@ +"""The migration runner against an empty scratch database.""" + +import threading + +import psycopg +import pytest + +from src.database import migrate as runner + + +def _write(directory, name, sql): + (directory / name).write_text(sql, encoding="utf-8") + + +def _rows(url, sql): + with psycopg.connect(url) as conn: + return conn.execute(sql).fetchall() + + +@pytest.fixture +def migrations(tmp_path): + _write(tmp_path, "0001_first.sql", "CREATE TABLE a (id bigint PRIMARY KEY);") + _write(tmp_path, "0002_second.sql", "ALTER TABLE a ADD COLUMN note text; INSERT INTO a VALUES (1, '100%');") + return tmp_path + + +def test_applies_in_order_once(scratch_database, migrations): + assert runner.migrate(scratch_database, directory=migrations) == ["0001", "0002"] + assert runner.migrate(scratch_database, directory=migrations) == [] + assert _rows(scratch_database, "SELECT id, note FROM a") == [(1, "100%")] + assert _rows(scratch_database, "SELECT version, name FROM schema_migrations ORDER BY version") == [ + ("0001", "first"), ("0002", "second")] + + +def test_a_failing_file_leaves_nothing_behind(scratch_database, migrations): + _write(migrations, "0003_broken.sql", "CREATE TABLE b (id bigint); SELECT 1/0;") + with pytest.raises(psycopg.errors.DivisionByZero): + runner.migrate(scratch_database, directory=migrations) + assert _rows(scratch_database, "SELECT version FROM schema_migrations ORDER BY version") == [ + ("0001",), ("0002",)] + assert _rows(scratch_database, "SELECT to_regclass('b')") == [(None,)] + + +def test_an_applied_file_must_not_change(scratch_database, migrations): + runner.migrate(scratch_database, directory=migrations) + _write(migrations, "0001_first.sql", "CREATE TABLE a (id bigint PRIMARY KEY, extra text);") + with pytest.raises(runner.MigrationError, match="0001_first changed"): + runner.migrate(scratch_database, directory=migrations) + + +def test_an_applied_version_needs_its_file(scratch_database, migrations): + runner.migrate(scratch_database, directory=migrations) + (migrations / "0002_second.sql").unlink() + with pytest.raises(runner.MigrationError, match="no file: 0002"): + runner.migrate(scratch_database, directory=migrations) + + +def test_a_new_file_must_be_numbered_above_the_applied_ones(scratch_database, migrations): + (migrations / "0002_second.sql").rename(migrations / "0003_second.sql") + runner.migrate(scratch_database, directory=migrations) + _write(migrations, "0002_late.sql", "CREATE TABLE late (id bigint);") + with pytest.raises(runner.MigrationError, match="0002_late is numbered below the newest applied one"): + runner.migrate(scratch_database, directory=migrations) + assert _rows(scratch_database, "SELECT to_regclass('late')") == [(None,)] + + +def test_seed_rows_are_inserted_once(scratch_database, migrations, tmp_path_factory, monkeypatch): + seeds = tmp_path_factory.mktemp("seeds") / "seeds.sql" + seeds.write_text("INSERT INTO a (id, note) VALUES (2, 'seed') ON CONFLICT (id) DO NOTHING;", encoding="utf-8") + monkeypatch.setattr(runner, "SEEDS_PATH", seeds) + runner.migrate(scratch_database, directory=migrations) + assert _rows(scratch_database, "SELECT id FROM a ORDER BY id") == [(1,)] + runner.migrate(scratch_database, directory=migrations, seed=True) + runner.migrate(scratch_database, directory=migrations, seed=True) + assert _rows(scratch_database, "SELECT id, note FROM a ORDER BY id") == [(1, "100%"), (2, "seed")] + + +@pytest.mark.parametrize("names", [["1_bad.sql"], ["0001_Upper.sql"], ["0001_a.sql", "0001_b.sql"]]) +def test_file_names_are_checked(tmp_path, names): + for name in names: + _write(tmp_path, name, "SELECT 1;") + with pytest.raises(runner.MigrationError): + runner.discover(tmp_path) + + +def test_concurrent_runners_apply_each_file_once(scratch_database, tmp_path): + _write(tmp_path, "0001_slow.sql", "SELECT pg_sleep(0.5); CREATE TABLE a (id bigint);") + results, errors = [], [] + + def run(): + try: + results.append(runner.migrate(scratch_database, directory=tmp_path)) + except Exception as exc: # pragma: no cover - reported below + errors.append(exc) + + threads = [threading.Thread(target=run) for _ in range(3)] + for thread in threads: + thread.start() + for thread in threads: + thread.join(10) + assert errors == [] + assert sorted(results) == [[], [], ["0001"]] + + +def test_status_and_cli(scratch_database, migrations, monkeypatch, capsys): + assert [(v, n, at) for v, n, at in runner.status(scratch_database, directory=migrations)] == [ + ("0001", "first", None), ("0002", "second", None)] + monkeypatch.setattr(runner, "MIGRATIONS_DIR", migrations) + monkeypatch.setenv("DATABASE_URL", scratch_database) + assert runner.main([]) == 0 + assert runner.main(["--status"]) == 0 + out = capsys.readouterr().out.splitlines() + assert out[0] == "applied 0001, 0002" + assert out[1].startswith("0001_first\t2") and out[2].startswith("0002_second\t2") + + +def test_real_migrations_apply_twice_cleanly(scratch_database): + assert runner.migrate(scratch_database) == [m.version for m in runner.discover()] + assert _rows(scratch_database, "SELECT count(*) FROM patreon_sync_state") == [(0,)] + assert runner.migrate(scratch_database, seed=True) == [] + assert runner.migrate(scratch_database, seed=True) == [] + assert _rows(scratch_database, "SELECT id FROM patreon_sync_state") == [(1,)] diff --git a/backend/src/database/test_schema.py b/backend/src/database/test_schema.py new file mode 100644 index 00000000..3069bb24 --- /dev/null +++ b/backend/src/database/test_schema.py @@ -0,0 +1,268 @@ +"""The baseline schema: parity with schema.sql, types, collation and admin_audit.""" + +import re +import sqlite3 + +import psycopg +import pytest + +from src import database +from src.database.testing import tables + +# Postgres types each SQLite affinity may become. +_ALLOWED = { + "INTEGER": {"bigint", "integer", "boolean"}, + "REAL": {"double precision"}, + "TEXT": {"text", "timestamptz", "jsonb"}, +} +_PG_TYPE = {"timestamp with time zone": "timestamptz"} +# Constraints Postgres adds for read-check-write paths that SQLite serialised +# with BEGIN IMMEDIATE instead. +_NEW_UNIQUE = { + ("microsoft_link_states", ("user_id",)), +} +_ON_DELETE = {"a": "NO ACTION", "r": "RESTRICT", "c": "CASCADE", "n": "SET NULL", "d": "SET DEFAULT"} +_CAST = re.compile(r"::(?:text|jsonb|integer|bigint|boolean)\b") + + +def _expr(text: str, booleans=()) -> str: + """An SQL expression in one spelling, so SQLite's and Postgres's can be compared.""" + text = re.sub(r"(\w+) = ANY \(ARRAY\[(.*?)\]\)", r"\1 IN (\2)", text) + text = _CAST.sub("", text) + for column in booleans: + text = re.sub(rf"\b{column} = 1\b", column, text) + text = re.sub(rf"\b{column} = 0\b", f"NOT {column}", text) + text = text.lower().replace("!=", "<>").replace("both from", "") + return re.sub(r"[\s()]", "", text) + + +def _default(value: str | None) -> str | None: + if value is None: + return None + value = {"false": "0", "true": "1"}.get(_CAST.sub("", value), _CAST.sub("", value)) + return value.strip("'") if re.fullmatch(r"'-?\d+'", value) else value + + +def _checks(create_sql: str) -> list[str]: + """The CHECK expressions in a SQLite CREATE TABLE statement.""" + out = [] + for match in re.finditer(r"\bCHECK\s*\(", create_sql): + depth, i = 1, match.end() + while depth: + depth += {"(": 1, ")": -1}.get(create_sql[i], 0) + i += 1 + out.append(create_sql[match.end():i - 1]) + return out + + +@pytest.fixture +def sqlite_schema(tmp_path, monkeypatch): + from src.skins import db + + monkeypatch.setattr(db, "DB_PATH", tmp_path / "province.db") + monkeypatch.setattr(db, "DATA_DIR", tmp_path) + for name in ("SKINS_DIR", "WARDROBE_DIR", "DRINKS_DIR"): + monkeypatch.setattr(db, name, tmp_path / name.lower()) + db.migrate() + conn = sqlite3.connect(tmp_path / "province.db") + try: + yield conn + finally: + conn.close() + + +def _sqlite_tables(conn) -> dict: + out = {} + names = [r[0] for r in conn.execute( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name <> 'sqlite_sequence' ORDER BY name")] + for table in names: + columns = conn.execute(f"PRAGMA table_info({table})").fetchall() + uniques = set() + for index in conn.execute(f"PRAGMA index_list({table})").fetchall(): + _, name, unique, origin, partial = index + if unique and not partial and origin != "pk": + uniques.add(tuple(r[2] for r in conn.execute(f"PRAGMA index_info({name})"))) + create_sql = conn.execute("SELECT sql FROM sqlite_master WHERE name = ?", (table,)).fetchone()[0] + partial = {name: sql.split(" WHERE ", 1)[1] for name, sql in conn.execute( + "SELECT name, sql FROM sqlite_master WHERE type = 'index' AND tbl_name = ? AND sql LIKE '% WHERE %'", + (table,))} + out[table] = { + "columns": [(c[1], c[2].upper()) for c in columns], + "not_null": {c[1] for c in columns if c[3] or c[5]}, + "defaults": {c[1]: c[4] for c in columns}, + "pk": tuple(c[1] for c in sorted(columns, key=lambda c: c[5]) if c[5]), + "unique": uniques, + "fk": {(f[3], f[2], f[4], f[6]) for f in conn.execute(f"PRAGMA foreign_key_list({table})")}, + "checks": _checks(create_sql), + "partial": partial, + } + return out + + +def _pg_tables(conn) -> dict: + out = {} + for table in tables(conn): + columns = conn.execute( + "SELECT column_name, data_type, is_nullable, collation_name, column_default " + "FROM information_schema.columns " + "WHERE table_schema = 'public' AND table_name = %s ORDER BY ordinal_position", (table,)).fetchall() + keys = conn.execute( + """ + SELECT c.contype, i.indisunique, array_agg(a.attname ORDER BY k.n) AS cols, i.indpred IS NOT NULL AS partial + FROM pg_index i + JOIN pg_class t ON t.oid = i.indrelid + CROSS JOIN LATERAL unnest(i.indkey) WITH ORDINALITY AS k(attnum, n) + JOIN pg_attribute a ON a.attrelid = t.oid AND a.attnum = k.attnum + LEFT JOIN pg_constraint c ON c.conindid = i.indexrelid AND c.conrelid = t.oid + WHERE t.relname = %s AND t.relnamespace = 'public'::regnamespace + GROUP BY i.indexrelid, c.contype, i.indisunique, i.indpred + """, (table,)).fetchall() + fks = conn.execute( + """ + SELECT a.attname AS col, f.relname AS ref, af.attname AS ref_col, c.confdeltype AS on_delete + FROM pg_constraint c + JOIN pg_class f ON f.oid = c.confrelid + JOIN pg_attribute a ON a.attrelid = c.conrelid AND a.attnum = c.conkey[1] + JOIN pg_attribute af ON af.attrelid = c.confrelid AND af.attnum = c.confkey[1] + WHERE c.contype = 'f' AND c.conrelid = %s::regclass + """, (table,)).fetchall() + out[table] = { + "columns": [(c["column_name"], _PG_TYPE.get(c["data_type"], c["data_type"])) for c in columns], + "not_null": {c["column_name"] for c in columns if c["is_nullable"] == "NO"}, + "collations": {c["column_name"]: c["collation_name"] for c in columns}, + "pk": next((tuple(k["cols"]) for k in keys if k["contype"] == "p"), ()), + "unique": {tuple(k["cols"]) for k in keys if k["indisunique"] and k["contype"] != "p" and not k["partial"]}, + "fk": {(f["col"], f["ref"], f["ref_col"], _ON_DELETE[f["on_delete"]]) for f in fks}, + "defaults": {c["column_name"]: c["column_default"] for c in columns}, + "checks": [r["def"][len("CHECK "):] for r in conn.execute( + "SELECT pg_get_constraintdef(oid) AS def FROM pg_constraint WHERE conrelid = %s::regclass " + "AND contype = 'c'", (table,))], + "partial": {r["name"]: r["pred"] for r in conn.execute( + "SELECT c.relname AS name, pg_get_expr(i.indpred, i.indrelid) AS pred FROM pg_index i " + "JOIN pg_class c ON c.oid = i.indexrelid WHERE i.indrelid = %s::regclass " + "AND i.indpred IS NOT NULL", (table,))}, + } + return out + + +@pytest.fixture +def schemas(sqlite_schema): + with database.connect() as conn: + return _sqlite_tables(sqlite_schema), _pg_tables(conn) + + +pytestmark = pytest.mark.usefixtures("postgres") + + +def test_same_tables_and_columns_as_schema_sql(schemas): + old, new = schemas + assert sorted(new) == sorted(old) + for table, spec in old.items(): + assert [c for c, _ in new[table]["columns"]] == [c for c, _ in spec["columns"]], table + for (column, sqlite_type), (_, pg_type) in zip(spec["columns"], new[table]["columns"]): + assert pg_type in _ALLOWED[sqlite_type], f"{table}.{column}: {sqlite_type} -> {pg_type}" + + +def test_same_keys_and_nullability(schemas): + old, new = schemas + for table, spec in old.items(): + assert new[table]["pk"] == spec["pk"], table + assert new[table]["not_null"] == spec["not_null"] | set(spec["pk"]), table + assert spec["unique"] <= new[table]["unique"], table + assert spec["fk"] <= new[table]["fk"], table + extra = {(table, cols) for cols in new[table]["unique"] - spec["unique"]} + assert extra <= _NEW_UNIQUE, extra + + +def test_same_defaults_checks_and_partial_indexes(schemas): + old, new = schemas + for table, spec in old.items(): + booleans = [c for c, ty in new[table]["columns"] if ty == "boolean"] + assert {c: _default(d) for c, d in new[table]["defaults"].items()} == spec["defaults"], table + assert sorted(_expr(c) for c in new[table]["checks"]) == sorted( + _expr(c, booleans) for c in spec["checks"]), table + for name, predicate in spec["partial"].items(): + assert _expr(new[table]["partial"].get(name, "missing")) == _expr(predicate, booleans), name + + +def test_text_sorts_by_code_point(schemas): + _, new = schemas + loose = [f"{t}.{c}" for t, spec in new.items() for c, ty in spec["columns"] + if ty == "text" and spec["collations"][c] != "C"] + assert loose == [] + with database.connect() as conn: + for i, name in enumerate(["a", "é", "Z", "_", "B"]): + conn.execute("INSERT INTO lp_users (uuid, name) VALUES (%s, %s)", (str(i), name)) + names = [r["name"] for r in conn.execute("SELECT name FROM lp_users ORDER BY name")] + conn.execute("INSERT INTO lp_users (uuid, name) VALUES ('x', 'ÉA')") + folded = conn.execute("SELECT lower(name) AS name FROM lp_users WHERE uuid = 'x'").fetchone()["name"] + assert names == ["B", "Z", "_", "a", "é"] + assert folded == "Éa" + + +def test_seed_rows(): + with database.connect() as conn: + rows = conn.execute("SELECT * FROM patreon_sync_state").fetchall() + assert rows == [{"id": 1, "last_sync_at": None, "last_sync_ok": None, + "consecutive_failures": 0, "brake_held": False}] + + +def test_identity_columns_take_explicit_ids(): + with database.connect() as conn: + conn.execute("INSERT INTO codes (id, code_hash, player_uuid, created_at, expires_at) " + "VALUES (41, 'h1', 'p', now(), now())") + conn.execute("SELECT setval(pg_get_serial_sequence('codes', 'id'), 41)") + row = conn.execute("INSERT INTO codes (code_hash, player_uuid, created_at, expires_at) " + "VALUES ('h2', 'p', now(), now()) RETURNING id, revoked, scope").fetchone() + assert row == {"id": 42, "revoked": False, "scope": "skin"} + + +def test_foreign_keys_are_enforced(): + with pytest.raises(database.ForeignKeyViolation): + with database.connect() as conn: + conn.execute("INSERT INTO sessions (token_hash, code_id, player_uuid, expires_at, created_at) " + "VALUES ('t', 99, 'p', now(), now())") + + +def _audit(conn): + return conn.execute( + "INSERT INTO admin_audit (created_at, actor_type, action, outcome, detail_json) " + "VALUES (now(), 'system', 'test', 'ok', %s) RETURNING id", (database.Jsonb({"database": "site"}),) + ).fetchone()["id"] + + +@pytest.mark.parametrize("statement", [ + "UPDATE admin_audit SET reason = 'changed'", + "DELETE FROM admin_audit", + "TRUNCATE admin_audit", + "INSERT INTO admin_audit (id, created_at, actor_type, action, outcome) VALUES (1, now(), 'system', 'x', 'ok') " + "ON CONFLICT (id) DO UPDATE SET action = 'x'", +]) +def test_admin_audit_is_append_only(statement): + with database.connect() as conn: + audit_id = _audit(conn) + with pytest.raises(psycopg.errors.RaiseException, match="admin_audit is append-only"): + with database.connect() as conn: + conn.execute(statement) + with database.connect() as conn: + rows = conn.execute("SELECT id, reason, detail_json FROM admin_audit").fetchall() + assert rows == [{"id": audit_id, "reason": None, "detail_json": {"database": "site"}}] + + +def test_reset_empties_tables_and_restores_seeds(postgres_session): + with database.connect() as conn: + _audit(conn) + conn.execute("UPDATE patreon_sync_state SET brake_held = true") + with pytest.raises(RuntimeError): + with database.connect() as conn: + conn.execute("INSERT INTO player_warnings (player_uuid, reason, created_at) VALUES ('p', 'r', now())") + raise RuntimeError + postgres_session() + with database.connect() as conn: + # The rolled-back insert advanced the sequence of a table that stayed empty. + assert conn.execute("INSERT INTO player_warnings (player_uuid, reason, created_at) " + "VALUES ('p', 'r', now()) RETURNING id").fetchone()["id"] == 1 + with database.connect() as conn: + assert conn.execute("SELECT count(*) AS n FROM admin_audit").fetchone()["n"] == 0 + assert conn.execute("SELECT brake_held FROM patreon_sync_state").fetchone()["brake_held"] is False + assert _audit(conn) == 1 diff --git a/backend/src/database/testing.py b/backend/src/database/testing.py new file mode 100644 index 00000000..b1ba3c18 --- /dev/null +++ b/backend/src/database/testing.py @@ -0,0 +1,164 @@ +"""Pytest fixtures for tests that run against a real Postgres. + +Set TEST_DATABASE_URL to a server where the role may create databases and is a +superuser (the reset skips the admin_audit TRUNCATE trigger). The session +creates its own database, applies the migrations once and points the pool at +it. A test opts in by taking the `postgres` fixture, or a whole module with +`pytestmark = pytest.mark.usefixtures("postgres")`; each such test starts with +every table empty apart from the seed rows, and identities reset. + +Without TEST_DATABASE_URL these tests skip, except under CI, where they fail. +""" + +from __future__ import annotations + +import os +from collections.abc import Iterator + +import psycopg +import pytest +from psycopg import sql +from psycopg.conninfo import make_conninfo +from psycopg.rows import tuple_row + +from . import connection +from .migrate import migrate + + +def server_url() -> str | None: + return os.environ.get("TEST_DATABASE_URL", "").strip() or None + + +def _require_server() -> str: + url = server_url() + if url is None: + if os.environ.get("CI"): + pytest.fail("TEST_DATABASE_URL is not set; CI must run the Postgres tests") + pytest.skip("TEST_DATABASE_URL is not set") + return url + + +def _database_name(suffix: str = "") -> str: + worker = os.environ.get("PYTEST_XDIST_WORKER", "main") + return f"ps_test_{worker}_{os.getpid()}{suffix}" + + +def create_database(name: str) -> str: + """Create an empty database `name` on the test server (dropping any old one) and return its URL.""" + base = _require_server() + with psycopg.connect(base, autocommit=True) as conn: + conn.execute(sql.SQL("DROP DATABASE IF EXISTS {} WITH (FORCE)").format(sql.Identifier(name))) + conn.execute(sql.SQL("CREATE DATABASE {} TEMPLATE template0 ENCODING 'UTF8'").format(sql.Identifier(name))) + return make_conninfo(base, dbname=name) + + +def drop_database(name: str) -> None: + with psycopg.connect(_require_server(), autocommit=True) as conn: + conn.execute(sql.SQL("DROP DATABASE IF EXISTS {} WITH (FORCE)").format(sql.Identifier(name))) + + +def tables(conn: psycopg.Connection) -> list[str]: + """The tables the migrations made, without schema_migrations.""" + cursor = conn.cursor(row_factory=tuple_row).execute( + "SELECT tablename FROM pg_tables WHERE schemaname = 'public' AND tablename <> 'schema_migrations' " + "ORDER BY tablename" + ) + return [row[0] for row in cursor] + + +def _copy_out(conn: psycopg.Connection, table: str) -> bytes: + with conn.cursor().copy(sql.SQL("COPY {} TO STDOUT").format(sql.Identifier(table))) as copy: + return b"".join(bytes(block) for block in copy) + + +class Reset: + """Empties the tables a test touched and puts back the seed rows. + + One query finds the tables with rows or a used identity sequence (a + rolled-back insert still advances it); one TRUNCATE empties them and + restarts their identities. Seeded tables are always reloaded. + """ + + def __init__(self, url: str): + self.url = url + with psycopg.connect(url, autocommit=True) as conn: + self.tables = tables(conn) + self.seeds = {t: data for t in self.tables if (data := _copy_out(conn, t))} + sequences = dict(conn.cursor(row_factory=tuple_row).execute( + "SELECT c.table_name, pg_get_serial_sequence(quote_ident(c.table_name), c.column_name) " + "FROM information_schema.columns c " + "WHERE c.table_schema = 'public' AND c.is_identity = 'YES'").fetchall()) + checks = [] + for table in self.tables: + used = sql.SQL("EXISTS (SELECT 1 FROM {})").format(sql.Identifier(table)) + if table in sequences: + used = sql.SQL("{} OR (SELECT is_called FROM {})").format( + used, sql.SQL(sequences[table])) + checks.append(sql.SQL("SELECT {} AS t WHERE {}").format(sql.Literal(table), used)) + self._dirty = sql.SQL(" UNION ALL ").join(checks) + self._conn: psycopg.Connection | None = None + + def close(self) -> None: + if self._conn is not None: + self._conn.close() + self._conn = None + + def __call__(self) -> None: + if self._conn is None or self._conn.closed: + self._conn = psycopg.connect(self.url) + conn = self._conn + with conn.transaction(): + dirty = {row[0] for row in conn.cursor(row_factory=tuple_row).execute(self._dirty)} + dirty |= set(self.seeds) + # replica skips the admin_audit TRUNCATE trigger; a test that kept a connection + # busy fails here instead of hanging. + conn.execute("SET LOCAL session_replication_role = replica") + conn.execute("SET LOCAL lock_timeout = '10s'") + conn.execute(sql.SQL("TRUNCATE {} RESTART IDENTITY CASCADE").format( + sql.SQL(", ").join(sql.Identifier(t) for t in sorted(dirty)))) + for table, data in self.seeds.items(): + with conn.cursor().copy(sql.SQL("COPY {} FROM STDIN").format(sql.Identifier(table))) as copy: + copy.write(data) + + +def _open_test_pool(url: str): + return connection.open_pool(url, min_size=0, max_size=4) + + +@pytest.fixture(scope="session") +def postgres_session() -> Iterator[Reset]: + """The session's migrated test database; the pool points at it.""" + name = _database_name() + url = create_database(name) + try: + migrate(url, seed=True) + reset = Reset(url) + reset.pool = _open_test_pool(url) + try: + yield reset + finally: + reset.close() + finally: + connection.close_pool() + drop_database(name) + + +@pytest.fixture +def postgres(postgres_session: Reset, monkeypatch) -> str: + """A freshly emptied test database; DATABASE_URL and the pool point at it. Returns its URL.""" + monkeypatch.setenv("DATABASE_URL", postgres_session.url) + if connection._pool is not postgres_session.pool: # a test closed or replaced it + postgres_session.pool = _open_test_pool(postgres_session.url) + postgres_session() + return postgres_session.url + + +@pytest.fixture +def scratch_database() -> Iterator[str]: + """An empty database with no migrations, dropped after the test. Returns its URL.""" + name = _database_name("_scratch") + url = create_database(name) + try: + yield url + finally: + drop_database(name) diff --git a/backend/tests/test_server_lifespan.py b/backend/tests/test_server_lifespan.py index 3110921e..22745081 100644 --- a/backend/tests/test_server_lifespan.py +++ b/backend/tests/test_server_lifespan.py @@ -5,18 +5,38 @@ from fastapi.testclient import TestClient -def test_startup_migrates_once_and_serves_ping(monkeypatch): +def _start(monkeypatch, database_url: str): from server import app + from src.database import migrate as postgres from src.skins import db - migrate = Mock() + monkeypatch.setenv("DATABASE_URL", database_url) + migrate, migrate_postgres = Mock(), Mock() monkeypatch.setattr(db, "migrate", migrate) + monkeypatch.setattr(postgres, "migrate", migrate_postgres) + return app, migrate, migrate_postgres + + +def test_startup_migrates_once_and_serves_ping(monkeypatch): + app, migrate, migrate_postgres = _start(monkeypatch, "postgresql://db/site") with TestClient(app) as client: migrate.assert_called_once_with() + migrate_postgres.assert_called_once_with(seed=True) response = client.get("/ping") assert response.status_code == 200 assert response.json() == {"ok": True} migrate.assert_called_once_with() migrate.assert_called_once_with() + migrate_postgres.assert_called_once_with(seed=True) + + +def test_startup_without_database_url_skips_postgres(monkeypatch): + app, migrate, migrate_postgres = _start(monkeypatch, "") + + with TestClient(app) as client: + assert client.get("/ping").status_code == 200 + + migrate.assert_called_once_with() + migrate_postgres.assert_not_called()