From fcaa86cd28de59ed5122ebb937e561617c2f5ddb Mon Sep 17 00:00:00 2001 From: Amir Golmoradi <132334913+Amir-Golmoradi@users.noreply.github.com> Date: Fri, 30 Oct 2026 17:59:52 +0330 Subject: [PATCH 1/2] Replace infrastructure scripts with Vault Agent and native tools --- .dockerignore | 14 + .env.example | 18 +- .github/workflows/_database-security.yml | 95 ++- README.md | 6 +- docker-compose.yaml | 663 ++++++++++++++---- docs/guides/postgresql-vault-migration.md | 312 +-------- docs/infrastructure-validation.md | 72 ++ docs/infrastructure.md | 302 ++++++++ docs/tasks/postgresql-vault-migration.md | 10 +- infrastructure/application.Dockerfile | 9 + infrastructure/compose.operations.yaml | 519 ++++++++++++++ infrastructure/compose.production.yaml | 142 ++++ .../callbacks/collateral/afterConnect.sql | 5 + .../callbacks/collateral/afterMigrate.sql | 29 + .../collateral/afterMigrateError.sql | 29 + .../callbacks/collateral/beforeMigrate.sql | 10 + .../callbacks/identity/afterConnect.sql | 5 + .../callbacks/identity/afterMigrate.sql | 36 + .../callbacks/identity/afterMigrateError.sql | 36 + .../callbacks/identity/beforeMigrate.sql | 10 + .../flyway/callbacks/loan/afterConnect.sql | 5 + .../flyway/callbacks/loan/afterMigrate.sql | 29 + .../callbacks/loan/afterMigrateError.sql | 29 + .../flyway/callbacks/loan/beforeMigrate.sql | 10 + .../flyway/callbacks/wallet/afterConnect.sql | 5 + .../flyway/callbacks/wallet/afterMigrate.sql | 22 + .../callbacks/wallet/afterMigrateError.sql | 22 + .../flyway/callbacks/wallet/beforeMigrate.sql | 10 + infrastructure/flyway/flyway.toml | 14 + infrastructure/migrations.sha256 | 4 + infrastructure/operations/Dockerfile | 12 + infrastructure/operations/adopt-artifacts.yml | 70 ++ infrastructure/operations/adopt.yml | 52 ++ infrastructure/operations/agent-handoff.yml | 3 + infrastructure/operations/allocate.yml | 36 + infrastructure/operations/ansible.cfg | 7 + infrastructure/operations/backup.yml | 56 ++ infrastructure/operations/import-identity.yml | 42 ++ infrastructure/operations/install-timer.yml | 38 + infrastructure/operations/migrate.yml | 249 +++++++ infrastructure/operations/preflight.yml | 105 +++ infrastructure/operations/prepare.yml | 115 +++ .../operations/provision-database.yml | 166 +++++ infrastructure/operations/provision-vault.yml | 437 ++++++++++++ infrastructure/operations/recover-attempt.yml | 116 +++ .../operations/refresh-identities.service | 7 + .../operations/refresh-identities.timer | 11 + .../operations/refresh-identities.yml | 105 +++ infrastructure/operations/requirements.txt | 12 + infrastructure/operations/requirements.yml | 3 + .../operations/restore-postgresql.yml | 76 ++ infrastructure/operations/seed-kafka.yml | 49 ++ infrastructure/operations/site.yml | 6 + infrastructure/operations/tasks/target.yml | 31 + .../operations/tasks/verify-migration.yml | 28 + infrastructure/operations/tests/agent.yml | 147 ++++ .../operations/tests/applications.yml | 35 + .../operations/tests/integration.yml | 62 ++ infrastructure/operations/tests/recovery.yml | 51 ++ infrastructure/operations/vault-lifecycle.yml | 217 ++++++ infrastructure/postgresql/entrypoint.sh | 25 +- infrastructure/postgresql/import-identity.sql | 63 ++ infrastructure/postgresql/pg_hba.conf | 5 +- infrastructure/tests-requirements.txt | 3 + infrastructure/tests/test_contracts.py | 180 +++++ infrastructure/vault/agent/collateral.hcl | 45 ++ infrastructure/vault/agent/identity.hcl | 45 ++ infrastructure/vault/agent/loan.hcl | 45 ++ infrastructure/vault/agent/operator.hcl | 18 + infrastructure/vault/agent/wallet.hcl | 45 ++ infrastructure/vault/entrypoint.sh | 8 +- .../vault/production/collateral.hcl | 44 ++ infrastructure/vault/production/identity.hcl | 44 ++ infrastructure/vault/production/loan.hcl | 44 ++ infrastructure/vault/production/wallet.hcl | 44 ++ infrastructure/vault/runtime/collateral.hcl | 40 ++ infrastructure/vault/runtime/identity.hcl | 40 ++ infrastructure/vault/runtime/loan.hcl | 40 ++ infrastructure/vault/runtime/wallet.hcl | 40 ++ payguard-collateral-service/pom.xml | 2 + .../src/main/resources/application.yaml | 23 +- payguard-identity-service/pom.xml | 2 + .../src/main/resources/application.yaml | 21 +- .../security/VaultSecretsIT.java | 14 +- payguard-loan-service/pom.xml | 2 + .../src/main/resources/application.yaml | 23 +- payguard-shared/pom.xml | 4 +- .../vault/IdentityDataImporter.java | 194 ----- .../vault/LeasedDataSource.java | 221 ------ .../infrastructure/vault/SchemaMigrator.java | 71 -- .../infrastructure/vault/VaultClient.java | 100 +-- .../vault/VaultDatabaseAutoConfiguration.java | 17 +- .../vault/AgentTokenClientTest.java | 189 +++++ .../PostgresqlMigrationRehearsalTest.java | 151 ---- .../infrastructure/vault/VaultDatabaseIT.java | 471 ------------- .../src/main/resources/application.yaml | 15 +- .../postgresql/V1__wallet_schema_marker.sql | 3 + pom.xml | 19 - scripts/local/common.py | 99 --- scripts/local/migrate.py | 46 -- scripts/local/prepare.py | 68 -- scripts/local/refresh_approle.py | 29 - scripts/local/vault_bootstrap.py | 194 ----- scripts/local/verify_applications.py | 66 -- scripts/local/verify_vault.py | 80 --- scripts/migration/convert_identity_export.py | 87 --- .../migration/test_convert_identity_export.py | 52 -- 107 files changed, 5401 insertions(+), 2496 deletions(-) create mode 100644 docs/infrastructure-validation.md create mode 100644 docs/infrastructure.md create mode 100644 infrastructure/application.Dockerfile create mode 100644 infrastructure/compose.operations.yaml create mode 100644 infrastructure/compose.production.yaml create mode 100644 infrastructure/flyway/callbacks/collateral/afterConnect.sql create mode 100644 infrastructure/flyway/callbacks/collateral/afterMigrate.sql create mode 100644 infrastructure/flyway/callbacks/collateral/afterMigrateError.sql create mode 100644 infrastructure/flyway/callbacks/collateral/beforeMigrate.sql create mode 100644 infrastructure/flyway/callbacks/identity/afterConnect.sql create mode 100644 infrastructure/flyway/callbacks/identity/afterMigrate.sql create mode 100644 infrastructure/flyway/callbacks/identity/afterMigrateError.sql create mode 100644 infrastructure/flyway/callbacks/identity/beforeMigrate.sql create mode 100644 infrastructure/flyway/callbacks/loan/afterConnect.sql create mode 100644 infrastructure/flyway/callbacks/loan/afterMigrate.sql create mode 100644 infrastructure/flyway/callbacks/loan/afterMigrateError.sql create mode 100644 infrastructure/flyway/callbacks/loan/beforeMigrate.sql create mode 100644 infrastructure/flyway/callbacks/wallet/afterConnect.sql create mode 100644 infrastructure/flyway/callbacks/wallet/afterMigrate.sql create mode 100644 infrastructure/flyway/callbacks/wallet/afterMigrateError.sql create mode 100644 infrastructure/flyway/callbacks/wallet/beforeMigrate.sql create mode 100644 infrastructure/flyway/flyway.toml create mode 100644 infrastructure/migrations.sha256 create mode 100644 infrastructure/operations/Dockerfile create mode 100644 infrastructure/operations/adopt-artifacts.yml create mode 100644 infrastructure/operations/adopt.yml create mode 100644 infrastructure/operations/agent-handoff.yml create mode 100644 infrastructure/operations/allocate.yml create mode 100644 infrastructure/operations/ansible.cfg create mode 100644 infrastructure/operations/backup.yml create mode 100644 infrastructure/operations/import-identity.yml create mode 100644 infrastructure/operations/install-timer.yml create mode 100644 infrastructure/operations/migrate.yml create mode 100644 infrastructure/operations/preflight.yml create mode 100644 infrastructure/operations/prepare.yml create mode 100644 infrastructure/operations/provision-database.yml create mode 100644 infrastructure/operations/provision-vault.yml create mode 100644 infrastructure/operations/recover-attempt.yml create mode 100644 infrastructure/operations/refresh-identities.service create mode 100644 infrastructure/operations/refresh-identities.timer create mode 100644 infrastructure/operations/refresh-identities.yml create mode 100644 infrastructure/operations/requirements.txt create mode 100644 infrastructure/operations/requirements.yml create mode 100644 infrastructure/operations/restore-postgresql.yml create mode 100644 infrastructure/operations/seed-kafka.yml create mode 100644 infrastructure/operations/site.yml create mode 100644 infrastructure/operations/tasks/target.yml create mode 100644 infrastructure/operations/tasks/verify-migration.yml create mode 100644 infrastructure/operations/tests/agent.yml create mode 100644 infrastructure/operations/tests/applications.yml create mode 100644 infrastructure/operations/tests/integration.yml create mode 100644 infrastructure/operations/tests/recovery.yml create mode 100644 infrastructure/operations/vault-lifecycle.yml mode change 100644 => 100755 infrastructure/postgresql/entrypoint.sh create mode 100644 infrastructure/postgresql/import-identity.sql create mode 100644 infrastructure/tests-requirements.txt create mode 100644 infrastructure/tests/test_contracts.py create mode 100644 infrastructure/vault/agent/collateral.hcl create mode 100644 infrastructure/vault/agent/identity.hcl create mode 100644 infrastructure/vault/agent/loan.hcl create mode 100644 infrastructure/vault/agent/operator.hcl create mode 100644 infrastructure/vault/agent/wallet.hcl mode change 100644 => 100755 infrastructure/vault/entrypoint.sh create mode 100644 infrastructure/vault/production/collateral.hcl create mode 100644 infrastructure/vault/production/identity.hcl create mode 100644 infrastructure/vault/production/loan.hcl create mode 100644 infrastructure/vault/production/wallet.hcl create mode 100644 infrastructure/vault/runtime/collateral.hcl create mode 100644 infrastructure/vault/runtime/identity.hcl create mode 100644 infrastructure/vault/runtime/loan.hcl create mode 100644 infrastructure/vault/runtime/wallet.hcl delete mode 100644 payguard-shared/src/main/java/infrastructure/vault/IdentityDataImporter.java delete mode 100644 payguard-shared/src/main/java/infrastructure/vault/LeasedDataSource.java delete mode 100644 payguard-shared/src/main/java/infrastructure/vault/SchemaMigrator.java create mode 100644 payguard-shared/src/test/java/infrastructure/vault/AgentTokenClientTest.java delete mode 100644 payguard-shared/src/test/java/infrastructure/vault/PostgresqlMigrationRehearsalTest.java delete mode 100644 payguard-shared/src/test/java/infrastructure/vault/VaultDatabaseIT.java create mode 100644 payguard-wallet-service/src/main/resources/db/migration/postgresql/V1__wallet_schema_marker.sql delete mode 100644 scripts/local/common.py delete mode 100644 scripts/local/migrate.py delete mode 100644 scripts/local/prepare.py delete mode 100644 scripts/local/refresh_approle.py delete mode 100644 scripts/local/vault_bootstrap.py delete mode 100644 scripts/local/verify_applications.py delete mode 100644 scripts/local/verify_vault.py delete mode 100644 scripts/migration/convert_identity_export.py delete mode 100644 scripts/migration/test_convert_identity_export.py diff --git a/.dockerignore b/.dockerignore index 54584d5..970db3b 100644 --- a/.dockerignore +++ b/.dockerignore @@ -9,3 +9,17 @@ .local **/target **/__pycache__ + +# Only the four packaged service JARs enter application build contexts. +!payguard-identity-service/target/ +payguard-identity-service/target/* +!payguard-identity-service/target/payguard-identity-service-1.0.0-SNAPSHOT.jar +!payguard-wallet-service/target/ +payguard-wallet-service/target/* +!payguard-wallet-service/target/payguard-wallet-service-1.0.0-SNAPSHOT.jar +!payguard-loan-service/target/ +payguard-loan-service/target/* +!payguard-loan-service/target/payguard-loan-service-1.0.0-SNAPSHOT.jar +!payguard-collateral-service/target/ +payguard-collateral-service/target/* +!payguard-collateral-service/target/payguard-collateral-service-1.0.0-SNAPSHOT.jar diff --git a/.env.example b/.env.example index 1a371d4..b32e3b5 100644 --- a/.env.example +++ b/.env.example @@ -1,7 +1,19 @@ -# Secrets are generated into ignored protected files by scripts/local/prepare.py. -# No runtime database passwords or HMAC keys belong here. +# Non-secret deployment identity. Never put passwords or tokens here. COMPOSE_PROJECT_NAME=payguard VAULT_PORT=8200 -KAFKA_PORT=9092 PAYGUARD_LOCAL_UID=1000 PAYGUARD_LOCAL_GID=1000 +PAYGUARD_DOCKER_GID=1000 +# Set to the SAME absolute path on the host and operations container. +PAYGUARD_REPO_ROOT=/workspace/payguard +PAYGUARD_RUNTIME_ROOT=/dev/shm/payguard +# PostgreSQL 18 image default. Inspect existing PG_VERSION locations before adoption. +PAYGUARD_PGDATA=/var/lib/postgresql/18/docker +# Existing volume names are explicit and external; they are never auto-created on startup. +PAYGUARD_POSTGRES_IDENTITY_VOLUME=payguard_postgres_identity_data +PAYGUARD_POSTGRES_WALLET_VOLUME=payguard_postgres_wallet_data +PAYGUARD_POSTGRES_LOAN_VOLUME=payguard_postgres_loan_data +PAYGUARD_POSTGRES_COLLATERAL_VOLUME=payguard_postgres_collateral_data +PAYGUARD_VAULT_DATA_VOLUME=payguard_vault_data +PAYGUARD_VAULT_AUDIT_VOLUME=payguard_vault_audit +PAYGUARD_KAFKA_DATA_VOLUME=payguard_kafka_data diff --git a/.github/workflows/_database-security.yml b/.github/workflows/_database-security.yml index 69b76b8..c055543 100644 --- a/.github/workflows/_database-security.yml +++ b/.github/workflows/_database-security.yml @@ -1,15 +1,22 @@ -name: PostgreSQL and Vault verification - +name: Native infrastructure verification on: workflow_call: - permissions: contents: read - jobs: - real-dependencies: + isolated-dependencies: runs-on: ubuntu-latest - timeout-minutes: 25 + timeout-minutes: 35 + env: + COMPOSE_PROJECT_NAME: payguard-it-ci + PAYGUARD_RUNTIME_ROOT: /dev/shm/payguard-it-ci + PAYGUARD_POSTGRES_IDENTITY_VOLUME: payguard-it-ci_identity + PAYGUARD_POSTGRES_WALLET_VOLUME: payguard-it-ci_wallet + PAYGUARD_POSTGRES_LOAN_VOLUME: payguard-it-ci_loan + PAYGUARD_POSTGRES_COLLATERAL_VOLUME: payguard-it-ci_collateral + PAYGUARD_VAULT_DATA_VOLUME: payguard-it-ci_vault + PAYGUARD_VAULT_AUDIT_VOLUME: payguard-it-ci_audit + PAYGUARD_KAFKA_DATA_VOLUME: payguard-it-ci_kafka steps: - uses: actions/checkout@v7 - uses: actions/setup-java@v6 @@ -17,39 +24,57 @@ jobs: distribution: temurin java-version: '25' cache: maven - - name: PostgreSQL migrations and competing transactions - run: >- - ./mvnw -B -ntp -pl payguard-identity-service,payguard-loan-service,payguard-collateral-service -am - -Dtest=PostgreSqlTlsTest,PostgresqlMigrationRehearsalTest,SanitizedDependencyErrorTest,OtpInfrastructureIntegrationTest,OtpApplicationContractsTest,LoanServiceApplicationTests,CollateralServiceApplicationTests - -Dsurefire.failIfNoSpecifiedTests=false clean test - - name: Build protected migration and integration runners - run: >- - ./mvnw -B -ntp -pl payguard-identity-service,payguard-wallet-service,payguard-loan-service,payguard-collateral-service -am -DskipTests package dependency:copy-dependencies -DincludeScope=test - - name: Verify synthetic export and approved identifier mapping - run: python3 -m unittest discover -s scripts/migration -p 'test_*.py' - - name: Prepare isolated TLS infrastructure + - name: Install native Ansible dependencies and offline test dependencies run: | - python3 scripts/local/prepare.py - docker compose --env-file .env.example up -d --wait postgres-wallet postgres-loan postgres-collateral postgres-identity vault - python3 scripts/local/vault_bootstrap.py - - name: Apply separately privileged Flyway migrations + python3 -m venv /tmp/payguard-ansible + /tmp/payguard-ansible/bin/pip install -r infrastructure/operations/requirements.txt -r infrastructure/tests-requirements.txt + /tmp/payguard-ansible/bin/ansible-galaxy collection install -r infrastructure/operations/requirements.yml + echo /tmp/payguard-ansible/bin >> "$GITHUB_PATH" + echo "PAYGUARD_REPO_ROOT=$GITHUB_WORKSPACE" >> "$GITHUB_ENV" + echo "PAYGUARD_LOCAL_UID=$(id -u)" >> "$GITHUB_ENV" + echo "PAYGUARD_LOCAL_GID=$(id -g)" >> "$GITHUB_ENV" + echo "PAYGUARD_DOCKER_GID=$(stat -c %g /var/run/docker.sock)" >> "$GITHUB_ENV" + mkdir -m 700 "$PAYGUARD_RUNTIME_ROOT" + - name: Verify source contracts and build applications run: | - python3 scripts/local/migrate.py identity - python3 scripts/local/migrate.py loan - python3 scripts/local/migrate.py collateral - - name: Verify actual pools, scoped secrets and Vault audit - run: python3 scripts/local/verify_vault.py - - name: Boot actual packaged services with scoped credentials - run: python3 scripts/local/verify_applications.py - - name: Preserve test reports + python3 -m unittest discover -s infrastructure/tests -v + ./mvnw -B -ntp -pl payguard-identity-service,payguard-wallet-service,payguard-loan-service,payguard-collateral-service -am test package + - name: Prepare isolated fixture and allocate only new explicit volumes + run: | + ansible-playbook -i localhost, -c local infrastructure/operations/prepare.yml -e fresh_target=true + ansible-playbook -i localhost, -c local infrastructure/operations/allocate.yml -e fresh_target=true + docker compose -f docker-compose.yaml -f infrastructure/compose.operations.yaml --profile operations build operations + docker compose up -d --wait postgres-identity postgres-wallet postgres-loan postgres-collateral + docker compose up -d vault + - name: Initialize and provision the new fixture through Ansible + run: | + docker compose -f docker-compose.yaml -f infrastructure/compose.operations.yaml run --rm --no-deps operations infrastructure/operations/vault-lifecycle.yml -e operation=initialize -e fresh_target=true -e local_rehearsal=true + docker compose -f docker-compose.yaml -f infrastructure/compose.operations.yaml run --rm --no-deps operations infrastructure/operations/site.yml -e fresh_target=true + docker compose -f docker-compose.yaml -f infrastructure/compose.operations.yaml up -d agent-operator + - name: Run all four CLI migrations twice + run: | + for pass in 1 2; do + for service in identity wallet loan collateral; do + docker compose -f docker-compose.yaml -f infrastructure/compose.operations.yaml run --rm --no-deps operations infrastructure/operations/migrate.yml -e service="$service" + done + done + - name: Verify privilege boundaries and native Agent security + run: | + docker compose -f docker-compose.yaml -f infrastructure/compose.operations.yaml run --rm --no-deps operations infrastructure/operations/tests/integration.yml + docker compose -f docker-compose.yaml -f infrastructure/compose.operations.yaml run --rm --no-deps operations infrastructure/operations/tests/agent.yml -e isolated_target=true + - name: Boot and verify actual packaged applications + run: | + docker compose --profile applications up -d --build + docker compose -f docker-compose.yaml -f infrastructure/compose.operations.yaml run --rm --no-deps operations infrastructure/operations/tests/applications.yml + - name: Rehearse an isolated Vault outage and recovery + run: docker compose -f docker-compose.yaml -f infrastructure/compose.operations.yaml run --rm --no-deps operations infrastructure/operations/tests/recovery.yml + - name: Preserve Java test reports if: always() uses: actions/upload-artifact@v7 with: - name: database-security-reports - path: | - **/target/surefire-reports/TEST-*.xml - payguard-shared/target/vault-test-reports/*.xml + name: native-infrastructure-java-reports + path: '**/target/surefire-reports/TEST-*.xml' if-no-files-found: warn - - name: Stop isolated infrastructure + - name: Stop only this isolated fixture if: always() - run: docker compose --env-file .env.example stop + run: docker compose -f docker-compose.yaml -f infrastructure/compose.operations.yaml --profile applications --profile operations stop diff --git a/README.md b/README.md index 7391626..ea1fe66 100644 --- a/README.md +++ b/README.md @@ -103,15 +103,15 @@ Domain rules remain in the domain, use-case coordination in application code, an ## Build and local rehearsal -Use JDK **25**, Docker/Compose, Python 3 and OpenSSL. The Maven wrapper is included. Database integration tests start real PostgreSQL containers; they do not require an unset opt-in flag. +Use JDK **25**, Docker/Compose, Ansible and OpenSSL. The Maven wrapper is included. Database integration tests start real PostgreSQL containers; they do not require an unset opt-in flag. ```bash ./mvnw -B -ntp clean verify ``` -The committed Identity integration passed full Maven verification with 91 tests and zero failures, errors or skips, including the existing domain regressions and real PostgreSQL tests. Formatting and static-analysis checks also passed. See [the publication repair log](docs/tasks/identity-publication-repair.md) for the exact source revision and evidence. +Historical Identity verification is recorded in [the publication repair log](docs/tasks/identity-publication-repair.md). It does not validate the new Vault Agent lifecycle; see [current infrastructure validation](docs/infrastructure-validation.md). -For the isolated, persistent TLS PostgreSQL/Vault rehearsal, follow [the implementation guide](docs/guides/postgresql-vault-migration.md). It gives the exact preparation, initialization, migration and test commands. Generated credentials, certificates and unseal material live under ignored `.local/payguard` with protected permissions. `.env.example` contains only public settings. Never copy secrets into an image or publish that directory. +For the Compose, Vault Agent, Ansible and Flyway CLI deployment, follow [the infrastructure runbook](docs/infrastructure.md). It gives the exact preparation, initialization, migration and test commands. Generated credentials, certificates and unseal material live under ignored `.local/payguard` with protected permissions. `.env.example` contains only public settings. Never copy secrets into an image or publish that directory. Runtime Flyway is disabled: apply migrations using the migration identity before starting applications. The `applications` Compose profile is separate from the database/Vault infrastructure. Provider delivery, customer enrollment, external authentication, production certificate issuance, HA/auto-unseal and durable event delivery are not implemented by this migration. diff --git a/docker-compose.yaml b/docker-compose.yaml index 8db1dc3..62a79fc 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,241 +1,602 @@ +# External volumes preserve the existing deployment identities. name: ${COMPOSE_PROJECT_NAME:-payguard} - services: + postgres-identity: + image: postgres:18-alpine@sha256:898c60cb390d1085bc97bdd6a1ae968757421df394b071848b52b1c09c89d6a5 + restart: unless-stopped + entrypoint: + - /bin/sh + - /opt/payguard/entrypoint.sh + environment: + POSTGRES_DB: payguard_identity + POSTGRES_PASSWORD_FILE: /run/payguard/admin-password + PGDATA: ${PAYGUARD_PGDATA:-/var/lib/postgresql/18/docker} + POSTGRES_INITDB_ARGS: --auth-host=scram-sha-256 --auth-local=peer + volumes: + - postgres_identity_data:/var/lib/postgresql + - type: bind + source: ./.local/payguard/identity + target: /run/payguard + read_only: true + bind: + create_host_path: false + - type: bind + source: ./infrastructure/postgresql/entrypoint.sh + target: /opt/payguard/entrypoint.sh + read_only: true + bind: + create_host_path: false + - type: bind + source: ./infrastructure/postgresql/pg_hba.conf + target: /etc/postgresql/pg_hba.conf + read_only: true + bind: + create_host_path: false + tmpfs: + - /run/postgresql:rw,nosuid,nodev + - /run/postgres-tls:rw,noexec,nosuid,nodev,mode=0700 + security_opt: + - no-new-privileges:true + healthcheck: + test: + - CMD + - pg_isready + - -U + - postgres + - -d + - payguard_identity + interval: 5s + timeout: 3s + retries: 30 + networks: + - database postgres-wallet: - image: postgres:17.11-bookworm@sha256:3645570cccdfa447589da9f57dd740faa29b30938e861289a5574b6ca6b03826 - entrypoint: ["/bin/sh", "/opt/payguard/entrypoint.sh"] + image: postgres:18-alpine@sha256:898c60cb390d1085bc97bdd6a1ae968757421df394b071848b52b1c09c89d6a5 + restart: unless-stopped + entrypoint: + - /bin/sh + - /opt/payguard/entrypoint.sh environment: POSTGRES_DB: payguard_wallet POSTGRES_PASSWORD_FILE: /run/payguard/admin-password + PGDATA: ${PAYGUARD_PGDATA:-/var/lib/postgresql/18/docker} + POSTGRES_INITDB_ARGS: --auth-host=scram-sha-256 --auth-local=peer volumes: - - postgres_wallet_data:/var/lib/postgresql/data - - ./.local/payguard/wallet:/run/payguard:ro - - ./infrastructure/postgresql/entrypoint.sh:/opt/payguard/entrypoint.sh:ro - - ./infrastructure/postgresql/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro + - postgres_wallet_data:/var/lib/postgresql + - type: bind + source: ./.local/payguard/wallet + target: /run/payguard + read_only: true + bind: + create_host_path: false + - type: bind + source: ./infrastructure/postgresql/entrypoint.sh + target: /opt/payguard/entrypoint.sh + read_only: true + bind: + create_host_path: false + - type: bind + source: ./infrastructure/postgresql/pg_hba.conf + target: /etc/postgresql/pg_hba.conf + read_only: true + bind: + create_host_path: false + tmpfs: + - /run/postgresql:rw,nosuid,nodev + - /run/postgres-tls:rw,noexec,nosuid,nodev,mode=0700 + security_opt: + - no-new-privileges:true healthcheck: - test: ["CMD", "pg_isready", "-U", "postgres", "-d", "payguard_wallet"] + test: + - CMD + - pg_isready + - -U + - postgres + - -d + - payguard_wallet interval: 5s timeout: 3s - retries: 20 - networks: [database] - + retries: 30 + networks: + - database postgres-loan: - image: postgres:17.11-bookworm@sha256:3645570cccdfa447589da9f57dd740faa29b30938e861289a5574b6ca6b03826 - entrypoint: ["/bin/sh", "/opt/payguard/entrypoint.sh"] + image: postgres:18-alpine@sha256:898c60cb390d1085bc97bdd6a1ae968757421df394b071848b52b1c09c89d6a5 + restart: unless-stopped + entrypoint: + - /bin/sh + - /opt/payguard/entrypoint.sh environment: POSTGRES_DB: payguard_loan POSTGRES_PASSWORD_FILE: /run/payguard/admin-password + PGDATA: ${PAYGUARD_PGDATA:-/var/lib/postgresql/18/docker} + POSTGRES_INITDB_ARGS: --auth-host=scram-sha-256 --auth-local=peer volumes: - - postgres_loan_data:/var/lib/postgresql/data - - ./.local/payguard/loan:/run/payguard:ro - - ./infrastructure/postgresql/entrypoint.sh:/opt/payguard/entrypoint.sh:ro - - ./infrastructure/postgresql/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro + - postgres_loan_data:/var/lib/postgresql + - type: bind + source: ./.local/payguard/loan + target: /run/payguard + read_only: true + bind: + create_host_path: false + - type: bind + source: ./infrastructure/postgresql/entrypoint.sh + target: /opt/payguard/entrypoint.sh + read_only: true + bind: + create_host_path: false + - type: bind + source: ./infrastructure/postgresql/pg_hba.conf + target: /etc/postgresql/pg_hba.conf + read_only: true + bind: + create_host_path: false + tmpfs: + - /run/postgresql:rw,nosuid,nodev + - /run/postgres-tls:rw,noexec,nosuid,nodev,mode=0700 + security_opt: + - no-new-privileges:true healthcheck: - test: ["CMD", "pg_isready", "-U", "postgres", "-d", "payguard_loan"] + test: + - CMD + - pg_isready + - -U + - postgres + - -d + - payguard_loan interval: 5s timeout: 3s - retries: 20 - networks: [database] - + retries: 30 + networks: + - database postgres-collateral: - image: postgres:17.11-bookworm@sha256:3645570cccdfa447589da9f57dd740faa29b30938e861289a5574b6ca6b03826 - entrypoint: ["/bin/sh", "/opt/payguard/entrypoint.sh"] + image: postgres:18-alpine@sha256:898c60cb390d1085bc97bdd6a1ae968757421df394b071848b52b1c09c89d6a5 + restart: unless-stopped + entrypoint: + - /bin/sh + - /opt/payguard/entrypoint.sh environment: POSTGRES_DB: payguard_collateral POSTGRES_PASSWORD_FILE: /run/payguard/admin-password + PGDATA: ${PAYGUARD_PGDATA:-/var/lib/postgresql/18/docker} + POSTGRES_INITDB_ARGS: --auth-host=scram-sha-256 --auth-local=peer volumes: - - postgres_collateral_data:/var/lib/postgresql/data - - ./.local/payguard/collateral:/run/payguard:ro - - ./infrastructure/postgresql/entrypoint.sh:/opt/payguard/entrypoint.sh:ro - - ./infrastructure/postgresql/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro + - postgres_collateral_data:/var/lib/postgresql + - type: bind + source: ./.local/payguard/collateral + target: /run/payguard + read_only: true + bind: + create_host_path: false + - type: bind + source: ./infrastructure/postgresql/entrypoint.sh + target: /opt/payguard/entrypoint.sh + read_only: true + bind: + create_host_path: false + - type: bind + source: ./infrastructure/postgresql/pg_hba.conf + target: /etc/postgresql/pg_hba.conf + read_only: true + bind: + create_host_path: false + tmpfs: + - /run/postgresql:rw,nosuid,nodev + - /run/postgres-tls:rw,noexec,nosuid,nodev,mode=0700 + security_opt: + - no-new-privileges:true healthcheck: - test: ["CMD", "pg_isready", "-U", "postgres", "-d", "payguard_collateral"] + test: + - CMD + - pg_isready + - -U + - postgres + - -d + - payguard_collateral interval: 5s timeout: 3s - retries: 20 - networks: [database] - - postgres-identity: - image: postgres:17.11-bookworm@sha256:3645570cccdfa447589da9f57dd740faa29b30938e861289a5574b6ca6b03826 - entrypoint: ["/bin/sh", "/opt/payguard/entrypoint.sh"] + retries: 30 + networks: + - database + vault: + image: hashicorp/vault:2.1.2@sha256:c2f666266f383d2cf424d86b8bb8ce7d065562173ffec2b476d762943608bb55 + restart: unless-stopped + user: 0:0 + entrypoint: + - /bin/sh + - /opt/payguard/entrypoint.sh + ports: + - 127.0.0.1:${VAULT_PORT:-8200}:8200 environment: - POSTGRES_DB: payguard_identity - POSTGRES_PASSWORD_FILE: /run/payguard/admin-password + VAULT_ADDR: https://127.0.0.1:8200 + VAULT_CACERT: /vault/tls/ca.crt volumes: - - postgres_identity_data:/var/lib/postgresql/data - - ./.local/payguard/identity:/run/payguard:ro - - ./infrastructure/postgresql/entrypoint.sh:/opt/payguard/entrypoint.sh:ro - - ./infrastructure/postgresql/pg_hba.conf:/etc/postgresql/pg_hba.conf:ro + - vault_data:/vault/data + - vault_audit:/vault/audit + - type: bind + source: ./.local/payguard/vault + target: /run/payguard + read_only: true + bind: + create_host_path: false + - type: bind + source: ./infrastructure/vault/server.hcl + target: /vault/config/server.hcl + read_only: true + bind: + create_host_path: false + - type: bind + source: ./infrastructure/vault/entrypoint.sh + target: /opt/payguard/entrypoint.sh + read_only: true + bind: + create_host_path: false + tmpfs: + - /vault/tls:rw,noexec,nosuid,nodev,mode=0700 + - /tmp:rw,noexec,nosuid,nodev + security_opt: + - no-new-privileges:true + networks: + database: {} + operations: + aliases: + - vault-invalid-san healthcheck: - test: ["CMD", "pg_isready", "-U", "postgres", "-d", "payguard_identity"] + test: + - CMD + - vault + - status interval: 5s timeout: 3s - retries: 20 - networks: [database] - - vault: - image: hashicorp/vault:2.1.2@sha256:c2f666266f383d2cf424d86b8bb8ce7d065562173ffec2b476d762943608bb55 - user: "0:0" - environment: - SKIP_SETCAP: "1" - SKIP_CHOWN: "1" - entrypoint: ["/bin/sh", "/opt/payguard/entrypoint.sh"] - ports: ["127.0.0.1:${VAULT_PORT:-8200}:8200"] - volumes: - - vault_data:/vault/data - - vault_audit:/vault/audit - - ./.local/payguard/vault:/run/payguard:ro - - ./infrastructure/vault/server.hcl:/vault/config/server.hcl:ro - - ./infrastructure/vault/entrypoint.sh:/opt/payguard/entrypoint.sh:ro - networks: [database, operations] - + retries: 30 redis: - image: redis:8.0.3-alpine + image: redis:8.0.3-alpine@sha256:25c0ae32c6c2301798579f5944af53729766a18eff5660bbef196fc2e6214a9c + restart: unless-stopped + command: + - redis-server + - /run/redis/redis.conf + volumes: + - type: bind + source: ./.local/payguard/redis + target: /run/redis + read_only: true + bind: + create_host_path: false + networks: + - database healthcheck: - test: ["CMD", "redis-cli", "ping"] + test: + - CMD-SHELL + - REDISCLI_AUTH="$$(cat /run/redis/password)" redis-cli ping | grep -q PONG interval: 5s timeout: 3s - retries: 10 - networks: [database] - + retries: 20 + security_opt: + - no-new-privileges:true + user: ${PAYGUARD_LOCAL_UID:-1000}:${PAYGUARD_LOCAL_GID:-1000} + tmpfs: + - /tmp:rw,noexec,nosuid,nodev + working_dir: /tmp kafka-init: - image: apache/kafka:4.0.1 - user: "0:0" - entrypoint: ["/bin/sh", "-c"] - command: ["chown 1000:1000 /tmp/kraft-combined-logs"] + image: apache/kafka:4.0.1@sha256:9a129e03121d46d74349a480f8f17082df3dd411b28281c9eabb95cf096370b9 + user: 0:0 + entrypoint: + - /bin/sh + - -ec + command: + - chown 1000:1000 /tmp/kraft-combined-logs volumes: - - kafka_data:/tmp/kraft-combined-logs - + - kafka_data:/tmp/kraft-combined-logs + network_mode: none + restart: 'no' kafka: - image: apache/kafka:4.0.1 - networks: [database] - container_name: ${COMPOSE_PROJECT_NAME:-payguard}-kafka + image: apache/kafka:4.0.1@sha256:9a129e03121d46d74349a480f8f17082df3dd411b28281c9eabb95cf096370b9 + restart: unless-stopped depends_on: - kafka-init: { condition: service_completed_successfully } - ports: - - "${KAFKA_PORT:-9092}:9092" + kafka-init: + condition: service_completed_successfully + networks: + - database environment: - KAFKA_NODE_ID: 1 + KAFKA_NODE_ID: '1' KAFKA_PROCESS_ROLES: broker,controller - KAFKA_LISTENERS: INTERNAL://0.0.0.0:29092,EXTERNAL://0.0.0.0:9092,CONTROLLER://0.0.0.0:9093 - KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka:29092,EXTERNAL://localhost:${KAFKA_PORT:-9092} + KAFKA_LISTENERS: INTERNAL://0.0.0.0:29092,CONTROLLER://0.0.0.0:9093 + KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka:29092 KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,CONTROLLER:PLAINTEXT + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,CONTROLLER:PLAINTEXT KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL - KAFKA_CONTROLLER_QUORUM_VOTERS: 1@localhost:9093 - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 - KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 - KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 + KAFKA_CONTROLLER_QUORUM_VOTERS: 1@kafka:9093 + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: '1' + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: '1' + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: '1' + KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: '0' KAFKA_LOG_DIRS: /tmp/kraft-combined-logs CLUSTER_ID: MDEyMzQ1Njc4OWFiY2RlZg + volumes: + - kafka_data:/tmp/kraft-combined-logs healthcheck: - test: ["CMD-SHELL", "/opt/kafka/bin/kafka-topics.sh --bootstrap-server kafka:29092 --list"] + test: + - CMD + - /opt/kafka/bin/kafka-topics.sh + - --bootstrap-server + - kafka:29092 + - --list interval: 10s timeout: 5s - retries: 10 - start_period: 20s - volumes: - - kafka_data:/tmp/kraft-combined-logs - - wallet-service: - profiles: [applications] + retries: 20 + identity-service: + read_only: true + cap_drop: &id001 + - ALL + security_opt: &id002 + - no-new-privileges:true + profiles: + - applications build: context: . - dockerfile: payguard-wallet-service/Dockerfile - user: "${PAYGUARD_LOCAL_UID:-1000}:${PAYGUARD_LOCAL_GID:-1000}" - ports: ["127.0.0.1:8081:8080"] + dockerfile: infrastructure/application.Dockerfile + args: + SERVICE: identity + restart: unless-stopped + user: ${PAYGUARD_LOCAL_UID:-1000}:${PAYGUARD_LOCAL_GID:-1000} + entrypoint: + - vault + - agent + - -config=/etc/vault/runtime.hcl + - -log-level=error + stop_grace_period: 40s + ports: + - 127.0.0.1:8084:8084 environment: - SPRING_DATASOURCE_URL: jdbc:postgresql://postgres-wallet:5432/payguard_wallet?sslmode=verify-full&sslrootcert=/run/payguard/ca.crt¤tSchema=payguard + SPRING_DATASOURCE_URL: jdbc:postgresql://postgres-identity:5432/payguard_identity?sslmode=verify-full&sslrootcert=/etc/vault/ca.crt¤tSchema=payguard VAULT_ADDR: https://vault:8200 - VAULT_CA_FILE: /run/payguard/ca.crt - VAULT_ROLE_ID_FILE: /run/payguard/approle/role-id - VAULT_SECRET_ID_FILE: /run/payguard/approle/secret-id + VAULT_CA_FILE: /etc/vault/ca.crt + VAULT_TOKEN_FILE: /run/agent/token REDIS_HOST: redis KAFKA_BOOTSTRAP_SERVERS: kafka:29092 volumes: - - ./.local/payguard/apps/wallet:/run/payguard/approle:ro - - ./.local/payguard/ca.crt:/run/payguard/ca.crt:ro + - type: bind + source: ./.local/payguard/apps/identity + target: /run/agent-auth + read_only: true + bind: + create_host_path: false + - type: bind + source: ./.local/payguard/ca.crt + target: /etc/vault/ca.crt + read_only: true + bind: + create_host_path: false + - type: bind + source: ./infrastructure/vault/runtime/identity.hcl + target: /etc/vault/runtime.hcl + read_only: true + bind: + create_host_path: false + tmpfs: + - /run/agent:rw,noexec,nosuid,nodev,uid=${PAYGUARD_LOCAL_UID:-1000},gid=${PAYGUARD_LOCAL_GID:-1000},mode=0700 + - /tmp:rw,noexec,nosuid,nodev,uid=${PAYGUARD_LOCAL_UID:-1000},gid=${PAYGUARD_LOCAL_GID:-1000},mode=0700 depends_on: - postgres-wallet: {condition: service_healthy} - networks: [database, operations] - - loan-service: - profiles: [applications] + postgres-identity: + condition: service_healthy + vault: + condition: service_healthy + redis: + condition: service_healthy + kafka: + condition: service_healthy + networks: + - database + - operations + wallet-service: + read_only: true + cap_drop: *id001 + security_opt: *id002 + profiles: + - applications build: context: . - dockerfile: payguard-loan-service/Dockerfile - user: "${PAYGUARD_LOCAL_UID:-1000}:${PAYGUARD_LOCAL_GID:-1000}" - ports: ["127.0.0.1:8082:8080"] + dockerfile: infrastructure/application.Dockerfile + args: + SERVICE: wallet + restart: unless-stopped + user: ${PAYGUARD_LOCAL_UID:-1000}:${PAYGUARD_LOCAL_GID:-1000} + entrypoint: + - vault + - agent + - -config=/etc/vault/runtime.hcl + - -log-level=error + stop_grace_period: 40s + ports: + - 127.0.0.1:8081:8080 environment: - SPRING_DATASOURCE_URL: jdbc:postgresql://postgres-loan:5432/payguard_loan?sslmode=verify-full&sslrootcert=/run/payguard/ca.crt¤tSchema=payguard + SPRING_DATASOURCE_URL: jdbc:postgresql://postgres-wallet:5432/payguard_wallet?sslmode=verify-full&sslrootcert=/etc/vault/ca.crt¤tSchema=payguard VAULT_ADDR: https://vault:8200 - VAULT_CA_FILE: /run/payguard/ca.crt - VAULT_ROLE_ID_FILE: /run/payguard/approle/role-id - VAULT_SECRET_ID_FILE: /run/payguard/approle/secret-id + VAULT_CA_FILE: /etc/vault/ca.crt + VAULT_TOKEN_FILE: /run/agent/token REDIS_HOST: redis KAFKA_BOOTSTRAP_SERVERS: kafka:29092 volumes: - - ./.local/payguard/apps/loan:/run/payguard/approle:ro - - ./.local/payguard/ca.crt:/run/payguard/ca.crt:ro + - type: bind + source: ./.local/payguard/apps/wallet + target: /run/agent-auth + read_only: true + bind: + create_host_path: false + - type: bind + source: ./.local/payguard/ca.crt + target: /etc/vault/ca.crt + read_only: true + bind: + create_host_path: false + - type: bind + source: ./infrastructure/vault/runtime/wallet.hcl + target: /etc/vault/runtime.hcl + read_only: true + bind: + create_host_path: false + tmpfs: + - /run/agent:rw,noexec,nosuid,nodev,uid=${PAYGUARD_LOCAL_UID:-1000},gid=${PAYGUARD_LOCAL_GID:-1000},mode=0700 + - /tmp:rw,noexec,nosuid,nodev,uid=${PAYGUARD_LOCAL_UID:-1000},gid=${PAYGUARD_LOCAL_GID:-1000},mode=0700 depends_on: - postgres-loan: {condition: service_healthy} - networks: [database, operations] - - collateral-service: - profiles: [applications] + postgres-wallet: + condition: service_healthy + vault: + condition: service_healthy + redis: + condition: service_healthy + kafka: + condition: service_healthy + networks: + - database + - operations + loan-service: + read_only: true + cap_drop: *id001 + security_opt: *id002 + profiles: + - applications build: context: . - dockerfile: payguard-collateral-service/Dockerfile - user: "${PAYGUARD_LOCAL_UID:-1000}:${PAYGUARD_LOCAL_GID:-1000}" - ports: ["127.0.0.1:8083:8080"] + dockerfile: infrastructure/application.Dockerfile + args: + SERVICE: loan + restart: unless-stopped + user: ${PAYGUARD_LOCAL_UID:-1000}:${PAYGUARD_LOCAL_GID:-1000} + entrypoint: + - vault + - agent + - -config=/etc/vault/runtime.hcl + - -log-level=error + stop_grace_period: 40s + ports: + - 127.0.0.1:8082:8080 environment: - SPRING_DATASOURCE_URL: jdbc:postgresql://postgres-collateral:5432/payguard_collateral?sslmode=verify-full&sslrootcert=/run/payguard/ca.crt¤tSchema=payguard + SPRING_DATASOURCE_URL: jdbc:postgresql://postgres-loan:5432/payguard_loan?sslmode=verify-full&sslrootcert=/etc/vault/ca.crt¤tSchema=payguard VAULT_ADDR: https://vault:8200 - VAULT_CA_FILE: /run/payguard/ca.crt - VAULT_ROLE_ID_FILE: /run/payguard/approle/role-id - VAULT_SECRET_ID_FILE: /run/payguard/approle/secret-id + VAULT_CA_FILE: /etc/vault/ca.crt + VAULT_TOKEN_FILE: /run/agent/token REDIS_HOST: redis KAFKA_BOOTSTRAP_SERVERS: kafka:29092 volumes: - - ./.local/payguard/apps/collateral:/run/payguard/approle:ro - - ./.local/payguard/ca.crt:/run/payguard/ca.crt:ro + - type: bind + source: ./.local/payguard/apps/loan + target: /run/agent-auth + read_only: true + bind: + create_host_path: false + - type: bind + source: ./.local/payguard/ca.crt + target: /etc/vault/ca.crt + read_only: true + bind: + create_host_path: false + - type: bind + source: ./infrastructure/vault/runtime/loan.hcl + target: /etc/vault/runtime.hcl + read_only: true + bind: + create_host_path: false + tmpfs: + - /run/agent:rw,noexec,nosuid,nodev,uid=${PAYGUARD_LOCAL_UID:-1000},gid=${PAYGUARD_LOCAL_GID:-1000},mode=0700 + - /tmp:rw,noexec,nosuid,nodev,uid=${PAYGUARD_LOCAL_UID:-1000},gid=${PAYGUARD_LOCAL_GID:-1000},mode=0700 depends_on: - postgres-collateral: {condition: service_healthy} - networks: [database, operations] - - identity-service: - profiles: [applications] + postgres-loan: + condition: service_healthy + vault: + condition: service_healthy + redis: + condition: service_healthy + kafka: + condition: service_healthy + networks: + - database + - operations + collateral-service: + read_only: true + cap_drop: *id001 + security_opt: *id002 + profiles: + - applications build: context: . - dockerfile: payguard-identity-service/Dockerfile - user: "${PAYGUARD_LOCAL_UID:-1000}:${PAYGUARD_LOCAL_GID:-1000}" - ports: ["127.0.0.1:8084:8084"] + dockerfile: infrastructure/application.Dockerfile + args: + SERVICE: collateral + restart: unless-stopped + user: ${PAYGUARD_LOCAL_UID:-1000}:${PAYGUARD_LOCAL_GID:-1000} + entrypoint: + - vault + - agent + - -config=/etc/vault/runtime.hcl + - -log-level=error + stop_grace_period: 40s + ports: + - 127.0.0.1:8083:8080 environment: - SPRING_DATASOURCE_URL: jdbc:postgresql://postgres-identity:5432/payguard_identity?sslmode=verify-full&sslrootcert=/run/payguard/ca.crt¤tSchema=payguard + SPRING_DATASOURCE_URL: jdbc:postgresql://postgres-collateral:5432/payguard_collateral?sslmode=verify-full&sslrootcert=/etc/vault/ca.crt¤tSchema=payguard VAULT_ADDR: https://vault:8200 - VAULT_CA_FILE: /run/payguard/ca.crt - VAULT_ROLE_ID_FILE: /run/payguard/approle/role-id - VAULT_SECRET_ID_FILE: /run/payguard/approle/secret-id + VAULT_CA_FILE: /etc/vault/ca.crt + VAULT_TOKEN_FILE: /run/agent/token REDIS_HOST: redis KAFKA_BOOTSTRAP_SERVERS: kafka:29092 volumes: - - ./.local/payguard/apps/identity:/run/payguard/approle:ro - - ./.local/payguard/ca.crt:/run/payguard/ca.crt:ro + - type: bind + source: ./.local/payguard/apps/collateral + target: /run/agent-auth + read_only: true + bind: + create_host_path: false + - type: bind + source: ./.local/payguard/ca.crt + target: /etc/vault/ca.crt + read_only: true + bind: + create_host_path: false + - type: bind + source: ./infrastructure/vault/runtime/collateral.hcl + target: /etc/vault/runtime.hcl + read_only: true + bind: + create_host_path: false + tmpfs: + - /run/agent:rw,noexec,nosuid,nodev,uid=${PAYGUARD_LOCAL_UID:-1000},gid=${PAYGUARD_LOCAL_GID:-1000},mode=0700 + - /tmp:rw,noexec,nosuid,nodev,uid=${PAYGUARD_LOCAL_UID:-1000},gid=${PAYGUARD_LOCAL_GID:-1000},mode=0700 depends_on: - postgres-identity: {condition: service_healthy} - networks: [database, operations] - + postgres-collateral: + condition: service_healthy + vault: + condition: service_healthy + redis: + condition: service_healthy + kafka: + condition: service_healthy + networks: + - database + - operations networks: - operations: database: internal: true - + operations: + internal: true volumes: + postgres_identity_data: + external: true + name: ${PAYGUARD_POSTGRES_IDENTITY_VOLUME:-payguard_postgres_identity_data} postgres_wallet_data: + external: true + name: ${PAYGUARD_POSTGRES_WALLET_VOLUME:-payguard_postgres_wallet_data} postgres_loan_data: + external: true + name: ${PAYGUARD_POSTGRES_LOAN_VOLUME:-payguard_postgres_loan_data} postgres_collateral_data: - postgres_identity_data: + external: true + name: ${PAYGUARD_POSTGRES_COLLATERAL_VOLUME:-payguard_postgres_collateral_data} vault_data: + external: true + name: ${PAYGUARD_VAULT_DATA_VOLUME:-payguard_vault_data} vault_audit: + external: true + name: ${PAYGUARD_VAULT_AUDIT_VOLUME:-payguard_vault_audit} kafka_data: + external: true + name: ${PAYGUARD_KAFKA_DATA_VOLUME:-payguard_kafka_data} diff --git a/docs/guides/postgresql-vault-migration.md b/docs/guides/postgresql-vault-migration.md index 71eb771..751c320 100644 --- a/docs/guides/postgresql-vault-migration.md +++ b/docs/guides/postgresql-vault-migration.md @@ -1,308 +1,6 @@ -# PostgreSQL and Vault implementation guide +# Native infrastructure operations -## Scope and evidence boundary - -This guide records the migration originally developed in `amg-postgresql-vault`, published in PR #35, and integrated with the owner's Identity replacement on `amg-complete-identity-service`. No existing-volume deletion or live-data cutover was performed. - -Available business services are Identity, Wallet, Loan and Collateral. Gateway and Proto have no owned database. Identity contains four package modules in one deployment, so it keeps one owned database. Source inspection found actual Identity JPA challenge/audit adapters, but only repository ports in Loan/Collateral and no Wallet schema, SQL adapter or jOOQ code-generation setup. Configuring their PostgreSQL/Vault infrastructure does not establish persisted financial workflows that do not yet exist. - -Normal runtime/build/service tests on the integration branch select PostgreSQL. Oracle migrations remain historical input, unchanged, in vendor-separated legacy locations. No running Oracle source or approved historical UUID-to-prefixed-ID mapping was supplied; live-data migration remains untested and blocked on those inputs. Absence of a running database was never treated as evidence that data could be discarded. - -## Publication scope and dependencies - -Migration-only PR #35 deliberately excluded the owner's Identity domain PR #33, OTP retirement, root module switch and unchanged application foundations. PRs #33 and #35 were subsequently merged. The integration branch preserves the owner's retirement commit, registers Identity in the reactor and includes its required application, bootstrap, adapter and test files. Five unused future identifier classes and unrelated local files remain outside the publication. - -The historical migration rehearsal results below came from the earlier full local checkout. The current integration was separately verified from a clean committed worktree: full Maven verification on `fd29d35` passed 91 tests and every configured quality gate. The first clean run reproduced two domain regressions; bounded production fixes restored stable identity hashing and rejection before challenge creation, with the existing test assertions retained. See [the publication repair log](../tasks/identity-publication-repair.md) for current publication and CI status. - -## Versions and decisions - -| Component | Selected/resolved version | Reason | -| --- | --- | --- | -| Java / Spring Boot | 25 / 4.1.1 | Actual parent; no unrelated framework upgrade | -| PostgreSQL | 17.11-bookworm, digest in Compose | Supported PostgreSQL major, reproducible server image | -| Vault Community | 2.1.2, digest in Compose | Self-hosted AppRole, KV v2, database engine, Raft and audit; no Enterprise dependency | -| Flyway / JDBC / Hikari | 12.4.0 / 42.7.13 / 7.0.2 | Actual Boot dependency management | -| Testcontainers / JUnit | 2.0.5 / 6.0.3 | Actual managed dependencies | -| jOOQ | OSS 3.19.15 | Existing Wallet dependency; POSTGRES dialect, no paid distribution | -| Redis / Kafka | 8.0.3-alpine / 4.0.1 | Explicit local tags, no `latest` infrastructure images | - -PostgreSQL 17 support is scheduled through November 2029. Vault 2.1.2's actual LICENSE is BUSL 1.1 with an additional noncompeting internal-production use grant and a later MPL 2.0 change license. Community is not described here as OSI open source. The selected features do not need an Enterprise subscription. Recheck the actual license before offering a competing hosted product. [PostgreSQL support](https://www.postgresql.org/support/versioning/), [Vault 2.1.2 LICENSE](https://github.com/hashicorp/vault/blob/v2.1.2/LICENSE), [HashiCorp licensing FAQ](https://www.hashicorp.com/en/license-faq), [jOOQ editions](https://www.jooq.org/download/). - -Accepted decisions: - -- [ADR 0011: PostgreSQL ownership, types and offline cutover](../decission/ADR-0011-postgresql-service-ownership-and-cutover.md) -- [ADR 0012: Vault classification, AppRole, key handling and license](../decission/ADR-0012-vault-service-secrets-and-bootstrap.md) -- [ADR 0013: Lease/pool replacement and failure policy](../decission/ADR-0013-database-leases-and-pool-replacement.md) -- [ADR 0014: OTP row/scope locking and rejection commits](../decission/ADR-0014-otp-postgresql-concurrency.md) - -## Ownership and credential flow - -| Service | Database | Durable owner / runtime group | Runtime Vault identity and role | Schema owner | -| --- | --- | --- | --- | --- | -| Identity | `payguard_identity` | `identity_owner` / `identity_runtime` | `local-identity` / `local-identity-runtime` | Verification migrations in Identity | -| Wallet | `payguard_wallet` | `wallet_owner` / `wallet_runtime` | `local-wallet` / `local-wallet-runtime` | No business schema supplied | -| Loan | `payguard_loan` | `loan_owner` / `loan_runtime` | `local-loan` / `local-loan-runtime` | Loan migrations | -| Collateral | `payguard_collateral` | `collateral_owner` / `collateral_runtime` | `local-collateral` / `local-collateral-runtime` | Collateral migrations | - -Before: independent Oracle containers and datasource passwords from environment variables. Identity additionally accepted a known development pepper fallback. After: four private PostgreSQL servers with TLS/SCRAM, protected server/bootstrap artifacts, separate migration AppRoles, and short-lived Vault-issued runtime logins. Four servers make each credential manager's CREATEROLE authority local to one service rather than relying on cluster-wide role names to isolate a shared server. - -`prepare.py` creates no-login owner/runtime groups, a separate bootstrap migrator and dedicated `vault_manager`. PUBLIC database and public-schema privileges are revoked. The owner creates schema `payguard`; runtime gets CONNECT, schema USAGE, table SELECT/INSERT/UPDATE/DELETE and sequence USAGE/SELECT through owner default privileges. It gets no schema CREATE, TEMP, role administration or migration-history access. Owner-created functions have no public EXECUTE by default. `SchemaMigrator` revokes audit UPDATE/DELETE and Identity scope DELETE after migrations. Runtime DML permissions do not establish a complete append-only financial adapter. - -Vault's manager has LOGIN/CREATEROLE and ADMIN membership only in this server's runtime/owner groups, with INHERIT/SET disabled. Applications never receive it. `database/rotate-root/local-` replaces its initial password; initial protected passwords are intentionally stale afterward. Manager rotation differs from runtime lease rotation. Losing the manager connection requires operator repair with separately protected PostgreSQL administrative authority, then an explicit rotate-root and issuance test. - -Migration logins inherit the durable owner group and execute `SET ROLE _owner` before DDL, so objects are owned by a no-login role. Revoking an expiring migration login can therefore clean it up without dropping business objects. Runtime logins inherit only their runtime DML group and own no schema objects. PostgreSQL plugin revocation explicitly removes membership and drops the login. [PostgreSQL privileges](https://www.postgresql.org/docs/17/ddl-priv.html), [Vault PostgreSQL engine](https://developer.hashicorp.com/vault/docs/secrets/databases/postgresql). - -The application reads its service-only RoleID/SecretID files, verifies Vault's HTTPS certificate, authenticates with AppRole, reads only its database role and installs a validated Hikari pool. Identity can additionally read `secret/data/local/identity/otp-keys` and its authenticator subtree. Wallet cannot obtain Identity/Loan/Collateral credentials or secrets. Runtime policies cannot issue migration credentials, administer Vault or revoke another role's leases. - -| Existing material | Classification and resulting source | Rotation / missing behavior | -| --- | --- | --- | -| Four runtime database logins | Dynamic database secrets engine, independently scoped runtime AppRoles | Pool generation replacement before expiry; no static fallback | -| Schema migration logins | Separate database roles and operator/build AppRoles | Expiring login owns no objects; durable owner retains schema | -| Vault database manager passwords | Separate operational credential per PostgreSQL server | `database/rotate-root`; never supplied to applications | -| PostgreSQL bootstrap passwords, Vault shares/operator token, AppRole SecretIDs, CA private key | Ignored mode-0600 operator artifacts for rehearsal | Operator-only provisioning, renewal/recovery and protected custody | -| Identity OTP/device-binding HMAC pepper | KV v2 named key ring at `secret/data/local/identity/otp-keys` | Required at startup; versioned/explicit legacy compatibility, bounded cache | -| Customer authenticator seeds | KV v2 customer path under `local/identity/authenticators` | Missing seed returns empty; denial/outage fails as dependency error before attempt mutation | -| Public customer/challenge IDs and AppRole RoleIDs | Identifiers | Preserve exact values; they are not authentication credentials | -| Delivery-provider/API signing credentials | No implemented credential-consuming provider/authentication adapter found | No fictitious secret path or integration added; delivery defaults remain synthetic/mock | - -Runtime and migration policies explicitly permit lookup/renew/revoke of their own token and do not inherit Vault's default policy. Lease renewal/revocation is constrained to the runtime role prefix. The operator policy enumerates its four migration credential paths; Vault ACL glob syntax must not be mistaken for arbitrary middle-of-path wildcard matching. - -## Schema and type mapping - -| Service / source type | PostgreSQL representation | Preserved meaning and limits | -| --- | --- | --- | -| Identity current challenge/customer IDs | `varchar(26)` / `varchar(255) COLLATE "C"` | Opaque `otp_`/`cus_` values, exact case and bytes; no trimming/lowercasing/UUID conversion | -| Identity audit `RAW(16)` / Java UUID | native `uuid` | UUID audit identity remains UUID; canonical RAW hex conversion uses network byte order, verified by synthetic vector | -| Identity attempts / limits | `integer` | Attempts bounded 0..max; ACTIVE cannot be exhausted; existing domain transitions retained | -| Identity hashes / enum names | bounded `varchar` | TOTP code hash nullable; device binding nullable; RANDOM_NUMERIC needs a nonblank hash | -| Identity creation/expiry/consumption/retention | `timestamptz(6)` | Instant semantics; consumed nullable except USED; expiry and consumption checks enforced | -| Loan account/collateral/event IDs | native `uuid` | Cross-service collateral identifier remains opaque reference with no cross-database foreign key | -| Loan amount / annual rate | `numeric(28,8)` / `numeric(20,12)` | Same decimal precision/scale; no floating point | -| Loan accrual day | `date` | Business LocalDate retained | -| Loan version / generated audit/outbox ID | `bigint` / bigint identity | Exact integer representation; generated IDs remain numeric | -| Collateral IDs / loan reference | native `uuid` | Owned valuation FK retained; no FK into Loan database | -| Collateral amount/value / FX / LTV thresholds | `numeric(28,8)` / `numeric(28,12)` / `numeric(10,8)` | Existing exact scales and currency column retained | -| Loan/Collateral CLOB payload | `text` | Outbox content retained as text, no invented JSON schema | -| Loan/Collateral timestamps | `timestamptz(6)` | Nullable retention/publication stays nullable; no original-offset preservation claim | -| Wallet Money | Existing BigDecimal + Currency | Domain currency fraction digits/HALF_EVEN unchanged; no database mapping exists to migrate | - -PostgreSQL stores an instant for timestamptz, not the original offset. Precision is microseconds; arbitrary Java nanoseconds are not preserved. Identity explicitly truncates application timestamps to microseconds, including the time re-evaluated after acquiring a verification lock. The offline converter also explicitly truncates source precision. Review/reconcile any historical sub-microsecond data before approving cutover. - -Oracle's empty-string-as-null behavior is not reproduced automatically. Existing domain validation and null fields are reviewed explicitly; PostgreSQL constraints reject important blank hashes. Unquoted names remain lower-case SQL identifiers. SYS_GUID defaults become PostgreSQL UUID generation only for actual UUID audit identity. PL/SQL is preserved historically rather than translated mechanically. There are no Oracle native business queries, jOOQ generated statements or code-generation configuration in the available runtime source. - -## Flyway history and OTP concurrency - -PostgreSQL Identity V1 creates the current string-ID shape directly. V2 adds persistent customer/purpose scope rows, backfills existing scopes and creates a partial unique ACTIVE index. Loan/Collateral each have PostgreSQL V1. These are new vendor histories: Oracle checksums/history remain separate and are never copied into PostgreSQL as proof of migration. - -Normal application Flyway is disabled. Provision the owner/schema, run separate migration tooling, then start the service; Hibernate validates Identity mappings instead of creating tables. Integration tests enable real Flyway before Hibernate validate. `flyway.clean` stays disabled. Duplicate historical ACTIVE challenges cause V2 to fail transactionally; the migration never deletes or guesses which challenge to retain. - -Verification uses a PostgreSQL row lock inside a bounded REQUIRES_NEW adapter transaction. The adapter restores current state, applies the callback, flushes and commits. The application throws its business rejection only after that return, so failed attempts, final lock and expiry transitions persist. One concurrent caller gets fresh VERIFIED; the competitor sees USED and gets ALREADY_USED. Failed attempts serialize; successful and failed final allowed attempts each produce a valid terminal transition. Lock/connection failures propagate as persistence/dependency failures and are not reported as verification success or automatically retried. - -Issuance upserts and locks an enduring `(user_id,purpose)` scope before invalidating locked ACTIVE rows and inserting a replacement. This also serializes first issuance when no challenge exists. The partial unique index is a final invariant. Other customers/purposes are unaffected. Insert-only `save` prevents merging stale snapshots over current state. - -Quota acquisition and Vault-backed TOTP verification occur outside the database lock. The locked mutation rechecks ownership, current state and current time. Audit, delivery and Kafka remain separate side effects; their existing failure/commit gap is documented as separate work. No exactly-once delivery, stored verification response idempotency or database/broker atomicity is claimed. - -## Persistent local setup: exact commands - -Prerequisites: clean source checkout with JDK 25 selected in JAVA_HOME/PATH, Docker/Compose, Python 3 and OpenSSL. Use a disposable rehearsal host. Existing `.local/payguard` or named volumes may contain data; preparation refuses to overwrite artifacts and no command below deletes volumes. - -```bash -./mvnw -B -ntp -pl payguard-identity-service,payguard-wallet-service,payguard-loan-service,payguard-collateral-service -am -DskipTests package dependency:copy-dependencies -DincludeScope=test -python3 scripts/local/prepare.py -docker compose --env-file .env.example up -d --wait postgres-wallet postgres-loan postgres-collateral postgres-identity vault -python3 scripts/local/vault_bootstrap.py -python3 scripts/local/migrate.py identity -python3 scripts/local/migrate.py loan -python3 scripts/local/migrate.py collateral -python3 scripts/local/verify_vault.py -python3 scripts/local/verify_applications.py -``` - -Skipping tests in the first command builds helper classpaths; it does not claim verification. Run the suites below. Wallet has no schema migration command because no business schema was supplied. - -The parent Maven Flyway plugin reads connection settings directly from `FLYWAY_URL`, `FLYWAY_USER`, `FLYWAY_PASSWORD` and `FLYWAY_INIT_SQL` in the invoking process. They are not Maven property references, so ordinary IDE imports and builds do not need migration credentials. An explicitly managed Maven migration must receive a verified TLS URL, a separately issued migration login and `FLYWAY_INIT_SQL` selecting the service's durable owner, for example `SET ROLE identity_owner`. That variable overrides the plugin's default SQL, which refuses database operations when owner-role selection is omitted. The supported local commands above use `migrate.py` and `SchemaMigrator`, which retain their internal `PAYGUARD_MIGRATION_*` variables, derive the owner from the validated service name, apply the post-migration grants and revoke the operator token. A raw Maven invocation does not perform that surrounding workflow. [Flyway Maven environment configuration](https://documentation.red-gate.com/fd/maven-goal-277579365.html). - -Preparation creates mode-0600 files under a mode-0700 ignored directory: synthetic random bootstrap passwords, CA/private keys and per-server TLS certificates. Both application-to-database and Vault-to-database URLs use `sslmode=verify-full` with the local CA. The CA/certificates expire after 30 days; this is explicit local rehearsal material, not production certificate lifecycle. TLS remains enabled on persistent Vault/Raft. Testcontainers tests use isolated disposable test passwords/plain local JDBC; they are not the non-disposable design. - -`PostgreSqlTls` validates the configured single-host JDBC URL before any connection. It requires an absolute CA path and exactly one `sslmode=verify-full`; duplicate/encoded overrides, URL passwords, userinfo and custom SSL factories/hostname verifiers are refused. Multi-host failover is not implemented by this local topology. Owner default EXECUTE is revoked globally for future functions because a per-schema revoke cannot remove PostgreSQL's global PUBLIC default grant. - -Vault startup remains sealed until initialization/unseal. Bootstrap initializes three shares with a two-share threshold, stores them in protected local operator artifacts, unseals and waits for leader readiness. It enables KV v2, the database engine, AppRole and a file audit device, provisions independent service policies, rotates manager passwords, creates a separately scoped orphan operator token and revokes/removes the initial root token. Apps mount only their AppRole directory and CA. Administrative/unseal/operator artifacts are not mounted into applications or copied into Docker images. - -RoleIDs are public identifiers; SecretIDs/tokens/unseal shares are secrets. Provisioning obtains a 60-second response-wrapped SecretID and unwraps once into a protected service file; service SecretIDs expire after one hour. Runtime tokens have 20-minute renewable leases with a four-hour maximum. The operator coordinates fresh files atomically before SecretID expiry: - -```bash -python3 scripts/local/refresh_approle.py -# Run under a supervised local operator process for a long rehearsal: -python3 scripts/local/refresh_approle.py --watch -``` - -The watcher renews its operator token and refreshes runtime files every 20 minutes. It stops on operator failure. It is an operator process, not application administration; production needs actual protected supervision/workload identity and alerting. Existing application tokens continue until their valid limits, then reauthenticate using the newest file. If refresh stops and material expires, startup/re-authentication fail; no default credential is accepted. Vault's operator token also has a server-imposed maximum lifetime; recover/re-provision before that boundary. - -After a Vault restart, rerun bootstrap to unseal using the protected local shares. If the operator token expired/lost, explicit threshold-key recovery is available: - -```bash -python3 scripts/local/vault_bootstrap.py --recover-operator -``` - -This uses Vault's threshold recovery protocol, creates an independent operator token and revokes the temporary recovery root token. It requires protected unseal shares; no long-lived root token is put in `.env`. Local share co-location is a rehearsal convenience; production needs separate custody and recovery storage. - -Ordinary bootstrap is repeatable: it retains an existing database engine configuration and rotated manager password, retains existing OTP key material, refreshes policies/AppRoles and never reinitializes Raft or business data. `--diagnostics` displays only failed bootstrap API paths and HTTP codes, never response bodies or credentials. After operator renewal fails, use explicit recovery; elapsed time is not authorization to regenerate or delete data. - -Applications start only after migration and secret provisioning. Set `PAYGUARD_LOCAL_UID`/`PAYGUARD_LOCAL_GID` in a public `.env` to the host owner of protected service files if different from 1000. Keep `PAYGUARD_ENVIRONMENT` aligned with independently provisioned environment policies/paths; the included bootstrap is explicitly local. - -```bash -docker compose --env-file .env.example --profile applications up -d --build -``` - -This is a local start command, not evidence that every financial workflow exists. Provider integration/configuration and missing repository adapters can block application business flows; inspect actual startup/readiness. Identity readiness includes db, Vault authentication and valid OTP keys. No service reports persistence readiness merely because a scheduler is alive. - -`verify_applications.py` instead boots each actual packaged JAR on the private network with only its own service AppRole directory and CA. It polls HTTP readiness inside the temporary container, checks secret markers in captured logs and removes only its own application probe containers. It does not mount administrative material, expose a public API, delete volumes or simulate a missing financial adapter. All four packaged services reached readiness UP locally. This proves startup/wiring, not Wallet ledger or Loan/Collateral repository flows. - -## Pool, key and failure lifecycle - -`LeasedDataSource` acquires a credential lease (default one hour, max four hours), validates a Hikari generation and switches future borrowers at half TTL. Old idle connections are evicted; checked-out old transactions drain for up to ten seconds, also bounded by lease safety deadline. Replacement uses a fresh credential rather than assuming Spring Cloud Vault automatically rotates/reconfigures a DataSource at max_ttl. It retries failed replacement once per second while old credentials remain valid, limits retained generations to eight and rejects borrowers at TTL minus two seconds. Lease timing starts conservatively before the issuing request. Old generations close/revoke after drain/deadline. [Spring Cloud Vault limitation](https://docs.spring.io/spring-cloud-vault/reference/secret-backends.html). - -Long transactions beyond drain/deadline can fail. PostgreSQL runtime statement/lock/idle-transaction timeouts are ten seconds. No transaction is replayed automatically: callers need their actual business idempotency/recovery contract. The pool has four maximum connections per generation and briefly overlaps generations. No capacity measurement is claimed. - -Revoking a PostgreSQL role stops new logins; existing established sessions can remain valid. Tests demonstrate this. Application-owned generations close after the bounded drain, but emergency external revocation does not imply immediate termination of every session. If forced termination is required, operators need an explicitly authorized database session-termination procedure. - -The pool validates CONNECT and schema USAGE on startup and each borrow, so an established session passing `SELECT 1` after role revocation cannot masquerade as a usable runtime connection. Failed borrowing requests asynchronous credential/pool replacement, with a one-second maintenance interval and bounded network/acquisition calls; legitimate work resumes only after validation. It does not retry the caller's transaction. Forced revocation may break in-flight DML even though the session itself survives; detection of every arbitrary table-grant configuration change is outside this check. - -Vault outage: existing valid database credentials work only to their safety deadline; new issuance/authentication fails, and readiness is down. Database outage: connection acquisition fails within the pool timeout, no verification succeeds, and recovery allows fresh legitimate work. Invalid required keys, missing provisioning, denied policy and expired authentication have sanitized dependency errors. There is no known password/pepper fallback. Secret values are omitted from errors, pool names and snapshot/key-ring diagnostics. - -Identity KV key ring has `current`, `keys` and an optional explicit `legacy`. New hashes are `keyId:salt:digest`; key material cannot change under an existing ID. To rotate, add a new random named key, select it as current and retain the previous key for at least max(random OTP TTL, TOTP TTL) plus ten seconds after its last use. Keep an explicit legacy key until all unversioned stored challenges/device bindings are expired or reconciled. Never guess the old key. Refresh happens every five seconds outside SQL locks; valid cached keys have a bounded five-minute outage lifetime, then fail closed. Removing a needed key early is rejected; do not overwrite pepper material under the same name. - -The explicit legacy alias cannot be reassigned to another key by a live refresh: unversioned hashes carry no information to select that different key. Removing the alias is also rejected during the compatibility window, even if the old key material remains in the ring. Removing key material and removing the alias are both retirement operations. - -Retirement is an operator-coordinated decision across instances, not a distributed retention registry. Allow the five-minute cached-key outage window plus maximum challenge TTL plus ten seconds after switching the Vault current key, and confirm all issuers have refreshed/stopped using the old key before removal. Local refresh guards reject unsafe removal for already running instances; newly started instances cannot recover key material removed from Vault. Failed refresh sets the OTP key readiness contributor DOWN while still-valid cached keys remain bounded; startup requires valid provisioned material. Preserve legacy key provenance across restart rather than guessing from a hash. - -OTP pepper rotation, AppRole token renewal, PostgreSQL login replacement and authenticator seed re-enrollment are separate operations. Seed rotation is not implemented as enrollment/recovery. Vault stores recoverable TOTP seeds in encrypted storage with scoped KV reads. No existing business field needing a new recoverable field-encryption adapter was identified; no Transit/card vault or indiscriminate column encryption was invented. Local TLS and Vault storage protection do not implement PostgreSQL volume/backup encryption; hosting must supply that layer. - -## Audit, backup and recovery - -Persistent Vault audit output is `/vault/audit/audit.json` on the protected `vault_audit` volume. Default HMAC protection remains enabled. The test runner verifies allowed/denied accesses and hashed credential values without printing raw audit output. Reading audit files belongs to operators; paths and access metadata remain sensitive even when values are protected. - -Vault can reject requests if it cannot write to any enabled audit device. This single-device local design therefore requires disk-capacity monitoring, retention/rotation and protected backups before production. No automatic audit retention, HA or auto-unseal was implemented. Do not solve disk failure by disabling audit or raw-value protection. PostgreSQL error-statement/parameter logging is suppressed to avoid CREATE ROLE password leakage. - -For a local backup with an installed compatible Vault CLI, supply the operator token via protected input; never print it or put it in command-line arguments. Set only public TLS settings normally: - -```bash -export VAULT_ADDR=https://127.0.0.1:8200 -export VAULT_CACERT="$PWD/.local/payguard/ca.crt" -umask 077 -read -r -s -p 'Operator token: ' VAULT_TOKEN -export VAULT_TOKEN -vault operator raft snapshot save .local/payguard/operator/raft.snapshot -unset VAULT_TOKEN -``` - -Back up PostgreSQL with separately authorized owner/admin tooling, protect/encrypt exports, retain the approved ID mapping, and store Vault recovery shares separately from encrypted snapshots. Rehearse restore into isolated infrastructure with the original key/certificate custody. Snapshot restore replaces Vault state and is an explicit operator recovery action, not an automatic command in setup. These backup/restore CLI steps were documented, not executed; live hosting/encryption/custody remain production-dependent. - -## Existing-data rehearsal and cutover - -1. Establish an authorized immutable Oracle export and inventory its vendor migration history, rows, numeric totals, UUID byte order, empty strings and timestamp precision. Export no secrets into ordinary logs. Keep Oracle histories unchanged. -2. Approve a persistent mapping artifact for every legacy customer/challenge identifier. Do not create new customer identities per request. Already-current prefixed IDs pass through exactly; mapping is still a provenance-reviewed artifact. -3. Freeze writes for an offline cutover. Preserve source export, mapping checksum, row counts and representative records. Do not copy Oracle `flyway_schema_history` into PostgreSQL. -4. Provision an empty owned PostgreSQL destination and apply its own Flyway migrations using the separate migration identity. -5. Convert a protected export with `scripts/migration/convert_identity_export.py`. Its JSON input has `challenges` and `audit` arrays named after current database columns. The mapping has `approved_by`, `approval_reference`, `customers` and `challenges` maps keyed by exact source identifiers. Actual audit UUID/RAW hex is preserved as UUID; timestamps require offsets and explicitly truncate to microseconds. Missing mapping, duplicate identity and orphan audit records are refused. -6. Import parents before children with an explicit command. The importer locks/checks an empty destination, inserts in one transaction, relies on constraints/ACTIVE uniqueness, verifies audit subject ownership, row counts and every supplied field, then commits. Populated destinations and reconciliation errors are refused; failures roll back without truncation. - -```bash -python3 scripts/migration/convert_identity_export.py \ - .local/payguard/operator/source-export.json \ - .local/payguard/operator/approved-id-mapping.json \ - .local/payguard/operator/converted-identity.json -python3 scripts/local/migrate.py identity \ - --import-identity .local/payguard/operator/converted-identity.json -``` - -The mapping/export must be supplied by an authorized operator; there is no live exporter credential or invented customer mapping. Conversion outputs are exclusive-create/mode 0600 and include mapping SHA-256 plus counts/provenance. Verify full source/destination counts, orphan queries, case-sensitive identity/reference equality, challenge state/attempts, expiry/consumption/retention and sampled audit records independently before opening traffic. - -Loan/Collateral tests rehearse their supplied schemas and representative exact numeric/UUID/date/time/null/idempotency/FK records. Live Oracle export/import and financial balance reconciliation were not executed. Wallet lacks a schema/adapter, so no Wallet ledger migration or platform-wide financial concurrency claim is made. Before a real Loan/Collateral cutover, supply authorized exports, import in owned parent/child order, preserve sequence identities or advance sequences deliberately and reconcile principal/outstanding/accrued/valuation totals and currency grouping. Those live tooling decisions remain blocked on actual source data and missing adapters. - -Before PostgreSQL accepts writes, rollback can redirect to the frozen Oracle source after verifying no target-only writes occurred. After it accepts writes, rollback requires reconciling/reverse-migrating target changes and side effects; retaining an Oracle volume alone is not sufficient. Prefer rehearsing recovery forward with backups and correction procedures before choosing a reverse cutover. Do not enable clean, drop volumes, truncate tables or regenerate identities to bypass a failed rehearsal. - -## Change ledger - -| Repository-relative path | Before → resulting behavior | Reason / evidence | -| --- | --- | --- | -| `pom.xml` | Oracle dependency management removed; PostgreSQL Flyway plugin reads native `FLYWAY_*` variables at execution time, with required owner-role selection | Managed compatible versions; Java/Spring unchanged; parent validation passed; isolated PostgreSQL 17.11 Maven migration applied Loan V1, verified five tables/two sequences owned by `loan_owner`, and rejected omitted owner initialization. The Vault-managed helper remains the complete local migration workflow | -| `payguard-{identity,loan,collateral,wallet}-service/pom.xml` | Oracle JDBC/Flyway/test containers → PostgreSQL and actuator | Actual service persistence approach retained; compilation and real database suites | -| `payguard-{identity,loan,collateral,wallet}-service/src/main/resources/application.yaml` | Oracle/env runtime passwords → PostgreSQL/Vault pool, explicit readiness | No pepper/password fallback; runtime Flyway disabled; separate migration path | -| `payguard-identity-service/src/main/resources/db/migration/postgresql/V1__identity_schema.sql` | New vendor history in final prefixed-ID shape | UUID audit/string identity/timestamp/check constraints | -| `payguard-identity-service/src/main/resources/db/migration/postgresql/V2__serialize_active_challenges.sql` | Scope-lock rows and partial ACTIVE index | First issuance race and populated V1 upgrade/rejection rehearsal | -| `payguard-{loan,collateral}-service/src/main/resources/db/migration/postgresql/V1__*_schema.sql` | Oracle types/defaults → PostgreSQL exact equivalents | Real Flyway and exact numeric/UUID/date/time/constraint tests | -| `payguard-{identity,loan,collateral}-service/src/main/resources/db/legacy/oracle` | Original scripts relocated unchanged | Preserve deployed Oracle histories/guards; never normal Flyway locations | -| `payguard-identity-service/src/main/java/dev/amg/payguard/identity/verification/domain/port/OtpRepository.java` | Separate load/save → locked mutation contract; save inserts only | Cross-process state freshness/explicit commit boundary | -| `payguard-identity-service/src/main/java/dev/amg/payguard/identity/verification/infrastructure/persistence/JpaOTPRepositoryAdapter.java` | Unserialized writes → row/scope locks and ordered replacement | Real concurrent verification/issuance and rollback tests | -| `payguard-identity-service/src/main/java/dev/amg/payguard/identity/verification/infrastructure/persistence/OtpChallengeJpaRepository.java` | Ordinary queries → explicit locked current-row/ACTIVE queries | PostgreSQL transactions | -| `payguard-identity-service/src/main/java/dev/amg/payguard/identity/verification/application/OtpApplicationService.java` | Stale verification save → committed result boundary, remote calls outside locks, microsecond/current locked time | Rejection commits, final attempts, ownership and expiry tests | -| `payguard-identity-service/src/main/java/dev/amg/payguard/identity/verification/domain/model/OtpSnapshot.java` | Nullable lifecycle contradictions → conditional consumed/binding validation and safe copies | Needed to restore real persisted states; no unrelated domain redesign | -| `payguard-identity-service/src/main/java/dev/amg/payguard/identity/verification/infrastructure/security` | Known pepper/unavailable seed → named HMAC ring and scoped Vault adapter | Real seed/rotation tests; enrollment remains absent | -| `payguard-shared/src/main/java/infrastructure/vault/VaultClient.java` | No client → verified TLS AppRole, bounded auth/renew/read calls, sanitized errors | Token renewal/re-authentication and policy/outage tests | -| `payguard-shared/src/main/java/infrastructure/vault/LeasedDataSource.java` | Static pool → validated credential/pool generations and bounded drain | Real short TTL/maxTTL, held transaction and outage tests | -| `payguard-shared/src/main/java/infrastructure/vault/VaultDatabaseAutoConfiguration.java` and `src/main/resources/META-INF/spring` | No shared wiring → infrastructure auto-configuration/readiness | Actual Spring wiring test | -| `payguard-shared/src/main/java/infrastructure/vault/SchemaMigrator.java` | No separate executable → real Flyway as owner role | Actual local Identity/Loan/Collateral migration commands and post-migration privilege restrictions | -| `payguard-shared/src/main/java/infrastructure/vault/IdentityDataImporter.java`, `scripts/migration` | No rehearsal path → approved mapping, empty transactional import/reconciliation | Synthetic Python/real PostgreSQL tests; no live source claim | -| `scripts/local/{common,prepare,vault_bootstrap,refresh_approle,migrate,verify_vault,verify_applications}.py` | Environment passwords/manual assumptions → protected provisioning and actual dependency/application runners | Local TLS initialization, recovery, manager rotation, credential tests and four real readiness probes | -| `payguard-identity-service/.../customer/domain/value_object/PrefixedIdValidator.java`, `package-info.java` | Missing referenced validator → exact existing prefix/22 ASCII character validation and package nullness | Current checkout failed compilation without this helper; existing typed-ID tests pass, no accessor/ID changes | -| `docker-compose.yaml`, `infrastructure/{postgresql,vault}`, `.env.example`, `.dockerignore` | Oracle/dev configuration → private TLS PostgreSQL and persistent non-dev Vault | No existing volume touched; secret directories excluded from images | -| `payguard-{identity,loan,collateral}-service/src/test` | Oracle/gated or datasource-excluding fixtures → actual PostgreSQL/Flyway tests | No opt-in test flag; disposable test profile is explicit | -| `.github/workflows/_database-security.yml`, `pipeline.yml`, `_publish.yml` | No Vault lifecycle CI → independent security suite, four real application probes and SHA-tagged service image builds | Workflow calls real test/migration/runner commands; remote CI execution not claimed | -| `README.md`, `payguard-identity-service/{README,ARCHITECTURE}.md`, `docs/decission/ADR-0011..0014` | Outdated Oracle/unresolved claims → implemented scope and explicit limits | Source-based handover; no compliance/production claim | - -## Verification commands and results - -Use JDK 25; a default older JDK fails `release version 25 not supported` and is not an application regression. - -```bash -./mvnw -B -ntp -pl payguard-identity-service,payguard-loan-service,payguard-collateral-service -am \ - -Dtest=PostgreSqlTlsTest,PostgresqlMigrationRehearsalTest,SanitizedDependencyErrorTest,OtpInfrastructureIntegrationTest,OtpApplicationContractsTest,LoanServiceApplicationTests,CollateralServiceApplicationTests \ - -Dsurefire.failIfNoSpecifiedTests=false clean test -python3 -m unittest discover -s scripts/migration -p 'test_*.py' -./mvnw -B -ntp -pl payguard-identity-service,payguard-wallet-service,payguard-loan-service,payguard-collateral-service -am -DskipTests package dependency:copy-dependencies -DincludeScope=test -python3 scripts/local/verify_vault.py -python3 scripts/local/verify_applications.py -./mvnw -B -ntp verify -``` - -Do not run clean builds concurrently with the Java Vault runner: it reads shared target classes/dependencies and writes reports there. The runner stops/restarts only the isolated Identity PostgreSQL container, seals/unseals only the local rehearsal Vault and restores its shortened role/key fixtures. It must never point to a production environment. - -Baseline full verify failed existing shared import formatting before tests. Baseline Identity diagnostic run executed 53 tests with 4 failures, 25 errors and 8 skips. Required nullable lifecycle fixes made persisted flows testable. An earlier verification executed 76 tests with two domain-contract failures; the current checkout contains owner changes to aggregate identity hashing and pre-creation validation, and those regressions now pass. They are recorded as historical findings, not unresolved defects. - -Current committed integration evidence: `./mvnw --batch-mode --no-transfer-progress verify` on `fd29d35` exited 0 in a clean worktree with **91 tests, zero failures/errors/skips** (Identity 74, other modules 17). Spotless, Checkstyle, PMD, CPD and SpotBugs passed. This includes the 14 PostgreSQL Identity infrastructure cases and existing concurrency/domain/application tests. Native Maven Flyway configuration was also verified on disposable PostgreSQL 17.11: Loan V1 installed, all five tables/two sequences were owned by `loan_owner`, and missing owner initialization was rejected. The Vault-managed helper remains the complete local migration workflow. - -Historical migration rehearsal evidence from the earlier full working checkout: - -- Earlier normal `./mvnw -B -ntp verify`: **95 tests passed, zero failures/errors/skips**. Shared 4, Proto 1, Gateway 2, Wallet 1, Loan 5, Collateral 4, Identity 78. Exit code **1** because Identity PMD reported six existing `AvoidFieldNameMatchingMethodName` findings; no checks were disabled. Log `/tmp/payguard-final-current-verify.log`. -- All modules before Identity passed normal Spotless/Checkstyle/PMD/CPD/SpotBugs. Independent Identity SpotBugs passed with zero bug/error instances; its six typed-ID PMD naming findings remain separate work. Package build with tests skipped succeeded for all four service JARs; it is not counted as test evidence. -- Identity real PostgreSQL suite: 14 passing tests, including competing verification, failed attempts, first issuance/replacement, final boundaries, restored states, ownership, case and rollback. Application contracts: 11 passing tests. Loan/Collateral tests exercise exact schema amounts/UUID/time/null/constraints; absent persisted financial flows remain unproved. -- Synthetic export conversion: 3 Python tests passed, zero skipped. -- Shared TLS parameter guard and secret-error redaction: one passing test each; populated V1/V2/import rehearsal: two passing tests. Expected duplicate ACTIVE upgrade refusal rolls back without deleting records. -- All four actual packaged applications reached HTTP readiness UP using dedicated AppRole files and verified PostgreSQL TLS (`/tmp/payguard-application-readiness.log`), and probe containers were stopped/removed. No financial API-flow claim follows from startup. -- Initial/recovered and ordinary repeat bootstrap, atomic SecretID refresh and all three local migration commands succeeded with retained volumes. Identity V1/V2, Loan V1 and Collateral V1 have their own Flyway histories; migration token revocation completed. -- Previous real Vault run XML contained 15 passing tests, zero skips (~115 seconds). The restored runner first failed 12 shared setup cases because the operator migration path used an invalid middle glob; explicit per-service paths corrected that defect. -- Final `python3 scripts/local/verify_vault.py` completed with exit **0** after legacy-alias hardening: **15 tests passed, zero failures/errors/skips**, approximately 116 seconds, including four runtime DML/default grants, denied DDL/admin/cross-service reads, ten-second statement defaults, real renewal/revocation, continuous pool replacement beyond 45-second max TTL, held transactions, token renewal/re-authentication, manager rotation, seal/unseal, database stop/start, forced revocation recovery, real scoped authenticator lookup and HMAC rotation. This includes refusal to reassign/remove the legacy alias prematurely while retaining old key material. Allowed/denied audit trace and plaintext secret-marker checks passed (`/tmp/payguard-vault-final-alias-tests.log` and `payguard-shared/target/vault-test-reports/TEST-junit-jupiter.xml`). -- Actual Identity Dockerfile build completed with exit **0** (`docker build -f payguard-identity-service/Dockerfile -t payguard/identity-service:rehearsal .`, `/tmp/payguard-identity-docker-build.log`). The image uses pinned Java 25 builder/runtime digests; protected local artifacts are excluded from build context. Tests skipped inside the image build are not counted as executed tests. -- Actual Wallet/Loan/Collateral Dockerfile builds completed with exit **0** (`docker compose --env-file .env.example --profile applications build wallet-service loan-service collateral-service`, `/tmp/payguard-financial-docker-build.log`). Thus all four service Dockerfiles built locally. CI tags output images with the source SHA; Compose derives local image names from its build configuration. No image was pushed or deployed. -- A fresh disposable PostgreSQL initialization using the current `prepare.py` SQL succeeded; runtime CONNECT/USAGE were allowed and schema CREATE/default function EXECUTE denied. The first proof waited on the temporary Unix bootstrap server and failed before initialization finished; a TCP readiness check corrected the fixture. This test used new temporary artifacts/storage with no network and stopped its own container; existing volumes were untouched (`/tmp/payguard-fresh-init-proof.log`). -- A final application probe after Vault/database recovery again reached readiness UP for all four services, scanning both stdout and stderr for secret markers (`/tmp/payguard-application-readiness-final.log`, exit 0). Rehearsal PostgreSQL/Vault containers retain their volumes; temporary application/provisioning probes were stopped/removed. -- After the final legacy-alias guard, normal verification again passed all 95 tests and stopped only at the same six PMD findings. Identity SpotBugs again passed with zero bugs/errors; its actual Docker image was rebuilt successfully from the final Java source. The final Identity-only application probe also reached readiness UP (`python3 scripts/local/verify_applications.py --service identity`, `/tmp/payguard-identity-final-readiness.log`, exit 0). - -Surefire reports are under each module's `target/surefire-reports`; the Vault runner writes `payguard-shared/target/vault-test-reports`. Dedicated CI invokes both paths without an unset environment gate. No remote GitHub run, live-data migration, production outage/restore or load/capacity measurement is claimed. - -## Remaining Oracle references and limits - -PR #35 retained the old OTP module because its retirement was separate owner work. The integration branch removes that module and selects Identity. The remaining Oracle references are historical migrations, documentation and unrelated price-oracle terminology: - -- `payguard-{identity,loan,collateral}-service/src/main/resources/db/legacy/oracle/*.sql`: exact historical vendor migrations and the legacy populated-data guard. -- `OtpInfrastructureIntegrationTest.archivedOracleMigrationRetainsPopulatedDataGuard`: reads the legacy guard as text, without an Oracle engine. `PostgresqlMigrationRehearsalTest` mentions the unavailable Oracle source in Javadoc; it executes PostgreSQL only. -- Historical ADRs, `CLAUDE.md`, migration ADRs/work log/this guide: original context, source-data handling and supersession rationale; no normal runtime Oracle configuration. -- `payguard-collateral-service/.../domain/PriceOracle.java` and its callers: price-feed domain terminology, unrelated to Oracle Database. -- Owner-retired `payguard-otp-service` tracked deletions and ignored stale `target/` output are not available runtime modules and are not migration evidence. - -Full remaining-reference search is `rg -n -i 'oracle|ojdbc|varchar2|sys_guid|gvenzl|pl/sql' --glob '!**/target/**' --glob '!.local/**'`. Treat matched market-oracle names and URLs to Java documentation separately from JDBC/database references. All original Oracle script bytes are retained; no clean/truncation command is added. - -Implemented: available PostgreSQL dependency/config/schema paths, complete Identity module registration, database concurrency, service-scoped Vault client/pool/secrets, persistent local TLS/SCRAM/Raft/audit provisioning, separate migrations and reviewable rehearsal tooling. Current committed Maven verification passes; five unpublished future ID classes remain local owner work. Production-dependent: certificate/SecretID/operator supervision, HA/auto-unseal, encrypted volume/backup hosting, custody, audit retention, capacity and disaster recovery. Not implemented: missing financial repositories, authenticator enrollment/recovery, external auth/Keycloak, durable event/delivery recovery or a new IAM system. - -Review [separate GitHub Issue drafts](../issues/postgresql-vault-follow-up.md) for acceptance criteria. Security references describe the PayGuard design, not invented internal company deployments: [OWASP secrets management](https://cheatsheetseries.owasp.org/cheatsheets/Secrets_Management_Cheat_Sheet.html), [cryptographic storage](https://cheatsheetseries.owasp.org/cheatsheets/Cryptographic_Storage_Cheat_Sheet.html), [Vault AppRole](https://developer.hashicorp.com/vault/docs/auth/approle), [KV v2](https://developer.hashicorp.com/vault/docs/secrets/kv/kv-v2), [Transit](https://developer.hashicorp.com/vault/docs/secrets/transit), [Stripe customer API-key guidance](https://docs.stripe.com/keys-best-practices). +The current runbook is [Infrastructure operations](../infrastructure.md). +The former Java/Python migration launchers have been retired. Historical test +results in `docs/tasks` and `specs` describe earlier implementations and are not +acceptance evidence for this replacement. See [current validation](../infrastructure-validation.md). diff --git a/docs/infrastructure-validation.md b/docs/infrastructure-validation.md new file mode 100644 index 0000000..2f79948 --- /dev/null +++ b/docs/infrastructure-validation.md @@ -0,0 +1,72 @@ +# Vault and native infrastructure validation + +This branch is based on `main` at `c904d71ef8a7541885cbbe887f25148e196ef768`. +It contains the infrastructure replacement, adapted to the committed service module names. +The owner's uncommitted module renames, Identity refactor, Wallet edits, and local tooling +remain outside the branch. No live database, Vault state, or deployment was accessed. + +## Scope + +- Compose manages PostgreSQL 18, Vault, dependencies, networks, application Agents and + independent Agent/Flyway migration jobs. Existing persistent volumes have explicit external names. +- Vault Agent handles AppRole authentication, token/credential renewal and application restart + on credential replacement. Java consumes Agent token files for scoped KV access. +- Ansible replaces preparation, provisioning, initialization, migration orchestration, import, + identity refresh, backup and recovery launchers. The root `scripts/` directory and custom + Java migration/import/leased-datasource implementations are removed. +- The four existing versioned PostgreSQL SQL files are byte-for-byte unchanged; their hashes + are recorded in `infrastructure/migrations.sha256`. Wallet receives a separate V1 marker, + not a financial persistence implementation. +- Identity import and privilege checks retain `identity_challenge`, `identity_audit_log`, and + `identity_challenge_scope`. No SQL from the owner's unfinished Identity refactor is imported. +- Existing Identity KV scoping and pepper-rotation tests are retained. Their fixture helper now + requires Agent-delivered token paths in `PAYGUARD_REHEARSAL_OPERATOR_TOKEN_FILE`, + `PAYGUARD_REHEARSAL_IDENTITY_TOKEN_FILE`, and `PAYGUARD_REHEARSAL_WALLET_TOKEN_FILE`. + Those live Vault tests were not executed here. + +## Executed checks + +| Check | Result | Limit | +| --- | --- | --- | +| Offline deployment/security contracts | PASS, 18 tests | Configuration and HCL/TOML parsing, not live leases | +| Compose base/application/migration/operations configuration | PASS | Compose 2.40.3; no daemon | +| Compose production overlay configuration | PASS | Dummy external broker hosts and CA paths | +| Ansible playbook syntax | PASS, 21 entry points | `preflight.yml` is an imported task list, not a playbook | +| Fresh preparation | PASS | OpenSSL and Ansible on a new isolated fixture | +| Repeated fresh preparation | PASS, safely refused | Generated bytes unchanged; private keys/passwords/config 0600, directories 0700 | +| Existing versioned PostgreSQL SQL | PASS, 4 files unchanged | No live Flyway history available | +| Shared Agent token/TLS/error-sanitization Java tests | PASS, 5 tests | No live Vault; repository style goals skipped for this run | +| Application packaging and test-source compilation | PASS, all four services | Runtime tests skipped; Spotless/Checkstyle bypassed in this packaging run | +| Reactor Maven validation | PASS | Spotless and Checkstyle for parent/shared/four services; PMD/SpotBugs not run | +| Changed Java formatting | PASS | Google Java Format 1.30.0 | +| Original checkout WIP preservation | PASS | Original status, staging state and file hashes unchanged | + +The Java test command used JDK 25 and the repository Maven wrapper: + +```bash +./mvnw -B -ntp -pl payguard-shared \ + -Dspotless.skip=true -Dcheckstyle.skip=true \ + -Dtest=AgentTokenClientTest,PostgreSqlTlsTest,SanitizedDependencyErrorTest test +python3 -m unittest discover -s infrastructure/tests -v +``` + +Dependency downloads used private, workspace-only proxy settings and the system Java trust +store. TLS verification remained enabled; proxy settings and generated secrets are not committed. + +## Required live acceptance + +The reusable native infrastructure CI workflow runs on an isolated Docker host. It prepares +separate volumes, provisions Vault/databases, migrates every database twice, verifies privileges, +starts the applications and rehearses a Vault outage. The workflow is executable test scaffolding, +not evidence that those jobs have passed. + +Production acceptance remains incomplete. Run integration, permission-denial, TLS failure, +Agent renewal/reauthentication, credential replacement, interrupted migration/recovery, +backup/restore, import rollback and business end-to-end tests before deployment. Test held SQL +transactions and Vault outages explicitly; credential expiration does not terminate every +already executing database statement. Complete paired PostgreSQL/Raft restoration and check +existing records, histories, ownership and secret versions on isolated recovery volumes. + +Base Kafka is a local plaintext rehearsal broker. The production overlay requires authenticated +TLS external Redis/Kafka and verified CA/hostname settings. This is a single-host deployment +building block, not a tested highly available database or Vault deployment. diff --git a/docs/infrastructure.md b/docs/infrastructure.md new file mode 100644 index 0000000..e9272f0 --- /dev/null +++ b/docs/infrastructure.md @@ -0,0 +1,302 @@ +# Infrastructure operations + +## Ownership + +Docker Compose manages process dependencies, persistent-volume identities and networks. +Ansible owns preparation, provisioning, import, backups, credential distribution and recovery. +Vault stores the existing Raft/KV state and issues separate runtime and migration PostgreSQL +credentials for Identity, Wallet, Loan and Collateral. Vault Agent owns AppRole authentication, +token renewal, database-lease renewal and credential delivery. Flyway CLI owns schema migration. +There is no application-startup migration and no Maven migration plugin. + +Runtime Agents supervise the service JARs. They wait for their environment templates before +starting Java, renew leases, and send SIGTERM when credentials change. Spring gracefully drains +for 20 seconds; Agent allows 30 seconds before forcing a stop. Run multiple application instances +behind a load balancer if credential replacement must not interrupt availability. The Java Vault +client only reads the Agent sink token for application KV lookups; it never authenticates, +renews, revokes or creates database credentials. + +## Requirements + +Use a Linux host with a local Unix-socket Docker daemon, Docker Compose >= 2.24.4, +Python 3.12, OpenSSL, coreutils, util-linux, and JDK 25 for application builds. The Docker socket +makes the operations runner an administrative component; never expose it to an application or +untrusted CI workload. Application images contain a JRE and Vault Agent, not Ansible or Docker. + +The checked-in image digests are fixed. Python/Ansible/PostgreSQL collection versions are fixed. +OS packages installed by `apk` and platform-specific wheel bytes are not a completely hermetic lock; +produce and scan an image SBOM and promote a built operations-image digest through deployment. +Do not substitute mutable tags during promotion. HashiCorp Vault is subject to its vendor license; +review it against your distribution/service model. The chosen Flyway image is the open-source +edition; no paid secrets resolver is used. + +Install the host tools for the preparation/adoption ceremony: + +```bash +python3 -m venv .local/ansible +.local/ansible/bin/pip install -r infrastructure/operations/requirements.txt +.local/ansible/bin/ansible-galaxy collection install -r infrastructure/operations/requirements.yml +``` + +Keep an existing `.env` and `.local/payguard` intact. Compare `.env.example` with your deployment +configuration; it contains names and paths only. Export the deployment environment before invoking +host Ansible or Compose. For a Bash session at the repository root: + +```bash +export PAYGUARD_REPO_ROOT="$(pwd -P)" +export PAYGUARD_LOCAL_UID="$(id -u)" +export PAYGUARD_LOCAL_GID="$(id -g)" +export PAYGUARD_DOCKER_GID="$(stat -c %g /var/run/docker.sock)" +export COMPOSE_PROJECT_NAME=payguard +export PAYGUARD_RUNTIME_ROOT=/dev/shm/payguard +mkdir -p -m 700 "$PAYGUARD_RUNTIME_ROOT" +export COMPOSE_FILE=docker-compose.yaml:infrastructure/compose.operations.yaml +``` + +Use a separate private tmpfs directory for every deployment. Swap must be disabled or protected +with encryption appropriate for secret material. The controller checks that the handoff directory +is tmpfs, owned by the operator, and mode 0700. Do not set DOCKER_HOST or a remote Docker context. + +Build the runner once: + +```bash +docker compose --profile operations build operations +``` + +In the commands below, `ops PLAYBOOK` means the following native Compose invocation; substitute +the actual playbook and extra variables rather than installing a wrapper: + +```bash +docker compose run --rm --no-deps operations infrastructure/operations/PLAYBOOK.yml +``` + +## Adopt existing PostgreSQL 18 and Vault state + +All seven persistent volumes are external and explicitly named. Normal startup fails if one is +missing, rather than creating an empty replacement. Preserve the existing Compose project and +volume names in `.env.example` or set their exact `PAYGUARD_*_VOLUME` overrides. Do not run +`allocate.yml`, `prepare.yml`, `docker compose down -v`, `flyway clean`, `repair`, or an implicit +baseline against an existing target. + +Inspect each volume's existing `PG_VERSION` location read-only before changing any container. +The PostgreSQL 18 image's standard PGDATA is `/var/lib/postgresql/18/docker`, with its named +volume mounted at `/var/lib/postgresql`. If your old container mounted a volume containing a +cluster directly at `/var/lib/postgresql/data`, that same volume will expose `PG_VERSION` at +`/var/lib/postgresql/PG_VERSION` under the new mount. In that case set +`PAYGUARD_PGDATA=/var/lib/postgresql`; do not initialize or move its files. If the four databases +have different layouts, use a reviewed Compose override with a per-service PGDATA. The entrypoint +refuses a version/layout mismatch. A PostgreSQL 17 cluster requires a separate, rehearsed major +upgrade; changing the image is not an upgrade plan. + +Take verified backups before cutover, quiescing writers for a consistent cross-service checkpoint. +Preserve current Vault recovery material outside this source tree. The archive contains no live +Docker volumes and cannot prove a live backup or restoration. Record the exact local daemon and +volume identities once: + +```bash +.local/ansible/bin/ansible-playbook -i localhost, -c local infrastructure/operations/adopt.yml +``` + +This only inspects and records existing volume names/daemon ID in ignored `target.yml`. On repeat, +it refuses identity drift. Stateful playbooks compare against that target before mutation. +The database reconciliation preserves existing manager passwords, including Vault-rotated ones. +It creates a manager password only when that manager is absent. Existing schema objects must +already have their stable `_owner`; unexpected ownership fails without reassignment. + +Start the four databases and the existing Vault volume. Vault startup does not initialize or +unseal it. If sealed, run the explicit unseal ceremony with two separately protected custodian +key files supplied through an Ansible variable file, or with the existing local rehearsal shares +only when `local_rehearsal=true`. Never pass shares or tokens on the command line. + +```bash +docker compose up -d --wait postgres-identity postgres-wallet postgres-loan postgres-collateral +docker compose up -d vault +# custodian-input.yml contains only protected key-file paths; keep it outside source control. +docker compose run --rm --no-deps operations infrastructure/operations/vault-lifecycle.yml \ + -e operation=unseal -e @/protected/custodian-input.yml +docker compose run --rm --no-deps operations infrastructure/operations/site.yml +docker compose up -d agent-operator +``` + +The site playbook preserves existing mounts, audit configuration, rotated connection passwords +and OTP KV versions. It adds missing service-scoped roles/policies and Redis authentication. +Existing database connection settings must have verified TLS. If an adopted mount/connection or +OTP key is missing, reconciliation fails rather than silently replacing that state. Keep existing +historical SQL and Flyway checksums unchanged. An incompatible history needs a separate reviewed +forward migration; baseline/repair is deliberately not automatic. + +The replacement import retains the committed `identity_challenge`, `identity_audit_log`, and +`identity_challenge_scope` tables. No old application data is +converted or rewritten during adoption. + +## Fresh isolated environment + +Choose a unique Compose project, unique names for all seven volumes and a separate artifact +checkout/runtime directory. Copy the variable names from `.env.example` and assign test-specific +values. A project name alone does not change the explicit default volume names. + +```bash +.local/ansible/bin/ansible-playbook -i localhost, -c local infrastructure/operations/prepare.yml -e fresh_target=true +.local/ansible/bin/ansible-playbook -i localhost, -c local infrastructure/operations/allocate.yml -e fresh_target=true +docker compose up -d --wait postgres-identity postgres-wallet postgres-loan postgres-collateral +docker compose up -d vault +docker compose run --rm --no-deps operations infrastructure/operations/vault-lifecycle.yml \ + -e operation=initialize -e fresh_target=true -e local_rehearsal=true +docker compose run --rm --no-deps operations infrastructure/operations/site.yml -e fresh_target=true +docker compose up -d agent-operator +``` + +`prepare.yml` refuses an existing artifact root, including a partial earlier run. It generates +certificates, bootstrap passwords and protected files only. It produces no secret-bearing SQL +or Docker environment credentials. `allocate.yml` refuses if any requested volume already exists. +These are explicit fresh-only operations; normal reconciliation and migration are repeatable. + +The local ceremony stores threshold shares together for disposable fixtures only. Production +initialization instead requires `recovery_pgp_keys` (three custodian public keys) and +`root_token_pgp_key`, supplied through a protected variable file. Encrypted shares and the encrypted +initial root token are saved; custodians must unseal Vault separately, decrypt the root token into +`operator/bootstrap-root` (0600), then run `operation=recover-operator`. That playbook creates the +operator token and revokes the temporary root in an `always` block. The normal startup path never +runs these ceremonies. Use organization-issued certificates and separate recovery custody for +production; the preparation CA is a local rehearsal convenience. + +## Independent migrations + +Build the committed service modules with JDK 25, then migrate each database explicitly. +This branch retains the existing module names and application persistence implementations: + +```bash +./mvnw -B -ntp -pl payguard-identity-service,payguard-wallet-service,payguard-loan-service,payguard-collateral-service -am package +for service in identity wallet loan collateral; do + docker compose run --rm --no-deps operations infrastructure/operations/migrate.yml -e service="$service" +done +docker compose --profile applications up -d --build +``` + +Each attempt acquires a per-service lock and a unique private tmpfs handoff directory. An +`exit_after_auth` Vault Agent renders JSON-escaped TOML containing one service's expiring login. +The controller admits only a verified target, protected files, and at least 840 seconds of +remaining token/lease budget. Flyway has a 600-second external deadline plus a 10-second kill +grace, database statement/lock timeouts, and bounded schema-history lock retries. + +`afterConnect.sql` selects the stable owner on every connection. Flyway 13 removed `initSql`; +that retired setting is not used. `beforeMigrate` removes runtime table defaults before Flyway +creates its metadata table. Success/error callbacks grant application-table DML, keep history +private, preserve append-only audit restrictions and protect Identity scope deletion. Functions +have no public EXECUTE default. Migration logins have no inherited owner privileges; they must +explicitly select the owner role. Runtime credentials cannot select the owner role or run DDL. + +The controller verifies ownership/history privileges, stops only its own Agent/Flyway containers, +and revokes the attempt token and associated leases. It deletes credential files and releases +the service lock only after confirmed revocation. No raw Flyway output is sent to logs. Protected +receipts distinguish `cleanup_verified` and `cleanup_pending`. A nonzero result is a failure, +including a migration that committed but whose cleanup could not be proved. + +Wallet currently has domain classes without a persistence adapter. Its independent V1 migration +is an explicit `SELECT 1` marker; this does not invent a financial ledger schema or imply that +Wallet persistence is implemented. Identity, Loan and Collateral SQL files retain their exact +supplied bytes; `infrastructure/migrations.sha256` records them. + +## Production broker configuration + +The base Compose topology includes local Redis authentication and an internal plaintext Kafka +broker for rehearsal. Use the production overlay for authenticated TLS external Redis and Kafka; +it disables local brokers and removes application dependencies on them. Set the production-only +hostnames, ports and CA paths required by that overlay. It enables Redis certificate validation, +Kafka SASL_SSL/PLAIN, and Kafka hostname verification, with explicit CA files. + +Supply the external Redis password in Vault at `secret/local/infrastructure/redis` or in the +protected `redis/password` input before running `site.yml -e external_redis=true`. The password +must match the external server. `seed-kafka.yml -e kafka_credentials_file=/protected/kafka.json` +imports missing per-service KV secrets with CAS=0. The JSON has `identity`, `wallet`, `loan`, and +`collateral` keys, each containing a `jaas_config` string for its externally provisioned principal. +Each principal must have only its service's topic permissions. Existing KV versions are preserved; +rotate credentials through a coordinated native Vault/broker procedure. + +```bash +export COMPOSE_FILE=docker-compose.yaml:infrastructure/compose.operations.yaml:infrastructure/compose.production.yaml +docker compose --profile applications config --quiet +docker compose --profile applications up -d --build +``` + +The operations controller always uses the base database/Vault topology for migration. Do not +manually start migration services or reuse a previous attempt root. The production overlay is a +single-host deployment building block, not an HA database/Vault design. Deployment acceptance +still requires live integration, security, recovery and business end-to-end checks. See the +current validation report for executed checks and remaining acceptance work. + +## Identity import + +`import-identity.yml` replaces both the Python converter and Java import runner. Supply protected +`source_file`, `mapping_file`, and `approved_mapping_sha256` variables in a protected variable file. +The mapping JSON requires `approved_by`, `approval_reference`, `customers`, and `challenges`. +Already-correct prefixed IDs pass through; historical IDs require explicit mappings. The source +contains `challenges` and `audit` arrays. Audit records retain their `challenge_id` references. + +The native PostgreSQL transaction sets the stable owner, locks the destination, refuses populated +challenge/audit/scope tables, binds JSON values as parameters, checks integer/timestamp inputs, +inserts parent rows before audit rows, derives scope rows, checks audit subjects/counts, and rolls +back on any error. Timestamp offsets are required; sub-microsecond digits are truncated before PostgreSQL conversion, +preserving the retired converter's precision rule. An +import is never a normal startup or migration side effect. Preserve approved exports and mappings +in protected backup custody, never in source or CI artifacts. + +## Backups and recovery + +`backup.yml -e backup_directory=/protected/new-checkpoint` creates four native custom-format +`pg_dump` archives and a native Raft snapshot, all mode 0600, plus checksums and a manifest. Quiesce +writers first when cross-service consistency is required. Snapshot bytes are sensitive, including +Vault's stored manager credentials; encrypt and control backup custody. The manifest starts with +`restore_verified: false`; file creation is not a restore test. + +`restore-postgresql.yml` requires an empty schema, the backup's exact SHA-256, a selected service, +and an independently recorded `payguard-recovery-*` target. It restores transactionally under the +stable owner, removes temporary schema-creation permission in `always`, then verifies ownership +and history privacy. Never point this workflow at the existing environment. + +For Vault recovery, restore the native Raft snapshot using Vault's documented +`vault operator raft snapshot restore` ceremony on a separate recovery cluster, using source +recovery custody and an explicitly authorized recovery operator. Restoring a snapshot replaces +a cluster's state and changes which unseal keys/tokens apply; it is deliberately not an automatic +normal-startup action. Preserve a combined database/Vault checkpoint: restoring Vault alone can +leave its stored manager passwords inconsistent with the databases. Rehearse the complete pair +before cutover; no successful Raft restore is claimed in this delivery. + +For an interrupted migration, keep the receipt and run: + +```bash +docker compose run --rm --no-deps operations infrastructure/operations/recover-attempt.yml -e attempt_id=RECORDED_ATTEMPT +``` + +Recovery validates the exact attempt path/project, stops only recorded containers, revokes only +the private attempt token, and removes only that attempt after confirmed cleanup. Repeated +verified recovery is a no-op. Missing tokens, failed revocation, or lost receipts fail closed. +Inspect native Vault leases and database sessions using the protected receipt; never broadly +revoke service leases to guess which one belonged to the failed run. A crash before any token +sink is recorded, or before the receipt exists, may require operator reconciliation of the +retained lock. Never remove a lock until the related processes and leases are accounted for. + +Use `install-timer.yml` on the Linux host to install the user service and 20-minute identity +refresh timer. Supply its non-secret environment variables first. Start `agent-operator` after +the first identity refresh; it owns ongoing operator authentication and token renewal. Refresh +uses wrapped one-hour SecretIDs and atomic replacements, without revoking active application +tokens. Monitor timer failures: once bootstrap material expires and the Agent reaches its token +maximum lifetime, threshold recovery may be necessary. + +## Verification + +```bash +python3 -m pip install -r infrastructure/tests-requirements.txt +python3 -m unittest discover -s infrastructure/tests -v +# Use the operations container for networked native checks. +docker compose run --rm --no-deps operations infrastructure/operations/tests/integration.yml +docker compose run --rm --no-deps operations infrastructure/operations/tests/agent.yml +docker compose run --rm --no-deps operations infrastructure/operations/tests/applications.yml +``` + +`tests/recovery.yml` refuses disruption outside a recorded `payguard-it-*` fixture. The GitHub +workflow creates explicit isolated volumes, runs migrations twice, exercises privileges/TLS, +starts packaged applications, and performs the Vault stop/start/unseal scenario. It is an +executable validation workflow, not a record of passing execution. Additional mandatory acceptance +work is listed in the current validation report, including restore, migration interruption, +lease max-TTL replacement, confidentiality, and business end-to-end scenarios. diff --git a/docs/tasks/postgresql-vault-migration.md b/docs/tasks/postgresql-vault-migration.md index 168adf7..d8ac5fe 100644 --- a/docs/tasks/postgresql-vault-migration.md +++ b/docs/tasks/postgresql-vault-migration.md @@ -1,3 +1,5 @@ +> Historical record. This document is preserved as prior evidence; current implementation and test status are in `docs/infrastructure.md` and `docs/infrastructure-validation.md`. + # PostgreSQL and Vault implementation work log Status: historical migration implementation/rehearsal record. Subsequent committed Identity integration passed full verification with 91 tests and all quality gates; see [the publication repair log](identity-publication-repair.md) for current source, publication and CI evidence. The results below describe the earlier full working checkout, including its then-unresolved PMD findings. Live-data and missing financial-adapter work remain outside the proven scope. Current source is authoritative. @@ -33,10 +35,10 @@ Status: historical migration implementation/rehearsal record. Subsequent committ | Service | Actual owned source | Previous credentials | Result and evidence boundary | | --- | --- | --- | --- | -| Identity | `payguard-identity-service/src/main/java/dev/amg/payguard/identity/verification/infrastructure/persistence`; actual JPA challenges/audit, Oracle V1/V2 | datasource environment and known pepper fallback | PostgreSQL final V1/scope V2, real adapter transactions/races, Vault pool/keys/seeds; one database for four package modules | -| Loan | `payguard-loan-service/src/main/resources/db/legacy/oracle/V1__loan_schema.sql`; repository port, no concrete entity/adapter | datasource environment | PostgreSQL exact schema/Flyway/SQL fixtures and Vault pool; absent persisted business flows not invented | -| Collateral | `payguard-collateral-service/src/main/resources/db/legacy/oracle/V1__collateral_schema.sql`; repository port, no concrete entity/adapter | datasource environment | PostgreSQL exact valuation schema/Flyway/fixtures and Vault pool; absent persisted business flows not invented | -| Wallet | `payguard-wallet-service/pom.xml` OSS jOOQ dependency/domain/ports; no schema/adapter/codegen/generated source | datasource environment | PostgreSQL driver/POSTGRES dialect and Vault pool; no fabricated ledger schema or platform-wide financial concurrency claim | +| Identity | `../../payguard-identity-service`; actual JPA challenges/audit, Oracle V1/V2 | datasource environment and known pepper fallback | PostgreSQL final V1/scope V2, real adapter transactions/races, Vault pool/keys/seeds; one database for four package modules | +| Loan | `../../payguard-loan-service`; repository port, no concrete entity/adapter | datasource environment | PostgreSQL exact schema/Flyway/SQL fixtures and Vault pool; absent persisted business flows not invented | +| Collateral | `../../payguard-collateral-service`; repository port, no concrete entity/adapter | datasource environment | PostgreSQL exact valuation schema/Flyway/fixtures and Vault pool; absent persisted business flows not invented | +| Wallet | `../../payguard-wallet-service` OSS jOOQ dependency/domain/ports; no schema/adapter/codegen/generated source | datasource environment | PostgreSQL driver/POSTGRES dialect and Vault pool; no fabricated ledger schema or platform-wide financial concurrency claim | | Gateway / Proto | database-free modules | no database credentials | Remain database-free | ## Decisions and versions diff --git a/infrastructure/application.Dockerfile b/infrastructure/application.Dockerfile new file mode 100644 index 0000000..895b362 --- /dev/null +++ b/infrastructure/application.Dockerfile @@ -0,0 +1,9 @@ +# Build the reactor outside this image; credentials never enter the build context. +FROM hashicorp/vault:2.1.2@sha256:c2f666266f383d2cf424d86b8bb8ce7d065562173ffec2b476d762943608bb55 AS vault +FROM eclipse-temurin:25-jre-alpine@sha256:b7528962e65d43064d8a8b67ba175367896bc948363ea2d3fcd7f10d34e1f048 +COPY --from=vault /bin/vault /usr/local/bin/vault +ARG SERVICE +WORKDIR /app +COPY payguard-${SERVICE}-service/target/payguard-${SERVICE}-service-1.0.0-SNAPSHOT.jar /app/service.jar +USER 1000:1000 +ENTRYPOINT ["vault", "agent"] diff --git a/infrastructure/compose.operations.yaml b/infrastructure/compose.operations.yaml new file mode 100644 index 0000000..9e3871d --- /dev/null +++ b/infrastructure/compose.operations.yaml @@ -0,0 +1,519 @@ +services: + agent-identity: + read_only: true + cap_drop: &id001 + - ALL + security_opt: &id002 + - no-new-privileges:true + profiles: + - migration + image: hashicorp/vault:2.1.2@sha256:c2f666266f383d2cf424d86b8bb8ce7d065562173ffec2b476d762943608bb55 + user: ${PAYGUARD_LOCAL_UID:-1000}:${PAYGUARD_LOCAL_GID:-1000} + entrypoint: + - vault + - agent + - -config=/etc/vault/agent.hcl + - -log-level=error + restart: 'no' + logging: &id003 + driver: none + networks: &id004 + - operations + volumes: + - type: bind + source: ./infrastructure/vault/agent/identity.hcl + target: /etc/vault/agent.hcl + read_only: true + bind: + create_host_path: false + - type: bind + source: ./.local/payguard/ca.crt + target: /etc/vault/ca.crt + read_only: true + bind: + create_host_path: false + - type: bind + source: ./.local/payguard/operator/identity + target: /run/agent-auth + read_only: true + bind: + create_host_path: false + - type: bind + source: ${PAYGUARD_ATTEMPT_ROOT:-/run/payguard-unset}/identity/private + target: /run/agent-private + read_only: false + bind: + create_host_path: false + - type: bind + source: ${PAYGUARD_ATTEMPT_ROOT:-/run/payguard-unset}/identity/handoff + target: /run/agent-handoff + read_only: false + bind: + create_host_path: false + tmpfs: + - /tmp:rw,noexec,nosuid,nodev,uid=${PAYGUARD_LOCAL_UID:-1000},gid=${PAYGUARD_LOCAL_GID:-1000},mode=0700 + migrate-identity: + read_only: true + cap_drop: *id001 + security_opt: *id002 + profiles: + - migration + image: flyway/flyway:13.7.0-alpine@sha256:4b95b4ba46a948262ee3420a112d233992cdbd3f73bab37335d8f3cd9e5e369d + user: ${PAYGUARD_LOCAL_UID:-1000}:${PAYGUARD_LOCAL_GID:-1000} + restart: 'no' + logging: + driver: none + command: + - -configFiles=/flyway/conf/flyway.toml,/run/agent-handoff/credential.toml + - migrate + depends_on: + agent-identity: + condition: service_completed_successfully + postgres-identity: + condition: service_healthy + volumes: + - type: bind + source: ./infrastructure/flyway/flyway.toml + target: /flyway/conf/flyway.toml + read_only: true + bind: + create_host_path: false + - type: bind + source: ./payguard-identity-service/src/main/resources/db/migration/postgresql + target: /flyway/sql + read_only: true + bind: + create_host_path: false + - type: bind + source: ./infrastructure/flyway/callbacks/identity + target: /flyway/callbacks + read_only: true + bind: + create_host_path: false + - type: bind + source: ${PAYGUARD_ATTEMPT_ROOT:-/run/payguard-unset}/identity/handoff + target: /run/agent-handoff + read_only: true + bind: + create_host_path: false + - type: bind + source: ./.local/payguard/ca.crt + target: /run/agent-ca/ca.crt + read_only: true + bind: + create_host_path: false + networks: + - database + tmpfs: + - /tmp:rw,noexec,nosuid,nodev,uid=${PAYGUARD_LOCAL_UID:-1000},gid=${PAYGUARD_LOCAL_GID:-1000},mode=0700 + agent-wallet: + read_only: true + cap_drop: *id001 + security_opt: *id002 + profiles: + - migration + image: hashicorp/vault:2.1.2@sha256:c2f666266f383d2cf424d86b8bb8ce7d065562173ffec2b476d762943608bb55 + user: ${PAYGUARD_LOCAL_UID:-1000}:${PAYGUARD_LOCAL_GID:-1000} + entrypoint: + - vault + - agent + - -config=/etc/vault/agent.hcl + - -log-level=error + restart: 'no' + logging: + driver: none + networks: + - operations + volumes: + - type: bind + source: ./infrastructure/vault/agent/wallet.hcl + target: /etc/vault/agent.hcl + read_only: true + bind: + create_host_path: false + - type: bind + source: ./.local/payguard/ca.crt + target: /etc/vault/ca.crt + read_only: true + bind: + create_host_path: false + - type: bind + source: ./.local/payguard/operator/wallet + target: /run/agent-auth + read_only: true + bind: + create_host_path: false + - type: bind + source: ${PAYGUARD_ATTEMPT_ROOT:-/run/payguard-unset}/wallet/private + target: /run/agent-private + read_only: false + bind: + create_host_path: false + - type: bind + source: ${PAYGUARD_ATTEMPT_ROOT:-/run/payguard-unset}/wallet/handoff + target: /run/agent-handoff + read_only: false + bind: + create_host_path: false + tmpfs: + - /tmp:rw,noexec,nosuid,nodev,uid=${PAYGUARD_LOCAL_UID:-1000},gid=${PAYGUARD_LOCAL_GID:-1000},mode=0700 + migrate-wallet: + read_only: true + cap_drop: *id001 + security_opt: *id002 + profiles: + - migration + image: flyway/flyway:13.7.0-alpine@sha256:4b95b4ba46a948262ee3420a112d233992cdbd3f73bab37335d8f3cd9e5e369d + user: ${PAYGUARD_LOCAL_UID:-1000}:${PAYGUARD_LOCAL_GID:-1000} + restart: 'no' + logging: + driver: none + command: + - -configFiles=/flyway/conf/flyway.toml,/run/agent-handoff/credential.toml + - migrate + depends_on: + agent-wallet: + condition: service_completed_successfully + postgres-wallet: + condition: service_healthy + volumes: + - type: bind + source: ./infrastructure/flyway/flyway.toml + target: /flyway/conf/flyway.toml + read_only: true + bind: + create_host_path: false + - type: bind + source: ./payguard-wallet-service/src/main/resources/db/migration/postgresql + target: /flyway/sql + read_only: true + bind: + create_host_path: false + - type: bind + source: ./infrastructure/flyway/callbacks/wallet + target: /flyway/callbacks + read_only: true + bind: + create_host_path: false + - type: bind + source: ${PAYGUARD_ATTEMPT_ROOT:-/run/payguard-unset}/wallet/handoff + target: /run/agent-handoff + read_only: true + bind: + create_host_path: false + - type: bind + source: ./.local/payguard/ca.crt + target: /run/agent-ca/ca.crt + read_only: true + bind: + create_host_path: false + networks: + - database + tmpfs: + - /tmp:rw,noexec,nosuid,nodev,uid=${PAYGUARD_LOCAL_UID:-1000},gid=${PAYGUARD_LOCAL_GID:-1000},mode=0700 + agent-loan: + read_only: true + cap_drop: *id001 + security_opt: *id002 + profiles: + - migration + image: hashicorp/vault:2.1.2@sha256:c2f666266f383d2cf424d86b8bb8ce7d065562173ffec2b476d762943608bb55 + user: ${PAYGUARD_LOCAL_UID:-1000}:${PAYGUARD_LOCAL_GID:-1000} + entrypoint: + - vault + - agent + - -config=/etc/vault/agent.hcl + - -log-level=error + restart: 'no' + logging: + driver: none + networks: + - operations + volumes: + - type: bind + source: ./infrastructure/vault/agent/loan.hcl + target: /etc/vault/agent.hcl + read_only: true + bind: + create_host_path: false + - type: bind + source: ./.local/payguard/ca.crt + target: /etc/vault/ca.crt + read_only: true + bind: + create_host_path: false + - type: bind + source: ./.local/payguard/operator/loan + target: /run/agent-auth + read_only: true + bind: + create_host_path: false + - type: bind + source: ${PAYGUARD_ATTEMPT_ROOT:-/run/payguard-unset}/loan/private + target: /run/agent-private + read_only: false + bind: + create_host_path: false + - type: bind + source: ${PAYGUARD_ATTEMPT_ROOT:-/run/payguard-unset}/loan/handoff + target: /run/agent-handoff + read_only: false + bind: + create_host_path: false + tmpfs: + - /tmp:rw,noexec,nosuid,nodev,uid=${PAYGUARD_LOCAL_UID:-1000},gid=${PAYGUARD_LOCAL_GID:-1000},mode=0700 + migrate-loan: + read_only: true + cap_drop: *id001 + security_opt: *id002 + profiles: + - migration + image: flyway/flyway:13.7.0-alpine@sha256:4b95b4ba46a948262ee3420a112d233992cdbd3f73bab37335d8f3cd9e5e369d + user: ${PAYGUARD_LOCAL_UID:-1000}:${PAYGUARD_LOCAL_GID:-1000} + restart: 'no' + logging: + driver: none + command: + - -configFiles=/flyway/conf/flyway.toml,/run/agent-handoff/credential.toml + - migrate + depends_on: + agent-loan: + condition: service_completed_successfully + postgres-loan: + condition: service_healthy + volumes: + - type: bind + source: ./infrastructure/flyway/flyway.toml + target: /flyway/conf/flyway.toml + read_only: true + bind: + create_host_path: false + - type: bind + source: ./payguard-loan-service/src/main/resources/db/migration/postgresql + target: /flyway/sql + read_only: true + bind: + create_host_path: false + - type: bind + source: ./infrastructure/flyway/callbacks/loan + target: /flyway/callbacks + read_only: true + bind: + create_host_path: false + - type: bind + source: ${PAYGUARD_ATTEMPT_ROOT:-/run/payguard-unset}/loan/handoff + target: /run/agent-handoff + read_only: true + bind: + create_host_path: false + - type: bind + source: ./.local/payguard/ca.crt + target: /run/agent-ca/ca.crt + read_only: true + bind: + create_host_path: false + networks: + - database + tmpfs: + - /tmp:rw,noexec,nosuid,nodev,uid=${PAYGUARD_LOCAL_UID:-1000},gid=${PAYGUARD_LOCAL_GID:-1000},mode=0700 + agent-collateral: + read_only: true + cap_drop: *id001 + security_opt: *id002 + profiles: + - migration + image: hashicorp/vault:2.1.2@sha256:c2f666266f383d2cf424d86b8bb8ce7d065562173ffec2b476d762943608bb55 + user: ${PAYGUARD_LOCAL_UID:-1000}:${PAYGUARD_LOCAL_GID:-1000} + entrypoint: + - vault + - agent + - -config=/etc/vault/agent.hcl + - -log-level=error + restart: 'no' + logging: + driver: none + networks: + - operations + volumes: + - type: bind + source: ./infrastructure/vault/agent/collateral.hcl + target: /etc/vault/agent.hcl + read_only: true + bind: + create_host_path: false + - type: bind + source: ./.local/payguard/ca.crt + target: /etc/vault/ca.crt + read_only: true + bind: + create_host_path: false + - type: bind + source: ./.local/payguard/operator/collateral + target: /run/agent-auth + read_only: true + bind: + create_host_path: false + - type: bind + source: ${PAYGUARD_ATTEMPT_ROOT:-/run/payguard-unset}/collateral/private + target: /run/agent-private + read_only: false + bind: + create_host_path: false + - type: bind + source: ${PAYGUARD_ATTEMPT_ROOT:-/run/payguard-unset}/collateral/handoff + target: /run/agent-handoff + read_only: false + bind: + create_host_path: false + tmpfs: + - /tmp:rw,noexec,nosuid,nodev,uid=${PAYGUARD_LOCAL_UID:-1000},gid=${PAYGUARD_LOCAL_GID:-1000},mode=0700 + migrate-collateral: + read_only: true + cap_drop: *id001 + security_opt: *id002 + profiles: + - migration + image: flyway/flyway:13.7.0-alpine@sha256:4b95b4ba46a948262ee3420a112d233992cdbd3f73bab37335d8f3cd9e5e369d + user: ${PAYGUARD_LOCAL_UID:-1000}:${PAYGUARD_LOCAL_GID:-1000} + restart: 'no' + logging: + driver: none + command: + - -configFiles=/flyway/conf/flyway.toml,/run/agent-handoff/credential.toml + - migrate + depends_on: + agent-collateral: + condition: service_completed_successfully + postgres-collateral: + condition: service_healthy + volumes: + - type: bind + source: ./infrastructure/flyway/flyway.toml + target: /flyway/conf/flyway.toml + read_only: true + bind: + create_host_path: false + - type: bind + source: ./payguard-collateral-service/src/main/resources/db/migration/postgresql + target: /flyway/sql + read_only: true + bind: + create_host_path: false + - type: bind + source: ./infrastructure/flyway/callbacks/collateral + target: /flyway/callbacks + read_only: true + bind: + create_host_path: false + - type: bind + source: ${PAYGUARD_ATTEMPT_ROOT:-/run/payguard-unset}/collateral/handoff + target: /run/agent-handoff + read_only: true + bind: + create_host_path: false + - type: bind + source: ./.local/payguard/ca.crt + target: /run/agent-ca/ca.crt + read_only: true + bind: + create_host_path: false + networks: + - database + tmpfs: + - /tmp:rw,noexec,nosuid,nodev,uid=${PAYGUARD_LOCAL_UID:-1000},gid=${PAYGUARD_LOCAL_GID:-1000},mode=0700 + operations: + read_only: true + cap_drop: *id001 + security_opt: *id002 + profiles: + - operations + build: + context: ./infrastructure/operations + user: ${PAYGUARD_LOCAL_UID:-1000}:${PAYGUARD_LOCAL_GID:-1000} + group_add: + - ${PAYGUARD_DOCKER_GID:-0} + working_dir: ${PAYGUARD_REPO_ROOT:-/workspace/payguard} + entrypoint: + - ansible-playbook + - -i + - localhost, + - -c + - local + environment: + PAYGUARD_REPO_ROOT: ${PAYGUARD_REPO_ROOT:-/workspace/payguard} + PAYGUARD_RUNTIME_ROOT: ${PAYGUARD_RUNTIME_ROOT:-/dev/shm/payguard} + PAYGUARD_LOCAL_UID: ${PAYGUARD_LOCAL_UID:-1000} + PAYGUARD_LOCAL_GID: ${PAYGUARD_LOCAL_GID:-1000} + PAYGUARD_DOCKER_GID: ${PAYGUARD_DOCKER_GID:-0} + COMPOSE_PROJECT_NAME: ${COMPOSE_PROJECT_NAME:-payguard} + ANSIBLE_CONFIG: ${PAYGUARD_REPO_ROOT:-/workspace/payguard}/infrastructure/operations/ansible.cfg + ANSIBLE_LOCAL_TEMP: /tmp/ansible + ANSIBLE_REMOTE_TEMP: /tmp/ansible + ANSIBLE_HOME: /tmp/ansible-home + volumes: + - type: bind + source: /var/run/docker.sock + target: /var/run/docker.sock + read_only: false + bind: + create_host_path: false + - type: bind + source: . + target: ${PAYGUARD_REPO_ROOT:-/workspace/payguard} + read_only: false + bind: + create_host_path: false + - type: bind + source: ${PAYGUARD_RUNTIME_ROOT:-/dev/shm/payguard} + target: ${PAYGUARD_RUNTIME_ROOT:-/dev/shm/payguard} + read_only: false + bind: + create_host_path: false + tmpfs: + - /tmp:rw,noexec,nosuid,nodev,uid=${PAYGUARD_LOCAL_UID:-1000},gid=${PAYGUARD_LOCAL_GID:-1000},mode=0700 + networks: + - database + - operations + agent-operator: + read_only: true + cap_drop: *id001 + security_opt: *id002 + image: hashicorp/vault:2.1.2@sha256:c2f666266f383d2cf424d86b8bb8ce7d065562173ffec2b476d762943608bb55 + user: ${PAYGUARD_LOCAL_UID:-1000}:${PAYGUARD_LOCAL_GID:-1000} + logging: *id003 + networks: *id004 + tmpfs: + - /tmp:rw,noexec,nosuid,nodev,uid=${PAYGUARD_LOCAL_UID:-1000},gid=${PAYGUARD_LOCAL_GID:-1000},mode=0700 + profiles: + - operations + restart: unless-stopped + entrypoint: + - vault + - agent + - -config=/etc/vault/agent.hcl + - -log-level=error + volumes: + - type: bind + source: ./infrastructure/vault/agent/operator.hcl + target: /etc/vault/agent.hcl + read_only: true + bind: + create_host_path: false + - type: bind + source: ./.local/payguard/ca.crt + target: /etc/vault/ca.crt + read_only: true + bind: + create_host_path: false + - type: bind + source: ./.local/payguard/operator/approle + target: /run/operator-auth + read_only: true + bind: + create_host_path: false + - type: bind + source: ./.local/payguard/operator + target: /run/operator + read_only: false + bind: + create_host_path: false diff --git a/infrastructure/compose.production.yaml b/infrastructure/compose.production.yaml new file mode 100644 index 0000000..6f918b0 --- /dev/null +++ b/infrastructure/compose.production.yaml @@ -0,0 +1,142 @@ +# Requires Docker Compose >= 2.24.4 (!override). External brokers must enforce TLS and authentication. +services: + kafka: + profiles: [local-brokers] + kafka-init: + profiles: [local-brokers] + redis: + profiles: [local-brokers] + identity-service: + depends_on: !override + postgres-identity: {condition: service_healthy} + vault: {condition: service_healthy} + networks: [database, operations, egress] + environment: + SPRING_DATA_REDIS_HOST: ${PAYGUARD_REDIS_HOST:?set authenticated TLS Redis hostname} + SPRING_DATA_REDIS_PORT: ${PAYGUARD_REDIS_PORT:-6380} + SPRING_DATA_REDIS_SSL_ENABLED: "true" + SPRING_DATA_REDIS_SSL_BUNDLE: redis + SPRING_SSL_BUNDLE_PEM_REDIS_TRUSTSTORE_CERTIFICATE: file:/etc/payguard/redis-ca.pem + KAFKA_BOOTSTRAP_SERVERS: ${PAYGUARD_KAFKA_BOOTSTRAP_SERVERS:?set TLS Kafka bootstrap servers} + SPRING_KAFKA_PROPERTIES_SECURITY_PROTOCOL: SASL_SSL + SPRING_KAFKA_PROPERTIES_SASL_MECHANISM: PLAIN + SPRING_KAFKA_PROPERTIES_SSL_TRUSTSTORE_TYPE: PEM + SPRING_KAFKA_PROPERTIES_SSL_TRUSTSTORE_LOCATION: /etc/payguard/kafka-ca.pem + SPRING_KAFKA_PROPERTIES_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: https + volumes: + - type: bind + source: ./infrastructure/vault/production/identity.hcl + target: /etc/vault/runtime.hcl + read_only: true + bind: {create_host_path: false} + - type: bind + source: ${PAYGUARD_REDIS_CA_FILE:?set Redis CA file} + target: /etc/payguard/redis-ca.pem + read_only: true + bind: {create_host_path: false} + - type: bind + source: ${PAYGUARD_KAFKA_CA_FILE:?set Kafka CA file} + target: /etc/payguard/kafka-ca.pem + read_only: true + bind: {create_host_path: false} + wallet-service: + depends_on: !override + postgres-wallet: {condition: service_healthy} + vault: {condition: service_healthy} + networks: [database, operations, egress] + environment: + SPRING_DATA_REDIS_HOST: ${PAYGUARD_REDIS_HOST:?set authenticated TLS Redis hostname} + SPRING_DATA_REDIS_PORT: ${PAYGUARD_REDIS_PORT:-6380} + SPRING_DATA_REDIS_SSL_ENABLED: "true" + SPRING_DATA_REDIS_SSL_BUNDLE: redis + SPRING_SSL_BUNDLE_PEM_REDIS_TRUSTSTORE_CERTIFICATE: file:/etc/payguard/redis-ca.pem + KAFKA_BOOTSTRAP_SERVERS: ${PAYGUARD_KAFKA_BOOTSTRAP_SERVERS:?set TLS Kafka bootstrap servers} + SPRING_KAFKA_PROPERTIES_SECURITY_PROTOCOL: SASL_SSL + SPRING_KAFKA_PROPERTIES_SASL_MECHANISM: PLAIN + SPRING_KAFKA_PROPERTIES_SSL_TRUSTSTORE_TYPE: PEM + SPRING_KAFKA_PROPERTIES_SSL_TRUSTSTORE_LOCATION: /etc/payguard/kafka-ca.pem + SPRING_KAFKA_PROPERTIES_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: https + volumes: + - type: bind + source: ./infrastructure/vault/production/wallet.hcl + target: /etc/vault/runtime.hcl + read_only: true + bind: {create_host_path: false} + - type: bind + source: ${PAYGUARD_REDIS_CA_FILE:?set Redis CA file} + target: /etc/payguard/redis-ca.pem + read_only: true + bind: {create_host_path: false} + - type: bind + source: ${PAYGUARD_KAFKA_CA_FILE:?set Kafka CA file} + target: /etc/payguard/kafka-ca.pem + read_only: true + bind: {create_host_path: false} + loan-service: + depends_on: !override + postgres-loan: {condition: service_healthy} + vault: {condition: service_healthy} + networks: [database, operations, egress] + environment: + SPRING_DATA_REDIS_HOST: ${PAYGUARD_REDIS_HOST:?set authenticated TLS Redis hostname} + SPRING_DATA_REDIS_PORT: ${PAYGUARD_REDIS_PORT:-6380} + SPRING_DATA_REDIS_SSL_ENABLED: "true" + SPRING_DATA_REDIS_SSL_BUNDLE: redis + SPRING_SSL_BUNDLE_PEM_REDIS_TRUSTSTORE_CERTIFICATE: file:/etc/payguard/redis-ca.pem + KAFKA_BOOTSTRAP_SERVERS: ${PAYGUARD_KAFKA_BOOTSTRAP_SERVERS:?set TLS Kafka bootstrap servers} + SPRING_KAFKA_PROPERTIES_SECURITY_PROTOCOL: SASL_SSL + SPRING_KAFKA_PROPERTIES_SASL_MECHANISM: PLAIN + SPRING_KAFKA_PROPERTIES_SSL_TRUSTSTORE_TYPE: PEM + SPRING_KAFKA_PROPERTIES_SSL_TRUSTSTORE_LOCATION: /etc/payguard/kafka-ca.pem + SPRING_KAFKA_PROPERTIES_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: https + volumes: + - type: bind + source: ./infrastructure/vault/production/loan.hcl + target: /etc/vault/runtime.hcl + read_only: true + bind: {create_host_path: false} + - type: bind + source: ${PAYGUARD_REDIS_CA_FILE:?set Redis CA file} + target: /etc/payguard/redis-ca.pem + read_only: true + bind: {create_host_path: false} + - type: bind + source: ${PAYGUARD_KAFKA_CA_FILE:?set Kafka CA file} + target: /etc/payguard/kafka-ca.pem + read_only: true + bind: {create_host_path: false} + collateral-service: + depends_on: !override + postgres-collateral: {condition: service_healthy} + vault: {condition: service_healthy} + networks: [database, operations, egress] + environment: + SPRING_DATA_REDIS_HOST: ${PAYGUARD_REDIS_HOST:?set authenticated TLS Redis hostname} + SPRING_DATA_REDIS_PORT: ${PAYGUARD_REDIS_PORT:-6380} + SPRING_DATA_REDIS_SSL_ENABLED: "true" + SPRING_DATA_REDIS_SSL_BUNDLE: redis + SPRING_SSL_BUNDLE_PEM_REDIS_TRUSTSTORE_CERTIFICATE: file:/etc/payguard/redis-ca.pem + KAFKA_BOOTSTRAP_SERVERS: ${PAYGUARD_KAFKA_BOOTSTRAP_SERVERS:?set TLS Kafka bootstrap servers} + SPRING_KAFKA_PROPERTIES_SECURITY_PROTOCOL: SASL_SSL + SPRING_KAFKA_PROPERTIES_SASL_MECHANISM: PLAIN + SPRING_KAFKA_PROPERTIES_SSL_TRUSTSTORE_TYPE: PEM + SPRING_KAFKA_PROPERTIES_SSL_TRUSTSTORE_LOCATION: /etc/payguard/kafka-ca.pem + SPRING_KAFKA_PROPERTIES_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM: https + volumes: + - type: bind + source: ./infrastructure/vault/production/collateral.hcl + target: /etc/vault/runtime.hcl + read_only: true + bind: {create_host_path: false} + - type: bind + source: ${PAYGUARD_REDIS_CA_FILE:?set Redis CA file} + target: /etc/payguard/redis-ca.pem + read_only: true + bind: {create_host_path: false} + - type: bind + source: ${PAYGUARD_KAFKA_CA_FILE:?set Kafka CA file} + target: /etc/payguard/kafka-ca.pem + read_only: true + bind: {create_host_path: false} +networks: + egress: {} diff --git a/infrastructure/flyway/callbacks/collateral/afterConnect.sql b/infrastructure/flyway/callbacks/collateral/afterConnect.sql new file mode 100644 index 0000000..3151657 --- /dev/null +++ b/infrastructure/flyway/callbacks/collateral/afterConnect.sql @@ -0,0 +1,5 @@ +-- Flyway 13 uses afterConnect in place of the removed initSql setting. +SET ROLE collateral_owner; +SET search_path TO payguard; +SET statement_timeout = '8min'; +SET lock_timeout = '30s'; diff --git a/infrastructure/flyway/callbacks/collateral/afterMigrate.sql b/infrastructure/flyway/callbacks/collateral/afterMigrate.sql new file mode 100644 index 0000000..5e23314 --- /dev/null +++ b/infrastructure/flyway/callbacks/collateral/afterMigrate.sql @@ -0,0 +1,29 @@ +-- The history table is private even between individual migration commits. +ALTER DEFAULT PRIVILEGES FOR ROLE collateral_owner IN SCHEMA payguard + REVOKE SELECT, INSERT, UPDATE, DELETE ON TABLES FROM collateral_runtime; +DO $guard$ +BEGIN + IF to_regclass('payguard.flyway_schema_history') IS NOT NULL THEN + REVOKE ALL ON payguard.flyway_schema_history FROM collateral_runtime; + END IF; +END +$guard$; +-- Grant only application tables after history hardening; never the metadata table. +DO $grant$ +DECLARE relation record; +BEGIN + FOR relation IN SELECT tablename FROM pg_tables + WHERE schemaname = 'payguard' AND tablename <> 'flyway_schema_history' + LOOP + EXECUTE format('GRANT SELECT, INSERT, UPDATE, DELETE ON TABLE payguard.%I TO collateral_runtime', relation.tablename); + END LOOP; +END +$grant$; +GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA payguard TO collateral_runtime; +DO $audit$ +BEGIN + IF to_regclass('payguard.collateral_audit_log') IS NOT NULL THEN + REVOKE UPDATE, DELETE ON payguard.collateral_audit_log FROM collateral_runtime; + END IF; +END +$audit$; diff --git a/infrastructure/flyway/callbacks/collateral/afterMigrateError.sql b/infrastructure/flyway/callbacks/collateral/afterMigrateError.sql new file mode 100644 index 0000000..5e23314 --- /dev/null +++ b/infrastructure/flyway/callbacks/collateral/afterMigrateError.sql @@ -0,0 +1,29 @@ +-- The history table is private even between individual migration commits. +ALTER DEFAULT PRIVILEGES FOR ROLE collateral_owner IN SCHEMA payguard + REVOKE SELECT, INSERT, UPDATE, DELETE ON TABLES FROM collateral_runtime; +DO $guard$ +BEGIN + IF to_regclass('payguard.flyway_schema_history') IS NOT NULL THEN + REVOKE ALL ON payguard.flyway_schema_history FROM collateral_runtime; + END IF; +END +$guard$; +-- Grant only application tables after history hardening; never the metadata table. +DO $grant$ +DECLARE relation record; +BEGIN + FOR relation IN SELECT tablename FROM pg_tables + WHERE schemaname = 'payguard' AND tablename <> 'flyway_schema_history' + LOOP + EXECUTE format('GRANT SELECT, INSERT, UPDATE, DELETE ON TABLE payguard.%I TO collateral_runtime', relation.tablename); + END LOOP; +END +$grant$; +GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA payguard TO collateral_runtime; +DO $audit$ +BEGIN + IF to_regclass('payguard.collateral_audit_log') IS NOT NULL THEN + REVOKE UPDATE, DELETE ON payguard.collateral_audit_log FROM collateral_runtime; + END IF; +END +$audit$; diff --git a/infrastructure/flyway/callbacks/collateral/beforeMigrate.sql b/infrastructure/flyway/callbacks/collateral/beforeMigrate.sql new file mode 100644 index 0000000..3d906f8 --- /dev/null +++ b/infrastructure/flyway/callbacks/collateral/beforeMigrate.sql @@ -0,0 +1,10 @@ +-- The history table is private even between individual migration commits. +ALTER DEFAULT PRIVILEGES FOR ROLE collateral_owner IN SCHEMA payguard + REVOKE SELECT, INSERT, UPDATE, DELETE ON TABLES FROM collateral_runtime; +DO $guard$ +BEGIN + IF to_regclass('payguard.flyway_schema_history') IS NOT NULL THEN + REVOKE ALL ON payguard.flyway_schema_history FROM collateral_runtime; + END IF; +END +$guard$; diff --git a/infrastructure/flyway/callbacks/identity/afterConnect.sql b/infrastructure/flyway/callbacks/identity/afterConnect.sql new file mode 100644 index 0000000..ac4aed6 --- /dev/null +++ b/infrastructure/flyway/callbacks/identity/afterConnect.sql @@ -0,0 +1,5 @@ +-- Flyway 13 uses afterConnect in place of the removed initSql setting. +SET ROLE identity_owner; +SET search_path TO payguard; +SET statement_timeout = '8min'; +SET lock_timeout = '30s'; diff --git a/infrastructure/flyway/callbacks/identity/afterMigrate.sql b/infrastructure/flyway/callbacks/identity/afterMigrate.sql new file mode 100644 index 0000000..2b5097a --- /dev/null +++ b/infrastructure/flyway/callbacks/identity/afterMigrate.sql @@ -0,0 +1,36 @@ +-- The history table is private even between individual migration commits. +ALTER DEFAULT PRIVILEGES FOR ROLE identity_owner IN SCHEMA payguard + REVOKE SELECT, INSERT, UPDATE, DELETE ON TABLES FROM identity_runtime; +DO $guard$ +BEGIN + IF to_regclass('payguard.flyway_schema_history') IS NOT NULL THEN + REVOKE ALL ON payguard.flyway_schema_history FROM identity_runtime; + END IF; +END +$guard$; +-- Grant only application tables after history hardening; never the metadata table. +DO $grant$ +DECLARE relation record; +BEGIN + FOR relation IN SELECT tablename FROM pg_tables + WHERE schemaname = 'payguard' AND tablename <> 'flyway_schema_history' + LOOP + EXECUTE format('GRANT SELECT, INSERT, UPDATE, DELETE ON TABLE payguard.%I TO identity_runtime', relation.tablename); + END LOOP; +END +$grant$; +GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA payguard TO identity_runtime; +DO $audit$ +BEGIN + IF to_regclass('payguard.identity_audit_log') IS NOT NULL THEN + REVOKE UPDATE, DELETE ON payguard.identity_audit_log FROM identity_runtime; + END IF; +END +$audit$; +DO $scope$ +BEGIN + IF to_regclass('payguard.identity_challenge_scope') IS NOT NULL THEN + REVOKE DELETE ON payguard.identity_challenge_scope FROM identity_runtime; + END IF; +END +$scope$; diff --git a/infrastructure/flyway/callbacks/identity/afterMigrateError.sql b/infrastructure/flyway/callbacks/identity/afterMigrateError.sql new file mode 100644 index 0000000..2b5097a --- /dev/null +++ b/infrastructure/flyway/callbacks/identity/afterMigrateError.sql @@ -0,0 +1,36 @@ +-- The history table is private even between individual migration commits. +ALTER DEFAULT PRIVILEGES FOR ROLE identity_owner IN SCHEMA payguard + REVOKE SELECT, INSERT, UPDATE, DELETE ON TABLES FROM identity_runtime; +DO $guard$ +BEGIN + IF to_regclass('payguard.flyway_schema_history') IS NOT NULL THEN + REVOKE ALL ON payguard.flyway_schema_history FROM identity_runtime; + END IF; +END +$guard$; +-- Grant only application tables after history hardening; never the metadata table. +DO $grant$ +DECLARE relation record; +BEGIN + FOR relation IN SELECT tablename FROM pg_tables + WHERE schemaname = 'payguard' AND tablename <> 'flyway_schema_history' + LOOP + EXECUTE format('GRANT SELECT, INSERT, UPDATE, DELETE ON TABLE payguard.%I TO identity_runtime', relation.tablename); + END LOOP; +END +$grant$; +GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA payguard TO identity_runtime; +DO $audit$ +BEGIN + IF to_regclass('payguard.identity_audit_log') IS NOT NULL THEN + REVOKE UPDATE, DELETE ON payguard.identity_audit_log FROM identity_runtime; + END IF; +END +$audit$; +DO $scope$ +BEGIN + IF to_regclass('payguard.identity_challenge_scope') IS NOT NULL THEN + REVOKE DELETE ON payguard.identity_challenge_scope FROM identity_runtime; + END IF; +END +$scope$; diff --git a/infrastructure/flyway/callbacks/identity/beforeMigrate.sql b/infrastructure/flyway/callbacks/identity/beforeMigrate.sql new file mode 100644 index 0000000..c5c2b18 --- /dev/null +++ b/infrastructure/flyway/callbacks/identity/beforeMigrate.sql @@ -0,0 +1,10 @@ +-- The history table is private even between individual migration commits. +ALTER DEFAULT PRIVILEGES FOR ROLE identity_owner IN SCHEMA payguard + REVOKE SELECT, INSERT, UPDATE, DELETE ON TABLES FROM identity_runtime; +DO $guard$ +BEGIN + IF to_regclass('payguard.flyway_schema_history') IS NOT NULL THEN + REVOKE ALL ON payguard.flyway_schema_history FROM identity_runtime; + END IF; +END +$guard$; diff --git a/infrastructure/flyway/callbacks/loan/afterConnect.sql b/infrastructure/flyway/callbacks/loan/afterConnect.sql new file mode 100644 index 0000000..5aeb260 --- /dev/null +++ b/infrastructure/flyway/callbacks/loan/afterConnect.sql @@ -0,0 +1,5 @@ +-- Flyway 13 uses afterConnect in place of the removed initSql setting. +SET ROLE loan_owner; +SET search_path TO payguard; +SET statement_timeout = '8min'; +SET lock_timeout = '30s'; diff --git a/infrastructure/flyway/callbacks/loan/afterMigrate.sql b/infrastructure/flyway/callbacks/loan/afterMigrate.sql new file mode 100644 index 0000000..2ce85c3 --- /dev/null +++ b/infrastructure/flyway/callbacks/loan/afterMigrate.sql @@ -0,0 +1,29 @@ +-- The history table is private even between individual migration commits. +ALTER DEFAULT PRIVILEGES FOR ROLE loan_owner IN SCHEMA payguard + REVOKE SELECT, INSERT, UPDATE, DELETE ON TABLES FROM loan_runtime; +DO $guard$ +BEGIN + IF to_regclass('payguard.flyway_schema_history') IS NOT NULL THEN + REVOKE ALL ON payguard.flyway_schema_history FROM loan_runtime; + END IF; +END +$guard$; +-- Grant only application tables after history hardening; never the metadata table. +DO $grant$ +DECLARE relation record; +BEGIN + FOR relation IN SELECT tablename FROM pg_tables + WHERE schemaname = 'payguard' AND tablename <> 'flyway_schema_history' + LOOP + EXECUTE format('GRANT SELECT, INSERT, UPDATE, DELETE ON TABLE payguard.%I TO loan_runtime', relation.tablename); + END LOOP; +END +$grant$; +GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA payguard TO loan_runtime; +DO $audit$ +BEGIN + IF to_regclass('payguard.loan_audit_log') IS NOT NULL THEN + REVOKE UPDATE, DELETE ON payguard.loan_audit_log FROM loan_runtime; + END IF; +END +$audit$; diff --git a/infrastructure/flyway/callbacks/loan/afterMigrateError.sql b/infrastructure/flyway/callbacks/loan/afterMigrateError.sql new file mode 100644 index 0000000..2ce85c3 --- /dev/null +++ b/infrastructure/flyway/callbacks/loan/afterMigrateError.sql @@ -0,0 +1,29 @@ +-- The history table is private even between individual migration commits. +ALTER DEFAULT PRIVILEGES FOR ROLE loan_owner IN SCHEMA payguard + REVOKE SELECT, INSERT, UPDATE, DELETE ON TABLES FROM loan_runtime; +DO $guard$ +BEGIN + IF to_regclass('payguard.flyway_schema_history') IS NOT NULL THEN + REVOKE ALL ON payguard.flyway_schema_history FROM loan_runtime; + END IF; +END +$guard$; +-- Grant only application tables after history hardening; never the metadata table. +DO $grant$ +DECLARE relation record; +BEGIN + FOR relation IN SELECT tablename FROM pg_tables + WHERE schemaname = 'payguard' AND tablename <> 'flyway_schema_history' + LOOP + EXECUTE format('GRANT SELECT, INSERT, UPDATE, DELETE ON TABLE payguard.%I TO loan_runtime', relation.tablename); + END LOOP; +END +$grant$; +GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA payguard TO loan_runtime; +DO $audit$ +BEGIN + IF to_regclass('payguard.loan_audit_log') IS NOT NULL THEN + REVOKE UPDATE, DELETE ON payguard.loan_audit_log FROM loan_runtime; + END IF; +END +$audit$; diff --git a/infrastructure/flyway/callbacks/loan/beforeMigrate.sql b/infrastructure/flyway/callbacks/loan/beforeMigrate.sql new file mode 100644 index 0000000..4253679 --- /dev/null +++ b/infrastructure/flyway/callbacks/loan/beforeMigrate.sql @@ -0,0 +1,10 @@ +-- The history table is private even between individual migration commits. +ALTER DEFAULT PRIVILEGES FOR ROLE loan_owner IN SCHEMA payguard + REVOKE SELECT, INSERT, UPDATE, DELETE ON TABLES FROM loan_runtime; +DO $guard$ +BEGIN + IF to_regclass('payguard.flyway_schema_history') IS NOT NULL THEN + REVOKE ALL ON payguard.flyway_schema_history FROM loan_runtime; + END IF; +END +$guard$; diff --git a/infrastructure/flyway/callbacks/wallet/afterConnect.sql b/infrastructure/flyway/callbacks/wallet/afterConnect.sql new file mode 100644 index 0000000..c3edb34 --- /dev/null +++ b/infrastructure/flyway/callbacks/wallet/afterConnect.sql @@ -0,0 +1,5 @@ +-- Flyway 13 uses afterConnect in place of the removed initSql setting. +SET ROLE wallet_owner; +SET search_path TO payguard; +SET statement_timeout = '8min'; +SET lock_timeout = '30s'; diff --git a/infrastructure/flyway/callbacks/wallet/afterMigrate.sql b/infrastructure/flyway/callbacks/wallet/afterMigrate.sql new file mode 100644 index 0000000..ab90770 --- /dev/null +++ b/infrastructure/flyway/callbacks/wallet/afterMigrate.sql @@ -0,0 +1,22 @@ +-- The history table is private even between individual migration commits. +ALTER DEFAULT PRIVILEGES FOR ROLE wallet_owner IN SCHEMA payguard + REVOKE SELECT, INSERT, UPDATE, DELETE ON TABLES FROM wallet_runtime; +DO $guard$ +BEGIN + IF to_regclass('payguard.flyway_schema_history') IS NOT NULL THEN + REVOKE ALL ON payguard.flyway_schema_history FROM wallet_runtime; + END IF; +END +$guard$; +-- Grant only application tables after history hardening; never the metadata table. +DO $grant$ +DECLARE relation record; +BEGIN + FOR relation IN SELECT tablename FROM pg_tables + WHERE schemaname = 'payguard' AND tablename <> 'flyway_schema_history' + LOOP + EXECUTE format('GRANT SELECT, INSERT, UPDATE, DELETE ON TABLE payguard.%I TO wallet_runtime', relation.tablename); + END LOOP; +END +$grant$; +GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA payguard TO wallet_runtime; diff --git a/infrastructure/flyway/callbacks/wallet/afterMigrateError.sql b/infrastructure/flyway/callbacks/wallet/afterMigrateError.sql new file mode 100644 index 0000000..ab90770 --- /dev/null +++ b/infrastructure/flyway/callbacks/wallet/afterMigrateError.sql @@ -0,0 +1,22 @@ +-- The history table is private even between individual migration commits. +ALTER DEFAULT PRIVILEGES FOR ROLE wallet_owner IN SCHEMA payguard + REVOKE SELECT, INSERT, UPDATE, DELETE ON TABLES FROM wallet_runtime; +DO $guard$ +BEGIN + IF to_regclass('payguard.flyway_schema_history') IS NOT NULL THEN + REVOKE ALL ON payguard.flyway_schema_history FROM wallet_runtime; + END IF; +END +$guard$; +-- Grant only application tables after history hardening; never the metadata table. +DO $grant$ +DECLARE relation record; +BEGIN + FOR relation IN SELECT tablename FROM pg_tables + WHERE schemaname = 'payguard' AND tablename <> 'flyway_schema_history' + LOOP + EXECUTE format('GRANT SELECT, INSERT, UPDATE, DELETE ON TABLE payguard.%I TO wallet_runtime', relation.tablename); + END LOOP; +END +$grant$; +GRANT USAGE, SELECT ON ALL SEQUENCES IN SCHEMA payguard TO wallet_runtime; diff --git a/infrastructure/flyway/callbacks/wallet/beforeMigrate.sql b/infrastructure/flyway/callbacks/wallet/beforeMigrate.sql new file mode 100644 index 0000000..f9a1450 --- /dev/null +++ b/infrastructure/flyway/callbacks/wallet/beforeMigrate.sql @@ -0,0 +1,10 @@ +-- The history table is private even between individual migration commits. +ALTER DEFAULT PRIVILEGES FOR ROLE wallet_owner IN SCHEMA payguard + REVOKE SELECT, INSERT, UPDATE, DELETE ON TABLES FROM wallet_runtime; +DO $guard$ +BEGIN + IF to_regclass('payguard.flyway_schema_history') IS NOT NULL THEN + REVOKE ALL ON payguard.flyway_schema_history FROM wallet_runtime; + END IF; +END +$guard$; diff --git a/infrastructure/flyway/flyway.toml b/infrastructure/flyway/flyway.toml new file mode 100644 index 0000000..af46b67 --- /dev/null +++ b/infrastructure/flyway/flyway.toml @@ -0,0 +1,14 @@ +[flyway] +locations = ["filesystem:/flyway/sql", "filesystem:/flyway/callbacks"] +defaultSchema = "payguard" +schemas = ["payguard"] +createSchemas = false +cleanDisabled = true +baselineOnMigrate = false +outOfOrder = false +validateOnMigrate = true +validateMigrationNaming = true +connectRetries = 3 +lockRetryCount = 10 +executeInTransaction = true +# Every connection selects the stable owner through afterConnect.sql. diff --git a/infrastructure/migrations.sha256 b/infrastructure/migrations.sha256 new file mode 100644 index 0000000..7871f0a --- /dev/null +++ b/infrastructure/migrations.sha256 @@ -0,0 +1,4 @@ +24bd7ec38e26ccece80748bea0b416404280616d9ed8314b8482b09001ccab37 payguard-identity-service/src/main/resources/db/migration/postgresql/V1__identity_schema.sql +80029ae2b4d46a8318e897600aae44b126640c0eb6ac7d41649a80ca965366a5 payguard-identity-service/src/main/resources/db/migration/postgresql/V2__serialize_active_challenges.sql +fd430eae5728d056e9cb023a56d7d10f82b7e3ef86b41e3da3191fa6b4ccd4dc payguard-loan-service/src/main/resources/db/migration/postgresql/V1__loan_schema.sql +c3c87edc1dfa52c1f96827a402009c2d6fccf33ca8e2ee4b7ee1ff3821d25471 payguard-collateral-service/src/main/resources/db/migration/postgresql/V1__collateral_schema.sql diff --git a/infrastructure/operations/Dockerfile b/infrastructure/operations/Dockerfile new file mode 100644 index 0000000..97bc90f --- /dev/null +++ b/infrastructure/operations/Dockerfile @@ -0,0 +1,12 @@ +FROM docker:29.8.0-cli@sha256:eccaacfeed644c7de222ff047483568cb988dde95476fbaaf10ea2d04921bb66 AS docker +FROM python:3.12.11-alpine3.22@sha256:efcdfa6a6b2fd2afb9c7dfa9a5b288a6f68338b5cfdebe6b637d986067d85757 +COPY --from=docker /usr/local/bin/docker /usr/local/bin/docker +COPY --from=docker /usr/local/libexec/docker/cli-plugins /usr/local/libexec/docker/cli-plugins +RUN apk add --no-cache libpq openssl coreutils util-linux curl && \ + python3 -m venv /opt/ansible +COPY requirements.txt requirements.yml /ops/ +RUN /opt/ansible/bin/pip install --no-cache-dir -r /ops/requirements.txt && \ + /opt/ansible/bin/ansible-galaxy collection install -r /ops/requirements.yml -p /usr/share/ansible/collections +ENV PATH="/opt/ansible/bin:$PATH" ANSIBLE_NOCOLOR=1 ANSIBLE_DISPLAY_ARGS_TO_STDOUT=False ANSIBLE_DIFF_ALWAYS=False +WORKDIR /ops +ENTRYPOINT ["ansible-playbook", "-i", "localhost,", "-c", "local"] diff --git a/infrastructure/operations/adopt-artifacts.yml b/infrastructure/operations/adopt-artifacts.yml new file mode 100644 index 0000000..49302d9 --- /dev/null +++ b/infrastructure/operations/adopt-artifacts.yml @@ -0,0 +1,70 @@ +--- +- name: Reconcile newly required protected artifacts without replacing existing credentials + hosts: localhost + gather_facts: false + vars: + repo: "{{ lookup('env','PAYGUARD_REPO_ROOT') }}" + local: "{{ repo }}/.local/payguard" + tasks: + - name: Validate recorded target + ansible.builtin.import_tasks: tasks/target.yml + - name: Create only required protected directories + ansible.builtin.file: + path: "{{ local }}/{{ item }}" + state: directory + mode: '0700' + loop: [operator/wallet, operator/receipts, operator/approle, apps/identity, apps/wallet, apps/loan, apps/collateral, redis] + - name: Read canonical Redis authentication without changing its KV version + ansible.builtin.uri: + url: https://vault:8200/v1/secret/data/local/infrastructure/redis + headers: + X-Vault-Token: "{{ lookup('file',local ~ '/operator/token') | trim }}" + ca_path: "{{ local }}/ca.crt" + validate_certs: true + status_code: [200, 404] + timeout: 5 + register: existing_redis + no_log: true + - name: Check the protected Redis input file + ansible.builtin.stat: + path: "{{ local }}/redis/password" + register: local_redis + no_log: true + - name: Require a supplied credential for externally managed Redis + ansible.builtin.assert: + that: existing_redis.status == 200 or local_redis.stat.exists + fail_msg: Supply the existing external Redis credential before production reconciliation + when: external_redis | default(false) | bool + no_log: true + - name: Generate only an absent local Redis credential + ansible.builtin.command: + argv: [openssl, rand, -hex, '32'] + when: existing_redis.status == 404 and not local_redis.stat.exists and not (external_redis | default(false) | bool) + register: fresh_redis + no_log: true + - name: Store the canonical or freshly generated Redis password once + ansible.builtin.copy: + dest: "{{ local }}/redis/password" + content: "{{ existing_redis.json.data.data.password if existing_redis.status == 200 else fresh_redis.stdout }}" + mode: '0600' + force: false + when: not local_redis.stat.exists + no_log: true + - name: Refuse mismatched existing Redis credentials + ansible.builtin.assert: + that: lookup('file',local ~ '/redis/password') | trim == existing_redis.json.data.data.password + fail_msg: Redis bootstrap file differs from the existing Vault secret; reconcile explicitly + when: existing_redis.status == 200 + no_log: true + - name: Render local Redis configuration with quoted password escaping + ansible.builtin.copy: + dest: "{{ local }}/redis/redis.conf" + content: | + bind 0.0.0.0 + protected-mode yes + requirepass {{ lookup('file',local ~ '/redis/password') | trim | to_json }} + save "" + appendonly no + loglevel warning + mode: '0600' + no_log: true diff --git a/infrastructure/operations/adopt.yml b/infrastructure/operations/adopt.yml new file mode 100644 index 0000000..8c149f9 --- /dev/null +++ b/infrastructure/operations/adopt.yml @@ -0,0 +1,52 @@ +--- +- name: Record an existing target without creating or changing its volumes + hosts: localhost + gather_facts: false + vars: + repo: "{{ lookup('env', 'PAYGUARD_REPO_ROOT') }}" + compose: [docker, compose, -f, "{{ repo }}/docker-compose.yaml"] + tasks: + - name: Resolve effective Compose names without displaying secret values + ansible.builtin.command: + argv: "{{ compose + ['config', '--format', 'json'] }}" + register: resolved + changed_when: false + no_log: true + + - name: Read local daemon identity + ansible.builtin.command: + argv: [docker, info, --format, "{{ '{{.ID}}' }}"] + register: daemon + changed_when: false + + - name: Select exact external persistent volumes + ansible.builtin.set_fact: + target_volumes: "{{ (resolved.stdout | from_json).volumes | dict2items | map(attribute='value.name') | list }}" + no_log: true + + - name: Require all persistent volumes already exist + ansible.builtin.command: + argv: [docker, volume, inspect, "{{ item }}"] + loop: "{{ target_volumes }}" + changed_when: false + no_log: true + + - name: Check whether a target is already recorded + ansible.builtin.stat: + path: "{{ repo }}/.local/payguard/target.yml" + register: recorded + + - name: Refuse target drift on repeated adoption + ansible.builtin.assert: + that: + - (lookup('file', repo ~ '/.local/payguard/target.yml') | from_yaml).daemon_id == daemon.stdout + - (lookup('file', repo ~ '/.local/payguard/target.yml') | from_yaml).project == lookup('env','COMPOSE_PROJECT_NAME') + - (lookup('file', repo ~ '/.local/payguard/target.yml') | from_yaml).volumes == target_volumes + when: recorded.stat.exists + + - name: Record the target once + ansible.builtin.copy: + dest: "{{ repo }}/.local/payguard/target.yml" + content: "{{ {'project': lookup('env','COMPOSE_PROJECT_NAME'), 'repository': repo, 'daemon_id': daemon.stdout, 'volumes': target_volumes} | to_nice_yaml }}" + mode: '0600' + force: false diff --git a/infrastructure/operations/agent-handoff.yml b/infrastructure/operations/agent-handoff.yml new file mode 100644 index 0000000..4adf6ea --- /dev/null +++ b/infrastructure/operations/agent-handoff.yml @@ -0,0 +1,3 @@ +--- +# Compatibility entry point. Migration admission, execution and cleanup are inseparable. +- ansible.builtin.import_playbook: migrate.yml diff --git a/infrastructure/operations/allocate.yml b/infrastructure/operations/allocate.yml new file mode 100644 index 0000000..393f061 --- /dev/null +++ b/infrastructure/operations/allocate.yml @@ -0,0 +1,36 @@ +--- +- name: Allocate explicitly named volumes only for a new isolated target + hosts: localhost + gather_facts: false + vars: + repo: "{{ lookup('env', 'PAYGUARD_REPO_ROOT') }}" + tasks: + - name: Require a new target ceremony + ansible.builtin.assert: + that: + - fresh_target | default(false) | bool + - lookup('env','COMPOSE_PROJECT_NAME') is match('^[a-z0-9][a-z0-9_-]+$') + - name: Resolve external volume names + ansible.builtin.command: + argv: [docker, compose, -f, "{{ repo }}/docker-compose.yaml", config, --format, json] + register: resolved + changed_when: false + no_log: true + - name: Inspect all requested volumes before allocating any + ansible.builtin.command: + argv: [docker, volume, inspect, "{{ item }}"] + loop: "{{ (resolved.stdout | from_json).volumes | dict2items | map(attribute='value.name') | list }}" + register: existing + failed_when: false + changed_when: false + no_log: true + - name: Refuse allocation if any requested volume already exists + ansible.builtin.assert: + that: existing.results | selectattr('rc','equalto',0) | list | length == 0 + fail_msg: Fresh allocation cannot adopt existing volumes; use adopt.yml + - name: Create the named fresh volumes + ansible.builtin.command: + argv: [docker, volume, create, --label, "payguard.project={{ lookup('env','COMPOSE_PROJECT_NAME') }}", "{{ item.item }}"] + loop: "{{ existing.results }}" + no_log: true +- ansible.builtin.import_playbook: adopt.yml diff --git a/infrastructure/operations/ansible.cfg b/infrastructure/operations/ansible.cfg new file mode 100644 index 0000000..2c8cc4f --- /dev/null +++ b/infrastructure/operations/ansible.cfg @@ -0,0 +1,7 @@ +[defaults] +retry_files_enabled = False +fact_caching = memory +stdout_callback = default +show_custom_stats = False +log_path = /dev/null +interpreter_python = auto_silent diff --git a/infrastructure/operations/backup.yml b/infrastructure/operations/backup.yml new file mode 100644 index 0000000..7e5bdc5 --- /dev/null +++ b/infrastructure/operations/backup.yml @@ -0,0 +1,56 @@ +--- +- name: Take protected PostgreSQL and Vault backups without changing source data + hosts: localhost + gather_facts: false + vars: + repo: "{{ lookup('env','PAYGUARD_REPO_ROOT') }}" + services: [identity, wallet, loan, collateral] + tasks: + - name: Validate recorded target + ansible.builtin.import_tasks: tasks/target.yml + - name: Require an absolute operator-selected destination + ansible.builtin.assert: + that: + - backup_directory is match('^/[^\n]+$') + - not (backup_directory is exists) + - name: Create a private backup directory + ansible.builtin.file: + path: "{{ backup_directory }}" + state: directory + mode: '0700' + - name: Dump each database using native PostgreSQL tooling + ansible.builtin.shell: >- + umask 077; + docker compose -f {{ (repo ~ '/docker-compose.yaml') | quote }} + exec -T -u postgres postgres-{{ item }} + pg_dump -U postgres -d payguard_{{ item }} --format=custom + > {{ (backup_directory ~ '/' ~ item ~ '.dump') | quote }} + args: + executable: /bin/sh + loop: "{{ services }}" + no_log: true + - name: Save a native Raft snapshot with verified TLS + ansible.builtin.uri: + url: https://vault:8200/v1/sys/storage/raft/snapshot + method: GET + headers: + X-Vault-Token: "{{ lookup('file', repo ~ '/.local/payguard/operator/token') | trim }}" + ca_path: "{{ repo }}/.local/payguard/ca.crt" + validate_certs: true + dest: "{{ backup_directory }}/vault.snap" + mode: '0600' + status_code: 200 + timeout: 60 + no_log: true + - name: Compute checksums without reading financial contents into output + ansible.builtin.stat: + path: "{{ backup_directory }}/{{ item }}" + checksum_algorithm: sha256 + loop: [identity.dump, wallet.dump, loan.dump, collateral.dump, vault.snap] + register: backups + no_log: true + - name: Record backup completeness and exact source identity + ansible.builtin.copy: + dest: "{{ backup_directory }}/manifest.yml" + content: "{{ {'target': lookup('file', repo ~ '/.local/payguard/target.yml') | from_yaml, 'files': backups.results | map(attribute='stat') | map(attribute='checksum') | list, 'restore_verified': false, 'cross_service_consistency': 'quiesce writers before invoking for a consistent application checkpoint'} | to_nice_yaml }}" + mode: '0600' diff --git a/infrastructure/operations/import-identity.yml b/infrastructure/operations/import-identity.yml new file mode 100644 index 0000000..98493f8 --- /dev/null +++ b/infrastructure/operations/import-identity.yml @@ -0,0 +1,42 @@ +--- +- name: Import an approved offline Identity export using native PostgreSQL transactions + hosts: localhost + gather_facts: false + vars: + repo: "{{ lookup('env','PAYGUARD_REPO_ROOT') }}" + tasks: + - name: Validate recorded target + ansible.builtin.import_tasks: tasks/target.yml + - name: Validate protected export and mapping inputs + ansible.builtin.stat: + path: "{{ item }}" + checksum_algorithm: sha256 + loop: ["{{ source_file }}", "{{ mapping_file }}"] + register: inputs + no_log: true + - name: Require private immutable approved input + ansible.builtin.assert: + that: + - inputs.results[0].stat.isreg + - inputs.results[1].stat.isreg + - inputs.results[0].stat.mode == '0600' + - inputs.results[1].stat.mode == '0600' + - inputs.results[1].stat.checksum == approved_mapping_sha256 + - (lookup('file',mapping_file) | from_json).approved_by | length > 0 + - (lookup('file',mapping_file) | from_json).approval_reference | length > 0 + no_log: true + - name: Import under stable Identity owner in one rollback-safe transaction + community.postgresql.postgresql_query: + login_host: postgres-identity + login_db: payguard_identity + login_user: postgres + login_password: "{{ lookup('file',repo ~ '/.local/payguard/identity/admin-password') | trim }}" + session_role: identity_owner + ssl_mode: verify-full + ca_cert: "{{ repo }}/.local/payguard/ca.crt" + autocommit: false + query: "{{ lookup('file',repo ~ '/infrastructure/postgresql/import-identity.sql') }}" + named_args: + source: "{{ lookup('file',source_file) }}" + mapping: "{{ lookup('file',mapping_file) }}" + no_log: true diff --git a/infrastructure/operations/install-timer.yml b/infrastructure/operations/install-timer.yml new file mode 100644 index 0000000..f99cbcd --- /dev/null +++ b/infrastructure/operations/install-timer.yml @@ -0,0 +1,38 @@ +--- +- name: Install scoped identity refresh for the current Linux user + hosts: localhost + gather_facts: false + vars: + repo: "{{ lookup('env','PAYGUARD_REPO_ROOT') }}" + config_dir: "{{ lookup('env','HOME') }}/.config" + tasks: + - name: Create user configuration directories + ansible.builtin.file: + path: "{{ config_dir }}/{{ item }}" + state: directory + mode: '0700' + loop: [payguard, systemd/user] + - name: Install the actual target environment without secret values + ansible.builtin.copy: + dest: "{{ config_dir }}/payguard/operations.env" + content: | + PAYGUARD_REPO_ROOT={{ repo }} + PAYGUARD_RUNTIME_ROOT={{ lookup('env','PAYGUARD_RUNTIME_ROOT') }} + PAYGUARD_LOCAL_UID={{ lookup('env','PAYGUARD_LOCAL_UID') }} + PAYGUARD_LOCAL_GID={{ lookup('env','PAYGUARD_LOCAL_GID') }} + PAYGUARD_DOCKER_GID={{ lookup('env','PAYGUARD_DOCKER_GID') }} + COMPOSE_PROJECT_NAME={{ lookup('env','COMPOSE_PROJECT_NAME') }} + mode: '0600' + - name: Install the user service and timer + ansible.builtin.copy: + src: "{{ repo }}/infrastructure/operations/{{ item }}" + dest: "{{ config_dir }}/systemd/user/{{ item }}" + mode: '0600' + loop: [refresh-identities.service, refresh-identities.timer] + - name: Enable automatic scoped identity refresh + ansible.builtin.systemd_service: + name: refresh-identities.timer + scope: user + daemon_reload: true + enabled: true + state: started diff --git a/infrastructure/operations/migrate.yml b/infrastructure/operations/migrate.yml new file mode 100644 index 0000000..2e1d30a --- /dev/null +++ b/infrastructure/operations/migrate.yml @@ -0,0 +1,249 @@ +--- +- name: Migrate one independently versioned database with native Agent and Flyway + hosts: localhost + gather_facts: false + vars: + repo: "{{ lookup('env', 'PAYGUARD_REPO_ROOT') }}" + runtime_root: "{{ lookup('env', 'PAYGUARD_RUNTIME_ROOT') }}" + operator_uid: "{{ lookup('env', 'PAYGUARD_LOCAL_UID') | int }}" + operator_gid: "{{ lookup('env', 'PAYGUARD_LOCAL_GID') | int }}" + compose: + - docker + - compose + - -f + - "{{ repo }}/docker-compose.yaml" + - -f + - "{{ repo }}/infrastructure/compose.operations.yaml" + tasks: + - name: Run bounded host and Vault preflight + ansible.builtin.import_tasks: preflight.yml + + - name: Acquire one service migration lock atomically + ansible.builtin.command: + argv: [mkdir, "{{ runtime_root }}/migration-{{ service }}.lock"] + changed_when: true + + - name: Create unique host-tmpfs attempt + ansible.builtin.tempfile: + state: directory + path: "{{ runtime_root }}" + prefix: "payguard-{{ service }}-" + register: attempt + + - name: Restrict attempt and service directories + ansible.builtin.file: + path: "{{ item.path }}" + state: directory + owner: "{{ operator_uid }}" + group: "{{ operator_gid }}" + mode: "{{ item.mode }}" + loop: + - {path: "{{ attempt.path }}", mode: '0700'} + - {path: "{{ attempt.path }}/{{ service }}", mode: '0700'} + - {path: "{{ attempt.path }}/{{ service }}/private", mode: '0700'} + - {path: "{{ attempt.path }}/{{ service }}/handoff", mode: '0700'} + no_log: true + + - name: Assign attempt identity + ansible.builtin.set_fact: + attempt_id: "{{ attempt.path | basename }}" + container_name: "payguard-agent-{{ attempt.path | basename }}" + receipt_path: "{{ repo }}/.local/payguard/operator/receipts/{{ attempt.path | basename }}.json" + no_log: true + + - name: Write protected pre-auth recovery receipt + ansible.builtin.copy: + dest: "{{ receipt_path }}" + content: "{{ {'attempt': attempt_id, 'service': service, 'target': lookup('env','COMPOSE_PROJECT_NAME'), 'path': attempt.path, 'container': container_name, 'state': 'agent_running', 'token_accessor': '', 'lease_id': '', 'expires_unix': 0} | to_json }}\n" + owner: "{{ operator_uid }}" + group: "{{ operator_gid }}" + mode: '0600' + no_log: true + + - name: Deliver and admit credential + block: + - name: Start bounded one-shot Agent + ansible.builtin.set_fact: + agent_start_unix: "{{ lookup('pipe', 'date +%s') | int }}" + + - name: Run only the selected Agent for at most 60 seconds + ansible.builtin.command: + argv: "{{ ['timeout', '-s', 'TERM', '-k', '5', (agent_timeout_seconds | default(60) | string)] + compose + ['run', '--rm', '--no-deps', '--name', container_name, 'agent-' ~ service] }}" + chdir: "{{ repo }}" + environment: + PAYGUARD_ATTEMPT_ROOT: "{{ attempt.path }}" + PAYGUARD_LOCAL_UID: "{{ operator_uid | string }}" + PAYGUARD_LOCAL_GID: "{{ operator_gid | string }}" + COMPOSE_PROJECT_NAME: "{{ lookup('env', 'COMPOSE_PROJECT_NAME') }}" + register: agent_run + failed_when: agent_run.rc != 0 + no_log: true + + - name: Check current credential and token file modes + ansible.builtin.stat: + path: "{{ item }}" + loop: + - "{{ attempt.path }}/{{ service }}/handoff/credential.toml" + - "{{ attempt.path }}/{{ service }}/private/token" + register: generated + no_log: true + + - name: Reject missing or exposed credential files + ansible.builtin.assert: + that: + - generated.results[0].stat.isreg + - generated.results[1].stat.isreg + - generated.results[0].stat.mode == '0440' + - generated.results[1].stat.mode == '0600' + - generated.results[0].stat.uid == operator_uid + - generated.results[1].stat.uid == operator_uid + - generated.results[0].stat.size > 100 + fail_msg: Agent credential handoff is incomplete + no_log: true + + - name: Read current credential metadata privately + ansible.builtin.slurp: + src: "{{ attempt.path }}/{{ service }}/handoff/credential.toml" + register: rendered + no_log: true + + - name: Validate fixed target and credential fields + ansible.builtin.set_fact: + rendered_text: "{{ rendered.content | b64decode }}" + lease_seconds: "{{ (rendered.content | b64decode | regex_findall('(?m)^# lease_seconds=([0-9]+)$') | first) | int }}" + lease_id: "{{ rendered.content | b64decode | regex_findall('(?m)^# lease_id=([A-Za-z0-9/_-]+)$') | first }}" + no_log: true + + - name: Reject stale or malformed handoff + ansible.builtin.assert: + that: + - rendered_text is search('(?m)^\[environments.default\]$') + - rendered_text is search('(?m)^url = "jdbc:postgresql://postgres-' ~ service ~ ':5432/payguard_' ~ service ~ '\?sslmode=verify-full&sslrootcert=/run/agent-ca/ca.crt¤tSchema=payguard"$') + - rendered_text is search('(?m)^user = ".+"$') + - rendered_text is search('(?m)^password = ".+"$') + - lease_seconds - (lookup('pipe', 'date +%s') | int) + (agent_start_unix | int) >= 840 + fail_msg: Credential is malformed or has insufficient lease lifetime + no_log: true + + - name: Check private Agent token lifetime over verified TLS + ansible.builtin.uri: + url: https://vault:8200/v1/auth/token/lookup-self + ca_path: "{{ repo }}/.local/payguard/ca.crt" + validate_certs: true + headers: + X-Vault-Token: "{{ lookup('file', attempt.path ~ '/' ~ service ~ '/private/token') | trim }}" + status_code: 200 + timeout: 5 + register: token_lookup + no_log: true + + - name: Reject insufficient token validity + ansible.builtin.assert: + that: + - token_lookup.json.data.ttl | int >= 840 + - token_lookup.json.data.accessor | length > 0 + fail_msg: Agent token has insufficient lifetime + no_log: true + + - name: Complete protected recovery receipt + ansible.builtin.copy: + dest: "{{ receipt_path }}" + content: "{{ {'attempt': attempt_id, 'service': service, 'target': lookup('env','COMPOSE_PROJECT_NAME'), 'path': attempt.path, 'container': container_name, 'state': 'credentials_ready', 'token_accessor': token_lookup.json.data.accessor, 'lease_id': lease_id, 'expires_unix': (agent_start_unix | int) + (lease_seconds | int)} | to_json }}\n" + owner: "{{ operator_uid }}" + group: "{{ operator_gid }}" + mode: '0600' + no_log: true + + - name: Keep the protected lease identifier out of the consumer handoff + ansible.builtin.copy: + dest: "{{ attempt.path }}/{{ service }}/handoff/credential.toml" + content: "{{ rendered_text | regex_replace('(?m)^# lease_id=[^\\n]*\\n', '') }}" + owner: "{{ operator_uid }}" + group: "{{ operator_gid }}" + mode: '0440' + no_log: true + + - name: Run native Flyway under the remaining lease budget + ansible.builtin.command: + argv: "{{ ['timeout', '-s', 'TERM', '-k', '10', '600'] + compose + ['run', '--rm', '--no-deps', '--name', 'payguard-flyway-' ~ attempt_id, 'migrate-' ~ service] }}" + chdir: "{{ repo }}" + environment: + PAYGUARD_ATTEMPT_ROOT: "{{ attempt.path }}" + PAYGUARD_LOCAL_UID: "{{ operator_uid | string }}" + PAYGUARD_LOCAL_GID: "{{ operator_gid | string }}" + COMPOSE_PROJECT_NAME: "{{ lookup('env', 'COMPOSE_PROJECT_NAME') }}" + register: flyway_run + no_log: true + + - name: Verify application grants and private history after Flyway + ansible.builtin.include_tasks: tasks/verify-migration.yml + + - name: Record migration success before cleanup + ansible.builtin.set_fact: + migration_success: true + + rescue: + - name: Retain migration failure without dependency diagnostics + ansible.builtin.set_fact: + migration_success: false + always: + - name: Stop only containers belonging to this attempt + ansible.builtin.command: + argv: [docker, rm, -f, "{{ item }}"] + loop: ["{{ container_name }}", "payguard-flyway-{{ attempt_id }}"] + failed_when: false + changed_when: false + no_log: true + + - name: Check for the Agent sink token + ansible.builtin.stat: + path: "{{ attempt.path }}/{{ service }}/private/token" + register: cleanup_token + no_log: true + + - name: Revoke this attempt token and its database leases + ansible.builtin.uri: + url: https://vault:8200/v1/auth/token/revoke-self + method: POST + ca_path: "{{ repo }}/.local/payguard/ca.crt" + validate_certs: true + headers: + X-Vault-Token: "{{ lookup('file', attempt.path ~ '/' ~ service ~ '/private/token') | trim }}" + status_code: [200, 204] + timeout: 5 + register: cleanup_revoke + ignore_errors: true + when: cleanup_token.stat.exists + no_log: true + + - name: Record recoverable cleanup outcome + ansible.builtin.copy: + dest: "{{ receipt_path }}" + content: "{{ {'attempt': attempt_id, 'service': service, 'target': lookup('env','COMPOSE_PROJECT_NAME'), 'path': attempt.path, 'container': container_name, 'state': ('cleanup_verified' if (cleanup_token.stat.exists and cleanup_revoke is succeeded) else 'cleanup_pending'), 'migration_success': migration_success | default(false), 'token_accessor': token_lookup.json.data.accessor | default(''), 'lease_id': lease_id | default('')} | to_json }}\n" + mode: '0600' + no_log: true + + - name: Remove temporary credentials only after confirmed revocation + ansible.builtin.file: + path: "{{ attempt.path }}" + state: absent + when: cleanup_token.stat.exists and cleanup_revoke is succeeded + no_log: true + + - name: Release service lock only after verified cleanup + ansible.builtin.file: + path: "{{ runtime_root }}/migration-{{ service }}.lock" + state: absent + when: cleanup_token.stat.exists and cleanup_revoke is succeeded + + - name: Require both migration and cleanup success + ansible.builtin.assert: + that: + - migration_success | default(false) + - cleanup_token.stat.exists + - cleanup_revoke is succeeded + fail_msg: Migration or cleanup failed; inspect the protected attempt receipt and use recover-attempt.yml + + - name: Report sanitized migration completion + ansible.builtin.debug: + msg: "migration_complete service={{ service }} cleanup=verified" diff --git a/infrastructure/operations/preflight.yml b/infrastructure/operations/preflight.yml new file mode 100644 index 0000000..b84f62a --- /dev/null +++ b/infrastructure/operations/preflight.yml @@ -0,0 +1,105 @@ +- name: Validate recorded target + ansible.builtin.import_tasks: tasks/target.yml + +- name: Reject unknown service or project + ansible.builtin.assert: + that: + - service | default('') in ['identity', 'wallet', 'loan', 'collateral'] + - repo | length > 0 + - runtime_root is match('^/[^ ]+$') + - operator_uid >= 0 + - operator_gid >= 0 + - lookup('env', 'COMPOSE_PROJECT_NAME') == (lookup('file', repo ~ '/.local/payguard/target.yml') | from_yaml).project + - (agent_timeout_seconds | default(60) | int) > 0 + - (agent_timeout_seconds | default(60) | int) <= 60 + fail_msg: Invalid service, project, or controller environment + +- name: Check local Docker daemon + ansible.builtin.command: + argv: [docker, context, inspect] + register: docker_context + changed_when: false + +- name: Reject a remote Docker daemon + ansible.builtin.assert: + that: + - (docker_context.stdout | from_json)[0].Endpoints.docker.Host.startswith('unix://') + - lookup('env', 'DOCKER_HOST') in ['', 'unix:///var/run/docker.sock'] + - lookup('env', 'DOCKER_CONTEXT') in ['', 'default'] + fail_msg: A local Unix-socket Docker daemon is required + +- name: Check protected handoff is on tmpfs + ansible.builtin.command: + argv: [findmnt, -T, "{{ runtime_root }}", -n, -o, FSTYPE] + register: runtime_fs + changed_when: false + failed_when: runtime_fs.rc != 0 or runtime_fs.stdout != 'tmpfs' + +- name: Check host paths and protected inputs + ansible.builtin.stat: + path: "{{ item }}" + loop: + - "{{ runtime_root }}" + - "{{ repo }}/.local/payguard/ca.crt" + - "{{ repo }}/.local/payguard/operator/{{ service }}/role-id" + - "{{ repo }}/.local/payguard/operator/{{ service }}/secret-id" + - "{{ repo }}/infrastructure/vault/agent/{{ service }}.hcl" + register: inputs + no_log: true + +- name: Reject missing or unprotected host paths + ansible.builtin.assert: + that: + - inputs.results | map(attribute='stat.exists') | min + - inputs.results[0].stat.isdir + - inputs.results[0].stat.writeable + - inputs.results[0].stat.mode == '0700' + - inputs.results[0].stat.uid == operator_uid + - not inputs.results[0].stat.islnk + - inputs.results[2].stat.mode == '0600' + - inputs.results[3].stat.mode == '0600' + - inputs.results[2].stat.uid == operator_uid + - inputs.results[3].stat.uid == operator_uid + fail_msg: Host mapping or scoped authentication input is unavailable + +- name: Validate Compose bind configuration + ansible.builtin.command: + argv: [docker, compose, -f, "{{ repo }}/docker-compose.yaml", -f, "{{ repo }}/infrastructure/compose.operations.yaml", --profile, migration, config, --quiet] + environment: + PAYGUARD_ATTEMPT_ROOT: "{{ runtime_root }}" + PAYGUARD_LOCAL_UID: "{{ operator_uid | string }}" + PAYGUARD_LOCAL_GID: "{{ operator_gid | string }}" + PAYGUARD_DOCKER_GID: "{{ lookup('env', 'PAYGUARD_DOCKER_GID') }}" + PAYGUARD_RUNTIME_ROOT: "{{ runtime_root }}" + COMPOSE_PROJECT_NAME: "{{ lookup('env', 'COMPOSE_PROJECT_NAME') }}" + changed_when: false + +- name: Check required Compose services are running + ansible.builtin.command: + argv: [docker, compose, -f, "{{ repo }}/docker-compose.yaml", ps, --status, running, --services] + register: running_services + changed_when: false + failed_when: running_services.rc != 0 or 'vault' not in running_services.stdout_lines or ('postgres-' ~ service) not in running_services.stdout_lines + +- name: Check unsealed Vault over verified TLS + ansible.builtin.uri: + url: https://vault:8200/v1/sys/health + ca_path: "{{ repo }}/.local/payguard/ca.crt" + validate_certs: true + status_code: 200 + timeout: 5 + changed_when: false + no_log: true + +- name: Read the local Docker daemon identity + ansible.builtin.command: + argv: [docker, info, --format, "{{ '{{.ID}}' }}"] + register: daemon_identity + changed_when: false + +- name: Verify independently recorded target identity + ansible.builtin.assert: + that: + - daemon_identity.stdout == (lookup('file', repo ~ '/.local/payguard/target.yml') | from_yaml).daemon_id + - (lookup('file', repo ~ '/.local/payguard/target.yml') | from_yaml).repository == repo + fail_msg: Docker daemon or repository differs from the adopted target diff --git a/infrastructure/operations/prepare.yml b/infrastructure/operations/prepare.yml new file mode 100644 index 0000000..92c9072 --- /dev/null +++ b/infrastructure/operations/prepare.yml @@ -0,0 +1,115 @@ +--- +- name: Prepare protected artifacts for a genuinely new local target + hosts: localhost + gather_facts: false + vars: + repo: "{{ lookup('env', 'PAYGUARD_REPO_ROOT') }}" + artifact_root: "{{ repo }}/.local/payguard" + services: [identity, loan, collateral, wallet] + tasks: + - name: Require explicit fresh target and refuse any existing artifact root + ansible.builtin.assert: + that: + - fresh_target | default(false) | bool + - lookup('env', 'COMPOSE_PROJECT_NAME') is match('^[a-z0-9][a-z0-9_-]+$') + - not (artifact_root is exists) + fail_msg: Fresh preparation refuses existing state or an unapproved project + + - name: Create protected directories + ansible.builtin.file: + path: "{{ artifact_root }}/{{ item }}" + state: directory + mode: '0700' + loop: "{{ [''] + services + ['vault', 'operator', 'apps', 'operator/approle', 'operator/identity', 'operator/loan', 'operator/collateral', 'operator/wallet', 'operator/receipts', 'redis', 'kafka'] + (services | map('regex_replace', '^', 'apps/') | list) }}" + + - name: Generate new local CA + ansible.builtin.command: + argv: [openssl, req, -x509, -newkey, rsa:3072, -nodes, -sha256, -days, '365', -subj, '/CN=PayGuard infrastructure CA', -addext, 'basicConstraints=critical,CA:TRUE', -addext, 'keyUsage=critical,keyCertSign,cRLSign', -keyout, "{{ artifact_root }}/ca.key", -out, "{{ artifact_root }}/ca.crt"] + no_log: true + + - name: Generate each service key and CSR + ansible.builtin.command: + argv: [openssl, req, -new, -newkey, rsa:3072, -nodes, -keyout, "{{ artifact_root }}/{{ item }}/server.key", -out, "{{ artifact_root }}/{{ item }}/server.csr", -subj, "/CN=postgres-{{ item }}"] + loop: "{{ services }}" + no_log: true + + - name: Sign each service certificate with its fixed SAN + ansible.builtin.command: + argv: [openssl, x509, -req, -in, "{{ artifact_root }}/{{ item }}/server.csr", -CA, "{{ artifact_root }}/ca.crt", -CAkey, "{{ artifact_root }}/ca.key", -CAcreateserial, -days, '365', -sha256, -extfile, /dev/stdin, -out, "{{ artifact_root }}/{{ item }}/server.crt"] + stdin: "basicConstraints=CA:FALSE\nkeyUsage=digitalSignature,keyEncipherment\nextendedKeyUsage=serverAuth\nsubjectAltName=DNS:postgres-{{ item }},DNS:localhost,IP:127.0.0.1\n" + loop: "{{ services }}" + no_log: true + + - name: Generate Vault key and CSR + ansible.builtin.command: + argv: [openssl, req, -new, -newkey, rsa:3072, -nodes, -keyout, "{{ artifact_root }}/vault/server.key", -out, "{{ artifact_root }}/vault/server.csr", -subj, /CN=vault] + no_log: true + + - name: Sign Vault certificate with local SANs + ansible.builtin.command: + argv: [openssl, x509, -req, -in, "{{ artifact_root }}/vault/server.csr", -CA, "{{ artifact_root }}/ca.crt", -CAkey, "{{ artifact_root }}/ca.key", -CAcreateserial, -days, '365', -sha256, -extfile, /dev/stdin, -out, "{{ artifact_root }}/vault/server.crt"] + stdin: "basicConstraints=CA:FALSE\nkeyUsage=digitalSignature,keyEncipherment\nextendedKeyUsage=serverAuth\nsubjectAltName=DNS:vault,DNS:localhost,IP:127.0.0.1\n" + no_log: true + + - name: Generate fresh PostgreSQL admin and Vault manager passwords + ansible.builtin.command: + argv: [openssl, rand, -hex, '32'] + loop: "{{ services | product(['admin-password', 'manager-password']) | list }}" + register: generated_passwords + no_log: true + + - name: Store passwords in protected files only + ansible.builtin.copy: + dest: "{{ artifact_root }}/{{ item.item.0 }}/{{ item.item.1 }}" + content: "{{ item.stdout }}\n" + mode: '0600' + loop: "{{ generated_passwords.results }}" + no_log: true + + - name: Copy CA for Vault server TLS + ansible.builtin.copy: + src: "{{ artifact_root }}/ca.crt" + dest: "{{ artifact_root }}/vault/ca.crt" + remote_src: true + mode: '0600' + + - name: Generate fresh Redis authentication + ansible.builtin.command: + argv: [openssl, rand, -hex, '32'] + register: redis_password + no_log: true + + - name: Store fresh Redis password + ansible.builtin.copy: + dest: "{{ artifact_root }}/redis/password" + content: "{{ redis_password.stdout }}" + mode: '0600' + no_log: true + + - name: Configure Redis with file-only authentication + ansible.builtin.copy: + dest: "{{ artifact_root }}/redis/redis.conf" + content: | + bind 0.0.0.0 + protected-mode yes + requirepass {{ redis_password.stdout }} + save "" + appendonly no + loglevel warning + mode: '0600' + no_log: true + + - name: Find generated files + ansible.builtin.find: + paths: "{{ artifact_root }}" + recurse: true + file_type: file + register: generated_files + no_log: true + + - name: Restrict generated files + ansible.builtin.file: + path: "{{ item.path }}" + mode: '0600' + loop: "{{ generated_files.files }}" + no_log: true diff --git a/infrastructure/operations/provision-database.yml b/infrastructure/operations/provision-database.yml new file mode 100644 index 0000000..9ca8d3e --- /dev/null +++ b/infrastructure/operations/provision-database.yml @@ -0,0 +1,166 @@ +--- +- name: Reconcile four isolated PostgreSQL databases without resetting data + hosts: localhost + gather_facts: false + vars: + repo: "{{ lookup('env', 'PAYGUARD_REPO_ROOT') }}" + services: [identity, loan, collateral, wallet] + tasks: + - name: Validate recorded target + ansible.builtin.import_tasks: tasks/target.yml + + - name: Require approved local project and protected admin inputs + ansible.builtin.assert: + that: + - lookup('env', 'COMPOSE_PROJECT_NAME') == (lookup('file', repo ~ '/.local/payguard/target.yml') | from_yaml).project + - repo | length > 0 + - lookup('file', repo ~ '/.local/payguard/ca.crt') | length > 0 + fail_msg: Database provisioning requires the approved local target and CA + no_log: true + + - name: Check PostgreSQL 18 and current database identity over verified TLS + community.postgresql.postgresql_query: + login_host: "postgres-{{ item }}" + login_db: "payguard_{{ item }}" + login_user: postgres + login_password: "{{ lookup('file', repo ~ '/.local/payguard/' ~ item ~ '/admin-password') | trim }}" + ssl_mode: verify-full + ca_cert: "{{ repo }}/.local/payguard/ca.crt" + query: "SELECT current_database() AS db, current_setting('server_version_num')::int AS version" + loop: "{{ services }}" + register: database_identity + no_log: true + + - name: Reject a wrong database or PostgreSQL major version + ansible.builtin.assert: + that: + - item.query_result[0].db == 'payguard_' ~ item.item + - item.query_result[0].version >= 180000 + - item.query_result[0].version < 190000 + loop: "{{ database_identity.results }}" + no_log: true + + - name: Create or preserve owner groups + community.postgresql.postgresql_user: + login_host: "postgres-{{ item }}" + login_db: "payguard_{{ item }}" + login_user: postgres + login_password: "{{ lookup('file', repo ~ '/.local/payguard/' ~ item ~ '/admin-password') | trim }}" + ssl_mode: verify-full + ca_cert: "{{ repo }}/.local/payguard/ca.crt" + name: "{{ item }}_owner" + role_attr_flags: NOLOGIN,NOSUPERUSER,NOCREATEDB,NOCREATEROLE,NOREPLICATION,NOBYPASSRLS + loop: "{{ services }}" + no_log: true + + - name: Create or preserve runtime groups + community.postgresql.postgresql_user: + login_host: "postgres-{{ item }}" + login_db: "payguard_{{ item }}" + login_user: postgres + login_password: "{{ lookup('file', repo ~ '/.local/payguard/' ~ item ~ '/admin-password') | trim }}" + ssl_mode: verify-full + ca_cert: "{{ repo }}/.local/payguard/ca.crt" + name: "{{ item }}_runtime" + role_attr_flags: NOLOGIN,NOSUPERUSER,NOCREATEDB,NOCREATEROLE,NOREPLICATION,NOBYPASSRLS + loop: "{{ services }}" + no_log: true + + - name: Check whether manager already exists + community.postgresql.postgresql_query: + login_host: "postgres-{{ item }}" + login_db: "payguard_{{ item }}" + login_user: postgres + login_password: "{{ lookup('file', repo ~ '/.local/payguard/' ~ item ~ '/admin-password') | trim }}" + ssl_mode: verify-full + ca_cert: "{{ repo }}/.local/payguard/ca.crt" + query: "SELECT count(*)::int AS found FROM pg_roles WHERE rolname = 'vault_manager'" + loop: "{{ services }}" + register: managers + no_log: true + + - name: Create manager only when absent + community.postgresql.postgresql_user: + login_host: "postgres-{{ item.item }}" + login_db: "payguard_{{ item.item }}" + login_user: postgres + login_password: "{{ lookup('file', repo ~ '/.local/payguard/' ~ item.item ~ '/admin-password') | trim }}" + ssl_mode: verify-full + ca_cert: "{{ repo }}/.local/payguard/ca.crt" + name: vault_manager + role_attr_flags: CREATEROLE,LOGIN,NOINHERIT,NOSUPERUSER,NOCREATEDB,NOREPLICATION,NOBYPASSRLS + password: "{{ lookup('file', repo ~ '/.local/payguard/' ~ item.item ~ '/manager-password', errors='ignore') | default((lookup('file', repo ~ '/.local/payguard/' ~ item.item ~ '/credentials.json', errors='ignore') | default('{}', true) | from_json).manager | default(''), true) | trim }}" + loop: "{{ managers.results }}" + when: item.query_result[0].found == 0 + no_log: true + + - name: Preserve manager password while reconciling role flags + community.postgresql.postgresql_user: + login_host: "postgres-{{ item.item }}" + login_db: "payguard_{{ item.item }}" + login_user: postgres + login_password: "{{ lookup('file', repo ~ '/.local/payguard/' ~ item.item ~ '/admin-password') | trim }}" + ssl_mode: verify-full + ca_cert: "{{ repo }}/.local/payguard/ca.crt" + name: vault_manager + role_attr_flags: CREATEROLE,LOGIN,NOINHERIT,NOSUPERUSER,NOCREATEDB,NOREPLICATION,NOBYPASSRLS + no_password_changes: true + loop: "{{ managers.results }}" + when: item.query_result[0].found > 0 + no_log: true + + - name: Reconcile private schema ownership + community.postgresql.postgresql_schema: + login_host: "postgres-{{ item }}" + login_db: "payguard_{{ item }}" + login_user: postgres + login_password: "{{ lookup('file', repo ~ '/.local/payguard/' ~ item ~ '/admin-password') | trim }}" + ssl_mode: verify-full + ca_cert: "{{ repo }}/.local/payguard/ca.crt" + name: payguard + owner: "{{ item }}_owner" + loop: "{{ services }}" + no_log: true + + - name: Reconcile only fixed database and default grants + community.postgresql.postgresql_query: + login_host: "postgres-{{ item }}" + login_db: "payguard_{{ item }}" + login_user: postgres + login_password: "{{ lookup('file', repo ~ '/.local/payguard/' ~ item ~ '/admin-password') | trim }}" + ssl_mode: verify-full + ca_cert: "{{ repo }}/.local/payguard/ca.crt" + query: + - "REVOKE ALL ON DATABASE payguard_{{ item }} FROM PUBLIC" + - "REVOKE ALL ON SCHEMA public FROM PUBLIC" + - "GRANT {{ item }}_owner, {{ item }}_runtime TO vault_manager WITH ADMIN TRUE, INHERIT FALSE, SET FALSE" + - "GRANT CONNECT ON DATABASE payguard_{{ item }} TO {{ item }}_owner, {{ item }}_runtime" + - "GRANT CONNECT ON DATABASE payguard_{{ item }} TO vault_manager WITH GRANT OPTION" + - "GRANT USAGE ON SCHEMA payguard TO {{ item }}_runtime, vault_manager" + - "ALTER DEFAULT PRIVILEGES FOR ROLE {{ item }}_owner IN SCHEMA payguard REVOKE SELECT, INSERT, UPDATE, DELETE ON TABLES FROM {{ item }}_runtime" + - "ALTER DEFAULT PRIVILEGES FOR ROLE {{ item }}_owner IN SCHEMA payguard GRANT USAGE, SELECT ON SEQUENCES TO {{ item }}_runtime" + - "ALTER DEFAULT PRIVILEGES FOR ROLE {{ item }}_owner REVOKE EXECUTE ON FUNCTIONS FROM PUBLIC" + loop: "{{ services }}" + no_log: true + + - name: Retain existing object ownership and refuse unexpected schema owners + community.postgresql.postgresql_query: + login_host: "postgres-{{ item }}" + login_db: "payguard_{{ item }}" + login_user: postgres + login_password: "{{ lookup('file', repo ~ '/.local/payguard/' ~ item ~ '/admin-password') | trim }}" + ssl_mode: verify-full + ca_cert: "{{ repo }}/.local/payguard/ca.crt" + query: | + SELECT count(*)::int AS unexpected FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace + WHERE n.nspname='payguard' AND pg_get_userbyid(c.relowner) <> '{{ item }}_owner' + loop: "{{ services }}" + register: object_owners + no_log: true + + - name: Refuse automatic ownership transfer of adopted data + ansible.builtin.assert: + that: item.query_result[0].unexpected == 0 + fail_msg: Existing schema objects have unexpected ownership; reconcile explicitly before migration + loop: "{{ object_owners.results }}" + no_log: true diff --git a/infrastructure/operations/provision-vault.yml b/infrastructure/operations/provision-vault.yml new file mode 100644 index 0000000..3004cb7 --- /dev/null +++ b/infrastructure/operations/provision-vault.yml @@ -0,0 +1,437 @@ +--- +- name: Reconcile scoped Vault configuration without replacing persisted secrets + hosts: localhost + gather_facts: false + vars: + repo: "{{ lookup('env', 'PAYGUARD_REPO_ROOT') }}" + ca: "{{ repo }}/.local/payguard/ca.crt" + token: "{{ lookup('file', repo ~ '/.local/payguard/operator/token') | trim }}" + services: [identity, loan, collateral, wallet] + role_pairs: + - [identity, runtime] + - [identity, migration] + - [loan, runtime] + - [loan, migration] + - [collateral, runtime] + - [collateral, migration] + - [wallet, runtime] + - [wallet, migration] + tasks: + - name: Validate recorded target + ansible.builtin.import_tasks: tasks/target.yml + + - name: Require approved project and scoped operator token + ansible.builtin.assert: + that: + - lookup('env', 'COMPOSE_PROJECT_NAME') == (lookup('file', repo ~ '/.local/payguard/target.yml') | from_yaml).project + - token | length > 0 + fail_msg: Vault reconciliation requires approved project and scoped operator access + no_log: true + + - name: Check unsealed Vault over verified HTTPS + ansible.builtin.uri: + url: https://vault:8200/v1/sys/health + ca_path: "{{ ca }}" + validate_certs: true + status_code: 200 + timeout: 5 + changed_when: false + no_log: true + + - name: Read persistent mount and audit inventory + ansible.builtin.uri: + url: "https://vault:8200/v1/{{ item }}" + headers: {X-Vault-Token: "{{ token }}"} + ca_path: "{{ ca }}" + validate_certs: true + status_code: 200 + timeout: 5 + loop: [sys/mounts, sys/auth, sys/audit] + register: vault_inventory + no_log: true + + - name: Enable only missing fresh database mount + ansible.builtin.uri: + url: https://vault:8200/v1/sys/mounts/database + method: POST + body_format: json + body: {type: database} + headers: {X-Vault-Token: "{{ token }}"} + ca_path: "{{ ca }}" + validate_certs: true + status_code: [200, 204] + when: fresh_target | default(false) | bool and 'database/' not in vault_inventory.results[0].json.data + no_log: true + + - name: Enable only missing fresh KV v2 mount + ansible.builtin.uri: + url: https://vault:8200/v1/sys/mounts/secret + method: POST + body_format: json + body: {type: kv, options: {version: '2'}} + headers: {X-Vault-Token: "{{ token }}"} + ca_path: "{{ ca }}" + validate_certs: true + status_code: [200, 204] + when: fresh_target | default(false) | bool and 'secret/' not in vault_inventory.results[0].json.data + no_log: true + + - name: Enable only missing fresh AppRole auth mount + ansible.builtin.uri: + url: https://vault:8200/v1/sys/auth/approle + method: POST + body_format: json + body: {type: approle} + headers: {X-Vault-Token: "{{ token }}"} + ca_path: "{{ ca }}" + validate_certs: true + status_code: [200, 204] + when: fresh_target | default(false) | bool and 'approle/' not in vault_inventory.results[1].json.data + no_log: true + + - name: Enable only missing fresh file audit + ansible.builtin.uri: + url: https://vault:8200/v1/sys/audit/file + method: POST + body_format: json + body: {type: file, options: {file_path: /vault/audit/audit.json}} + headers: {X-Vault-Token: "{{ token }}"} + ca_path: "{{ ca }}" + validate_certs: true + status_code: [200, 204] + when: fresh_target | default(false) | bool and 'file/' not in vault_inventory.results[2].json.data + no_log: true + + - name: Re-read mounts and audit after fresh-only changes + ansible.builtin.uri: + url: "https://vault:8200/v1/{{ item }}" + headers: {X-Vault-Token: "{{ token }}"} + ca_path: "{{ ca }}" + validate_certs: true + status_code: 200 + timeout: 5 + loop: [sys/mounts, sys/auth, sys/audit] + register: vault_inventory + no_log: true + + - name: Refuse incompatible or missing adopted-state mounts + ansible.builtin.assert: + that: + - vault_inventory.results[0].json.data['database/'].type == 'database' + - vault_inventory.results[0].json.data['secret/'].type == 'kv' + - vault_inventory.results[0].json.data['secret/'].options.version == '2' + - vault_inventory.results[1].json.data['approle/'].type == 'approle' + - vault_inventory.results[2].json.data['file/'].type == 'file' + fail_msg: Required Vault mount or audit state is missing or incompatible; run the explicit fresh ceremony on a new target + no_log: true + + - name: Read existing database connections + ansible.builtin.uri: + url: "https://vault:8200/v1/database/config/local-{{ item }}" + headers: {X-Vault-Token: "{{ token }}"} + ca_path: "{{ ca }}" + validate_certs: true + status_code: [200, 404] + timeout: 5 + loop: "{{ services }}" + register: db_configs + no_log: true + + - name: Refuse missing adopted-state connections + ansible.builtin.assert: + that: + - item.status in [200, 404] + - item.status != 200 or 'sslmode=verify-full' in item.json.data.connection_details.connection_url + fail_msg: Missing or incompatible Vault database connection requires explicit fresh setup or operator repair + loop: "{{ db_configs.results }}" + no_log: true + + - name: Create only missing fresh database connections + ansible.builtin.uri: + url: "https://vault:8200/v1/database/config/local-{{ item.item }}" + method: POST + body_format: json + body: + plugin_name: postgresql-database-plugin + allowed_roles: ["local-{{ item.item }}-runtime", "local-{{ item.item }}-migration"] + connection_url: "postgresql://{{ '{{username}}' }}:{{ '{{password}}' }}@postgres-{{ item.item }}:5432/payguard_{{ item.item }}?sslmode=verify-full&sslrootcert=/vault/tls/ca.crt" + username: vault_manager + password: "{{ lookup('file', repo ~ '/.local/payguard/' ~ item.item ~ '/manager-password', errors='ignore') | default((lookup('file', repo ~ '/.local/payguard/' ~ item.item ~ '/credentials.json', errors='ignore') | default('{}', true) | from_json).manager | default(''), true) | trim }}" + verify_connection: true + headers: {X-Vault-Token: "{{ token }}"} + ca_path: "{{ ca }}" + validate_certs: true + status_code: [200, 204] + timeout: 10 + loop: "{{ db_configs.results }}" + when: item.status == 404 and fresh_target | default(false) | bool + no_log: true + + - name: Rotate only newly created manager connection roots + ansible.builtin.uri: + url: "https://vault:8200/v1/database/rotate-root/local-{{ item.item }}" + method: POST + headers: {X-Vault-Token: "{{ token }}"} + ca_path: "{{ ca }}" + validate_certs: true + status_code: [200, 204] + timeout: 10 + loop: "{{ db_configs.results }}" + when: item.status == 404 and fresh_target | default(false) | bool + no_log: true + + - name: Extend only allowed role names without supplying manager credentials + ansible.builtin.uri: + url: "https://vault:8200/v1/database/config/local-{{ item.item }}" + method: POST + body_format: json + body: + plugin_name: "{{ item.json.data.plugin_name }}" + allowed_roles: ["local-{{ item.item }}-runtime", "local-{{ item.item }}-migration"] + headers: {X-Vault-Token: "{{ token }}"} + ca_path: "{{ ca }}" + validate_certs: true + status_code: [200, 204] + when: item.status == 200 and ('local-' ~ item.item ~ '-migration') not in item.json.data.allowed_roles + loop: "{{ db_configs.results }}" + no_log: true + + - name: Read existing dynamic role definitions + ansible.builtin.uri: + url: "https://vault:8200/v1/database/roles/local-{{ item.0 }}-{{ item.1 }}" + headers: {X-Vault-Token: "{{ token }}"} + ca_path: "{{ ca }}" + validate_certs: true + status_code: [200, 404] + timeout: 5 + loop: "{{ role_pairs }}" + register: db_roles + no_log: true + + - name: Require adopted-state roles or explicit fresh setup + ansible.builtin.assert: + that: + - item.status in [200, 404] + - item.status != 200 or item.json.data.db_name == 'local-' ~ item.item.0 + - item.status != 200 or item.item.1 == 'runtime' or 'WITH INHERIT FALSE, SET TRUE, ADMIN FALSE' in (item.json.data.creation_statements | join(' ')) + - item.status != 200 or item.item.1 == 'runtime' or 'GRANT CONNECT ON DATABASE payguard_' ~ item.item.0 in (item.json.data.creation_statements | join(' ')) + - item.status != 200 or item.item.1 == 'migration' or item.item.0 ~ '_runtime' in (item.json.data.creation_statements | join(' ')) + fail_msg: Missing or cross-service Vault role requires explicit fresh setup + loop: "{{ db_roles.results }}" + no_log: true + + - name: Create only missing fresh dynamic roles + ansible.builtin.uri: + url: "https://vault:8200/v1/database/roles/local-{{ item.item.0 }}-{{ item.item.1 }}" + method: POST + body_format: json + body: + db_name: "local-{{ item.item.0 }}" + creation_statements: + - "CREATE ROLE \"{{ '{{name}}' }}\" WITH LOGIN PASSWORD '{{ '{{password}}' }}' VALID UNTIL '{{ '{{expiration}}' }}'; GRANT {{ item.item.0 }}_{{ 'runtime' if item.item.1 == 'runtime' else 'owner' }} TO \"{{ '{{name}}' }}\"{{ '' if item.item.1 == 'runtime' else ' WITH INHERIT FALSE, SET TRUE, ADMIN FALSE' }}; {{ '' if item.item.1 == 'runtime' else 'GRANT CONNECT ON DATABASE payguard_' ~ item.item.0 ~ ' TO \"' ~ '{{name}}' ~ '\";' }}" + revocation_statements: + - "{{ 'REVOKE CONNECT ON DATABASE payguard_' ~ item.item.0 ~ ' FROM \"' ~ '{{name}}' ~ '\"; ' if item.item.1 == 'migration' else '' }}REVOKE {{ item.item.0 }}_{{ 'runtime' if item.item.1 == 'runtime' else 'owner' }} FROM \"{{ '{{name}}' }}\"; DROP ROLE IF EXISTS \"{{ '{{name}}' }}\";" + renew_statements: ["ALTER ROLE \"{{ '{{name}}' }}\" VALID UNTIL '{{ '{{expiration}}' }}';"] + default_ttl: "{{ '1h' if item.item.1 == 'runtime' else '15m' }}" + max_ttl: "{{ '4h' if item.item.1 == 'runtime' else '30m' }}" + headers: {X-Vault-Token: "{{ token }}"} + ca_path: "{{ ca }}" + validate_certs: true + status_code: [200, 204] + timeout: 10 + loop: "{{ db_roles.results }}" + when: item.status == 404 + no_log: true + + - name: Read scoped policy definitions + ansible.builtin.uri: + url: "https://vault:8200/v1/sys/policies/acl/local-{{ item.0 }}{{ '-migration' if item.1 == 'migration' else '' }}" + headers: {X-Vault-Token: "{{ token }}"} + ca_path: "{{ ca }}" + validate_certs: true + status_code: [200, 404] + timeout: 5 + loop: "{{ role_pairs }}" + register: policies + no_log: true + + - name: Reject missing adopted policies and cross-service grants + ansible.builtin.assert: + that: + - item.status in [200, 404] + - item.status != 200 or 'database/creds/local-' ~ item.item.0 ~ '-' ~ item.item.1 in item.json.data.policy + - item.status != 200 or (services | reject('equalto', item.item.0) | map('regex_replace', '^(.*)$', 'database/creds/local-\\1-') | select('in', item.json.data.policy) | list | length) == 0 + fail_msg: Missing or cross-service Vault policy must not be adopted + loop: "{{ policies.results }}" + no_log: true + + - name: Create only missing fresh scoped policies + ansible.builtin.uri: + url: "https://vault:8200/v1/sys/policies/acl/local-{{ item.item.0 }}{{ '-migration' if item.item.1 == 'migration' else '' }}" + method: POST + body_format: json + body: + policy: | + path "auth/token/lookup-self" { capabilities = ["read"] } + path "auth/token/renew-self" { capabilities = ["update"] } + path "auth/token/revoke-self" { capabilities = ["update"] } + path "database/creds/local-{{ item.item.0 }}-{{ item.item.1 }}" { capabilities = ["read"] } + {% if item.item.1 == 'runtime' %} + path "secret/data/local/{{ item.item.0 }}/kafka" { capabilities = ["read"] } + path "secret/data/local/infrastructure/redis" { capabilities = ["read"] } + path "sys/leases/renew" { capabilities = ["update"] allowed_parameters = { "lease_id" = ["database/creds/local-{{ item.item.0 }}-runtime/*"], "increment" = ["*"] } } + path "sys/leases/revoke" { capabilities = ["update"] allowed_parameters = { "lease_id" = ["database/creds/local-{{ item.item.0 }}-runtime/*"], "increment" = ["*"] } } + {% if item.item.0 == 'identity' %} + path "secret/data/local/identity/otp-keys" { capabilities = ["read"] } + path "secret/data/local/identity/authenticators/*" { capabilities = ["read"] } + {% endif %} + {% endif %} + headers: {X-Vault-Token: "{{ token }}"} + ca_path: "{{ ca }}" + validate_certs: true + status_code: [200, 204] + timeout: 5 + loop: "{{ policies.results }}" + no_log: true + + - name: Read existing AppRole contracts + ansible.builtin.uri: + url: "https://vault:8200/v1/auth/approle/role/local-{{ item.0 }}{{ '-migration' if item.1 == 'migration' else '' }}" + headers: {X-Vault-Token: "{{ token }}"} + ca_path: "{{ ca }}" + validate_certs: true + status_code: [200, 404] + timeout: 5 + loop: "{{ role_pairs }}" + register: approles + no_log: true + + - name: Reject absent or broad adopted AppRoles + ansible.builtin.assert: + that: + - item.status in [200, 404] + - item.status != 200 or item.json.data.token_policies == ['local-' ~ item.item.0 ~ ('-migration' if item.item.1 == 'migration' else '')] + fail_msg: Missing or broad AppRole cannot be adopted + loop: "{{ approles.results }}" + no_log: true + + - name: Create only missing fresh AppRoles + ansible.builtin.uri: + url: "https://vault:8200/v1/auth/approle/role/local-{{ item.item.0 }}{{ '-migration' if item.item.1 == 'migration' else '' }}" + method: POST + body_format: json + body: + token_policies: ["local-{{ item.item.0 }}{{ '-migration' if item.item.1 == 'migration' else '' }}"] + secret_id_ttl: 1h + secret_id_num_uses: 0 + token_ttl: 20m + token_max_ttl: 4h + token_no_default_policy: true + headers: {X-Vault-Token: "{{ token }}"} + ca_path: "{{ ca }}" + validate_certs: true + status_code: [200, 204] + timeout: 5 + loop: "{{ approles.results }}" + when: item.status == 404 + no_log: true + + - name: Read scoped operator AppRole + ansible.builtin.uri: + url: https://vault:8200/v1/auth/approle/role/local-operator + headers: {X-Vault-Token: "{{ token }}"} + ca_path: "{{ ca }}" + validate_certs: true + status_code: [200, 404] + register: operator_approle + no_log: true + + - name: Require existing or explicitly fresh scoped operator AppRole + ansible.builtin.assert: + that: + - operator_approle.status == 200 or fresh_target | default(false) | bool + - operator_approle.status != 200 or operator_approle.json.data.token_policies == ['local-operator'] + fail_msg: Operator AppRole is missing or broader than approved + no_log: true + + - name: Create only missing fresh operator AppRole + ansible.builtin.uri: + url: https://vault:8200/v1/auth/approle/role/local-operator + method: POST + body_format: json + body: {token_policies: [local-operator], secret_id_ttl: 1h, secret_id_num_uses: 0, token_ttl: 20m, token_max_ttl: 4h, token_no_default_policy: true} + headers: {X-Vault-Token: "{{ token }}"} + ca_path: "{{ ca }}" + validate_certs: true + status_code: [200, 204] + when: operator_approle.status == 404 and fresh_target | default(false) | bool + no_log: true + + - name: Read current Identity OTP key without exposing it + ansible.builtin.uri: + url: https://vault:8200/v1/secret/data/local/identity/otp-keys + headers: {X-Vault-Token: "{{ token }}"} + ca_path: "{{ ca }}" + validate_certs: true + status_code: [200, 404] + timeout: 5 + register: otp_key + no_log: true + + - name: Reject missing adopted-state OTP key + ansible.builtin.assert: + that: + - otp_key.status == 200 or fresh_target | default(false) | bool + fail_msg: Adopted Identity OTP key is missing; do not silently replace it + + - name: Generate only a missing fresh OTP key + ansible.builtin.command: + argv: [openssl, rand, -base64, '48'] + register: fresh_otp_key + when: otp_key.status == 404 and fresh_target | default(false) | bool + no_log: true + + - name: Write missing fresh OTP key with KV create-only CAS + ansible.builtin.uri: + url: https://vault:8200/v1/secret/data/local/identity/otp-keys + method: POST + body_format: json + body: + options: {cas: 0} + data: {current: localInitial, keys: {localInitial: "{{ fresh_otp_key.stdout | trim }}"}} + headers: {X-Vault-Token: "{{ token }}"} + ca_path: "{{ ca }}" + validate_certs: true + status_code: 200 + timeout: 5 + when: otp_key.status == 404 and fresh_target | default(false) | bool + no_log: true + + - name: Report protected reconciliation result + ansible.builtin.debug: + msg: Vault configuration checked; existing connection passwords, Raft, audit and OTP key preserved + + - name: Read centralized Redis authentication + ansible.builtin.uri: + url: https://vault:8200/v1/secret/data/local/infrastructure/redis + headers: {X-Vault-Token: "{{ token }}"} + ca_path: "{{ ca }}" + validate_certs: true + status_code: [200, 404] + register: redis_secret + no_log: true + + - name: Seed only missing Redis authentication using KV create-only CAS + ansible.builtin.uri: + url: https://vault:8200/v1/secret/data/local/infrastructure/redis + method: POST + body_format: json + body: + options: {cas: 0} + data: {password: "{{ lookup('file', repo ~ '/.local/payguard/redis/password') | trim }}"} + headers: {X-Vault-Token: "{{ token }}"} + ca_path: "{{ ca }}" + validate_certs: true + status_code: 200 + when: redis_secret.status == 404 + no_log: true diff --git a/infrastructure/operations/recover-attempt.yml b/infrastructure/operations/recover-attempt.yml new file mode 100644 index 0000000..9d49c5d --- /dev/null +++ b/infrastructure/operations/recover-attempt.yml @@ -0,0 +1,116 @@ +--- +- name: Reconcile one interrupted Agent attempt + hosts: localhost + gather_facts: false + vars: + repo: "{{ lookup('env', 'PAYGUARD_REPO_ROOT') }}" + runtime_root: "{{ lookup('env', 'PAYGUARD_RUNTIME_ROOT') }}" + operator_uid: "{{ lookup('env', 'PAYGUARD_LOCAL_UID') | int }}" + operator_gid: "{{ lookup('env', 'PAYGUARD_LOCAL_GID') | int }}" + tasks: + - name: Validate recorded target + ansible.builtin.import_tasks: tasks/target.yml + + - name: Reject unsafe receipt selector + ansible.builtin.assert: + that: + - attempt_id | default('') is match('^payguard-(identity|wallet|loan|collateral)-[A-Za-z0-9_.-]+$') + fail_msg: Select a recorded Agent attempt ID + + - name: Read protected receipt + ansible.builtin.slurp: + src: "{{ repo }}/.local/payguard/operator/receipts/{{ attempt_id }}.json" + register: receipt_source + no_log: true + + - name: Parse and verify receipt scope + ansible.builtin.set_fact: + receipt: "{{ receipt_source.content | b64decode | from_json }}" + no_log: true + + - name: Validate a previously completed recovery record + ansible.builtin.assert: + that: + - receipt.attempt == attempt_id + - receipt.target == lookup('env','COMPOSE_PROJECT_NAME') + when: receipt.state == 'cleanup_verified' + + - name: Finish an already verified recovery without changing state + ansible.builtin.meta: end_play + when: receipt.state == 'cleanup_verified' + + - name: Reject unrelated or tampered attempt + ansible.builtin.assert: + that: + - receipt.attempt == attempt_id + - receipt.service in ['identity', 'wallet', 'loan', 'collateral'] + - receipt.target == lookup('env', 'COMPOSE_PROJECT_NAME') + - receipt.path == runtime_root ~ '/' ~ attempt_id + - receipt.state != 'cleanup_verified' + - lookup('env', 'COMPOSE_PROJECT_NAME') == (lookup('file', repo ~ '/.local/payguard/target.yml') | from_yaml).project + - receipt.container == 'payguard-agent-' ~ attempt_id + fail_msg: Receipt scope does not match this Compose project + + - name: Stop only the recorded Agent container + ansible.builtin.command: + argv: [docker, rm, -f, "{{ receipt.container }}"] + failed_when: false + changed_when: false + no_log: true + + - name: Stop only the recorded Flyway container + ansible.builtin.command: + argv: [docker, rm, -f, "payguard-flyway-{{ attempt_id }}"] + failed_when: false + changed_when: false + no_log: true + + - name: Check for the private attempt token + ansible.builtin.stat: + path: "{{ receipt.path }}/{{ receipt.service }}/private/token" + register: private_token + no_log: true + + - name: Revoke this attempt token and child database leases + ansible.builtin.uri: + url: https://vault:8200/v1/auth/token/revoke-self + method: POST + ca_path: "{{ repo }}/.local/payguard/ca.crt" + validate_certs: true + headers: + X-Vault-Token: "{{ lookup('file', receipt.path ~ '/' ~ receipt.service ~ '/private/token') | trim }}" + status_code: [200, 204] + timeout: 5 + register: revoked + ignore_errors: true + when: private_token.stat.exists + no_log: true + + - name: Fail closed when revocation cannot be proved + ansible.builtin.fail: + msg: "cleanup_unknown for {{ attempt_id }}; inspect protected receipt, token expiry and database sessions before reuse" + when: not private_token.stat.exists or revoked is failed + + - name: Remove only the recorded tmpfs attempt + ansible.builtin.file: + path: "{{ receipt.path }}" + state: absent + no_log: true + + - name: Release only this service lock after confirmed cleanup + ansible.builtin.file: + path: "{{ runtime_root }}/migration-{{ receipt.service }}.lock" + state: absent + + - name: Retain sanitized completion evidence without protected identifiers + ansible.builtin.copy: + dest: "{{ repo }}/.local/payguard/operator/receipts/{{ attempt_id }}.json" + content: "{{ {'attempt': attempt_id, 'service': receipt.service, 'target': receipt.target, 'state': 'cleanup_verified'} | to_json }}\n" + owner: "{{ operator_uid }}" + group: "{{ operator_gid }}" + mode: '0600' + no_log: true + + - name: Report sanitized recovery result + ansible.builtin.debug: + msg: "cleanup_verified service={{ receipt.service }} attempt={{ attempt_id }}" diff --git a/infrastructure/operations/refresh-identities.service b/infrastructure/operations/refresh-identities.service new file mode 100644 index 0000000..ae7173e --- /dev/null +++ b/infrastructure/operations/refresh-identities.service @@ -0,0 +1,7 @@ +[Unit] +Description=Refresh PayGuard scoped Vault AppRole files + +[Service] +Type=oneshot +EnvironmentFile=%h/.config/payguard/operations.env +ExecStart=/usr/bin/bash -lc 'cd "$PAYGUARD_REPO_ROOT" && docker compose --profile operations -f docker-compose.yaml -f infrastructure/compose.operations.yaml run --rm --no-deps operations infrastructure/operations/refresh-identities.yml' diff --git a/infrastructure/operations/refresh-identities.timer b/infrastructure/operations/refresh-identities.timer new file mode 100644 index 0000000..7299272 --- /dev/null +++ b/infrastructure/operations/refresh-identities.timer @@ -0,0 +1,11 @@ +[Unit] +Description=Refresh PayGuard scoped Vault AppRoles every 20 minutes + +[Timer] +OnBootSec=2min +OnUnitActiveSec=20min +Unit=refresh-identities.service +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/infrastructure/operations/refresh-identities.yml b/infrastructure/operations/refresh-identities.yml new file mode 100644 index 0000000..0a666df --- /dev/null +++ b/infrastructure/operations/refresh-identities.yml @@ -0,0 +1,105 @@ +--- +- name: Refresh existing scoped AppRole identities without broad revocation + hosts: localhost + gather_facts: false + vars: + repo: "{{ lookup('env', 'PAYGUARD_REPO_ROOT') }}" + ca: "{{ repo }}/.local/payguard/ca.crt" + operator_token: "{{ lookup('file', repo ~ '/.local/payguard/operator/token') | trim }}" + identities: + - {role: local-identity, path: apps/identity} + - {role: local-loan, path: apps/loan} + - {role: local-collateral, path: apps/collateral} + - {role: local-wallet, path: apps/wallet} + - {role: local-identity-migration, path: operator/identity} + - {role: local-loan-migration, path: operator/loan} + - {role: local-collateral-migration, path: operator/collateral} + - {role: local-wallet-migration, path: operator/wallet} + - {role: local-operator, path: operator/approle} + tasks: + - name: Validate recorded target + ansible.builtin.import_tasks: tasks/target.yml + + - name: Pin approved project and require protected operator token + ansible.builtin.assert: + that: + - lookup('env', 'COMPOSE_PROJECT_NAME') == (lookup('file', repo ~ '/.local/payguard/target.yml') | from_yaml).project + - operator_token | length > 0 + fail_msg: Scoped refresh requires the approved project and operator token + no_log: true + + - name: Read existing scoped role IDs + ansible.builtin.uri: + url: "https://vault:8200/v1/auth/approle/role/{{ item.role }}/role-id" + headers: {X-Vault-Token: "{{ operator_token }}"} + ca_path: "{{ ca }}" + validate_certs: true + status_code: 200 + timeout: 5 + loop: "{{ identities }}" + register: role_ids + no_log: true + + - name: Issue a 60-second response-wrapped SecretID for each existing role + ansible.builtin.uri: + url: "https://vault:8200/v1/auth/approle/role/{{ item.role }}/secret-id" + method: POST + body_format: json + body: {} + headers: + X-Vault-Token: "{{ operator_token }}" + X-Vault-Wrap-TTL: 60s + ca_path: "{{ ca }}" + validate_certs: true + status_code: 200 + timeout: 5 + loop: "{{ identities }}" + register: wrapped + no_log: true + + - name: Unwrap each SecretID exactly once + ansible.builtin.uri: + url: https://vault:8200/v1/sys/wrapping/unwrap + method: POST + headers: {X-Vault-Token: "{{ item.json.wrap_info.token }}"} + ca_path: "{{ ca }}" + validate_certs: true + status_code: 200 + timeout: 5 + loop: "{{ wrapped.results }}" + register: unwrapped + no_log: true + + - name: Validate reusable one-hour SecretID contract before replacing files + ansible.builtin.assert: + that: + - item.json.data.secret_id | length > 0 + - item.json.data.secret_id_ttl | int == 3600 + - item.json.data.secret_id_num_uses | int == 0 + fail_msg: Wrapped identity does not match the approved one-hour contract + loop: "{{ unwrapped.results }}" + no_log: true + + - name: Replace role ID files atomically + ansible.builtin.copy: + dest: "{{ repo }}/.local/payguard/{{ item.path }}/role-id" + content: "{{ role_ids.results[index].json.data.role_id }}\n" + mode: '0600' + loop: "{{ identities }}" + loop_control: + index_var: index + no_log: true + + - name: Replace SecretID files atomically after all unwraps pass + ansible.builtin.copy: + dest: "{{ repo }}/.local/payguard/{{ item.path }}/secret-id" + content: "{{ unwrapped.results[index].json.data.secret_id }}\n" + mode: '0600' + loop: "{{ identities }}" + loop_control: + index_var: index + no_log: true + + - name: Report scoped refresh without identifiers + ansible.builtin.debug: + msg: "scoped_identity_refresh=complete roles={{ identities | length }}" diff --git a/infrastructure/operations/requirements.txt b/infrastructure/operations/requirements.txt new file mode 100644 index 0000000..ca48f97 --- /dev/null +++ b/infrastructure/operations/requirements.txt @@ -0,0 +1,12 @@ +# Runtime dependencies, including the transitive versions used for local Ansible validation. +ansible-core==2.19.3 +cffi==2.1.1 +cryptography==50.0.2 +Jinja2==3.1.6 +MarkupSafe==3.0.4 +packaging==26.3 +psycopg[binary]==3.2.10 +pycparser==3.12 +PyYAML==6.0.3 +resolvelib==1.2.1 +typing_extensions==4.16.0 diff --git a/infrastructure/operations/requirements.yml b/infrastructure/operations/requirements.yml new file mode 100644 index 0000000..b3ea7ef --- /dev/null +++ b/infrastructure/operations/requirements.yml @@ -0,0 +1,3 @@ +collections: + - name: community.postgresql + version: 4.2.0 diff --git a/infrastructure/operations/restore-postgresql.yml b/infrastructure/operations/restore-postgresql.yml new file mode 100644 index 0000000..848c7f4 --- /dev/null +++ b/infrastructure/operations/restore-postgresql.yml @@ -0,0 +1,76 @@ +--- +- name: Restore a database backup into an empty isolated recovery target + hosts: localhost + gather_facts: false + vars: + repo: "{{ lookup('env','PAYGUARD_REPO_ROOT') }}" + tasks: + - name: Validate recorded target + ansible.builtin.import_tasks: tasks/target.yml + - name: Require an isolated recovery project and an exact expected backup hash + ansible.builtin.assert: + that: + - lookup('env','COMPOSE_PROJECT_NAME') is match('^payguard-recovery-[a-z0-9-]+$') + - service in ['identity','wallet','loan','collateral'] + - backup_sha256 is match('^[a-f0-9]{64}$') + - backup_file is match('^/[^\n]+$') + - name: Verify backup bytes before restoration + ansible.builtin.stat: + path: "{{ backup_file }}" + checksum_algorithm: sha256 + register: backup + no_log: true + - name: Refuse changed backup bytes + ansible.builtin.assert: + that: + - backup.stat.isreg + - backup.stat.checksum == backup_sha256 + no_log: true + - name: Require an empty destination schema + community.postgresql.postgresql_query: + login_host: "postgres-{{ service }}" + login_db: "payguard_{{ service }}" + login_user: postgres + login_password: "{{ lookup('file',repo ~ '/.local/payguard/' ~ service ~ '/admin-password') | trim }}" + ssl_mode: verify-full + ca_cert: "{{ repo }}/.local/payguard/ca.crt" + query: SELECT count(*)::int AS objects FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace WHERE n.nspname='payguard' + register: destination + no_log: true + - name: Refuse replacing any populated destination + ansible.builtin.assert: + that: destination.query_result[0].objects == 0 + no_log: true + - name: Restore under stable owner and always remove temporary schema-creation permission + block: + - name: Temporarily permit native schema restoration + community.postgresql.postgresql_query: + login_host: "postgres-{{ service }}" + login_db: "payguard_{{ service }}" + login_user: postgres + login_password: "{{ lookup('file',repo ~ '/.local/payguard/' ~ service ~ '/admin-password') | trim }}" + ssl_mode: verify-full + ca_cert: "{{ repo }}/.local/payguard/ca.crt" + query: "GRANT CREATE ON DATABASE payguard_{{ service }} TO {{ service }}_owner; DROP SCHEMA payguard" + no_log: true + - name: Restore backup atomically using native pg_restore + ansible.builtin.shell: >- + docker compose -f {{ (repo ~ '/docker-compose.yaml') | quote }} + exec -T -u postgres postgres-{{ service }} + pg_restore -U postgres -d payguard_{{ service }} --single-transaction + --no-owner --role={{ service }}_owner + < {{ backup_file | quote }} + no_log: true + always: + - name: Remove the temporary database permission + community.postgresql.postgresql_query: + login_host: "postgres-{{ service }}" + login_db: "payguard_{{ service }}" + login_user: postgres + login_password: "{{ lookup('file',repo ~ '/.local/payguard/' ~ service ~ '/admin-password') | trim }}" + ssl_mode: verify-full + ca_cert: "{{ repo }}/.local/payguard/ca.crt" + query: "REVOKE CREATE ON DATABASE payguard_{{ service }} FROM {{ service }}_owner" + no_log: true + - name: Verify restored history and stable ownership + ansible.builtin.include_tasks: tasks/verify-migration.yml diff --git a/infrastructure/operations/seed-kafka.yml b/infrastructure/operations/seed-kafka.yml new file mode 100644 index 0000000..6cd9305 --- /dev/null +++ b/infrastructure/operations/seed-kafka.yml @@ -0,0 +1,49 @@ +--- +- name: Import externally provisioned per-service Kafka credentials once + hosts: localhost + gather_facts: false + vars: + repo: "{{ lookup('env','PAYGUARD_REPO_ROOT') }}" + services: [identity, wallet, loan, collateral] + tasks: + - name: Validate recorded target + ansible.builtin.import_tasks: tasks/target.yml + - name: Require a protected credential file + ansible.builtin.stat: + path: "{{ kafka_credentials_file }}" + register: credentials + no_log: true + - name: Refuse exposed Kafka input + ansible.builtin.assert: + that: + - credentials.stat.isreg + - credentials.stat.mode == '0600' + no_log: true + - name: Read current per-service Kafka secrets + ansible.builtin.uri: + url: "https://vault:8200/v1/secret/data/local/{{ item }}/kafka" + headers: + X-Vault-Token: "{{ lookup('file',repo ~ '/.local/payguard/operator/token') | trim }}" + ca_path: "{{ repo }}/.local/payguard/ca.crt" + validate_certs: true + status_code: [200, 404] + loop: "{{ services }}" + register: kafka_secrets + no_log: true + - name: Create only missing Kafka secrets with CAS and preserve existing versions + ansible.builtin.uri: + url: "https://vault:8200/v1/secret/data/local/{{ item.item }}/kafka" + method: POST + body_format: json + body: + options: {cas: 0} + data: + jaas_config: "{{ (lookup('file',kafka_credentials_file) | from_json)[item.item].jaas_config }}" + headers: + X-Vault-Token: "{{ lookup('file',repo ~ '/.local/payguard/operator/token') | trim }}" + ca_path: "{{ repo }}/.local/payguard/ca.crt" + validate_certs: true + status_code: 200 + loop: "{{ kafka_secrets.results }}" + when: item.status == 404 + no_log: true diff --git a/infrastructure/operations/site.yml b/infrastructure/operations/site.yml new file mode 100644 index 0000000..7d03d7e --- /dev/null +++ b/infrastructure/operations/site.yml @@ -0,0 +1,6 @@ +--- +# Existing-state reconciliation only. Initialization and migration are separate explicit actions. +- ansible.builtin.import_playbook: adopt-artifacts.yml +- ansible.builtin.import_playbook: provision-database.yml +- ansible.builtin.import_playbook: provision-vault.yml +- ansible.builtin.import_playbook: refresh-identities.yml diff --git a/infrastructure/operations/tasks/target.yml b/infrastructure/operations/tasks/target.yml new file mode 100644 index 0000000..686a68a --- /dev/null +++ b/infrastructure/operations/tasks/target.yml @@ -0,0 +1,31 @@ +--- +- name: Require an independently recorded local target + ansible.builtin.assert: + that: + - repo is match('^/[^\n]+$') + - lookup('env','COMPOSE_PROJECT_NAME') == (lookup('file', repo ~ '/.local/payguard/target.yml') | from_yaml).project + - lookup('env','DOCKER_HOST') in ['', 'unix:///var/run/docker.sock'] + - lookup('env','DOCKER_CONTEXT') in ['', 'default'] + fail_msg: Repository, project or Docker endpoint differs from the recorded local target +- name: Read daemon identity before any target mutation + ansible.builtin.command: + argv: [docker, info, --format, "{{ '{{.ID}}' }}"] + register: current_daemon + changed_when: false +- name: Pin target daemon and repository + ansible.builtin.assert: + that: + - current_daemon.stdout == (lookup('file', repo ~ '/.local/payguard/target.yml') | from_yaml).daemon_id + - repo == (lookup('file', repo ~ '/.local/payguard/target.yml') | from_yaml).repository + fail_msg: Target daemon or repository has changed +- name: Resolve current persistent volume mapping + ansible.builtin.command: + argv: [docker, compose, -f, "{{ repo }}/docker-compose.yaml", config, --format, json] + register: target_compose + changed_when: false + no_log: true +- name: Refuse changed volume identities + ansible.builtin.assert: + that: + - (target_compose.stdout | from_json).volumes | dict2items | map(attribute='value.name') | list == (lookup('file', repo ~ '/.local/payguard/target.yml') | from_yaml).volumes + fail_msg: Compose volumes differ from the recorded target diff --git a/infrastructure/operations/tasks/verify-migration.yml b/infrastructure/operations/tasks/verify-migration.yml new file mode 100644 index 0000000..b5f41d2 --- /dev/null +++ b/infrastructure/operations/tasks/verify-migration.yml @@ -0,0 +1,28 @@ +--- +- name: Verify stable ownership and prevent runtime access to migration history + community.postgresql.postgresql_query: + login_host: "postgres-{{ service }}" + login_db: "payguard_{{ service }}" + login_user: postgres + login_password: "{{ lookup('file', repo ~ '/.local/payguard/' ~ service ~ '/admin-password') | trim }}" + ssl_mode: verify-full + ca_cert: "{{ repo }}/.local/payguard/ca.crt" + query: | + SELECT + EXISTS (SELECT 1 FROM payguard.flyway_schema_history WHERE success) AS migrated, + has_table_privilege('{{ service }}_runtime','payguard.flyway_schema_history','SELECT') AS history_read, + has_table_privilege('{{ service }}_runtime','payguard.flyway_schema_history','INSERT') AS history_write, + (SELECT count(*) FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace + WHERE n.nspname='payguard' AND pg_get_userbyid(c.relowner)<>'{{ service }}_owner') AS wrong_owner + register: migration_verified + no_log: true + +- name: Reject incomplete migrations or unsafe object privileges + ansible.builtin.assert: + that: + - migration_verified.query_result[0].migrated + - not migration_verified.query_result[0].history_read + - not migration_verified.query_result[0].history_write + - migration_verified.query_result[0].wrong_owner | int == 0 + fail_msg: Migration history, grants or stable ownership verification failed + no_log: true diff --git a/infrastructure/operations/tests/agent.yml b/infrastructure/operations/tests/agent.yml new file mode 100644 index 0000000..caed228 --- /dev/null +++ b/infrastructure/operations/tests/agent.yml @@ -0,0 +1,147 @@ +--- +- name: Verify scoped Agent security without changing Vault configuration + hosts: localhost + gather_facts: false + vars: + repo: "{{ lookup('env', 'PAYGUARD_REPO_ROOT') }}" + ca: "{{ repo }}/.local/payguard/ca.crt" + tasks: + - name: Validate recorded target + ansible.builtin.import_tasks: ../tasks/target.yml + - name: Require the approved project + ansible.builtin.assert: + that: + - lookup('env', 'COMPOSE_PROJECT_NAME') == (lookup('file', repo ~ '/.local/payguard/target.yml') | from_yaml).project + + - name: Restrict authentication failure probes to an actual isolated project + ansible.builtin.assert: + that: lookup('env','COMPOSE_PROJECT_NAME') is match('^payguard-it-[a-z0-9-]+$') + when: isolated_target | default(false) | bool + + - name: Reject an unrelated CA before HTTP + ansible.builtin.uri: + url: https://vault:8200/v1/sys/health + ca_path: /etc/ssl/cert.pem + validate_certs: true + timeout: 5 + register: wrong_ca + ignore_errors: true + no_log: true + + - name: Reject a wrong Vault hostname before HTTP + ansible.builtin.uri: + url: https://vault-invalid-san:8200/v1/sys/health + ca_path: "{{ ca }}" + validate_certs: true + timeout: 5 + register: wrong_san + ignore_errors: true + no_log: true + + - name: Require both TLS refusals + ansible.builtin.assert: + that: + - wrong_ca is failed + - wrong_san is failed + - "'certificate' in (wrong_ca.msg | lower)" + - "'certificate' in (wrong_san.msg | lower)" + fail_msg: Vault TLS accepted an unrelated CA or hostname + + - name: Log in once with the existing Identity migration AppRole + ansible.builtin.uri: + url: https://vault:8200/v1/auth/approle/login + method: POST + body_format: json + body: + role_id: "{{ lookup('file', repo ~ '/.local/payguard/operator/identity/role-id') | trim }}" + secret_id: "{{ lookup('file', repo ~ '/.local/payguard/operator/identity/secret-id') | trim }}" + ca_path: "{{ ca }}" + validate_certs: true + status_code: 200 + timeout: 5 + register: identity_login + no_log: true + + - name: Probe cross-service denial and always revoke the probe token + block: + - name: Deny cross-service migration credential read + ansible.builtin.uri: + url: https://vault:8200/v1/database/creds/local-loan-migration + headers: {X-Vault-Token: "{{ identity_login.json.auth.client_token }}"} + ca_path: "{{ ca }}" + validate_certs: true + status_code: 403 + timeout: 5 + register: denied_cross_service + no_log: true + always: + - name: Revoke only this test token + ansible.builtin.uri: + url: https://vault:8200/v1/auth/token/revoke-self + method: POST + headers: {X-Vault-Token: "{{ identity_login.json.auth.client_token }}"} + ca_path: "{{ ca }}" + validate_certs: true + status_code: [200, 204] + timeout: 5 + no_log: true + + - name: Report only sanitized security checks + ansible.builtin.debug: + msg: wrong_ca=denied wrong_san=denied cross_service=denied token=revoked + + - name: Reject missing SecretID only on an isolated target + ansible.builtin.uri: + url: https://vault:8200/v1/auth/approle/login + method: POST + body_format: json + body: + role_id: "{{ lookup('file', repo ~ '/.local/payguard/operator/identity/role-id') | trim }}" + secret_id: 00000000-0000-0000-0000-000000000000 + ca_path: "{{ ca }}" + validate_certs: true + status_code: [400, 403] + timeout: 5 + when: isolated_target | default(false) | bool + no_log: true + + - name: Issue an isolated one-second SecretID for expiry testing + ansible.builtin.uri: + url: https://vault:8200/v1/auth/approle/role/local-identity-migration/secret-id + method: POST + body_format: json + body: {ttl: 1s} + headers: + X-Vault-Token: "{{ lookup('file', repo ~ '/.local/payguard/operator/token') | trim }}" + ca_path: "{{ ca }}" + validate_certs: true + status_code: 200 + timeout: 5 + register: expiring_secret + when: isolated_target | default(false) | bool + no_log: true + + - name: Wait only for the isolated test SecretID to expire + ansible.builtin.pause: + seconds: 2 + when: isolated_target | default(false) | bool + + - name: Reject the expired isolated SecretID + ansible.builtin.uri: + url: https://vault:8200/v1/auth/approle/login + method: POST + body_format: json + body: + role_id: "{{ lookup('file', repo ~ '/.local/payguard/operator/identity/role-id') | trim }}" + secret_id: "{{ expiring_secret.json.data.secret_id }}" + ca_path: "{{ ca }}" + validate_certs: true + status_code: [400, 403] + timeout: 5 + when: isolated_target | default(false) | bool + no_log: true + + - name: Report isolated authentication refusals + ansible.builtin.debug: + msg: missing_secret=denied expired_secret=denied + when: isolated_target | default(false) | bool diff --git a/infrastructure/operations/tests/applications.yml b/infrastructure/operations/tests/applications.yml new file mode 100644 index 0000000..f1741ad --- /dev/null +++ b/infrastructure/operations/tests/applications.yml @@ -0,0 +1,35 @@ +--- +- name: Verify actual packaged application readiness + hosts: localhost + gather_facts: false + vars: + repo: "{{ lookup('env','PAYGUARD_REPO_ROOT') }}" + services: [identity, wallet, loan, collateral] + tasks: + - name: Validate recorded target + ansible.builtin.import_tasks: ../tasks/target.yml + - name: Wait for every application readiness group + ansible.builtin.uri: + url: "http://{{ item }}-service:{{ '8084' if item == 'identity' else '8080' }}/actuator/health/readiness" + status_code: 200 + return_content: true + timeout: 3 + register: readiness + until: readiness.status == 200 and readiness.json.status == 'UP' + retries: 30 + delay: 2 + loop: "{{ services }}" + no_log: true + - name: Confirm Compose never receives injected database credentials + ansible.builtin.command: + argv: [docker, compose, -f, "{{ repo }}/docker-compose.yaml", --profile, applications, config, --format, json] + register: config + changed_when: false + no_log: true + - name: Require credentials to remain Agent child-process values + ansible.builtin.assert: + that: + - "'SPRING_DATASOURCE_PASSWORD' not in (config.stdout | from_json).services[item ~ '-service'].environment" + - "'SPRING_DATASOURCE_USERNAME' not in (config.stdout | from_json).services[item ~ '-service'].environment" + loop: "{{ services }}" + no_log: true diff --git a/infrastructure/operations/tests/integration.yml b/infrastructure/operations/tests/integration.yml new file mode 100644 index 0000000..04289c4 --- /dev/null +++ b/infrastructure/operations/tests/integration.yml @@ -0,0 +1,62 @@ +--- +- name: Verify real database ownership, histories and runtime privilege boundaries + hosts: localhost + gather_facts: false + vars: + repo: "{{ lookup('env','PAYGUARD_REPO_ROOT') }}" + services: [identity, wallet, loan, collateral] + tasks: + - name: Validate recorded target + ansible.builtin.import_tasks: ../tasks/target.yml + - name: Check every independently migrated database + ansible.builtin.include_tasks: ../tasks/verify-migration.yml + loop: "{{ services }}" + loop_control: + loop_var: service + - name: Reject runtime DDL and role escalation + community.postgresql.postgresql_query: + login_host: "postgres-{{ item }}" + login_db: "payguard_{{ item }}" + login_user: postgres + login_password: "{{ lookup('file',repo ~ '/.local/payguard/' ~ item ~ '/admin-password') | trim }}" + ssl_mode: verify-full + ca_cert: "{{ repo }}/.local/payguard/ca.crt" + query: | + SELECT has_schema_privilege('{{ item }}_runtime','payguard','CREATE') AS ddl, + has_database_privilege('{{ item }}_runtime',current_database(),'CREATE') AS create_schema, + pg_has_role('{{ item }}_runtime','{{ item }}_owner','MEMBER') AS owner_member + register: runtime_privileges + loop: "{{ services }}" + no_log: true + - name: Assert runtime cannot perform DDL or select owner role + ansible.builtin.assert: + that: + - not item.query_result[0].ddl + - not item.query_result[0].create_schema + - not item.query_result[0].owner_member + loop: "{{ runtime_privileges.results }}" + no_log: true + - name: Confirm application tables have runtime data privileges + community.postgresql.postgresql_query: + login_host: "postgres-{{ item }}" + login_db: "payguard_{{ item }}" + login_user: postgres + login_password: "{{ lookup('file',repo ~ '/.local/payguard/' ~ item ~ '/admin-password') | trim }}" + ssl_mode: verify-full + ca_cert: "{{ repo }}/.local/payguard/ca.crt" + query: | + SELECT count(*)::int AS missing FROM pg_tables + WHERE schemaname='payguard' AND tablename<>'flyway_schema_history' + AND tablename NOT LIKE '%_audit_log' AND tablename <> 'identity_challenge_scope' + AND NOT (has_table_privilege('{{ item }}_runtime',format('payguard.%I',tablename),'SELECT') + AND has_table_privilege('{{ item }}_runtime',format('payguard.%I',tablename),'INSERT') + AND has_table_privilege('{{ item }}_runtime',format('payguard.%I',tablename),'UPDATE') + AND has_table_privilege('{{ item }}_runtime',format('payguard.%I',tablename),'DELETE')) + register: table_privileges + loop: "{{ services }}" + no_log: true + - name: Require application data access + ansible.builtin.assert: + that: item.query_result[0].missing == 0 + loop: "{{ table_privileges.results }}" + no_log: true diff --git a/infrastructure/operations/tests/recovery.yml b/infrastructure/operations/tests/recovery.yml new file mode 100644 index 0000000..4224148 --- /dev/null +++ b/infrastructure/operations/tests/recovery.yml @@ -0,0 +1,51 @@ +--- +- name: Test a real Vault outage and non-destructive recovery on an isolated fixture + hosts: localhost + gather_facts: false + vars: + repo: "{{ lookup('env','PAYGUARD_REPO_ROOT') }}" + compose: [docker, compose, -f, "{{ repo }}/docker-compose.yaml"] + tasks: + - name: Validate recorded target + ansible.builtin.import_tasks: ../tasks/target.yml + - name: Restrict disruption to an isolated test project + ansible.builtin.assert: + that: lookup('env','COMPOSE_PROJECT_NAME') is match('^payguard-it-[a-z0-9-]+$') + - name: Exercise dependency loss and always restore the Vault process + block: + - name: Stop Vault without removing its Raft volume + ansible.builtin.command: + argv: "{{ compose + ['stop', 'vault'] }}" + - name: Expect verified-TLS health to become unavailable + ansible.builtin.uri: + url: https://vault:8200/v1/sys/health + ca_path: "{{ repo }}/.local/payguard/ca.crt" + validate_certs: true + timeout: 3 + register: unavailable + ignore_errors: true + no_log: true + - name: Require an actual outage + ansible.builtin.assert: + that: unavailable is failed + always: + - name: Start the existing Vault container + ansible.builtin.command: + argv: "{{ compose + ['start','vault'] }}" + - name: Wait for the existing seal-state endpoint + ansible.builtin.uri: + url: https://vault:8200/v1/sys/seal-status + ca_path: "{{ repo }}/.local/payguard/ca.crt" + validate_certs: true + status_code: 200 + timeout: 3 + register: seal_status + until: seal_status is succeeded + retries: 10 + delay: 2 + no_log: true +- ansible.builtin.import_playbook: ../vault-lifecycle.yml + vars: + operation: unseal + local_rehearsal: true +- ansible.builtin.import_playbook: integration.yml diff --git a/infrastructure/operations/vault-lifecycle.yml b/infrastructure/operations/vault-lifecycle.yml new file mode 100644 index 0000000..80b216b --- /dev/null +++ b/infrastructure/operations/vault-lifecycle.yml @@ -0,0 +1,217 @@ +--- +- name: Explicit Vault first initialization, unseal, or scoped operator recovery + hosts: localhost + gather_facts: false + vars: + repo: "{{ lookup('env', 'PAYGUARD_REPO_ROOT') }}" + ca: "{{ repo }}/.local/payguard/ca.crt" + operator_dir: "{{ repo }}/.local/payguard/operator" + tasks: + - name: Validate recorded target + ansible.builtin.import_tasks: tasks/target.yml + + - name: Require one explicit ceremony and the approved project + ansible.builtin.assert: + that: + - operation | default('') in ['initialize', 'unseal', 'recover-operator'] + - lookup('env', 'COMPOSE_PROJECT_NAME') == (lookup('file', repo ~ '/.local/payguard/target.yml') | from_yaml).project + fail_msg: Select an explicit Vault ceremony on the approved project + + - name: Read Vault initialization state over verified TLS + ansible.builtin.uri: + url: https://vault:8200/v1/sys/init + ca_path: "{{ ca }}" + validate_certs: true + status_code: 200 + timeout: 5 + register: init_state + until: init_state.status == 200 + retries: 10 + delay: 2 + changed_when: false + no_log: true + + - name: Refuse initialization of persisted Raft state + ansible.builtin.assert: + that: + - not init_state.json.initialized + - fresh_target | default(false) | bool + fail_msg: Existing Vault data cannot be initialized again + when: operation == 'initialize' + + - name: Refuse unseal or recovery before first initialization + ansible.builtin.assert: + that: + - init_state.json.initialized + fail_msg: Vault has no initialized state + when: operation != 'initialize' + + - name: Require separated PGP custodians for production initialization + ansible.builtin.assert: + that: + - (recovery_pgp_keys | default([]) | length) == 3 + - (root_token_pgp_key | default('') | length) > 0 + fail_msg: Production initialization requires three custodian PGP keys and a root-token PGP key; local_rehearsal is for disposable fixtures only + when: operation == 'initialize' and not (local_rehearsal | default(false) | bool) + no_log: true + + - name: Initialize only a new Vault with the approved threshold + ansible.builtin.uri: + url: https://vault:8200/v1/sys/init + method: POST + body_format: json + body: + secret_shares: 3 + secret_threshold: 2 + pgp_keys: "{{ recovery_pgp_keys | default(omit) }}" + root_token_pgp_key: "{{ root_token_pgp_key | default(omit) }}" + ca_path: "{{ ca }}" + validate_certs: true + status_code: 200 + timeout: 15 + register: initialized + when: operation == 'initialize' + no_log: true + + - name: Persist fresh recovery shares only in the protected operator directory + ansible.builtin.copy: + dest: "{{ operator_dir }}/initialization.json" + content: "{{ {'keys': initialized.json['keys'], 'keys_base64': initialized.json['keys_base64'], 'secret_threshold': 2, 'encrypted_root_token': (initialized.json.root_token if not (local_rehearsal | default(false) | bool) else '')} | to_json }}\n" + mode: '0600' + when: operation == 'initialize' + no_log: true + + - name: Read current seal state + ansible.builtin.uri: + url: https://vault:8200/v1/sys/seal-status + ca_path: "{{ ca }}" + validate_certs: true + status_code: 200 + timeout: 5 + register: seal_state + changed_when: false + no_log: true + + - name: Require protected custodian inputs for production unseal + ansible.builtin.assert: + that: (unseal_key_files | default([]) | length) >= 2 + fail_msg: Supply two separate protected custodian key files; automatic fixture-share loading is disabled for production + when: operation == 'unseal' and not (local_rehearsal | default(false) | bool) + no_log: true + + - name: Unseal explicitly with the existing protected threshold shares + ansible.builtin.uri: + url: https://vault:8200/v1/sys/unseal + method: POST + body_format: json + body: + key: "{{ lookup('file', unseal_key_files[item]) | trim if operation == 'unseal' and not (local_rehearsal | default(false) | bool) else (initialized.json['keys'] if operation == 'initialize' else (lookup('file', operator_dir ~ '/initialization.json') | from_json)['keys'])[item] }}" + ca_path: "{{ ca }}" + validate_certs: true + status_code: 200 + timeout: 5 + loop: [0, 1] + when: (operation == 'unseal' or (operation == 'initialize' and local_rehearsal | default(false) | bool)) and seal_state.json.sealed + no_log: true + + - name: Require a protected temporary root token for recovery + ansible.builtin.stat: + path: "{{ operator_dir }}/bootstrap-root" + register: recovery_root + when: operation == 'recover-operator' + no_log: true + + - name: Refuse recovery without a protected native generate-root result + ansible.builtin.assert: + that: + - recovery_root.stat.exists + - recovery_root.stat.mode == '0600' + fail_msg: Complete the native Vault threshold generate-root ceremony and protect bootstrap-root before recovery + when: operation == 'recover-operator' + + - name: Select only the temporary root for scoped operator issuance + ansible.builtin.set_fact: + temporary_root: "{{ initialized.json.root_token if operation == 'initialize' else lookup('file', operator_dir ~ '/bootstrap-root') | trim }}" + when: operation == 'recover-operator' or (operation == 'initialize' and local_rehearsal | default(false) | bool) + no_log: true + + - name: Issue scoped operator authority and always revoke temporary root + block: + - name: Install the scoped operator policy under temporary root authority + ansible.builtin.uri: + url: https://vault:8200/v1/sys/policies/acl/local-operator + method: POST + body_format: json + body: + policy: | + path "auth/token/lookup-self" { capabilities = ["read"] } + path "auth/token/renew-self" { capabilities = ["update"] } + path "auth/token/revoke-self" { capabilities = ["update"] } + path "sys/storage/raft/snapshot" { capabilities = ["read", "sudo"] } + path "sys/mounts" { capabilities = ["read", "sudo"] } + path "sys/auth" { capabilities = ["read", "sudo"] } + path "sys/audit" { capabilities = ["read", "sudo"] } + path "sys/mounts/*" { capabilities = ["create", "read", "update", "sudo"] } + path "sys/auth/approle" { capabilities = ["create", "read", "update", "sudo"] } + path "sys/audit/file" { capabilities = ["create", "read", "update", "sudo"] } + path "sys/policies/acl/local-*" { capabilities = ["create", "read", "update", "sudo"] } + path "database/config/local-*" { capabilities = ["create", "read", "update", "list"] } + path "database/roles/local-*" { capabilities = ["create", "read", "update", "list"] } + path "database/rotate-root/local-*" { capabilities = ["update"] } + path "auth/approle/role/local-*" { capabilities = ["create", "read", "update", "list"] } + path "secret/data/local/*" { capabilities = ["create", "read", "update", "list"] } + headers: {X-Vault-Token: "{{ temporary_root }}"} + ca_path: "{{ ca }}" + validate_certs: true + status_code: [200, 204] + timeout: 5 + when: operation == 'recover-operator' or (operation == 'initialize' and local_rehearsal | default(false) | bool) + no_log: true + + - name: Issue a scoped renewable operator token + ansible.builtin.uri: + url: https://vault:8200/v1/auth/token/create-orphan + method: POST + body_format: json + body: {policies: [local-operator], ttl: 8h, renewable: true} + headers: {X-Vault-Token: "{{ temporary_root }}"} + ca_path: "{{ ca }}" + validate_certs: true + status_code: 200 + timeout: 5 + register: scoped_operator + when: operation == 'recover-operator' or (operation == 'initialize' and local_rehearsal | default(false) | bool) + no_log: true + + - name: Store only the scoped operator token + ansible.builtin.copy: + dest: "{{ operator_dir }}/token" + content: "{{ scoped_operator.json.auth.client_token }}\n" + mode: '0600' + when: operation == 'recover-operator' or (operation == 'initialize' and local_rehearsal | default(false) | bool) + no_log: true + + always: + - name: Revoke the temporary root token + ansible.builtin.uri: + url: https://vault:8200/v1/auth/token/revoke-self + method: POST + headers: {X-Vault-Token: "{{ temporary_root }}"} + ca_path: "{{ ca }}" + validate_certs: true + status_code: [200, 204] + timeout: 5 + when: operation == 'recover-operator' or (operation == 'initialize' and local_rehearsal | default(false) | bool) + no_log: true + + - name: Remove recovered temporary root file after revocation + ansible.builtin.file: + path: "{{ operator_dir }}/bootstrap-root" + state: absent + when: operation == 'recover-operator' + no_log: true + + + - name: Report ceremony without recovery material + ansible.builtin.debug: + msg: "Vault ceremony {{ operation }} completed; no root token retained" diff --git a/infrastructure/postgresql/entrypoint.sh b/infrastructure/postgresql/entrypoint.sh old mode 100644 new mode 100755 index 33d4f5d..1011afe --- a/infrastructure/postgresql/entrypoint.sh +++ b/infrastructure/postgresql/entrypoint.sh @@ -1,14 +1,23 @@ #!/bin/sh set -eu -mkdir -p /var/lib/postgresql/tls -cp /run/payguard/server.crt /var/lib/postgresql/tls/server.crt -cp /run/payguard/server.key /var/lib/postgresql/tls/server.key -cp /run/payguard/init.sql /docker-entrypoint-initdb.d/010-payguard.sql -chown -R postgres:postgres /var/lib/postgresql/tls /docker-entrypoint-initdb.d/010-payguard.sql -chmod 600 /var/lib/postgresql/tls/server.key /docker-entrypoint-initdb.d/010-payguard.sql +# Refuse a different existing layout instead of initializing an empty replacement cluster. +# PGDATA may be overridden only after inspecting the existing volume read-only. +: "${PGDATA:=/var/lib/postgresql/18/docker}" +for version in $(find /var/lib/postgresql -name PG_VERSION -type f); do + if [ "$version" != "$PGDATA/PG_VERSION" ] || [ "$(cat "$version")" != 18 ]; then + echo 'Existing PostgreSQL layout/version differs from PGDATA; refusing initialization' >&2 + exit 1 + fi +done +mkdir -p /run/postgres-tls +cp /run/payguard/server.crt /run/postgres-tls/server.crt +cp /run/payguard/server.key /run/postgres-tls/server.key +chown -R postgres:postgres /run/postgres-tls +chmod 700 /run/postgres-tls +chmod 600 /run/postgres-tls/server.key exec docker-entrypoint.sh postgres -c ssl=on \ - -c ssl_cert_file=/var/lib/postgresql/tls/server.crt \ - -c ssl_key_file=/var/lib/postgresql/tls/server.key \ + -c ssl_cert_file=/run/postgres-tls/server.crt \ + -c ssl_key_file=/run/postgres-tls/server.key \ -c password_encryption=scram-sha-256 \ -c log_min_error_statement=panic \ -c log_parameter_max_length=0 \ diff --git a/infrastructure/postgresql/import-identity.sql b/infrastructure/postgresql/import-identity.sql new file mode 100644 index 0000000..5e880c9 --- /dev/null +++ b/infrastructure/postgresql/import-identity.sql @@ -0,0 +1,63 @@ +-- Parameters are bound through psycopg; source values never become SQL syntax. +-- Mapping: customers and challenges map old identifiers to approved prefixed IDs. +-- Timestamps require an explicit UTC offset. PostgreSQL stores microsecond precision. +SET LOCAL search_path TO payguard; +SET LOCAL statement_timeout = '5min'; +LOCK TABLE identity_challenge, identity_audit_log, identity_challenge_scope IN ACCESS EXCLUSIVE MODE; +DO $guard$ +BEGIN + IF EXISTS (SELECT 1 FROM identity_challenge) OR EXISTS (SELECT 1 FROM identity_audit_log) + OR EXISTS (SELECT 1 FROM identity_challenge_scope) + THEN RAISE EXCEPTION 'Identity import destination is populated'; END IF; +END +$guard$; +CREATE TEMP TABLE pg_temp.import_document ON COMMIT DROP AS +SELECT %(source)s::jsonb AS source, %(mapping)s::jsonb AS mapping; +DO $guard$ +DECLARE document jsonb; +BEGIN + SELECT source INTO document FROM pg_temp.import_document; + IF jsonb_typeof(document->'challenges') IS DISTINCT FROM 'array' + OR jsonb_typeof(document->'audit') IS DISTINCT FROM 'array' + THEN RAISE EXCEPTION 'Export arrays required'; END IF; + IF EXISTS ( + SELECT 1 FROM jsonb_array_elements(document->'challenges') c + WHERE jsonb_typeof(c->'id') IS DISTINCT FROM 'string' + OR jsonb_typeof(c->'user_id') IS DISTINCT FROM 'string' + OR jsonb_typeof(c->'attempts') IS DISTINCT FROM 'number' + OR jsonb_typeof(c->'max_attempts') IS DISTINCT FROM 'number' + OR (c->>'attempts') !~ '^[0-9]+$' OR (c->>'max_attempts') !~ '^[0-9]+$' + OR (c->>'created_at') !~ '(Z|[+-][0-9]{2}:[0-9]{2})$' + OR (c->>'expires_at') !~ '(Z|[+-][0-9]{2}:[0-9]{2})$' + OR (c->>'retention_until') !~ '(Z|[+-][0-9]{2}:[0-9]{2})$' + OR (c->>'consumed_at' IS NOT NULL AND (c->>'consumed_at') !~ '(Z|[+-][0-9]{2}:[0-9]{2})$') + ) THEN RAISE EXCEPTION 'Invalid integer or timestamp input'; END IF; + IF EXISTS ( + SELECT 1 FROM jsonb_array_elements(document->'audit') a + WHERE a->>'occurred_at' IS NULL OR (a->>'occurred_at') !~ '(Z|[+-][0-9]{2}:[0-9]{2})$' + ) THEN RAISE EXCEPTION 'Audit timestamp offset required'; END IF; +END +$guard$; +INSERT INTO identity_challenge +(id,user_id,purpose,channel,verification_mode,code_hash,device_binding_hash,attempts,max_attempts,status,created_at,expires_at,consumed_at,retention_until) +SELECT CASE WHEN c->>'id' ~ '^otp_[A-Za-z0-9]{22}$' THEN c->>'id' ELSE mapping->'challenges'->>(c->>'id') END, + CASE WHEN c->>'user_id' ~ '^cus_[A-Za-z0-9]{22}$' THEN c->>'user_id' ELSE mapping->'customers'->>(c->>'user_id') END, + c->>'purpose',c->>'channel',c->>'verification_mode',c->>'code_hash',c->>'device_binding_hash', + (c->>'attempts')::integer,(c->>'max_attempts')::integer,c->>'status', + regexp_replace(c->>'created_at', '(\.[0-9]{6})[0-9]+', '\1')::timestamptz,regexp_replace(c->>'expires_at', '(\.[0-9]{6})[0-9]+', '\1')::timestamptz,regexp_replace(c->>'consumed_at', '(\.[0-9]{6})[0-9]+', '\1')::timestamptz,regexp_replace(c->>'retention_until', '(\.[0-9]{6})[0-9]+', '\1')::timestamptz +FROM pg_temp.import_document, LATERAL jsonb_array_elements(source->'challenges') c; +INSERT INTO identity_audit_log (id,challenge_id,user_id,action,reason,source_ip,occurred_at) +SELECT (a->>'id')::uuid, + CASE WHEN a->>'challenge_id' ~ '^otp_[A-Za-z0-9]{22}$' THEN a->>'challenge_id' ELSE mapping->'challenges'->>(a->>'challenge_id') END, + CASE WHEN a->>'user_id' ~ '^cus_[A-Za-z0-9]{22}$' THEN a->>'user_id' ELSE mapping->'customers'->>(a->>'user_id') END, + a->>'action',a->>'reason',a->>'source_ip',regexp_replace(a->>'occurred_at', '(\.[0-9]{6})[0-9]+', '\1')::timestamptz +FROM pg_temp.import_document, LATERAL jsonb_array_elements(source->'audit') a; +INSERT INTO identity_challenge_scope (user_id,purpose) SELECT DISTINCT user_id,purpose FROM identity_challenge; +DO $reconcile$ +BEGIN + IF EXISTS (SELECT 1 FROM identity_audit_log a JOIN identity_challenge c ON c.id=a.challenge_id WHERE a.user_id<>c.user_id) + OR (SELECT count(*) FROM identity_challenge)<>(SELECT jsonb_array_length(source->'challenges') FROM pg_temp.import_document) + OR (SELECT count(*) FROM identity_audit_log)<>(SELECT jsonb_array_length(source->'audit') FROM pg_temp.import_document) + THEN RAISE EXCEPTION 'Identity import reconciliation failed'; END IF; +END +$reconcile$; diff --git a/infrastructure/postgresql/pg_hba.conf b/infrastructure/postgresql/pg_hba.conf index 548abca..0a4a8a1 100644 --- a/infrastructure/postgresql/pg_hba.conf +++ b/infrastructure/postgresql/pg_hba.conf @@ -1,5 +1,6 @@ -# Unix sockets are reserved for local bootstrap/operator maintenance inside the container. -local all all trust +# Bootstrap maintenance is restricted to the postgres OS account inside the container. +local all postgres peer +local all all reject hostnossl all all 0.0.0.0/0 reject hostnossl all all ::/0 reject hostssl all all 0.0.0.0/0 scram-sha-256 diff --git a/infrastructure/tests-requirements.txt b/infrastructure/tests-requirements.txt new file mode 100644 index 0000000..3dbf2e1 --- /dev/null +++ b/infrastructure/tests-requirements.txt @@ -0,0 +1,3 @@ +# Offline contract tests only; never loaded into an application or operations image. +PyYAML==6.0.3 +python-hcl2==7.3.1 diff --git a/infrastructure/tests/test_contracts.py b/infrastructure/tests/test_contracts.py new file mode 100644 index 0000000..df743a5 --- /dev/null +++ b/infrastructure/tests/test_contracts.py @@ -0,0 +1,180 @@ +"""Offline contracts for the native deployment; run with unittest, never for provisioning.""" +from pathlib import Path +import hashlib +import json +import re +import tomllib +import unittest +import xml.etree.ElementTree as ET +import yaml +import hcl2 + +ROOT = Path(__file__).resolve().parents[2] +SERVICES = ("identity", "wallet", "loan", "collateral") + +class DeploymentContracts(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.compose = yaml.safe_load((ROOT / "docker-compose.yaml").read_text()) + cls.ops = yaml.safe_load((ROOT / "infrastructure/compose.operations.yaml").read_text()) + + def test_existing_data_volumes_are_external_and_named(self): + volumes = self.compose["volumes"] + self.assertEqual(7, len(volumes)) + for value in volumes.values(): + self.assertTrue(value["external"]) + self.assertIn("payguard_", value["name"]) + + def test_postgres_18_volume_and_tls_contract(self): + for service in SERVICES: + database = self.compose["services"]["postgres-" + service] + self.assertTrue(database["image"].startswith("postgres:18-alpine@sha256:")) + self.assertIn(f"postgres_{service}_data:/var/lib/postgresql", database["volumes"]) + self.assertNotIn("ports", database) + self.assertIn("POSTGRES_PASSWORD_FILE", database["environment"]) + hba = (ROOT / "infrastructure/postgresql/pg_hba.conf").read_text() + self.assertNotRegex(hba, r"(?m)^.*\btrust\b") + self.assertIn("scram-sha-256", hba) + self.assertIn("refusing initialization", (ROOT / "infrastructure/postgresql/entrypoint.sh").read_text()) + + def test_no_bootstrap_credentials_in_application_configuration(self): + for service in SERVICES: + application = self.compose["services"][service + "-service"] + self.assertEqual(["applications"], application["profiles"]) + self.assertEqual("vault", application["entrypoint"][0]) + self.assertNotIn("SPRING_DATASOURCE_PASSWORD", application["environment"]) + self.assertNotIn("VAULT_SECRET_ID_FILE", application["environment"]) + for volume in application["volumes"]: + self.assertTrue(volume["read_only"]) + self.assertFalse(volume["bind"]["create_host_path"]) + + def test_all_hcl_documents_parse(self): + for path in (ROOT / "infrastructure/vault").rglob("*.hcl"): + with self.subTest(path=path.name), path.open() as stream: + hcl2.load(stream) + + def test_agent_owns_authentication_and_runtime_lifecycle(self): + for service in SERVICES: + text = (ROOT / f"infrastructure/vault/runtime/{service}.hcl").read_text() + self.assertIn('restart_on_secret_changes = "always"', text) + self.assertIn('database/creds/local-' + service + '-runtime', text) + self.assertIn('SPRING_DATA_REDIS_PASSWORD', text) + self.assertNotIn('exit_after_auth = true', text) + self.assertNotIn('tls_skip_verify', text) + + def test_each_flyway_job_is_scoped_and_immutable(self): + for service in SERVICES: + job = self.ops["services"]["migrate-" + service] + self.assertTrue(job["read_only"]) + self.assertEqual(["database"], job["networks"]) + self.assertEqual({"condition": "service_completed_successfully"}, job["depends_on"]["agent-" + service]) + self.assertEqual({"driver": "none"}, job["logging"]) + self.assertTrue(all(m["read_only"] for m in job["volumes"])) + + def test_flyway_disallows_destructive_and_implicit_adoption_commands(self): + config = tomllib.loads((ROOT / "infrastructure/flyway/flyway.toml").read_text())["flyway"] + self.assertTrue(config["cleanDisabled"]) + self.assertFalse(config["baselineOnMigrate"]) + self.assertFalse(config["outOfOrder"]) + self.assertFalse(config["createSchemas"]) + self.assertTrue(config["validateOnMigrate"]) + + def test_hostile_credentials_remain_single_toml_string_values(self): + # Agent uses Go toJSON. JSON string escaping is valid TOML for these markers. + markers = ['quote"marker', 'backslash\\marker', 'line\nbreak', 'x"\ninitSql = "DROP TABLE x', 'tab\tmarker'] + for value in markers: + data = tomllib.loads("[environments.default]\npassword = " + json.dumps(value) + "\n") + self.assertEqual(value, data["environments"]["default"]["password"]) + self.assertEqual(["password"], list(data["environments"]["default"])) + for service in SERVICES: + text = (ROOT / f"infrastructure/vault/agent/{service}.hcl").read_text() + self.assertIn(".Data.username | toJSON", text) + self.assertIn(".Data.password | toJSON", text) + self.assertNotIn("initSql", text) + after_connect = (ROOT / f"infrastructure/flyway/callbacks/{service}/afterConnect.sql").read_text() + self.assertIn(f"SET ROLE {service}_owner", after_connect) + + def test_history_is_never_in_runtime_default_table_grants(self): + for service in SERVICES: + for event in ["beforeMigrate", "afterMigrate", "afterMigrateError"]: + sql = (ROOT / f"infrastructure/flyway/callbacks/{service}/{event}.sql").read_text() + self.assertIn(f"REVOKE ALL ON payguard.flyway_schema_history FROM {service}_runtime", sql) + self.assertIn("REVOKE SELECT, INSERT, UPDATE, DELETE ON TABLES", sql) + self.assertIn("tablename <> 'flyway_schema_history'", sql) + + def test_original_versioned_sql_hashes_are_preserved(self): + for line in (ROOT / "infrastructure/migrations.sha256").read_text().splitlines(): + checksum, name = line.split(" ", 1) + self.assertEqual(checksum, hashlib.sha256((ROOT / name).read_bytes()).hexdigest(), name) + + def test_no_retired_production_tooling(self): + self.assertFalse((ROOT / "scripts").exists()) + for name in ["SchemaMigrator", "IdentityDataImporter", "LeasedDataSource"]: + self.assertFalse((ROOT / f"payguard-shared/src/main/java/infrastructure/vault/{name}.java").exists()) + client = (ROOT / "payguard-shared/src/main/java/infrastructure/vault/VaultClient.java").read_text() + for endpoint in ["auth/approle/login", "auth/token/renew-self", "auth/token/revoke-self", "sys/leases"]: + self.assertNotIn(endpoint, client) + + def test_parent_reactor_modules_exist(self): + ns = {"m": "http://maven.apache.org/POM/4.0.0"} + root = ET.parse(ROOT / "pom.xml").getroot() + for node in root.findall("m:modules/m:module", ns): + self.assertTrue((ROOT / node.text / "pom.xml").is_file()) + + def test_production_brokers_require_authentication_tls_and_hostname_checks(self): + class Loader(yaml.SafeLoader): + pass + Loader.add_constructor("!override", lambda loader, node: loader.construct_mapping(node, deep=True)) + production = yaml.load((ROOT / "infrastructure/compose.production.yaml").read_text(), Loader=Loader) + self.assertEqual(["local-brokers"], production["services"]["kafka"]["profiles"]) + self.assertEqual(["local-brokers"], production["services"]["redis"]["profiles"]) + for service in SERVICES: + app = production["services"][service + "-service"] + self.assertNotIn("kafka", app["depends_on"]) + self.assertNotIn("redis", app["depends_on"]) + self.assertEqual("SASL_SSL", app["environment"]["SPRING_KAFKA_PROPERTIES_SECURITY_PROTOCOL"]) + self.assertEqual("https", app["environment"]["SPRING_KAFKA_PROPERTIES_SSL_ENDPOINT_IDENTIFICATION_ALGORITHM"]) + self.assertEqual("true", app["environment"]["SPRING_DATA_REDIS_SSL_ENABLED"]) + + def test_agent_and_flyway_tmpfs_are_writable_by_the_configured_nonroot_user(self): + for service in SERVICES: + for prefix in ["agent-", "migrate-"]: + tmpfs = self.ops["services"][prefix + service]["tmpfs"][0] + self.assertIn("uid=${PAYGUARD_LOCAL_UID", tmpfs) + self.assertIn("gid=${PAYGUARD_LOCAL_GID", tmpfs) + self.assertIn("mode=0700", tmpfs) + + def test_production_initialization_requires_separate_custodians(self): + ceremony = (ROOT / "infrastructure/operations/vault-lifecycle.yml").read_text() + self.assertIn("root_token_pgp_key", ceremony) + self.assertIn("recovery_pgp_keys", ceremony) + self.assertIn("unseal_key_files", ceremony) + self.assertIn("always:", ceremony) + + def test_security_sensitive_ansible_tasks_suppress_output(self): + for path in (ROOT / "infrastructure/operations").glob("*.yml"): + plays = yaml.safe_load(path.read_text()) + if not isinstance(plays, list): + continue + for play in plays: + for task in play.get("tasks", []) + play.get("pre_tasks", []): + encoded = json.dumps(task) + # A secret-bearing action must not print its body on failure. + if any(key in task for key in ["ansible.builtin.uri", "community.postgresql.postgresql_query", "community.postgresql.postgresql_user"]): + if any(secret in encoded for secret in ["X-Vault-Token", "login_password", "root_token", "secret_id"]): + self.assertTrue(task.get("no_log"), (path.name, task.get("name"))) + + def test_migration_has_bounded_execution_and_verified_cleanup(self): + text = (ROOT / "infrastructure/operations/migrate.yml").read_text() + for value in ["'600'", "token_lookup.json.data.ttl", "revoke-self", "cleanup_pending", "cleanup_verified", "migration-{{ service }}.lock", "tasks/verify-migration.yml"]: + self.assertIn(value, text) + self.assertIn("always:", text) + + def test_recovery_requires_exact_receipt_path(self): + text = (ROOT / "infrastructure/operations/recover-attempt.yml").read_text() + self.assertIn("receipt.path == runtime_root ~ '/' ~ attempt_id", text) + self.assertNotIn("receipt.path.startswith", text) + self.assertIn("cleanup_unknown", text) + +if __name__ == "__main__": + unittest.main() diff --git a/infrastructure/vault/agent/collateral.hcl b/infrastructure/vault/agent/collateral.hcl new file mode 100644 index 0000000..aa254a2 --- /dev/null +++ b/infrastructure/vault/agent/collateral.hcl @@ -0,0 +1,45 @@ +vault { + address = "https://vault:8200" + ca_cert = "/etc/vault/ca.crt" +} + +exit_after_auth = true + +pid_file = "/run/agent-private/agent.pid" + +auto_auth { + method "approle" { + config = { + role_id_file_path = "/run/agent-auth/role-id" + secret_id_file_path = "/run/agent-auth/secret-id" + remove_secret_id_file_after_reading = false + } + } + sink "file" { + config = { + path = "/run/agent-private/token" + mode = 0600 + } + } +} + +template_config { + exit_on_retry_failure = true +} + +template { + destination = "/run/agent-handoff/credential.toml" + perms = 0440 + backup = false + error_on_missing_key = true + contents = < org.flywaydb flyway-database-postgresql + test org.springframework.boot @@ -32,6 +33,7 @@ org.springframework.boot spring-boot-starter-flyway + test org.postgresql diff --git a/payguard-collateral-service/src/main/resources/application.yaml b/payguard-collateral-service/src/main/resources/application.yaml index fe479db..d55b682 100644 --- a/payguard-collateral-service/src/main/resources/application.yaml +++ b/payguard-collateral-service/src/main/resources/application.yaml @@ -4,6 +4,12 @@ spring: datasource: url: ${SPRING_DATASOURCE_URL} driver-class-name: org.postgresql.Driver + hikari: + maximum-pool-size: 10 + connection-timeout: 5000 + validation-timeout: 2000 + max-lifetime: 300000 + keepalive-time: 30000 jpa: properties: hibernate.default_schema: payguard @@ -12,23 +18,13 @@ spring: ddl-auto: validate flyway: enabled: false - locations: classpath:db/migration/postgresql - clean-disabled: true - url: ${SPRING_DATASOURCE_URL} - user: ${PAYGUARD_MIGRATION_USERNAME} - password: ${PAYGUARD_MIGRATION_PASSWORD} - default-schema: payguard - schemas: payguard - create-schemas: false payguard: vault: enabled: true address: ${VAULT_ADDR} ca-file: ${VAULT_CA_FILE} - role-id-file: ${VAULT_ROLE_ID_FILE} - secret-id-file: ${VAULT_SECRET_ID_FILE} - database-role: ${PAYGUARD_ENVIRONMENT:local}-collateral-runtime + token-file: ${VAULT_TOKEN_FILE} management: endpoint: @@ -42,3 +38,8 @@ management: web: exposure: include: health,info + +server: + shutdown: graceful + +spring.lifecycle.timeout-per-shutdown-phase: 20s diff --git a/payguard-identity-service/pom.xml b/payguard-identity-service/pom.xml index 66092c3..1ac4088 100644 --- a/payguard-identity-service/pom.xml +++ b/payguard-identity-service/pom.xml @@ -59,10 +59,12 @@ org.springframework.boot spring-boot-starter-flyway + test org.flywaydb flyway-database-postgresql + test org.postgresql diff --git a/payguard-identity-service/src/main/resources/application.yaml b/payguard-identity-service/src/main/resources/application.yaml index 6acf651..9aee22a 100644 --- a/payguard-identity-service/src/main/resources/application.yaml +++ b/payguard-identity-service/src/main/resources/application.yaml @@ -4,6 +4,12 @@ spring: datasource: url: ${SPRING_DATASOURCE_URL} driver-class-name: org.postgresql.Driver + hikari: + maximum-pool-size: 10 + connection-timeout: 5000 + validation-timeout: 2000 + max-lifetime: 300000 + keepalive-time: 30000 jpa: properties: hibernate.default_schema: payguard @@ -12,14 +18,6 @@ spring: ddl-auto: validate flyway: enabled: false - locations: classpath:db/migration/postgresql - clean-disabled: true - url: ${SPRING_DATASOURCE_URL} - user: ${PAYGUARD_MIGRATION_USERNAME} - password: ${PAYGUARD_MIGRATION_PASSWORD} - default-schema: payguard - schemas: payguard - create-schemas: false data: redis: host: ${REDIS_HOST:localhost} @@ -31,6 +29,7 @@ spring: value-serializer: org.apache.kafka.common.serialization.StringSerializer server: + shutdown: graceful port: ${OTP_SERVICE_PORT:8084} identity: @@ -54,10 +53,8 @@ payguard: enabled: true address: ${VAULT_ADDR} ca-file: ${VAULT_CA_FILE} - role-id-file: ${VAULT_ROLE_ID_FILE} - secret-id-file: ${VAULT_SECRET_ID_FILE} + token-file: ${VAULT_TOKEN_FILE} secret-prefix: ${PAYGUARD_ENVIRONMENT:local}/identity - database-role: ${PAYGUARD_ENVIRONMENT:local}-identity-runtime management: endpoint: @@ -71,3 +68,5 @@ management: web: exposure: include: health,info + +spring.lifecycle.timeout-per-shutdown-phase: 20s diff --git a/payguard-identity-service/src/test/java/dev/amg/payguard/identity/verification/infrastructure/security/VaultSecretsIT.java b/payguard-identity-service/src/test/java/dev/amg/payguard/identity/verification/infrastructure/security/VaultSecretsIT.java index 2130342..0ebee7f 100644 --- a/payguard-identity-service/src/test/java/dev/amg/payguard/identity/verification/infrastructure/security/VaultSecretsIT.java +++ b/payguard-identity-service/src/test/java/dev/amg/payguard/identity/verification/infrastructure/security/VaultSecretsIT.java @@ -137,11 +137,15 @@ void invalidMissingKeyConfigurationFailsStartupWithoutDefault() { } private static VaultClient client(String folder) { - Path scope = LOCAL.resolve(folder); + String scope = + folder.equals("operator/approle") + ? "OPERATOR" + : folder.substring(5).toUpperCase(java.util.Locale.ROOT); + String tokenFile = System.getenv("PAYGUARD_REHEARSAL_" + scope + "_TOKEN_FILE"); + if (tokenFile == null || tokenFile.isBlank()) { + throw new IllegalStateException("Agent token fixture path required for " + scope); + } return new VaultClient( - URI.create("https://vault:8200"), - LOCAL.resolve("ca.crt"), - scope.resolve("role-id"), - scope.resolve("secret-id")); + URI.create("https://vault:8200"), LOCAL.resolve("ca.crt"), Path.of(tokenFile)); } } diff --git a/payguard-loan-service/pom.xml b/payguard-loan-service/pom.xml index a5c3101..42ff186 100644 --- a/payguard-loan-service/pom.xml +++ b/payguard-loan-service/pom.xml @@ -20,6 +20,7 @@ org.flywaydb flyway-database-postgresql + test dev.amg.payguard @@ -41,6 +42,7 @@ org.springframework.boot spring-boot-starter-flyway + test org.postgresql diff --git a/payguard-loan-service/src/main/resources/application.yaml b/payguard-loan-service/src/main/resources/application.yaml index b99e8c7..feafdc4 100644 --- a/payguard-loan-service/src/main/resources/application.yaml +++ b/payguard-loan-service/src/main/resources/application.yaml @@ -4,6 +4,12 @@ spring: datasource: url: ${SPRING_DATASOURCE_URL} driver-class-name: org.postgresql.Driver + hikari: + maximum-pool-size: 10 + connection-timeout: 5000 + validation-timeout: 2000 + max-lifetime: 300000 + keepalive-time: 30000 jpa: properties: hibernate.default_schema: payguard @@ -12,23 +18,13 @@ spring: ddl-auto: validate flyway: enabled: false - locations: classpath:db/migration/postgresql - clean-disabled: true - url: ${SPRING_DATASOURCE_URL} - user: ${PAYGUARD_MIGRATION_USERNAME} - password: ${PAYGUARD_MIGRATION_PASSWORD} - default-schema: payguard - schemas: payguard - create-schemas: false payguard: vault: enabled: true address: ${VAULT_ADDR} ca-file: ${VAULT_CA_FILE} - role-id-file: ${VAULT_ROLE_ID_FILE} - secret-id-file: ${VAULT_SECRET_ID_FILE} - database-role: ${PAYGUARD_ENVIRONMENT:local}-loan-runtime + token-file: ${VAULT_TOKEN_FILE} management: endpoint: @@ -42,3 +38,8 @@ management: web: exposure: include: health,info + +server: + shutdown: graceful + +spring.lifecycle.timeout-per-shutdown-phase: 20s diff --git a/payguard-shared/pom.xml b/payguard-shared/pom.xml index ee0691f..e702a39 100644 --- a/payguard-shared/pom.xml +++ b/payguard-shared/pom.xml @@ -15,12 +15,12 @@ org.flywaydb flyway-core - true + test org.flywaydb flyway-database-postgresql - true + test org.junit.platform diff --git a/payguard-shared/src/main/java/infrastructure/vault/IdentityDataImporter.java b/payguard-shared/src/main/java/infrastructure/vault/IdentityDataImporter.java deleted file mode 100644 index c5de9cb..0000000 --- a/payguard-shared/src/main/java/infrastructure/vault/IdentityDataImporter.java +++ /dev/null @@ -1,194 +0,0 @@ -package infrastructure.vault; - -import java.nio.file.Files; -import java.nio.file.Path; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; -import java.time.OffsetDateTime; -import java.util.List; -import java.util.UUID; -import tools.jackson.databind.JsonNode; -import tools.jackson.databind.json.JsonMapper; - -/** Offline parent-before-child import into an empty, separately migrated destination. */ -public final class IdentityDataImporter { - private static final org.slf4j.Logger LOG = - org.slf4j.LoggerFactory.getLogger(IdentityDataImporter.class); - private static final String MANIFEST = "manifest"; - private static final String AUDIT_TABLE = "identity_audit_log"; - private static final List CHALLENGE = - List.of( - "id", - "user_id", - "purpose", - "channel", - "verification_mode", - "code_hash", - "device_binding_hash", - "attempts", - "max_attempts", - "status", - "created_at", - "expires_at", - "consumed_at", - "retention_until"); - private static final List AUDIT = - List.of("id", "challenge_id", "user_id", "action", "reason", "source_ip", "occurred_at"); - - private IdentityDataImporter() {} - - public static void main(String[] arguments) { - try { - String url = System.getenv("PAYGUARD_MIGRATION_JDBC_URL"); - if (arguments.length != 1 || url == null) { - throw new IllegalArgumentException( - "Explicit protected input and verified PostgreSQL TLS required"); - } - PostgreSqlTls.requireVerified(url); - JsonNode input = - JsonMapper.builder().build().readTree(Files.readString(Path.of(arguments[0]))); - try (Connection connection = - DriverManager.getConnection( - url, - System.getenv("PAYGUARD_MIGRATION_USERNAME"), - System.getenv("PAYGUARD_MIGRATION_PASSWORD"))) { - try (var statement = connection.createStatement()) { - statement.execute("SET ROLE identity_owner"); - statement.execute("SET search_path TO payguard"); - } - importData(connection, input); - } - LOG.info( - "Identity import committed: row counts, references and representative fields reconciled"); - } catch (Exception exception) { - LOG.error("Identity import refused; protected diagnostics suppressed"); - System.exit(1); - } - } - - public static void importData(Connection connection, JsonNode input) throws SQLException { - boolean originalAutoCommit = connection.getAutoCommit(); - if (!originalAutoCommit - || input.path(MANIFEST).path("approval_reference").asText().isBlank() - || !input.path(MANIFEST).path("mapping_sha256").asText().matches("[a-f0-9]{64}")) { - throw new SQLException("Separate import connection and approved mapping manifest required"); - } - connection.setAutoCommit(false); - try { - try (var statement = connection.createStatement()) { - statement.execute( - "LOCK TABLE identity_challenge, identity_audit_log IN ACCESS EXCLUSIVE MODE"); - for (String table : List.of("identity_challenge", AUDIT_TABLE)) { - try (var rows = statement.executeQuery("SELECT count(*) FROM " + table)) { - if (!rows.next()) throw new SQLException("Reconciliation query returned no result"); - if (rows.getLong(1) != 0) throw new SQLException("Import destination is populated"); - } - } - } - insert(connection, "identity_challenge", CHALLENGE, input.path("challenges")); - insert(connection, AUDIT_TABLE, AUDIT, input.path("audit")); - try (var statement = connection.createStatement()) { - statement.executeUpdate( - "INSERT INTO identity_challenge_scope(user_id,purpose) SELECT DISTINCT user_id,purpose FROM identity_challenge ON CONFLICT DO NOTHING"); - try (var rows = - statement.executeQuery( - "SELECT count(*) FROM identity_audit_log a JOIN identity_challenge c ON c.id=a.challenge_id WHERE c.user_id<>a.user_id")) { - if (!rows.next()) throw new SQLException("Reconciliation query returned no result"); - if (rows.getLong(1) != 0) throw new SQLException("Audit subject mismatch"); - } - } - reconcile( - connection, - "identity_challenge", - CHALLENGE, - input.path("challenges"), - input.path(MANIFEST).path("challenge_count").asInt(-1)); - reconcile( - connection, - AUDIT_TABLE, - AUDIT, - input.path("audit"), - input.path(MANIFEST).path("audit_count").asInt(-1)); - connection.commit(); - } catch (SQLException | RuntimeException exception) { - connection.rollback(); - throw exception; - } finally { - connection.setAutoCommit(originalAutoCommit); - } - } - - private static void insert( - Connection connection, String table, List columns, JsonNode data) - throws SQLException { - if (!data.isArray()) throw new SQLException("Expected export rows"); - String placeholders = String.join(",", java.util.Collections.nCopies(columns.size(), "?")); - try (var statement = - connection.prepareStatement( - "INSERT INTO " - + table - + "(" - + String.join(",", columns) - + ") VALUES (" - + placeholders - + ")")) { - for (JsonNode row : data) { - for (int i = 0; i < columns.size(); i++) - statement.setObject(i + 1, value(table, columns.get(i), row)); - statement.executeUpdate(); - } - } - } - - private static Object value(String table, String column, JsonNode row) throws SQLException { - JsonNode node = row.get(column); - if (node == null) throw new SQLException("Export column missing"); - if (node.isNull()) return null; - if (column.endsWith("_at") || "retention_until".equals(column)) - return OffsetDateTime.parse(node.asText()); - if ("attempts".equals(column) || "max_attempts".equals(column)) { - if (!node.isIntegralNumber() || !node.canConvertToInt()) - throw new SQLException("Invalid exported integer"); - return node.asInt(); - } - if (!node.isTextual()) throw new SQLException("Invalid exported text"); - if (AUDIT_TABLE.equals(table) && "id".equals(column)) return UUID.fromString(node.asText()); - return node.asText(); - } - - private static void reconcile( - Connection connection, String table, List columns, JsonNode data, int expected) - throws SQLException { - try (var statement = connection.createStatement(); - var rows = statement.executeQuery("SELECT count(*) FROM " + table)) { - if (!rows.next()) throw new SQLException("Reconciliation query returned no result"); - if (rows.getInt(1) != expected || expected != data.size()) - throw new SQLException("Export count mismatch"); - } - try (var statement = - connection.prepareStatement( - "SELECT " + String.join(",", columns) + " FROM " + table + " WHERE id=?")) { - for (JsonNode row : data) { - statement.setObject(1, value(table, "id", row)); - try (var rows = statement.executeQuery()) { - if (!rows.next()) throw new SQLException("Export identity missing"); - for (String column : columns) { - Object expectedValue = value(table, column, row); - Object stored = - expectedValue instanceof OffsetDateTime - ? rows.getObject(column, OffsetDateTime.class) - : rows.getObject(column); - if (stored instanceof OffsetDateTime timestamp - && expectedValue instanceof OffsetDateTime source) { - if (!timestamp.toInstant().equals(source.toInstant())) - throw new SQLException("Export instant mismatch"); - } else if (!java.util.Objects.equals(expectedValue, stored)) { - throw new SQLException("Export field mismatch"); - } - } - } - } - } - } -} diff --git a/payguard-shared/src/main/java/infrastructure/vault/LeasedDataSource.java b/payguard-shared/src/main/java/infrastructure/vault/LeasedDataSource.java deleted file mode 100644 index 9e682ca..0000000 --- a/payguard-shared/src/main/java/infrastructure/vault/LeasedDataSource.java +++ /dev/null @@ -1,221 +0,0 @@ -package infrastructure.vault; - -import com.zaxxer.hikari.HikariConfig; -import com.zaxxer.hikari.HikariDataSource; -import java.sql.Connection; -import java.sql.SQLException; -import java.time.Duration; -import java.time.Instant; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.TimeUnit; -import org.jspecify.annotations.Nullable; -import org.springframework.jdbc.datasource.AbstractDataSource; - -/** - * Replaces validated pool generations before lease expiry. Borrowers already in a transaction keep - * their generation during a bounded drain. This class never retries application transactions. - */ -public final class LeasedDataSource extends AbstractDataSource implements AutoCloseable { - private static final int MAX_RETIRED_GENERATIONS = 8; - private static final Duration RETRY_DELAY = Duration.ofSeconds(1); - private final VaultClient vault; - private final String role; - private final String jdbcUrl; - private final Duration drain; - private final List retired = new ArrayList<>(); - private final ScheduledExecutorService scheduler = - Executors.newSingleThreadScheduledExecutor( - Thread.ofPlatform().daemon().name("database-lease-maintenance").factory()); - private @Nullable Generation current; - private boolean closed; - private long replacements; - private Instant nextAttempt = Instant.EPOCH; - private boolean rotationRequested; - - public LeasedDataSource(VaultClient vault, String role, String jdbcUrl, Duration drain) { - PostgreSqlTls.requireVerified(jdbcUrl); - if (!role.matches("[a-z][a-z0-9-]+-runtime")) { - throw new IllegalArgumentException( - "Runtime database requires a scoped role and verified PostgreSQL TLS"); - } - this.vault = vault; - this.role = role; - this.jdbcUrl = jdbcUrl; - this.drain = drain; - replace(); - scheduler.scheduleWithFixedDelay(this::maintain, 1, 1, TimeUnit.SECONDS); - } - - private synchronized void replace() { - if (retired.size() >= MAX_RETIRED_GENERATIONS) { - throw new VaultUnavailableException(); - } - Instant issued = Instant.now(); - var secret = vault.read("database/creds/" + role); - String lease = secret.path("lease_id").asText(); - long ttl = secret.path("lease_duration").asLong(); - if (ttl < 20 || !lease.startsWith("database/creds/" + role + "/")) { - throw new VaultUnavailableException(); - } - HikariConfig config = new HikariConfig(); - config.setJdbcUrl(jdbcUrl); - config.setUsername(secret.path("data").path("username").asText()); - config.setPassword(secret.path("data").path("password").asText()); - config.setPoolName("payguard-runtime-" + replacements); - config.setMaximumPoolSize(4); - config.setMinimumIdle(0); - config.setConnectionTimeout(1000); - config.setValidationTimeout(1000); - config.setInitializationFailTimeout(1000); - config.setConnectionInitSql("SET search_path TO payguard"); - HikariDataSource candidate; - try { - candidate = new HikariDataSource(config); - } catch (RuntimeException exception) { - revoke(lease); - throw new VaultUnavailableException(exception); - } - try (var connection = candidate.getConnection()) { - if (!connection.isValid(1) || !Instant.now().isBefore(issued.plusSeconds(ttl - 2))) { - throw new SQLException("Database validation failed"); - } - validateRuntime(connection); - } catch (SQLException exception) { - candidate.close(); - revoke(lease); - throw new VaultUnavailableException(exception); - } - Generation previous = current; - current = - new Generation(candidate, lease, issued.plusSeconds(ttl / 2), issued.plusSeconds(ttl - 2)); - replacements++; - nextAttempt = Instant.EPOCH; - rotationRequested = false; - if (previous != null) { - previous.retireAt = Instant.now().plus(drain); - previous.pool.getHikariPoolMXBean().softEvictConnections(); - retired.add(previous); - } - } - - private synchronized void maintain() { - if (closed) { - return; - } - Instant now = Instant.now(); - Generation generation = current; - if (generation != null - && (rotationRequested || !now.isBefore(generation.replaceAt)) - && !now.isBefore(nextAttempt)) { - try { - replace(); - } catch (VaultUnavailableException exception) { - /* Retry once per scheduled interval while the prior lease remains valid. */ - nextAttempt = Instant.now().plus(RETRY_DELAY); - } - } - retired.removeIf( - old -> { - if (old.pool.isClosed() - || old.pool.getHikariPoolMXBean().getActiveConnections() == 0 - || !now.isBefore(old.retireAt) - || !now.isBefore(old.deadline)) { - old.pool.close(); - return revoke(old.lease); - } - return false; - }); - if (generation != null && !now.isBefore(generation.deadline)) { - generation.pool.close(); - } - } - - private boolean revoke(String lease) { - try { - vault.write("sys/leases/revoke", Map.of("lease_id", lease)); - return true; - } catch (VaultUnavailableException exception) { - /* The engine also revokes expired leases. Retry graceful revocation during maintenance. */ - return false; - } - } - - @Override - public synchronized Connection getConnection() throws SQLException { - Generation generation = current; - if (closed || generation == null || !Instant.now().isBefore(generation.deadline)) { - throw new SQLException("Runtime database credential is unavailable"); - } - try { - Connection connection = generation.pool.getConnection(); - try { - validateRuntime(connection); - return connection; - } catch (SQLException exception) { - connection.close(); - throw exception; - } - } catch (SQLException exception) { - rotationRequested = true; - throw new SQLException( - "Runtime database connection is unavailable", - VaultUnavailableException.sanitizedCause(exception)); - } - } - - private static void validateRuntime(Connection connection) throws SQLException { - try (var statement = connection.createStatement()) { - statement.setQueryTimeout(1); - try (var rows = - statement.executeQuery( - "SELECT has_schema_privilege(current_user,'payguard','USAGE') AND has_database_privilege(current_user,current_database(),'CONNECT')")) { - if (!rows.next() || !rows.getBoolean(1)) - throw new SQLException("Runtime database privileges unavailable"); - } - } - } - - @Override - public Connection getConnection(String username, String password) throws SQLException { - throw new SQLException("Caller-supplied database credentials are not supported"); - } - - /** Non-sensitive count used to verify continuous replacement without exposing credentials. */ - public synchronized long replacementCount() { - return replacements; - } - - @Override - public synchronized void close() { - closed = true; - scheduler.shutdownNow(); - if (current != null) { - current.pool.close(); - revoke(current.lease); - } - for (Generation old : retired) { - old.pool.close(); - revoke(old.lease); - } - retired.clear(); - } - - private static final class Generation { - private final HikariDataSource pool; - private final String lease; - private final Instant replaceAt; - private final Instant deadline; - private Instant retireAt = Instant.MAX; - - private Generation(HikariDataSource pool, String lease, Instant replaceAt, Instant deadline) { - this.pool = pool; - this.lease = lease; - this.replaceAt = replaceAt; - this.deadline = deadline; - } - } -} diff --git a/payguard-shared/src/main/java/infrastructure/vault/SchemaMigrator.java b/payguard-shared/src/main/java/infrastructure/vault/SchemaMigrator.java deleted file mode 100644 index c6f3630..0000000 --- a/payguard-shared/src/main/java/infrastructure/vault/SchemaMigrator.java +++ /dev/null @@ -1,71 +0,0 @@ -package infrastructure.vault; - -import java.sql.DriverManager; -import java.util.Set; -import org.flywaydb.core.Flyway; - -/** Separate operator entry point. Migration logins SET ROLE to the no-login owner before DDL. */ -public final class SchemaMigrator { - private static final String IDENTITY = "identity"; - private static final org.slf4j.Logger LOG = - org.slf4j.LoggerFactory.getLogger(SchemaMigrator.class); - - private SchemaMigrator() {} - - public static void main(String[] arguments) { - try { - String service = required("PAYGUARD_MIGRATION_SERVICE"); - if (!Set.of("identity", "loan", "collateral").contains(service)) { - throw new IllegalArgumentException("No supplied schema for this service"); - } - String url = required("PAYGUARD_MIGRATION_JDBC_URL"); - PostgreSqlTls.requireVerified(url); - String user = required("PAYGUARD_MIGRATION_USERNAME"); - String password = required("PAYGUARD_MIGRATION_PASSWORD"); - Flyway.configure() - .dataSource(url, user, password) - .initSql("SET ROLE " + service + "_owner") - .schemas("payguard") - .defaultSchema("payguard") - .createSchemas(false) - .cleanDisabled(true) - .locations( - "filesystem:/workspace/payguard-" - + service - + "-service/src/main/resources/db/migration/postgresql") - .load() - .migrate(); - try (var connection = DriverManager.getConnection(url, user, password); - var statement = connection.createStatement()) { - statement.execute("SET ROLE " + service + "_owner"); - statement.execute("ALTER DEFAULT PRIVILEGES REVOKE EXECUTE ON FUNCTIONS FROM PUBLIC"); - statement.execute( - "REVOKE ALL ON payguard.flyway_schema_history FROM " + service + "_runtime"); - if (IDENTITY.equals(service)) { - statement.execute( - "REVOKE UPDATE, DELETE ON payguard.identity_audit_log FROM identity_runtime"); - statement.execute( - "REVOKE DELETE ON payguard.identity_challenge_scope FROM identity_runtime"); - } else { - statement.execute( - "REVOKE UPDATE, DELETE ON payguard." - + service - + "_audit_log FROM " - + service - + "_runtime"); - } - } - } catch (RuntimeException | java.sql.SQLException exception) { - LOG.error("Migration failed; secret-bearing connection diagnostics suppressed"); - System.exit(1); - } - } - - private static String required(String name) { - String value = System.getenv(name); - if (value == null || value.isBlank()) { - throw new IllegalArgumentException("Missing protected migration configuration"); - } - return value; - } -} diff --git a/payguard-shared/src/main/java/infrastructure/vault/VaultClient.java b/payguard-shared/src/main/java/infrastructure/vault/VaultClient.java index 037a30c..70620c3 100644 --- a/payguard-shared/src/main/java/infrastructure/vault/VaultClient.java +++ b/payguard-shared/src/main/java/infrastructure/vault/VaultClient.java @@ -10,7 +10,6 @@ import java.security.KeyStore; import java.security.cert.CertificateFactory; import java.time.Duration; -import java.time.Instant; import java.util.Map; import java.util.Optional; import javax.net.ssl.SSLContext; @@ -20,28 +19,31 @@ import tools.jackson.databind.json.JsonMapper; /** - * Scoped AppRole client with TLS verification, bounded calls and sanitized failures. Authentication - * material lives in provisioned files. A missing KV item is distinct from denial or an outage. + * Application KV client using a token delivered and renewed exclusively by Vault Agent. This Java + * client never reads AppRole bootstrap material or manages Vault leases. A missing KV item is + * distinct from denial or an outage. */ public final class VaultClient implements AutoCloseable { - private static final org.slf4j.Logger LOG = org.slf4j.LoggerFactory.getLogger(VaultClient.class); private static final String POST = "POST"; private final URI address; - private final Path roleIdFile; - private final Path secretIdFile; + private final Path tokenFile; private final HttpClient http; private final JsonMapper json = JsonMapper.builder().build(); - private String token = ""; - private Instant renewAt = Instant.EPOCH; - private Instant expiresAt = Instant.EPOCH; - public VaultClient(URI address, Path caFile, Path roleIdFile, Path secretIdFile) { + public VaultClient(URI address, Path caFile, Path tokenFile) { + this(address, tokenFile, verifiedHttpClient(caFile)); + } + + VaultClient(URI address, Path tokenFile, HttpClient http) { if (!"https".equals(address.getScheme()) || address.getUserInfo() != null) { throw new IllegalArgumentException("Vault requires an HTTPS endpoint without credentials"); } this.address = address; - this.roleIdFile = roleIdFile; - this.secretIdFile = secretIdFile; + this.tokenFile = tokenFile; + this.http = http; + } + + private static HttpClient verifiedHttpClient(Path caFile) { try { KeyStore trust = KeyStore.getInstance(KeyStore.getDefaultType()); trust.load(null); @@ -58,7 +60,7 @@ public VaultClient(URI address, Path caFile, Path roleIdFile, Path secretIdFile) managers.init(trust); SSLContext tls = SSLContext.getInstance("TLS"); tls.init(null, managers.getTrustManagers(), null); - http = HttpClient.newBuilder().sslContext(tls).connectTimeout(Duration.ofSeconds(3)).build(); + return HttpClient.newBuilder().sslContext(tls).connectTimeout(Duration.ofSeconds(3)).build(); } catch (Exception exception) { throw new VaultUnavailableException(exception); } @@ -76,56 +78,24 @@ public JsonNode write(String path, Map body) { return request(POST, path, body, false).orElseThrow(VaultUnavailableException::new); } - private synchronized Optional request( - String method, String path, @Nullable Map body, boolean missingAllowed) { - authenticate(); - return exchange(method, path, body, token, missingAllowed); + @Override + public void close() { + // The supervising Agent owns token and lease lifecycle. + http.close(); } - private void authenticate() { - Instant now = Instant.now(); - if (!token.isEmpty() && now.isBefore(renewAt)) { - return; - } - if (!token.isEmpty() && now.isBefore(expiresAt)) { - try { - acceptAuthentication( - exchange(POST, "auth/token/renew-self", Map.of(), token, false).orElseThrow()); - return; - } catch (VaultUnavailableException exception) { - /* A still-valid token can be used briefly; expired tokens require reauthentication. */ - renewAt = now.plusSeconds(1); - return; - } - } + private Optional request( + String method, String path, @Nullable Map body, boolean missingAllowed) { try { - acceptAuthentication( - exchange( - POST, - "auth/approle/login", - Map.of( - "role_id", Files.readString(roleIdFile).strip(), - "secret_id", Files.readString(secretIdFile).strip()), - "", - false) - .orElseThrow()); + // Reopen the file: Agent replaces sinks atomically during reauthentication. + String token = Files.readString(tokenFile).strip(); + if (token.isBlank()) throw new VaultUnavailableException(); + return exchange(method, path, body, token, missingAllowed); } catch (IOException exception) { throw new VaultUnavailableException(exception); } } - private void acceptAuthentication(JsonNode response) { - JsonNode auth = response.path("auth"); - String nextToken = auth.path("client_token").asText(); - long ttl = auth.path("lease_duration").asLong(); - if (nextToken.isEmpty() || ttl < 2 || auth.path("policies").toString().contains("\"root\"")) { - throw new VaultUnavailableException(); - } - token = nextToken; - expiresAt = Instant.now().plusSeconds(ttl); - renewAt = Instant.now().plusSeconds(Math.max(1, ttl / 2)); - } - private Optional exchange( String method, String path, @@ -159,12 +129,6 @@ private Optional exchange( throw new VaultUnavailableException(); } if (response.statusCode() < 200 || response.statusCode() >= 300) { - if ((response.statusCode() == 401 || response.statusCode() == 403) - && authentication.equals(token)) { - token = ""; - renewAt = Instant.EPOCH; - expiresAt = Instant.EPOCH; - } throw new VaultUnavailableException(); } return Optional.of( @@ -176,18 +140,4 @@ private Optional exchange( throw new VaultUnavailableException(exception); } } - - @Override - public synchronized void close() { - if (!token.isEmpty()) { - try { - exchange(POST, "auth/token/revoke-self", Map.of(), token, false); - } catch (VaultUnavailableException exception) { - /* Vault expires unreachable tokens and their remaining leases independently. */ - LOG.warn("Vault token cleanup deferred to its expiry"); - } - } - http.close(); - token = ""; - } } diff --git a/payguard-shared/src/main/java/infrastructure/vault/VaultDatabaseAutoConfiguration.java b/payguard-shared/src/main/java/infrastructure/vault/VaultDatabaseAutoConfiguration.java index d5dcce1..0baa7f6 100644 --- a/payguard-shared/src/main/java/infrastructure/vault/VaultDatabaseAutoConfiguration.java +++ b/payguard-shared/src/main/java/infrastructure/vault/VaultDatabaseAutoConfiguration.java @@ -2,35 +2,24 @@ import java.net.URI; import java.nio.file.Path; -import java.time.Duration; import org.springframework.boot.autoconfigure.AutoConfiguration; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.boot.health.contributor.Health; import org.springframework.boot.health.contributor.HealthIndicator; -import org.springframework.boot.jdbc.autoconfigure.DataSourceAutoConfiguration; import org.springframework.context.annotation.Bean; import org.springframework.core.env.Environment; /** Infrastructure-only composition shared by the four database-owning services. */ -@AutoConfiguration(before = DataSourceAutoConfiguration.class) +@AutoConfiguration @ConditionalOnProperty(name = "payguard.vault.enabled", havingValue = "true") public class VaultDatabaseAutoConfiguration { @Bean(destroyMethod = "close") VaultClient vaultClient(Environment environment) { + PostgreSqlTls.requireVerified(environment.getRequiredProperty("spring.datasource.url")); return new VaultClient( URI.create(environment.getRequiredProperty("payguard.vault.address")), Path.of(environment.getRequiredProperty("payguard.vault.ca-file")), - Path.of(environment.getRequiredProperty("payguard.vault.role-id-file")), - Path.of(environment.getRequiredProperty("payguard.vault.secret-id-file"))); - } - - @Bean(destroyMethod = "close") - LeasedDataSource dataSource(VaultClient vault, Environment environment) { - return new LeasedDataSource( - vault, - environment.getRequiredProperty("payguard.vault.database-role"), - environment.getRequiredProperty("spring.datasource.url"), - Duration.ofSeconds(10)); + Path.of(environment.getRequiredProperty("payguard.vault.token-file"))); } @Bean diff --git a/payguard-shared/src/test/java/infrastructure/vault/AgentTokenClientTest.java b/payguard-shared/src/test/java/infrastructure/vault/AgentTokenClientTest.java new file mode 100644 index 0000000..13ba2b9 --- /dev/null +++ b/payguard-shared/src/test/java/infrastructure/vault/AgentTokenClientTest.java @@ -0,0 +1,189 @@ +package infrastructure.vault; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.net.Authenticator; +import java.net.CookieHandler; +import java.net.ProxySelector; +import java.net.URI; +import java.net.http.HttpClient; +import java.net.http.HttpHeaders; +import java.net.http.HttpRequest; +import java.net.http.HttpResponse; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.StandardCopyOption; +import java.time.Duration; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.Executor; +import javax.net.ssl.SSLContext; +import javax.net.ssl.SSLParameters; +import javax.net.ssl.SSLSession; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +/** Tests Agent token handoff using a transport fixture without sockets or JVM instrumentation. */ +class AgentTokenClientTest { + @TempDir Path directory; + + @Test + void observesAtomicTokenReplacementAndNeverCallsAuthenticationEndpoints() throws Exception { + Path token = directory.resolve("token"); + Files.writeString(token, "synthetic-first-token\n"); + var transport = new Transport(new Response(200, "{}"), new Response(200, "{}")); + try (VaultClient client = new VaultClient(URI.create("https://vault:8200"), token, transport)) { + client.read("secret/data/local/identity/otp-keys"); + Path replacement = directory.resolve("replacement"); + Files.writeString(replacement, "synthetic-next-token\n"); + Files.move( + replacement, token, StandardCopyOption.ATOMIC_MOVE, StandardCopyOption.REPLACE_EXISTING); + client.read("secret/data/local/identity/otp-keys"); + } + assertEquals(2, transport.requests.size()); + assertEquals( + "synthetic-first-token", + transport.requests.get(0).headers().firstValue("X-Vault-Token").orElseThrow()); + assertEquals( + "synthetic-next-token", + transport.requests.get(1).headers().firstValue("X-Vault-Token").orElseThrow()); + assertTrue( + transport.requests.stream() + .allMatch(request -> request.uri().getPath().startsWith("/v1/secret/data/"))); + assertTrue(transport.requests.stream().allMatch(request -> request.method().equals("GET"))); + assertTrue(transport.closed); + } + + @Test + void distinguishesMissingKvFromPolicyDenialWithoutLeakingResponse() throws Exception { + Path token = directory.resolve("token"); + Files.writeString(token, "synthetic-token"); + var transport = + new Transport( + new Response(404, "{\"errors\":[]}"), new Response(403, "secret-response-marker")); + try (VaultClient client = new VaultClient(URI.create("https://vault:8200"), token, transport)) { + assertTrue( + client.readOptional("secret/data/local/identity/authenticators/cus_example").isEmpty()); + var failure = + assertThrows( + VaultUnavailableException.class, + () -> client.readOptional("secret/data/local/identity/authenticators/cus_example")); + assertFalse(failure.toString().contains("secret-response-marker")); + } + } + + @Test + void missingOrEmptyAgentTokenFailsBeforeSendingARequest() throws Exception { + var transport = new Transport(); + Path token = directory.resolve("token"); + try (VaultClient client = new VaultClient(URI.create("https://vault:8200"), token, transport)) { + assertThrows(VaultUnavailableException.class, () -> client.read("auth/token/lookup-self")); + Files.writeString(token, "\n"); + assertThrows(VaultUnavailableException.class, () -> client.read("auth/token/lookup-self")); + } + assertTrue(transport.requests.isEmpty()); + } + + private record Response(int statusCode, String body) implements HttpResponse { + public HttpRequest request() { + return HttpRequest.newBuilder(uri()).build(); + } + + public Optional> previousResponse() { + return Optional.empty(); + } + + public HttpHeaders headers() { + return HttpHeaders.of(Map.of(), (name, value) -> true); + } + + public Optional sslSession() { + return Optional.empty(); + } + + public URI uri() { + return URI.create("https://vault:8200"); + } + + public HttpClient.Version version() { + return HttpClient.Version.HTTP_1_1; + } + } + + private static final class Transport extends HttpClient { + private final ArrayDeque responses = new ArrayDeque<>(); + private final List requests = new ArrayList<>(); + private boolean closed; + + private Transport(Response... responses) { + this.responses.addAll(List.of(responses)); + } + + @Override + @SuppressWarnings("unchecked") + public HttpResponse send(HttpRequest request, HttpResponse.BodyHandler handler) { + requests.add(request); + return (HttpResponse) responses.removeFirst(); + } + + public Optional cookieHandler() { + return Optional.empty(); + } + + public Optional connectTimeout() { + return Optional.empty(); + } + + public Redirect followRedirects() { + return Redirect.NEVER; + } + + public Optional proxy() { + return Optional.empty(); + } + + public SSLContext sslContext() { + throw new UnsupportedOperationException(); + } + + public SSLParameters sslParameters() { + return new SSLParameters(); + } + + public Optional authenticator() { + return Optional.empty(); + } + + public Version version() { + return Version.HTTP_1_1; + } + + public Optional executor() { + return Optional.empty(); + } + + public CompletableFuture> sendAsync( + HttpRequest request, HttpResponse.BodyHandler handler) { + throw new UnsupportedOperationException(); + } + + public CompletableFuture> sendAsync( + HttpRequest request, + HttpResponse.BodyHandler handler, + HttpResponse.PushPromiseHandler pushHandler) { + throw new UnsupportedOperationException(); + } + + @Override + public void close() { + closed = true; + } + } +} diff --git a/payguard-shared/src/test/java/infrastructure/vault/PostgresqlMigrationRehearsalTest.java b/payguard-shared/src/test/java/infrastructure/vault/PostgresqlMigrationRehearsalTest.java deleted file mode 100644 index 485680c..0000000 --- a/payguard-shared/src/test/java/infrastructure/vault/PostgresqlMigrationRehearsalTest.java +++ /dev/null @@ -1,151 +0,0 @@ -package infrastructure.vault; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import java.sql.Connection; -import java.sql.DriverManager; -import org.flywaydb.core.Flyway; -import org.junit.jupiter.api.Test; -import org.testcontainers.postgresql.PostgreSQLContainer; -import tools.jackson.databind.json.JsonMapper; - -/** Synthetic offline import and populated PostgreSQL upgrade, without live Oracle access. */ -class PostgresqlMigrationRehearsalTest { - private static final String LOCATION = - "filesystem:../payguard-identity-service/src/main/resources/db/migration/postgresql"; - - @Test - void v1ToV2UpgradePreservesRecordsAndRejectsTwoHistoricalActiveRows() throws Exception { - try (var database = new PostgreSQLContainer("postgres:17.11-bookworm")) { - database.start(); - Flyway.configure() - .dataSource(database.getJdbcUrl(), database.getUsername(), database.getPassword()) - .schemas("payguard") - .defaultSchema("payguard") - .locations(LOCATION) - .target("1") - .load() - .migrate(); - try (Connection connection = connect(database); - var statement = connection.createStatement()) { - statement.executeUpdate(insert("A", "ACTIVE")); - statement.executeUpdate(insert("B", "ACTIVE")); - var next = - Flyway.configure() - .dataSource(database.getJdbcUrl(), database.getUsername(), database.getPassword()) - .schemas("payguard") - .defaultSchema("payguard") - .locations(LOCATION) - .load(); - assertThrows(org.flywaydb.core.api.FlywayException.class, next::migrate); - assertEquals(2, count(connection, "identity_challenge")); - /* This fixture explicitly resolves the synthetic duplicate; the migration never deletes it. */ - statement.executeUpdate( - "UPDATE identity_challenge SET status='INVALIDATED' WHERE id='otp_" - + "B".repeat(22) - + "'"); - next.migrate(); - assertEquals(2, count(connection, "identity_challenge")); - assertEquals(1, count(connection, "identity_challenge_scope")); - assertEquals(2, count(connection, "flyway_schema_history")); - } - } - } - - @Test - void importerPreservesUuidReferencesNullableFieldsAndInstantsAndRefusesPopulatedDestination() - throws Exception { - try (var database = new PostgreSQLContainer("postgres:17.11-bookworm")) { - database.start(); - Flyway.configure() - .dataSource(database.getJdbcUrl(), database.getUsername(), database.getPassword()) - .schemas("payguard") - .defaultSchema("payguard") - .locations(LOCATION) - .load() - .migrate(); - var json = JsonMapper.builder().build(); - var input = - json.readTree( - """ - {"challenges":[{"id":"otp_AAAAAAAAAAAAAAAAAAAAAA","user_id":"cus_aaaaaaaaaaaaaaaaaaaaaa", - "purpose":"LOGIN","channel":"SMS","verification_mode":"RANDOM_NUMERIC","code_hash":"synthetic-hash", - "device_binding_hash":null,"attempts":1,"max_attempts":5,"status":"USED", - "created_at":"2026-01-01T03:30:00.123456+03:30","expires_at":"2026-01-01T00:05:00Z", - "consumed_at":"2026-01-01T00:00:01.123456Z","retention_until":"2029-01-02T00:00:00Z"}], - "audit":[{"id":"00112233-4455-6677-8899-aabbccddeeff","challenge_id":"otp_AAAAAAAAAAAAAAAAAAAAAA", - "user_id":"cus_aaaaaaaaaaaaaaaaaaaaaa","action":"VERIFY","reason":"verified","source_ip":null, - "occurred_at":"2026-01-01T00:00:01.123456Z"}], - "manifest":{"challenge_count":1,"audit_count":1,"approval_reference":"synthetic-only", - "mapping_sha256":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}} - """); - try (Connection connection = connect(database)) { - var orphan = input.deepCopy(); - ((tools.jackson.databind.node.ObjectNode) orphan.path("audit").get(0)) - .put("challenge_id", "otp_" + "B".repeat(22)); - assertThrows( - java.sql.SQLException.class, () -> IdentityDataImporter.importData(connection, orphan)); - assertEquals(0, count(connection, "identity_challenge")); - assertEquals(0, count(connection, "identity_audit_log")); - var badInteger = input.deepCopy(); - ((tools.jackson.databind.node.ObjectNode) badInteger.path("challenges").get(0)) - .put("attempts", "not-an-integer"); - assertThrows( - java.sql.SQLException.class, - () -> IdentityDataImporter.importData(connection, badInteger)); - assertEquals(0, count(connection, "identity_challenge")); - IdentityDataImporter.importData(connection, input); - assertEquals(1, count(connection, "identity_challenge")); - assertEquals(1, count(connection, "identity_audit_log")); - assertThrows( - java.sql.SQLException.class, () -> IdentityDataImporter.importData(connection, input)); - try (var statement = connection.createStatement(); - var rows = - statement.executeQuery( - "SELECT id,created_at,consumed_at,device_binding_hash,user_id FROM identity_challenge")) { - assertTrue(rows.next()); - assertEquals("cus_" + "a".repeat(22), rows.getString("user_id")); - assertEquals( - java.time.Instant.parse("2026-01-01T00:00:00.123456Z"), - rows.getObject("created_at", java.time.OffsetDateTime.class).toInstant()); - assertTrue(rows.getObject("consumed_at") != null); - assertTrue(rows.getObject("device_binding_hash") == null); - } - } - } - } - - private static Connection connect(PostgreSQLContainer database) throws java.sql.SQLException { - return DriverManager.getConnection( - database.getJdbcUrl() + "¤tSchema=payguard", - database.getUsername(), - database.getPassword()); - } - - private static int count(Connection connection, String table) throws java.sql.SQLException { - try (var statement = connection.createStatement(); - var rows = - statement.executeQuery( - "SELECT count(*) FROM " - + table - + " WHERE " - + (table.equals("flyway_schema_history") - ? "success AND version IS NOT NULL" - : "true"))) { - rows.next(); - return rows.getInt(1); - } - } - - private static String insert(String suffix, String status) { - return "INSERT INTO identity_challenge(id,user_id,purpose,channel,verification_mode,code_hash,max_attempts,status,created_at,expires_at,retention_until) VALUES ('otp_" - + suffix.repeat(22) - + "','cus_" - + "A".repeat(22) - + "','LOGIN','SMS','RANDOM_NUMERIC','synthetic',5,'" - + status - + "','2026-01-01T00:00:00Z','2026-01-01T00:05:00Z','2029-01-02T00:00:00Z')"; - } -} diff --git a/payguard-shared/src/test/java/infrastructure/vault/VaultDatabaseIT.java b/payguard-shared/src/test/java/infrastructure/vault/VaultDatabaseIT.java deleted file mode 100644 index 40e9105..0000000 --- a/payguard-shared/src/test/java/infrastructure/vault/VaultDatabaseIT.java +++ /dev/null @@ -1,471 +0,0 @@ -package infrastructure.vault; - -import static org.awaitility.Awaitility.await; -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertFalse; -import static org.junit.jupiter.api.Assertions.assertThrows; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import java.net.URI; -import java.nio.file.Files; -import java.nio.file.Path; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; -import java.time.Duration; -import java.time.Instant; -import java.util.Map; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.MethodOrderer; -import org.junit.jupiter.api.Order; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestMethodOrder; -import tools.jackson.databind.JsonNode; -import tools.jackson.databind.json.JsonMapper; - -/** - * Runs only through the explicit local/CI Vault runner against provisioned, isolated TLS engines. - */ -@TestMethodOrder(MethodOrderer.OrderAnnotation.class) -class VaultDatabaseIT { - private static final Path LOCAL = Path.of("/workspace/.local/payguard"); - private static final String PROBE = - "vault_probe_" + java.util.UUID.randomUUID().toString().replace("-", ""); - private static final String ROLE = "local-identity-runtime"; - private static VaultClient operator; - private static JsonNode originalRole; - - @BeforeAll - static void prepare() throws Exception { - assertEquals("true", System.getenv("PAYGUARD_REHEARSAL")); - operator = client(LOCAL.resolve("operator/approle")); - originalRole = operator.read("database/roles/" + ROLE).path("data"); - operator.write( - "database/roles/" + ROLE, - Map.of( - "db_name", - "local-identity", - "creation_statements", - originalRole.path("creation_statements"), - "revocation_statements", - originalRole.path("revocation_statements"), - "renew_statements", - originalRole.path("renew_statements"), - "default_ttl", - "30s", - "max_ttl", - "45s")); - try (Connection connection = migration("identity"); - var statement = connection.createStatement()) { - statement.execute( - "CREATE TABLE payguard." - + PROBE - + " (id integer PRIMARY KEY, marker varchar(64) NOT NULL)"); - } - } - - @AfterAll - static void restore() throws Exception { - if (operator != null) { - try { - if (originalRole != null) { - operator.write( - "database/roles/" + ROLE, - Map.of( - "db_name", - "local-identity", - "creation_statements", - originalRole.path("creation_statements"), - "revocation_statements", - originalRole.path("revocation_statements"), - "renew_statements", - originalRole.path("renew_statements"), - "default_ttl", - "1h", - "max_ttl", - "4h")); - } - try (Connection connection = migration("identity"); - var statement = connection.createStatement()) { - statement.execute("DROP TABLE IF EXISTS payguard." + PROBE); - } - } finally { - operator.close(); - } - } - } - - @Test - @Order(1) - void independentlyScopedRuntimeRolesAreUsableAndDeniedDdlAndOtherServicePaths() throws Exception { - for (String service : new String[] {"identity", "wallet", "loan", "collateral"}) { - try (VaultClient vault = client(LOCAL.resolve("apps/" + service))) { - try (Connection owner = migration(service); - var statement = owner.createStatement()) { - statement.execute("CREATE TABLE payguard." + PROBE + "_dml(id integer PRIMARY KEY)"); - } - var credential = vault.read("database/creds/local-" + service + "-runtime"); - try (Connection connection = login(service, credential); - var statement = connection.createStatement()) { - assertTrue(connection.isValid(1)); - try (var rows = statement.executeQuery("SHOW statement_timeout")) { - assertTrue(rows.next()); - assertEquals("10s", rows.getString(1)); - } - assertEquals( - 1, statement.executeUpdate("INSERT INTO payguard." + PROBE + "_dml VALUES (1)")); - assertEquals( - 1, statement.executeUpdate("UPDATE payguard." + PROBE + "_dml SET id=2 WHERE id=1")); - try (var rows = statement.executeQuery("SELECT id FROM payguard." + PROBE + "_dml")) { - assertTrue(rows.next()); - assertEquals(2, rows.getInt(1)); - } - assertEquals( - 1, statement.executeUpdate("DELETE FROM payguard." + PROBE + "_dml WHERE id=2")); - assertThrows( - SQLException.class, - () -> statement.execute("CREATE TABLE payguard.unauthorized_probe(id integer)")); - assertThrows( - SQLException.class, - () -> statement.execute("CREATE TABLE public.unauthorized_probe(id integer)")); - assertThrows( - SQLException.class, - () -> statement.executeQuery("SELECT rolpassword FROM pg_authid")); - } finally { - try (Connection owner = migration(service); - var statement = owner.createStatement()) { - statement.execute("DROP TABLE payguard." + PROBE + "_dml"); - } - } - String other = service.equals("identity") ? "wallet" : "identity"; - assertThrows(SQLException.class, () -> login(other, credential)); - assertThrows( - VaultUnavailableException.class, - () -> vault.read("database/creds/local-" + service + "-migration")); - assertThrows( - VaultUnavailableException.class, - () -> vault.read("database/creds/local-" + other + "-runtime")); - assertThrows( - VaultUnavailableException.class, - () -> vault.readOptional("secret/data/local/" + other + "/otp-keys")); - } - } - } - - @Test - @Order(2) - void issuedCredentialCanRenewAndRevokeWithoutClaimingExistingSessionsAreDisconnected() - throws Exception { - try (VaultClient vault = client(LOCAL.resolve("apps/identity"))) { - var credential = vault.read("database/creds/" + ROLE); - String lease = credential.path("lease_id").asText(); - try (Connection existing = login("identity", credential)) { - assertTrue( - vault - .write("sys/leases/renew", Map.of("lease_id", lease)) - .path("lease_duration") - .asLong() - > 0); - vault.write("sys/leases/revoke", Map.of("lease_id", lease)); - assertThrows(SQLException.class, () -> login("identity", credential)); - /* PostgreSQL role removal prevents new login; an established session can still complete work. */ - assertTrue(existing.isValid(1)); - } - } - } - - @Test - @Order(3) - void leasePolicyCannotRevokeAnotherServiceCredential() { - try (VaultClient wallet = client(LOCAL.resolve("apps/wallet")); - VaultClient identity = client(LOCAL.resolve("apps/identity"))) { - var credential = wallet.read("database/creds/local-wallet-runtime"); - assertThrows( - VaultUnavailableException.class, - () -> - identity.write( - "sys/leases/revoke", Map.of("lease_id", credential.path("lease_id").asText()))); - } - } - - @Test - @Order(4) - void actualPoolContinuesBeyondInitialMaxTtlAndDrainsAnInFlightTransaction() throws Exception { - try (VaultClient vault = client(LOCAL.resolve("apps/identity")); - LeasedDataSource pool = - new LeasedDataSource(vault, ROLE, url("identity"), Duration.ofSeconds(10))) { - Instant began = Instant.now(); - await() - .atMost(Duration.ofSeconds(12)) - .until(() -> Instant.now().isAfter(began.plusSeconds(10))); - try (Connection held = pool.getConnection(); - var statement = held.createStatement()) { - held.setAutoCommit(false); - statement.executeUpdate( - "INSERT INTO " + PROBE + "(id,marker) VALUES (1,'synthetic-lease-probe')"); - await().atMost(Duration.ofSeconds(10)).until(() -> pool.replacementCount() >= 2); - held.commit(); - } - await() - .atMost(Duration.ofSeconds(60)) - .pollInterval(Duration.ofMillis(200)) - .until( - () -> { - try (Connection connection = pool.getConnection(); - var statement = connection.createStatement()) { - var rows = - statement.executeQuery("SELECT count(*) FROM " + PROBE + " WHERE id=1"); - return rows.next() - && rows.getInt(1) == 1 - && pool.replacementCount() >= 4 - && Instant.now().isAfter(began.plusSeconds(45)); - } - }); - } - } - - @Test - @Order(5) - void tokenRevocationReauthenticatesAndAProcessRestartUsesItsProvisionedAppRole() { - try (VaultClient vault = client(LOCAL.resolve("apps/identity"))) { - vault.read("secret/data/local/identity/otp-keys"); - vault.write("auth/token/revoke-self", Map.of()); - assertThrows( - VaultUnavailableException.class, () -> vault.read("secret/data/local/identity/otp-keys")); - assertTrue(vault.read("secret/data/local/identity/otp-keys").has("data")); - } - try (VaultClient restarted = client(LOCAL.resolve("apps/identity"))) { - assertTrue(restarted.read("secret/data/local/identity/otp-keys").has("data")); - } - } - - @Test - @Order(6) - void managerCredentialRotationInvalidatesBootstrapPasswordAndPreservesIssuance() - throws Exception { - var data = - JsonMapper.builder() - .build() - .readTree(Files.readString(LOCAL.resolve("identity/credentials.json"))); - assertThrows( - SQLException.class, - () -> - DriverManager.getConnection( - url("identity"), "vault_manager", data.path("manager").asText())); - operator.write("database/rotate-root/local-identity", Map.of()); - try (VaultClient vault = client(LOCAL.resolve("apps/identity")); - Connection connection = login("identity", vault.read("database/creds/" + ROLE))) { - assertTrue(connection.isValid(1)); - } - } - - @Test - @Order(7) - void vaultOutageUsesOnlyValidCachedCredentialsThenFailsClosedAndRecovers() throws Exception { - try (VaultClient vault = client(LOCAL.resolve("apps/identity")); - LeasedDataSource pool = - new LeasedDataSource(vault, ROLE, url("identity"), Duration.ofSeconds(10))) { - operator.write("sys/seal", Map.of()); - try { - assertThrows( - VaultUnavailableException.class, - () -> - vault.readOptional( - "secret/data/local/identity/authenticators/cus_" + "A".repeat(22))); - try (Connection connection = pool.getConnection()) { - assertTrue(connection.isValid(1)); - } - await() - .atMost(Duration.ofSeconds(35)) - .pollInterval(Duration.ofMillis(200)) - .until( - () -> { - try (Connection ignored = pool.getConnection()) { - return false; - } catch (SQLException expected) { - return true; - } - }); - } finally { - var initialization = - JsonMapper.builder() - .build() - .readTree(Files.readString(LOCAL.resolve("operator/initialization.json"))); - for (int key = 0; key < 2; key++) - operator.write( - "sys/unseal", Map.of("key", initialization.path("keys").get(key).asText())); - } - await() - .atMost(Duration.ofSeconds(20)) - .until( - () -> { - try (Connection connection = pool.getConnection()) { - return connection.isValid(1); - } catch (SQLException expected) { - return false; - } - }); - } - } - - @Test - @Order(8) - void failuresAndDiagnosticsExcludeSecretValues() { - assertThrows( - VaultUnavailableException.class, - () -> operator.readOptional("missing-vault-mount/data/no-secret")); - try (VaultClient vault = client(LOCAL.resolve("apps/wallet"))) { - var failure = - assertThrows( - VaultUnavailableException.class, - () -> vault.read("secret/data/local/identity/otp-keys")); - assertEquals( - "Required secret service is unavailable or access is denied", failure.getMessage()); - assertFalse(failure.getMessage().contains("hvs.")); - } - assertThrows( - VaultUnavailableException.class, - () -> - new VaultClient( - URI.create("https://vault:8200"), - LOCAL.resolve("ca.crt"), - LOCAL.resolve("missing-role"), - LOCAL.resolve("missing-secret")) - .read("secret/data/local/identity/otp-keys")); - } - - private static VaultClient client(Path folder) { - return new VaultClient( - URI.create("https://vault:8200"), - LOCAL.resolve("ca.crt"), - folder.resolve("role-id"), - folder.resolve("secret-id")); - } - - @Test - @Order(9) - void shortLivedAppRoleTokenRenewsWithoutReplacingItsIdentity() { - operator.write( - "auth/approle/role/local-identity", Map.of("token_ttl", "8s", "token_max_ttl", "30s")); - try (VaultClient vault = client(LOCAL.resolve("apps/identity"))) { - String accessor = vault.read("auth/token/lookup-self").path("data").path("accessor").asText(); - Instant started = Instant.now(); - await() - .atMost(Duration.ofSeconds(7)) - .until(() -> Instant.now().isAfter(started.plusSeconds(5))); - var renewed = vault.read("auth/token/lookup-self").path("data"); - assertTrue(accessor.equals(renewed.path("accessor").asText())); - assertTrue(renewed.path("ttl").asLong() >= 5); - } finally { - operator.write( - "auth/approle/role/local-identity", Map.of("token_ttl", "20m", "token_max_ttl", "4h")); - } - } - - @Test - @Order(10) - void springAutoConfigurationSuppliesTheActualVaultPoolAndReadinessContributor() { - Path scope = LOCAL.resolve("apps/identity"); - new org.springframework.boot.test.context.runner.ApplicationContextRunner() - .withConfiguration( - org.springframework.boot.autoconfigure.AutoConfigurations.of( - VaultDatabaseAutoConfiguration.class)) - .withPropertyValues( - "payguard.vault.enabled=true", - "payguard.vault.address=https://vault:8200", - "payguard.vault.ca-file=" + LOCAL.resolve("ca.crt"), - "payguard.vault.role-id-file=" + scope.resolve("role-id"), - "payguard.vault.secret-id-file=" + scope.resolve("secret-id"), - "payguard.vault.database-role=" + ROLE, - "spring.datasource.url=" + url("identity")) - .run( - context -> { - assertTrue(context.getStartupFailure() == null); - var source = context.getBean(javax.sql.DataSource.class); - assertTrue(source instanceof LeasedDataSource); - try (Connection connection = source.getConnection()) { - assertTrue(connection.isValid(1)); - } - var health = - context.getBean( - "vaultHealthIndicator", - org.springframework.boot.health.contributor.HealthIndicator.class); - assertEquals("UP", health.health().getStatus().getCode()); - }); - } - - private static String url(String service) { - return "jdbc:postgresql://postgres-" - + service - + ":5432/payguard_" - + service - + "?sslmode=verify-full&sslrootcert=/workspace/.local/payguard/ca.crt¤tSchema=payguard"; - } - - private static Connection login(String service, JsonNode credential) throws SQLException { - return DriverManager.getConnection( - url(service), - credential.path("data").path("username").asText(), - credential.path("data").path("password").asText()); - } - - private static Connection migration(String service) throws SQLException { - var credential = operator.read("database/creds/local-" + service + "-migration"); - Connection connection = login(service, credential); - try (var statement = connection.createStatement()) { - statement.execute("SET ROLE " + service + "_owner"); - } - return connection; - } - - @Test - @Order(11) - void databaseOutageDeniesConnectionsAndRecoveryDoesNotReplayATransaction() throws Exception { - Path control = Path.of("target/vault-test-reports/control"); - try (VaultClient vault = client(LOCAL.resolve("apps/identity")); - LeasedDataSource pool = - new LeasedDataSource(vault, ROLE, url("identity"), Duration.ofSeconds(10))) { - Files.writeString(control.resolve("stop.request"), "requested"); - try { - await() - .atMost(Duration.ofSeconds(15)) - .until(() -> Files.exists(control.resolve("stop.ack"))); - assertThrows(SQLException.class, pool::getConnection); - } finally { - Files.writeString(control.resolve("start.request"), "requested"); - await() - .atMost(Duration.ofSeconds(15)) - .until(() -> Files.exists(control.resolve("start.ack"))); - } - await() - .atMost(Duration.ofSeconds(20)) - .until( - () -> { - try (Connection connection = pool.getConnection()) { - return connection.isValid(1); - } catch (SQLException expected) { - return false; - } - }); - } - } - - @Test - @Order(12) - void forcedRevocationFailsBorrowingUntilAReplacementRestoresRuntimePrivileges() throws Exception { - try (VaultClient vault = client(LOCAL.resolve("apps/identity")); - LeasedDataSource pool = - new LeasedDataSource(vault, ROLE, url("identity"), Duration.ofSeconds(10))) { - try (Connection connection = pool.getConnection()) { - assertTrue(connection.isValid(1)); - } - operator.write("sys/leases/revoke-prefix/database/creds/" + ROLE, Map.of()); - assertThrows(SQLException.class, pool::getConnection); - await().atMost(Duration.ofSeconds(15)).until(() -> pool.replacementCount() >= 2); - try (Connection connection = pool.getConnection()) { - assertTrue(connection.isValid(1)); - } - } - } -} diff --git a/payguard-wallet-service/src/main/resources/application.yaml b/payguard-wallet-service/src/main/resources/application.yaml index f0db46a..963bee0 100644 --- a/payguard-wallet-service/src/main/resources/application.yaml +++ b/payguard-wallet-service/src/main/resources/application.yaml @@ -4,6 +4,12 @@ spring: datasource: url: ${SPRING_DATASOURCE_URL} driver-class-name: org.postgresql.Driver + hikari: + maximum-pool-size: 10 + connection-timeout: 5000 + validation-timeout: 2000 + max-lifetime: 300000 + keepalive-time: 30000 jooq: sql-dialect: POSTGRES @@ -12,9 +18,7 @@ payguard: enabled: true address: ${VAULT_ADDR} ca-file: ${VAULT_CA_FILE} - role-id-file: ${VAULT_ROLE_ID_FILE} - secret-id-file: ${VAULT_SECRET_ID_FILE} - database-role: ${PAYGUARD_ENVIRONMENT:local}-wallet-runtime + token-file: ${VAULT_TOKEN_FILE} management: endpoint: @@ -28,3 +32,8 @@ management: web: exposure: include: health,info + +server: + shutdown: graceful + +spring.lifecycle.timeout-per-shutdown-phase: 20s diff --git a/payguard-wallet-service/src/main/resources/db/migration/postgresql/V1__wallet_schema_marker.sql b/payguard-wallet-service/src/main/resources/db/migration/postgresql/V1__wallet_schema_marker.sql new file mode 100644 index 0000000..b0fe140 --- /dev/null +++ b/payguard-wallet-service/src/main/resources/db/migration/postgresql/V1__wallet_schema_marker.sql @@ -0,0 +1,3 @@ +-- Wallet currently contains domain objects and no persistence adapter. +-- Version its independent schema without inventing an unimplemented ledger model. +SELECT 1; diff --git a/pom.xml b/pom.xml index 72b5783..e5ee5f6 100644 --- a/pom.xml +++ b/pom.xml @@ -75,25 +75,6 @@ - - org.flywaydb - flyway-maven-plugin - 12.4.0 - - - - DO 'BEGIN RAISE EXCEPTION ''FLYWAY_INIT_SQL must select the service owner''; END;' - payguard - payguard - false - true - filesystem:${project.basedir}/src/main/resources/db/migration/postgresql - - - org.flywaydbflyway-database-postgresql12.4.0 - org.postgresqlpostgresql42.7.13 - - org.springframework.boot spring-boot-maven-plugin diff --git a/scripts/local/common.py b/scripts/local/common.py deleted file mode 100644 index 48aaef2..0000000 --- a/scripts/local/common.py +++ /dev/null @@ -1,99 +0,0 @@ -"""Protected local-rehearsal I/O; never include dependency responses in diagnostics.""" -import json -import os -from pathlib import Path -import ssl -import subprocess -import tempfile -import urllib.error -import urllib.request - -ROOT = Path(__file__).resolve().parents[2] -LOCAL = ROOT / ".local/payguard" -SERVICES = ("wallet", "loan", "collateral", "identity") -JAVA_IMAGE = "eclipse-temurin:25-jdk-alpine@sha256:3fd2d245c4e0eba615fe366a71b8bd25f5db7104f53e4026b24bf508b880bd2a" - - -class DependencyError(RuntimeError): - def __init__(self, status=0): - self.status = status - super().__init__("Local dependency unavailable or access denied; details suppressed") - - -def write(path, value): - """Atomic replacement avoids partially read SecretIDs during reauthentication.""" - path = Path(path) - path.parent.mkdir(parents=True, exist_ok=True, mode=0o700) - descriptor, temporary = tempfile.mkstemp(dir=path.parent) - try: - with os.fdopen(descriptor, "w") as output: - output.write(value) - os.chmod(temporary, 0o600) - os.replace(temporary, path) - finally: - if os.path.exists(temporary): - os.unlink(temporary) - - -def read(path): - return Path(path).read_text().strip() - - -def run(arguments, **kwargs): - result = subprocess.run(arguments, cwd=ROOT, capture_output=True, **kwargs) - if result.returncode: - raise DependencyError() - return result.stdout - - -class Vault: - def __init__(self, token=""): - self.token = token - port = int(os.environ.get("VAULT_PORT", "8200")) - self.address = f"https://127.0.0.1:{port}/v1/" - self.http = urllib.request.build_opener( - urllib.request.ProxyHandler({}), - urllib.request.HTTPSHandler(context=ssl.create_default_context(cafile=LOCAL / "ca.crt")), - ) - - def request(self, method, path, body=None, wrap=False): - headers = {"Content-Type": "application/json"} - if self.token: - headers["X-Vault-Token"] = self.token - if wrap: - headers["X-Vault-Wrap-TTL"] = "60s" - payload = None if body is None else json.dumps(body).encode() - request = urllib.request.Request(self.address + path, payload, headers, method=method) - try: - with self.http.open(request, timeout=10) as response: - data = response.read() - return json.loads(data) if data else {} - except urllib.error.HTTPError as error: - raise DependencyError(error.code) from None - except (OSError, ValueError): - raise DependencyError() from None - - def get(self, path): - return self.request("GET", path) - - def post(self, path, body=None): - return self.request("POST", path, {} if body is None else body) - - def provision(self, role, folder): - role_id = self.get(f"auth/approle/role/{role}/role-id")["data"]["role_id"] - wrapped = self.request("POST", f"auth/approle/role/{role}/secret-id", {}, wrap=True) - one_use = Vault(wrapped["wrap_info"]["token"]) - secret_id = one_use.post("sys/wrapping/unwrap")["data"]["secret_id"] - write(folder / "role-id", role_id) - write(folder / "secret-id", secret_id) - - -def operator(): - return Vault(read(LOCAL / "operator/token")) - - -def main(action): - try: - action() - except (DependencyError, OSError, ValueError, KeyError): - raise SystemExit("Local rehearsal failed; protected dependency details suppressed") from None diff --git a/scripts/local/migrate.py b/scripts/local/migrate.py deleted file mode 100644 index 706ac44..0000000 --- a/scripts/local/migrate.py +++ /dev/null @@ -1,46 +0,0 @@ -"""Run supplied Flyway schemas with separate migration leases, never runtime privileges.""" -import argparse -import os -from pathlib import Path -from common import JAVA_IMAGE, LOCAL, ROOT, Vault, main, read, run - - -def migrate(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("service", choices=("identity", "loan", "collateral")) - parser.add_argument("--import-identity", type=Path) - arguments = parser.parse_args() - service = arguments.service - imported = arguments.import_identity - if imported is not None: - imported = imported.resolve() - if service != "identity" or not imported.is_file() or not imported.is_relative_to(LOCAL): - raise ValueError("Explicit protected Identity export required") - scope = LOCAL / "operator" / service - vault = Vault() - auth = vault.post("auth/approle/login", { - "role_id": read(scope / "role-id"), "secret_id": read(scope / "secret-id")}) - vault.token = auth["auth"]["client_token"] - try: - credentials = vault.get("database/creds/local-" + service + "-migration")["data"] - environment = os.environ.copy() - environment.update({ - "PAYGUARD_MIGRATION_SERVICE": service, - "PAYGUARD_MIGRATION_JDBC_URL": f"jdbc:postgresql://postgres-{service}:5432/payguard_{service}?sslmode=verify-full&sslrootcert=/workspace/.local/payguard/ca.crt¤tSchema=payguard", - "PAYGUARD_MIGRATION_USERNAME": credentials["username"], - "PAYGUARD_MIGRATION_PASSWORD": credentials["password"]}) - command = ["docker", "run", "--rm", "--network", "payguard_database", - "--user", f"{os.getuid()}:{os.getgid()}", "-v", f"{ROOT}:/workspace:ro", "-w", "/workspace"] - for name in ("PAYGUARD_MIGRATION_SERVICE", "PAYGUARD_MIGRATION_JDBC_URL", "PAYGUARD_MIGRATION_USERNAME", "PAYGUARD_MIGRATION_PASSWORD"): - command += ["-e", name] - command += [JAVA_IMAGE, "java", "-cp", "payguard-shared/target/classes:payguard-shared/target/dependency/*"] - entry = ["infrastructure.vault.SchemaMigrator"] if imported is None else [ - "infrastructure.vault.IdentityDataImporter", "/workspace/" + str(imported.relative_to(ROOT))] - run(command + entry, env=environment) - print(f"{service}: separately privileged migration/import completed") - finally: - vault.post("auth/token/revoke-self") - - -if __name__ == "__main__": - main(migrate) diff --git a/scripts/local/prepare.py b/scripts/local/prepare.py deleted file mode 100644 index 4563255..0000000 --- a/scripts/local/prepare.py +++ /dev/null @@ -1,68 +0,0 @@ -"""Create new protected artifacts. Existing data and artifacts are never overwritten.""" -import json -import os -import secrets -from common import LOCAL, SERVICES, main, run, write - - -def certificate(folder, name, alternatives): - folder.mkdir(mode=0o700, parents=True) - run(["openssl", "req", "-new", "-newkey", "rsa:3072", "-nodes", - "-keyout", str(folder / "server.key"), "-out", str(folder / "server.csr"), - "-subj", f"/CN={name}"]) - write(folder / "extensions.cnf", "basicConstraints=CA:FALSE\nkeyUsage=digitalSignature,keyEncipherment\nextendedKeyUsage=serverAuth\nsubjectAltName=" + alternatives + "\n") - run(["openssl", "x509", "-req", "-in", str(folder / "server.csr"), - "-CA", str(LOCAL / "ca.crt"), "-CAkey", str(LOCAL / "ca.key"), - "-CAcreateserial", "-days", "30", "-sha256", "-extfile", str(folder / "extensions.cnf"), - "-out", str(folder / "server.crt")]) - for path in folder.iterdir(): - path.chmod(0o600) - - -def sql(service, credentials): - owner, runtime = f"{service}_owner", f"{service}_runtime" - return f"""REVOKE ALL ON DATABASE payguard_{service} FROM PUBLIC; -REVOKE ALL ON SCHEMA public FROM PUBLIC; -CREATE ROLE {owner} NOLOGIN; -CREATE ROLE {runtime} NOLOGIN; -CREATE ROLE vault_manager LOGIN CREATEROLE PASSWORD '{credentials['manager']}'; -CREATE ROLE bootstrap_migrator LOGIN PASSWORD '{credentials['migration']}'; -GRANT {owner} TO bootstrap_migrator; -GRANT {owner}, {runtime} TO vault_manager WITH ADMIN TRUE, INHERIT FALSE, SET FALSE; -GRANT CONNECT ON DATABASE payguard_{service} TO {owner}, {runtime}, vault_manager; -CREATE SCHEMA payguard AUTHORIZATION {owner}; -GRANT USAGE ON SCHEMA payguard TO {runtime}, vault_manager; -ALTER DEFAULT PRIVILEGES FOR ROLE {owner} IN SCHEMA payguard - GRANT SELECT, INSERT, UPDATE, DELETE ON TABLES TO {runtime}; -ALTER DEFAULT PRIVILEGES FOR ROLE {owner} IN SCHEMA payguard - GRANT USAGE, SELECT ON SEQUENCES TO {runtime}; -ALTER DEFAULT PRIVILEGES FOR ROLE {owner} REVOKE EXECUTE ON FUNCTIONS FROM PUBLIC; -""" - - -def prepare(): - if LOCAL.exists(): - raise ValueError("Refuse existing protected artifacts") - os.umask(0o077) - LOCAL.mkdir(parents=True, mode=0o700) - run(["openssl", "req", "-x509", "-newkey", "rsa:3072", "-nodes", "-sha256", "-days", "30", - "-subj", "/CN=PayGuard disposable rehearsal CA", "-addext", "basicConstraints=critical,CA:TRUE", - "-addext", "keyUsage=critical,keyCertSign,cRLSign", "-keyout", str(LOCAL / "ca.key"), - "-out", str(LOCAL / "ca.crt")]) - (LOCAL / "ca.key").chmod(0o600) - (LOCAL / "ca.crt").chmod(0o600) - for service in SERVICES: - folder = LOCAL / service - certificate(folder, "postgres-" + service, "DNS:postgres-" + service) - credentials = {kind: secrets.token_hex(32) for kind in ("admin", "manager", "migration")} - write(folder / "credentials.json", json.dumps(credentials)) - write(folder / "admin-password", credentials["admin"]) - write(folder / "init.sql", sql(service, credentials)) - (LOCAL / "apps" / service).mkdir(parents=True, mode=0o700) - certificate(LOCAL / "vault", "vault", "DNS:vault,DNS:localhost,IP:127.0.0.1") - write(LOCAL / "vault/ca.crt", (LOCAL / "ca.crt").read_text()) - print("Created protected local TLS and bootstrap artifacts; no secret values displayed") - - -if __name__ == "__main__": - main(prepare) diff --git a/scripts/local/refresh_approle.py b/scripts/local/refresh_approle.py deleted file mode 100644 index d403829..0000000 --- a/scripts/local/refresh_approle.py +++ /dev/null @@ -1,29 +0,0 @@ -"""Operator-only SecretID replacement; supervise --watch for extended local rehearsal.""" -import argparse -import time -from common import LOCAL, SERVICES, main, operator - - -def refresh(): - vault = operator() - vault.post("auth/token/renew-self") - for service in SERVICES: - vault.provision("local-" + service, LOCAL / "apps" / service) - vault.provision("local-" + service + "-migration", LOCAL / "operator" / service) - vault.provision("local-operator", LOCAL / "operator/approle") - print("Renewed operator token and atomically replaced scoped SecretID files") - - -def action(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--watch", action="store_true") - watch = parser.parse_args().watch - while True: - refresh() - if not watch: - return - time.sleep(20 * 60) - - -if __name__ == "__main__": - main(action) diff --git a/scripts/local/vault_bootstrap.py b/scripts/local/vault_bootstrap.py deleted file mode 100644 index f6b2361..0000000 --- a/scripts/local/vault_bootstrap.py +++ /dev/null @@ -1,194 +0,0 @@ -"""Explicit local operator bootstrap/recovery, verified HTTPS and protected artifacts.""" -import argparse -import base64 -import json -import secrets -import time -from common import DependencyError, LOCAL, SERVICES, Vault, main, read, write - - -def rule(path, capabilities): - return f'path "{path}" {{ capabilities = {json.dumps(capabilities)} }}\n' - - -def own_token_policy(): - return (rule("auth/token/lookup-self", ["read"]) - + rule("auth/token/renew-self", ["update"]) - + rule("auth/token/revoke-self", ["update"])) - - -def runtime_policy(service): - role = "local-" + service + "-runtime" - policy = own_token_policy() + rule("database/creds/" + role, ["read"]) - for operation in ("renew", "revoke"): - policy += f'''path "sys/leases/{operation}" {{ - capabilities = ["update"] - allowed_parameters = {{ "lease_id" = ["database/creds/{role}/*"], "increment" = ["*"] }} -}}\n''' - if service == "identity": - policy += rule("secret/data/local/identity/otp-keys", ["read"]) - policy += rule("secret/data/local/identity/authenticators/*", ["read"]) - return policy - - -def operator_policy(): - policy = own_token_policy() - for path in ("sys/mounts", "sys/auth", "sys/audit"): - policy += rule(path, ["read", "sudo"]) - for path in ("sys/mounts/secret", "sys/mounts/database", "sys/auth/approle", - "sys/audit/file", "sys/seal", "sys/policies/acl/local-*", - "auth/token/create-orphan", "sys/leases/revoke-prefix/database/creds/local-*"): - policy += rule(path, ["create", "read", "update", "sudo"]) - for path in ("database/config/local-*", "database/roles/local-*", "database/rotate-root/local-*", - "auth/approle/role/local-*", "secret/data/local/*", - "secret/destroy/local/*"): - policy += rule(path, ["create", "read", "update", "delete", "list"]) - for service in SERVICES: - policy += rule("database/creds/local-" + service + "-migration", ["read"]) - for operation in ("renew", "revoke"): - policy += rule("sys/leases/" + operation, ["update"]) - return policy - - -def recover(vault, initialization): - progress = vault.get("sys/generate-root/attempt") - if progress.get("started"): - raise ValueError("An operator recovery is already in progress") - started = vault.post("sys/generate-root/attempt") - try: - result = {} - for key in initialization["keys"][:2]: - result = vault.post("sys/generate-root/update", {"key": key, "nonce": started["nonce"]}) - if not result.get("complete"): - raise ValueError("Recovery threshold not met") - encoded = result.get("encoded_token") or result.get("encoded_root_token") - raw = base64.b64decode(encoded + "=" * (-len(encoded) % 4)) - otp = started["otp"].encode() - return Vault(bytes(left ^ right for left, right in zip(raw, otp)).decode()) - except Exception: - vault.request("DELETE", "sys/generate-root/attempt") - raise - - -def configure(vault): - vault.post("sys/policies/acl/local-operator", {"policy": operator_policy()}) - for path, body in (("sys/mounts/secret", {"type": "kv", "options": {"version": "2"}}), - ("sys/mounts/database", {"type": "database"}), - ("sys/auth/approle", {"type": "approle"}), - ("sys/audit/file", {"type": "file", "options": {"file_path": "/vault/audit/audit.json"}})): - listing = "sys/auth" if path.startswith("sys/auth") else "sys/audit" if path.startswith("sys/audit") else "sys/mounts" - name = path.rsplit("/", 1)[1] + "/" - existing = vault.get(listing) - if name not in existing.get("data", existing): - vault.post(path, body) - vault.post("sys/policies/acl/local-operator", {"policy": operator_policy()}) - for service in SERVICES: - name = "local-" + service - credentials = json.loads(read(LOCAL / service / "credentials.json")) - try: - vault.get("database/config/" + name) - except DependencyError as failure: - if failure.status != 404: - raise - vault.post("database/config/" + name, { - "plugin_name": "postgresql-database-plugin", "allowed_roles": [name + "-runtime", name + "-migration"], - "connection_url": "postgresql://{{username}}:{{password}}@postgres-" + service + ":5432/payguard_" + service + "?sslmode=verify-full&sslrootcert=/vault/tls/ca.crt", - "username": "vault_manager", "password": credentials["manager"], "verify_connection": True}) - vault.post("database/rotate-root/" + name) - for kind, ttl, maximum in (("runtime", "1h", "4h"), ("migration", "15m", "30m")): - group = service + ("_runtime" if kind == "runtime" else "_owner") - role = name + "-" + kind - timeouts = (''' ALTER ROLE "{{name}}" SET statement_timeout='10s'; - ALTER ROLE "{{name}}" SET lock_timeout='10s'; - ALTER ROLE "{{name}}" SET idle_in_transaction_session_timeout='10s';''' if kind == "runtime" else "") - vault.post("database/roles/" + role, { - "db_name": name, - "creation_statements": [f'''CREATE ROLE "{{{{name}}}}" WITH LOGIN PASSWORD '{{{{password}}}}' VALID UNTIL '{{{{expiration}}}}'; GRANT {group} TO "{{{{name}}}}";''' + timeouts], - "revocation_statements": [f'REVOKE {group} FROM "{{{{name}}}}"; DROP ROLE IF EXISTS "{{{{name}}}}";'], - "renew_statements": ['ALTER ROLE "{{name}}" VALID UNTIL \'{{expiration}}\';'], - "default_ttl": ttl, "max_ttl": maximum}) - policy = runtime_policy(service) if kind == "runtime" else own_token_policy() + rule("database/creds/" + role, ["read"]) - policy_name = name if kind == "runtime" else name + "-migration" - vault.post("sys/policies/acl/" + policy_name, {"policy": policy}) - vault.post("auth/approle/role/" + policy_name, { - "token_policies": [policy_name], "secret_id_ttl": "1h", "secret_id_num_uses": 0, - "token_ttl": "20m", "token_max_ttl": "4h", "token_no_default_policy": True}) - destination = LOCAL / "apps" / service if kind == "runtime" else LOCAL / "operator" / service - vault.provision(policy_name, destination) - vault.post("auth/approle/role/local-operator", { - "token_policies": ["local-operator"], "secret_id_ttl": "1h", "secret_id_num_uses": 0, - "token_ttl": "20m", "token_max_ttl": "4h", "token_no_default_policy": True}) - vault.provision("local-operator", LOCAL / "operator/approle") - try: - vault.get("secret/data/local/identity/otp-keys") - except DependencyError as failure: - if failure.status != 404: - raise - vault.post("secret/data/local/identity/otp-keys", { - "data": {"current": "localInitial", "keys": {"localInitial": secrets.token_urlsafe(48)}}}) - - -def bootstrap(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--recover-operator", action="store_true") - parser.add_argument("--diagnostics", action="store_true", help="Display only failed API path/status; never response bodies or credentials") - arguments = parser.parse_args() - if arguments.diagnostics: - request = Vault.request - - def diagnosed(self, method, path, *args, **kwargs): - try: - return request(self, method, path, *args, **kwargs) - except DependencyError as failure: - print(f"Failed bootstrap API: {method} {path}, HTTP {failure.status}", flush=True) - raise - - Vault.request = diagnosed - vault = Vault() - deadline = time.monotonic() + 45 - while True: - try: - status = vault.get("sys/init") - break - except DependencyError: - if time.monotonic() >= deadline: - raise - time.sleep(0.2) - temporary_root = LOCAL / "operator/bootstrap-root" - if not status["initialized"]: - initialized = vault.post("sys/init", {"secret_shares": 3, "secret_threshold": 2}) - write(temporary_root, initialized.pop("root_token")) - write(LOCAL / "operator/initialization.json", json.dumps(initialized)) - initialization = json.loads(read(LOCAL / "operator/initialization.json")) - if vault.get("sys/seal-status")["sealed"]: - for key in initialization["keys"][:2]: - vault.post("sys/unseal", {"key": key}) - while not vault.get("sys/leader").get("is_self"): - if time.monotonic() >= deadline: - raise DependencyError() - time.sleep(0.2) - root = False - if temporary_root.exists(): - vault = Vault(read(temporary_root)) - root = True - elif arguments.recover_operator: - vault = recover(vault, initialization) - root = True - else: - vault = Vault(read(LOCAL / "operator/token")) - vault.post("auth/token/renew-self") - try: - if root: - vault.post("sys/policies/acl/local-operator", {"policy": operator_policy()}) - issued = vault.post("auth/token/create-orphan", {"policies": ["local-operator"], "ttl": "8h", "renewable": True}) - write(LOCAL / "operator/token", issued["auth"]["client_token"]) - configure(vault) - finally: - if root: - vault.post("auth/token/revoke-self") - temporary_root.unlink(missing_ok=True) - print("Vault initialized/unsealed and scoped local identities provisioned; no root token retained") - - -if __name__ == "__main__": - main(bootstrap) diff --git a/scripts/local/verify_applications.py b/scripts/local/verify_applications.py deleted file mode 100644 index 542a9ff..0000000 --- a/scripts/local/verify_applications.py +++ /dev/null @@ -1,66 +0,0 @@ -"""Boot actual packaged applications with only their own AppRole files and verified CA.""" -import argparse -import os -import subprocess -import time -import uuid -from common import DependencyError, LOCAL, ROOT, SERVICES, main, run - -JRE_IMAGE = "eclipse-temurin:25-jre-alpine@sha256:3c0a9084927a221ccd1d007fcaf614465672c0af37aaa834c5184483afe56d61" - - -def verify(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--service", choices=SERVICES) - selected = parser.parse_args().service - for service in SERVICES if selected is None else (selected,): - module = ROOT / f"payguard-{service}-service" - jars = list((module / "target").glob(f"payguard-{service}-service-*.jar")) - if len(jars) != 1: - raise ValueError("Build actual service JARs before runtime verification") - name = "payguard-application-probe-" + service + "-" + uuid.uuid4().hex[:8] - run(["docker", "run", "-d", "--name", name, "--network", "payguard_database", - "--user", f"{os.getuid()}:{os.getgid()}", "-v", f"{jars[0]}:/app.jar:ro", - "-v", f"{LOCAL / 'apps' / service}:/run/payguard/approle:ro", - "-v", f"{LOCAL / 'ca.crt'}:/run/payguard/ca.crt:ro", - "-e", f"SPRING_DATASOURCE_URL=jdbc:postgresql://postgres-{service}:5432/payguard_{service}?sslmode=verify-full&sslrootcert=/run/payguard/ca.crt¤tSchema=payguard", - "-e", "VAULT_ADDR=https://vault:8200", "-e", "VAULT_CA_FILE=/run/payguard/ca.crt", - "-e", "VAULT_ROLE_ID_FILE=/run/payguard/approle/role-id", "-e", "VAULT_SECRET_ID_FILE=/run/payguard/approle/secret-id", - "-e", "REDIS_HOST=redis", "-e", "KAFKA_BOOTSTRAP_SERVERS=kafka:29092", - JRE_IMAGE, "java", "-jar", "/app.jar"]) - try: - port = 8084 if service == "identity" else 8080 - deadline = time.monotonic() + 60 - ready = False - while time.monotonic() < deadline: - status = run(["docker", "inspect", "-f", "{{.State.Running}}", name]).strip() - if status != b"true": - break - try: - health = run(["docker", "exec", name, "wget", "-qO-", f"http://127.0.0.1:{port}/actuator/health/readiness"]) - if b'"status":"UP"' in health: - ready = True - break - except DependencyError: - pass - time.sleep(0.2) - captured = subprocess.run(["docker", "logs", name], capture_output=True, check=True) - logs = captured.stdout + captured.stderr - markers = [(LOCAL / "apps" / service / "secret-id").read_bytes().strip(), - (LOCAL / "operator/token").read_bytes().strip()] - if any(marker in logs for marker in markers): - raise ValueError("Secret-bearing application logs detected") - if not ready: - # Only exception class names are emitted, never arbitrary application messages. - import re - classes = sorted(set(re.findall(rb"(?:[a-z]+\.)+[A-Za-z]+Exception", logs))) - print(service + ": readiness failed; exception types " + ", ".join(value.decode() for value in classes), flush=True) - raise DependencyError() - print(service + ": actual packaged application readiness UP; scoped AppRole and PostgreSQL TLS active", flush=True) - finally: - run(["docker", "stop", "-t", "10", name]) - run(["docker", "rm", name]) - - -if __name__ == "__main__": - main(verify) diff --git a/scripts/local/verify_vault.py b/scripts/local/verify_vault.py deleted file mode 100644 index 618a0de..0000000 --- a/scripts/local/verify_vault.py +++ /dev/null @@ -1,80 +0,0 @@ -"""Real isolated TLS dependency tests. Controls only the local rehearsal database/Vault.""" -import json -import os -import subprocess -import time -import uuid -import xml.etree.ElementTree as ET -from common import DependencyError, JAVA_IMAGE, LOCAL, ROOT, main, run - - -def verify(): - reports = ROOT / "payguard-shared/target/vault-test-reports" - control = reports / "control" - control.mkdir(parents=True, exist_ok=True) - for name in ("stop.request", "stop.ack", "start.request", "start.ack"): - (control / name).unlink(missing_ok=True) - compose = ["docker", "compose", "--env-file", ".env.example"] - name = "payguard-vault-verification-" + uuid.uuid4().hex[:12] - command = ["docker", "run", "--rm", "--name", name, "--network", "payguard_database", - "--user", f"{os.getuid()}:{os.getgid()}", "-v", f"{ROOT}:/workspace", - "-w", "/workspace/payguard-shared", "-e", "PAYGUARD_REHEARSAL=true", JAVA_IMAGE, - "java", "-cp", "target/classes:target/test-classes:target/dependency/*:/workspace/payguard-identity-service/target/classes:/workspace/payguard-identity-service/target/test-classes:/workspace/payguard-identity-service/target/dependency/*", - "org.junit.platform.console.ConsoleLauncher", "execute", "--disable-ansi-colors", - "--select-class", "infrastructure.vault.VaultDatabaseIT", - "--select-class", "dev.amg.payguard.identity.verification.infrastructure.security.VaultSecretsIT", - "--reports-dir", "target/vault-test-reports"] - began = time.monotonic() - stopped = False - with open(os.devnull, "wb") as null: - process = subprocess.Popen(command, cwd=ROOT, stdout=null, stderr=null) - try: - while process.poll() is None: - if time.monotonic() - began > 240: - run(["docker", "stop", "-t", "5", name]) - raise DependencyError() - if (control / "stop.request").exists() and not (control / "stop.ack").exists(): - run(compose + ["stop", "postgres-identity"]) - stopped = True - (control / "stop.ack").write_text("stopped") - if (control / "start.request").exists() and not (control / "start.ack").exists(): - run(compose + ["up", "-d", "--wait", "postgres-identity"]) - stopped = False - (control / "start.ack").write_text("healthy") - time.sleep(0.2) - finally: - if stopped: - run(compose + ["up", "-d", "--wait", "postgres-identity"]) - if process.poll() is None: - run(["docker", "stop", "-t", "5", name]) - process.wait(timeout=15) - counts = {key: 0 for key in ("tests", "failures", "errors", "skipped")} - for report in reports.glob("TEST-*.xml"): - suite = ET.parse(report).getroot() - for key in counts: - counts[key] += int(suite.get(key, "0")) - print("Vault/PostgreSQL tests:", json.dumps(counts), flush=True) - if process.returncode or counts != {"tests": 15, "failures": 0, "errors": 0, "skipped": 0}: - raise DependencyError() - audit = run(compose + ["exec", "-T", "vault", "cat", "/vault/audit/audit.json"]) - records = [json.loads(line) for line in audit.splitlines() if line.strip()] - if not any(record.get("error") and record.get("request", {}).get("path", "").startswith("secret/data/local/") for record in records): - raise ValueError("Denied audit evidence missing") - if not any(record.get("type") == "response" and not record.get("error") and record.get("request", {}).get("path") == "database/creds/local-identity-runtime" for record in records): - raise ValueError("Allowed audit evidence missing") - markers = [b"synthetic-first-pepper-marker-123456789", b"synthetic-second-pepper-marker-123456789", - b"GEZDGNBVGY3TQOJQGEZDGNBVGY3TQOJQ"] - for path in (LOCAL / "operator/token", LOCAL / "apps/identity/secret-id"): - markers.append(path.read_bytes().strip()) - if any(marker in audit for marker in markers): - raise ValueError("Audit sensitive-value protection failed") - if not any(str(record.get("auth", {}).get("client_token", "")).startswith("hmac-sha256:") for record in records): - raise ValueError("Audit token protection evidence missing") - for report in reports.glob("TEST-*.xml"): - if any(marker in report.read_bytes() for marker in markers): - raise ValueError("Secret-bearing diagnostics found") - print("Allowed/denied accesses traceable; audit and report secret-marker checks passed") - - -if __name__ == "__main__": - main(verify) diff --git a/scripts/migration/convert_identity_export.py b/scripts/migration/convert_identity_export.py deleted file mode 100644 index 913bbac..0000000 --- a/scripts/migration/convert_identity_export.py +++ /dev/null @@ -1,87 +0,0 @@ -"""Convert authorized offline Identity JSON using a persistent approved ID mapping.""" -import argparse -from datetime import datetime, timezone -import hashlib -import json -import os -from pathlib import Path -import re -import uuid - - -def identifier(value, prefix, mapping): - if not isinstance(value, str): - raise ValueError("Identifier must be an exact string") - if re.fullmatch(prefix + r"_[A-Za-z0-9]{22}", value): - return value - converted = mapping.get(value) - if not isinstance(converted, str) or not re.fullmatch(prefix + r"_[A-Za-z0-9]{22}", converted): - raise ValueError("Approved historical identifier mapping required") - return converted - - -def timestamp(value): - if value is None: - return None - if not isinstance(value, str) or not re.search(r"(?:Z|[+-]\d{2}:\d{2})$", value): - raise ValueError("Timestamp offset required") - value = re.sub(r"(\.\d{6})\d+", r"\1", value) - return datetime.fromisoformat(value.replace("Z", "+00:00")).astimezone(timezone.utc).isoformat(timespec="microseconds") - - -def convert(source, mapping, mapping_bytes): - if not mapping.get("approved_by") or not mapping.get("approval_reference"): - raise ValueError("Approved provenance required") - customers, challenges = mapping.get("customers", {}), mapping.get("challenges", {}) - result = {"challenges": [], "audit": []} - identities = {} - for original in source["challenges"]: - row = dict(original) - row["id"] = identifier(row["id"], "otp", challenges) - row["user_id"] = identifier(row["user_id"], "cus", customers) - if row["id"] in identities: - raise ValueError("Duplicate converted challenge") - identities[row["id"]] = row["user_id"] - for field in ("created_at", "expires_at", "consumed_at", "retention_until"): - row[field] = timestamp(row.get(field)) - for field in ("attempts", "max_attempts"): - if type(row.get(field)) is not int: - raise ValueError("Integer attempts required") - result["challenges"].append(row) - audit_ids = set() - for original in source["audit"]: - row = dict(original) - row["id"] = str(uuid.UUID(row["id"])) - row["challenge_id"] = identifier(row["challenge_id"], "otp", challenges) - row["user_id"] = identifier(row["user_id"], "cus", customers) - if row["id"] in audit_ids or identities.get(row["challenge_id"]) != row["user_id"]: - raise ValueError("Duplicate or orphan/mismatched audit reference") - audit_ids.add(row["id"]) - row["occurred_at"] = timestamp(row["occurred_at"]) - result["audit"].append(row) - result["manifest"] = {"approved_by": mapping["approved_by"], "approval_reference": mapping["approval_reference"], - "mapping_sha256": hashlib.sha256(mapping_bytes).hexdigest(), - "challenge_count": len(result["challenges"]), "audit_count": len(result["audit"]), - "timestamp_policy": "UTC microseconds; truncate sub-microsecond precision"} - return result - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("source", type=Path) - parser.add_argument("mapping", type=Path) - parser.add_argument("destination", type=Path) - arguments = parser.parse_args() - try: - mapping_bytes = arguments.mapping.read_bytes() - converted = convert(json.loads(arguments.source.read_text()), json.loads(mapping_bytes), mapping_bytes) - descriptor = os.open(arguments.destination, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600) - with os.fdopen(descriptor, "w") as output: - json.dump(converted, output, indent=2) - print("Protected export converted with approved mapping and reconciliation manifest") - except (OSError, ValueError, KeyError, TypeError): - raise SystemExit("Conversion refused; protected source details suppressed") from None - - -if __name__ == "__main__": - main() diff --git a/scripts/migration/test_convert_identity_export.py b/scripts/migration/test_convert_identity_export.py deleted file mode 100644 index 09bee40..0000000 --- a/scripts/migration/test_convert_identity_export.py +++ /dev/null @@ -1,52 +0,0 @@ -"""Synthetic conversion contract; no live source or invented customer mappings.""" -import unittest -from convert_identity_export import convert - - -class IdentityExportTest(unittest.TestCase): - def fixture(self): - self.customer = "cus_" + "A" * 22 - self.challenge = "otp_" + "b" * 22 - source = {"challenges": [{"id": "old-challenge", "user_id": "old-customer", "attempts": 0, "max_attempts": 5, - "created_at": "2026-01-01T03:30:00.123456789+03:30", "expires_at": "2026-01-01T00:35:00Z"}], - "audit": [{"id": "00112233445566778899aabbccddeeff", "challenge_id": "old-challenge", "user_id": "old-customer", "occurred_at": "2026-01-01T00:00:00Z"}]} - mapping = {"approved_by": "synthetic operator", "approval_reference": "fixture approval", - "customers": {"old-customer": self.customer}, "challenges": {"old-challenge": self.challenge}} - return source, mapping - - def test_approved_mapping_preserves_uuid_byte_order_and_truncates_microseconds(self): - source, mapping = self.fixture() - result = convert(source, mapping, b"synthetic approved mapping bytes") - self.assertEqual(self.challenge, result["challenges"][0]["id"]) - self.assertEqual("2026-01-01T00:00:00.123456+00:00", result["challenges"][0]["created_at"]) - self.assertEqual("00112233-4455-6677-8899-aabbccddeeff", result["audit"][0]["id"]) - self.assertEqual(1, result["manifest"]["audit_count"]) - - def test_current_case_sensitive_ids_pass_through_and_missing_mapping_is_refused(self): - source, mapping = self.fixture() - source["challenges"][0]["id"] = self.challenge - result = convert(source, mapping, b"mapping") - self.assertEqual(self.challenge, result["challenges"][0]["id"]) - mapping["customers"] = {} - with self.assertRaises(ValueError): - convert(source, mapping, b"mapping") - - def test_orphans_duplicates_unapproved_mapping_and_offsetless_time_are_refused(self): - for defect in ("orphan", "duplicate", "approval", "offset", "integer"): - source, mapping = self.fixture() - if defect == "orphan": - source["audit"][0]["challenge_id"] = "otp_" + "Q" * 22 - elif defect == "duplicate": - source["challenges"].append(dict(source["challenges"][0])) - elif defect == "approval": - mapping.pop("approval_reference") - elif defect == "offset": - source["challenges"][0]["created_at"] = "2026-01-01T00:00:00" - else: - source["challenges"][0]["attempts"] = "0" - with self.assertRaises(ValueError): - convert(source, mapping, b"mapping") - - -if __name__ == "__main__": - unittest.main() From 76f3293cf5aa62274a4dcb0fec673807c9a627d9 Mon Sep 17 00:00:00 2001 From: Amir Golmoradi <132334913+Amir-Golmoradi@users.noreply.github.com> Date: Fri, 30 Oct 2026 18:40:46 +0330 Subject: [PATCH 2/2] Keep Flyway test migrations in the payguard schema --- .../src/test/resources/application-test.yaml | 5 +++++ .../src/test/resources/application-test.yaml | 5 +++++ .../src/test/resources/application-test.yaml | 5 +++++ 3 files changed, 15 insertions(+) diff --git a/payguard-collateral-service/src/test/resources/application-test.yaml b/payguard-collateral-service/src/test/resources/application-test.yaml index 3f74d54..66ad59a 100644 --- a/payguard-collateral-service/src/test/resources/application-test.yaml +++ b/payguard-collateral-service/src/test/resources/application-test.yaml @@ -4,6 +4,11 @@ payguard: spring: flyway: enabled: true + locations: classpath:db/migration/postgresql + default-schema: payguard + schemas: payguard + create-schemas: false + clean-disabled: true management: endpoint: diff --git a/payguard-identity-service/src/test/resources/application-test.yaml b/payguard-identity-service/src/test/resources/application-test.yaml index 56269a9..cde5cb2 100644 --- a/payguard-identity-service/src/test/resources/application-test.yaml +++ b/payguard-identity-service/src/test/resources/application-test.yaml @@ -4,6 +4,11 @@ payguard: spring: flyway: enabled: true + locations: classpath:db/migration/postgresql + default-schema: payguard + schemas: payguard + create-schemas: false + clean-disabled: true jpa: open-in-view: false diff --git a/payguard-loan-service/src/test/resources/application-test.yaml b/payguard-loan-service/src/test/resources/application-test.yaml index 3f74d54..66ad59a 100644 --- a/payguard-loan-service/src/test/resources/application-test.yaml +++ b/payguard-loan-service/src/test/resources/application-test.yaml @@ -4,6 +4,11 @@ payguard: spring: flyway: enabled: true + locations: classpath:db/migration/postgresql + default-schema: payguard + schemas: payguard + create-schemas: false + clean-disabled: true management: endpoint: