From 5955b58f0adbe27b280cb127346b4725ff2da527 Mon Sep 17 00:00:00 2001 From: Lazar Kanelov Date: Tue, 27 Oct 2026 11:09:02 +0200 Subject: [PATCH 1/2] Add the Snowflake Next application samples Move the eight samples from localstack/snowflake-next-sample-apps into this repository as top-level directories: claims-service, fleet-telemetry, metering-api, migrate-and-swap, order-to-cash-cdc, phi-guardrails, slim-ci-lab and tenantboard. - add install, start, stop (and logs where missing) targets so the existing CI matrix drives them like the other samples - extend the CI workflow with the uv, Java, Go and .NET toolchains, a nightly run against the newest snowflake-next image, and the auth token for the test and log steps (Testcontainers starts its own emulator) - rename the Go module of metering-api to this repository's path - ignore generated key pairs and toolchain output, and pin line endings for scripts, mvnw and mvnw.cmd --- .gitattributes | 8 + .github/workflows/main.yml | 44 + .gitignore | 18 + README.md | 15 +- claims-service/.env.example | 31 + .../.mvn/wrapper/maven-wrapper.properties | 3 + claims-service/Makefile | 77 + claims-service/README.md | 136 ++ claims-service/claims-api/pom.xml | 105 + .../com/example/claims/ClaimsApplication.java | 14 + .../example/claims/api/ClaimsController.java | 73 + .../com/example/claims/api/ErrorHandler.java | 72 + .../com/example/claims/api/KpiController.java | 33 + .../example/claims/config/PrivateKeys.java | 52 + .../config/SnowflakeDataSourceConfig.java | 43 + .../claims/config/SnowflakeProperties.java | 41 + .../claims/domain/BatchIntakeResult.java | 22 + .../java/com/example/claims/domain/Claim.java | 20 + .../example/claims/domain/ClaimStatus.java | 8 + .../com/example/claims/domain/MonthlyKpi.java | 13 + .../com/example/claims/domain/NewClaim.java | 26 + .../example/claims/repo/ClaimsRepository.java | 217 ++ .../example/claims/repo/KpiRepository.java | 41 + .../com/example/claims/seed/ClaimSeeder.java | 73 + .../example/claims/seed/SyntheticClaims.java | 53 + .../claims/service/AdjudicationService.java | 47 + .../claims/service/ClaimIntakeService.java | 64 + .../service/ClaimNotAdjudicableException.java | 25 + .../service/ClaimNotFoundException.java | 15 + .../service/DuplicateClaimException.java | 15 + .../claims/service/InvalidClaimException.java | 36 + .../example/claims/service/KpiService.java | 29 + .../src/main/resources/application-cloud.yml | 11 + .../src/main/resources/application.yml | 39 + .../db/changelog/db.changelog-master.yaml | 20 + .../db/changelog/sql/001-icd10-function.sql | 16 + .../resources/db/changelog/sql/001-schema.sql | 85 + .../sql/002-adjudicate-procedure.sql | 111 + .../db/changelog/sql/003-kpi-stream-task.sql | 58 + .../db/changelog/sql/004-reference-data.sql | 42 + .../claims/AdjudicationErrorPathTest.java | 121 ++ .../example/claims/ClaimsIntegrationTest.java | 201 ++ .../com/example/claims/Icd10FunctionTest.java | 53 + .../claims/IntegrationTestSupport.java | 88 + .../claims/SnowflakeEmulatorContainer.java | 47 + .../example/claims/SnowflakeTestTarget.java | 228 +++ .../src/test/resources/logback-test.xml | 15 + claims-service/docker-compose.yml | 20 + claims-service/docs/architecture.md | 93 + claims-service/init/01_platform.sf.sql | 29 + claims-service/mvnw | 295 +++ claims-service/mvnw.cmd | 189 ++ claims-service/pom.xml | 32 + claims-service/scripts/demo.sh | 42 + claims-service/scripts/test-summary.sh | 22 + claims-service/scripts/wait-for-emulator.sh | 24 + fleet-telemetry/.editorconfig | 29 + fleet-telemetry/.env.example | 29 + fleet-telemetry/.gitignore | 5 + fleet-telemetry/Directory.Build.props | 15 + fleet-telemetry/Directory.Packages.props | 16 + fleet-telemetry/FleetTelemetry.slnx | 12 + fleet-telemetry/Makefile | 80 + fleet-telemetry/README.md | 143 ++ fleet-telemetry/docker-compose.yml | 19 + fleet-telemetry/docs/architecture.md | 106 + fleet-telemetry/global.json | 9 + fleet-telemetry/init/01_platform.sf.sql | 31 + fleet-telemetry/init/02_landing.sf.sql | 21 + fleet-telemetry/init/03_fleet_registry.sf.sql | 60 + fleet-telemetry/init/04_typed_views.sf.sql | 39 + fleet-telemetry/init/05_fleet_kpis.sf.sql | 24 + fleet-telemetry/scripts/demo.sh | 78 + fleet-telemetry/scripts/wait-for-emulator.sh | 23 + .../BadRequestExceptionHandler.cs | 33 + .../FleetTelemetry.Api/Endpoints/Contracts.cs | 46 + .../Endpoints/FleetEndpoints.cs | 31 + .../FleetTelemetry.Api/Endpoints/Problems.cs | 24 + .../Endpoints/TelemetryEndpoints.cs | 44 + .../Endpoints/VehicleEndpoints.cs | 138 ++ .../FleetTelemetry.Api.csproj | 11 + .../FleetTelemetry.Api/HttpQueryTagSource.cs | 18 + .../src/FleetTelemetry.Api/Program.cs | 26 + .../Properties/launchSettings.json | 13 + .../SnowflakeHealthCheck.cs | 23 + .../appsettings.Development.json | 17 + .../src/FleetTelemetry.Api/appsettings.json | 24 + .../src/FleetTelemetry.Api/packages.lock.json | 287 +++ .../FleetTelemetry.Core.csproj | 8 + .../Kpis/FleetDailyKpis.cs | 51 + .../src/FleetTelemetry.Core/Kpis/KpiMath.cs | 24 + .../Rules/DrivingEventRules.cs | 28 + .../FleetTelemetry.Core/Rules/TripRules.cs | 15 + .../Simulation/FleetSimulator.cs | 311 +++ .../src/FleetTelemetry.Core/Simulation/Geo.cs | 34 + .../Simulation/SimulationModel.cs | 34 + .../Simulation/TelemetryExport.cs | 39 + .../Telemetry/TelemetryBatchParser.cs | 168 ++ .../Telemetry/TelemetryJson.cs | 29 + .../Telemetry/TelemetryPing.cs | 24 + .../FleetTelemetry.Core/packages.lock.json | 6 + .../src/FleetTelemetry.Data/AnalyticsRows.cs | 67 + .../src/FleetTelemetry.Data/FleetAnalytics.cs | 57 + .../src/FleetTelemetry.Data/FleetDatabase.cs | 77 + .../src/FleetTelemetry.Data/FleetRegistry.cs | 47 + .../FleetTelemetry.Data.csproj | 20 + .../ServiceCollectionExtensions.cs | 24 + .../FleetTelemetry.Data/SnowflakeOptions.cs | 121 ++ .../FleetTelemetry.Data/TelemetryBackfill.cs | 138 ++ .../FleetTelemetry.Data/TelemetryWriter.cs | 23 + .../FleetTelemetry.Data/VehicleAnalytics.cs | 156 ++ .../FleetTelemetry.Data/packages.lock.json | 402 ++++ .../src/FleetTelemetry.Ingest/Commands.cs | 89 + .../FleetTelemetry.Ingest.csproj | 20 + .../src/FleetTelemetry.Ingest/Program.cs | 31 + .../appsettings.Development.json | 8 + .../FleetTelemetry.Ingest/appsettings.json | 26 + .../FleetTelemetry.Ingest/packages.lock.json | 586 ++++++ .../BackfillTests.cs | 94 + .../FleetKpiTests.cs | 85 + .../FleetTelemetry.IntegrationTests.csproj | 26 + .../Infrastructure/Expected.cs | 62 + .../Infrastructure/FleetDatabaseFixture.cs | 113 ++ .../SnowflakeOptionsExtensions.cs | 34 + .../Infrastructure/SnowflakeTarget.cs | 172 ++ .../Infrastructure/SqlScripts.cs | 60 + .../TelemetryIngestTests.cs | 139 ++ .../VehicleAnalyticsTests.cs | 193 ++ .../packages.lock.json | 823 ++++++++ .../xunit.runner.json | 4 + .../DrivingEventRulesTests.cs | 33 + .../FleetDailyKpisTests.cs | 65 + .../FleetSimulatorTests.cs | 144 ++ .../FleetTelemetry.UnitTests.csproj | 18 + .../FleetTelemetry.UnitTests/KpiMathTests.cs | 39 + .../TelemetryBatchParserTests.cs | 117 ++ .../TripRulesTests.cs | 22 + .../packages.lock.json | 137 ++ metering-api/.env.example | 33 + metering-api/.gitignore | 6 + metering-api/Makefile | 72 + metering-api/README.md | 142 ++ metering-api/cmd/api/main.go | 84 + metering-api/cmd/demo/main.go | 255 +++ metering-api/cmd/seed/main.go | 73 + metering-api/docker-compose.yml | 21 + metering-api/docs/architecture.md | 114 ++ metering-api/go.mod | 112 + metering-api/go.sum | 266 +++ metering-api/init/01_platform.sf.sql | 31 + metering-api/init/02_schema.sf.sql | 74 + metering-api/init/03_price_plans.sf.sql | 32 + metering-api/internal/config/config.go | 171 ++ metering-api/internal/config/config_test.go | 127 ++ metering-api/internal/metering/dates.go | 87 + metering-api/internal/metering/events.go | 162 ++ .../metering/export_integration_test.go | 14 + .../metering/harness_integration_test.go | 344 ++++ metering-api/internal/metering/http.go | 265 +++ metering-api/internal/metering/http_test.go | 80 + metering-api/internal/metering/invoices.go | 237 +++ .../metering/metering_integration_test.go | 411 ++++ metering-api/internal/metering/problem.go | 46 + metering-api/internal/metering/rollups.go | 144 ++ metering-api/internal/metering/service.go | 141 ++ metering-api/internal/metering/usage.go | 63 + metering-api/internal/metering/validate.go | 101 + .../internal/metering/validate_test.go | 111 + metering-api/internal/pricing/pricing.go | 107 + metering-api/internal/pricing/pricing_test.go | 128 ++ metering-api/internal/seed/seed.go | 128 ++ metering-api/internal/seed/seed_test.go | 87 + metering-api/scripts/demo.sh | 27 + metering-api/scripts/wait-for-emulator.sh | 23 + migrate-and-swap/.env.example | 35 + migrate-and-swap/.python-version | 1 + migrate-and-swap/Makefile | 73 + migrate-and-swap/README.md | 119 ++ .../demos/V1.3.0__bad_negative_stock.sql | 15 + migrate-and-swap/docker-compose.yml | 18 + migrate-and-swap/docs/architecture.md | 95 + migrate-and-swap/init/01_bootstrap.sf.sql | 24 + migrate-and-swap/migrations/A__grants.sql | 10 + .../migrations/R__inventory_views.sql | 42 + .../migrations/V1.0.0__baseline.sql | 99 + .../migrations/V1.1.0__add_reorder_point.sql | 24 + .../migrations/V1.2.0__split_location.sql | 27 + migrate-and-swap/pyproject.toml | 44 + migrate-and-swap/release/__init__.py | 1 + migrate-and-swap/release/__main__.py | 5 + migrate-and-swap/release/dataset.py | 192 ++ migrate-and-swap/release/demo.py | 115 ++ migrate-and-swap/release/deploy.py | 127 ++ migrate-and-swap/release/gate.py | 110 + migrate-and-swap/release/recovery.py | 51 + migrate-and-swap/release/release.py | 206 ++ migrate-and-swap/release/settings.py | 148 ++ migrate-and-swap/release/validations.sql | 67 + migrate-and-swap/schemachange-config.yml | 9 + migrate-and-swap/scripts/seed.py | 23 + migrate-and-swap/scripts/wait-for-emulator.sh | 24 + migrate-and-swap/tests/conftest.py | 117 ++ migrate-and-swap/tests/test_recovery.py | 95 + migrate-and-swap/tests/test_release.py | 284 +++ migrate-and-swap/uv.lock | 547 +++++ order-to-cash-cdc/.env.example | 30 + order-to-cash-cdc/Makefile | 72 + order-to-cash-cdc/README.md | 114 ++ order-to-cash-cdc/connect/Dockerfile | 35 + order-to-cash-cdc/connect/snowflake-sink.json | 26 + order-to-cash-cdc/connect/worker.properties | 25 + order-to-cash-cdc/docker-compose.yml | 64 + order-to-cash-cdc/docs/architecture.md | 91 + order-to-cash-cdc/init/01_landing.sf.sql | 45 + order-to-cash-cdc/init/02_core.sf.sql | 94 + order-to-cash-cdc/init/03_pipeline.sf.sql | 243 +++ order-to-cash-cdc/pipeline/__init__.py | 0 order-to-cash-cdc/pipeline/demo.py | 109 + order-to-cash-cdc/pipeline/harness.py | 166 ++ order-to-cash-cdc/pipeline/kafka_connect.py | 152 ++ order-to-cash-cdc/pipeline/seed.py | 39 + order-to-cash-cdc/pipeline/settings.py | 96 + order-to-cash-cdc/pipeline/snowflake_db.py | 97 + order-to-cash-cdc/producer/__init__.py | 0 order-to-cash-cdc/producer/produce.py | 356 ++++ order-to-cash-cdc/pyproject.toml | 26 + .../scripts/wait-for-emulator.sh | 24 + order-to-cash-cdc/tests/__init__.py | 0 order-to-cash-cdc/tests/conftest.py | 65 + order-to-cash-cdc/tests/model.py | 105 + order-to-cash-cdc/tests/test_model.py | 38 + order-to-cash-cdc/tests/test_pipeline.py | 176 ++ order-to-cash-cdc/tests/test_transactions.py | 47 + order-to-cash-cdc/uv.lock | 323 +++ phi-guardrails/.env.example | 29 + phi-guardrails/.python-version | 1 + phi-guardrails/Makefile | 59 + phi-guardrails/README.md | 125 ++ phi-guardrails/access-contract.yml | 111 + .../demos/broken-current-role-policy.sql | 22 + phi-guardrails/docker-compose.yml | 18 + phi-guardrails/docs/architecture.md | 107 + phi-guardrails/init/01_bootstrap.sf.sql | 18 + .../migrations/R__analyst_views.sql | 51 + .../migrations/V1.0.0__roles_and_schema.sql | 122 ++ .../migrations/V1.1.0__tags_and_masking.sql | 71 + .../V1.2.0__facility_row_access.sql | 34 + .../migrations/V1.3.0__add_patient_email.sql | 13 + phi-guardrails/pyproject.toml | 43 + phi-guardrails/schemachange-config.yml | 9 + phi-guardrails/scripts/load_synthea.py | 36 + phi-guardrails/scripts/wait-for-emulator.sh | 24 + phi-guardrails/src/phi_guardrails/__init__.py | 1 + phi-guardrails/src/phi_guardrails/contract.py | 132 ++ phi-guardrails/src/phi_guardrails/coverage.py | 119 ++ phi-guardrails/src/phi_guardrails/demo.py | 116 ++ phi-guardrails/src/phi_guardrails/deploy.py | 92 + phi-guardrails/src/phi_guardrails/drill.py | 47 + phi-guardrails/src/phi_guardrails/load.py | 41 + phi-guardrails/src/phi_guardrails/matrix.py | 227 +++ phi-guardrails/src/phi_guardrails/settings.py | 129 ++ phi-guardrails/src/phi_guardrails/synthea.py | 183 ++ phi-guardrails/tests/conftest.py | 78 + phi-guardrails/tests/test_access_matrix.py | 80 + phi-guardrails/tests/test_deploy.py | 32 + phi-guardrails/tests/test_load.py | 21 + phi-guardrails/tests/test_policy_coverage.py | 137 ++ phi-guardrails/uv.lock | 523 +++++ slim-ci-lab/.env.example | 27 + .../.github-example/workflows/dbt-ci.yml | 137 ++ slim-ci-lab/.gitignore | 17 + slim-ci-lab/Makefile | 67 + slim-ci-lab/README.md | 133 ++ slim-ci-lab/dbt/dbt_project.yml | 47 + slim-ci-lab/dbt/macros/drop_pr_schema.sql | 10 + .../dbt/macros/generate_schema_name.sql | 30 + slim-ci-lab/dbt/macros/surrogate_key.sql | 12 + .../intermediate/int_trips_enriched.sql | 82 + .../models/marts/fct_daily_zone_revenue.sql | 15 + slim-ci-lab/dbt/models/schema.yml | 87 + slim-ci-lab/dbt/models/sources.yml | 17 + .../dbt/models/staging/stg_taxi_zones.sql | 7 + slim-ci-lab/dbt/models/staging/stg_trips.sql | 38 + slim-ci-lab/dbt/profiles.yml | 48 + slim-ci-lab/dbt/seeds/taxi_zone_lookup.csv | 266 +++ slim-ci-lab/dbt/seeds/taxi_zone_lookup.yml | 10 + slim-ci-lab/dbt/snapshots/dim_zone.yml | 16 + slim-ci-lab/demos/bad-unique-key.patch | 20 + slim-ci-lab/demos/example-pr.patch | 19 + slim-ci-lab/docker-compose.yml | 18 + slim-ci-lab/docs/architecture.md | 92 + slim-ci-lab/init/01_platform.sf.sql | 27 + slim-ci-lab/pyproject.toml | 36 + slim-ci-lab/scripts/ci_local.sh | 78 + slim-ci-lab/scripts/wait-for-emulator.sh | 24 + slim-ci-lab/src/slim_ci_lab/__init__.py | 0 slim-ci-lab/src/slim_ci_lab/bad_pr_demo.py | 92 + slim-ci-lab/src/slim_ci_lab/ci_summary.py | 60 + slim-ci-lab/src/slim_ci_lab/dbt_runner.py | 68 + slim-ci-lab/src/slim_ci_lab/loader.py | 124 ++ slim-ci-lab/src/slim_ci_lab/settings.py | 122 ++ slim-ci-lab/src/slim_ci_lab/tlc.py | 89 + slim-ci-lab/tests/conftest.py | 71 + slim-ci-lab/tests/test_load.py | 39 + slim-ci-lab/tests/test_models.py | 95 + slim-ci-lab/tests/test_slim_ci.py | 89 + slim-ci-lab/uv.lock | 1209 +++++++++++ tenantboard/.env.example | 41 + tenantboard/.gitignore | 13 + tenantboard/Makefile | 77 + tenantboard/README.md | 135 ++ tenantboard/app/__init__.py | 0 tenantboard/app/db.py | 197 ++ tenantboard/app/errors.py | 102 + tenantboard/app/main.py | 175 ++ tenantboard/app/models.py | 91 + tenantboard/app/provisioning.py | 206 ++ tenantboard/app/repository.py | 235 +++ tenantboard/app/settings.py | 97 + tenantboard/app/tenancy.py | 38 + tenantboard/app/uploads.py | 100 + tenantboard/docker-compose.yml | 18 + tenantboard/docs/architecture.md | 118 ++ tenantboard/init/01_platform.sf.sql | 42 + tenantboard/init/02_policies.sf.sql | 25 + tenantboard/init/03_tables.sf.sql | 59 + tenantboard/init/04_grants.sf.sql | 31 + tenantboard/pyproject.toml | 45 + tenantboard/scripts/__init__.py | 0 tenantboard/scripts/demo.py | 92 + tenantboard/scripts/run.sh | 41 + tenantboard/scripts/seed.py | 271 +++ tenantboard/scripts/wait-for-emulator.sh | 24 + tenantboard/scripts/wait_for_init.py | 53 + tenantboard/tests/__init__.py | 0 tenantboard/tests/conftest.py | 102 + tenantboard/tests/helpers.py | 75 + tenantboard/tests/test_api.py | 194 ++ tenantboard/tests/test_isolation.py | 124 ++ tenantboard/tests/test_session_cache.py | 93 + tenantboard/tests/test_uploads.py | 119 ++ tenantboard/uv.lock | 724 +++++++ tenantboard/web/index.html | 13 + tenantboard/web/package-lock.json | 1794 +++++++++++++++++ tenantboard/web/package.json | 27 + tenantboard/web/src/App.tsx | 287 +++ tenantboard/web/src/RevenueChart.tsx | 129 ++ tenantboard/web/src/api.ts | 123 ++ tenantboard/web/src/format.ts | 29 + tenantboard/web/src/main.tsx | 12 + tenantboard/web/src/styles.css | 177 ++ tenantboard/web/tsconfig.json | 17 + tenantboard/web/vite.config.ts | 20 + 353 files changed, 32263 insertions(+), 2 deletions(-) create mode 100644 .gitattributes create mode 100644 claims-service/.env.example create mode 100644 claims-service/.mvn/wrapper/maven-wrapper.properties create mode 100644 claims-service/Makefile create mode 100644 claims-service/README.md create mode 100644 claims-service/claims-api/pom.xml create mode 100644 claims-service/claims-api/src/main/java/com/example/claims/ClaimsApplication.java create mode 100644 claims-service/claims-api/src/main/java/com/example/claims/api/ClaimsController.java create mode 100644 claims-service/claims-api/src/main/java/com/example/claims/api/ErrorHandler.java create mode 100644 claims-service/claims-api/src/main/java/com/example/claims/api/KpiController.java create mode 100644 claims-service/claims-api/src/main/java/com/example/claims/config/PrivateKeys.java create mode 100644 claims-service/claims-api/src/main/java/com/example/claims/config/SnowflakeDataSourceConfig.java create mode 100644 claims-service/claims-api/src/main/java/com/example/claims/config/SnowflakeProperties.java create mode 100644 claims-service/claims-api/src/main/java/com/example/claims/domain/BatchIntakeResult.java create mode 100644 claims-service/claims-api/src/main/java/com/example/claims/domain/Claim.java create mode 100644 claims-service/claims-api/src/main/java/com/example/claims/domain/ClaimStatus.java create mode 100644 claims-service/claims-api/src/main/java/com/example/claims/domain/MonthlyKpi.java create mode 100644 claims-service/claims-api/src/main/java/com/example/claims/domain/NewClaim.java create mode 100644 claims-service/claims-api/src/main/java/com/example/claims/repo/ClaimsRepository.java create mode 100644 claims-service/claims-api/src/main/java/com/example/claims/repo/KpiRepository.java create mode 100644 claims-service/claims-api/src/main/java/com/example/claims/seed/ClaimSeeder.java create mode 100644 claims-service/claims-api/src/main/java/com/example/claims/seed/SyntheticClaims.java create mode 100644 claims-service/claims-api/src/main/java/com/example/claims/service/AdjudicationService.java create mode 100644 claims-service/claims-api/src/main/java/com/example/claims/service/ClaimIntakeService.java create mode 100644 claims-service/claims-api/src/main/java/com/example/claims/service/ClaimNotAdjudicableException.java create mode 100644 claims-service/claims-api/src/main/java/com/example/claims/service/ClaimNotFoundException.java create mode 100644 claims-service/claims-api/src/main/java/com/example/claims/service/DuplicateClaimException.java create mode 100644 claims-service/claims-api/src/main/java/com/example/claims/service/InvalidClaimException.java create mode 100644 claims-service/claims-api/src/main/java/com/example/claims/service/KpiService.java create mode 100644 claims-service/claims-api/src/main/resources/application-cloud.yml create mode 100644 claims-service/claims-api/src/main/resources/application.yml create mode 100644 claims-service/claims-api/src/main/resources/db/changelog/db.changelog-master.yaml create mode 100644 claims-service/claims-api/src/main/resources/db/changelog/sql/001-icd10-function.sql create mode 100644 claims-service/claims-api/src/main/resources/db/changelog/sql/001-schema.sql create mode 100644 claims-service/claims-api/src/main/resources/db/changelog/sql/002-adjudicate-procedure.sql create mode 100644 claims-service/claims-api/src/main/resources/db/changelog/sql/003-kpi-stream-task.sql create mode 100644 claims-service/claims-api/src/main/resources/db/changelog/sql/004-reference-data.sql create mode 100644 claims-service/claims-api/src/test/java/com/example/claims/AdjudicationErrorPathTest.java create mode 100644 claims-service/claims-api/src/test/java/com/example/claims/ClaimsIntegrationTest.java create mode 100644 claims-service/claims-api/src/test/java/com/example/claims/Icd10FunctionTest.java create mode 100644 claims-service/claims-api/src/test/java/com/example/claims/IntegrationTestSupport.java create mode 100644 claims-service/claims-api/src/test/java/com/example/claims/SnowflakeEmulatorContainer.java create mode 100644 claims-service/claims-api/src/test/java/com/example/claims/SnowflakeTestTarget.java create mode 100644 claims-service/claims-api/src/test/resources/logback-test.xml create mode 100644 claims-service/docker-compose.yml create mode 100644 claims-service/docs/architecture.md create mode 100644 claims-service/init/01_platform.sf.sql create mode 100755 claims-service/mvnw create mode 100644 claims-service/mvnw.cmd create mode 100644 claims-service/pom.xml create mode 100755 claims-service/scripts/demo.sh create mode 100755 claims-service/scripts/test-summary.sh create mode 100755 claims-service/scripts/wait-for-emulator.sh create mode 100644 fleet-telemetry/.editorconfig create mode 100644 fleet-telemetry/.env.example create mode 100644 fleet-telemetry/.gitignore create mode 100644 fleet-telemetry/Directory.Build.props create mode 100644 fleet-telemetry/Directory.Packages.props create mode 100644 fleet-telemetry/FleetTelemetry.slnx create mode 100644 fleet-telemetry/Makefile create mode 100644 fleet-telemetry/README.md create mode 100644 fleet-telemetry/docker-compose.yml create mode 100644 fleet-telemetry/docs/architecture.md create mode 100644 fleet-telemetry/global.json create mode 100644 fleet-telemetry/init/01_platform.sf.sql create mode 100644 fleet-telemetry/init/02_landing.sf.sql create mode 100644 fleet-telemetry/init/03_fleet_registry.sf.sql create mode 100644 fleet-telemetry/init/04_typed_views.sf.sql create mode 100644 fleet-telemetry/init/05_fleet_kpis.sf.sql create mode 100755 fleet-telemetry/scripts/demo.sh create mode 100755 fleet-telemetry/scripts/wait-for-emulator.sh create mode 100644 fleet-telemetry/src/FleetTelemetry.Api/BadRequestExceptionHandler.cs create mode 100644 fleet-telemetry/src/FleetTelemetry.Api/Endpoints/Contracts.cs create mode 100644 fleet-telemetry/src/FleetTelemetry.Api/Endpoints/FleetEndpoints.cs create mode 100644 fleet-telemetry/src/FleetTelemetry.Api/Endpoints/Problems.cs create mode 100644 fleet-telemetry/src/FleetTelemetry.Api/Endpoints/TelemetryEndpoints.cs create mode 100644 fleet-telemetry/src/FleetTelemetry.Api/Endpoints/VehicleEndpoints.cs create mode 100644 fleet-telemetry/src/FleetTelemetry.Api/FleetTelemetry.Api.csproj create mode 100644 fleet-telemetry/src/FleetTelemetry.Api/HttpQueryTagSource.cs create mode 100644 fleet-telemetry/src/FleetTelemetry.Api/Program.cs create mode 100644 fleet-telemetry/src/FleetTelemetry.Api/Properties/launchSettings.json create mode 100644 fleet-telemetry/src/FleetTelemetry.Api/SnowflakeHealthCheck.cs create mode 100644 fleet-telemetry/src/FleetTelemetry.Api/appsettings.Development.json create mode 100644 fleet-telemetry/src/FleetTelemetry.Api/appsettings.json create mode 100644 fleet-telemetry/src/FleetTelemetry.Api/packages.lock.json create mode 100644 fleet-telemetry/src/FleetTelemetry.Core/FleetTelemetry.Core.csproj create mode 100644 fleet-telemetry/src/FleetTelemetry.Core/Kpis/FleetDailyKpis.cs create mode 100644 fleet-telemetry/src/FleetTelemetry.Core/Kpis/KpiMath.cs create mode 100644 fleet-telemetry/src/FleetTelemetry.Core/Rules/DrivingEventRules.cs create mode 100644 fleet-telemetry/src/FleetTelemetry.Core/Rules/TripRules.cs create mode 100644 fleet-telemetry/src/FleetTelemetry.Core/Simulation/FleetSimulator.cs create mode 100644 fleet-telemetry/src/FleetTelemetry.Core/Simulation/Geo.cs create mode 100644 fleet-telemetry/src/FleetTelemetry.Core/Simulation/SimulationModel.cs create mode 100644 fleet-telemetry/src/FleetTelemetry.Core/Simulation/TelemetryExport.cs create mode 100644 fleet-telemetry/src/FleetTelemetry.Core/Telemetry/TelemetryBatchParser.cs create mode 100644 fleet-telemetry/src/FleetTelemetry.Core/Telemetry/TelemetryJson.cs create mode 100644 fleet-telemetry/src/FleetTelemetry.Core/Telemetry/TelemetryPing.cs create mode 100644 fleet-telemetry/src/FleetTelemetry.Core/packages.lock.json create mode 100644 fleet-telemetry/src/FleetTelemetry.Data/AnalyticsRows.cs create mode 100644 fleet-telemetry/src/FleetTelemetry.Data/FleetAnalytics.cs create mode 100644 fleet-telemetry/src/FleetTelemetry.Data/FleetDatabase.cs create mode 100644 fleet-telemetry/src/FleetTelemetry.Data/FleetRegistry.cs create mode 100644 fleet-telemetry/src/FleetTelemetry.Data/FleetTelemetry.Data.csproj create mode 100644 fleet-telemetry/src/FleetTelemetry.Data/ServiceCollectionExtensions.cs create mode 100644 fleet-telemetry/src/FleetTelemetry.Data/SnowflakeOptions.cs create mode 100644 fleet-telemetry/src/FleetTelemetry.Data/TelemetryBackfill.cs create mode 100644 fleet-telemetry/src/FleetTelemetry.Data/TelemetryWriter.cs create mode 100644 fleet-telemetry/src/FleetTelemetry.Data/VehicleAnalytics.cs create mode 100644 fleet-telemetry/src/FleetTelemetry.Data/packages.lock.json create mode 100644 fleet-telemetry/src/FleetTelemetry.Ingest/Commands.cs create mode 100644 fleet-telemetry/src/FleetTelemetry.Ingest/FleetTelemetry.Ingest.csproj create mode 100644 fleet-telemetry/src/FleetTelemetry.Ingest/Program.cs create mode 100644 fleet-telemetry/src/FleetTelemetry.Ingest/appsettings.Development.json create mode 100644 fleet-telemetry/src/FleetTelemetry.Ingest/appsettings.json create mode 100644 fleet-telemetry/src/FleetTelemetry.Ingest/packages.lock.json create mode 100644 fleet-telemetry/tests/FleetTelemetry.IntegrationTests/BackfillTests.cs create mode 100644 fleet-telemetry/tests/FleetTelemetry.IntegrationTests/FleetKpiTests.cs create mode 100644 fleet-telemetry/tests/FleetTelemetry.IntegrationTests/FleetTelemetry.IntegrationTests.csproj create mode 100644 fleet-telemetry/tests/FleetTelemetry.IntegrationTests/Infrastructure/Expected.cs create mode 100644 fleet-telemetry/tests/FleetTelemetry.IntegrationTests/Infrastructure/FleetDatabaseFixture.cs create mode 100644 fleet-telemetry/tests/FleetTelemetry.IntegrationTests/Infrastructure/SnowflakeOptionsExtensions.cs create mode 100644 fleet-telemetry/tests/FleetTelemetry.IntegrationTests/Infrastructure/SnowflakeTarget.cs create mode 100644 fleet-telemetry/tests/FleetTelemetry.IntegrationTests/Infrastructure/SqlScripts.cs create mode 100644 fleet-telemetry/tests/FleetTelemetry.IntegrationTests/TelemetryIngestTests.cs create mode 100644 fleet-telemetry/tests/FleetTelemetry.IntegrationTests/VehicleAnalyticsTests.cs create mode 100644 fleet-telemetry/tests/FleetTelemetry.IntegrationTests/packages.lock.json create mode 100644 fleet-telemetry/tests/FleetTelemetry.IntegrationTests/xunit.runner.json create mode 100644 fleet-telemetry/tests/FleetTelemetry.UnitTests/DrivingEventRulesTests.cs create mode 100644 fleet-telemetry/tests/FleetTelemetry.UnitTests/FleetDailyKpisTests.cs create mode 100644 fleet-telemetry/tests/FleetTelemetry.UnitTests/FleetSimulatorTests.cs create mode 100644 fleet-telemetry/tests/FleetTelemetry.UnitTests/FleetTelemetry.UnitTests.csproj create mode 100644 fleet-telemetry/tests/FleetTelemetry.UnitTests/KpiMathTests.cs create mode 100644 fleet-telemetry/tests/FleetTelemetry.UnitTests/TelemetryBatchParserTests.cs create mode 100644 fleet-telemetry/tests/FleetTelemetry.UnitTests/TripRulesTests.cs create mode 100644 fleet-telemetry/tests/FleetTelemetry.UnitTests/packages.lock.json create mode 100644 metering-api/.env.example create mode 100644 metering-api/.gitignore create mode 100644 metering-api/Makefile create mode 100644 metering-api/README.md create mode 100644 metering-api/cmd/api/main.go create mode 100644 metering-api/cmd/demo/main.go create mode 100644 metering-api/cmd/seed/main.go create mode 100644 metering-api/docker-compose.yml create mode 100644 metering-api/docs/architecture.md create mode 100644 metering-api/go.mod create mode 100644 metering-api/go.sum create mode 100644 metering-api/init/01_platform.sf.sql create mode 100644 metering-api/init/02_schema.sf.sql create mode 100644 metering-api/init/03_price_plans.sf.sql create mode 100644 metering-api/internal/config/config.go create mode 100644 metering-api/internal/config/config_test.go create mode 100644 metering-api/internal/metering/dates.go create mode 100644 metering-api/internal/metering/events.go create mode 100644 metering-api/internal/metering/export_integration_test.go create mode 100644 metering-api/internal/metering/harness_integration_test.go create mode 100644 metering-api/internal/metering/http.go create mode 100644 metering-api/internal/metering/http_test.go create mode 100644 metering-api/internal/metering/invoices.go create mode 100644 metering-api/internal/metering/metering_integration_test.go create mode 100644 metering-api/internal/metering/problem.go create mode 100644 metering-api/internal/metering/rollups.go create mode 100644 metering-api/internal/metering/service.go create mode 100644 metering-api/internal/metering/usage.go create mode 100644 metering-api/internal/metering/validate.go create mode 100644 metering-api/internal/metering/validate_test.go create mode 100644 metering-api/internal/pricing/pricing.go create mode 100644 metering-api/internal/pricing/pricing_test.go create mode 100644 metering-api/internal/seed/seed.go create mode 100644 metering-api/internal/seed/seed_test.go create mode 100755 metering-api/scripts/demo.sh create mode 100755 metering-api/scripts/wait-for-emulator.sh create mode 100644 migrate-and-swap/.env.example create mode 100644 migrate-and-swap/.python-version create mode 100644 migrate-and-swap/Makefile create mode 100644 migrate-and-swap/README.md create mode 100644 migrate-and-swap/demos/V1.3.0__bad_negative_stock.sql create mode 100644 migrate-and-swap/docker-compose.yml create mode 100644 migrate-and-swap/docs/architecture.md create mode 100644 migrate-and-swap/init/01_bootstrap.sf.sql create mode 100644 migrate-and-swap/migrations/A__grants.sql create mode 100644 migrate-and-swap/migrations/R__inventory_views.sql create mode 100644 migrate-and-swap/migrations/V1.0.0__baseline.sql create mode 100644 migrate-and-swap/migrations/V1.1.0__add_reorder_point.sql create mode 100644 migrate-and-swap/migrations/V1.2.0__split_location.sql create mode 100644 migrate-and-swap/pyproject.toml create mode 100644 migrate-and-swap/release/__init__.py create mode 100644 migrate-and-swap/release/__main__.py create mode 100644 migrate-and-swap/release/dataset.py create mode 100644 migrate-and-swap/release/demo.py create mode 100644 migrate-and-swap/release/deploy.py create mode 100644 migrate-and-swap/release/gate.py create mode 100644 migrate-and-swap/release/recovery.py create mode 100644 migrate-and-swap/release/release.py create mode 100644 migrate-and-swap/release/settings.py create mode 100644 migrate-and-swap/release/validations.sql create mode 100644 migrate-and-swap/schemachange-config.yml create mode 100644 migrate-and-swap/scripts/seed.py create mode 100755 migrate-and-swap/scripts/wait-for-emulator.sh create mode 100644 migrate-and-swap/tests/conftest.py create mode 100644 migrate-and-swap/tests/test_recovery.py create mode 100644 migrate-and-swap/tests/test_release.py create mode 100644 migrate-and-swap/uv.lock create mode 100644 order-to-cash-cdc/.env.example create mode 100644 order-to-cash-cdc/Makefile create mode 100644 order-to-cash-cdc/README.md create mode 100644 order-to-cash-cdc/connect/Dockerfile create mode 100644 order-to-cash-cdc/connect/snowflake-sink.json create mode 100644 order-to-cash-cdc/connect/worker.properties create mode 100644 order-to-cash-cdc/docker-compose.yml create mode 100644 order-to-cash-cdc/docs/architecture.md create mode 100644 order-to-cash-cdc/init/01_landing.sf.sql create mode 100644 order-to-cash-cdc/init/02_core.sf.sql create mode 100644 order-to-cash-cdc/init/03_pipeline.sf.sql create mode 100644 order-to-cash-cdc/pipeline/__init__.py create mode 100644 order-to-cash-cdc/pipeline/demo.py create mode 100644 order-to-cash-cdc/pipeline/harness.py create mode 100644 order-to-cash-cdc/pipeline/kafka_connect.py create mode 100644 order-to-cash-cdc/pipeline/seed.py create mode 100644 order-to-cash-cdc/pipeline/settings.py create mode 100644 order-to-cash-cdc/pipeline/snowflake_db.py create mode 100644 order-to-cash-cdc/producer/__init__.py create mode 100644 order-to-cash-cdc/producer/produce.py create mode 100644 order-to-cash-cdc/pyproject.toml create mode 100755 order-to-cash-cdc/scripts/wait-for-emulator.sh create mode 100644 order-to-cash-cdc/tests/__init__.py create mode 100644 order-to-cash-cdc/tests/conftest.py create mode 100644 order-to-cash-cdc/tests/model.py create mode 100644 order-to-cash-cdc/tests/test_model.py create mode 100644 order-to-cash-cdc/tests/test_pipeline.py create mode 100644 order-to-cash-cdc/tests/test_transactions.py create mode 100644 order-to-cash-cdc/uv.lock create mode 100644 phi-guardrails/.env.example create mode 100644 phi-guardrails/.python-version create mode 100644 phi-guardrails/Makefile create mode 100644 phi-guardrails/README.md create mode 100644 phi-guardrails/access-contract.yml create mode 100644 phi-guardrails/demos/broken-current-role-policy.sql create mode 100644 phi-guardrails/docker-compose.yml create mode 100644 phi-guardrails/docs/architecture.md create mode 100644 phi-guardrails/init/01_bootstrap.sf.sql create mode 100644 phi-guardrails/migrations/R__analyst_views.sql create mode 100644 phi-guardrails/migrations/V1.0.0__roles_and_schema.sql create mode 100644 phi-guardrails/migrations/V1.1.0__tags_and_masking.sql create mode 100644 phi-guardrails/migrations/V1.2.0__facility_row_access.sql create mode 100644 phi-guardrails/migrations/V1.3.0__add_patient_email.sql create mode 100644 phi-guardrails/pyproject.toml create mode 100644 phi-guardrails/schemachange-config.yml create mode 100644 phi-guardrails/scripts/load_synthea.py create mode 100755 phi-guardrails/scripts/wait-for-emulator.sh create mode 100644 phi-guardrails/src/phi_guardrails/__init__.py create mode 100644 phi-guardrails/src/phi_guardrails/contract.py create mode 100644 phi-guardrails/src/phi_guardrails/coverage.py create mode 100644 phi-guardrails/src/phi_guardrails/demo.py create mode 100644 phi-guardrails/src/phi_guardrails/deploy.py create mode 100644 phi-guardrails/src/phi_guardrails/drill.py create mode 100644 phi-guardrails/src/phi_guardrails/load.py create mode 100644 phi-guardrails/src/phi_guardrails/matrix.py create mode 100644 phi-guardrails/src/phi_guardrails/settings.py create mode 100644 phi-guardrails/src/phi_guardrails/synthea.py create mode 100644 phi-guardrails/tests/conftest.py create mode 100644 phi-guardrails/tests/test_access_matrix.py create mode 100644 phi-guardrails/tests/test_deploy.py create mode 100644 phi-guardrails/tests/test_load.py create mode 100644 phi-guardrails/tests/test_policy_coverage.py create mode 100644 phi-guardrails/uv.lock create mode 100644 slim-ci-lab/.env.example create mode 100644 slim-ci-lab/.github-example/workflows/dbt-ci.yml create mode 100644 slim-ci-lab/.gitignore create mode 100644 slim-ci-lab/Makefile create mode 100644 slim-ci-lab/README.md create mode 100644 slim-ci-lab/dbt/dbt_project.yml create mode 100644 slim-ci-lab/dbt/macros/drop_pr_schema.sql create mode 100644 slim-ci-lab/dbt/macros/generate_schema_name.sql create mode 100644 slim-ci-lab/dbt/macros/surrogate_key.sql create mode 100644 slim-ci-lab/dbt/models/intermediate/int_trips_enriched.sql create mode 100644 slim-ci-lab/dbt/models/marts/fct_daily_zone_revenue.sql create mode 100644 slim-ci-lab/dbt/models/schema.yml create mode 100644 slim-ci-lab/dbt/models/sources.yml create mode 100644 slim-ci-lab/dbt/models/staging/stg_taxi_zones.sql create mode 100644 slim-ci-lab/dbt/models/staging/stg_trips.sql create mode 100644 slim-ci-lab/dbt/profiles.yml create mode 100644 slim-ci-lab/dbt/seeds/taxi_zone_lookup.csv create mode 100644 slim-ci-lab/dbt/seeds/taxi_zone_lookup.yml create mode 100644 slim-ci-lab/dbt/snapshots/dim_zone.yml create mode 100644 slim-ci-lab/demos/bad-unique-key.patch create mode 100644 slim-ci-lab/demos/example-pr.patch create mode 100644 slim-ci-lab/docker-compose.yml create mode 100644 slim-ci-lab/docs/architecture.md create mode 100644 slim-ci-lab/init/01_platform.sf.sql create mode 100644 slim-ci-lab/pyproject.toml create mode 100755 slim-ci-lab/scripts/ci_local.sh create mode 100755 slim-ci-lab/scripts/wait-for-emulator.sh create mode 100644 slim-ci-lab/src/slim_ci_lab/__init__.py create mode 100644 slim-ci-lab/src/slim_ci_lab/bad_pr_demo.py create mode 100644 slim-ci-lab/src/slim_ci_lab/ci_summary.py create mode 100644 slim-ci-lab/src/slim_ci_lab/dbt_runner.py create mode 100644 slim-ci-lab/src/slim_ci_lab/loader.py create mode 100644 slim-ci-lab/src/slim_ci_lab/settings.py create mode 100644 slim-ci-lab/src/slim_ci_lab/tlc.py create mode 100644 slim-ci-lab/tests/conftest.py create mode 100644 slim-ci-lab/tests/test_load.py create mode 100644 slim-ci-lab/tests/test_models.py create mode 100644 slim-ci-lab/tests/test_slim_ci.py create mode 100644 slim-ci-lab/uv.lock create mode 100644 tenantboard/.env.example create mode 100644 tenantboard/.gitignore create mode 100644 tenantboard/Makefile create mode 100644 tenantboard/README.md create mode 100644 tenantboard/app/__init__.py create mode 100644 tenantboard/app/db.py create mode 100644 tenantboard/app/errors.py create mode 100644 tenantboard/app/main.py create mode 100644 tenantboard/app/models.py create mode 100644 tenantboard/app/provisioning.py create mode 100644 tenantboard/app/repository.py create mode 100644 tenantboard/app/settings.py create mode 100644 tenantboard/app/tenancy.py create mode 100644 tenantboard/app/uploads.py create mode 100644 tenantboard/docker-compose.yml create mode 100644 tenantboard/docs/architecture.md create mode 100644 tenantboard/init/01_platform.sf.sql create mode 100644 tenantboard/init/02_policies.sf.sql create mode 100644 tenantboard/init/03_tables.sf.sql create mode 100644 tenantboard/init/04_grants.sf.sql create mode 100644 tenantboard/pyproject.toml create mode 100644 tenantboard/scripts/__init__.py create mode 100644 tenantboard/scripts/demo.py create mode 100755 tenantboard/scripts/run.sh create mode 100644 tenantboard/scripts/seed.py create mode 100755 tenantboard/scripts/wait-for-emulator.sh create mode 100644 tenantboard/scripts/wait_for_init.py create mode 100644 tenantboard/tests/__init__.py create mode 100644 tenantboard/tests/conftest.py create mode 100644 tenantboard/tests/helpers.py create mode 100644 tenantboard/tests/test_api.py create mode 100644 tenantboard/tests/test_isolation.py create mode 100644 tenantboard/tests/test_session_cache.py create mode 100644 tenantboard/tests/test_uploads.py create mode 100644 tenantboard/uv.lock create mode 100644 tenantboard/web/index.html create mode 100644 tenantboard/web/package-lock.json create mode 100644 tenantboard/web/package.json create mode 100644 tenantboard/web/src/App.tsx create mode 100644 tenantboard/web/src/RevenueChart.tsx create mode 100644 tenantboard/web/src/api.ts create mode 100644 tenantboard/web/src/format.ts create mode 100644 tenantboard/web/src/main.tsx create mode 100644 tenantboard/web/src/styles.css create mode 100644 tenantboard/web/tsconfig.json create mode 100644 tenantboard/web/vite.config.ts diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..b98d449 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,8 @@ +*.sh text eol=lf +mvnw text eol=lf +Makefile text eol=lf +*.cmd text eol=crlf +*.jar binary +*.parquet binary +*.png binary +*.gz binary diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5656f2f..f7f2d45 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -9,6 +9,9 @@ on: pull_request: branches: - main + schedule: + # nightly regression run against the newest snowflake-next image + - cron: '17 3 * * *' workflow_dispatch: permissions: @@ -25,14 +28,22 @@ jobs: sample: - airflow-dbt-transformation - citi-bike-data-app + - claims-service - credit-card-fraud-detection-with-snowpark - credit-scoring-with-snowpark + - fleet-telemetry - glue-snowflake-integration - lambda-snowpark-connector + - metering-api + - migrate-and-swap - multi-container + - order-to-cash-cdc + - phi-guardrails - predicting-customer-spend + - slim-ci-lab - soda-data-quality-checks - streamlit-snowpark-dynamic-filters + - tenantboard defaults: run: working-directory: ${{ matrix.sample }} @@ -60,6 +71,33 @@ jobs: with: terraform_wrapper: false + - name: Setup uv + if: hashFiles(format('{0}/uv.lock', matrix.sample)) != '' + uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + + - name: Setup Java + if: hashFiles(format('{0}/pom.xml', matrix.sample)) != '' + uses: actions/setup-java@v4 + with: + distribution: temurin + java-version: '21' + cache: maven + + - name: Setup Go + if: hashFiles(format('{0}/go.mod', matrix.sample)) != '' + uses: actions/setup-go@v5 + with: + go-version-file: ${{ matrix.sample }}/go.mod + cache-dependency-path: ${{ matrix.sample }}/go.sum + + - name: Setup .NET + if: hashFiles(format('{0}/global.json', matrix.sample)) != '' + uses: actions/setup-dotnet@v4 + with: + global-json-file: ${{ matrix.sample }}/global.json + - name: Install lstk and the Snowflake and AWS CLIs run: | npm install -g @localstack/lstk @@ -80,8 +118,14 @@ jobs: if grep -q '^start-localstack:' Makefile; then make start-localstack; else make start; fi - name: Run tests + env: + # samples tested with Testcontainers start their own emulator + LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }} run: make test - name: Show LocalStack logs if: failure() + env: + # docker compose refuses to read a compose file that requires the token + LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }} run: make logs || true diff --git a/.gitignore b/.gitignore index 66683c7..43e39d9 100644 --- a/.gitignore +++ b/.gitignore @@ -172,3 +172,21 @@ cython_debug/ **/tempdata/*.txt .idea/ + +# Throwaway key pairs and local config generated by `make up` +.secrets/ +*.p8 +*.pem +!**/fixtures/**/*.pem + +# Node, Java, dbt and tool caches +node_modules/ +.mvn/wrapper/maven-wrapper.jar +.ruff_cache/ +**/dbt/target/ +**/dbt/logs/ +**/dbt/dbt_packages/ +prod-artifacts/ +data/raw/ +.vscode/ +*.swp diff --git a/README.md b/README.md index 60e815d..7c76e6e 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,7 @@ The samples run on the next-generation Snowflake emulator, [`localstack/snowflak - [`lstk`](https://docs.localstack.cloud/snowflake/developer-tools/lstk/) 1.3.0 or later, which starts the emulator with `lstk start --type snowflake --image localstack/snowflake-next:latest` - [Docker Compose](https://docs.docker.com/compose/) and the [`awslocal` CLI](https://docs.localstack.cloud/user-guide/integrations/aws-cli/) for the samples that also use AWS services: `airflow-dbt-transformation`, `glue-snowflake-integration`, `lambda-snowpark-connector` and `multi-container`. Each of them has a `docker-compose.yml` that starts LocalStack AWS on port 4566 and Snowflake Next on a second port. - [Snowflake CLI](https://docs.snowflake.com/en/developer-guide/snowflake-cli/installation/installation) (`snow`) with a `local` and a `localstack` connection, see below +- For the application samples (`claims-service`, `fleet-telemetry`, `metering-api`, `migrate-and-swap`, `order-to-cash-cdc`, `phi-guardrails`, `slim-ci-lab` and `tenantboard`), Docker Compose v2 and the sample's own toolchain: [`uv`](https://docs.astral.sh/uv/) for the Python samples, Node.js 22 for the Tenantboard UI, JDK 21 for Claims Service, Go for Metering API and the .NET 10 SDK for Fleet Telemetry. They start the emulator themselves and do not need `lstk` or `snow`. On Windows, run them inside WSL 2. ## Configuration @@ -20,7 +21,7 @@ Set your auth token before running any sample: export LOCALSTACK_AUTH_TOKEN= ``` -Alternatively, log in with `lstk login`. +Alternatively, log in with `lstk login`. The application samples read the token from the environment or from `~/.localstack/default.env`. Several samples use the Snowflake CLI with a connection named `local` or `localstack`. Create both, pointing at the emulator: @@ -37,20 +38,30 @@ done |---|---| | [airflow-dbt-transformation](./airflow-dbt-transformation) | Run local data transformation pipelines with Airflow, dbt, and Snowpark. | | [citi-bike-data-app](./citi-bike-data-app) | Seed Citibike data into Snowflake and query it from a web application. | +| [claims-service](./claims-service) | A Java 21 Spring Boot service with stored procedures, a Java UDF and integration tests on Testcontainers. | | [credit-card-fraud-detection-with-snowpark](./credit-card-fraud-detection-with-snowpark) | Feature engineering and ML-oriented fraud workflows with Snowpark. | | [credit-scoring-with-snowpark](./credit-scoring-with-snowpark) | Exploratory data analysis for credit scoring using Snowpark. | +| [fleet-telemetry](./fleet-telemetry) | An ASP.NET Core service that ingests vehicle telemetry as JSON and analyzes it with FLATTEN, window functions and a dynamic table. | | [glue-snowflake-integration](./glue-snowflake-integration) | Integrate AWS Glue ETL jobs with Snowflake in a local environment. | | [lambda-snowpark-connector](./lambda-snowpark-connector) | Use Snowpark from an AWS Lambda function running on LocalStack. | +| [metering-api](./metering-api) | A Go usage-metering and invoicing service with idempotent ingestion, transactional invoice close and async rollups. | +| [migrate-and-swap](./migrate-and-swap) | Zero-downtime schema releases with schemachange: migrate a clone, validate it, swap it in, roll back in one step. | | [multi-container](./multi-container) | Run LocalStack for Snowflake and LocalStack AWS emulation in separate containers. | +| [order-to-cash-cdc](./order-to-cash-cdc) | Kafka change events processed into current-state and history tables with streams, tasks and dynamic tables. | +| [phi-guardrails](./phi-guardrails) | Governance as code: tag-based masking, row access policies and a tested access matrix. | | [predicting-customer-spend](./predicting-customer-spend) | Predict customer spend using Snowpark and Python ML tooling. | +| [slim-ci-lab](./slim-ci-lab) | A dbt Core project with a Slim CI pull-request pipeline: state selection, defer and per-PR schemas. | | [soda-data-quality-checks](./soda-data-quality-checks) | Execute Soda data quality checks against local Snowflake tables. | | [streamlit-snowpark-dynamic-filters](./streamlit-snowpark-dynamic-filters) | Streamlit application with dynamic Snowpark-powered filters. | +| [tenantboard](./tenantboard) | A multi-tenant FastAPI and React analytics app with tenant isolation and PII masking enforced by Snowflake policies. | ## Testing A fresh Snowflake Next instance only has the `SNOWFLAKE` database, as in a new Snowflake account, so each sample creates the database it uses before loading data. -Every sample has `make start` (or `make start-localstack` where `make start` runs the app) to start LocalStack, and `make test` to check that the sample works. The [CI workflow](.github/workflows/main.yml) runs `make install`, the start target and `make test` for each sample on every pull request. +Every sample has `make start` (or `make start-localstack` where `make start` runs the app) to start LocalStack, and `make test` to check that the sample works. The [CI workflow](.github/workflows/main.yml) runs `make install`, the start target and `make test` for each sample on every pull request, and every night against the newest `localstack/snowflake-next` image to catch regressions. + +The application samples document `make up`, `make seed`, `make demo`, `make test` and `make down` in their READMEs, with `make start` and `make stop` as aliases for `make up` and `make down`. They bind the emulator to `127.0.0.1:4566`; set `SNOWFLAKE_PORT` to use another port. Their test suites also run against a real Snowflake account with `make test-cloud`, using the `SNOWFLAKE_*` settings in each sample's `.env.example`. ## Checking Out A Single Sample diff --git a/claims-service/.env.example b/claims-service/.env.example new file mode 100644 index 0000000..7b6a77a --- /dev/null +++ b/claims-service/.env.example @@ -0,0 +1,31 @@ +# Every variable the sample reads. Nothing here is required for `./mvnw verify` +# except LOCALSTACK_AUTH_TOKEN; the Makefile also reads it from ~/.localstack/default.env. + +# LocalStack auth token with Snowflake access (never commit a real value). +LOCALSTACK_AUTH_TOKEN= + +# Emulator image; defaults to the newest build. Pin a tag or digest to freeze it. +SNOWFLAKE_IMAGE=localstack/snowflake-next:latest + +# Host port of the compose emulator (make up / seed / run). Testcontainers picks its own. +SNOWFLAKE_PORT=4566 + +# EMULATOR (default) or SNOWFLAKE_CLOUD (make test-cloud). +TEST_TARGET=EMULATOR + +# Key pair used by make seed / make run (make up generates it). +SNOWFLAKE_PRIVATE_KEY_PATH=.secrets/rsa_key.p8 + +# Real Snowflake only: TEST_TARGET=SNOWFLAKE_CLOUD, or SPRING_PROFILES_ACTIVE=cloud for the app. +SNOWFLAKE_ACCOUNT= +SNOWFLAKE_HOST= +SNOWFLAKE_USER= +SNOWFLAKE_ROLE= +SNOWFLAKE_WAREHOUSE= +# App only (the test suite creates and drops its own CLAIMS_IT_ database). +SNOWFLAKE_DATABASE=CLAIMS +SNOWFLAKE_SCHEMA=CORE +SPRING_PROFILES_ACTIVE= + +# Liquibase contexts; "default,scheduled" also resumes the KPI task (every 5 minutes). +CLAIMS_LIQUIBASE_CONTEXTS=default diff --git a/claims-service/.mvn/wrapper/maven-wrapper.properties b/claims-service/.mvn/wrapper/maven-wrapper.properties new file mode 100644 index 0000000..c0bcafe --- /dev/null +++ b/claims-service/.mvn/wrapper/maven-wrapper.properties @@ -0,0 +1,3 @@ +wrapperVersion=3.3.4 +distributionType=only-script +distributionUrl=https://repo.maven.apache.org/maven2/org/apache/maven/apache-maven/3.9.11/apache-maven-3.9.11-bin.zip diff --git a/claims-service/Makefile b/claims-service/Makefile new file mode 100644 index 0000000..c440423 --- /dev/null +++ b/claims-service/Makefile @@ -0,0 +1,77 @@ +# Claims Service: Spring Boot on Snowflake JDBC, tested against the Snowflake-Next emulator. +# +# make up generate the dev key, pull the emulator image, start a long-lived emulator +# make seed migrate that emulator and load deterministic sample claims (idempotent) +# make run run the service on http://localhost:8080 against that emulator +# make test ./mvnw -q verify: the full suite, on its own Testcontainers emulator +# make demo the demo moment: clean build plus full suite, timed, with a per-test summary +# make down stop and remove the long-lived emulator +# make test-cloud the same suite against real Snowflake (consumes credits) +# +# `make test` and `make demo` need only Docker and JDK 21: Testcontainers starts, provisions +# and removes a fresh emulator for every run, so they do not depend on `make up`. + +-include $(HOME)/.localstack/default.env +export LOCALSTACK_AUTH_TOKEN +SNOWFLAKE_PORT ?= 4566 +SNOWFLAKE_IMAGE ?= localstack/snowflake-next:latest +export SNOWFLAKE_PORT SNOWFLAKE_IMAGE + +COMPOSE := docker compose -p snfs-claims-service +MVNW := ./mvnw -B +SECRETS := .secrets +DEV_KEY := $(SECRETS)/rsa_key.p8 +APP_JAR := claims-api/target/claims-api-1.0.0.jar +JAVA_OPTS := -Dnet.snowflake.jdbc.disablePlatformDetection=true +# The default (no profile) config, which targets the compose emulator. +APP := java $(JAVA_OPTS) -jar $(APP_JAR) --claims.snowflake.private-key-path=$(DEV_KEY) + +.PHONY: up seed run test demo down test-cloud build logs install start stop + +up: $(DEV_KEY) + $(COMPOSE) pull + $(COMPOSE) up -d + scripts/wait-for-emulator.sh 180 + +build: + $(MVNW) -q -DskipTests package + +seed: build + $(APP) --spring.main.web-application-type=none --claims.seed.enabled=true + +run: build + $(APP) + +test: + ./mvnw -q verify + @scripts/test-summary.sh + +demo: + scripts/demo.sh + +down: + $(COMPOSE) down -v --remove-orphans + +# Reads SNOWFLAKE_ACCOUNT, SNOWFLAKE_USER, SNOWFLAKE_PRIVATE_KEY_PATH, SNOWFLAKE_ROLE and +# SNOWFLAKE_WAREHOUSE; creates and drops a CLAIMS_IT_ database. +test-cloud: + TEST_TARGET=SNOWFLAKE_CLOUD ./mvnw -q verify + @scripts/test-summary.sh + +logs: + $(COMPOSE) logs --tail=200 snowflake + +# Throwaway unencrypted PKCS#8 key pair for key-pair (SNOWFLAKE_JWT) auth. +$(DEV_KEY): + mkdir -p $(SECRETS) + openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 -out $@ + openssl rsa -in $@ -pubout -out $(SECRETS)/rsa_key.pub + chmod 600 $@ + +# The targets the repository CI runs for every sample (see the root README) +install: + @echo "Nothing to install, make up and make test fetch the dependencies" + +start: up + +stop: down diff --git a/claims-service/README.md b/claims-service/README.md new file mode 100644 index 0000000..439c52f --- /dev/null +++ b/claims-service/README.md @@ -0,0 +1,136 @@ +# Claims Service + +A Spring Boot insurance-claims service built on the Snowflake JDBC driver, with SQL Scripting, a SQL UDF, streams and tasks. It is for Java backend engineers who want Snowflake in their integration tests: `./mvnw verify` runs the whole suite on a laptop with only Docker and JDK 21, against the LocalStack Snowflake emulator. + +## Architecture + +```mermaid +flowchart LR + client["HTTP client
(curl, tests)"] -- "REST + JSON" --> api["claims-api
Spring Boot 3.5"] + subgraph svc["Spring service"] + api --> services["ClaimIntakeService
AdjudicationService
KpiService"] + services --> repo["JdbcTemplate repositories
HikariCP pool"] + lb["Liquibase
at start-up"] + end + repo -- "JDBC, key-pair auth" --> sf + lb -- "DDL and
reference data" --> sf + subgraph sf["Snowflake: CLAIMS.CORE"] + claims[("CLAIMS")] + udf["IS_VALID_ICD10
SQL UDF"] + proc["ADJUDICATE_CLAIM
SQL Scripting"] + task["CLAIMS_CHANGES stream
REFRESH_CLAIM_KPIS task"] + kpis[("CLAIM_KPIS_MONTHLY")] + udf -. "validates codes for" .-> claims + proc --> claims --> task --> kpis + end +``` + +The business rules live in Snowflake, so the API, bulk loads and ad-hoc SQL all apply them from one implementation. The service connects as `SAMPLE_SVC` with role `CLAIMS_APP`, a least-privilege role that owns the `CLAIMS.CORE` schema and its objects; Liquibase migrates that schema at start-up, so the deployed objects always match the code. Design notes are in [docs/architecture.md](docs/architecture.md). + +## How it works + +1. **Provisioning.** An administrator runs `init/01_platform.sf.sql` once: warehouse `CLAIMS_WH`, role `CLAIMS_APP`, database `CLAIMS`, schema `CORE` and service user `SAMPLE_SVC`. +2. **Migrations.** Liquibase creates the tables, the `IS_VALID_ICD10` SQL UDF (one `REGEXP_LIKE` over the ICD-10-CM code structure), the procedure, stream, task and reference data. +3. **Submission** (`POST /api/claims`). One query checks the code with `IS_VALID_ICD10` and looks up the policy and provider; a `MERGE` inserts the claim unless the id already exists. +4. **Bulk intake** (`POST /api/claims/batch`). One JDBC batch into a staging table, one `UPDATE` that classifies every row, one `MERGE` of the accepted rows. +5. **Adjudication** (`POST /api/claims/{id}/adjudication`). `CALL ADJUDICATE_CLAIM(?)` applies the rules in priority order and records the decision in one transaction. +6. **KPIs** (`GET /api/kpis/monthly`). The task folds the stream's changes into the monthly KPI table every 5 minutes, or now with `POST /api/kpis/refresh`. + +```mermaid +sequenceDiagram + autonumber + participant C as Client + participant S as Spring service + participant P as ADJUDICATE_CLAIM + C->>S: POST /api/claims/CLM-1/adjudication + S->>P: CALL ADJUDICATE_CLAIM('CLM-1') + alt claim is not SUBMITTED + P-->>S: RAISE claim_not_pending (-20002) + S-->>C: 422 problem details + else claim is SUBMITTED + P->>P: evaluate rules, compute payable amount + P->>P: UPDATE CLAIMS, INSERT ADJUDICATION_LOG, COMMIT + P-->>S: 'APPROVED' or 'DENIED' + S-->>C: 200 claim with status and payableAmount + end +``` + +## Prerequisites + +- Docker (with Compose v2 for `make up`) +- JDK 21 (Maven is not needed: `./mvnw` downloads it) +- A LocalStack auth token with Snowflake access, in `LOCALSTACK_AUTH_TOKEN` or `~/.localstack/default.env` +- `openssl` and `curl` for `make up` and the examples + +## Quickstart + +```sh +make up # dev key in .secrets/, start the emulator on 127.0.0.1:4566 (SNOWFLAKE_PORT to change) +make seed # migrate, load 300 synthetic claims, adjudicate 25, refresh KPIs +make run # the service on http://localhost:8080 +``` + +In a second terminal: + +```sh +curl -s -X POST localhost:8080/api/claims -H 'Content-Type: application/json' -d '{ + "claimId": "CLM-DEMO-0001", "policyId": "POL-1001", "providerId": "PRV-201", + "diagnosisCode": "E11.9", "serviceDate": "2026-02-10", "billedAmount": 1200.00}' +# 201 {"claimId":"CLM-DEMO-0001",...,"status":"SUBMITTED","payableAmount":null,...} + +curl -s -X POST localhost:8080/api/claims/CLM-DEMO-0001/adjudication +# 200 {...,"status":"APPROVED","payableAmount":855.00,...} (1200 - 250 deductible) x 0.90 + +curl -s -X POST localhost:8080/api/claims/CLM-DEMO-0001/adjudication +# 422 {"type":"urn:claims:problem:claim-not-pending",...,"snowflakeErrorCode":-20002} + +curl -s -X POST localhost:8080/api/claims -H 'Content-Type: application/json' -d '{ + "claimId": "CLM-DEMO-0002", "policyId": "POL-1001", "providerId": "PRV-201", + "diagnosisCode": "E11.", "serviceDate": "2026-02-10", "billedAmount": 1200.00}' +# 422 {"type":"urn:claims:problem:invalid-diagnosis-code","detail":"'E11.' is not a valid ICD-10-CM code",...} + +curl -s -X POST localhost:8080/api/kpis/refresh # 202, runs EXECUTE TASK REFRESH_CLAIM_KPIS +curl -s localhost:8080/api/kpis/monthly +# [{"serviceMonth":"2026-01-01","status":"APPROVED","claimCount":...},...] +``` + +Then stop the service and finish: + +```sh +make test # the full suite on its own Testcontainers emulator (does not need make up) +make down # remove the emulator +``` + +## Tests + +`./mvnw verify` (or `make test`) starts a fresh emulator in Testcontainers, provisions it with `init/01_platform.sf.sql`, migrates it with Liquibase and drives the service through its real HTTP API and JDBC driver: + +- **Adjudication:** the payable amount from the policy terms, every rule in priority order and its boundaries (last covered day, one cent over the limit), and the audit row. +- **Bulk intake:** 250 claims loaded with one JDBC batch, each bad row reported with its row number and reason. +- **Error paths:** `422` for an invalid ICD-10 code or a claim adjudicated twice, `409` for a duplicate id, `404` for an unknown claim, with nothing written. +- **KPIs:** the stream and task move a claim between status buckets after `EXECUTE TASK`. +- **ICD-10 UDF:** `IS_VALID_ICD10` called directly with canonical, malformed and `NULL` codes. + +To run the same suite against a Snowflake account (consumes credits), set `SNOWFLAKE_ACCOUNT`, `SNOWFLAKE_USER`, `SNOWFLAKE_PRIVATE_KEY_PATH`, `SNOWFLAKE_ROLE` and `SNOWFLAKE_WAREHOUSE` and run `make test-cloud`. It creates a `CLAIMS_IT_` database, migrates it and drops it when the run ends; the role needs `CREATE DATABASE` and `EXECUTE TASK`. + +## Project structure + +``` +claims-service/ + Makefile, docker-compose.yml up, seed, run, test, demo, down, test-cloud + init/01_platform.sf.sql administrator provisioning: warehouse, role, schema, user + claims-api/ + src/main/java/.../claims/ api, config, domain, repo, seed, service + src/main/resources/ application.yml, Liquibase changelogs + src/test/java/.../claims/ Testcontainers set-up and integration tests + scripts/ wait-for-emulator.sh, demo.sh, test-summary.sh + docs/architecture.md design notes +``` + +## Learn more + +- [Snowflake JDBC driver](https://docs.snowflake.com/en/developer-guide/jdbc/jdbc) and [key-pair authentication](https://docs.snowflake.com/en/user-guide/key-pair-auth) +- [Snowflake Scripting cursors](https://docs.snowflake.com/en/developer-guide/snowflake-scripting/cursors) and [exceptions](https://docs.snowflake.com/en/developer-guide/snowflake-scripting/exceptions) +- [SQL UDFs](https://docs.snowflake.com/en/developer-guide/udf/sql/udf-sql-introduction) and [`REGEXP_LIKE`](https://docs.snowflake.com/en/sql-reference/functions/regexp_like) +- [Streams](https://docs.snowflake.com/en/user-guide/streams-intro) and [tasks](https://docs.snowflake.com/en/user-guide/tasks-intro) +- [Access control overview](https://docs.snowflake.com/en/user-guide/security-access-control-overview) diff --git a/claims-service/claims-api/pom.xml b/claims-service/claims-api/pom.xml new file mode 100644 index 0000000..9abcff3 --- /dev/null +++ b/claims-service/claims-api/pom.xml @@ -0,0 +1,105 @@ + + + 4.0.0 + + + com.example + claims-service-parent + 1.0.0 + + + claims-api + Claims API + + + + --add-opens=java.base/java.nio=ALL-UNNAMED + + + + + org.springframework.boot + spring-boot-starter-web + + + org.springframework.boot + spring-boot-starter-jdbc + + + org.springframework.boot + spring-boot-starter-validation + + + org.springframework.boot + spring-boot-starter-actuator + + + org.liquibase + liquibase-core + + + net.snowflake + snowflake-jdbc + ${snowflake-jdbc.version} + + + + org.springframework.boot + spring-boot-starter-test + test + + + org.springframework.boot + spring-boot-testcontainers + test + + + org.testcontainers + junit-jupiter + test + + + + + + + org.apache.maven.plugins + maven-surefire-plugin + + + 1 + true + + ${snowflake.jvm.args} -XX:+EnableDynamicAgentLoading + + ${project.basedir}/../init + + true + + + + + org.springframework.boot + spring-boot-maven-plugin + + ${snowflake.jvm.args} -Dnet.snowflake.jdbc.disablePlatformDetection=true + + + + org.apache.maven.plugins + maven-jar-plugin + + + + + java.base/java.nio + + + + + + + diff --git a/claims-service/claims-api/src/main/java/com/example/claims/ClaimsApplication.java b/claims-service/claims-api/src/main/java/com/example/claims/ClaimsApplication.java new file mode 100644 index 0000000..e804ba2 --- /dev/null +++ b/claims-service/claims-api/src/main/java/com/example/claims/ClaimsApplication.java @@ -0,0 +1,14 @@ +package com.example.claims; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.boot.context.properties.ConfigurationPropertiesScan; + +@SpringBootApplication +@ConfigurationPropertiesScan +public class ClaimsApplication { + + public static void main(String[] args) { + SpringApplication.run(ClaimsApplication.class, args); + } +} diff --git a/claims-service/claims-api/src/main/java/com/example/claims/api/ClaimsController.java b/claims-service/claims-api/src/main/java/com/example/claims/api/ClaimsController.java new file mode 100644 index 0000000..9cbfe77 --- /dev/null +++ b/claims-service/claims-api/src/main/java/com/example/claims/api/ClaimsController.java @@ -0,0 +1,73 @@ +package com.example.claims.api; + +import com.example.claims.domain.BatchIntakeResult; +import com.example.claims.domain.Claim; +import com.example.claims.domain.ClaimStatus; +import com.example.claims.domain.NewClaim; +import com.example.claims.repo.ClaimsRepository; +import com.example.claims.service.AdjudicationService; +import com.example.claims.service.ClaimIntakeService; +import com.example.claims.service.ClaimNotFoundException; +import jakarta.validation.Valid; +import jakarta.validation.constraints.Max; +import jakarta.validation.constraints.Min; +import jakarta.validation.constraints.NotEmpty; +import jakarta.validation.constraints.Size; +import java.net.URI; +import java.util.List; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PathVariable; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RequestParam; +import org.springframework.web.bind.annotation.RestController; + +@RestController +@RequestMapping("/api/claims") +public class ClaimsController { + + static final int MAX_BATCH_SIZE = 10_000; + + private final ClaimIntakeService intake; + private final AdjudicationService adjudication; + private final ClaimsRepository claims; + + public ClaimsController(ClaimIntakeService intake, AdjudicationService adjudication, ClaimsRepository claims) { + this.intake = intake; + this.adjudication = adjudication; + this.claims = claims; + } + + /** 201 with the stored claim; 409 for a duplicate id; 422 for a bad code or reference. */ + @PostMapping + public ResponseEntity submit(@Valid @RequestBody NewClaim claim) { + Claim stored = intake.submit(claim); + return ResponseEntity.created(URI.create("/api/claims/" + stored.claimId())).body(stored); + } + + /** Bulk intake: valid rows are loaded, the rest are reported with a reason. */ + @PostMapping("/batch") + public BatchIntakeResult submitBatch( + @RequestBody @NotEmpty @Size(max = MAX_BATCH_SIZE) List<@Valid NewClaim> batch) { + return intake.intake(batch); + } + + @GetMapping("/{claimId}") + public Claim get(@PathVariable String claimId) { + return claims.findById(claimId).orElseThrow(() -> new ClaimNotFoundException(claimId)); + } + + @GetMapping + public List list(@RequestParam(required = false) ClaimStatus status, + @RequestParam(defaultValue = "50") @Min(1) @Max(500) int limit) { + return claims.findRecent(status, limit); + } + + /** Runs ADJUDICATE_CLAIM; 404 for an unknown claim, 422 if it was already adjudicated. */ + @PostMapping("/{claimId}/adjudication") + public Claim adjudicate(@PathVariable String claimId) { + return adjudication.adjudicate(claimId); + } +} diff --git a/claims-service/claims-api/src/main/java/com/example/claims/api/ErrorHandler.java b/claims-service/claims-api/src/main/java/com/example/claims/api/ErrorHandler.java new file mode 100644 index 0000000..d0ef326 --- /dev/null +++ b/claims-service/claims-api/src/main/java/com/example/claims/api/ErrorHandler.java @@ -0,0 +1,72 @@ +package com.example.claims.api; + +import com.example.claims.service.ClaimNotAdjudicableException; +import com.example.claims.service.ClaimNotFoundException; +import com.example.claims.service.DuplicateClaimException; +import com.example.claims.service.InvalidClaimException; +import java.net.URI; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.dao.DataAccessException; +import org.springframework.http.HttpStatus; +import org.springframework.http.ProblemDetail; +import org.springframework.web.bind.annotation.ExceptionHandler; +import org.springframework.web.bind.annotation.RestControllerAdvice; +import org.springframework.web.servlet.mvc.method.annotation.ResponseEntityExceptionHandler; + +/** + * Maps domain and database errors to RFC 9457 problem details. Request validation + * errors (400, including Spring MVC's built-in method validation) come from + * {@link ResponseEntityExceptionHandler}. + */ +@RestControllerAdvice +public class ErrorHandler extends ResponseEntityExceptionHandler { + + static final String PROBLEM_BASE = "urn:claims:problem:"; + + private static final Logger log = LoggerFactory.getLogger(ErrorHandler.class); + + @ExceptionHandler + ProblemDetail duplicate(DuplicateClaimException e) { + ProblemDetail problem = problem(HttpStatus.CONFLICT, "duplicate-claim", "Duplicate claim", e.getMessage()); + problem.setProperty("claimId", e.claimId()); + return problem; + } + + @ExceptionHandler + ProblemDetail invalid(InvalidClaimException e) { + return problem(HttpStatus.UNPROCESSABLE_ENTITY, e.problem().slug(), "Claim rejected", e.getMessage()); + } + + @ExceptionHandler + ProblemDetail notFound(ClaimNotFoundException e) { + ProblemDetail problem = problem(HttpStatus.NOT_FOUND, "claim-not-found", "Claim not found", e.getMessage()); + problem.setProperty("claimId", e.claimId()); + return problem; + } + + /** The procedure's custom exception: the claim exists but cannot be adjudicated now. */ + @ExceptionHandler + ProblemDetail notAdjudicable(ClaimNotAdjudicableException e) { + ProblemDetail problem = problem(HttpStatus.UNPROCESSABLE_ENTITY, "claim-not-pending", + "Claim cannot be adjudicated", e.getMessage()); + problem.setProperty("claimId", e.claimId()); + problem.setProperty("snowflakeErrorCode", e.snowflakeErrorCode()); + return problem; + } + + /** Anything else from Snowflake: log it, but never leak SQL or driver details. */ + @ExceptionHandler + ProblemDetail database(DataAccessException e) { + log.error("Snowflake call failed", e); + return problem(HttpStatus.INTERNAL_SERVER_ERROR, "database-error", "Database error", + "The claims database could not complete the request"); + } + + private static ProblemDetail problem(HttpStatus status, String slug, String title, String detail) { + ProblemDetail problem = ProblemDetail.forStatusAndDetail(status, detail); + problem.setType(URI.create(PROBLEM_BASE + slug)); + problem.setTitle(title); + return problem; + } +} diff --git a/claims-service/claims-api/src/main/java/com/example/claims/api/KpiController.java b/claims-service/claims-api/src/main/java/com/example/claims/api/KpiController.java new file mode 100644 index 0000000..c5e39ad --- /dev/null +++ b/claims-service/claims-api/src/main/java/com/example/claims/api/KpiController.java @@ -0,0 +1,33 @@ +package com.example.claims.api; + +import com.example.claims.domain.MonthlyKpi; +import com.example.claims.service.KpiService; +import java.util.List; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +@RestController +@RequestMapping("/api/kpis") +public class KpiController { + + private final KpiService kpis; + + public KpiController(KpiService kpis) { + this.kpis = kpis; + } + + @GetMapping("/monthly") + public List monthly() { + return kpis.monthly(); + } + + /** 202: the task run is queued; poll GET /api/kpis/monthly for the result. */ + @PostMapping("/refresh") + public ResponseEntity refresh() { + kpis.refreshNow(); + return ResponseEntity.accepted().build(); + } +} diff --git a/claims-service/claims-api/src/main/java/com/example/claims/config/PrivateKeys.java b/claims-service/claims-api/src/main/java/com/example/claims/config/PrivateKeys.java new file mode 100644 index 0000000..015f5b1 --- /dev/null +++ b/claims-service/claims-api/src/main/java/com/example/claims/config/PrivateKeys.java @@ -0,0 +1,52 @@ +package com.example.claims.config; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.security.GeneralSecurityException; +import java.security.KeyFactory; +import java.security.PrivateKey; +import java.security.spec.PKCS8EncodedKeySpec; +import java.util.Base64; + +/** + * Loads an unencrypted PKCS#8 PEM private key ({@code -----BEGIN PRIVATE KEY-----}), + * the format {@code openssl genpkey} writes. + * + *

The key is handed to the JDBC driver as a {@link PrivateKey} object rather than a + * file path. That keeps the driver's key-file permission checks out of the picture + * (a key on a shared or mounted volume is often group-readable) and works the same + * whether the key came from a file, a secret manager or a test fixture. + */ +public final class PrivateKeys { + + private static final String BEGIN = "-----BEGIN PRIVATE KEY-----"; + private static final String END = "-----END PRIVATE KEY-----"; + + private PrivateKeys() { + } + + public static PrivateKey readPkcs8Pem(Path path) { + String pem; + try { + pem = Files.readString(path); + } catch (IOException e) { + throw new UncheckedIOException("Cannot read Snowflake private key " + path, e); + } + int begin = pem.indexOf(BEGIN); + int end = pem.indexOf(END); + if (begin < 0 || end < begin) { + throw new IllegalArgumentException(path + " is not an unencrypted PKCS#8 PEM key (" + + BEGIN + "). Generate one with: openssl genpkey -algorithm RSA " + + "-pkeyopt rsa_keygen_bits:2048 -out rsa_key.p8"); + } + String base64 = pem.substring(begin + BEGIN.length(), end).replaceAll("\\s", ""); + try { + return KeyFactory.getInstance("RSA") + .generatePrivate(new PKCS8EncodedKeySpec(Base64.getDecoder().decode(base64))); + } catch (GeneralSecurityException | IllegalArgumentException e) { + throw new IllegalArgumentException(path + " does not contain a valid RSA private key", e); + } + } +} diff --git a/claims-service/claims-api/src/main/java/com/example/claims/config/SnowflakeDataSourceConfig.java b/claims-service/claims-api/src/main/java/com/example/claims/config/SnowflakeDataSourceConfig.java new file mode 100644 index 0000000..8c447a9 --- /dev/null +++ b/claims-service/claims-api/src/main/java/com/example/claims/config/SnowflakeDataSourceConfig.java @@ -0,0 +1,43 @@ +package com.example.claims.config; + +import com.zaxxer.hikari.HikariConfig; +import com.zaxxer.hikari.HikariDataSource; +import java.time.Duration; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * HikariCP pool of Snowflake JDBC connections using key-pair (SNOWFLAKE_JWT) auth. + * + *

Role, warehouse, database and schema are set on every connection, so no + * statement depends on a user's defaults. Snowflake has no default warehouse and a + * new user's default role is PUBLIC. + */ +@Configuration(proxyBeanMethods = false) +public class SnowflakeDataSourceConfig { + + @Bean(destroyMethod = "close") + public HikariDataSource dataSource(SnowflakeProperties props) { + HikariConfig config = new HikariConfig(); + config.setPoolName("snowflake"); + config.setJdbcUrl(props.url()); + config.setMaximumPoolSize(props.maxPoolSize()); + config.setMinimumIdle(1); + // Snowflake sessions expire after 4 hours of inactivity by default; recycle well before that. + config.setMaxLifetime(Duration.ofMinutes(30).toMillis()); + + config.addDataSourceProperty("account", props.account()); + config.addDataSourceProperty("user", props.user()); + config.addDataSourceProperty("authenticator", "SNOWFLAKE_JWT"); + config.addDataSourceProperty("privateKey", PrivateKeys.readPkcs8Pem(props.privateKeyPath())); + config.addDataSourceProperty("role", props.role()); + config.addDataSourceProperty("warehouse", props.warehouse()); + config.addDataSourceProperty("db", props.database()); + config.addDataSourceProperty("schema", props.schema()); + config.addDataSourceProperty("application", "claims-service"); + config.addDataSourceProperty("QUERY_TAG", props.queryTag()); + // Fail fast on a bad role/warehouse/database instead of running with a NULL context. + config.addDataSourceProperty("validateDefaultParameters", "true"); + return new HikariDataSource(config); + } +} diff --git a/claims-service/claims-api/src/main/java/com/example/claims/config/SnowflakeProperties.java b/claims-service/claims-api/src/main/java/com/example/claims/config/SnowflakeProperties.java new file mode 100644 index 0000000..6e2fc65 --- /dev/null +++ b/claims-service/claims-api/src/main/java/com/example/claims/config/SnowflakeProperties.java @@ -0,0 +1,41 @@ +package com.example.claims.config; + +import jakarta.validation.constraints.Min; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; +import java.nio.file.Path; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.boot.context.properties.bind.DefaultValue; +import org.springframework.validation.annotation.Validated; + +/** + * Every Snowflake connection setting in one place. Values come from + * {@code application.yml}, which maps the shared sample environment variables + * ({@code SNOWFLAKE_ACCOUNT}, {@code SNOWFLAKE_USER}, ...) onto these fields. + * + * @param url JDBC URL without parameters, e.g. + * {@code jdbc:snowflake://snowflake.localhost.localstack.cloud:4566/} + * @param account account identifier ({@code snowflake} on the emulator) + * @param user service user that authenticates with a key pair + * @param privateKeyPath unencrypted PKCS#8 PEM file holding the user's private key + * @param role role the service runs as; never ACCOUNTADMIN in production + * @param warehouse warehouse for queries, the procedure and the KPI task + * @param database database that holds the service schema + * @param schema schema owned by the service; Liquibase migrates it + * @param queryTag QUERY_TAG for every statement, for cost attribution in QUERY_HISTORY + * @param maxPoolSize HikariCP pool size + */ +@Validated +@ConfigurationProperties("claims.snowflake") +public record SnowflakeProperties( + @NotBlank String url, + @NotBlank String account, + @NotBlank String user, + @NotNull Path privateKeyPath, + @NotBlank String role, + @NotBlank String warehouse, + @NotBlank String database, + @NotBlank String schema, + @DefaultValue("claims-service") String queryTag, + @DefaultValue("5") @Min(1) int maxPoolSize) { +} diff --git a/claims-service/claims-api/src/main/java/com/example/claims/domain/BatchIntakeResult.java b/claims-service/claims-api/src/main/java/com/example/claims/domain/BatchIntakeResult.java new file mode 100644 index 0000000..5ddc61c --- /dev/null +++ b/claims-service/claims-api/src/main/java/com/example/claims/domain/BatchIntakeResult.java @@ -0,0 +1,22 @@ +package com.example.claims.domain; + +import java.util.List; + +/** + * Outcome of a bulk intake: every submitted row is either accepted or listed in + * {@code rejected} with a reason. + */ +public record BatchIntakeResult(String batchId, int received, int accepted, List rejected) { + + /** Why a row of a batch was not loaded. */ + public enum Reason { + INVALID_DIAGNOSIS_CODE, + UNKNOWN_POLICY, + UNKNOWN_PROVIDER, + DUPLICATE_CLAIM_ID, + DUPLICATE_IN_BATCH + } + + public record Rejection(int row, String claimId, Reason reason) { + } +} diff --git a/claims-service/claims-api/src/main/java/com/example/claims/domain/Claim.java b/claims-service/claims-api/src/main/java/com/example/claims/domain/Claim.java new file mode 100644 index 0000000..e5671ff --- /dev/null +++ b/claims-service/claims-api/src/main/java/com/example/claims/domain/Claim.java @@ -0,0 +1,20 @@ +package com.example.claims.domain; + +import java.math.BigDecimal; +import java.time.LocalDate; +import java.time.Instant; + +/** A claim as stored in {@code CLAIMS}. Adjudication fills the last four fields. */ +public record Claim( + String claimId, + String policyId, + String providerId, + String diagnosisCode, + LocalDate serviceDate, + BigDecimal billedAmount, + ClaimStatus status, + BigDecimal payableAmount, + String denialReason, + Instant submittedAt, + Instant adjudicatedAt) { +} diff --git a/claims-service/claims-api/src/main/java/com/example/claims/domain/ClaimStatus.java b/claims-service/claims-api/src/main/java/com/example/claims/domain/ClaimStatus.java new file mode 100644 index 0000000..5d1c8b0 --- /dev/null +++ b/claims-service/claims-api/src/main/java/com/example/claims/domain/ClaimStatus.java @@ -0,0 +1,8 @@ +package com.example.claims.domain; + +/** Lifecycle of a claim. Only {@code SUBMITTED} claims can be adjudicated. */ +public enum ClaimStatus { + SUBMITTED, + APPROVED, + DENIED +} diff --git a/claims-service/claims-api/src/main/java/com/example/claims/domain/MonthlyKpi.java b/claims-service/claims-api/src/main/java/com/example/claims/domain/MonthlyKpi.java new file mode 100644 index 0000000..9c84d7f --- /dev/null +++ b/claims-service/claims-api/src/main/java/com/example/claims/domain/MonthlyKpi.java @@ -0,0 +1,13 @@ +package com.example.claims.domain; + +import java.math.BigDecimal; +import java.time.LocalDate; + +/** One row of {@code CLAIM_KPIS_MONTHLY}: claims per service month and current status. */ +public record MonthlyKpi( + LocalDate serviceMonth, + ClaimStatus status, + long claimCount, + BigDecimal billedTotal, + BigDecimal payableTotal) { +} diff --git a/claims-service/claims-api/src/main/java/com/example/claims/domain/NewClaim.java b/claims-service/claims-api/src/main/java/com/example/claims/domain/NewClaim.java new file mode 100644 index 0000000..2b3148a --- /dev/null +++ b/claims-service/claims-api/src/main/java/com/example/claims/domain/NewClaim.java @@ -0,0 +1,26 @@ +package com.example.claims.domain; + +import jakarta.validation.constraints.Digits; +import jakarta.validation.constraints.NotBlank; +import jakarta.validation.constraints.NotNull; +import jakarta.validation.constraints.PastOrPresent; +import jakarta.validation.constraints.Pattern; +import jakarta.validation.constraints.Positive; +import jakarta.validation.constraints.Size; +import java.math.BigDecimal; +import java.time.LocalDate; + +/** + * A claim submitted by a provider. Bean Validation checks the shape; whether the + * diagnosis code is a valid ICD-10 code, and whether the policy and provider exist, + * is decided in Snowflake (the {@code IS_VALID_ICD10} SQL UDF and reference tables), + * so batch loads and ad-hoc SQL apply exactly the same rules. + */ +public record NewClaim( + @NotBlank @Pattern(regexp = "CLM-[A-Z0-9-]{4,26}", message = "must look like CLM-2026-000001") String claimId, + @NotBlank @Size(max = 20) String policyId, + @NotBlank @Size(max = 20) String providerId, + @NotBlank @Size(max = 8) String diagnosisCode, + @NotNull @PastOrPresent LocalDate serviceDate, + @NotNull @Positive @Digits(integer = 10, fraction = 2) BigDecimal billedAmount) { +} diff --git a/claims-service/claims-api/src/main/java/com/example/claims/repo/ClaimsRepository.java b/claims-service/claims-api/src/main/java/com/example/claims/repo/ClaimsRepository.java new file mode 100644 index 0000000..388a620 --- /dev/null +++ b/claims-service/claims-api/src/main/java/com/example/claims/repo/ClaimsRepository.java @@ -0,0 +1,217 @@ +package com.example.claims.repo; + +import com.example.claims.domain.BatchIntakeResult.Reason; +import com.example.claims.domain.BatchIntakeResult.Rejection; +import com.example.claims.domain.Claim; +import com.example.claims.domain.ClaimStatus; +import com.example.claims.domain.NewClaim; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.time.Instant; +import java.util.List; +import java.util.Optional; +import org.springframework.jdbc.core.BatchPreparedStatementSetter; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.RowMapper; +import org.springframework.stereotype.Repository; + +/** + * All SQL against the claim tables. Every value is a bind parameter; object names + * are unqualified and resolve against the connection's database and schema. + */ +@Repository +public class ClaimsRepository { + + private static final String CLAIM_COLUMNS = """ + CLAIM_ID, POLICY_ID, PROVIDER_ID, DIAGNOSIS_CODE, SERVICE_DATE, BILLED_AMOUNT, + STATUS, PAYABLE_AMOUNT, DENIAL_REASON, SUBMITTED_AT, ADJUDICATED_AT"""; + + /** + * Snowflake does not enforce PRIMARY KEY constraints, so "insert unless the id + * exists" is a MERGE: concurrent MERGEs on one table serialize on its lock, so a + * second submission of the same id sees the first and inserts nothing. + */ + private static final String INSERT_IF_ABSENT = """ + MERGE INTO CLAIMS c + USING (SELECT ? AS CLAIM_ID, ? AS POLICY_ID, ? AS PROVIDER_ID, ? AS DIAGNOSIS_CODE, + ?::DATE AS SERVICE_DATE, ?::NUMBER(12, 2) AS BILLED_AMOUNT) s + ON c.CLAIM_ID = s.CLAIM_ID + WHEN NOT MATCHED THEN INSERT + (CLAIM_ID, POLICY_ID, PROVIDER_ID, DIAGNOSIS_CODE, SERVICE_DATE, BILLED_AMOUNT, STATUS) + VALUES + (s.CLAIM_ID, s.POLICY_ID, s.PROVIDER_ID, s.DIAGNOSIS_CODE, s.SERVICE_DATE, s.BILLED_AMOUNT, 'SUBMITTED') + """; + + private static final String CHECK_REFERENCES = """ + SELECT IS_VALID_ICD10(?) AS DIAGNOSIS_VALID, + (SELECT COUNT(*) FROM POLICIES WHERE POLICY_ID = ?) > 0 AS POLICY_EXISTS, + (SELECT COUNT(*) FROM PROVIDERS WHERE PROVIDER_ID = ?) > 0 AS PROVIDER_EXISTS + """; + + private static final String STAGE_INTAKE_ROW = """ + INSERT INTO CLAIM_INTAKE + (BATCH_ID, ROW_NO, CLAIM_ID, POLICY_ID, PROVIDER_ID, DIAGNOSIS_CODE, SERVICE_DATE, BILLED_AMOUNT) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + """; + + /** Marks every staged row that must not be loaded, in rule order. */ + private static final String CLASSIFY_INTAKE = """ + UPDATE CLAIM_INTAKE t + SET REJECT_REASON = s.REASON + FROM ( + SELECT i.ROW_NO, + CASE + WHEN NOT IS_VALID_ICD10(i.DIAGNOSIS_CODE) THEN 'INVALID_DIAGNOSIS_CODE' + WHEN p.POLICY_ID IS NULL THEN 'UNKNOWN_POLICY' + WHEN v.PROVIDER_ID IS NULL THEN 'UNKNOWN_PROVIDER' + WHEN c.CLAIM_ID IS NOT NULL THEN 'DUPLICATE_CLAIM_ID' + WHEN ROW_NUMBER() OVER (PARTITION BY i.CLAIM_ID ORDER BY i.ROW_NO) > 1 + THEN 'DUPLICATE_IN_BATCH' + END AS REASON + FROM CLAIM_INTAKE i + LEFT JOIN POLICIES p ON p.POLICY_ID = i.POLICY_ID + LEFT JOIN PROVIDERS v ON v.PROVIDER_ID = i.PROVIDER_ID + LEFT JOIN CLAIMS c ON c.CLAIM_ID = i.CLAIM_ID + WHERE i.BATCH_ID = ? + ) s + WHERE t.BATCH_ID = ? AND t.ROW_NO = s.ROW_NO AND s.REASON IS NOT NULL + """; + + private static final String PROMOTE_INTAKE = """ + MERGE INTO CLAIMS c + USING (SELECT CLAIM_ID, POLICY_ID, PROVIDER_ID, DIAGNOSIS_CODE, SERVICE_DATE, BILLED_AMOUNT + FROM CLAIM_INTAKE + WHERE BATCH_ID = ? AND REJECT_REASON IS NULL) s + ON c.CLAIM_ID = s.CLAIM_ID + WHEN NOT MATCHED THEN INSERT + (CLAIM_ID, POLICY_ID, PROVIDER_ID, DIAGNOSIS_CODE, SERVICE_DATE, BILLED_AMOUNT, STATUS) + VALUES + (s.CLAIM_ID, s.POLICY_ID, s.PROVIDER_ID, s.DIAGNOSIS_CODE, s.SERVICE_DATE, s.BILLED_AMOUNT, 'SUBMITTED') + """; + + private final JdbcTemplate jdbc; + + public ClaimsRepository(JdbcTemplate jdbc) { + this.jdbc = jdbc; + } + + public Optional findById(String claimId) { + return jdbc.query("SELECT " + CLAIM_COLUMNS + " FROM CLAIMS WHERE CLAIM_ID = ?", CLAIM_ROW, claimId) + .stream().findFirst(); + } + + public List findRecent(ClaimStatus status, int limit) { + String statusName = status == null ? null : status.name(); + return jdbc.query("SELECT " + CLAIM_COLUMNS + """ + FROM CLAIMS + WHERE (?::VARCHAR IS NULL OR STATUS = ?) + ORDER BY SUBMITTED_AT DESC, CLAIM_ID + LIMIT ?""", CLAIM_ROW, statusName, statusName, limit); + } + + public ReferenceCheck checkReferences(NewClaim claim) { + return jdbc.queryForObject(CHECK_REFERENCES, + (rs, n) -> new ReferenceCheck( + rs.getBoolean("DIAGNOSIS_VALID"), + rs.getBoolean("POLICY_EXISTS"), + rs.getBoolean("PROVIDER_EXISTS")), + claim.diagnosisCode(), claim.policyId(), claim.providerId()); + } + + /** Returns {@code false} when a claim with this id already exists. */ + public boolean insertIfAbsent(NewClaim claim) { + return jdbc.update(INSERT_IF_ABSENT, + claim.claimId(), claim.policyId(), claim.providerId(), claim.diagnosisCode(), + claim.serviceDate().toString(), claim.billedAmount()) == 1; + } + + /** + * Stages a batch with one {@code JdbcTemplate.batchUpdate}: the Snowflake driver + * sends all rows as array binds of a single INSERT (and switches to a temporary + * stage upload for very large batches) instead of one round trip per row. + */ + public void stageIntake(String batchId, List claims) { + jdbc.batchUpdate(STAGE_INTAKE_ROW, new BatchPreparedStatementSetter() { + @Override + public void setValues(PreparedStatement ps, int i) throws SQLException { + NewClaim claim = claims.get(i); + ps.setString(1, batchId); + ps.setInt(2, i + 1); + ps.setString(3, claim.claimId()); + ps.setString(4, claim.policyId()); + ps.setString(5, claim.providerId()); + ps.setString(6, claim.diagnosisCode()); + ps.setString(7, claim.serviceDate().toString()); + ps.setBigDecimal(8, claim.billedAmount()); + } + + @Override + public int getBatchSize() { + return claims.size(); + } + }); + } + + public void classifyIntake(String batchId) { + jdbc.update(CLASSIFY_INTAKE, batchId, batchId); + } + + /** Moves the batch's accepted rows into CLAIMS and returns how many were inserted. */ + public int promoteIntake(String batchId) { + return jdbc.update(PROMOTE_INTAKE, batchId); + } + + public List intakeRejections(String batchId) { + return jdbc.query(""" + SELECT ROW_NO, CLAIM_ID, REJECT_REASON + FROM CLAIM_INTAKE + WHERE BATCH_ID = ? AND REJECT_REASON IS NOT NULL + ORDER BY ROW_NO""", + (rs, n) -> new Rejection(rs.getInt(1), rs.getString(2), Reason.valueOf(rs.getString(3))), + batchId); + } + + public void clearIntake(String batchId) { + jdbc.update("DELETE FROM CLAIM_INTAKE WHERE BATCH_ID = ?", batchId); + } + + /** + * Runs the {@code ADJUDICATE_CLAIM} stored procedure, which owns its transaction, + * and returns its decision ({@code APPROVED} or {@code DENIED}). + */ + public String adjudicate(String claimId) { + return jdbc.queryForObject("CALL ADJUDICATE_CLAIM(?)", String.class, claimId); + } + + public List policyIds() { + return jdbc.queryForList("SELECT POLICY_ID FROM POLICIES ORDER BY POLICY_ID", String.class); + } + + public List providerIds() { + return jdbc.queryForList("SELECT PROVIDER_ID FROM PROVIDERS ORDER BY PROVIDER_ID", String.class); + } + + /** Result of the pre-insert checks that run in Snowflake. */ + public record ReferenceCheck(boolean diagnosisValid, boolean policyExists, boolean providerExists) { + } + + private static final RowMapper CLAIM_ROW = (rs, n) -> new Claim( + rs.getString("CLAIM_ID"), + rs.getString("POLICY_ID"), + rs.getString("PROVIDER_ID"), + rs.getString("DIAGNOSIS_CODE"), + rs.getDate("SERVICE_DATE").toLocalDate(), + rs.getBigDecimal("BILLED_AMOUNT"), + ClaimStatus.valueOf(rs.getString("STATUS")), + rs.getBigDecimal("PAYABLE_AMOUNT"), + rs.getString("DENIAL_REASON"), + instant(rs, "SUBMITTED_AT"), + instant(rs, "ADJUDICATED_AT")); + + private static Instant instant(ResultSet rs, String column) throws SQLException { + Timestamp ts = rs.getTimestamp(column); + return ts == null ? null : ts.toInstant(); + } +} diff --git a/claims-service/claims-api/src/main/java/com/example/claims/repo/KpiRepository.java b/claims-service/claims-api/src/main/java/com/example/claims/repo/KpiRepository.java new file mode 100644 index 0000000..5aca7a5 --- /dev/null +++ b/claims-service/claims-api/src/main/java/com/example/claims/repo/KpiRepository.java @@ -0,0 +1,41 @@ +package com.example.claims.repo; + +import com.example.claims.domain.ClaimStatus; +import com.example.claims.domain.MonthlyKpi; +import java.util.List; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.stereotype.Repository; + +/** Reads the incrementally maintained KPI table and triggers its refresh task. */ +@Repository +public class KpiRepository { + + static final String REFRESH_TASK = "REFRESH_CLAIM_KPIS"; + + private final JdbcTemplate jdbc; + + public KpiRepository(JdbcTemplate jdbc) { + this.jdbc = jdbc; + } + + public List findMonthly() { + return jdbc.query(""" + SELECT SERVICE_MONTH, STATUS, CLAIM_COUNT, BILLED_TOTAL, PAYABLE_TOTAL + FROM CLAIM_KPIS_MONTHLY + ORDER BY SERVICE_MONTH, STATUS""", + (rs, n) -> new MonthlyKpi( + rs.getDate(1).toLocalDate(), + ClaimStatus.valueOf(rs.getString(2)), + rs.getLong(3), + rs.getBigDecimal(4), + rs.getBigDecimal(5))); + } + + /** + * Starts one run of the KPI task now instead of waiting for its schedule. + * On Snowflake the run is asynchronous: the call returns once the run is queued. + */ + public void runRefreshTask() { + jdbc.execute("EXECUTE TASK " + REFRESH_TASK); + } +} diff --git a/claims-service/claims-api/src/main/java/com/example/claims/seed/ClaimSeeder.java b/claims-service/claims-api/src/main/java/com/example/claims/seed/ClaimSeeder.java new file mode 100644 index 0000000..63721af --- /dev/null +++ b/claims-service/claims-api/src/main/java/com/example/claims/seed/ClaimSeeder.java @@ -0,0 +1,73 @@ +package com.example.claims.seed; + +import com.example.claims.domain.BatchIntakeResult; +import com.example.claims.domain.NewClaim; +import com.example.claims.repo.ClaimsRepository; +import com.example.claims.service.AdjudicationService; +import com.example.claims.service.ClaimIntakeService; +import com.example.claims.service.KpiService; +import java.util.HashSet; +import java.util.List; +import java.util.Set; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.boot.ApplicationArguments; +import org.springframework.boot.ApplicationRunner; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.boot.context.properties.bind.DefaultValue; +import org.springframework.stereotype.Component; + +/** + * Loads deterministic sample claims through the bulk intake path, adjudicates the + * first {@code claims.seed.adjudicate} newly accepted ones (each is one procedure call) + * and refreshes the KPIs. Enabled with {@code claims.seed.enabled=true} + * ({@code make seed}); safe to re-run: existing claims are rejected as duplicates. + */ +@Component +@ConditionalOnProperty(name = "claims.seed.enabled", havingValue = "true") +public class ClaimSeeder implements ApplicationRunner { + + private static final Logger log = LoggerFactory.getLogger(ClaimSeeder.class); + + private final SeedProperties props; + private final ClaimsRepository claims; + private final ClaimIntakeService intake; + private final AdjudicationService adjudication; + private final KpiService kpis; + + public ClaimSeeder(SeedProperties props, ClaimsRepository claims, ClaimIntakeService intake, + AdjudicationService adjudication, KpiService kpis) { + this.props = props; + this.claims = claims; + this.intake = intake; + this.adjudication = adjudication; + this.kpis = kpis; + } + + @Override + public void run(ApplicationArguments args) { + List batch = SyntheticClaims.generate(props.randomSeed(), props.claims(), "CLM-SEED-", + claims.policyIds(), claims.providerIds()); + BatchIntakeResult result = intake.intake(batch); + Set rejected = new HashSet<>(); + result.rejected().forEach(r -> rejected.add(r.claimId())); + List toAdjudicate = batch.stream() + .map(NewClaim::claimId) + .filter(id -> !rejected.contains(id)) + .limit(props.adjudicate()) + .toList(); + toAdjudicate.forEach(adjudication::adjudicate); + kpis.refreshNow(); + log.info("Seeded {} claims: {} accepted, {} rejected, {} adjudicated", + result.received(), result.accepted(), result.rejected().size(), toAdjudicate.size()); + } + + @ConfigurationProperties("claims.seed") + public record SeedProperties( + @DefaultValue("false") boolean enabled, + @DefaultValue("300") int claims, + @DefaultValue("25") int adjudicate, + @DefaultValue("20261024") long randomSeed) { + } +} diff --git a/claims-service/claims-api/src/main/java/com/example/claims/seed/SyntheticClaims.java b/claims-service/claims-api/src/main/java/com/example/claims/seed/SyntheticClaims.java new file mode 100644 index 0000000..fc32d39 --- /dev/null +++ b/claims-service/claims-api/src/main/java/com/example/claims/seed/SyntheticClaims.java @@ -0,0 +1,53 @@ +package com.example.claims.seed; + +import com.example.claims.domain.NewClaim; +import java.math.BigDecimal; +import java.time.LocalDate; +import java.util.List; +import java.util.Random; +import java.util.stream.IntStream; + +/** + * Deterministic synthetic claims: the same seed always yields the same claims, so + * seeding is repeatable and re-seeding is rejected as duplicates. + */ +public final class SyntheticClaims { + + /** Real ICD-10-CM codes of common outpatient diagnoses. */ + static final List DIAGNOSIS_CODES = List.of( + "I10", // essential (primary) hypertension + "E11.9", // type 2 diabetes mellitus without complications + "J06.9", // acute upper respiratory infection, unspecified + "M54.50", // low back pain, unspecified + "Z00.00", // general adult medical examination without abnormal findings + "K21.9", // gastro-esophageal reflux disease without esophagitis + "F41.1", // generalized anxiety disorder + "J45.909", // unspecified asthma, uncomplicated + "N39.0", // urinary tract infection, site not specified + "S93.401A", // sprain of unspecified ligament of right ankle, initial encounter + "R51.9", // headache, unspecified + "Z41.1"); // encounter for cosmetic surgery (excluded from coverage) + + static final LocalDate FIRST_SERVICE_DATE = LocalDate.of(2026, 1, 1); + static final int SERVICE_DAYS = 181; // 2026-01-01 .. 2026-06-30 + + private SyntheticClaims() { + } + + public static List generate(long seed, int count, String idPrefix, + List policyIds, List providerIds) { + if (policyIds.isEmpty() || providerIds.isEmpty()) { + throw new IllegalArgumentException("reference data (policies, providers) is not loaded"); + } + Random random = new Random(seed); + return IntStream.rangeClosed(1, count) + .mapToObj(i -> new NewClaim( + "%s%06d".formatted(idPrefix, i), + policyIds.get(random.nextInt(policyIds.size())), + providerIds.get(random.nextInt(providerIds.size())), + DIAGNOSIS_CODES.get(random.nextInt(DIAGNOSIS_CODES.size())), + FIRST_SERVICE_DATE.plusDays(random.nextInt(SERVICE_DAYS)), + BigDecimal.valueOf(5_000L + random.nextInt(495_000), 2))) // 50.00 .. 4999.99 + .toList(); + } +} diff --git a/claims-service/claims-api/src/main/java/com/example/claims/service/AdjudicationService.java b/claims-service/claims-api/src/main/java/com/example/claims/service/AdjudicationService.java new file mode 100644 index 0000000..fd93aeb --- /dev/null +++ b/claims-service/claims-api/src/main/java/com/example/claims/service/AdjudicationService.java @@ -0,0 +1,47 @@ +package com.example.claims.service; + +import com.example.claims.domain.Claim; +import com.example.claims.repo.ClaimsRepository; +import java.sql.SQLException; +import org.springframework.dao.DataAccessException; +import org.springframework.stereotype.Service; + +/** + * Adjudicates a claim by calling the {@code ADJUDICATE_CLAIM} SQL Scripting procedure. + * + *

The procedure owns its transaction (BEGIN ... COMMIT, ROLLBACK on error), so this + * service deliberately runs it outside a Spring transaction. Business errors come back + * as the procedure's custom exceptions, which Snowflake reports with the exception's + * number as the JDBC error code and SQLSTATE {@code P0001}; they are matched by code, + * never by message text (Snowflake prefixes the message with the raising line). + */ +@Service +public class AdjudicationService { + + /** {@code claim_not_found EXCEPTION (-20001, ...)} in 002-adjudicate-procedure.sql. */ + static final int CLAIM_NOT_FOUND = -20001; + /** {@code claim_not_pending EXCEPTION (-20002, ...)} in 002-adjudicate-procedure.sql. */ + static final int CLAIM_NOT_PENDING = -20002; + + private final ClaimsRepository claims; + + public AdjudicationService(ClaimsRepository claims) { + this.claims = claims; + } + + public Claim adjudicate(String claimId) { + try { + claims.adjudicate(claimId); + } catch (DataAccessException e) { + if (e.getMostSpecificCause() instanceof SQLException sql) { + switch (sql.getErrorCode()) { + case CLAIM_NOT_FOUND -> throw new ClaimNotFoundException(claimId); + case CLAIM_NOT_PENDING -> throw new ClaimNotAdjudicableException(claimId, sql.getErrorCode(), e); + default -> throw e; + } + } + throw e; + } + return claims.findById(claimId).orElseThrow(() -> new ClaimNotFoundException(claimId)); + } +} diff --git a/claims-service/claims-api/src/main/java/com/example/claims/service/ClaimIntakeService.java b/claims-service/claims-api/src/main/java/com/example/claims/service/ClaimIntakeService.java new file mode 100644 index 0000000..47ec6f3 --- /dev/null +++ b/claims-service/claims-api/src/main/java/com/example/claims/service/ClaimIntakeService.java @@ -0,0 +1,64 @@ +package com.example.claims.service; + +import com.example.claims.domain.BatchIntakeResult; +import com.example.claims.domain.Claim; +import com.example.claims.domain.NewClaim; +import com.example.claims.repo.ClaimsRepository; +import com.example.claims.repo.ClaimsRepository.ReferenceCheck; +import com.example.claims.service.InvalidClaimException.Problem; +import java.util.List; +import java.util.UUID; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +/** Accepts claims one at a time (REST) or in bulk (batch intake). */ +@Service +public class ClaimIntakeService { + + private final ClaimsRepository claims; + + public ClaimIntakeService(ClaimsRepository claims) { + this.claims = claims; + } + + /** + * Validates the claim in Snowflake and inserts it unless its id exists. + * + * @throws InvalidClaimException bad ICD-10 code, unknown policy or provider (HTTP 422) + * @throws DuplicateClaimException a claim with this id was already submitted (HTTP 409) + */ + public Claim submit(NewClaim claim) { + ReferenceCheck check = claims.checkReferences(claim); + if (!check.diagnosisValid()) { + throw new InvalidClaimException(Problem.INVALID_DIAGNOSIS_CODE, + "'" + claim.diagnosisCode() + "' is not a valid ICD-10-CM code"); + } + if (!check.policyExists()) { + throw new InvalidClaimException(Problem.UNKNOWN_POLICY, "Policy " + claim.policyId() + " does not exist"); + } + if (!check.providerExists()) { + throw new InvalidClaimException(Problem.UNKNOWN_PROVIDER, + "Provider " + claim.providerId() + " does not exist"); + } + if (!claims.insertIfAbsent(claim)) { + throw new DuplicateClaimException(claim.claimId()); + } + return claims.findById(claim.claimId()).orElseThrow(() -> new ClaimNotFoundException(claim.claimId())); + } + + /** + * Loads a batch in one transaction: stage every row with a JDBC batch insert, + * classify the rows in SQL (same UDF and reference checks as {@link #submit}), + * MERGE the accepted ones into CLAIMS, and report the rest. + */ + @Transactional + public BatchIntakeResult intake(List batch) { + String batchId = UUID.randomUUID().toString(); + claims.stageIntake(batchId, batch); + claims.classifyIntake(batchId); + int accepted = claims.promoteIntake(batchId); + List rejected = claims.intakeRejections(batchId); + claims.clearIntake(batchId); + return new BatchIntakeResult(batchId, batch.size(), accepted, rejected); + } +} diff --git a/claims-service/claims-api/src/main/java/com/example/claims/service/ClaimNotAdjudicableException.java b/claims-service/claims-api/src/main/java/com/example/claims/service/ClaimNotAdjudicableException.java new file mode 100644 index 0000000..bcd451f --- /dev/null +++ b/claims-service/claims-api/src/main/java/com/example/claims/service/ClaimNotAdjudicableException.java @@ -0,0 +1,25 @@ +package com.example.claims.service; + +/** + * {@code ADJUDICATE_CLAIM} raised its {@code claim_not_pending} exception: the claim + * exists but is no longer {@code SUBMITTED}, so adjudicating it again is refused. + */ +public class ClaimNotAdjudicableException extends RuntimeException { + + private final String claimId; + private final int snowflakeErrorCode; + + public ClaimNotAdjudicableException(String claimId, int snowflakeErrorCode, Throwable cause) { + super("Claim " + claimId + " is not pending adjudication", cause); + this.claimId = claimId; + this.snowflakeErrorCode = snowflakeErrorCode; + } + + public String claimId() { + return claimId; + } + + public int snowflakeErrorCode() { + return snowflakeErrorCode; + } +} diff --git a/claims-service/claims-api/src/main/java/com/example/claims/service/ClaimNotFoundException.java b/claims-service/claims-api/src/main/java/com/example/claims/service/ClaimNotFoundException.java new file mode 100644 index 0000000..c0e62e7 --- /dev/null +++ b/claims-service/claims-api/src/main/java/com/example/claims/service/ClaimNotFoundException.java @@ -0,0 +1,15 @@ +package com.example.claims.service; + +public class ClaimNotFoundException extends RuntimeException { + + private final String claimId; + + public ClaimNotFoundException(String claimId) { + super("Claim " + claimId + " does not exist"); + this.claimId = claimId; + } + + public String claimId() { + return claimId; + } +} diff --git a/claims-service/claims-api/src/main/java/com/example/claims/service/DuplicateClaimException.java b/claims-service/claims-api/src/main/java/com/example/claims/service/DuplicateClaimException.java new file mode 100644 index 0000000..5e9233b --- /dev/null +++ b/claims-service/claims-api/src/main/java/com/example/claims/service/DuplicateClaimException.java @@ -0,0 +1,15 @@ +package com.example.claims.service; + +public class DuplicateClaimException extends RuntimeException { + + private final String claimId; + + public DuplicateClaimException(String claimId) { + super("Claim " + claimId + " was already submitted"); + this.claimId = claimId; + } + + public String claimId() { + return claimId; + } +} diff --git a/claims-service/claims-api/src/main/java/com/example/claims/service/InvalidClaimException.java b/claims-service/claims-api/src/main/java/com/example/claims/service/InvalidClaimException.java new file mode 100644 index 0000000..272286b --- /dev/null +++ b/claims-service/claims-api/src/main/java/com/example/claims/service/InvalidClaimException.java @@ -0,0 +1,36 @@ +package com.example.claims.service; + +/** + * The claim is well-formed but cannot be accepted: its diagnosis code fails the + * {@code IS_VALID_ICD10} UDF, or it references an unknown policy or provider. + */ +public class InvalidClaimException extends RuntimeException { + + /** Machine-readable problem codes, also used as the problem type suffix. */ + public enum Problem { + INVALID_DIAGNOSIS_CODE("invalid-diagnosis-code"), + UNKNOWN_POLICY("unknown-policy"), + UNKNOWN_PROVIDER("unknown-provider"); + + private final String slug; + + Problem(String slug) { + this.slug = slug; + } + + public String slug() { + return slug; + } + } + + private final Problem problem; + + public InvalidClaimException(Problem problem, String detail) { + super(detail); + this.problem = problem; + } + + public Problem problem() { + return problem; + } +} diff --git a/claims-service/claims-api/src/main/java/com/example/claims/service/KpiService.java b/claims-service/claims-api/src/main/java/com/example/claims/service/KpiService.java new file mode 100644 index 0000000..33b9208 --- /dev/null +++ b/claims-service/claims-api/src/main/java/com/example/claims/service/KpiService.java @@ -0,0 +1,29 @@ +package com.example.claims.service; + +import com.example.claims.domain.MonthlyKpi; +import com.example.claims.repo.KpiRepository; +import java.util.List; +import org.springframework.stereotype.Service; + +/** + * KPIs are maintained in Snowflake by a stream on CLAIMS and a scheduled task that + * MERGEs only the changed rows into CLAIM_KPIS_MONTHLY; the service only reads them. + */ +@Service +public class KpiService { + + private final KpiRepository kpis; + + public KpiService(KpiRepository kpis) { + this.kpis = kpis; + } + + public List monthly() { + return kpis.findMonthly(); + } + + /** Runs the refresh task now (operators, tests); normally it runs on its schedule. */ + public void refreshNow() { + kpis.runRefreshTask(); + } +} diff --git a/claims-service/claims-api/src/main/resources/application-cloud.yml b/claims-service/claims-api/src/main/resources/application-cloud.yml new file mode 100644 index 0000000..ff5b152 --- /dev/null +++ b/claims-service/claims-api/src/main/resources/application-cloud.yml @@ -0,0 +1,11 @@ +# Real Snowflake account: SPRING_PROFILES_ACTIVE=cloud. Consumes credits. +claims: + snowflake: + url: jdbc:snowflake://${SNOWFLAKE_HOST:${SNOWFLAKE_ACCOUNT}.snowflakecomputing.com}:443/ + account: ${SNOWFLAKE_ACCOUNT} + user: ${SNOWFLAKE_USER} + private-key-path: ${SNOWFLAKE_PRIVATE_KEY_PATH} + role: ${SNOWFLAKE_ROLE} + warehouse: ${SNOWFLAKE_WAREHOUSE} + database: ${SNOWFLAKE_DATABASE:CLAIMS} + schema: ${SNOWFLAKE_SCHEMA:CORE} diff --git a/claims-service/claims-api/src/main/resources/application.yml b/claims-service/claims-api/src/main/resources/application.yml new file mode 100644 index 0000000..7440afa --- /dev/null +++ b/claims-service/claims-api/src/main/resources/application.yml @@ -0,0 +1,39 @@ +# Defaults target the Snowflake-Next emulator. For a real account activate the +# "cloud" profile (application-cloud.yml), which reads the SNOWFLAKE_* variables. +spring: + application: + name: claims-service + mvc: + problemdetails: + enabled: true + liquibase: + change-log: classpath:db/changelog/db.changelog-master.yaml + default-schema: ${claims.snowflake.schema} + # "scheduled" also RESUMEs the KPI task so it runs every 5 minutes; without it the + # task stays suspended and runs only on EXECUTE TASK (tests, POST /api/kpis/refresh). + contexts: ${CLAIMS_LIQUIBASE_CONTEXTS:default} + parameters: + warehouse: ${claims.snowflake.warehouse} + appSchema: ${claims.snowflake.database}.${claims.snowflake.schema} + appRole: ${claims.snowflake.role} + +claims: + snowflake: + url: jdbc:snowflake://snowflake.localhost.localstack.cloud:${SNOWFLAKE_PORT:4566}/ + account: snowflake + user: SAMPLE_SVC + private-key-path: ${SNOWFLAKE_PRIVATE_KEY_PATH:.secrets/rsa_key.p8} + role: CLAIMS_APP + warehouse: CLAIMS_WH + database: CLAIMS + schema: CORE + +management: + endpoints: + web: + exposure: + include: health,info + +logging: + level: + net.snowflake: WARN diff --git a/claims-service/claims-api/src/main/resources/db/changelog/db.changelog-master.yaml b/claims-service/claims-api/src/main/resources/db/changelog/db.changelog-master.yaml new file mode 100644 index 0000000..eeec831 --- /dev/null +++ b/claims-service/claims-api/src/main/resources/db/changelog/db.changelog-master.yaml @@ -0,0 +1,20 @@ +# Liquibase OSS (built-in Snowflake support) migrates the service schema at startup. +# The database, schema, warehouse and role are provisioned by an administrator +# beforehand (init/01_platform.sf.sql); the service only owns objects inside its schema. +databaseChangeLog: + - include: + file: sql/001-schema.sql + relativeToChangelogFile: true + + - include: + file: sql/001-icd10-function.sql + relativeToChangelogFile: true + - include: + file: sql/002-adjudicate-procedure.sql + relativeToChangelogFile: true + - include: + file: sql/003-kpi-stream-task.sql + relativeToChangelogFile: true + - include: + file: sql/004-reference-data.sql + relativeToChangelogFile: true diff --git a/claims-service/claims-api/src/main/resources/db/changelog/sql/001-icd10-function.sql b/claims-service/claims-api/src/main/resources/db/changelog/sql/001-icd10-function.sql new file mode 100644 index 0000000..940ae93 --- /dev/null +++ b/claims-service/claims-api/src/main/resources/db/changelog/sql/001-icd10-function.sql @@ -0,0 +1,16 @@ +--liquibase formatted sql + +--changeset claims:001-icd10-function runOnChange:true splitStatements:false +-- TRUE when CODE is a structurally valid ICD-10-CM code in canonical dotted form: +-- * a category of 3 characters: a letter, a digit, then a digit or letter (E11, C4A); +-- * optionally a dot and 1 to 4 more digits or letters (E11.9, S93.401A). +-- Codes are stored trimmed and upper case, so anything else, including NULL, is FALSE. +-- REGEXP_LIKE matches the whole string. +CREATE OR REPLACE FUNCTION IS_VALID_ICD10(CODE VARCHAR) +RETURNS BOOLEAN +COMMENT = 'TRUE when CODE is a structurally valid ICD-10-CM code in canonical dotted form' +AS +$$ + COALESCE(REGEXP_LIKE(CODE, '[A-Z][0-9][0-9A-Z]([.][0-9A-Z]{1,4})?'), FALSE) +$$ +--rollback DROP FUNCTION IS_VALID_ICD10(VARCHAR); diff --git a/claims-service/claims-api/src/main/resources/db/changelog/sql/001-schema.sql b/claims-service/claims-api/src/main/resources/db/changelog/sql/001-schema.sql new file mode 100644 index 0000000..4f0c918 --- /dev/null +++ b/claims-service/claims-api/src/main/resources/db/changelog/sql/001-schema.sql @@ -0,0 +1,85 @@ +--liquibase formatted sql + +-- Snowflake records PRIMARY KEY constraints but does not enforce them; the service +-- prevents duplicate claim ids with MERGE (see ClaimsRepository). + +--changeset claims:001-policies +CREATE TABLE POLICIES ( + POLICY_ID VARCHAR(20) NOT NULL PRIMARY KEY, + MEMBER_NAME VARCHAR(100) NOT NULL, + PLAN_TIER VARCHAR(10) NOT NULL, + EFFECTIVE_DATE DATE NOT NULL, + TERMINATION_DATE DATE NOT NULL, + DEDUCTIBLE NUMBER(12, 2) NOT NULL, + COINSURANCE_RATE NUMBER(4, 3) NOT NULL, + CLAIM_LIMIT NUMBER(12, 2) NOT NULL +) COMMENT = 'Member policies: coverage period and cost-sharing terms (synthetic)'; +--rollback DROP TABLE POLICIES; + +--changeset claims:001-providers +CREATE TABLE PROVIDERS ( + PROVIDER_ID VARCHAR(20) NOT NULL PRIMARY KEY, + NAME VARCHAR(100) NOT NULL, + NPI VARCHAR(10) NOT NULL, + SPECIALTY VARCHAR(50) NOT NULL, + IN_NETWORK BOOLEAN NOT NULL +) COMMENT = 'Billing providers (synthetic)'; +--rollback DROP TABLE PROVIDERS; + +--changeset claims:001-claims +CREATE TABLE CLAIMS ( + CLAIM_ID VARCHAR(30) NOT NULL PRIMARY KEY, + POLICY_ID VARCHAR(20) NOT NULL, + PROVIDER_ID VARCHAR(20) NOT NULL, + DIAGNOSIS_CODE VARCHAR(8) NOT NULL, + SERVICE_DATE DATE NOT NULL, + BILLED_AMOUNT NUMBER(12, 2) NOT NULL, + STATUS VARCHAR(12) NOT NULL, + PAYABLE_AMOUNT NUMBER(12, 2), + DENIAL_REASON VARCHAR(30), + SUBMITTED_AT TIMESTAMP_LTZ NOT NULL DEFAULT CURRENT_TIMESTAMP(), + ADJUDICATED_AT TIMESTAMP_LTZ +) COMMENT = 'Insurance claims: one row per claim, STATUS SUBMITTED, APPROVED or DENIED'; +--rollback DROP TABLE CLAIMS; + +--changeset claims:001-claim-intake +CREATE TRANSIENT TABLE CLAIM_INTAKE ( + BATCH_ID VARCHAR(36) NOT NULL, + ROW_NO NUMBER(9, 0) NOT NULL, + CLAIM_ID VARCHAR(30) NOT NULL, + POLICY_ID VARCHAR(20) NOT NULL, + PROVIDER_ID VARCHAR(20) NOT NULL, + DIAGNOSIS_CODE VARCHAR(8) NOT NULL, + SERVICE_DATE DATE NOT NULL, + BILLED_AMOUNT NUMBER(12, 2) NOT NULL, + REJECT_REASON VARCHAR(30) +) COMMENT = 'Staging for bulk intake; rows live only for the duration of one batch transaction'; +--rollback DROP TABLE CLAIM_INTAKE; + +--changeset claims:001-adjudication-rules +CREATE TABLE ADJUDICATION_RULES ( + RULE_CODE VARCHAR(30) NOT NULL PRIMARY KEY, + PRIORITY NUMBER(4, 0) NOT NULL, + DENIAL_REASON VARCHAR(30) NOT NULL, + DESCRIPTION VARCHAR(200) NOT NULL, + ENABLED BOOLEAN NOT NULL DEFAULT TRUE +) COMMENT = 'Rules ADJUDICATE_CLAIM evaluates in PRIORITY order; the first failing rule denies the claim'; +--rollback DROP TABLE ADJUDICATION_RULES; + +--changeset claims:001-coverage-exclusions +CREATE TABLE COVERAGE_EXCLUSIONS ( + ICD10_CATEGORY VARCHAR(3) NOT NULL PRIMARY KEY, + DESCRIPTION VARCHAR(200) NOT NULL +) COMMENT = 'ICD-10 categories (first three characters) that policies do not cover'; +--rollback DROP TABLE COVERAGE_EXCLUSIONS; + +--changeset claims:001-adjudication-log +CREATE TABLE ADJUDICATION_LOG ( + CLAIM_ID VARCHAR(30) NOT NULL, + DECISION VARCHAR(12) NOT NULL, + DENIAL_REASON VARCHAR(30), + PAYABLE_AMOUNT NUMBER(12, 2) NOT NULL, + RULES_EVALUATED NUMBER(4, 0) NOT NULL, + ADJUDICATED_AT TIMESTAMP_LTZ NOT NULL DEFAULT CURRENT_TIMESTAMP() +) COMMENT = 'Audit trail: one row per adjudication decision'; +--rollback DROP TABLE ADJUDICATION_LOG; diff --git a/claims-service/claims-api/src/main/resources/db/changelog/sql/002-adjudicate-procedure.sql b/claims-service/claims-api/src/main/resources/db/changelog/sql/002-adjudicate-procedure.sql new file mode 100644 index 0000000..02c3a49 --- /dev/null +++ b/claims-service/claims-api/src/main/resources/db/changelog/sql/002-adjudicate-procedure.sql @@ -0,0 +1,111 @@ +--liquibase formatted sql + +--changeset claims:002-adjudicate-claim runOnChange:true splitStatements:false +-- Adjudicates one SUBMITTED claim: +-- 1. raises claim_not_found (-20001) or claim_not_pending (-20002) for bad input; +-- 2. walks the enabled ADJUDICATION_RULES with a declared cursor, in PRIORITY order, +-- and denies the claim with the first failing rule's DENIAL_REASON; +-- 3. computes the payable amount for approved claims from the policy terms; +-- 4. writes the decision and an audit row in one transaction, guarding against a +-- concurrent adjudication with SQLROWCOUNT. +-- The claim's values are read into variables once with SELECT ... INTO, so the rules +-- cursor is a plain query over ADJUDICATION_RULES and the loop only compares variables. +-- The procedure returns the decision; callers read the updated claim row itself. +CREATE OR REPLACE PROCEDURE ADJUDICATE_CLAIM(P_CLAIM_ID VARCHAR) +RETURNS VARCHAR +LANGUAGE SQL +EXECUTE AS OWNER +AS +$$ +DECLARE + claim_not_found EXCEPTION (-20001, 'Claim not found'); + claim_not_pending EXCEPTION (-20002, 'Claim is not pending adjudication'); + + rule_rs RESULTSET DEFAULT ( + SELECT RULE_CODE, DENIAL_REASON + FROM ADJUDICATION_RULES + WHERE ENABLED + ORDER BY PRIORITY, RULE_CODE); + rule_cursor CURSOR FOR rule_rs; + + v_found INTEGER DEFAULT 0; + v_status VARCHAR; + v_service_date DATE; + v_billed NUMBER(12, 2); + v_diagnosis VARCHAR; + v_effective DATE; + v_termination DATE; + v_deductible NUMBER(12, 2); + v_coinsurance NUMBER(4, 3); + v_claim_limit NUMBER(12, 2); + v_in_network BOOLEAN; + v_excluded INTEGER DEFAULT 0; + + v_decision VARCHAR DEFAULT 'APPROVED'; + v_denial_reason VARCHAR DEFAULT NULL; + v_member_share NUMBER(4, 3); + v_payable NUMBER(12, 2) DEFAULT 0; + v_rules_evaluated INTEGER DEFAULT 0; +BEGIN + SELECT COUNT(*) INTO :v_found FROM CLAIMS WHERE CLAIM_ID = :p_claim_id; + IF (v_found = 0) THEN + RAISE claim_not_found; + END IF; + + SELECT c.STATUS, c.SERVICE_DATE, c.BILLED_AMOUNT, c.DIAGNOSIS_CODE, + p.EFFECTIVE_DATE, p.TERMINATION_DATE, p.DEDUCTIBLE, p.COINSURANCE_RATE, p.CLAIM_LIMIT, + v.IN_NETWORK + INTO :v_status, :v_service_date, :v_billed, :v_diagnosis, + :v_effective, :v_termination, :v_deductible, :v_coinsurance, :v_claim_limit, + :v_in_network + FROM CLAIMS c + JOIN POLICIES p ON p.POLICY_ID = c.POLICY_ID + JOIN PROVIDERS v ON v.PROVIDER_ID = c.PROVIDER_ID + WHERE c.CLAIM_ID = :p_claim_id; + IF (v_status <> 'SUBMITTED') THEN + RAISE claim_not_pending; + END IF; + + SELECT COUNT(*) INTO :v_excluded + FROM COVERAGE_EXCLUSIONS + WHERE ICD10_CATEGORY = LEFT(:v_diagnosis, 3); + + FOR r IN rule_cursor DO + v_rules_evaluated := v_rules_evaluated + 1; + IF ((r.RULE_CODE = 'POLICY_ACTIVE' AND (v_service_date < v_effective OR v_service_date > v_termination)) + OR (r.RULE_CODE = 'DIAGNOSIS_COVERED' AND v_excluded > 0) + OR (r.RULE_CODE = 'WITHIN_CLAIM_LIMIT' AND v_billed > v_claim_limit)) THEN + v_decision := 'DENIED'; + v_denial_reason := r.DENIAL_REASON; + BREAK; + END IF; + END FOR; + + IF (v_decision = 'APPROVED') THEN + -- Out-of-network care carries 30 points more coinsurance. + v_member_share := IFF(v_in_network, v_coinsurance, LEAST(v_coinsurance + 0.3, 1)); + v_payable := ROUND(GREATEST(v_billed - v_deductible, 0) * (1 - v_member_share), 2); + END IF; + + BEGIN TRANSACTION; + UPDATE CLAIMS + SET STATUS = :v_decision, + PAYABLE_AMOUNT = :v_payable, + DENIAL_REASON = :v_denial_reason, + ADJUDICATED_AT = CURRENT_TIMESTAMP() + WHERE CLAIM_ID = :p_claim_id AND STATUS = 'SUBMITTED'; + IF (SQLROWCOUNT = 0) THEN + -- Another session adjudicated the claim since we read it. + ROLLBACK; + RAISE claim_not_pending; + END IF; + INSERT INTO ADJUDICATION_LOG (CLAIM_ID, DECISION, DENIAL_REASON, PAYABLE_AMOUNT, RULES_EVALUATED) + VALUES (:p_claim_id, :v_decision, :v_denial_reason, :v_payable, :v_rules_evaluated); + COMMIT; + RETURN v_decision; +EXCEPTION + WHEN OTHER THEN + ROLLBACK; + RAISE; +END; +$$ diff --git a/claims-service/claims-api/src/main/resources/db/changelog/sql/003-kpi-stream-task.sql b/claims-service/claims-api/src/main/resources/db/changelog/sql/003-kpi-stream-task.sql new file mode 100644 index 0000000..8f7ceb2 --- /dev/null +++ b/claims-service/claims-api/src/main/resources/db/changelog/sql/003-kpi-stream-task.sql @@ -0,0 +1,58 @@ +--liquibase formatted sql + +--changeset claims:003-kpi-table +CREATE TABLE CLAIM_KPIS_MONTHLY ( + SERVICE_MONTH DATE NOT NULL, + STATUS VARCHAR(12) NOT NULL, + CLAIM_COUNT NUMBER(18, 0) NOT NULL, + BILLED_TOTAL NUMBER(18, 2) NOT NULL, + PAYABLE_TOTAL NUMBER(18, 2) NOT NULL, + UPDATED_AT TIMESTAMP_LTZ NOT NULL, + PRIMARY KEY (SERVICE_MONTH, STATUS) +) COMMENT = 'Claims per service month and current status, maintained incrementally by REFRESH_CLAIM_KPIS'; +--rollback DROP TABLE CLAIM_KPIS_MONTHLY; + +--changeset claims:003-claims-stream +CREATE STREAM CLAIMS_CHANGES ON TABLE CLAIMS COMMENT = 'Changes to CLAIMS not yet folded into CLAIM_KPIS_MONTHLY'; +-- The KPI task reads the stream as the service role. +GRANT SELECT ON STREAM CLAIMS_CHANGES TO ROLE ${appRole}; +--rollback DROP STREAM CLAIMS_CHANGES; + +--changeset claims:003-kpi-task runOnChange:true splitStatements:false +-- One MERGE over the stream's net changes: an INSERT row adds a claim to its +-- (month, status) bucket, a DELETE row (the old image of an UPDATE) removes it, so a +-- status change moves the claim between buckets. Committing the MERGE advances the +-- stream offset, so the task body is that one statement. +CREATE OR REPLACE TASK REFRESH_CLAIM_KPIS + WAREHOUSE = ${warehouse} + SCHEDULE = '5 MINUTE' + COMMENT = 'Folds CLAIMS_CHANGES into CLAIM_KPIS_MONTHLY' + WHEN SYSTEM$STREAM_HAS_DATA('${appSchema}.CLAIMS_CHANGES') +AS +MERGE INTO ${appSchema}.CLAIM_KPIS_MONTHLY k +USING ( + SELECT DATE_TRUNC('MONTH', SERVICE_DATE) AS SERVICE_MONTH, + STATUS, + SUM(IFF(METADATA$ACTION = 'INSERT', 1, -1)) AS D_COUNT, + SUM(IFF(METADATA$ACTION = 'INSERT', BILLED_AMOUNT, -BILLED_AMOUNT)) AS D_BILLED, + SUM(IFF(METADATA$ACTION = 'INSERT', 1, -1) * COALESCE(PAYABLE_AMOUNT, 0)) AS D_PAYABLE + FROM ${appSchema}.CLAIMS_CHANGES + GROUP BY 1, 2 +) s +ON k.SERVICE_MONTH = s.SERVICE_MONTH AND k.STATUS = s.STATUS +WHEN MATCHED AND k.CLAIM_COUNT + s.D_COUNT = 0 THEN DELETE +WHEN MATCHED THEN UPDATE SET + CLAIM_COUNT = k.CLAIM_COUNT + s.D_COUNT, + BILLED_TOTAL = k.BILLED_TOTAL + s.D_BILLED, + PAYABLE_TOTAL = k.PAYABLE_TOTAL + s.D_PAYABLE, + UPDATED_AT = CURRENT_TIMESTAMP() +WHEN NOT MATCHED AND s.D_COUNT <> 0 THEN INSERT + (SERVICE_MONTH, STATUS, CLAIM_COUNT, BILLED_TOTAL, PAYABLE_TOTAL, UPDATED_AT) +VALUES + (s.SERVICE_MONTH, s.STATUS, s.D_COUNT, s.D_BILLED, s.D_PAYABLE, CURRENT_TIMESTAMP()); +--rollback DROP TASK REFRESH_CLAIM_KPIS; + +--changeset claims:003-kpi-task-resume context:scheduled runAlways:true +-- Production only (Liquibase context "scheduled"): CREATE OR REPLACE TASK leaves the +-- task suspended, so resume it after every deployment. +ALTER TASK REFRESH_CLAIM_KPIS RESUME; diff --git a/claims-service/claims-api/src/main/resources/db/changelog/sql/004-reference-data.sql b/claims-service/claims-api/src/main/resources/db/changelog/sql/004-reference-data.sql new file mode 100644 index 0000000..ce5e931 --- /dev/null +++ b/claims-service/claims-api/src/main/resources/db/changelog/sql/004-reference-data.sql @@ -0,0 +1,42 @@ +--liquibase formatted sql + +-- Synthetic reference data. Names, NPIs and policies are invented. + +--changeset claims:004-policies +INSERT INTO POLICIES + (POLICY_ID, MEMBER_NAME, PLAN_TIER, EFFECTIVE_DATE, TERMINATION_DATE, DEDUCTIBLE, COINSURANCE_RATE, CLAIM_LIMIT) +VALUES + ('POL-1001', 'Avery Chen', 'GOLD', '2025-01-01', '2026-12-31', 250.00, 0.100, 25000.00), + ('POL-1002', 'Jordan Patel', 'SILVER', '2025-07-01', '2027-06-30', 1000.00, 0.200, 15000.00), + ('POL-1003', 'Sam Okafor', 'BRONZE', '2026-01-01', '2026-12-31', 2500.00, 0.400, 10000.00), + ('POL-1004', 'Riley Novak', 'SILVER', '2024-01-01', '2026-03-31', 1000.00, 0.200, 15000.00), + ('POL-1005', 'Morgan Silva', 'GOLD', '2026-04-01', '2027-03-31', 250.00, 0.100, 25000.00), + ('POL-1006', 'Casey Lindqvist','BRONZE', '2025-01-01', '2026-12-31', 2500.00, 0.400, 4000.00), + ('POL-1007', 'Taylor Moreau', 'SILVER', '2026-01-01', '2026-12-31', 1000.00, 0.200, 15000.00), + ('POL-1008', 'Drew Haddad', 'GOLD', '2025-01-01', '2027-12-31', 250.00, 0.100, 25000.00); +--rollback DELETE FROM POLICIES WHERE POLICY_ID LIKE 'POL-10%'; + +--changeset claims:004-providers +INSERT INTO PROVIDERS (PROVIDER_ID, NAME, NPI, SPECIALTY, IN_NETWORK) +VALUES + ('PRV-201', 'Lakeside Family Medicine', '1902837465', 'Family Medicine', TRUE), + ('PRV-202', 'Northgate Orthopedics', '1548293071', 'Orthopedic Surgery', TRUE), + ('PRV-203', 'Harbor Urgent Care', '1265019384', 'Urgent Care', TRUE), + ('PRV-204', 'Summit Behavioral Health', '1730468259', 'Psychiatry', TRUE), + ('PRV-205', 'Westfield Imaging Center', '1384726150', 'Diagnostic Radiology', FALSE), + ('PRV-206', 'Riverbend Medical Group', '1659204837', 'Internal Medicine', FALSE); +--rollback DELETE FROM PROVIDERS WHERE PROVIDER_ID LIKE 'PRV-2%'; + +--changeset claims:004-adjudication-rules +INSERT INTO ADJUDICATION_RULES (RULE_CODE, PRIORITY, DENIAL_REASON, DESCRIPTION) +VALUES + ('POLICY_ACTIVE', 10, 'POLICY_INACTIVE', 'The service date falls within the policy period'), + ('DIAGNOSIS_COVERED', 20, 'NOT_COVERED', 'The diagnosis category is not excluded from coverage'), + ('WITHIN_CLAIM_LIMIT', 30, 'OVER_CLAIM_LIMIT', 'The billed amount does not exceed the policy per-claim limit'); +--rollback DELETE FROM ADJUDICATION_RULES; + +--changeset claims:004-coverage-exclusions +INSERT INTO COVERAGE_EXCLUSIONS (ICD10_CATEGORY, DESCRIPTION) +VALUES + ('Z41', 'Encounter for procedures for purposes other than remedying health state (cosmetic and elective)'); +--rollback DELETE FROM COVERAGE_EXCLUSIONS; diff --git a/claims-service/claims-api/src/test/java/com/example/claims/AdjudicationErrorPathTest.java b/claims-service/claims-api/src/test/java/com/example/claims/AdjudicationErrorPathTest.java new file mode 100644 index 0000000..b081a6a --- /dev/null +++ b/claims-service/claims-api/src/test/java/com/example/claims/AdjudicationErrorPathTest.java @@ -0,0 +1,121 @@ +package com.example.claims; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.catchThrowableOfType; + +import com.fasterxml.jackson.databind.JsonNode; +import java.sql.Connection; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.Map; +import javax.sql.DataSource; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpStatus; +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; + +/** + * Error paths: each one asserts the HTTP problem the client sees and that nothing was + * written that should not have been. + */ +class AdjudicationErrorPathTest extends IntegrationTestSupport { + + @Autowired + private DataSource dataSource; + + @ParameterizedTest + @ValueSource(strings = {"E11.", "E119", "e11.9", "XYZ", "1E1.9"}) + void invalidIcd10CodeIsRejectedByTheUdfWith422AndNotStored(String diagnosisCode) { + String id = newClaimId("DX"); + + ResponseEntity response = submit(claim(id, "POL-1001", "PRV-201", diagnosisCode, "2026-02-10", + "300.00")); + + assertThat(response.getStatusCode()).isEqualTo(HttpStatus.UNPROCESSABLE_ENTITY); + assertThat(response.getHeaders().getContentType()).isEqualTo(MediaType.APPLICATION_PROBLEM_JSON); + assertThat(response.getBody().get("type").asText()).isEqualTo("urn:claims:problem:invalid-diagnosis-code"); + assertThat(response.getBody().get("detail").asText()).contains(diagnosisCode); + assertThat(getClaim(id).getStatusCode()).isEqualTo(HttpStatus.NOT_FOUND); + } + + @Test + void duplicateClaimIdIsRejectedWith409AndTheOriginalIsUnchanged() { + String id = newClaimId("DUP"); + assertThat(submit(claim(id, "POL-1001", "PRV-201", "I10", "2026-03-01", "100.00")).getStatusCode()) + .isEqualTo(HttpStatus.CREATED); + + ResponseEntity second = submit(claim(id, "POL-1002", "PRV-202", "J06.9", "2026-03-02", "999.00")); + + assertThat(second.getStatusCode()).isEqualTo(HttpStatus.CONFLICT); + assertThat(second.getBody().get("type").asText()).isEqualTo("urn:claims:problem:duplicate-claim"); + assertThat(second.getBody().get("claimId").asText()).isEqualTo(id); + JsonNode stored = getClaim(id).getBody(); + assertThat(stored.get("policyId").asText()).isEqualTo("POL-1001"); + assertThat(stored.get("billedAmount").decimalValue()).isEqualByComparingTo("100.00"); + assertThat(jdbc.queryForObject("SELECT COUNT(*) FROM CLAIMS WHERE CLAIM_ID = ?", Long.class, id)).isEqualTo(1L); + } + + @Test + void adjudicatingTwiceRaisesTheProcedureExceptionMappedTo422() { + String id = newClaimId("TWICE"); + submit(claim(id, "POL-1001", "PRV-201", "I10", "2026-03-05", "1000.00")); + assertThat(adjudicate(id).getStatusCode()).isEqualTo(HttpStatus.OK); + + ResponseEntity second = adjudicate(id); + + assertThat(second.getStatusCode()).isEqualTo(HttpStatus.UNPROCESSABLE_ENTITY); + assertThat(second.getBody().get("type").asText()).isEqualTo("urn:claims:problem:claim-not-pending"); + assertThat(second.getBody().get("snowflakeErrorCode").asInt()).isEqualTo(-20002); + // The failed call changed nothing: still one decision, still the first payout. + Map log = jdbc.queryForMap( + "SELECT COUNT(*) AS N, MAX(PAYABLE_AMOUNT) AS PAYABLE FROM ADJUDICATION_LOG WHERE CLAIM_ID = ?", id); + assertThat(log).containsEntry("N", 1L); + assertThat(log.get("PAYABLE").toString()).isEqualTo("675.00"); // (1000-250)*0.90 + assertThat(getClaim(id).getBody().get("status").asText()).isEqualTo("APPROVED"); + } + + @Test + void adjudicatingAnUnknownClaimIs404FromTheProcedureClaimNotFoundException() throws SQLException { + ResponseEntity response = adjudicate("CLM-DOES-NOT-EXIST"); + + assertThat(response.getStatusCode()).isEqualTo(HttpStatus.NOT_FOUND); + assertThat(response.getBody().get("type").asText()).isEqualTo("urn:claims:problem:claim-not-found"); + + // The wire contract the service maps on: the custom exception's number and SQLSTATE P0001. + try (Connection connection = dataSource.getConnection(); Statement statement = connection.createStatement()) { + SQLException error = catchThrowableOfType(SQLException.class, + () -> statement.execute("CALL ADJUDICATE_CLAIM('CLM-DOES-NOT-EXIST')")); + assertThat(error.getErrorCode()).isEqualTo(-20001); + assertThat(error.getSQLState()).isEqualTo("P0001"); + assertThat(error.getMessage()).contains("Claim not found"); + } + } + + @Test + void unknownPolicyIsRejectedWith422() { + String id = newClaimId("POL"); + + ResponseEntity response = submit(claim(id, "POL-0000", "PRV-201", "I10", "2026-03-05", "100.00")); + + assertThat(response.getStatusCode()).isEqualTo(HttpStatus.UNPROCESSABLE_ENTITY); + assertThat(response.getBody().get("type").asText()).isEqualTo("urn:claims:problem:unknown-policy"); + assertThat(getClaim(id).getStatusCode()).isEqualTo(HttpStatus.NOT_FOUND); + } + + @Test + void malformedRequestIsRejectedWith400BeforeReachingSnowflake() { + ResponseEntity response = submit(Map.of( + "claimId", "not-a-claim-id", + "policyId", "POL-1001", + "providerId", "PRV-201", + "diagnosisCode", "I10", + "serviceDate", "2026-03-05", + "billedAmount", -5)); + + assertThat(response.getStatusCode()).isEqualTo(HttpStatus.BAD_REQUEST); + assertThat(response.getHeaders().getContentType()).isEqualTo(MediaType.APPLICATION_PROBLEM_JSON); + } +} diff --git a/claims-service/claims-api/src/test/java/com/example/claims/ClaimsIntegrationTest.java b/claims-service/claims-api/src/test/java/com/example/claims/ClaimsIntegrationTest.java new file mode 100644 index 0000000..8bcd688 --- /dev/null +++ b/claims-service/claims-api/src/test/java/com/example/claims/ClaimsIntegrationTest.java @@ -0,0 +1,201 @@ +package com.example.claims; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.tuple; + +import com.example.claims.domain.BatchIntakeResult; +import com.example.claims.domain.BatchIntakeResult.Reason; +import com.example.claims.domain.MonthlyKpi; +import com.example.claims.domain.NewClaim; +import com.example.claims.seed.SyntheticClaims; +import com.fasterxml.jackson.databind.JsonNode; +import java.math.BigDecimal; +import java.time.Duration; +import java.time.LocalDate; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import org.springframework.core.ParameterizedTypeReference; +import org.springframework.http.HttpEntity; +import org.springframework.http.HttpMethod; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; + +/** + * Happy paths end to end: REST -> JDBC -> Snowflake (UDF, procedure, stream, task). + * + *

Expected amounts are worked out by hand from the policy terms in + * {@code 004-reference-data.sql} and the payout rule: payable = max(billed - deductible, 0) + * x (1 - member share), where the member share is the policy coinsurance in network and + * coinsurance + 0.30 out of network. + */ +class ClaimsIntegrationTest extends IntegrationTestSupport { + + @Test + void approvesInNetworkClaimAndPaysBilledLessDeductibleLessCoinsurance() { + // POL-1001: deductible 250.00, coinsurance 0.10; PRV-201 is in network. + String id = newClaimId("OK"); + ResponseEntity created = submit(claim(id, "POL-1001", "PRV-201", "E11.9", "2026-02-10", "1200.00")); + assertThat(created.getStatusCode()).isEqualTo(HttpStatus.CREATED); + assertThat(created.getHeaders().getLocation()).hasToString("/api/claims/" + id); + assertThat(created.getBody().get("status").asText()).isEqualTo("SUBMITTED"); + + JsonNode adjudicated = adjudicate(id).getBody(); + + assertThat(adjudicated.get("status").asText()).isEqualTo("APPROVED"); + assertThat(adjudicated.get("payableAmount").decimalValue()).isEqualByComparingTo("855.00"); // (1200-250)*0.90 + assertThat(adjudicated.get("denialReason").isNull()).isTrue(); + assertThat(adjudicated.get("adjudicatedAt").isNull()).isFalse(); + assertThat(jdbc.queryForMap( + "SELECT DECISION, PAYABLE_AMOUNT, RULES_EVALUATED FROM ADJUDICATION_LOG WHERE CLAIM_ID = ?", id)) + .containsEntry("DECISION", "APPROVED") + .containsEntry("RULES_EVALUATED", 3L) + .hasEntrySatisfying("PAYABLE_AMOUNT", v -> assertThat((BigDecimal) v).isEqualByComparingTo("855.00")); + } + + @ParameterizedTest(name = "{0}") + @CsvSource(textBlock = """ + # case, policy, provider, dx, service date, billed, status, reason, payable + out of network adds 30 points of share, POL-1002, PRV-205, M54.50, 2026-03-02, 3000.00, APPROVED, , 1000.00 + billed below deductible pays nothing, POL-1003, PRV-203, J06.9, 2026-02-01, 800.00, APPROVED, , 0.00 + last day of the policy period is covered,POL-1004, PRV-201, I10, 2026-03-31, 1300.00, APPROVED, , 240.00 + day after termination is denied, POL-1004, PRV-201, I10, 2026-04-01, 1300.00, DENIED, POLICY_INACTIVE, 0.00 + day before the policy starts is denied, POL-1005, PRV-201, I10, 2026-03-31, 400.00, DENIED, POLICY_INACTIVE, 0.00 + excluded diagnosis category is denied, POL-1001, PRV-202, Z41.1, 2026-02-14, 2500.00, DENIED, NOT_COVERED, 0.00 + billed exactly at the claim limit, POL-1006, PRV-201, S93.401A,2026-05-05, 4000.00, APPROVED, , 900.00 + one cent over the claim limit is denied, POL-1006, PRV-201, S93.401A,2026-05-05, 4000.01, DENIED, OVER_CLAIM_LIMIT, 0.00 + first failing rule by priority wins, POL-1004, PRV-202, Z41.1, 2026-04-15, 9000.00, DENIED, POLICY_INACTIVE, 0.00 + """) + void adjudicationAppliesRulesInPriorityOrder(String scenario, String policy, String provider, String dx, + String serviceDate, String billed, String status, String reason, + String payable) { + // POL-1002 1000.00/0.20 -> OON share 0.50: (3000-1000)*0.50 = 1000.00 + // POL-1003 2500.00/0.40: 800 < 2500 -> 0.00 + // POL-1004 1000.00/0.20, period ends 2026-03-31: (1300-1000)*0.80 = 240.00 + // POL-1005 period starts 2026-04-01 + // POL-1006 2500.00/0.40, limit 4000.00: (4000-2500)*0.60 = 900.00 + String id = newClaimId("RULE"); + assertThat(submit(claim(id, policy, provider, dx, serviceDate, billed)).getStatusCode()) + .isEqualTo(HttpStatus.CREATED); + + JsonNode adjudicated = adjudicate(id).getBody(); + + assertThat(adjudicated.get("status").asText()).as(scenario).isEqualTo(status); + assertThat(adjudicated.get("denialReason").isNull() ? null : adjudicated.get("denialReason").asText()) + .as(scenario).isEqualTo(reason); + assertThat(adjudicated.get("payableAmount").decimalValue()).as(scenario).isEqualByComparingTo(payable); + } + + @Test + void bulkIntakeLoadsValidRowsWithOneBatchAndReportsEveryRejectedRow() { + String prefix = newClaimId("B") + "-"; + List valid = SyntheticClaims.generate(7, 250, prefix, + List.of("POL-1001", "POL-1002", "POL-1003", "POL-1007", "POL-1008"), + List.of("PRV-201", "PRV-202", "PRV-203", "PRV-204", "PRV-205", "PRV-206")); + String existing = newClaimId("EXIST"); + assertThat(submit(claim(existing, "POL-1001", "PRV-201", "I10", "2026-01-20", "150.00")).getStatusCode()) + .isEqualTo(HttpStatus.CREATED); + + List batch = new ArrayList<>(valid); + batch.add(claim(prefix + "BAD-DX", "POL-1001", "PRV-201", "E11.", "2026-02-01", "100.00")); // row 251 + batch.add(claim(prefix + "BAD-POL", "POL-9999", "PRV-201", "I10", "2026-02-01", "100.00")); // row 252 + batch.add(claim(prefix + "BAD-PRV", "POL-1001", "PRV-999", "I10", "2026-02-01", "100.00")); // row 253 + batch.add(valid.get(0)); // row 254 + batch.add(claim(existing, "POL-1001", "PRV-201", "I10", "2026-01-20", "150.00")); // row 255 + + ResponseEntity response = http.exchange("/api/claims/batch", HttpMethod.POST, + new HttpEntity<>(batch), BatchIntakeResult.class); + + assertThat(response.getStatusCode()).isEqualTo(HttpStatus.OK); + BatchIntakeResult result = response.getBody(); + assertThat(result.received()).isEqualTo(255); + assertThat(result.accepted()).isEqualTo(250); + assertThat(result.rejected()) + .extracting(BatchIntakeResult.Rejection::row, BatchIntakeResult.Rejection::claimId, + BatchIntakeResult.Rejection::reason) + .containsExactly( + tuple(251, prefix + "BAD-DX", Reason.INVALID_DIAGNOSIS_CODE), + tuple(252, prefix + "BAD-POL", Reason.UNKNOWN_POLICY), + tuple(253, prefix + "BAD-PRV", Reason.UNKNOWN_PROVIDER), + tuple(254, valid.get(0).claimId(), Reason.DUPLICATE_IN_BATCH), + tuple(255, existing, Reason.DUPLICATE_CLAIM_ID)); + + BigDecimal expectedBilled = valid.stream().map(NewClaim::billedAmount).reduce(BigDecimal.ZERO, BigDecimal::add); + Map stored = jdbc.queryForMap(""" + SELECT COUNT(*) AS N, COUNT(DISTINCT CLAIM_ID) AS IDS, SUM(BILLED_AMOUNT) AS BILLED + FROM CLAIMS WHERE STARTSWITH(CLAIM_ID, ?) AND STATUS = 'SUBMITTED'""", prefix); + assertThat(stored).containsEntry("N", 250L).containsEntry("IDS", 250L); + assertThat((BigDecimal) stored.get("BILLED")).isEqualByComparingTo(expectedBilled); + assertThat(jdbc.queryForObject("SELECT COUNT(*) FROM CLAIM_INTAKE WHERE BATCH_ID = ?", Long.class, + result.batchId())).isZero(); + } + + /** + * Headline test: CLAIM_KPIS_MONTHLY is maintained incrementally by the + * CLAIMS_CHANGES stream and the REFRESH_CLAIM_KPIS task, triggered with EXECUTE TASK. + * The second refresh only sees one UPDATE (a DELETE plus an INSERT in the stream), + * which must move the claim from SUBMITTED to APPROVED. + */ + @Test + void kpiTableReflectsNewAndAdjudicatedClaimsAfterExecuteTask() { + // September 2026 is used by no other test. + String a = newClaimId("KPIA"); + String b = newClaimId("KPIB"); + String c = newClaimId("KPIC"); + submit(claim(a, "POL-1001", "PRV-201", "I10", "2026-09-03", "500.00")); // approve: (500-250)*0.90 = 225.00 + submit(claim(b, "POL-1002", "PRV-202", "M54.50", "2026-09-10", "1500.00")); // later: (1500-1000)*0.80 = 400.00 + submit(claim(c, "POL-1001", "PRV-203", "Z41.1", "2026-09-21", "900.00")); // deny: NOT_COVERED + adjudicate(a); + adjudicate(c); + + assertThat(refreshAndAwaitSeptember("APPROVED=1/500.00/225.00 DENIED=1/900.00/0.00 SUBMITTED=1/1500.00/0.00")) + .isEqualTo("APPROVED=1/500.00/225.00 DENIED=1/900.00/0.00 SUBMITTED=1/1500.00/0.00"); + + adjudicate(b); + + assertThat(refreshAndAwaitSeptember("APPROVED=2/2000.00/625.00 DENIED=1/900.00/0.00")) + .isEqualTo("APPROVED=2/2000.00/625.00 DENIED=1/900.00/0.00"); + } + + private String refreshAndAwaitSeptember(String expected) { + assertThat(http.postForEntity("/api/kpis/refresh", null, Void.class).getStatusCode()) + .isEqualTo(HttpStatus.ACCEPTED); + return awaitValue("September 2026 KPIs " + expected, Duration.ofSeconds(90), + () -> september(http.exchange("/api/kpis/monthly", HttpMethod.GET, null, + new ParameterizedTypeReference>() { }).getBody()), + expected::equals); + } + + /** Renders September's rows as "STATUS=count/billed/payable ..." for one readable comparison. */ + private static String september(List kpis) { + return kpis.stream() + .filter(k -> k.serviceMonth().equals(LocalDate.of(2026, 9, 1))) + .map(k -> k.status() + "=" + k.claimCount() + "/" + k.billedTotal().setScale(2) + "/" + + k.payableTotal().setScale(2)) + .sorted() + .collect(Collectors.joining(" ")); + } + + @Test + void listsClaimsByStatus() { + String id = newClaimId("LIST"); + submit(claim(id, "POL-1008", "PRV-204", "F41.1", "2026-06-30", "640.00")); + + JsonNode submitted = http.getForEntity("/api/claims?status=SUBMITTED&limit=500", JsonNode.class).getBody(); + + List statuses = new ArrayList<>(); + List ids = new ArrayList<>(); + submitted.forEach(node -> { + statuses.add(node.get("status").asText()); + ids.add(node.get("claimId").asText()); + }); + assertThat(ids).contains(id); + assertThat(statuses).containsOnly("SUBMITTED"); + assertThat(http.getForEntity("/api/claims?limit=0", JsonNode.class).getStatusCode()) + .isEqualTo(HttpStatus.BAD_REQUEST); + } +} diff --git a/claims-service/claims-api/src/test/java/com/example/claims/Icd10FunctionTest.java b/claims-service/claims-api/src/test/java/com/example/claims/Icd10FunctionTest.java new file mode 100644 index 0000000..6820cd1 --- /dev/null +++ b/claims-service/claims-api/src/test/java/com/example/claims/Icd10FunctionTest.java @@ -0,0 +1,53 @@ +package com.example.claims; + +import static org.assertj.core.api.Assertions.assertThat; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.ValueSource; + +/** + * The {@code IS_VALID_ICD10} SQL UDF, called directly. The REST tests cover how the + * service uses it; these pin the ICD-10-CM structure rules themselves. + */ +class Icd10FunctionTest extends IntegrationTestSupport { + + @ParameterizedTest + @ValueSource(strings = { + "I10", // essential hypertension: category only + "E11.9", // type 2 diabetes without complications + "J45.909", // unspecified asthma, uncomplicated + "S93.401A", // ankle sprain, initial encounter: 7 characters with extension + "U07.1", // COVID-19 (special-purpose chapter U) + "C4A.9", // Merkel cell carcinoma: letter in position 3 + }) + void acceptsCanonicalCodes(String code) { + assertThat(isValid(code)).as(code).isTrue(); + } + + @ParameterizedTest + @ValueSource(strings = { + "", // empty + "E1", // too short + "E11.", // dot without subcategory + "E119", // undotted form is not canonical + "e11.9", // lower case is not canonical + " E11.9", // untrimmed + "1E1.9", // chapter must be a letter + "EE1.9", // second character must be a digit + "S93.401AB", // more than 7 characters + "E11-9", // wrong separator + }) + void rejectsMalformedCodes(String code) { + assertThat(isValid(code)).as(code).isFalse(); + } + + @Test + void nullIsNotValid() { + assertThat(isValid(null)).isFalse(); + } + + private Boolean isValid(String code) { + return jdbc.queryForObject("SELECT IS_VALID_ICD10(?::VARCHAR)", Boolean.class, code); + } +} diff --git a/claims-service/claims-api/src/test/java/com/example/claims/IntegrationTestSupport.java b/claims-service/claims-api/src/test/java/com/example/claims/IntegrationTestSupport.java new file mode 100644 index 0000000..54c462a --- /dev/null +++ b/claims-service/claims-api/src/test/java/com/example/claims/IntegrationTestSupport.java @@ -0,0 +1,88 @@ +package com.example.claims; + +import com.example.claims.domain.NewClaim; +import com.fasterxml.jackson.databind.JsonNode; +import java.math.BigDecimal; +import java.time.Duration; +import java.time.Instant; +import java.time.LocalDate; +import java.util.HexFormat; +import java.util.Locale; +import java.util.concurrent.ThreadLocalRandom; +import java.util.function.Predicate; +import java.util.function.Supplier; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.boot.test.web.client.TestRestTemplate; +import org.springframework.http.ResponseEntity; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.test.context.DynamicPropertyRegistry; +import org.springframework.test.context.DynamicPropertySource; + +/** + * Boots the whole service (HTTP on a random port, HikariCP, Liquibase) against the + * target chosen by {@code TEST_TARGET}. All test classes share this configuration, so + * Spring caches one context and the suite migrates one schema on one emulator. + */ +@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT, properties = "spring.main.banner-mode=off") +abstract class IntegrationTestSupport { + + @Autowired + protected TestRestTemplate http; + + @Autowired + protected JdbcTemplate jdbc; + + @DynamicPropertySource + static void snowflake(DynamicPropertyRegistry registry) { + SnowflakeTestTarget.get().register(registry); + } + + /** A claim id no other test (or earlier run on a shared database) uses. */ + protected static String newClaimId(String tag) { + return "CLM-" + tag + "-" + HexFormat.of().toHexDigits(ThreadLocalRandom.current().nextInt()) + .toUpperCase(Locale.ROOT); + } + + protected static NewClaim claim(String claimId, String policyId, String providerId, String diagnosisCode, + String serviceDate, String billedAmount) { + return new NewClaim(claimId, policyId, providerId, diagnosisCode, LocalDate.parse(serviceDate), + new BigDecimal(billedAmount)); + } + + protected ResponseEntity submit(Object claim) { + return http.postForEntity("/api/claims", claim, JsonNode.class); + } + + protected ResponseEntity adjudicate(String claimId) { + return http.postForEntity("/api/claims/{id}/adjudication", null, JsonNode.class, claimId); + } + + protected ResponseEntity getClaim(String claimId) { + return http.getForEntity("/api/claims/{id}", JsonNode.class, claimId); + } + + /** + * Polls until the value satisfies the condition. EXECUTE TASK queues a task run and + * returns, so tests wait for the observable result with a hard timeout instead of + * sleeping. + */ + protected static T awaitValue(String what, Duration timeout, Supplier probe, Predicate done) { + Instant deadline = Instant.now().plus(timeout); + T value = probe.get(); + while (!done.test(value)) { + if (Instant.now().isAfter(deadline)) { + throw new AssertionError("Timed out after " + timeout.toSeconds() + "s waiting for " + what + + "; last value: " + value); + } + try { + Thread.sleep(1_000); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new AssertionError(e); + } + value = probe.get(); + } + return value; + } +} diff --git a/claims-service/claims-api/src/test/java/com/example/claims/SnowflakeEmulatorContainer.java b/claims-service/claims-api/src/test/java/com/example/claims/SnowflakeEmulatorContainer.java new file mode 100644 index 0000000..c529056 --- /dev/null +++ b/claims-service/claims-api/src/test/java/com/example/claims/SnowflakeEmulatorContainer.java @@ -0,0 +1,47 @@ +package com.example.claims; + +import java.time.Duration; +import org.testcontainers.containers.GenericContainer; +import org.testcontainers.containers.wait.strategy.Wait; +import org.testcontainers.utility.DockerImageName; + +/** + * The Snowflake-Next emulator as a Testcontainers container. + * + *

Readiness is checked from the host: an HTTPS + * GET of {@code /} on the mapped port must answer below 500 (the TLS certificate is for + * {@code *.localhost.localstack.cloud}, hence {@code allowInsecure()} for this probe only; + * JDBC verifies it normally). Clients connect to + * {@code snowflake.localhost.localstack.cloud}, which resolves to 127.0.0.1, on the + * mapped port. + */ +public final class SnowflakeEmulatorContainer extends GenericContainer { + + public static final String DEFAULT_IMAGE = "localstack/snowflake-next:latest"; + public static final String HOST = "snowflake.localhost.localstack.cloud"; + private static final int PORT = 4566; + + public SnowflakeEmulatorContainer() { + super(DockerImageName.parse(System.getenv().getOrDefault("SNOWFLAKE_IMAGE", DEFAULT_IMAGE))); + withExposedPorts(PORT); + withEnv("LOCALSTACK_AUTH_TOKEN", authToken()); + waitingFor(Wait.forHttps("/") + .forPort(PORT) + .allowInsecure() + .forStatusCodeMatching(status -> status < 500) + .withStartupTimeout(Duration.ofMinutes(3))); + } + + public String jdbcUrl() { + return "jdbc:snowflake://" + HOST + ":" + getMappedPort(PORT) + "/"; + } + + private static String authToken() { + String token = System.getenv("LOCALSTACK_AUTH_TOKEN"); + if (token == null || token.isBlank()) { + throw new IllegalStateException("LOCALSTACK_AUTH_TOKEN is not set. Export it, or run `make test`, " + + "which reads it from ~/.localstack/default.env."); + } + return token; + } +} diff --git a/claims-service/claims-api/src/test/java/com/example/claims/SnowflakeTestTarget.java b/claims-service/claims-api/src/test/java/com/example/claims/SnowflakeTestTarget.java new file mode 100644 index 0000000..419e4e7 --- /dev/null +++ b/claims-service/claims-api/src/test/java/com/example/claims/SnowflakeTestTarget.java @@ -0,0 +1,228 @@ +package com.example.claims; + +import com.example.claims.config.PrivateKeys; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.PosixFilePermissions; +import java.security.KeyPairGenerator; +import java.security.NoSuchAlgorithmException; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.SQLException; +import java.sql.Statement; +import java.time.Duration; +import java.time.Instant; +import java.util.Arrays; +import java.util.Base64; +import java.util.HexFormat; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.ThreadLocalRandom; +import java.util.stream.Collectors; +import java.util.stream.Stream; +import org.springframework.test.context.DynamicPropertyRegistry; + +/** + * Where the suite runs, chosen by {@code TEST_TARGET}: + *

    + *
  • {@code EMULATOR} (default): starts one {@link SnowflakeEmulatorContainer} for the + * whole run, generates a throwaway key pair and runs the {@code init/*.sf.sql} + * platform provisioning scripts as ACCOUNTADMIN;
    • + *
  • {@code SNOWFLAKE_CLOUD}: connects to the account in the {@code SNOWFLAKE_*} + * variables, creates a {@code CLAIMS_IT_} database for this run and drops it + * when the JVM exits.
    • + *
+ * Nothing else differs between the two targets. + */ +final class SnowflakeTestTarget { + + private static final boolean EMULATOR = !"SNOWFLAKE_CLOUD".equals( + System.getenv().getOrDefault("TEST_TARGET", "EMULATOR").toUpperCase(Locale.ROOT)); + + private static final Duration READY_TIMEOUT = Duration.ofSeconds(90); + private static SnowflakeTestTarget instance; + + private final Map properties; + + private SnowflakeTestTarget(Map properties) { + this.properties = properties; + } + + static synchronized SnowflakeTestTarget get() { + if (instance == null) { + instance = EMULATOR ? emulator() : cloud(); + } + return instance; + } + + void register(DynamicPropertyRegistry registry) { + properties.forEach((name, value) -> registry.add(name, () -> value)); + } + + private static SnowflakeTestTarget emulator() { + SnowflakeEmulatorContainer container = new SnowflakeEmulatorContainer(); + container.start(); // stopped by Testcontainers' Ryuk when the JVM exits + Path key = generateThrowawayKey(); + String url = container.jdbcUrl(); + try (Connection admin = connectWhenReady(url, "snowflake", "SAMPLE_SVC", key, "ACCOUNTADMIN")) { + for (Path script : initScripts(Path.of(System.getProperty("claims.init-dir", "../init")))) { + runScript(admin, script); + } + } catch (SQLException e) { + throw new IllegalStateException("Provisioning the emulator failed", e); + } + return new SnowflakeTestTarget(settings(url, "snowflake", "SAMPLE_SVC", key, "CLAIMS_APP", "CLAIMS_WH", + "CLAIMS")); + } + + private static SnowflakeTestTarget cloud() { + String account = requireEnv("SNOWFLAKE_ACCOUNT"); + String host = System.getenv().getOrDefault("SNOWFLAKE_HOST", account + ".snowflakecomputing.com"); + String url = "jdbc:snowflake://" + host + ":443/"; + String user = requireEnv("SNOWFLAKE_USER"); + Path key = Path.of(requireEnv("SNOWFLAKE_PRIVATE_KEY_PATH")); + String role = requireEnv("SNOWFLAKE_ROLE"); + String warehouse = requireEnv("SNOWFLAKE_WAREHOUSE"); + String database = "CLAIMS_IT_" + HexFormat.of().toHexDigits(ThreadLocalRandom.current().nextInt()) + .toUpperCase(Locale.ROOT); + try (Connection admin = connectWhenReady(url, account, user, key, role); + Statement statement = admin.createStatement()) { + statement.execute("CREATE DATABASE " + database); + statement.execute("CREATE SCHEMA " + database + ".CORE"); + } catch (SQLException e) { + throw new IllegalStateException("Creating the test database failed", e); + } + Runtime.getRuntime().addShutdownHook(new Thread(() -> { + try (Connection admin = connect(url, account, user, key, role); + Statement statement = admin.createStatement()) { + statement.execute("DROP DATABASE IF EXISTS " + database); + } catch (SQLException e) { + System.err.println("Could not drop test database " + database + ": " + e.getMessage()); + } + })); + return new SnowflakeTestTarget(settings(url, account, user, key, role, warehouse, database)); + } + + private static Map settings(String url, String account, String user, Path key, String role, + String warehouse, String database) { + Map settings = new LinkedHashMap<>(); + settings.put("claims.snowflake.url", url); + settings.put("claims.snowflake.account", account); + settings.put("claims.snowflake.user", user); + settings.put("claims.snowflake.private-key-path", key.toString()); + settings.put("claims.snowflake.role", role); + settings.put("claims.snowflake.warehouse", warehouse); + settings.put("claims.snowflake.database", database); + settings.put("claims.snowflake.schema", "CORE"); + settings.put("claims.snowflake.query-tag", "claims-service-it"); + return settings; + } + + /** The first connection retries until the emulator answers SQL, with a hard timeout. */ + private static Connection connectWhenReady(String url, String account, String user, Path key, String role) + throws SQLException { + Instant deadline = Instant.now().plus(READY_TIMEOUT); + while (true) { + try { + Connection connection = connect(url, account, user, key, role); + try (Statement statement = connection.createStatement()) { + statement.execute("SELECT 1"); + } + return connection; + } catch (SQLException e) { + if (Instant.now().isAfter(deadline)) { + throw new SQLException("Snowflake at " + url + " did not answer SELECT 1 within " + + READY_TIMEOUT.toSeconds() + "s", e); + } + pause(); + } + } + } + + private static Connection connect(String url, String account, String user, Path key, String role) + throws SQLException { + Properties props = new Properties(); + props.put("account", account); + props.put("user", user); + props.put("authenticator", "SNOWFLAKE_JWT"); + props.put("privateKey", PrivateKeys.readPkcs8Pem(key)); + props.put("role", role); + return DriverManager.getConnection(url, props); + } + + /** The same {@code init/NN_name.sf.sql} files the compose emulator runs at startup, in name order. */ + private static List initScripts(Path dir) { + try (Stream files = Files.list(dir)) { + return files.filter(f -> f.getFileName().toString().endsWith(".sf.sql")).sorted().toList(); + } catch (IOException e) { + throw new UncheckedIOException("Cannot list init scripts in " + dir.toAbsolutePath(), e); + } + } + + private static void runScript(Connection connection, Path script) throws SQLException { + String sql; + try { + sql = Files.readString(script); + } catch (IOException e) { + throw new UncheckedIOException("Cannot read platform script " + script.toAbsolutePath(), e); + } + String withoutComments = sql.lines() + .filter(line -> !line.strip().startsWith("--")) + .collect(Collectors.joining("\n")); + List statements = Arrays.stream(withoutComments.split(";\\s*(\\R|$)")) + .map(String::strip) + .filter(s -> !s.isEmpty()) + .toList(); + try (Statement statement = connection.createStatement()) { + for (String each : statements) { + statement.execute(each); + } + } + } + + /** A fresh RSA key for this run, written outside the repository with owner-only permissions. */ + private static Path generateThrowawayKey() { + try { + KeyPairGenerator generator = KeyPairGenerator.getInstance("RSA"); + generator.initialize(2048); + byte[] pkcs8 = generator.generateKeyPair().getPrivate().getEncoded(); + String pem = "-----BEGIN PRIVATE KEY-----\n" + + Base64.getMimeEncoder(64, new byte[] {'\n'}).encodeToString(pkcs8) + + "\n-----END PRIVATE KEY-----\n"; + Path file = FileSystems.getDefault().supportedFileAttributeViews().contains("posix") + ? Files.createTempFile("claims-it-", ".p8", + PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rw-------"))) + : Files.createTempFile("claims-it-", ".p8"); + Files.writeString(file, pem); + file.toFile().deleteOnExit(); + return file; + } catch (IOException e) { + throw new UncheckedIOException(e); + } catch (NoSuchAlgorithmException e) { + throw new IllegalStateException(e); + } + } + + private static String requireEnv(String name) { + String value = System.getenv(name); + if (value == null || value.isBlank()) { + throw new IllegalStateException("TEST_TARGET=SNOWFLAKE_CLOUD needs " + name); + } + return value; + } + + private static void pause() { + try { + Thread.sleep(2_000); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new IllegalStateException(e); + } + } +} diff --git a/claims-service/claims-api/src/test/resources/logback-test.xml b/claims-service/claims-api/src/test/resources/logback-test.xml new file mode 100644 index 0000000..37b1833 --- /dev/null +++ b/claims-service/claims-api/src/test/resources/logback-test.xml @@ -0,0 +1,15 @@ + + + + + + %d{HH:mm:ss.SSS} %-5level %logger{36} - %msg%n + + + + + + + + + diff --git a/claims-service/docker-compose.yml b/claims-service/docker-compose.yml new file mode 100644 index 0000000..8cee204 --- /dev/null +++ b/claims-service/docker-compose.yml @@ -0,0 +1,20 @@ +# A long-lived emulator for `make seed` and `make run` (and for SnowSQL, DBeaver or +# any JDBC tool). The test suite does not use it: `./mvnw verify` starts its own +# emulator with Testcontainers. +services: + snowflake: + image: ${SNOWFLAKE_IMAGE:-localstack/snowflake-next:latest} + pull_policy: always + ports: + - "127.0.0.1:${SNOWFLAKE_PORT:-4566}:4566" + environment: + LOCALSTACK_AUTH_TOKEN: ${LOCALSTACK_AUTH_TOKEN:?set LOCALSTACK_AUTH_TOKEN} + volumes: + # 01_platform: warehouse, roles, database, schema, users and grants. + - ./init:/etc/localstack/init/ready.d:ro + networks: + default: + aliases: + - snowflake.localhost.localstack.cloud + - stages.s3.localhost.localstack.cloud + - stages-enc.s3.localhost.localstack.cloud diff --git a/claims-service/docs/architecture.md b/claims-service/docs/architecture.md new file mode 100644 index 0000000..129c4a4 --- /dev/null +++ b/claims-service/docs/architecture.md @@ -0,0 +1,93 @@ +# Claims Service: design notes + +How the [Claims Service](../README.md) divides work between Spring and Snowflake, and why. + +## Responsibilities + +| Layer | Owns | +| --- | --- | +| Platform (`init/01_platform.sf.sql`, run by an administrator) | Warehouse `CLAIMS_WH`, role `CLAIMS_APP`, database `CLAIMS`, schema `CORE`, service user `SAMPLE_SVC` | +| Liquibase (`claims-api/src/main/resources/db/changelog`) | Tables, the `IS_VALID_ICD10` SQL UDF, the `ADJUDICATE_CLAIM` procedure, stream, task, reference data | +| Snowflake at run time | Adjudication rules, ICD-10 validation, incremental KPIs | +| Spring service (`claims-api`) | REST API, request validation, error mapping, connection pool, bulk-intake transaction | + +Rules live in Snowflake so that the single-claim path, bulk intake and ad-hoc SQL apply them from one implementation. The adjudication rules are rows in `ADJUDICATION_RULES`, so operations can disable one with an `UPDATE` instead of a deployment. + +## Data model + +```mermaid +erDiagram + POLICIES ||--o{ CLAIMS : covers + PROVIDERS ||--o{ CLAIMS : bills + CLAIMS ||--o{ ADJUDICATION_LOG : "is decided in" + CLAIMS ||--o{ CLAIM_KPIS_MONTHLY : "is counted in" + POLICIES { + varchar POLICY_ID PK + date EFFECTIVE_DATE + date TERMINATION_DATE + number DEDUCTIBLE + number COINSURANCE_RATE + number CLAIM_LIMIT + } + PROVIDERS { + varchar PROVIDER_ID PK + boolean IN_NETWORK + } + CLAIMS { + varchar CLAIM_ID PK + varchar DIAGNOSIS_CODE + date SERVICE_DATE + number BILLED_AMOUNT + varchar STATUS + number PAYABLE_AMOUNT + } + ADJUDICATION_LOG { + varchar CLAIM_ID + varchar DECISION + number RULES_EVALUATED + } + CLAIM_KPIS_MONTHLY { + date SERVICE_MONTH PK + varchar STATUS PK + number CLAIM_COUNT + number PAYABLE_TOTAL + } +``` + +Snowflake records primary keys but does not enforce them, so uniqueness of `CLAIM_ID` comes from how rows are written: a submission is one `MERGE ... WHEN NOT MATCHED THEN INSERT`, and a second request with the same id inserts nothing and becomes `409 Conflict`. Bulk intake stages the batch with `JdbcTemplate.batchUpdate`, classifies every row with one `UPDATE ... FROM` (invalid code, unknown policy or provider, duplicate id), merges the accepted rows and returns the rejected ones with their row numbers, all in one transaction. + +## Adjudication + +| Rule | Priority | Fails when | Denial reason | +| --- | --- | --- | --- | +| `POLICY_ACTIVE` | 10 | the service date is outside the policy period | `POLICY_INACTIVE` | +| `DIAGNOSIS_COVERED` | 20 | the code's category is in `COVERAGE_EXCLUSIONS` | `NOT_COVERED` | +| `WITHIN_CLAIM_LIMIT` | 30 | `BILLED_AMOUNT > CLAIM_LIMIT` | `OVER_CLAIM_LIMIT` | + +An approved claim pays `max(billed - deductible, 0) x (1 - member share)`, where the member share is the policy's coinsurance rate, plus 0.30 (capped at 1) out of network. + +- **Variables first, then the rules.** The procedure reads the claim and policy values once with `SELECT ... INTO`, then walks a declared cursor over the enabled rules; the first failing rule denies the claim. +- **A plain result.** It returns the decision as a `VARCHAR`; the service then reads the stored claim, so the API response is always the row as written. +- **It owns its transaction.** `BEGIN TRANSACTION ... COMMIT` wraps the `UPDATE` and the audit insert, and `EXCEPTION WHEN OTHER THEN ROLLBACK; RAISE;` undoes a partial write. +- **Concurrency guard.** The `UPDATE` repeats `STATUS = 'SUBMITTED'` and checks `SQLROWCOUNT`, so two concurrent calls cannot both decide one claim. +- **Errors by number.** Custom exceptions reach JDBC as `SQLException.getErrorCode()` with SQLSTATE `P0001`; the service maps `-20001` to 404 and `-20002` to 422 and never parses the message. + +## Incremental KPIs + +The `CLAIMS_CHANGES` stream returns the net change since its offset. A new claim is an `INSERT` row; an adjudication of a counted claim is a `DELETE` of the old image plus an `INSERT` of the new one. The `REFRESH_CLAIM_KPIS` task body is one `MERGE` that adds +1 per `INSERT` and -1 per `DELETE` to each (month, status) bucket, so a claim moves from `SUBMITTED` to `APPROVED` or `DENIED`; committing it advances the stream offset. The task is created suspended and resumed only under the Liquibase `scheduled` context; tests and `POST /api/kpis/refresh` use `EXECUTE TASK`, and tests poll for the result. + +## ICD-10 validation + +`IS_VALID_ICD10(CODE)` is a SQL UDF: `COALESCE(REGEXP_LIKE(CODE, '[A-Z][0-9][0-9A-Z]([.][0-9A-Z]{1,4})?'), FALSE)`. It accepts the canonical dotted ICD-10-CM form: a 3-character category (letter, digit, digit or letter), optionally followed by a dot and 1 to 4 more characters, such as `I10`, `E11.9` or `S93.401A`. `REGEXP_LIKE` matches the whole string, and `NULL` is `FALSE`. The single-claim check, the bulk-intake `UPDATE` and ad-hoc SQL all call it, and its changeset is `runOnChange`, so editing the rule redeploys it. + +## Connection and authentication + +- One HikariCP pool as `SAMPLE_SVC` with role `CLAIMS_APP`, the owner of `CLAIMS.CORE`; Liquibase migrates through the same pool at start-up. `CLAIMS_APP` holds only `USAGE` on the warehouse and database and `EXECUTE TASK`, and rolls up to `SYSADMIN` like any custom role. +- Key-pair auth (`SNOWFLAKE_JWT`): the private key is parsed into a `PrivateKey` and handed to the driver, so a secret manager can supply it the same way as a file. +- Every connection sets role, warehouse, database, schema, `QUERY_TAG` and `application`, and `validateDefaultParameters=true` fails fast on a bad role or warehouse. +- The driver's Arrow reader needs `--add-opens=java.base/java.nio=ALL-UNNAMED` on Java 17+; the build sets it for tests, `spring-boot:run` and `java -jar`. + +## Moving to a real account + +1. An administrator runs `init/01_platform.sf.sql` and registers `RSA_PUBLIC_KEY` on `SAMPLE_SVC`. +2. Start the service with `SPRING_PROFILES_ACTIVE=cloud`, the `SNOWFLAKE_*` variables (`application-cloud.yml`), and `CLAIMS_LIQUIBASE_CONTEXTS=default,scheduled` to run the KPI task every 5 minutes. diff --git a/claims-service/init/01_platform.sf.sql b/claims-service/init/01_platform.sf.sql new file mode 100644 index 0000000..0c8c421 --- /dev/null +++ b/claims-service/init/01_platform.sf.sql @@ -0,0 +1,29 @@ +-- Platform provisioning for the claims service: what a Snowflake administrator runs +-- once per environment (in production usually Terraform or a platform repo). +-- The service itself only migrates objects inside CLAIMS.CORE with Liquibase. +-- +-- Used in two places: +-- * docker-compose.yml mounts init/ as the emulator's init hooks (make up); +-- * the Testcontainers suite runs init/*.sf.sql over JDBC after the container starts. +-- Keep it to plain statements, each ending with ';' at the end of a line. + +USE ROLE ACCOUNTADMIN; + +CREATE WAREHOUSE IF NOT EXISTS CLAIMS_WH + WAREHOUSE_SIZE = XSMALL AUTO_SUSPEND = 60 AUTO_RESUME = TRUE INITIALLY_SUSPENDED = TRUE; + +CREATE ROLE IF NOT EXISTS CLAIMS_APP; +GRANT USAGE ON WAREHOUSE CLAIMS_WH TO ROLE CLAIMS_APP; +GRANT EXECUTE TASK ON ACCOUNT TO ROLE CLAIMS_APP; +-- Custom roles roll up to SYSADMIN, so administrators can manage what CLAIMS_APP owns. +GRANT ROLE CLAIMS_APP TO ROLE SYSADMIN; + +CREATE DATABASE IF NOT EXISTS CLAIMS; +CREATE SCHEMA IF NOT EXISTS CLAIMS.CORE; +GRANT USAGE ON DATABASE CLAIMS TO ROLE CLAIMS_APP; +GRANT OWNERSHIP ON SCHEMA CLAIMS.CORE TO ROLE CLAIMS_APP COPY CURRENT GRANTS; + +-- Service user with key-pair auth only. On a real account also run: +-- ALTER USER SAMPLE_SVC SET RSA_PUBLIC_KEY = ''; +CREATE USER IF NOT EXISTS SAMPLE_SVC TYPE = SERVICE DEFAULT_ROLE = CLAIMS_APP DEFAULT_WAREHOUSE = CLAIMS_WH; +GRANT ROLE CLAIMS_APP TO USER SAMPLE_SVC; diff --git a/claims-service/mvnw b/claims-service/mvnw new file mode 100755 index 0000000..bd8896b --- /dev/null +++ b/claims-service/mvnw @@ -0,0 +1,295 @@ +#!/bin/sh +# ---------------------------------------------------------------------------- +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# ---------------------------------------------------------------------------- + +# ---------------------------------------------------------------------------- +# Apache Maven Wrapper startup batch script, version 3.3.4 +# +# Optional ENV vars +# ----------------- +# JAVA_HOME - location of a JDK home dir, required when download maven via java source +# MVNW_REPOURL - repo url base for downloading maven distribution +# MVNW_USERNAME/MVNW_PASSWORD - user and password for downloading maven +# MVNW_VERBOSE - true: enable verbose log; debug: trace the mvnw script; others: silence the output +# ---------------------------------------------------------------------------- + +set -euf +[ "${MVNW_VERBOSE-}" != debug ] || set -x + +# OS specific support. +native_path() { printf %s\\n "$1"; } +case "$(uname)" in +CYGWIN* | MINGW*) + [ -z "${JAVA_HOME-}" ] || JAVA_HOME="$(cygpath --unix "$JAVA_HOME")" + native_path() { cygpath --path --windows "$1"; } + ;; +esac + +# set JAVACMD and JAVACCMD +set_java_home() { + # For Cygwin and MinGW, ensure paths are in Unix format before anything is touched + if [ -n "${JAVA_HOME-}" ]; then + if [ -x "$JAVA_HOME/jre/sh/java" ]; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + JAVACCMD="$JAVA_HOME/jre/sh/javac" + else + JAVACMD="$JAVA_HOME/bin/java" + JAVACCMD="$JAVA_HOME/bin/javac" + + if [ ! -x "$JAVACMD" ] || [ ! -x "$JAVACCMD" ]; then + echo "The JAVA_HOME environment variable is not defined correctly, so mvnw cannot run." >&2 + echo "JAVA_HOME is set to \"$JAVA_HOME\", but \"\$JAVA_HOME/bin/java\" or \"\$JAVA_HOME/bin/javac\" does not exist." >&2 + return 1 + fi + fi + else + JAVACMD="$( + 'set' +e + 'unset' -f command 2>/dev/null + 'command' -v java + )" || : + JAVACCMD="$( + 'set' +e + 'unset' -f command 2>/dev/null + 'command' -v javac + )" || : + + if [ ! -x "${JAVACMD-}" ] || [ ! -x "${JAVACCMD-}" ]; then + echo "The java/javac command does not exist in PATH nor is JAVA_HOME set, so mvnw cannot run." >&2 + return 1 + fi + fi +} + +# hash string like Java String::hashCode +hash_string() { + str="${1:-}" h=0 + while [ -n "$str" ]; do + char="${str%"${str#?}"}" + h=$(((h * 31 + $(LC_CTYPE=C printf %d "'$char")) % 4294967296)) + str="${str#?}" + done + printf %x\\n $h +} + +verbose() { :; } +[ "${MVNW_VERBOSE-}" != true ] || verbose() { printf %s\\n "${1-}"; } + +die() { + printf %s\\n "$1" >&2 + exit 1 +} + +trim() { + # MWRAPPER-139: + # Trims trailing and leading whitespace, carriage returns, tabs, and linefeeds. + # Needed for removing poorly interpreted newline sequences when running in more + # exotic environments such as mingw bash on Windows. + printf "%s" "${1}" | tr -d '[:space:]' +} + +scriptDir="$(dirname "$0")" +scriptName="$(basename "$0")" + +# parse distributionUrl and optional distributionSha256Sum, requires .mvn/wrapper/maven-wrapper.properties +while IFS="=" read -r key value; do + case "${key-}" in + distributionUrl) distributionUrl=$(trim "${value-}") ;; + distributionSha256Sum) distributionSha256Sum=$(trim "${value-}") ;; + esac +done <"$scriptDir/.mvn/wrapper/maven-wrapper.properties" +[ -n "${distributionUrl-}" ] || die "cannot read distributionUrl property in $scriptDir/.mvn/wrapper/maven-wrapper.properties" + +case "${distributionUrl##*/}" in +maven-mvnd-*bin.*) + MVN_CMD=mvnd.sh _MVNW_REPO_PATTERN=/maven/mvnd/ + case "${PROCESSOR_ARCHITECTURE-}${PROCESSOR_ARCHITEW6432-}:$(uname -a)" in + *AMD64:CYGWIN* | *AMD64:MINGW*) distributionPlatform=windows-amd64 ;; + :Darwin*x86_64) distributionPlatform=darwin-amd64 ;; + :Darwin*arm64) distributionPlatform=darwin-aarch64 ;; + :Linux*x86_64*) distributionPlatform=linux-amd64 ;; + *) + echo "Cannot detect native platform for mvnd on $(uname)-$(uname -m), use pure java version" >&2 + distributionPlatform=linux-amd64 + ;; + esac + distributionUrl="${distributionUrl%-bin.*}-$distributionPlatform.zip" + ;; +maven-mvnd-*) MVN_CMD=mvnd.sh _MVNW_REPO_PATTERN=/maven/mvnd/ ;; +*) MVN_CMD="mvn${scriptName#mvnw}" _MVNW_REPO_PATTERN=/org/apache/maven/ ;; +esac + +# apply MVNW_REPOURL and calculate MAVEN_HOME +# maven home pattern: ~/.m2/wrapper/dists/{apache-maven-,maven-mvnd--}/ +[ -z "${MVNW_REPOURL-}" ] || distributionUrl="$MVNW_REPOURL$_MVNW_REPO_PATTERN${distributionUrl#*"$_MVNW_REPO_PATTERN"}" +distributionUrlName="${distributionUrl##*/}" +distributionUrlNameMain="${distributionUrlName%.*}" +distributionUrlNameMain="${distributionUrlNameMain%-bin}" +MAVEN_USER_HOME="${MAVEN_USER_HOME:-${HOME}/.m2}" +MAVEN_HOME="${MAVEN_USER_HOME}/wrapper/dists/${distributionUrlNameMain-}/$(hash_string "$distributionUrl")" + +exec_maven() { + unset MVNW_VERBOSE MVNW_USERNAME MVNW_PASSWORD MVNW_REPOURL || : + exec "$MAVEN_HOME/bin/$MVN_CMD" "$@" || die "cannot exec $MAVEN_HOME/bin/$MVN_CMD" +} + +if [ -d "$MAVEN_HOME" ]; then + verbose "found existing MAVEN_HOME at $MAVEN_HOME" + exec_maven "$@" +fi + +case "${distributionUrl-}" in +*?-bin.zip | *?maven-mvnd-?*-?*.zip) ;; +*) die "distributionUrl is not valid, must match *-bin.zip or maven-mvnd-*.zip, but found '${distributionUrl-}'" ;; +esac + +# prepare tmp dir +if TMP_DOWNLOAD_DIR="$(mktemp -d)" && [ -d "$TMP_DOWNLOAD_DIR" ]; then + clean() { rm -rf -- "$TMP_DOWNLOAD_DIR"; } + trap clean HUP INT TERM EXIT +else + die "cannot create temp dir" +fi + +mkdir -p -- "${MAVEN_HOME%/*}" + +# Download and Install Apache Maven +verbose "Couldn't find MAVEN_HOME, downloading and installing it ..." +verbose "Downloading from: $distributionUrl" +verbose "Downloading to: $TMP_DOWNLOAD_DIR/$distributionUrlName" + +# select .zip or .tar.gz +if ! command -v unzip >/dev/null; then + distributionUrl="${distributionUrl%.zip}.tar.gz" + distributionUrlName="${distributionUrl##*/}" +fi + +# verbose opt +__MVNW_QUIET_WGET=--quiet __MVNW_QUIET_CURL=--silent __MVNW_QUIET_UNZIP=-q __MVNW_QUIET_TAR='' +[ "${MVNW_VERBOSE-}" != true ] || __MVNW_QUIET_WGET='' __MVNW_QUIET_CURL='' __MVNW_QUIET_UNZIP='' __MVNW_QUIET_TAR=v + +# normalize http auth +case "${MVNW_PASSWORD:+has-password}" in +'') MVNW_USERNAME='' MVNW_PASSWORD='' ;; +has-password) [ -n "${MVNW_USERNAME-}" ] || MVNW_USERNAME='' MVNW_PASSWORD='' ;; +esac + +if [ -z "${MVNW_USERNAME-}" ] && command -v wget >/dev/null; then + verbose "Found wget ... using wget" + wget ${__MVNW_QUIET_WGET:+"$__MVNW_QUIET_WGET"} "$distributionUrl" -O "$TMP_DOWNLOAD_DIR/$distributionUrlName" || die "wget: Failed to fetch $distributionUrl" +elif [ -z "${MVNW_USERNAME-}" ] && command -v curl >/dev/null; then + verbose "Found curl ... using curl" + curl ${__MVNW_QUIET_CURL:+"$__MVNW_QUIET_CURL"} -f -L -o "$TMP_DOWNLOAD_DIR/$distributionUrlName" "$distributionUrl" || die "curl: Failed to fetch $distributionUrl" +elif set_java_home; then + verbose "Falling back to use Java to download" + javaSource="$TMP_DOWNLOAD_DIR/Downloader.java" + targetZip="$TMP_DOWNLOAD_DIR/$distributionUrlName" + cat >"$javaSource" <<-END + public class Downloader extends java.net.Authenticator + { + protected java.net.PasswordAuthentication getPasswordAuthentication() + { + return new java.net.PasswordAuthentication( System.getenv( "MVNW_USERNAME" ), System.getenv( "MVNW_PASSWORD" ).toCharArray() ); + } + public static void main( String[] args ) throws Exception + { + setDefault( new Downloader() ); + java.nio.file.Files.copy( java.net.URI.create( args[0] ).toURL().openStream(), java.nio.file.Paths.get( args[1] ).toAbsolutePath().normalize() ); + } + } + END + # For Cygwin/MinGW, switch paths to Windows format before running javac and java + verbose " - Compiling Downloader.java ..." + "$(native_path "$JAVACCMD")" "$(native_path "$javaSource")" || die "Failed to compile Downloader.java" + verbose " - Running Downloader.java ..." + "$(native_path "$JAVACMD")" -cp "$(native_path "$TMP_DOWNLOAD_DIR")" Downloader "$distributionUrl" "$(native_path "$targetZip")" +fi + +# If specified, validate the SHA-256 sum of the Maven distribution zip file +if [ -n "${distributionSha256Sum-}" ]; then + distributionSha256Result=false + if [ "$MVN_CMD" = mvnd.sh ]; then + echo "Checksum validation is not supported for maven-mvnd." >&2 + echo "Please disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." >&2 + exit 1 + elif command -v sha256sum >/dev/null; then + if echo "$distributionSha256Sum $TMP_DOWNLOAD_DIR/$distributionUrlName" | sha256sum -c - >/dev/null 2>&1; then + distributionSha256Result=true + fi + elif command -v shasum >/dev/null; then + if echo "$distributionSha256Sum $TMP_DOWNLOAD_DIR/$distributionUrlName" | shasum -a 256 -c >/dev/null 2>&1; then + distributionSha256Result=true + fi + else + echo "Checksum validation was requested but neither 'sha256sum' or 'shasum' are available." >&2 + echo "Please install either command, or disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." >&2 + exit 1 + fi + if [ $distributionSha256Result = false ]; then + echo "Error: Failed to validate Maven distribution SHA-256, your Maven distribution might be compromised." >&2 + echo "If you updated your Maven version, you need to update the specified distributionSha256Sum property." >&2 + exit 1 + fi +fi + +# unzip and move +if command -v unzip >/dev/null; then + unzip ${__MVNW_QUIET_UNZIP:+"$__MVNW_QUIET_UNZIP"} "$TMP_DOWNLOAD_DIR/$distributionUrlName" -d "$TMP_DOWNLOAD_DIR" || die "failed to unzip" +else + tar xzf${__MVNW_QUIET_TAR:+"$__MVNW_QUIET_TAR"} "$TMP_DOWNLOAD_DIR/$distributionUrlName" -C "$TMP_DOWNLOAD_DIR" || die "failed to untar" +fi + +# Find the actual extracted directory name (handles snapshots where filename != directory name) +actualDistributionDir="" + +# First try the expected directory name (for regular distributions) +if [ -d "$TMP_DOWNLOAD_DIR/$distributionUrlNameMain" ]; then + if [ -f "$TMP_DOWNLOAD_DIR/$distributionUrlNameMain/bin/$MVN_CMD" ]; then + actualDistributionDir="$distributionUrlNameMain" + fi +fi + +# If not found, search for any directory with the Maven executable (for snapshots) +if [ -z "$actualDistributionDir" ]; then + # enable globbing to iterate over items + set +f + for dir in "$TMP_DOWNLOAD_DIR"/*; do + if [ -d "$dir" ]; then + if [ -f "$dir/bin/$MVN_CMD" ]; then + actualDistributionDir="$(basename "$dir")" + break + fi + fi + done + set -f +fi + +if [ -z "$actualDistributionDir" ]; then + verbose "Contents of $TMP_DOWNLOAD_DIR:" + verbose "$(ls -la "$TMP_DOWNLOAD_DIR")" + die "Could not find Maven distribution directory in extracted archive" +fi + +verbose "Found extracted Maven distribution directory: $actualDistributionDir" +printf %s\\n "$distributionUrl" >"$TMP_DOWNLOAD_DIR/$actualDistributionDir/mvnw.url" +mv -- "$TMP_DOWNLOAD_DIR/$actualDistributionDir" "$MAVEN_HOME" || [ -d "$MAVEN_HOME" ] || die "fail to move MAVEN_HOME" + +clean || : +exec_maven "$@" diff --git a/claims-service/mvnw.cmd b/claims-service/mvnw.cmd new file mode 100644 index 0000000..5761d94 --- /dev/null +++ b/claims-service/mvnw.cmd @@ -0,0 +1,189 @@ +<# : batch portion +@REM ---------------------------------------------------------------------------- +@REM Licensed to the Apache Software Foundation (ASF) under one +@REM or more contributor license agreements. See the NOTICE file +@REM distributed with this work for additional information +@REM regarding copyright ownership. The ASF licenses this file +@REM to you under the Apache License, Version 2.0 (the +@REM "License"); you may not use this file except in compliance +@REM with the License. You may obtain a copy of the License at +@REM +@REM http://www.apache.org/licenses/LICENSE-2.0 +@REM +@REM Unless required by applicable law or agreed to in writing, +@REM software distributed under the License is distributed on an +@REM "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +@REM KIND, either express or implied. See the License for the +@REM specific language governing permissions and limitations +@REM under the License. +@REM ---------------------------------------------------------------------------- + +@REM ---------------------------------------------------------------------------- +@REM Apache Maven Wrapper startup batch script, version 3.3.4 +@REM +@REM Optional ENV vars +@REM MVNW_REPOURL - repo url base for downloading maven distribution +@REM MVNW_USERNAME/MVNW_PASSWORD - user and password for downloading maven +@REM MVNW_VERBOSE - true: enable verbose log; others: silence the output +@REM ---------------------------------------------------------------------------- + +@IF "%__MVNW_ARG0_NAME__%"=="" (SET __MVNW_ARG0_NAME__=%~nx0) +@SET __MVNW_CMD__= +@SET __MVNW_ERROR__= +@SET __MVNW_PSMODULEP_SAVE=%PSModulePath% +@SET PSModulePath= +@FOR /F "usebackq tokens=1* delims==" %%A IN (`powershell -noprofile "& {$scriptDir='%~dp0'; $script='%__MVNW_ARG0_NAME__%'; icm -ScriptBlock ([Scriptblock]::Create((Get-Content -Raw '%~f0'))) -NoNewScope}"`) DO @( + IF "%%A"=="MVN_CMD" (set __MVNW_CMD__=%%B) ELSE IF "%%B"=="" (echo %%A) ELSE (echo %%A=%%B) +) +@SET PSModulePath=%__MVNW_PSMODULEP_SAVE% +@SET __MVNW_PSMODULEP_SAVE= +@SET __MVNW_ARG0_NAME__= +@SET MVNW_USERNAME= +@SET MVNW_PASSWORD= +@IF NOT "%__MVNW_CMD__%"=="" ("%__MVNW_CMD__%" %*) +@echo Cannot start maven from wrapper >&2 && exit /b 1 +@GOTO :EOF +: end batch / begin powershell #> + +$ErrorActionPreference = "Stop" +if ($env:MVNW_VERBOSE -eq "true") { + $VerbosePreference = "Continue" +} + +# calculate distributionUrl, requires .mvn/wrapper/maven-wrapper.properties +$distributionUrl = (Get-Content -Raw "$scriptDir/.mvn/wrapper/maven-wrapper.properties" | ConvertFrom-StringData).distributionUrl +if (!$distributionUrl) { + Write-Error "cannot read distributionUrl property in $scriptDir/.mvn/wrapper/maven-wrapper.properties" +} + +switch -wildcard -casesensitive ( $($distributionUrl -replace '^.*/','') ) { + "maven-mvnd-*" { + $USE_MVND = $true + $distributionUrl = $distributionUrl -replace '-bin\.[^.]*$',"-windows-amd64.zip" + $MVN_CMD = "mvnd.cmd" + break + } + default { + $USE_MVND = $false + $MVN_CMD = $script -replace '^mvnw','mvn' + break + } +} + +# apply MVNW_REPOURL and calculate MAVEN_HOME +# maven home pattern: ~/.m2/wrapper/dists/{apache-maven-,maven-mvnd--}/ +if ($env:MVNW_REPOURL) { + $MVNW_REPO_PATTERN = if ($USE_MVND -eq $False) { "/org/apache/maven/" } else { "/maven/mvnd/" } + $distributionUrl = "$env:MVNW_REPOURL$MVNW_REPO_PATTERN$($distributionUrl -replace "^.*$MVNW_REPO_PATTERN",'')" +} +$distributionUrlName = $distributionUrl -replace '^.*/','' +$distributionUrlNameMain = $distributionUrlName -replace '\.[^.]*$','' -replace '-bin$','' + +$MAVEN_M2_PATH = "$HOME/.m2" +if ($env:MAVEN_USER_HOME) { + $MAVEN_M2_PATH = "$env:MAVEN_USER_HOME" +} + +if (-not (Test-Path -Path $MAVEN_M2_PATH)) { + New-Item -Path $MAVEN_M2_PATH -ItemType Directory | Out-Null +} + +$MAVEN_WRAPPER_DISTS = $null +if ((Get-Item $MAVEN_M2_PATH).Target[0] -eq $null) { + $MAVEN_WRAPPER_DISTS = "$MAVEN_M2_PATH/wrapper/dists" +} else { + $MAVEN_WRAPPER_DISTS = (Get-Item $MAVEN_M2_PATH).Target[0] + "/wrapper/dists" +} + +$MAVEN_HOME_PARENT = "$MAVEN_WRAPPER_DISTS/$distributionUrlNameMain" +$MAVEN_HOME_NAME = ([System.Security.Cryptography.SHA256]::Create().ComputeHash([byte[]][char[]]$distributionUrl) | ForEach-Object {$_.ToString("x2")}) -join '' +$MAVEN_HOME = "$MAVEN_HOME_PARENT/$MAVEN_HOME_NAME" + +if (Test-Path -Path "$MAVEN_HOME" -PathType Container) { + Write-Verbose "found existing MAVEN_HOME at $MAVEN_HOME" + Write-Output "MVN_CMD=$MAVEN_HOME/bin/$MVN_CMD" + exit $? +} + +if (! $distributionUrlNameMain -or ($distributionUrlName -eq $distributionUrlNameMain)) { + Write-Error "distributionUrl is not valid, must end with *-bin.zip, but found $distributionUrl" +} + +# prepare tmp dir +$TMP_DOWNLOAD_DIR_HOLDER = New-TemporaryFile +$TMP_DOWNLOAD_DIR = New-Item -Itemtype Directory -Path "$TMP_DOWNLOAD_DIR_HOLDER.dir" +$TMP_DOWNLOAD_DIR_HOLDER.Delete() | Out-Null +trap { + if ($TMP_DOWNLOAD_DIR.Exists) { + try { Remove-Item $TMP_DOWNLOAD_DIR -Recurse -Force | Out-Null } + catch { Write-Warning "Cannot remove $TMP_DOWNLOAD_DIR" } + } +} + +New-Item -Itemtype Directory -Path "$MAVEN_HOME_PARENT" -Force | Out-Null + +# Download and Install Apache Maven +Write-Verbose "Couldn't find MAVEN_HOME, downloading and installing it ..." +Write-Verbose "Downloading from: $distributionUrl" +Write-Verbose "Downloading to: $TMP_DOWNLOAD_DIR/$distributionUrlName" + +$webclient = New-Object System.Net.WebClient +if ($env:MVNW_USERNAME -and $env:MVNW_PASSWORD) { + $webclient.Credentials = New-Object System.Net.NetworkCredential($env:MVNW_USERNAME, $env:MVNW_PASSWORD) +} +[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 +$webclient.DownloadFile($distributionUrl, "$TMP_DOWNLOAD_DIR/$distributionUrlName") | Out-Null + +# If specified, validate the SHA-256 sum of the Maven distribution zip file +$distributionSha256Sum = (Get-Content -Raw "$scriptDir/.mvn/wrapper/maven-wrapper.properties" | ConvertFrom-StringData).distributionSha256Sum +if ($distributionSha256Sum) { + if ($USE_MVND) { + Write-Error "Checksum validation is not supported for maven-mvnd. `nPlease disable validation by removing 'distributionSha256Sum' from your maven-wrapper.properties." + } + Import-Module $PSHOME\Modules\Microsoft.PowerShell.Utility -Function Get-FileHash + if ((Get-FileHash "$TMP_DOWNLOAD_DIR/$distributionUrlName" -Algorithm SHA256).Hash.ToLower() -ne $distributionSha256Sum) { + Write-Error "Error: Failed to validate Maven distribution SHA-256, your Maven distribution might be compromised. If you updated your Maven version, you need to update the specified distributionSha256Sum property." + } +} + +# unzip and move +Expand-Archive "$TMP_DOWNLOAD_DIR/$distributionUrlName" -DestinationPath "$TMP_DOWNLOAD_DIR" | Out-Null + +# Find the actual extracted directory name (handles snapshots where filename != directory name) +$actualDistributionDir = "" + +# First try the expected directory name (for regular distributions) +$expectedPath = Join-Path "$TMP_DOWNLOAD_DIR" "$distributionUrlNameMain" +$expectedMvnPath = Join-Path "$expectedPath" "bin/$MVN_CMD" +if ((Test-Path -Path $expectedPath -PathType Container) -and (Test-Path -Path $expectedMvnPath -PathType Leaf)) { + $actualDistributionDir = $distributionUrlNameMain +} + +# If not found, search for any directory with the Maven executable (for snapshots) +if (!$actualDistributionDir) { + Get-ChildItem -Path "$TMP_DOWNLOAD_DIR" -Directory | ForEach-Object { + $testPath = Join-Path $_.FullName "bin/$MVN_CMD" + if (Test-Path -Path $testPath -PathType Leaf) { + $actualDistributionDir = $_.Name + } + } +} + +if (!$actualDistributionDir) { + Write-Error "Could not find Maven distribution directory in extracted archive" +} + +Write-Verbose "Found extracted Maven distribution directory: $actualDistributionDir" +Rename-Item -Path "$TMP_DOWNLOAD_DIR/$actualDistributionDir" -NewName $MAVEN_HOME_NAME | Out-Null +try { + Move-Item -Path "$TMP_DOWNLOAD_DIR/$MAVEN_HOME_NAME" -Destination $MAVEN_HOME_PARENT | Out-Null +} catch { + if (! (Test-Path -Path "$MAVEN_HOME" -PathType Container)) { + Write-Error "fail to move MAVEN_HOME" + } +} finally { + try { Remove-Item $TMP_DOWNLOAD_DIR -Recurse -Force | Out-Null } + catch { Write-Warning "Cannot remove $TMP_DOWNLOAD_DIR" } +} + +Write-Output "MVN_CMD=$MAVEN_HOME/bin/$MVN_CMD" diff --git a/claims-service/pom.xml b/claims-service/pom.xml new file mode 100644 index 0000000..dd130c2 --- /dev/null +++ b/claims-service/pom.xml @@ -0,0 +1,32 @@ + + + 4.0.0 + + + + org.springframework.boot + spring-boot-starter-parent + 3.5.16 + + + + com.example + claims-service-parent + 1.0.0 + pom + Claims Service (parent) + Spring Boot insurance-claims service on Snowflake, tested against the Snowflake-Next emulator + + + claims-api + + + + 21 + 4.4.0 + UTF-8 + + diff --git a/claims-service/scripts/demo.sh b/claims-service/scripts/demo.sh new file mode 100755 index 0000000..1b8d1e4 --- /dev/null +++ b/claims-service/scripts/demo.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# The demo moment: from a clean build, `./mvnw verify` runs the whole integration +# suite against a fresh Snowflake-Next emulator. No Snowflake account, no key pair to +# request, no MFA prompt: only Docker, JDK 21 and a LocalStack token. +set -euo pipefail +cd "$(dirname "${BASH_SOURCE[0]}")/.." + +if [[ -z "${LOCALSTACK_AUTH_TOKEN:-}" ]]; then + echo "LOCALSTACK_AUTH_TOKEN is not set (run through 'make demo', which reads ~/.localstack/default.env)." >&2 + exit 1 +fi + +echo "==> ./mvnw clean verify (Testcontainers starts ${SNOWFLAKE_IMAGE:-localstack/snowflake-next:latest})" +start=$SECONDS +status=0 +./mvnw -B -q clean verify || status=$? +elapsed=$((SECONDS - start)) + +echo +echo "==> Test results" +# One line per test case. +for report in claims-api/target/surefire-reports/TEST-*.xml; do + [[ -f "$report" ]] || continue + awk ' + function attr(line, name, m) { if (match(line, name "=\"[^\"]*\"")) { m = substr(line, RSTART + length(name) + 2, RLENGTH - length(name) - 3); return m } return "" } + /[[:space:]]*$/) { print_case() } next } + open && // { print_case() } + function print_case() { n = split(cls, parts, "."); printf " %-4s %6.2fs %s.%s\n", st, t, parts[n], name; open = 0 } + ' "$report" +done + +echo +scripts/test-summary.sh +if (( status == 0 )); then + echo "==> PASS in $((elapsed / 60))m $((elapsed % 60))s" +else + echo "==> FAILED after $((elapsed / 60))m $((elapsed % 60))s (see the Maven output above)" +fi +exit "$status" diff --git a/claims-service/scripts/test-summary.sh b/claims-service/scripts/test-summary.sh new file mode 100755 index 0000000..99e4b82 --- /dev/null +++ b/claims-service/scripts/test-summary.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# One Surefire-style totals line over every module's XML reports, for `make test` +# (./mvnw -q prints nothing on success). +set -euo pipefail +cd "$(dirname "${BASH_SOURCE[0]}")/.." +shopt -s nullglob +reports=(claims-api/target/surefire-reports/TEST-*.xml) +if (( ${#reports[@]} == 0 )); then + echo "No test reports found; run ./mvnw verify first." >&2 + exit 1 +fi +awk '//dev/null; do + if (( SECONDS >= deadline )); then + echo "Emulator at ${url} did not become ready within ${timeout}s." >&2 + echo "Check 'docker compose logs snowflake' (is LOCALSTACK_AUTH_TOKEN valid? is port ${port} free?)." >&2 + exit 1 + fi + sleep 2 +done +echo "Emulator ready at ${url}" diff --git a/fleet-telemetry/.editorconfig b/fleet-telemetry/.editorconfig new file mode 100644 index 0000000..7117117 --- /dev/null +++ b/fleet-telemetry/.editorconfig @@ -0,0 +1,29 @@ +root = true + +[*] +charset = utf-8 +end_of_line = lf +insert_final_newline = true +trim_trailing_whitespace = true +indent_style = space +indent_size = 2 + +[*.cs] +indent_size = 4 +csharp_style_namespace_declarations = file_scoped:warning +csharp_style_var_for_built_in_types = true:suggestion +csharp_style_var_when_type_is_apparent = true:suggestion +csharp_prefer_braces = when_multiline:suggestion +dotnet_sort_system_directives_first = true +dotnet_style_qualification_for_field = false:warning +dotnet_diagnostic.IDE0005.severity = warning +# Analyzer findings that do not fit a sample app. +dotnet_diagnostic.CA1515.severity = none +dotnet_diagnostic.CA1848.severity = none +dotnet_diagnostic.CA2007.severity = none + +[Makefile] +indent_style = tab + +[*.md] +trim_trailing_whitespace = false diff --git a/fleet-telemetry/.env.example b/fleet-telemetry/.env.example new file mode 100644 index 0000000..c103ac2 --- /dev/null +++ b/fleet-telemetry/.env.example @@ -0,0 +1,29 @@ +# Every variable the sample reads. The Makefile reads LOCALSTACK_AUTH_TOKEN from +# ~/.localstack/default.env when it is not exported. + +# LocalStack auth token (never commit a real value). +LOCALSTACK_AUTH_TOKEN= + +# Emulator image; defaults to the newest build. Pin a tag or digest to freeze it. +SNOWFLAKE_IMAGE=localstack/snowflake-next:latest + +# Host port of the compose emulator (make up, seed, run, demo). Testcontainers picks its own. +SNOWFLAKE_PORT=4566 + +# EMULATOR (default) or SNOWFLAKE_CLOUD (make test-cloud). +TEST_TARGET=EMULATOR + +# Key for key-pair (JWT) authentication; make up generates .secrets/rsa_key.p8. +SNOWFLAKE_PRIVATE_KEY_PATH=.secrets/rsa_key.p8 + +# A Snowflake account: the app reads these over appsettings.json (or Snowflake__Account and so on). +SNOWFLAKE_ACCOUNT= +SNOWFLAKE_HOST= +SNOWFLAKE_USER=FLEET_SVC +SNOWFLAKE_ROLE=FLEET_APP +SNOWFLAKE_WAREHOUSE=FLEET_WH +SNOWFLAKE_DATABASE=FLEET +SNOWFLAKE_SCHEMA=TELEMETRY + +# make demo: port of the API it starts. +DEMO_PORT=5080 diff --git a/fleet-telemetry/.gitignore b/fleet-telemetry/.gitignore new file mode 100644 index 0000000..8fcf503 --- /dev/null +++ b/fleet-telemetry/.gitignore @@ -0,0 +1,5 @@ +bin/ +obj/ +.secrets/ +.cache/ +TestResults/ diff --git a/fleet-telemetry/Directory.Build.props b/fleet-telemetry/Directory.Build.props new file mode 100644 index 0000000..5a9d664 --- /dev/null +++ b/fleet-telemetry/Directory.Build.props @@ -0,0 +1,15 @@ + + + net10.0 + enable + enable + true + latest-recommended + true + true + true + + true + $(NoWarn);CS1591 + + diff --git a/fleet-telemetry/Directory.Packages.props b/fleet-telemetry/Directory.Packages.props new file mode 100644 index 0000000..83c1fca --- /dev/null +++ b/fleet-telemetry/Directory.Packages.props @@ -0,0 +1,16 @@ + + + true + + + + + + + + + + + + + diff --git a/fleet-telemetry/FleetTelemetry.slnx b/fleet-telemetry/FleetTelemetry.slnx new file mode 100644 index 0000000..f52f3ef --- /dev/null +++ b/fleet-telemetry/FleetTelemetry.slnx @@ -0,0 +1,12 @@ + + + + + + + + + + + + diff --git a/fleet-telemetry/Makefile b/fleet-telemetry/Makefile new file mode 100644 index 0000000..8586dfb --- /dev/null +++ b/fleet-telemetry/Makefile @@ -0,0 +1,80 @@ +# Fleet Telemetry: an ASP.NET Core service on Snowflake.Data, developed against the +# Snowflake-Next emulator. +# +# make up dev key, pull the emulator image, start it, wait until it answers SQL +# make seed generate 3 days of synthetic telemetry and bulk-load it (idempotent) +# make run the API on http://localhost:5080 against the emulator +# make demo walkthrough: backfill, live POST, trips, events, faults, daily KPIs +# make test unit tests, then integration tests on their own Testcontainers emulator +# make test-cloud the same suite against Snowflake (TEST_TARGET=SNOWFLAKE_CLOUD) +# make check build with warnings as errors and verify formatting +# make down stop and remove the emulator + +-include $(HOME)/.localstack/default.env +export LOCALSTACK_AUTH_TOKEN +SNOWFLAKE_PORT ?= 4566 +SNOWFLAKE_IMAGE ?= localstack/snowflake-next:latest +export SNOWFLAKE_PORT SNOWFLAKE_IMAGE + +COMPOSE := docker compose -p snfs-fleet-telemetry +DEV_KEY := .secrets/rsa_key.p8 +DATA_DIR := .cache/telemetry +# make run, seed and demo use appsettings.Development.json (the local emulator) and the dev key. +DEV_ENV := DOTNET_ENVIRONMENT=Development ASPNETCORE_ENVIRONMENT=Development \ + SNOWFLAKE_PRIVATE_KEY_PATH=$(CURDIR)/$(DEV_KEY) +INGEST := $(DEV_ENV) dotnet run --project src/FleetTelemetry.Ingest -- + +.PHONY: up seed run demo test test-unit test-integration test-cloud check down logs install start stop + +up: $(DEV_KEY) + $(COMPOSE) pull + $(COMPOSE) up -d + scripts/wait-for-emulator.sh 180 + +seed: + $(INGEST) generate $(DATA_DIR) + $(INGEST) backfill $(DATA_DIR) + +run: + $(DEV_ENV) dotnet run --project src/FleetTelemetry.Api --urls http://localhost:5080 + +demo: + scripts/demo.sh + +test: test-unit test-integration + +test-unit: + dotnet test --project tests/FleetTelemetry.UnitTests + +test-integration: + dotnet test --project tests/FleetTelemetry.IntegrationTests + +# Reads SNOWFLAKE_ACCOUNT, SNOWFLAKE_USER, SNOWFLAKE_PRIVATE_KEY_PATH, SNOWFLAKE_ROLE and +# SNOWFLAKE_WAREHOUSE; each test class creates and drops a FLEET_IT_* database. +test-cloud: + TEST_TARGET=SNOWFLAKE_CLOUD dotnet test --project tests/FleetTelemetry.IntegrationTests + +check: + dotnet build -warnaserror + dotnet format --verify-no-changes + +down: + $(COMPOSE) down -v --remove-orphans + +logs: + $(COMPOSE) logs --tail=200 snowflake + +# Throwaway unencrypted PKCS#8 key pair for key-pair (JWT) authentication. +$(DEV_KEY): + mkdir -p $(dir $@) + openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 -out $@ + openssl rsa -in $@ -pubout -out $(dir $@)rsa_key.pub + chmod 600 $@ + +# The targets the repository CI runs for every sample (see the root README) +install: + @echo "Nothing to install, make up and make test fetch the dependencies" + +start: up + +stop: down diff --git a/fleet-telemetry/README.md b/fleet-telemetry/README.md new file mode 100644 index 0000000..baab7ac --- /dev/null +++ b/fleet-telemetry/README.md @@ -0,0 +1,143 @@ +# Fleet Telemetry + +A fleet telemetry service for a logistics company, built with ASP.NET Core minimal APIs on the official `Snowflake.Data` driver: vehicles post JSON pings with GPS, speed, fuel and engine diagnostics, and operations dashboards query trips, harsh braking, idling, fault codes and daily fleet KPIs. It shows how a .NET team develops and tests against Snowflake-Next on a laptop, with `dotnet run`, xUnit and Testcontainers. + +## Architecture + +```mermaid +flowchart LR + vehicles["Vehicles
JSON pings every 5 s"] -- "POST /api/telemetry" --> api["FleetTelemetry.Api
ASP.NET Core, Dapper"] + exports["Gateway exports
NDJSON per vehicle-day"] --> ingest["FleetTelemetry.Ingest
backfill command"] + dash["Dashboards"] -- "GET trips, events,
faults, daily KPIs" --> api + api -- "Snowflake.Data, key-pair JWT
QUERY_TAG per request" --> sf + ingest -- "PUT + COPY INTO" --> sf + subgraph sf["Snowflake: FLEET.TELEMETRY"] + raw[("TELEMETRY_RAW
PAYLOAD VARIANT")] + view["TELEMETRY view
typed, deduplicated"] + faults["TELEMETRY_FAULTS view
LATERAL FLATTEN"] + dt[("FLEET_DAILY_KPIS
dynamic table")] + reg[("VEHICLES, DEPOTS")] + raw --> view --> faults + view --> dt + end + tests["xUnit + Testcontainers
+ WebApplicationFactory"] -. "own emulator,
database per class" .-> sf +``` + +Both ingestion paths land the device JSON unchanged in one `VARIANT` column. A typed view turns it into columns, the API's analytics are SQL over that view, and a dynamic table keeps the daily KPIs current without a scheduler. + +## How it works + +1. **Provisioning.** `init/01_platform.sf.sql` creates the warehouse `FLEET_WH`, the role `FLEET_APP`, the database `FLEET` and the service user `FLEET_SVC`. Files 02 to 05 create the landing table, the NDJSON file format and stage, the fleet registry, the typed views and the dynamic table. The emulator runs them all as init hooks. +2. **Live ingestion.** `POST /api/telemetry` takes a JSON array of up to 1,000 pings, validates every field (400 problem details otherwise) and inserts the whole batch with one bound parameter: `INSERT ... SELECT value FROM TABLE(FLATTEN(PARSE_JSON(?)))`. +3. **Backfill.** `dotnet run --project src/FleetTelemetry.Ingest -- backfill ` uploads `*.ndjson` files with `PUT`, loads them with `COPY INTO ... ON_ERROR = CONTINUE`, prints rows loaded and the first bad line per file, and refreshes the KPIs. Snowflake's load history makes a re-run load nothing twice. +4. **Analytics.** Trips: `LAG` finds gaps of more than 10 minutes between pings and a running `SUM` numbers the trips. Harsh braking: `LAG` over speed, at least 2.5 m/s². Idling: runs of stationary pings of 5 minutes or more. Faults: the `TELEMETRY_FAULTS` view unnests each ping's `engine.dtc` array. +5. **Daily KPIs.** `FLEET_DAILY_KPIS` aggregates distance, fuel, idling and faults per vehicle and day with `TARGET_LAG = '15 minutes'`; `GET /api/fleet/daily` rolls it up to fleet totals. + +```mermaid +sequenceDiagram + autonumber + participant D as Dashboard + participant A as FleetTelemetry.Api + participant R as FleetRegistry + participant Q as VehicleAnalytics + participant S as Snowflake + D->>A: GET /api/vehicles/TRK-203/trips?date=2026-09-08 + A->>R: VehicleExistsAsync("TRK-203") + R->>S: ALTER SESSION SET QUERY_TAG, then SELECT COUNT(*) FROM VEHICLES + alt unknown vehicle + A-->>D: 404 problem details + else known vehicle + A->>Q: GetTripsAsync("TRK-203", 2026-09-08) + Q->>S: LAG(TS) for gaps, SUM(new trip) OVER (ORDER BY TS), GROUP BY trip + S-->>Q: one row per trip + Q-->>A: start, end, distance, fuel, idle seconds + A-->>D: 200 trips with fuel economy and average speed + end +``` + +Design notes (data model, the semi-structured design, trip segmentation, the dynamic table, configuration) are in [docs/architecture.md](docs/architecture.md). + +## Prerequisites + +- Docker with Compose v2 +- The .NET 10 SDK (`global.json` pins 10.0.402 and rolls forward to newer feature bands) +- `make`, `openssl` and `curl` +- A LocalStack auth token, exported as `LOCALSTACK_AUTH_TOKEN` or stored in `~/.localstack/default.env` + +## Quickstart + +```sh +make up # dev key pair in .secrets/, start the emulator, wait until it answers SQL +make seed # 24 vehicles over 3 days (67,812 pings) as 72 NDJSON files, then the backfill +make demo # backfill, a live POST, trips, events, faults, KPIs and two error responses +make run # the API on http://localhost:5080, for your own requests +make down # stop and remove the emulator +``` + +If port 4566 is taken, set `SNOWFLAKE_PORT=14566` (or any free port) for every target. + +With `make run` going: + +```sh +curl -s -X POST localhost:5080/api/telemetry -H 'Content-Type: application/json' \ + --data-binary @.cache/telemetry/live/TRK-101-2026-09-10.json +# 202 {"accepted":329} +curl -s 'localhost:5080/api/vehicles/TRK-203/events?type=harsh_braking' +# {"vehicleId":"TRK-203","type":"harsh_braking","events":[{"occurredAt":"2026-09-08T09:02:20Z", +# "fromSpeedKmh":52,"toSpeedKmh":0.3,"decelerationMs2":2.87,...},...]} +curl -s 'localhost:5080/api/fleet/daily?from=2026-09-07&to=2026-09-09' +# {"days":[{"day":"2026-09-07","activeVehicles":24,"distanceKm":1696.8,"fuelEconomyLPer100Km":25.3,...},...]} +``` + +Other endpoints: `GET /api/vehicles/{id}/trips?date=`, `GET /api/vehicles/{id}/events?type=idling`, `GET /api/vehicles/{id}/faults`, `GET /health`. Event and fault queries take optional `from` and `to` dates. + +The API reads `appsettings.json`, then `appsettings.Development.json` (the emulator: account, host, port and key), then environment variables (`Snowflake__Port`, or the conventional `SNOWFLAKE_PORT`, `SNOWFLAKE_ACCOUNT` and so on). + +## Tests + +```sh +make test # unit tests, then integration tests (about 1.5 minutes) +``` + +`make test` does not need `make up`: the integration tests start their own emulator with Testcontainers on a random port, run the same init hooks, and give each test class its own database (`FLEET_IT__`, dropped afterwards). The API runs in-process through `WebApplicationFactory`, so every HTTP test goes through routing, validation, Dapper and the driver. + +- **Unit tests** (46): trip segmentation and harsh-braking thresholds at their boundaries, KPI math (fleet fuel economy is total fuel over total distance), batch validation, and the simulator's ground truth. +- **Backfill** (4): every file loads one row per ping, a truncated line is skipped and reported with its line number, a re-run loads nothing. +- **Vehicle analytics** (14): trips, harsh braking, idling and fault codes for 4 vehicles over 2 days match what the simulator drove; every query carries the route and the caller's `traceparent` trace id in its `QUERY_TAG`; unknown vehicles are 404 and bad parameters 400 problem details. +- **Fleet KPIs** (3): daily KPIs equal a LINQ computation over the pings, and live pings appear after `ALTER DYNAMIC TABLE ... REFRESH`. +- **Ingestion** (5): a posted batch becomes a trip, a retried batch counts once, a pause of exactly 10 minutes keeps the trip and 601 seconds splits it, malformed and invalid batches are 400 and write nothing. + +Expected values come from the simulator's ground truth or from LINQ over the generated pings, never from the code under test. + +**Against Snowflake** (consumes credits): run `init/01_platform.sf.sql` once as `ACCOUNTADMIN`, register the public key with `ALTER USER FLEET_SVC SET RSA_PUBLIC_KEY = '...'`, then: + +```sh +export SNOWFLAKE_ACCOUNT= SNOWFLAKE_PRIVATE_KEY_PATH= +make test-cloud # TEST_TARGET=SNOWFLAKE_CLOUD; user, role and warehouse default to FLEET_SVC, FLEET_APP, FLEET_WH +``` + +## Project structure + +``` +fleet-telemetry/ + FleetTelemetry.slnx, global.json, Directory.Build.props, Directory.Packages.props + Makefile, docker-compose.yml, .env.example + init/ 01 platform, 02 landing, 03 registry, 04 typed views, 05 dynamic table + src/FleetTelemetry.Core/ telemetry model and validation, trip and event rules, KPI math, simulator + src/FleetTelemetry.Data/ Snowflake options, sessions with query tags, Dapper queries, backfill + src/FleetTelemetry.Api/ minimal API endpoints, problem details, health check, appsettings + src/FleetTelemetry.Ingest/ generate and backfill commands + tests/FleetTelemetry.UnitTests/ xUnit, no Snowflake + tests/FleetTelemetry.IntegrationTests/ Testcontainers emulator, per-class databases, HTTP tests + scripts/ wait-for-emulator.sh, demo.sh + docs/architecture.md design notes +``` + +## Learn more + +- [Snowflake .NET driver](https://docs.snowflake.com/en/developer-guide/dotnet/dotnet-driver) and [key-pair authentication](https://docs.snowflake.com/en/user-guide/key-pair-auth) +- [Semi-structured data](https://docs.snowflake.com/en/user-guide/semistructured-intro), [`FLATTEN`](https://docs.snowflake.com/en/sql-reference/functions/flatten) and [`PARSE_JSON`](https://docs.snowflake.com/en/sql-reference/functions/parse_json) +- [`PUT`](https://docs.snowflake.com/en/sql-reference/sql/put) and [`COPY INTO `](https://docs.snowflake.com/en/sql-reference/sql/copy-into-table) +- [Window functions](https://docs.snowflake.com/en/sql-reference/functions-analytic) and [`LAG`](https://docs.snowflake.com/en/sql-reference/functions/lag) +- [Dynamic tables](https://docs.snowflake.com/en/user-guide/dynamic-tables-about) +- [Testcontainers for .NET](https://dotnet.testcontainers.org/) and [integration tests in ASP.NET Core](https://learn.microsoft.com/aspnet/core/test/integration-tests) diff --git a/fleet-telemetry/docker-compose.yml b/fleet-telemetry/docker-compose.yml new file mode 100644 index 0000000..d2e5f97 --- /dev/null +++ b/fleet-telemetry/docker-compose.yml @@ -0,0 +1,19 @@ +# The Snowflake-Next emulator for local development: `make up` starts it, the API +# (`make run`), the ingest tool (`make seed`) and SnowSQL or DBeaver connect to it. +# The integration tests start their own emulator with Testcontainers. +services: + snowflake: + image: ${SNOWFLAKE_IMAGE:-localstack/snowflake-next:latest} + pull_policy: always + ports: + - "127.0.0.1:${SNOWFLAKE_PORT:-4566}:4566" + environment: + LOCALSTACK_AUTH_TOKEN: ${LOCALSTACK_AUTH_TOKEN:?set LOCALSTACK_AUTH_TOKEN} + volumes: + - ./init:/etc/localstack/init/ready.d:ro + networks: + default: + aliases: + - snowflake.localhost.localstack.cloud + - stages.s3.localhost.localstack.cloud + - stages-enc.s3.localhost.localstack.cloud diff --git a/fleet-telemetry/docs/architecture.md b/fleet-telemetry/docs/architecture.md new file mode 100644 index 0000000..aaab1bb --- /dev/null +++ b/fleet-telemetry/docs/architecture.md @@ -0,0 +1,106 @@ +# Architecture + +Design notes for the fleet telemetry service: the data model, why telemetry lands as semi-structured JSON, how trips and driving events are derived, and how the daily KPIs stay current. + +## Data model + +```mermaid +erDiagram + DEPOTS ||--o{ VEHICLES : "home depot" + VEHICLES ||--o{ TELEMETRY_RAW : "vehicle_id in PAYLOAD" + TELEMETRY_RAW ||--|| TELEMETRY : "typed view" + TELEMETRY ||--o{ TELEMETRY_FAULTS : "FLATTEN engine.dtc" + TELEMETRY ||--o{ FLEET_DAILY_KPIS : "per vehicle and day" + DEPOTS { + varchar DEPOT_ID PK + varchar NAME + float LAT + float LON + } + VEHICLES { + varchar VEHICLE_ID PK + varchar REGISTRATION + varchar VEHICLE_CLASS + number FUEL_TANK_L + varchar DEPOT_ID FK + } + TELEMETRY_RAW { + variant PAYLOAD + } + FLEET_DAILY_KPIS { + date DAY + varchar VEHICLE_ID + number SAMPLES + number IDLE_SAMPLES + float DISTANCE_KM + float FUEL_USED_L + number FAULT_SAMPLES + } +``` + +| Object | Kind | Purpose | +| --- | --- | --- | +| `TELEMETRY_RAW` | table | One row per ping, the device JSON in `PAYLOAD VARIANT` | +| `TELEMETRY_NDJSON`, `TELEMETRY_STAGE` | file format, internal stage | Backfill files, one JSON document per line | +| `DEPOTS`, `VEHICLES` | tables | Fleet registry (reference data, synced from fleet management in production) | +| `TELEMETRY` | view | Typed columns, UTC timestamps, one row per `message_id` | +| `TELEMETRY_FAULTS` | view | One row per active fault code per ping | +| `FLEET_DAILY_KPIS` | dynamic table | Distance, fuel, idling and faults per vehicle and UTC day | + +All timestamps are UTC. Ping times are normalized with `CONVERT_TIMEZONE('UTC', ts::TIMESTAMP_TZ)` because device clocks may report any offset; trips and KPIs use UTC days. + +## Semi-structured design + +A ping looks like this (one line of an export file, or one element of a `POST /api/telemetry` batch): + +```json +{"message_id":"TRK-101-20260907054800","vehicle_id":"TRK-101","ts":"2026-09-07T05:48:00Z", + "gps":{"lat":51.889,"lon":4.431,"heading_deg":0},"speed_kmh":0,"odometer_km":309199.518, + "fuel":{"level_pct":98.3,"used_l":75416.832}, + "engine":{"rpm":640,"coolant_c":32.8,"dtc":[{"code":"P0301","severity":"warning"}]}} +``` + +- **Land first, type later.** Firmware adds fields over time. The landing table stores the payload exactly as received, so a new field never breaks ingestion and history can be re-read when the view gains a column. +- **One shape for both paths.** The API validates a batch and inserts it with a single bound parameter, `PARSE_JSON(?)` flattened server-side, so a batch lands completely or not at all. The backfill loads the same JSON with `COPY INTO` into the same table. +- **Retries are expected.** Devices resend a batch when they miss the response. `message_id` identifies a ping; the `TELEMETRY` view keeps one row per `message_id` with `QUALIFY ROW_NUMBER() OVER (PARTITION BY VEHICLE_ID, MESSAGE_ID ...) = 1`. Including `VEHICLE_ID` in the partition lets a query's vehicle filter apply before the deduplication. +- **Nested arrays stay nested.** Active diagnostic trouble codes are an array inside `engine`. `TELEMETRY_FAULTS` unnests it with `LATERAL FLATTEN`, and `GET /api/vehicles/{id}/faults` aggregates occurrences, first and last seen per code. +- **Bad lines do not stop a load.** `COPY INTO ... ON_ERROR = CONTINUE` skips a corrupt line and reports it (`errors_seen`, `first_error`, `first_error_line`) per file; the backfill prints that report. Gateways export one file per vehicle per day, so a problem file is easy to trace back to its source. + +## Trip segmentation + +Devices report every 5 seconds while the ignition is on and stay silent when it is off, so trips are separated by gaps. The rule lives in `TripRules` and is applied in SQL: + +```sql +WITH pings AS ( + SELECT TS, DATEDIFF('second', LAG(TS) OVER (ORDER BY TS), TS) AS GAP_S, ... + FROM TELEMETRY WHERE VEHICLE_ID = :vehicleId AND TS >= :dayStart AND TS < :dayEnd +), +numbered AS ( + SELECT *, SUM(IFF(GAP_S IS NULL OR GAP_S > :maxGapS, 1, 0)) + OVER (ORDER BY TS ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS TRIP_NO + FROM pings +) +SELECT TRIP_NO, MIN(TS), MAX(TS), MAX(ODOMETER_KM) - MIN(ODOMETER_KM), ... FROM numbered GROUP BY TRIP_NO +``` + +- A gap of more than 10 minutes starts a new trip; exactly 10 minutes does not (a long traffic jam or a level crossing). Unit tests pin the boundary, and an integration test posts pings 600 and 601 seconds apart. +- Distance and fuel are odometer and fuel-counter differences, so lost pings do not distort them. +- Idle seconds in a trip are the time between consecutive stationary pings. +- **Harsh braking** compares each ping with the previous one (`LAG(SPEED_KMH)`): an average deceleration of at least 2.5 m/s² (about 0.25 g) is an event. +- **Idling** is a gaps-and-islands query: consecutive stationary pings form an island, and islands of 5 minutes or more are reported. + +The API adds the derived figures in C# (`KpiMath`): fuel economy in L/100 km (none below 1 km), average speed including stops, idle share. + +## Daily KPIs + +`FLEET_DAILY_KPIS` is a dynamic table over the `TELEMETRY` view, one row per vehicle and day, with `TARGET_LAG = '15 minutes'`. Snowflake keeps it current; nothing in the application schedules refreshes. Two places refresh it on demand with `ALTER DYNAMIC TABLE FLEET_DAILY_KPIS REFRESH`: the backfill, so a dashboard shows a backfill as soon as it finishes, and the tests, so they need not wait for the lag. + +`GET /api/fleet/daily` reads the vehicle-days for a range of at most 92 days and rolls them up in `FleetDailyKpis`. Fleet fuel economy is total fuel over total distance, not an average of vehicle ratios, so a van that moved 2 km does not weigh as much as a truck that drove 400. + +## Configuration and sessions + +- `SnowflakeOptions` binds the `Snowflake` section. `appsettings.json` holds the production shape; `appsettings.Development.json` changes only account, host, port and key path for the emulator. `SNOWFLAKE_*` environment variables override both. +- Authentication is key-pair only (`authenticator=snowflake_jwt`). `make up` writes a throwaway key to `.secrets/`; on an account, `ALTER USER FLEET_SVC SET RSA_PUBLIC_KEY = ...` registers its public half. +- `FleetDatabase` opens pooled sessions and sets a JSON `QUERY_TAG` with the application, the HTTP route and the request id, so a query in Snowflake's history leads back to its request. +- Every query uses bound parameters (`:name`, through Dapper). Every endpoint takes the request's `CancellationToken` down to the driver. +- Errors are RFC 9457 problem details: 404 for an unknown vehicle, 400 for malformed JSON, invalid pings (field errors keyed like `[3].speed_kmh`) and bad query parameters. diff --git a/fleet-telemetry/global.json b/fleet-telemetry/global.json new file mode 100644 index 0000000..9e9a685 --- /dev/null +++ b/fleet-telemetry/global.json @@ -0,0 +1,9 @@ +{ + "sdk": { + "version": "10.0.402", + "rollForward": "latestFeature" + }, + "test": { + "runner": "Microsoft.Testing.Platform" + } +} diff --git a/fleet-telemetry/init/01_platform.sf.sql b/fleet-telemetry/init/01_platform.sf.sql new file mode 100644 index 0000000..2c1d2ca --- /dev/null +++ b/fleet-telemetry/init/01_platform.sf.sql @@ -0,0 +1,31 @@ +-- Account-level provisioning: what a Snowflake administrator runs once per +-- environment (often from Terraform or a platform repository). +-- +-- The emulator runs every init/*.sf.sql file at startup (docker compose mounts +-- this directory as its init hooks). The integration tests run this file on +-- their own emulator, then the database-scoped files 02 to 05 once per test +-- class in a fresh database. Keep one statement per line ending in ';'. + +USE ROLE ACCOUNTADMIN; + +CREATE WAREHOUSE IF NOT EXISTS FLEET_WH + WAREHOUSE_SIZE = XSMALL AUTO_SUSPEND = 60 AUTO_RESUME = TRUE INITIALLY_SUSPENDED = TRUE; + +-- The service role owns the telemetry schema. Granting it to SYSADMIN keeps it +-- inside the standard role hierarchy. +CREATE ROLE IF NOT EXISTS FLEET_APP; +GRANT ROLE FLEET_APP TO ROLE SYSADMIN; +GRANT USAGE ON WAREHOUSE FLEET_WH TO ROLE FLEET_APP; +-- Development and CI only: the integration tests create one database per test class. +GRANT CREATE DATABASE ON ACCOUNT TO ROLE FLEET_APP; + +CREATE DATABASE IF NOT EXISTS FLEET; +CREATE SCHEMA IF NOT EXISTS FLEET.TELEMETRY; +GRANT OWNERSHIP ON DATABASE FLEET TO ROLE FLEET_APP COPY CURRENT GRANTS; +GRANT OWNERSHIP ON SCHEMA FLEET.TELEMETRY TO ROLE FLEET_APP COPY CURRENT GRANTS; + +-- Service user for the API and the ingest tool, key-pair authentication only. +-- On a Snowflake account, register the public key that `make up` writes: +-- ALTER USER FLEET_SVC SET RSA_PUBLIC_KEY = '<.secrets/rsa_key.pub without the PEM header lines>'; +CREATE USER IF NOT EXISTS FLEET_SVC TYPE = SERVICE DEFAULT_ROLE = FLEET_APP DEFAULT_WAREHOUSE = FLEET_WH; +GRANT ROLE FLEET_APP TO USER FLEET_SVC; diff --git a/fleet-telemetry/init/02_landing.sf.sql b/fleet-telemetry/init/02_landing.sf.sql new file mode 100644 index 0000000..ed628d6 --- /dev/null +++ b/fleet-telemetry/init/02_landing.sf.sql @@ -0,0 +1,21 @@ +-- Landing zone for vehicle telemetry. Both ingestion paths write here: +-- * POST /api/telemetry inserts each ping with PARSE_JSON(?); +-- * the backfill command PUTs NDJSON files to TELEMETRY_STAGE and runs COPY INTO. +-- The payload is kept exactly as the device sent it; typed views pick the fields. + +USE ROLE FLEET_APP; +USE WAREHOUSE FLEET_WH; +USE SCHEMA FLEET.TELEMETRY; + +CREATE TABLE IF NOT EXISTS TELEMETRY_RAW ( + PAYLOAD VARIANT +) COMMENT = 'One row per telemetry ping, as received from the vehicle'; + +-- One JSON document per line, gzip-compressed by PUT. +CREATE FILE FORMAT IF NOT EXISTS TELEMETRY_NDJSON + TYPE = JSON COMPRESSION = AUTO + COMMENT = 'Newline-delimited JSON telemetry exports'; + +CREATE STAGE IF NOT EXISTS TELEMETRY_STAGE + FILE_FORMAT = TELEMETRY_NDJSON + COMMENT = 'Backfill files uploaded by FleetTelemetry.Ingest'; diff --git a/fleet-telemetry/init/03_fleet_registry.sf.sql b/fleet-telemetry/init/03_fleet_registry.sf.sql new file mode 100644 index 0000000..8c4fddd --- /dev/null +++ b/fleet-telemetry/init/03_fleet_registry.sf.sql @@ -0,0 +1,60 @@ +-- Fleet registry: depots and vehicles. In production this is synced from the +-- fleet management system; here it is reference data. The telemetry generator +-- reads it to decide which vehicles to simulate and where they start. + +USE ROLE FLEET_APP; +USE WAREHOUSE FLEET_WH; +USE SCHEMA FLEET.TELEMETRY; + +CREATE TABLE IF NOT EXISTS DEPOTS ( + DEPOT_ID VARCHAR(8) NOT NULL, + NAME VARCHAR(64) NOT NULL, + LAT FLOAT NOT NULL, + LON FLOAT NOT NULL, + CONSTRAINT PK_DEPOTS PRIMARY KEY (DEPOT_ID) +); + +CREATE TABLE IF NOT EXISTS VEHICLES ( + VEHICLE_ID VARCHAR(16) NOT NULL, + REGISTRATION VARCHAR(16) NOT NULL, + MAKE VARCHAR(32) NOT NULL, + MODEL VARCHAR(32) NOT NULL, + VEHICLE_CLASS VARCHAR(16) NOT NULL, + FUEL_TANK_L NUMBER(5, 0) NOT NULL, + DEPOT_ID VARCHAR(8) NOT NULL, + CONSTRAINT PK_VEHICLES PRIMARY KEY (VEHICLE_ID), + CONSTRAINT FK_VEHICLES_DEPOT FOREIGN KEY (DEPOT_ID) REFERENCES DEPOTS (DEPOT_ID) +); + +INSERT OVERWRITE INTO DEPOTS (DEPOT_ID, NAME, LAT, LON) +SELECT column1, column2, column3, column4 FROM VALUES + ('RTM', 'Rotterdam Waalhaven', 51.8890, 4.4310), + ('ANR', 'Antwerp Noorderlaan', 51.2600, 4.3600), + ('DUI', 'Duisburg Logport', 51.3870, 6.6900); + +INSERT OVERWRITE INTO VEHICLES (VEHICLE_ID, REGISTRATION, MAKE, MODEL, VEHICLE_CLASS, FUEL_TANK_L, DEPOT_ID) +SELECT column1, column2, column3, column4, column5, column6, column7 FROM VALUES + ('TRK-101', '12-BXR-4', 'DAF', 'XF 480', 'tractor', 845, 'RTM'), + ('TRK-102', '31-BXT-7', 'Scania', 'R 450', 'tractor', 700, 'RTM'), + ('TRK-103', '58-BZD-2', 'Volvo', 'FH 460', 'tractor', 720, 'RTM'), + ('TRK-104', '76-BZF-9', 'DAF', 'CF 410', 'rigid', 430, 'RTM'), + ('TRK-105', '83-BZH-1', 'Volvo', 'FL 280', 'rigid', 300, 'RTM'), + ('VAN-106', 'V-402-KP', 'Mercedes-Benz', 'Sprinter 317', 'van', 93, 'RTM'), + ('VAN-107', 'V-417-KR', 'Ford', 'Transit 350', 'van', 80, 'RTM'), + ('VAN-108', 'V-433-KS', 'Renault', 'Master 165', 'van', 80, 'RTM'), + ('TRK-201', '1-XKD-118', 'DAF', 'XF 480', 'tractor', 845, 'ANR'), + ('TRK-202', '1-XKF-207', 'MAN', 'TGX 18.470', 'tractor', 760, 'ANR'), + ('TRK-203', '1-XLA-334', 'Scania', 'R 450', 'tractor', 700, 'ANR'), + ('TRK-204', '1-XLC-409', 'Renault', 'D Wide', 'rigid', 400, 'ANR'), + ('TRK-205', '1-XLE-552', 'MAN', 'TGL 12.250', 'rigid', 220, 'ANR'), + ('VAN-206', '1-VNA-611', 'Mercedes-Benz', 'Sprinter 317', 'van', 93, 'ANR'), + ('VAN-207', '1-VNC-728', 'Iveco', 'Daily 35S16', 'van', 90, 'ANR'), + ('VAN-208', '1-VNE-845', 'Ford', 'Transit 350', 'van', 80, 'ANR'), + ('TRK-301', 'DU-FL 3011', 'Mercedes-Benz', 'Actros 1848', 'tractor', 780, 'DUI'), + ('TRK-302', 'DU-FL 3022', 'MAN', 'TGX 18.470', 'tractor', 760, 'DUI'), + ('TRK-303', 'DU-FL 3033', 'Volvo', 'FH 460', 'tractor', 720, 'DUI'), + ('TRK-304', 'DU-FL 3044', 'Mercedes-Benz', 'Atego 1530', 'rigid', 290, 'DUI'), + ('TRK-305', 'DU-FL 3055', 'DAF', 'CF 410', 'rigid', 430, 'DUI'), + ('VAN-306', 'DU-FL 3066', 'Volkswagen', 'Crafter 35', 'van', 75, 'DUI'), + ('VAN-307', 'DU-FL 3077', 'Mercedes-Benz', 'Sprinter 317', 'van', 93, 'DUI'), + ('VAN-308', 'DU-FL 3088', 'Iveco', 'Daily 35S16', 'van', 90, 'DUI'); diff --git a/fleet-telemetry/init/04_typed_views.sf.sql b/fleet-telemetry/init/04_typed_views.sf.sql new file mode 100644 index 0000000..fef456f --- /dev/null +++ b/fleet-telemetry/init/04_typed_views.sf.sql @@ -0,0 +1,39 @@ +-- Typed views over the landing table. Every query in the API reads these, so a +-- new device field only needs a column here, never a change to the landing table. + +USE ROLE FLEET_APP; +USE WAREHOUSE FLEET_WH; +USE SCHEMA FLEET.TELEMETRY; + +-- One row per ping. Devices retry uploads, so a message can arrive more than +-- once; the view keeps one copy per MESSAGE_ID. Partitioning by VEHICLE_ID as well +-- lets a query's VEHICLE_ID filter apply before the deduplication. Device clocks may +-- report any UTC offset; TS is normalized to UTC. +CREATE OR REPLACE VIEW TELEMETRY AS +SELECT + PAYLOAD:message_id::VARCHAR AS MESSAGE_ID, + PAYLOAD:vehicle_id::VARCHAR AS VEHICLE_ID, + CONVERT_TIMEZONE('UTC', PAYLOAD:ts::TIMESTAMP_TZ)::TIMESTAMP_NTZ AS TS, + PAYLOAD:gps.lat::FLOAT AS LAT, + PAYLOAD:gps.lon::FLOAT AS LON, + PAYLOAD:gps.heading_deg::INTEGER AS HEADING_DEG, + PAYLOAD:speed_kmh::FLOAT AS SPEED_KMH, + PAYLOAD:odometer_km::FLOAT AS ODOMETER_KM, + PAYLOAD:fuel.level_pct::FLOAT AS FUEL_LEVEL_PCT, + PAYLOAD:fuel.used_l::FLOAT AS FUEL_USED_L, + PAYLOAD:engine.rpm::INTEGER AS ENGINE_RPM, + PAYLOAD:engine.coolant_c::FLOAT AS COOLANT_C, + COALESCE(PAYLOAD:engine.dtc::ARRAY, ARRAY_CONSTRUCT()) AS FAULT_CODES +FROM TELEMETRY_RAW +QUALIFY ROW_NUMBER() OVER (PARTITION BY VEHICLE_ID, MESSAGE_ID ORDER BY TS) = 1; + +-- One row per active diagnostic trouble code (DTC) per ping. +CREATE OR REPLACE VIEW TELEMETRY_FAULTS AS +SELECT + t.VEHICLE_ID, + t.TS, + f.value:code::VARCHAR AS CODE, + f.value:severity::VARCHAR AS SEVERITY, + t.COOLANT_C +FROM TELEMETRY t, + LATERAL FLATTEN(INPUT => t.FAULT_CODES) f; diff --git a/fleet-telemetry/init/05_fleet_kpis.sf.sql b/fleet-telemetry/init/05_fleet_kpis.sf.sql new file mode 100644 index 0000000..657992e --- /dev/null +++ b/fleet-telemetry/init/05_fleet_kpis.sf.sql @@ -0,0 +1,24 @@ +-- Daily KPIs per vehicle, kept up to date by Snowflake from the typed view. +-- Dashboards read this table instead of scanning raw telemetry. The backfill +-- command and the tests refresh it on demand with ALTER DYNAMIC TABLE ... REFRESH. + +USE ROLE FLEET_APP; +USE WAREHOUSE FLEET_WH; +USE SCHEMA FLEET.TELEMETRY; + +CREATE OR REPLACE DYNAMIC TABLE FLEET_DAILY_KPIS + TARGET_LAG = '15 minutes' + WAREHOUSE = FLEET_WH + COMMENT = 'Daily distance, fuel, idling and fault KPIs per vehicle (UTC days)' +AS +SELECT + TS::DATE AS DAY, + VEHICLE_ID, + COUNT(*) AS SAMPLES, + COUNT_IF(SPEED_KMH = 0) AS IDLE_SAMPLES, + MAX(ODOMETER_KM) - MIN(ODOMETER_KM) AS DISTANCE_KM, + MAX(FUEL_USED_L) - MIN(FUEL_USED_L) AS FUEL_USED_L, + MAX(SPEED_KMH) AS MAX_SPEED_KMH, + COUNT_IF(ARRAY_SIZE(FAULT_CODES) > 0) AS FAULT_SAMPLES +FROM TELEMETRY +GROUP BY TS::DATE, VEHICLE_ID; diff --git a/fleet-telemetry/scripts/demo.sh b/fleet-telemetry/scripts/demo.sh new file mode 100755 index 0000000..caee76d --- /dev/null +++ b/fleet-telemetry/scripts/demo.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +# A walkthrough of the service against the emulator started by `make up`: +# bulk backfill, a live POST, then the analytics endpoints and two error paths. +# Starts the API in the background and stops it on exit. +set -euo pipefail +cd "$(dirname "${BASH_SOURCE[0]}")/.." + +export DOTNET_ENVIRONMENT=Development ASPNETCORE_ENVIRONMENT=Development +export SNOWFLAKE_PRIVATE_KEY_PATH="${SNOWFLAKE_PRIVATE_KEY_PATH:-$PWD/.secrets/rsa_key.p8}" +api="http://localhost:${DEMO_PORT:-5080}" +data=.cache/telemetry +ingest=(dotnet run --project src/FleetTelemetry.Ingest --) + +step() { printf '\n\033[1m== %s\033[0m\n' "$*"; } +pretty() { + if command -v jq >/dev/null; then jq .; elif command -v python3 >/dev/null; then python3 -m json.tool; else cat; echo; fi +} +call() { # METHOD PATH [curl args...] + local method="$1" path="$2"; shift 2 + printf '$ curl -X %s %s\n' "$method" "$path" + local body status + body="$(curl -sS -X "$method" "${api}${path}" -w '\n%{http_code}' "$@")" + status="${body##*$'\n'}" + printf 'HTTP %s\n' "$status" + printf '%s' "${body%$'\n'*}" | pretty +} + +step "Build" +for project in src/FleetTelemetry.Api src/FleetTelemetry.Ingest; do dotnet build --nologo -v quiet "$project" >/dev/null; done + +step "Backfill: PUT the NDJSON exports to @TELEMETRY_STAGE, COPY INTO TELEMETRY_RAW (re-runs load nothing twice)" +[[ -d "$data" ]] || "${ingest[@]}" generate "$data" +"${ingest[@]}" backfill "$data" 2>&1 | grep -v '^info:' + +step "Start the API on ${api}" +dotnet run --no-build --project src/FleetTelemetry.Api --urls "$api" >.cache/demo-api.log 2>&1 & +api_pid=$! +trap 'kill "$api_pid" 2>/dev/null || true; wait "$api_pid" 2>/dev/null || true' EXIT +deadline=$((SECONDS + 90)) +until curl -fsS "${api}/health" >/dev/null 2>&1; do + if (( SECONDS >= deadline )) || ! kill -0 "$api_pid" 2>/dev/null; then + echo "The API did not start; see .cache/demo-api.log" >&2 + exit 1 + fi + sleep 1 +done +curl -fsS "${api}/health"; echo + +live="$(find "$data/live" -maxdepth 1 -name '*.json' | sort | head -n 1)" +vehicle="$(basename "$live" | cut -d- -f1-2)" +day="$(basename "$live" .json | cut -d- -f3-)" + +step "Live ingestion: POST one trip of ${vehicle} (a JSON array of pings) to /api/telemetry" +call POST /api/telemetry -H 'Content-Type: application/json' --data-binary "@${live}" + +step "Trips of ${vehicle} on ${day}: window functions split the pings at gaps over 10 minutes" +call GET "/api/vehicles/${vehicle}/trips?date=${day}" + +step "Trips of TRK-203 on 2026-09-08, from the backfill" +call GET "/api/vehicles/TRK-203/trips?date=2026-09-08" + +step "Harsh braking: LAG over speed, at least 2.5 m/s² between consecutive pings" +call GET "/api/vehicles/TRK-203/events?type=harsh_braking" + +step "Idling: five minutes or more standing still with the engine running" +call GET "/api/vehicles/TRK-203/events?type=idling" + +step "Fault codes: LATERAL FLATTEN over each ping's engine.dtc array" +call GET "/api/vehicles/TRK-203/faults" + +step "Daily fleet KPIs from the FLEET_DAILY_KPIS dynamic table" +call GET "/api/fleet/daily?from=2026-09-07&to=2026-09-09" + +step "Errors are problem details: an unknown vehicle, a malformed batch" +call GET "/api/vehicles/TRK-999/trips?date=2026-09-08" +call POST /api/telemetry -H 'Content-Type: application/json' --data '[{"message_id": "TRK-101-1", "vehicle_id": ' + +step "Done. The API log is in .cache/demo-api.log" diff --git a/fleet-telemetry/scripts/wait-for-emulator.sh b/fleet-telemetry/scripts/wait-for-emulator.sh new file mode 100755 index 0000000..eecb632 --- /dev/null +++ b/fleet-telemetry/scripts/wait-for-emulator.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# Wait until the Snowflake-Next emulator answers a SQL statement. +# +# Usage: scripts/wait-for-emulator.sh [timeout-seconds] +# Reads SNOWFLAKE_PORT (default 4566). Readiness is checked from the host. +set -euo pipefail + +port="${SNOWFLAKE_PORT:-4566}" +timeout="${1:-120}" +url="https://snowflake.localhost.localstack.cloud:${port}" +deadline=$((SECONDS + timeout)) + +until curl -fsS --compressed -o /dev/null -X POST "${url}/api/v2/statements" \ + -H 'Authorization: Bearer local' -H 'Content-Type: application/json' \ + -H 'Accept: application/json' -d '{"statement":"SELECT 1"}' 2>/dev/null; do + if (( SECONDS >= deadline )); then + echo "Emulator at ${url} did not become ready within ${timeout}s." >&2 + echo "Check 'docker compose logs snowflake' (is LOCALSTACK_AUTH_TOKEN valid? is port ${port} free?)." >&2 + exit 1 + fi + sleep 2 +done +echo "Emulator ready at ${url}" diff --git a/fleet-telemetry/src/FleetTelemetry.Api/BadRequestExceptionHandler.cs b/fleet-telemetry/src/FleetTelemetry.Api/BadRequestExceptionHandler.cs new file mode 100644 index 0000000..9477b76 --- /dev/null +++ b/fleet-telemetry/src/FleetTelemetry.Api/BadRequestExceptionHandler.cs @@ -0,0 +1,33 @@ +using Microsoft.AspNetCore.Diagnostics; +using Microsoft.AspNetCore.Mvc; + +namespace FleetTelemetry.Api; + +/// +/// Turns request binding failures (a missing or unparsable query parameter) into a 400 +/// problem response that says which parameter was wrong. +/// +internal sealed class BadRequestExceptionHandler(IProblemDetailsService problemDetails) : IExceptionHandler +{ + public async ValueTask TryHandleAsync(HttpContext httpContext, Exception exception, CancellationToken cancellationToken) + { + if (exception is not BadHttpRequestException badRequest) + { + return false; + } + + httpContext.Response.StatusCode = badRequest.StatusCode; + return await problemDetails.TryWriteAsync(new ProblemDetailsContext + { + HttpContext = httpContext, + Exception = exception, + ProblemDetails = new ProblemDetails + { + Status = badRequest.StatusCode, + Title = "Invalid request", + Detail = badRequest.Message, + Type = "urn:fleet-telemetry:problem:validation", + }, + }); + } +} diff --git a/fleet-telemetry/src/FleetTelemetry.Api/Endpoints/Contracts.cs b/fleet-telemetry/src/FleetTelemetry.Api/Endpoints/Contracts.cs new file mode 100644 index 0000000..bc507e4 --- /dev/null +++ b/fleet-telemetry/src/FleetTelemetry.Api/Endpoints/Contracts.cs @@ -0,0 +1,46 @@ +using FleetTelemetry.Kpis; + +namespace FleetTelemetry.Api.Endpoints; + +// Response bodies. Timestamps are UTC. + +public sealed record TelemetryAccepted(int Accepted); + +public sealed record Trip( + int TripNo, + DateTime StartedAt, + DateTime EndedAt, + long DurationS, + double DistanceKm, + double FuelUsedL, + double? FuelEconomyLPer100Km, + double? AvgSpeedKmh, + double MaxSpeedKmh, + long IdleS, + long Samples); + +public sealed record VehicleTrips(string VehicleId, DateOnly Date, IReadOnlyList Trips); + +public sealed record HarshBrakingEvent( + DateTime OccurredAt, + double Lat, + double Lon, + double FromSpeedKmh, + double ToSpeedKmh, + double DecelerationMs2); + +public sealed record IdlingEvent(DateTime StartedAt, DateTime EndedAt, long DurationS, double Lat, double Lon); + +public sealed record VehicleEvents(string VehicleId, string Type, IReadOnlyList Events); + +public sealed record ActiveFault( + string Code, + string Severity, + long Occurrences, + DateTime FirstSeen, + DateTime LastSeen, + double MaxCoolantC); + +public sealed record VehicleFaults(string VehicleId, IReadOnlyList Faults); + +public sealed record FleetDaily(DateOnly From, DateOnly To, IReadOnlyList Days); diff --git a/fleet-telemetry/src/FleetTelemetry.Api/Endpoints/FleetEndpoints.cs b/fleet-telemetry/src/FleetTelemetry.Api/Endpoints/FleetEndpoints.cs new file mode 100644 index 0000000..e957455 --- /dev/null +++ b/fleet-telemetry/src/FleetTelemetry.Api/Endpoints/FleetEndpoints.cs @@ -0,0 +1,31 @@ +using FleetTelemetry.Data; +using FleetTelemetry.Kpis; + +namespace FleetTelemetry.Api.Endpoints; + +internal static class FleetEndpoints +{ + public const int MaxDays = 92; + + public static void MapFleetEndpoints(this IEndpointRouteBuilder app) => + app.MapGet("/api/fleet/daily", GetDailyAsync); + + /// Fleet KPIs per UTC day in [from, to], from the FLEET_DAILY_KPIS dynamic table. + private static async Task GetDailyAsync( + DateOnly from, + DateOnly to, + FleetAnalytics analytics, + CancellationToken cancellationToken) + { + if (to < from || to.DayNumber - from.DayNumber >= MaxDays) + { + return Problems.Invalid(new Dictionary + { + ["to"] = [$"Must be on or after 'from' and at most {MaxDays} days later."], + }); + } + + var vehicleDays = await analytics.GetVehicleDaysAsync(from, to, cancellationToken); + return TypedResults.Ok(new FleetDaily(from, to, FleetDailyKpis.Summarize(vehicleDays))); + } +} diff --git a/fleet-telemetry/src/FleetTelemetry.Api/Endpoints/Problems.cs b/fleet-telemetry/src/FleetTelemetry.Api/Endpoints/Problems.cs new file mode 100644 index 0000000..acd83d0 --- /dev/null +++ b/fleet-telemetry/src/FleetTelemetry.Api/Endpoints/Problems.cs @@ -0,0 +1,24 @@ +namespace FleetTelemetry.Api.Endpoints; + +/// Problem details (RFC 9457) the API returns, beyond framework validation errors. +internal static class Problems +{ + public static IResult VehicleNotFound(string vehicleId) => TypedResults.Problem( + statusCode: StatusCodes.Status404NotFound, + title: "Vehicle not found", + detail: $"No vehicle '{vehicleId}' in the fleet registry.", + type: "urn:fleet-telemetry:problem:vehicle-not-found"); + + public static IResult MalformedTelemetry(string detail) => TypedResults.Problem( + statusCode: StatusCodes.Status400BadRequest, + title: "Malformed telemetry", + detail: detail, + type: "urn:fleet-telemetry:problem:malformed-telemetry"); + + public static IResult Invalid(IReadOnlyDictionary errors) => TypedResults.ValidationProblem( + errors, + title: "Invalid request", + type: "urn:fleet-telemetry:problem:validation"); + + public static DateTime Utc(DateTime value) => DateTime.SpecifyKind(value, DateTimeKind.Utc); +} diff --git a/fleet-telemetry/src/FleetTelemetry.Api/Endpoints/TelemetryEndpoints.cs b/fleet-telemetry/src/FleetTelemetry.Api/Endpoints/TelemetryEndpoints.cs new file mode 100644 index 0000000..27145e6 --- /dev/null +++ b/fleet-telemetry/src/FleetTelemetry.Api/Endpoints/TelemetryEndpoints.cs @@ -0,0 +1,44 @@ +using FleetTelemetry.Data; +using FleetTelemetry.Telemetry; + +namespace FleetTelemetry.Api.Endpoints; + +internal static partial class TelemetryEndpoints +{ + public static void MapTelemetryEndpoints(this IEndpointRouteBuilder app) => + app.MapPost("/api/telemetry", IngestAsync); + + /// + /// Accepts a JSON array of pings. The batch is validated as a whole, then written in one + /// statement; the landing table keeps each ping exactly as sent. + /// + private static async Task IngestAsync( + HttpRequest request, + TelemetryWriter writer, + TimeProvider clock, + ILoggerFactory loggers, + CancellationToken cancellationToken) + { + using var reader = new StreamReader(request.Body); + var body = await reader.ReadToEndAsync(cancellationToken); + + switch (TelemetryBatchParser.Parse(body, clock.GetUtcNow())) + { + case TelemetryBatchResult.Malformed malformed: + return Problems.MalformedTelemetry(malformed.Detail); + case TelemetryBatchResult.Invalid invalid: + return Problems.Invalid(invalid.Errors); + case TelemetryBatchResult.Valid valid: + var inserted = await writer.InsertBatchAsync(body, cancellationToken); + var vehicles = valid.Pings.Select(p => p.VehicleId).Distinct().Count(); + var logger = loggers.CreateLogger("FleetTelemetry.Api.Telemetry"); + LogAccepted(logger, inserted, vehicles); + return TypedResults.Accepted((string?)null, new TelemetryAccepted(inserted)); + default: + throw new InvalidOperationException("Unexpected parse result."); + } + } + + [LoggerMessage(Level = LogLevel.Information, Message = "Accepted {Pings} pings from {Vehicles} vehicles")] + private static partial void LogAccepted(ILogger logger, int pings, int vehicles); +} diff --git a/fleet-telemetry/src/FleetTelemetry.Api/Endpoints/VehicleEndpoints.cs b/fleet-telemetry/src/FleetTelemetry.Api/Endpoints/VehicleEndpoints.cs new file mode 100644 index 0000000..8eb383f --- /dev/null +++ b/fleet-telemetry/src/FleetTelemetry.Api/Endpoints/VehicleEndpoints.cs @@ -0,0 +1,138 @@ +using FleetTelemetry.Data; +using FleetTelemetry.Kpis; +using FleetTelemetry.Rules; + +namespace FleetTelemetry.Api.Endpoints; + +internal static class VehicleEndpoints +{ + public const string HarshBraking = "harsh_braking"; + public const string Idling = "idling"; + + public static void MapVehicleEndpoints(this IEndpointRouteBuilder app) + { + var vehicles = app.MapGroup("/api/vehicles/{vehicleId}"); + vehicles.MapGet("/trips", GetTripsAsync); + vehicles.MapGet("/events", GetEventsAsync); + vehicles.MapGet("/faults", GetFaultsAsync); + } + + /// Trips on one UTC day; see . + private static async Task GetTripsAsync( + string vehicleId, + DateOnly date, + FleetRegistry registry, + VehicleAnalytics analytics, + CancellationToken cancellationToken) + { + if (!await registry.VehicleExistsAsync(vehicleId, cancellationToken)) + { + return Problems.VehicleNotFound(vehicleId); + } + + var trips = await analytics.GetTripsAsync(vehicleId, date, cancellationToken); + return TypedResults.Ok(new VehicleTrips( + vehicleId, + date, + trips.Select(t => new Trip( + TripNo: (int)t.TripNo, + StartedAt: Problems.Utc(t.StartedAt), + EndedAt: Problems.Utc(t.EndedAt), + DurationS: t.DurationS, + DistanceKm: Math.Round(t.DistanceKm, 3), + FuelUsedL: Math.Round(t.FuelUsedL, 3), + FuelEconomyLPer100Km: KpiMath.FuelEconomyLPer100Km(t.FuelUsedL, t.DistanceKm), + AvgSpeedKmh: KpiMath.AverageSpeedKmh(t.DistanceKm, t.DurationS), + MaxSpeedKmh: t.MaxSpeedKmh, + IdleS: t.IdleS, + Samples: t.Samples)) + .ToList())); + } + + /// Driving events: harsh_braking or idling, optionally within [from, to] (UTC days). + private static async Task GetEventsAsync( + string vehicleId, + string? type, + DateOnly? from, + DateOnly? to, + FleetRegistry registry, + VehicleAnalytics analytics, + CancellationToken cancellationToken) + { + if (type is not (HarshBraking or Idling)) + { + return Problems.Invalid(new Dictionary + { + ["type"] = [$"Must be '{HarshBraking}' or '{Idling}'."], + }); + } + + if (Range(from, to) is not var (start, end)) + { + return InvalidRange(); + } + + if (!await registry.VehicleExistsAsync(vehicleId, cancellationToken)) + { + return Problems.VehicleNotFound(vehicleId); + } + + if (type == HarshBraking) + { + var events = await analytics.GetHarshBrakingAsync(vehicleId, start, end, cancellationToken); + return TypedResults.Ok(new VehicleEvents( + vehicleId, + type, + events.Select(e => new HarshBrakingEvent( + Problems.Utc(e.OccurredAt), e.Lat, e.Lon, e.FromSpeedKmh, e.ToSpeedKmh, Math.Round(e.DecelerationMs2, 2))) + .ToList())); + } + + var idling = await analytics.GetIdlingAsync(vehicleId, start, end, cancellationToken); + return TypedResults.Ok(new VehicleEvents( + vehicleId, + type, + idling.Select(e => new IdlingEvent(Problems.Utc(e.StartedAt), Problems.Utc(e.EndedAt), e.DurationS, e.Lat, e.Lon)) + .ToList())); + } + + /// Diagnostic trouble codes reported by the vehicle, optionally within [from, to] (UTC days). + private static async Task GetFaultsAsync( + string vehicleId, + DateOnly? from, + DateOnly? to, + FleetRegistry registry, + VehicleAnalytics analytics, + CancellationToken cancellationToken) + { + if (Range(from, to) is not var (start, end)) + { + return InvalidRange(); + } + + if (!await registry.VehicleExistsAsync(vehicleId, cancellationToken)) + { + return Problems.VehicleNotFound(vehicleId); + } + + var faults = await analytics.GetFaultsAsync(vehicleId, start, end, cancellationToken); + return TypedResults.Ok(new VehicleFaults( + vehicleId, + faults.Select(f => new ActiveFault( + f.Code, f.Severity, f.Occurrences, Problems.Utc(f.FirstSeen), Problems.Utc(f.LastSeen), f.MaxCoolantC)) + .ToList())); + } + + /// [from, to] as UTC day bounds, half-open; both are optional. + private static (DateTime Start, DateTime End)? Range(DateOnly? from, DateOnly? to) + { + var start = (from ?? DateOnly.MinValue).ToDateTime(TimeOnly.MinValue); + var end = to is { } last ? last.AddDays(1).ToDateTime(TimeOnly.MinValue) : DateTime.MaxValue; + return start < end ? (start, end) : null; + } + + private static IResult InvalidRange() => Problems.Invalid(new Dictionary + { + ["to"] = ["Must not be before 'from'."], + }); +} diff --git a/fleet-telemetry/src/FleetTelemetry.Api/FleetTelemetry.Api.csproj b/fleet-telemetry/src/FleetTelemetry.Api/FleetTelemetry.Api.csproj new file mode 100644 index 0000000..f1c6802 --- /dev/null +++ b/fleet-telemetry/src/FleetTelemetry.Api/FleetTelemetry.Api.csproj @@ -0,0 +1,11 @@ + + + + FleetTelemetry.Api + + + + + + + diff --git a/fleet-telemetry/src/FleetTelemetry.Api/HttpQueryTagSource.cs b/fleet-telemetry/src/FleetTelemetry.Api/HttpQueryTagSource.cs new file mode 100644 index 0000000..c0c7532 --- /dev/null +++ b/fleet-telemetry/src/FleetTelemetry.Api/HttpQueryTagSource.cs @@ -0,0 +1,18 @@ +using System.Diagnostics; +using FleetTelemetry.Data; + +namespace FleetTelemetry.Api; + +/// +/// Tags Snowflake queries with the HTTP route and the request's W3C trace id (taken from an +/// incoming traceparent header when there is one), so a query in Snowflake's history +/// can be matched to its trace. +/// +internal sealed class HttpQueryTagSource(IHttpContextAccessor accessor) : IQueryTagSource +{ + public string Operation => accessor.HttpContext is { } context + ? $"{context.Request.Method} {(context.GetEndpoint() as RouteEndpoint)?.RoutePattern.RawText ?? context.Request.Path.Value}" + : "background"; + + public string? CorrelationId => Activity.Current?.TraceId.ToHexString() ?? accessor.HttpContext?.TraceIdentifier; +} diff --git a/fleet-telemetry/src/FleetTelemetry.Api/Program.cs b/fleet-telemetry/src/FleetTelemetry.Api/Program.cs new file mode 100644 index 0000000..ab47d07 --- /dev/null +++ b/fleet-telemetry/src/FleetTelemetry.Api/Program.cs @@ -0,0 +1,26 @@ +using FleetTelemetry.Api; +using FleetTelemetry.Api.Endpoints; +using FleetTelemetry.Data; + +var builder = WebApplication.CreateBuilder(args); +builder.Configuration.AddSnowflakeEnvironmentVariables(); + +builder.Services.AddProblemDetails(); +builder.Services.AddExceptionHandler(); +builder.Services.AddHttpContextAccessor(); +builder.Services.AddSingleton(TimeProvider.System); +builder.Services.AddFleetSnowflake(); +builder.Services.AddScoped(); +builder.Services.AddHealthChecks().AddCheck("snowflake"); + +var app = builder.Build(); + +app.UseExceptionHandler(); +app.UseStatusCodePages(); + +app.MapHealthChecks("/health"); +app.MapTelemetryEndpoints(); +app.MapVehicleEndpoints(); +app.MapFleetEndpoints(); + +app.Run(); diff --git a/fleet-telemetry/src/FleetTelemetry.Api/Properties/launchSettings.json b/fleet-telemetry/src/FleetTelemetry.Api/Properties/launchSettings.json new file mode 100644 index 0000000..804e684 --- /dev/null +++ b/fleet-telemetry/src/FleetTelemetry.Api/Properties/launchSettings.json @@ -0,0 +1,13 @@ +{ + "$schema": "https://json.schemastore.org/launchsettings.json", + "profiles": { + "http": { + "commandName": "Project", + "launchBrowser": false, + "applicationUrl": "http://localhost:5080", + "environmentVariables": { + "ASPNETCORE_ENVIRONMENT": "Development" + } + } + } +} diff --git a/fleet-telemetry/src/FleetTelemetry.Api/SnowflakeHealthCheck.cs b/fleet-telemetry/src/FleetTelemetry.Api/SnowflakeHealthCheck.cs new file mode 100644 index 0000000..b3acc8b --- /dev/null +++ b/fleet-telemetry/src/FleetTelemetry.Api/SnowflakeHealthCheck.cs @@ -0,0 +1,23 @@ +using Dapper; +using FleetTelemetry.Data; +using Microsoft.Extensions.Diagnostics.HealthChecks; + +namespace FleetTelemetry.Api; + +/// Healthy when a session can be opened and answers a query. +internal sealed class SnowflakeHealthCheck(FleetDatabase database) : IHealthCheck +{ + public async Task CheckHealthAsync(HealthCheckContext context, CancellationToken cancellationToken = default) + { + try + { + await using var connection = await database.OpenAsync(cancellationToken); + await connection.ExecuteScalarAsync(new CommandDefinition("SELECT 1", cancellationToken: cancellationToken)); + return HealthCheckResult.Healthy(); + } + catch (Exception e) when (e is not OperationCanceledException) + { + return HealthCheckResult.Unhealthy("Snowflake is not reachable", e); + } + } +} diff --git a/fleet-telemetry/src/FleetTelemetry.Api/appsettings.Development.json b/fleet-telemetry/src/FleetTelemetry.Api/appsettings.Development.json new file mode 100644 index 0000000..76573f9 --- /dev/null +++ b/fleet-telemetry/src/FleetTelemetry.Api/appsettings.Development.json @@ -0,0 +1,17 @@ +{ + "Snowflake": { + "Account": "snowflake", + "Host": "snowflake.localhost.localstack.cloud", + "Port": 4566, + "PrivateKeyPath": "../../.secrets/rsa_key.p8" + }, + "Logging": { + "Console": { + "FormatterName": "simple", + "FormatterOptions": { + "SingleLine": true, + "TimestampFormat": "HH:mm:ss " + } + } + } +} diff --git a/fleet-telemetry/src/FleetTelemetry.Api/appsettings.json b/fleet-telemetry/src/FleetTelemetry.Api/appsettings.json new file mode 100644 index 0000000..e576c88 --- /dev/null +++ b/fleet-telemetry/src/FleetTelemetry.Api/appsettings.json @@ -0,0 +1,24 @@ +{ + "Snowflake": { + "Account": "myorg-fleet", + "Host": "myorg-fleet.snowflakecomputing.com", + "Port": 443, + "User": "FLEET_SVC", + "PrivateKeyPath": "/run/secrets/snowflake_rsa_key.p8", + "Role": "FLEET_APP", + "Warehouse": "FLEET_WH", + "Database": "FLEET", + "Schema": "TELEMETRY", + "Application": "fleet-telemetry-api" + }, + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.AspNetCore": "Warning" + }, + "Console": { + "FormatterName": "json" + } + }, + "AllowedHosts": "*" +} diff --git a/fleet-telemetry/src/FleetTelemetry.Api/packages.lock.json b/fleet-telemetry/src/FleetTelemetry.Api/packages.lock.json new file mode 100644 index 0000000..a180b71 --- /dev/null +++ b/fleet-telemetry/src/FleetTelemetry.Api/packages.lock.json @@ -0,0 +1,287 @@ +{ + "version": 2, + "dependencies": { + "net10.0": { + "Apache.Arrow": { + "type": "Transitive", + "resolved": "23.0.0", + "contentHash": "cU4Zm7byFwnb82Ea9ZRvVnq2nJNf/TF9j+rj/MHlvu9RBYMqYtu4sJaR1JmMh3MmSIuwMafVg7PfJxPAMv9dGw==", + "dependencies": { + "Apache.Arrow.Scalars": "23.0.0" + } + }, + "Apache.Arrow.Scalars": { + "type": "Transitive", + "resolved": "23.0.0", + "contentHash": "i53VmQ505ASXsStf7y29Z5qvhN6PGgFGNb6UUU7z1WmbzEgSjKQuIJkKKVbh+K5rtXRiKyr1Ah3Rmvbeb2BEHQ==" + }, + "AWSSDK.Core": { + "type": "Transitive", + "resolved": "4.0.100.8", + "contentHash": "xnuBVLQBmYQXsDZJ9mq2UDSFZm3xgO5oUb4/UR8p0UO7tG6heDhsLLI1NzZhIVAKyfW0tXg9hn9a/ek018TOVw==" + }, + "AWSSDK.S3": { + "type": "Transitive", + "resolved": "4.0.101.4", + "contentHash": "TYFuatWECzCbj/Lu1SsANucgpU/Br5YJ5Padl3YEdNJGfZqAcx3QemJw2BUopAsYTJ3e7TFpWhpNWha3dB9/dw==", + "dependencies": { + "AWSSDK.Core": "[4.0.100.8, 5.0.0)" + } + }, + "Azure.Core": { + "type": "Transitive", + "resolved": "1.55.0", + "contentHash": "c7femvYS/xEUrLP1sNZN+DoQZmx3X+G3ZXtOOz0RL/7cGMCM3JVqepVmRd3AwM4IK8AtTGS4GOigCO+d/zaSsQ==", + "dependencies": { + "Microsoft.Bcl.AsyncInterfaces": "10.0.3", + "Microsoft.Identity.Client": "4.83.1", + "Microsoft.Identity.Client.Extensions.Msal": "4.83.1", + "System.ClientModel": "1.11.0", + "System.Memory.Data": "10.0.3" + } + }, + "Azure.Storage.Blobs": { + "type": "Transitive", + "resolved": "12.29.1", + "contentHash": "pWh7xZBto8YcwiO853AB3uijTfVqs9PDte0Bu9/Zd8O9nwKiitWm0Jej1vsNkmYUzeG1552f8vJPjmtW30pBUQ==", + "dependencies": { + "Azure.Core": "1.55.0", + "Azure.Storage.Common": "12.28.0" + } + }, + "Azure.Storage.Common": { + "type": "Transitive", + "resolved": "12.28.0", + "contentHash": "5l8YhNrks38zKLGFW2BBFLwieyamH/xauABSASsdpZv79G0f+n2n22IjkRmUqCmALSupkwRHh2LOhbW3yj5Qgw==", + "dependencies": { + "Azure.Core": "1.55.0", + "System.IO.Hashing": "10.0.3" + } + }, + "BouncyCastle.Cryptography": { + "type": "Transitive", + "resolved": "2.7.0", + "contentHash": "U+12df8UEWHgBi04YVf/Lgi2dy3SItlIYvHjjEVa/BngCQIzDCDRBk50DDByCfDvSbe5pRNFr3b7UrVK2kMcLw==" + }, + "Google.Api.Gax": { + "type": "Transitive", + "resolved": "4.13.1", + "contentHash": "ujDpZk7O2VqAEIqcSlhH0FKzVpGZWly/qcNjHxpNUrL445Tei9PHnu4V1pwW2WDiMDvo3TpL1Bi4DeU525Afrg==", + "dependencies": { + "Microsoft.Bcl.AsyncInterfaces": "6.0.0", + "Newtonsoft.Json": "13.0.4" + } + }, + "Google.Api.Gax.Rest": { + "type": "Transitive", + "resolved": "4.13.1", + "contentHash": "wDfFZXKoHC6sy+Yfub/4cLQOb5ozBVPqDOrxCUAeTfku4d96ILqgzulJSdkPHbiT3oWB3hlYTdJgUwNSJjyo+A==", + "dependencies": { + "Google.Api.Gax": "4.13.1", + "Google.Apis.Auth": "1.73.0" + } + }, + "Google.Apis": { + "type": "Transitive", + "resolved": "1.74.0", + "contentHash": "Z6hSkmRD6V8/raSBqn/EsPpNop6suaLaGE9DbSY3yRE5G47l4zYqrJgapcLARk2Z3TsQUDmWz+w/tob3wuVt3g==", + "dependencies": { + "Google.Apis.Core": "1.74.0" + } + }, + "Google.Apis.Auth": { + "type": "Transitive", + "resolved": "1.74.0", + "contentHash": "43H4foh3zfWgboK8QnJjIfPQuBiS794TxqQLDfeE3erQSluKmEN0qeDuwxLTpSr9Mtwj76S4LWxPneuSbC6fNg==", + "dependencies": { + "Google.Apis": "1.74.0", + "Google.Apis.Core": "1.74.0", + "System.Management": "7.0.2" + } + }, + "Google.Apis.Core": { + "type": "Transitive", + "resolved": "1.74.0", + "contentHash": "gWOIiks6yo2TRliIyy10V18BJOO2BfUSPJFiTW6Tfap4tm3hFj7nvYU3XIfwzAxwoIy6nsBoqVDt9iBDSXrvYQ==", + "dependencies": { + "Newtonsoft.Json": "13.0.4" + } + }, + "Google.Apis.Storage.v1": { + "type": "Transitive", + "resolved": "1.74.0.4115", + "contentHash": "9J3YPPSG53xwQLzdQ304dI+DMtVxJDpEazA2QQ2LA8IJoM4VUu4DpIxyhTGxpnAK4Db7s4LwzeNi6NkPKM3HzQ==", + "dependencies": { + "Google.Apis": "1.74.0", + "Google.Apis.Auth": "1.74.0" + } + }, + "Google.Cloud.Storage.V1": { + "type": "Transitive", + "resolved": "4.15.0", + "contentHash": "TVlJsaIkMDwHbZPCcLFFKduFcFD0yJL1jsZmV/V/FMjv5iok/eoC0oy8/VHjxfmWmNGkJLPLMAe5MrFLtY7hzg==", + "dependencies": { + "Google.Api.Gax.Rest": "[4.13.1, 5.0.0)", + "Google.Apis.Storage.v1": "[1.74.0.4115, 2.0.0)" + } + }, + "Microsoft.Bcl.AsyncInterfaces": { + "type": "Transitive", + "resolved": "10.0.3", + "contentHash": "TV62UsrJZPX6gbt3c4WrtXh7bmaDIcMqf9uft1cc4L6gJXOU07hDGEh+bFQh/L2Az0R1WVOkiT66lFqS6G2NmA==" + }, + "Microsoft.Bcl.Cryptography": { + "type": "Transitive", + "resolved": "10.0.2", + "contentHash": "LG9Yll3B5aNpxv0+D47g6LiOiKBIlodhcHdQwcYzo8VeexFLGqx5ymetmA2aBRyo9cCcWsQWrFsdbsr8LvmWDw==" + }, + "Microsoft.Identity.Client": { + "type": "Transitive", + "resolved": "4.83.1", + "contentHash": "jOLIrZ3cynoqHLLO1cXplFFabrhrMEYs/EuKHvmCyrOm1axqiVFT6nCSnHxk7w5+d2BeQfCdM12Yf/0X7OeS1g==", + "dependencies": { + "Microsoft.IdentityModel.Abstractions": "8.14.0" + } + }, + "Microsoft.Identity.Client.Extensions.Msal": { + "type": "Transitive", + "resolved": "4.83.1", + "contentHash": "I3k4J4Hj4KbLEFanjeUzzDOVecukETaTgEkJ7h2pP/Yazs6SLp6TVUTo/Eo+ptPXMwvc+iX7rBFtMSUrA7R+Mg==", + "dependencies": { + "Microsoft.Identity.Client": "4.83.1", + "System.Security.Cryptography.ProtectedData": "4.5.0" + } + }, + "Microsoft.IdentityModel.Abstractions": { + "type": "Transitive", + "resolved": "8.21.0", + "contentHash": "XYgEmYpUvng0oYIagDJ+uebboQSnWtCV7q9zMyXlBw6ZqLLCHcmGUb7TpXvQ3HWY4OzbOCUQm2Ac0saPSnwqSw==" + }, + "Microsoft.IdentityModel.JsonWebTokens": { + "type": "Transitive", + "resolved": "8.21.0", + "contentHash": "iiA5uimwlRe4Drd8YKOaB8IQkgGOjxGjO6mu5zCGFjaEBpB6yNOluZ33zJFXiW5MQYRC6ypMnOXsiR5SPrO+tw==", + "dependencies": { + "Microsoft.IdentityModel.Tokens": "8.21.0" + } + }, + "Microsoft.IdentityModel.Logging": { + "type": "Transitive", + "resolved": "8.21.0", + "contentHash": "LgIWUUX386xCbvjq2ex8iXGIu2raDWrrIQ4OyYZKaNWMhzvN8YOTbAYMFJC+iHmKnkNBM8g7mLak9KQodldH3g==", + "dependencies": { + "Microsoft.IdentityModel.Abstractions": "8.21.0" + } + }, + "Microsoft.IdentityModel.Tokens": { + "type": "Transitive", + "resolved": "8.21.0", + "contentHash": "xFnFEl4VrhAJEbqtRrpgfAACgkm6NYo7cWYKYROH00N9/M5OYrRwTP0lHsginnLrqPrseLcSqAueQNRCyGNsdw==", + "dependencies": { + "Microsoft.Bcl.Cryptography": "10.0.2", + "Microsoft.IdentityModel.Logging": "8.21.0" + } + }, + "Mono.Unix": { + "type": "Transitive", + "resolved": "7.1.0-final.1.21458.1", + "contentHash": "Rhxz4A7By8Q0wEgDqR+mioDsYXGrcYMYPiWE9bSaUKMpG8yAGArhetEQV5Ms6KhKCLdQTlPYLBKPZYoKbAvT/g==" + }, + "Newtonsoft.Json": { + "type": "Transitive", + "resolved": "13.0.4", + "contentHash": "pdgNNMai3zv51W5aq268sujXUyx7SNdE2bj1wZcWjAQrKMFZV260lbqYop1d2GM67JI1huLRwxo9ZqnfF/lC6A==" + }, + "System.ClientModel": { + "type": "Transitive", + "resolved": "1.11.0", + "contentHash": "1Wl32zh7TbvN+HAO8NqDuaN0Ao2Qu/0j0NJSrXGDtpUQsTXQYJ3C6hd9/Ds2IrgP4agMQYFoXB35GZfC21ByHw==", + "dependencies": { + "System.Memory.Data": "10.0.3" + } + }, + "System.CodeDom": { + "type": "Transitive", + "resolved": "7.0.0", + "contentHash": "GLltyqEsE5/3IE+zYRP5sNa1l44qKl9v+bfdMcwg+M9qnQf47wK3H0SUR/T+3N4JEQXF3vV4CSuuo0rsg+nq2A==" + }, + "System.IdentityModel.Tokens.Jwt": { + "type": "Transitive", + "resolved": "8.21.0", + "contentHash": "dKsTkmlpUf/I6g1Aw9joWaZAahXvC0gWGQvkXW1IjZLSefC/4LhQI0tL0VIs64/NZQbl4XnQvLQyu6c7z1pMxg==", + "dependencies": { + "Microsoft.IdentityModel.JsonWebTokens": "8.21.0", + "Microsoft.IdentityModel.Tokens": "8.21.0" + } + }, + "System.IO.Hashing": { + "type": "Transitive", + "resolved": "10.0.3", + "contentHash": "La6ICwsdTKhVX+LKN+pvFjQRR3LhLwq3uKdi2knjLzRyPYBSydF4cjXidYxIiTcDD6XVYdsBWQEI8ZxiZ/OdIg==" + }, + "System.Management": { + "type": "Transitive", + "resolved": "7.0.2", + "contentHash": "/qEUN91mP/MUQmJnM5y5BdT7ZoPuVrtxnFlbJ8a3kBJGhe2wCzBfnPFtK2wTtEEcf3DMGR9J00GZZfg6HRI6yA==", + "dependencies": { + "System.CodeDom": "7.0.0" + } + }, + "System.Memory.Data": { + "type": "Transitive", + "resolved": "10.0.3", + "contentHash": "MaGhRfGunmrj/nHjtsi9XkhlYJ/ERGWrbA+BiSKNtGnAjc9XlG5EhAvak6VRcX5LYzPF6pBO8nJ613dTgzabig==" + }, + "System.Security.Cryptography.ProtectedData": { + "type": "Transitive", + "resolved": "4.5.0", + "contentHash": "wLBKzFnDCxP12VL9ANydSYhk59fC4cvOr9ypYQLPnAj48NQIhqnjdD2yhP8yEKyBJEjERWS9DisKL7rX5eU25Q==" + }, + "TimeZoneConverter": { + "type": "Transitive", + "resolved": "7.2.0", + "contentHash": "4Poqu79dTmacVETqUJ53g3x4U21V/cmWPSlehdsVE4Q00akavvlRq+SRjcg/CmfO9ibOCjRl6TeUV7hkC5RC4w==" + }, + "Tomlyn.Signed": { + "type": "Transitive", + "resolved": "2.10.1", + "contentHash": "+BHtb+YzcZ8uV9KVxjQE2uq7CJoYfd1q9gDuU2lx+wrrUppsj41lm2/uYQKTchz5Cs0pzvhf8t71bScDKVi8zA==" + }, + "fleettelemetry.core": { + "type": "Project" + }, + "fleettelemetry.data": { + "type": "Project", + "dependencies": { + "Dapper": "[2.1.89, )", + "FleetTelemetry.Core": "[1.0.0, )", + "Snowflake.Data": "[6.4.0, )" + } + }, + "Dapper": { + "type": "CentralTransitive", + "requested": "[2.1.89, )", + "resolved": "2.1.89", + "contentHash": "1VjFE664wnxvuhlgP8kS4PeFFF4jqD3VkbIHW6Zlc52zuibgcz5ICnIHcQXlTp6pURpPNGLV0ZpNe5x9KePJnA==" + }, + "Snowflake.Data": { + "type": "CentralTransitive", + "requested": "[6.4.0, )", + "resolved": "6.4.0", + "contentHash": "9zGrm/bD5h+DVha41TKy7soNDqw1bObAXCz9fQdhMJ0wKbj9IRuZvE+ZUvflBMq2mnjkgbNdiGeKz7qixgqbJA==", + "dependencies": { + "AWSSDK.S3": "4.0.101.4", + "Apache.Arrow": "23.0.0", + "Azure.Storage.Blobs": "12.29.1", + "BouncyCastle.Cryptography": "2.7.0", + "Google.Cloud.Storage.V1": "4.15.0", + "Mono.Unix": "7.1.0-final.1.21458.1", + "Newtonsoft.Json": "13.0.4", + "System.IdentityModel.Tokens.Jwt": "8.21.0", + "TimeZoneConverter": "7.2.0", + "Tomlyn.Signed": "2.10.1" + } + } + } + } +} \ No newline at end of file diff --git a/fleet-telemetry/src/FleetTelemetry.Core/FleetTelemetry.Core.csproj b/fleet-telemetry/src/FleetTelemetry.Core/FleetTelemetry.Core.csproj new file mode 100644 index 0000000..ad8ef52 --- /dev/null +++ b/fleet-telemetry/src/FleetTelemetry.Core/FleetTelemetry.Core.csproj @@ -0,0 +1,8 @@ + + + + + FleetTelemetry + + + diff --git a/fleet-telemetry/src/FleetTelemetry.Core/Kpis/FleetDailyKpis.cs b/fleet-telemetry/src/FleetTelemetry.Core/Kpis/FleetDailyKpis.cs new file mode 100644 index 0000000..545af2d --- /dev/null +++ b/fleet-telemetry/src/FleetTelemetry.Core/Kpis/FleetDailyKpis.cs @@ -0,0 +1,51 @@ +namespace FleetTelemetry.Kpis; + +/// One row of the FLEET_DAILY_KPIS dynamic table: one vehicle on one UTC day. +public sealed record VehicleDayKpi( + DateOnly Day, + string VehicleId, + long Samples, + long IdleSamples, + double DistanceKm, + double FuelUsedL, + double MaxSpeedKmh, + long FaultSamples); + +/// Fleet-wide KPIs for one day. +public sealed record FleetDayKpi( + DateOnly Day, + int ActiveVehicles, + double DistanceKm, + double FuelUsedL, + double? FuelEconomyLPer100Km, + double IdleShare, + double MaxSpeedKmh, + int VehiclesWithFaults); + +public static class FleetDailyKpis +{ + /// + /// Rolls vehicle-days up to fleet days. Ratios are computed from the fleet totals + /// (total fuel over total distance), not averaged across vehicles, so a van that + /// drove 2 km does not weigh as much as a truck that drove 400. + /// + public static IReadOnlyList Summarize(IEnumerable vehicleDays) => + vehicleDays + .GroupBy(v => v.Day) + .OrderBy(g => g.Key) + .Select(day => + { + var distance = day.Sum(v => v.DistanceKm); + var fuel = day.Sum(v => v.FuelUsedL); + return new FleetDayKpi( + day.Key, + ActiveVehicles: day.Select(v => v.VehicleId).Distinct().Count(), + DistanceKm: Math.Round(distance, 1), + FuelUsedL: Math.Round(fuel, 1), + FuelEconomyLPer100Km: KpiMath.FuelEconomyLPer100Km(fuel, distance), + IdleShare: KpiMath.IdleShare(day.Sum(v => v.IdleSamples), day.Sum(v => v.Samples)), + MaxSpeedKmh: day.Max(v => v.MaxSpeedKmh), + VehiclesWithFaults: day.Count(v => v.FaultSamples > 0)); + }) + .ToList(); +} diff --git a/fleet-telemetry/src/FleetTelemetry.Core/Kpis/KpiMath.cs b/fleet-telemetry/src/FleetTelemetry.Core/Kpis/KpiMath.cs new file mode 100644 index 0000000..1720dd4 --- /dev/null +++ b/fleet-telemetry/src/FleetTelemetry.Core/Kpis/KpiMath.cs @@ -0,0 +1,24 @@ +namespace FleetTelemetry.Kpis; + +/// Derived fleet metrics, shared by trip and daily KPI responses. +public static class KpiMath +{ + /// Below this distance a fuel economy figure is noise (a vehicle moved in the yard). + public const double MinDistanceForEconomyKm = 1.0; + + /// Litres per 100 km, rounded to 0.1, or null when the distance is too short to say. + public static double? FuelEconomyLPer100Km(double fuelUsedL, double distanceKm) => + distanceKm < MinDistanceForEconomyKm ? null : Math.Round(fuelUsedL / distanceKm * 100, 1); + + /// Share of ignition-on samples with the vehicle stationary, 0 to 1, rounded to 0.001. + public static double IdleShare(long idleSamples, long samples) + { + ArgumentOutOfRangeException.ThrowIfNegative(idleSamples); + ArgumentOutOfRangeException.ThrowIfGreaterThan(idleSamples, samples); + return samples == 0 ? 0 : Math.Round((double)idleSamples / samples, 3); + } + + /// Average speed over a trip including stops, rounded to 0.1, or null for a zero-length trip. + public static double? AverageSpeedKmh(double distanceKm, long durationSeconds) => + durationSeconds <= 0 ? null : Math.Round(distanceKm / (durationSeconds / 3600.0), 1); +} diff --git a/fleet-telemetry/src/FleetTelemetry.Core/Rules/DrivingEventRules.cs b/fleet-telemetry/src/FleetTelemetry.Core/Rules/DrivingEventRules.cs new file mode 100644 index 0000000..c2bf0db --- /dev/null +++ b/fleet-telemetry/src/FleetTelemetry.Core/Rules/DrivingEventRules.cs @@ -0,0 +1,28 @@ +namespace FleetTelemetry.Rules; + +/// +/// Thresholds for the driving events dashboards report. The events queries compute the +/// same values in SQL from consecutive pings. +/// +public static class DrivingEventRules +{ + /// About 0.25 g, a common harsh-braking threshold for trucks and vans. + public const double HarshBrakingMs2 = 2.5; + + /// A stationary vehicle with the engine running for this long is idling. + public static readonly TimeSpan MinIdling = TimeSpan.FromMinutes(5); + + /// Average deceleration in m/s² between two pings (negative when accelerating). + public static double DecelerationMs2(double fromKmh, double toKmh, TimeSpan elapsed) + { + if (elapsed <= TimeSpan.Zero) + { + throw new ArgumentOutOfRangeException(nameof(elapsed), elapsed, "Pings must be in time order."); + } + + return (fromKmh - toKmh) / 3.6 / elapsed.TotalSeconds; + } + + public static bool IsHarshBraking(double fromKmh, double toKmh, TimeSpan elapsed) => + DecelerationMs2(fromKmh, toKmh, elapsed) >= HarshBrakingMs2; +} diff --git a/fleet-telemetry/src/FleetTelemetry.Core/Rules/TripRules.cs b/fleet-telemetry/src/FleetTelemetry.Core/Rules/TripRules.cs new file mode 100644 index 0000000..2ebf512 --- /dev/null +++ b/fleet-telemetry/src/FleetTelemetry.Core/Rules/TripRules.cs @@ -0,0 +1,15 @@ +namespace FleetTelemetry.Rules; + +/// +/// How pings are grouped into trips. Devices report every few seconds while the ignition +/// is on and stop reporting when it is off, so a silence longer than +/// ends a trip. The trips query applies the same rule in SQL. +/// +public static class TripRules +{ + public static readonly TimeSpan MaxPingGap = TimeSpan.FromMinutes(10); + + /// Time since the vehicle's previous ping, or null for its first ping. + public static bool StartsNewTrip(TimeSpan? sincePreviousPing) => + sincePreviousPing is not { } gap || gap > MaxPingGap; +} diff --git a/fleet-telemetry/src/FleetTelemetry.Core/Simulation/FleetSimulator.cs b/fleet-telemetry/src/FleetTelemetry.Core/Simulation/FleetSimulator.cs new file mode 100644 index 0000000..fb7a48e --- /dev/null +++ b/fleet-telemetry/src/FleetTelemetry.Core/Simulation/FleetSimulator.cs @@ -0,0 +1,311 @@ +using FleetTelemetry.Telemetry; + +namespace FleetTelemetry.Simulation; + +/// +/// Deterministic telemetry for a fleet: the same seed, vehicles and dates always produce +/// the same pings. Each vehicle leaves its depot in the morning, drives 2 to 4 delivery +/// trips with traffic stops, the occasional hard stop and loading-dock wait, and returns +/// to the depot. Devices report every while the ignition is on. +/// +public sealed class FleetSimulator(int seed) +{ + public static readonly TimeSpan PingInterval = TimeSpan.FromSeconds(5); + + private static readonly FaultCode[] FaultCatalog = + [ + new("P0420", "info"), + new("P0301", "warning"), + new("P0087", "warning"), + new("U0100", "warning"), + new("P0217", "critical"), + ]; + + /// Simulates every vehicle for consecutive days. + public IReadOnlyList Simulate(IReadOnlyList vehicles, DateOnly firstDay, int days) + { + ArgumentNullException.ThrowIfNull(vehicles); + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(days); + + var result = new List(vehicles.Count * days); + for (var v = 0; v < vehicles.Count; v++) + { + var vehicle = vehicles[v]; + var state = VehicleState.Initial(vehicle, new Random(unchecked((seed * 1_000_003) + v))); + for (var d = 0; d < days; d++) + { + var rng = new Random(unchecked((seed * 1_000_003) + (v * 7_919) + ((d + 1) * 104_729))); + result.Add(SimulateDay(vehicle, state, firstDay.AddDays(d), rng)); + } + } + + return result; + } + + private static VehicleDay SimulateDay(SimulatedVehicle vehicle, VehicleState state, DateOnly day, Random rng) + { + var driver = new Driver(vehicle, state, rng); + state.Refuel(rng); + state.Faults = rng.NextDouble() < 0.3 ? PlanFaults(rng) : null; + + var start = new DateTimeOffset(day.ToDateTime(new TimeOnly(5, 30)), TimeSpan.Zero) + .AddMinutes(rng.Next(0, 90)); + var tripCount = rng.Next(2, 5); + var time = start; + for (var trip = 0; trip < tripCount; trip++) + { + var returning = trip == tripCount - 1; + var (destLat, destLon, distanceKm) = returning + ? (vehicle.DepotLat, vehicle.DepotLon, Math.Max(3, Geo.DistanceKm(state.Lat, state.Lon, vehicle.DepotLat, vehicle.DepotLon) * 1.25)) + : PickDestination(vehicle, state, rng); + time = driver.DriveTrip(time, destLat, destLon, distanceKm); + // Ignition off while unloading: 35 to 120 minutes without pings. + time = time.AddMinutes(rng.Next(35, 121)); + } + + return new VehicleDay(vehicle.VehicleId, day, driver.Pings, driver.Trips, driver.HarshBrakes, driver.Idling); + } + + private static (double Lat, double Lon, double DistanceKm) PickDestination( + SimulatedVehicle vehicle, VehicleState state, Random rng) + { + var (min, max) = vehicle.VehicleClass == "van" ? (4.0, 18.0) : (10.0, 35.0); + var straightKm = min + (rng.NextDouble() * (max - min)); + // Stay within about 40 km of the depot so a day's route remains regional. + var bearing = Geo.DistanceKm(state.Lat, state.Lon, vehicle.DepotLat, vehicle.DepotLon) > 25 + ? Geo.BearingDeg(state.Lat, state.Lon, vehicle.DepotLat, vehicle.DepotLon) + ((rng.NextDouble() - 0.5) * 90) + : rng.NextDouble() * 360; + var (lat, lon) = Geo.Offset(state.Lat, state.Lon, bearing, straightKm); + return (lat, lon, straightKm * 1.25); + } + + private static FaultPlan PlanFaults(Random rng) + { + var first = FaultCatalog[rng.Next(FaultCatalog.Length)]; + var second = rng.NextDouble() < 0.25 ? FaultCatalog[rng.Next(FaultCatalog.Length)] : null; + return new FaultPlan( + ActiveFrom: rng.NextDouble() * 0.7, + Codes: second is null || second == first ? [first] : [first, second]); + } + + private sealed record FaultPlan(double ActiveFrom, FaultCode[] Codes); + + private sealed class VehicleState + { + public double Lat { get; set; } + + public double Lon { get; set; } + + public double OdometerKm { get; set; } + + public double FuelUsedL { get; set; } + + public double FuelInTankL { get; set; } + + public double CoolantC { get; set; } + + public FaultPlan? Faults { get; set; } + + public required double TankL { get; init; } + + public static VehicleState Initial(SimulatedVehicle vehicle, Random rng) => new() + { + Lat = vehicle.DepotLat, + Lon = vehicle.DepotLon, + OdometerKm = Math.Round(40_000 + (rng.NextDouble() * 360_000), 3), + FuelUsedL = Math.Round(10_000 + (rng.NextDouble() * 90_000), 3), + TankL = vehicle.FuelTankL, + }; + + public void Refuel(Random rng) + { + FuelInTankL = TankL * (0.9 + (rng.NextDouble() * 0.1)); + CoolantC = 30 + (rng.NextDouble() * 10); + } + } + + /// Emits pings for one vehicle-day and records the ground truth as it goes. + private sealed class Driver(SimulatedVehicle vehicle, VehicleState state, Random rng) + { + private const double NormalBrakingKmhPerTick = 14; + + private readonly double _litresPerKm = vehicle.VehicleClass switch + { + "tractor" => 0.31, + "rigid" => 0.23, + _ => 0.105, + }; + + private readonly double _idleLitresPerTick = (vehicle.VehicleClass == "van" ? 0.8 : 2.4) / 720; + + private double _speed; + private double _heading; + private DateTimeOffset _time; + private int _dayTick; + + public List Pings { get; } = []; + + public List Trips { get; } = []; + + public List HarshBrakes { get; } = []; + + public List Idling { get; } = []; + + public DateTimeOffset DriveTrip(DateTimeOffset start, double destLat, double destLon, double distanceKm) + { + _time = start; + _speed = 0; + var firstPing = Pings.Count; + var (fromLat, fromLon) = (state.Lat, state.Lon); + var baseHeading = Geo.BearingDeg(fromLat, fromLon, destLat, destLon); + var startOdometer = state.OdometerKm; + // Urban, regional or highway cruising speed; trucks are limited to about 80 km/h. + var cruise = (vehicle.VehicleClass == "van" ? 50 : 55) + (rng.Next(0, 3) * 12.5); + double? harshAtKm = rng.NextDouble() < 0.4 ? distanceKm * (0.2 + (rng.NextDouble() * 0.5)) : null; + double? dockAtKm = rng.NextDouble() < 0.3 ? distanceKm * (0.3 + (rng.NextDouble() * 0.4)) : null; + + // Ignition on: a few stationary pings before pulling away. + Emit(); + Idle(rng.Next(2, 6)); + + while (true) + { + var travelled = state.OdometerKm - startOdometer; + var remainingM = (distanceKm - travelled) * 1000; + Steer(fromLat, fromLon, destLat, destLon, baseHeading, travelled / distanceKm); + + if (remainingM <= StoppingDistanceM(_speed) + 30) + { + BrakeToStop(); + Idle(rng.Next(1, 4)); + break; + } + + if (dockAtKm is { } dock && travelled >= dock) + { + dockAtKm = null; + BrakeToStop(); + var idleStart = _time; + Idle(rng.Next(72, 109)); + Idling.Add(new IdlingTruth(idleStart, _time)); + continue; + } + + if (harshAtKm is { } harsh && travelled >= harsh && _speed >= 48) + { + harshAtKm = null; + var from = Rounded(_speed); + Tick(Math.Max(0, _speed - (48 + (rng.NextDouble() * 14)))); + HarshBrakes.Add(new HarshBrakeTruth(_time, from, Rounded(_speed))); + continue; + } + + if (_speed > 20 && rng.NextDouble() < 0.004) + { + BrakeToStop(); + Idle(rng.Next(3, 11)); + continue; + } + + Tick(_speed < cruise - 3 + ? Math.Min(cruise, _speed + 6 + (rng.NextDouble() * 4)) + : Math.Clamp(_speed + ((rng.NextDouble() - 0.5) * 4), cruise - 3, cruise + 3)); + } + + var trip = Pings.Skip(firstPing).ToList(); + Trips.Add(new TripTruth( + trip[0].Ts, + trip[^1].Ts, + Math.Round(trip[^1].OdometerKm - trip[0].OdometerKm, 3), + trip.Max(p => p.SpeedKmh), + trip.Count)); + return _time; + } + + private static double StoppingDistanceM(double speedKmh) + { + var metresPerSecond = speedKmh / 3.6; + var deceleration = NormalBrakingKmhPerTick / 3.6 / PingInterval.TotalSeconds; + return metresPerSecond * metresPerSecond / (2 * deceleration); + } + + private static double Rounded(double speedKmh) => Math.Round(speedKmh, 1); + + private void Steer(double fromLat, double fromLon, double destLat, double destLon, double baseHeading, double progress) + { + var p = Math.Clamp(progress, 0, 1); + // A gentle S-curve around the straight line, like a road would. + var wobbleKm = Math.Sin(p * Math.PI * 3) * 0.4; + var (lat, lon) = Geo.Offset( + fromLat + ((destLat - fromLat) * p), + fromLon + ((destLon - fromLon) * p), + baseHeading + 90, + wobbleKm); + state.Lat = lat; + state.Lon = lon; + _heading = baseHeading + (Math.Cos(p * Math.PI * 3) * 15); + } + + private void BrakeToStop() + { + while (_speed > 0) + { + Tick(Math.Max(0, _speed - (10 + (rng.NextDouble() * (NormalBrakingKmhPerTick - 10))))); + } + } + + private void Idle(int ticks) + { + for (var i = 0; i < ticks; i++) + { + Tick(0); + } + } + + private void Tick(double newSpeed) + { + var km = (_speed + newSpeed) / 2 / 3600 * PingInterval.TotalSeconds; + state.OdometerKm += km; + var litres = (km * _litresPerKm) + (newSpeed == 0 ? _idleLitresPerTick : 0); + state.FuelUsedL += litres; + state.FuelInTankL = Math.Max(0, state.FuelInTankL - litres); + _speed = newSpeed; + _time += PingInterval; + Emit(); + } + + private void Emit() + { + _dayTick++; + state.CoolantC = state.CoolantC < 86 + ? state.CoolantC + 1.1 + : 86 + (rng.NextDouble() * 6); + var faults = ActiveFaults(); + var coolant = faults.Any(f => f.Code == "P0217") ? 106 + (rng.NextDouble() * 6) : state.CoolantC; + var rpm = _speed == 0 ? 620 + rng.Next(0, 60) : 900 + (int)(_speed * 14) + rng.Next(-80, 80); + + Pings.Add(new TelemetryPing( + MessageId: $"{vehicle.VehicleId}-{_time:yyyyMMddHHmmss}", + VehicleId: vehicle.VehicleId, + Ts: _time, + Gps: new GpsFix(Math.Round(state.Lat, 5), Math.Round(state.Lon, 5), (int)Math.Round(((_heading % 360) + 360) % 360) % 360), + SpeedKmh: Rounded(_speed), + OdometerKm: Math.Round(state.OdometerKm, 3), + Fuel: new FuelReading( + Math.Round(state.FuelInTankL / state.TankL * 100, 1), + Math.Round(state.FuelUsedL, 3)), + Engine: new EngineReading(rpm, Math.Round(coolant, 1), faults))); + } + + private FaultCode[] ActiveFaults() + { + // Faults appear part-way through the day (measured in pings) and stay active. + if (state.Faults is not { } plan || _dayTick < plan.ActiveFrom * 2_000) + { + return []; + } + + return plan.Codes; + } + } +} diff --git a/fleet-telemetry/src/FleetTelemetry.Core/Simulation/Geo.cs b/fleet-telemetry/src/FleetTelemetry.Core/Simulation/Geo.cs new file mode 100644 index 0000000..5f41498 --- /dev/null +++ b/fleet-telemetry/src/FleetTelemetry.Core/Simulation/Geo.cs @@ -0,0 +1,34 @@ +namespace FleetTelemetry.Simulation; + +/// Small-distance geodesy, accurate enough for regional routes. +internal static class Geo +{ + private const double EarthRadiusKm = 6371.0; + + public static double DistanceKm(double lat1, double lon1, double lat2, double lon2) + { + var dLat = ToRadians(lat2 - lat1); + var dLon = ToRadians(lon2 - lon1); + var a = (Math.Sin(dLat / 2) * Math.Sin(dLat / 2)) + + (Math.Cos(ToRadians(lat1)) * Math.Cos(ToRadians(lat2)) * Math.Sin(dLon / 2) * Math.Sin(dLon / 2)); + return 2 * EarthRadiusKm * Math.Asin(Math.Sqrt(a)); + } + + public static double BearingDeg(double lat1, double lon1, double lat2, double lon2) + { + var y = Math.Sin(ToRadians(lon2 - lon1)) * Math.Cos(ToRadians(lat2)); + var x = (Math.Cos(ToRadians(lat1)) * Math.Sin(ToRadians(lat2))) + - (Math.Sin(ToRadians(lat1)) * Math.Cos(ToRadians(lat2)) * Math.Cos(ToRadians(lon2 - lon1))); + return (Math.Atan2(y, x) * 180 / Math.PI + 360) % 360; + } + + public static (double Lat, double Lon) Offset(double lat, double lon, double bearingDeg, double distanceKm) + { + var bearing = ToRadians(bearingDeg); + var dLat = distanceKm * Math.Cos(bearing) / EarthRadiusKm; + var dLon = distanceKm * Math.Sin(bearing) / (EarthRadiusKm * Math.Cos(ToRadians(lat))); + return (lat + (dLat * 180 / Math.PI), lon + (dLon * 180 / Math.PI)); + } + + private static double ToRadians(double degrees) => degrees * Math.PI / 180; +} diff --git a/fleet-telemetry/src/FleetTelemetry.Core/Simulation/SimulationModel.cs b/fleet-telemetry/src/FleetTelemetry.Core/Simulation/SimulationModel.cs new file mode 100644 index 0000000..ec19d12 --- /dev/null +++ b/fleet-telemetry/src/FleetTelemetry.Core/Simulation/SimulationModel.cs @@ -0,0 +1,34 @@ +using FleetTelemetry.Telemetry; + +namespace FleetTelemetry.Simulation; + +/// A vehicle from the fleet registry, with the depot it starts each day from. +public sealed record SimulatedVehicle( + string VehicleId, + string VehicleClass, + double FuelTankL, + double DepotLat, + double DepotLon); + +/// What the simulator knows about a trip it drove: first and last ping, odometer distance. +public sealed record TripTruth( + DateTimeOffset StartedAt, + DateTimeOffset EndedAt, + double DistanceKm, + double MaxSpeedKmh, + int Samples); + +/// A planned hard stop: the speed dropped from FromKmh to ToKmh at At. +public sealed record HarshBrakeTruth(DateTimeOffset At, double FromKmh, double ToKmh); + +/// A planned wait with the engine running (a loading dock), first to last stationary ping. +public sealed record IdlingTruth(DateTimeOffset StartedAt, DateTimeOffset EndedAt); + +/// One vehicle on one UTC day: the pings it sent and the ground truth behind them. +public sealed record VehicleDay( + string VehicleId, + DateOnly Date, + IReadOnlyList Pings, + IReadOnlyList Trips, + IReadOnlyList HarshBrakes, + IReadOnlyList Idling); diff --git a/fleet-telemetry/src/FleetTelemetry.Core/Simulation/TelemetryExport.cs b/fleet-telemetry/src/FleetTelemetry.Core/Simulation/TelemetryExport.cs new file mode 100644 index 0000000..5adf18d --- /dev/null +++ b/fleet-telemetry/src/FleetTelemetry.Core/Simulation/TelemetryExport.cs @@ -0,0 +1,39 @@ +using System.Text.Json; +using FleetTelemetry.Telemetry; + +namespace FleetTelemetry.Simulation; + +/// A written export file and the number of pings in it. +public sealed record ExportFile(string Path, int Lines); + +/// +/// Writes telemetry the way vehicle gateways export it: NDJSON, one file per vehicle per +/// UTC day, named yyyy-MM-dd_VEHICLE.ndjson. +/// +public static class TelemetryExport +{ + public static string FileName(DateOnly day, string vehicleId) => $"{day:yyyy-MM-dd}_{vehicleId}.ndjson"; + + public static IReadOnlyList WriteNdjson(string directory, IEnumerable vehicleDays) + { + Directory.CreateDirectory(directory); + return vehicleDays + .OrderBy(d => d.Date) + .ThenBy(d => d.VehicleId, StringComparer.Ordinal) + .Select(day => + { + var path = Path.Combine(directory, FileName(day.Date, day.VehicleId)); + using var writer = new StreamWriter(path) { NewLine = "\n" }; + foreach (var ping in day.Pings) + { + writer.WriteLine(JsonSerializer.Serialize(ping, TelemetryJson.Options)); + } + + return new ExportFile(path, day.Pings.Count); + }) + .ToList(); + } + + public static string ToJsonArray(IEnumerable pings) => + JsonSerializer.Serialize(pings, TelemetryJson.Options); +} diff --git a/fleet-telemetry/src/FleetTelemetry.Core/Telemetry/TelemetryBatchParser.cs b/fleet-telemetry/src/FleetTelemetry.Core/Telemetry/TelemetryBatchParser.cs new file mode 100644 index 0000000..4f0fb36 --- /dev/null +++ b/fleet-telemetry/src/FleetTelemetry.Core/Telemetry/TelemetryBatchParser.cs @@ -0,0 +1,168 @@ +using System.Globalization; +using System.Text.Json; +using System.Text.RegularExpressions; + +namespace FleetTelemetry.Telemetry; + +/// Outcome of parsing a POST /api/telemetry body. +public abstract record TelemetryBatchResult +{ + private TelemetryBatchResult() + { + } + + /// The body is not a JSON array of objects. + public sealed record Malformed(string Detail) : TelemetryBatchResult; + + /// The body is a JSON array but some pings break the rules; keys are like [3].speed_kmh. + public sealed record Invalid(IReadOnlyDictionary Errors) : TelemetryBatchResult; + + public sealed record Valid(IReadOnlyList Pings) : TelemetryBatchResult; +} + +/// +/// Validates a batch of pings before it is written to Snowflake. A batch is accepted or +/// rejected as a whole, so a device can safely retry it. +/// +public static partial class TelemetryBatchParser +{ + public const int MaxBatchSize = 1000; + + /// Devices may buffer while offline, but their clock must not run ahead of ours. + public static readonly TimeSpan MaxClockSkew = TimeSpan.FromMinutes(5); + + public static TelemetryBatchResult Parse(string body, DateTimeOffset receivedAt) + { + JsonDocument document; + try + { + document = JsonDocument.Parse(body); + } + catch (JsonException e) + { + return new TelemetryBatchResult.Malformed($"The request body is not valid JSON: {e.Message}"); + } + + using (document) + { + if (document.RootElement.ValueKind != JsonValueKind.Array) + { + return new TelemetryBatchResult.Malformed("The request body must be a JSON array of telemetry pings."); + } + + var count = document.RootElement.GetArrayLength(); + if (count is 0 or > MaxBatchSize) + { + return new TelemetryBatchResult.Invalid(new Dictionary + { + ["$"] = [$"A batch must contain between 1 and {MaxBatchSize} pings, got {count}."], + }); + } + + var errors = new Dictionary(); + var pings = new List(count); + var index = 0; + foreach (var element in document.RootElement.EnumerateArray()) + { + var prefix = $"[{index.ToString(CultureInfo.InvariantCulture)}]"; + try + { + var ping = element.Deserialize(TelemetryJson.Options); + if (ping is null) + { + errors[prefix] = ["A ping must be a JSON object."]; + } + else + { + foreach (var (field, message) in Validate(ping, receivedAt)) + { + errors[$"{prefix}.{field}"] = [message]; + } + + pings.Add(ping); + } + } + catch (JsonException e) + { + errors[prefix] = [e.Message]; + } + + index++; + } + + return errors.Count > 0 + ? new TelemetryBatchResult.Invalid(errors) + : new TelemetryBatchResult.Valid(pings); + } + } + + /// Field rules for a single ping. Field names use the wire format. + public static IEnumerable<(string Field, string Message)> Validate(TelemetryPing ping, DateTimeOffset receivedAt) + { + ArgumentNullException.ThrowIfNull(ping); + + if (string.IsNullOrWhiteSpace(ping.MessageId) || ping.MessageId.Length > 64) + { + yield return ("message_id", "Must be 1 to 64 characters."); + } + + if (!VehicleIdPattern().IsMatch(ping.VehicleId)) + { + yield return ("vehicle_id", "Must look like TRK-101 or VAN-206."); + } + + if (ping.Ts > receivedAt + MaxClockSkew) + { + yield return ("ts", "Is in the future."); + } + + if (ping.Gps.Lat is < -90 or > 90 || double.IsNaN(ping.Gps.Lat)) + { + yield return ("gps.lat", "Must be between -90 and 90."); + } + + if (ping.Gps.Lon is < -180 or > 180 || double.IsNaN(ping.Gps.Lon)) + { + yield return ("gps.lon", "Must be between -180 and 180."); + } + + if (ping.Gps.HeadingDeg is < 0 or > 359) + { + yield return ("gps.heading_deg", "Must be between 0 and 359."); + } + + if (ping.SpeedKmh is < 0 or > 200 || double.IsNaN(ping.SpeedKmh)) + { + yield return ("speed_kmh", "Must be between 0 and 200."); + } + + if (ping.OdometerKm < 0 || double.IsNaN(ping.OdometerKm)) + { + yield return ("odometer_km", "Must not be negative."); + } + + if (ping.Fuel.LevelPct is < 0 or > 100 || double.IsNaN(ping.Fuel.LevelPct)) + { + yield return ("fuel.level_pct", "Must be between 0 and 100."); + } + + if (ping.Fuel.UsedL < 0 || double.IsNaN(ping.Fuel.UsedL)) + { + yield return ("fuel.used_l", "Must not be negative."); + } + + for (var i = 0; i < ping.Engine.Dtc.Count; i++) + { + if (!DtcPattern().IsMatch(ping.Engine.Dtc[i].Code)) + { + yield return ($"engine.dtc[{i.ToString(CultureInfo.InvariantCulture)}].code", "Must be an OBD-II code such as P0301."); + } + } + } + + [GeneratedRegex("^[A-Z]{3}-[0-9]{3}$")] + private static partial Regex VehicleIdPattern(); + + [GeneratedRegex("^[PCBU][0-9A-F]{4}$")] + private static partial Regex DtcPattern(); +} diff --git a/fleet-telemetry/src/FleetTelemetry.Core/Telemetry/TelemetryJson.cs b/fleet-telemetry/src/FleetTelemetry.Core/Telemetry/TelemetryJson.cs new file mode 100644 index 0000000..5411691 --- /dev/null +++ b/fleet-telemetry/src/FleetTelemetry.Core/Telemetry/TelemetryJson.cs @@ -0,0 +1,29 @@ +using System.Globalization; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace FleetTelemetry.Telemetry; + +/// The device wire format: snake_case properties, required fields enforced. +public static class TelemetryJson +{ + public static JsonSerializerOptions Options { get; } = new(JsonSerializerDefaults.Web) + { + PropertyNamingPolicy = JsonNamingPolicy.SnakeCaseLower, + RespectNullableAnnotations = true, + RespectRequiredConstructorParameters = true, + NumberHandling = JsonNumberHandling.Strict, + WriteIndented = false, + Converters = { new UtcTimestampConverter() }, + }; + + /// Reads any ISO 8601 offset; writes UTC with a Z suffix, as devices do. + private sealed class UtcTimestampConverter : JsonConverter + { + public override DateTimeOffset Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) => + reader.GetDateTimeOffset(); + + public override void Write(Utf8JsonWriter writer, DateTimeOffset value, JsonSerializerOptions options) => + writer.WriteStringValue(value.UtcDateTime.ToString("yyyy-MM-dd'T'HH:mm:ss.FFFFFFF'Z'", CultureInfo.InvariantCulture)); + } +} diff --git a/fleet-telemetry/src/FleetTelemetry.Core/Telemetry/TelemetryPing.cs b/fleet-telemetry/src/FleetTelemetry.Core/Telemetry/TelemetryPing.cs new file mode 100644 index 0000000..00cd5e6 --- /dev/null +++ b/fleet-telemetry/src/FleetTelemetry.Core/Telemetry/TelemetryPing.cs @@ -0,0 +1,24 @@ +namespace FleetTelemetry.Telemetry; + +/// +/// One telemetry message as a vehicle sends it. Serialized with +/// (snake_case), which is also the shape stored in the landing table. +/// +public sealed record TelemetryPing( + string MessageId, + string VehicleId, + DateTimeOffset Ts, + GpsFix Gps, + double SpeedKmh, + double OdometerKm, + FuelReading Fuel, + EngineReading Engine); + +public sealed record GpsFix(double Lat, double Lon, int HeadingDeg); + +public sealed record FuelReading(double LevelPct, double UsedL); + +/// Engine diagnostics, including the active diagnostic trouble codes (DTCs). +public sealed record EngineReading(int Rpm, double CoolantC, IReadOnlyList Dtc); + +public sealed record FaultCode(string Code, string Severity); diff --git a/fleet-telemetry/src/FleetTelemetry.Core/packages.lock.json b/fleet-telemetry/src/FleetTelemetry.Core/packages.lock.json new file mode 100644 index 0000000..6afd678 --- /dev/null +++ b/fleet-telemetry/src/FleetTelemetry.Core/packages.lock.json @@ -0,0 +1,6 @@ +{ + "version": 2, + "dependencies": { + "net10.0": {} + } +} \ No newline at end of file diff --git a/fleet-telemetry/src/FleetTelemetry.Data/AnalyticsRows.cs b/fleet-telemetry/src/FleetTelemetry.Data/AnalyticsRows.cs new file mode 100644 index 0000000..7142e56 --- /dev/null +++ b/fleet-telemetry/src/FleetTelemetry.Data/AnalyticsRows.cs @@ -0,0 +1,67 @@ +namespace FleetTelemetry.Data; + +// Result rows as Snowflake returns them. Timestamps are UTC (TIMESTAMP_NTZ). + +public sealed class TripRow +{ + public long TripNo { get; init; } + + public DateTime StartedAt { get; init; } + + public DateTime EndedAt { get; init; } + + public long DurationS { get; init; } + + public double DistanceKm { get; init; } + + public double FuelUsedL { get; init; } + + public double MaxSpeedKmh { get; init; } + + public long IdleS { get; init; } + + public long Samples { get; init; } +} + +public sealed class HarshBrakingRow +{ + public DateTime OccurredAt { get; init; } + + public double Lat { get; init; } + + public double Lon { get; init; } + + public double FromSpeedKmh { get; init; } + + public double ToSpeedKmh { get; init; } + + public double DecelerationMs2 { get; init; } +} + +public sealed class IdlingRow +{ + public DateTime StartedAt { get; init; } + + public DateTime EndedAt { get; init; } + + public long DurationS { get; init; } + + public double Lat { get; init; } + + public double Lon { get; init; } +} + +public sealed class FaultRow +{ + public string Code { get; init; } = ""; + + public string Severity { get; init; } = ""; + + public long Occurrences { get; init; } + + public DateTime FirstSeen { get; init; } + + public DateTime LastSeen { get; init; } + + public double MaxCoolantC { get; init; } +} diff --git a/fleet-telemetry/src/FleetTelemetry.Data/FleetAnalytics.cs b/fleet-telemetry/src/FleetTelemetry.Data/FleetAnalytics.cs new file mode 100644 index 0000000..76ad969 --- /dev/null +++ b/fleet-telemetry/src/FleetTelemetry.Data/FleetAnalytics.cs @@ -0,0 +1,57 @@ +using Dapper; +using FleetTelemetry.Kpis; + +namespace FleetTelemetry.Data; + +/// Fleet-wide KPIs, read from the FLEET_DAILY_KPIS dynamic table. +public sealed class FleetAnalytics(FleetDatabase database) +{ + public const string DailyKpisTable = "FLEET_DAILY_KPIS"; + + public async Task> GetVehicleDaysAsync( + DateOnly from, DateOnly to, CancellationToken cancellationToken) + { + await using var connection = await database.OpenAsync(cancellationToken); + var rows = await connection.QueryAsync(new CommandDefinition( + $""" + SELECT DAY, VEHICLE_ID, SAMPLES, IDLE_SAMPLES, DISTANCE_KM, FUEL_USED_L, MAX_SPEED_KMH, FAULT_SAMPLES + FROM {DailyKpisTable} + WHERE DAY BETWEEN :fromDay AND :toDay + ORDER BY DAY, VEHICLE_ID + """, + new { fromDay = from.ToDateTime(TimeOnly.MinValue), toDay = to.ToDateTime(TimeOnly.MinValue) }, + cancellationToken: cancellationToken)); + return rows + .Select(r => new VehicleDayKpi( + DateOnly.FromDateTime(r.Day), r.VehicleId, r.Samples, r.IdleSamples, + r.DistanceKm, r.FuelUsedL, r.MaxSpeedKmh, r.FaultSamples)) + .ToList(); + } + + /// Brings the dynamic table up to date now instead of waiting for its target lag. + public async Task RefreshAsync(CancellationToken cancellationToken) + { + await using var connection = await database.OpenAsync(cancellationToken); + await connection.ExecuteAsync(new CommandDefinition( + $"ALTER DYNAMIC TABLE {DailyKpisTable} REFRESH", cancellationToken: cancellationToken)); + } + + private sealed class VehicleDayRow + { + public DateTime Day { get; init; } + + public string VehicleId { get; init; } = ""; + + public long Samples { get; init; } + + public long IdleSamples { get; init; } + + public double DistanceKm { get; init; } + + public double FuelUsedL { get; init; } + + public double MaxSpeedKmh { get; init; } + + public long FaultSamples { get; init; } + } +} diff --git a/fleet-telemetry/src/FleetTelemetry.Data/FleetDatabase.cs b/fleet-telemetry/src/FleetTelemetry.Data/FleetDatabase.cs new file mode 100644 index 0000000..2728378 --- /dev/null +++ b/fleet-telemetry/src/FleetTelemetry.Data/FleetDatabase.cs @@ -0,0 +1,77 @@ +using System.Data.Common; +using System.Text; +using Dapper; +using Microsoft.Extensions.Options; +using Snowflake.Data.Client; + +namespace FleetTelemetry.Data; + +/// Describes the unit of work running queries, for Snowflake's QUERY_TAG. +public interface IQueryTagSource +{ + /// Short, stable description of the operation, such as an HTTP route or a command name. + string Operation { get; } + + /// Correlation id of the request or run, or null. + string? CorrelationId { get; } +} + +/// A fixed query tag, for command-line tools. +public sealed class FixedQueryTag(string operation) : IQueryTagSource +{ + public string Operation { get; } = operation; + + public string? CorrelationId { get; } = Guid.NewGuid().ToString("N"); +} + +/// +/// Opens pooled Snowflake sessions. Every session is tagged with the application, the +/// operation and a correlation id, so a slow query in Snowflake's query history leads +/// straight back to the request that ran it. +/// +public sealed class FleetDatabase(IOptions options, IQueryTagSource tagSource) +{ + static FleetDatabase() + { + // Snowflake returns upper-case SNAKE_CASE column names; map them to PascalCase properties. + DefaultTypeMap.MatchNamesWithUnderscores = true; + } + + private readonly string _connectionString = options.Value.ToConnectionString(); + + public async Task OpenAsync(CancellationToken cancellationToken) + { + var connection = new SnowflakeDbConnection(_connectionString); + try + { + await connection.OpenAsync(cancellationToken); + var tag = QueryTag(options.Value.Application, tagSource); + await connection.ExecuteAsync(new CommandDefinition( + $"ALTER SESSION SET QUERY_TAG = '{tag}'", cancellationToken: cancellationToken)); + return connection; + } + catch + { + await connection.DisposeAsync(); + throw; + } + } + + /// A JSON query tag. Values are reduced to a safe character set, so the literal needs no escaping. + internal static string QueryTag(string application, IQueryTagSource source) + { + var tag = new StringBuilder("{\"app\":\"").Append(Safe(application)) + .Append("\",\"op\":\"").Append(Safe(source.Operation)).Append('"'); + if (source.CorrelationId is { } id) + { + tag.Append(",\"id\":\"").Append(Safe(id)).Append('"'); + } + + return tag.Append('}').ToString(); + + static string Safe(string value) => new(value + .Where(c => char.IsAsciiLetterOrDigit(c) || c is ' ' or '/' or '{' or '}' or '-' or '_' or '.' or ':') + .Take(200) + .ToArray()); + } +} diff --git a/fleet-telemetry/src/FleetTelemetry.Data/FleetRegistry.cs b/fleet-telemetry/src/FleetTelemetry.Data/FleetRegistry.cs new file mode 100644 index 0000000..01bf19b --- /dev/null +++ b/fleet-telemetry/src/FleetTelemetry.Data/FleetRegistry.cs @@ -0,0 +1,47 @@ +using Dapper; +using FleetTelemetry.Simulation; + +namespace FleetTelemetry.Data; + +/// Reads the fleet registry (VEHICLES and DEPOTS). +public sealed class FleetRegistry(FleetDatabase database) +{ + public async Task VehicleExistsAsync(string vehicleId, CancellationToken cancellationToken) + { + await using var connection = await database.OpenAsync(cancellationToken); + return await connection.ExecuteScalarAsync(new CommandDefinition( + "SELECT COUNT(*) FROM VEHICLES WHERE VEHICLE_ID = :vehicleId", + new { vehicleId }, + cancellationToken: cancellationToken)) > 0; + } + + /// Every vehicle with its depot location, in vehicle id order. + public async Task> ListVehiclesAsync(CancellationToken cancellationToken) + { + await using var connection = await database.OpenAsync(cancellationToken); + var rows = await connection.QueryAsync(new CommandDefinition( + """ + SELECT v.VEHICLE_ID, v.VEHICLE_CLASS, v.FUEL_TANK_L::FLOAT AS FUEL_TANK_L, + d.LAT AS DEPOT_LAT, d.LON AS DEPOT_LON + FROM VEHICLES v + JOIN DEPOTS d ON d.DEPOT_ID = v.DEPOT_ID + ORDER BY v.VEHICLE_ID + """, + cancellationToken: cancellationToken)); + return rows.Select(r => new SimulatedVehicle(r.VehicleId, r.VehicleClass, r.FuelTankL, r.DepotLat, r.DepotLon)) + .ToList(); + } + + private sealed class VehicleRow + { + public string VehicleId { get; init; } = ""; + + public string VehicleClass { get; init; } = ""; + + public double FuelTankL { get; init; } + + public double DepotLat { get; init; } + + public double DepotLon { get; init; } + } +} diff --git a/fleet-telemetry/src/FleetTelemetry.Data/FleetTelemetry.Data.csproj b/fleet-telemetry/src/FleetTelemetry.Data/FleetTelemetry.Data.csproj new file mode 100644 index 0000000..c009c9a --- /dev/null +++ b/fleet-telemetry/src/FleetTelemetry.Data/FleetTelemetry.Data.csproj @@ -0,0 +1,20 @@ + + + + + FleetTelemetry.Data + + + + + + + + + + + + + + + diff --git a/fleet-telemetry/src/FleetTelemetry.Data/ServiceCollectionExtensions.cs b/fleet-telemetry/src/FleetTelemetry.Data/ServiceCollectionExtensions.cs new file mode 100644 index 0000000..49bcf84 --- /dev/null +++ b/fleet-telemetry/src/FleetTelemetry.Data/ServiceCollectionExtensions.cs @@ -0,0 +1,24 @@ +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Options; + +namespace FleetTelemetry.Data; + +public static class ServiceCollectionExtensions +{ + /// + /// Registers the Snowflake data access. The caller registers an + /// describing its unit of work (an HTTP request, a command-line run). + /// + public static IServiceCollection AddFleetSnowflake(this IServiceCollection services) + { + services.AddOptions().BindConfiguration(SnowflakeOptions.Section).ValidateOnStart(); + services.AddSingleton, SnowflakeOptionsValidator>(); + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); + services.AddScoped(); + return services; + } +} diff --git a/fleet-telemetry/src/FleetTelemetry.Data/SnowflakeOptions.cs b/fleet-telemetry/src/FleetTelemetry.Data/SnowflakeOptions.cs new file mode 100644 index 0000000..4a2e3c9 --- /dev/null +++ b/fleet-telemetry/src/FleetTelemetry.Data/SnowflakeOptions.cs @@ -0,0 +1,121 @@ +using System.Data.Common; +using System.Globalization; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.Options; + +namespace FleetTelemetry.Data; + +/// +/// Connection settings, bound from the Snowflake configuration section. The same +/// shape serves a local emulator and a Snowflake account; only account, host, port and +/// key differ (see appsettings.json and appsettings.Development.json). +/// +public sealed class SnowflakeOptions +{ + public const string Section = "Snowflake"; + + public string Account { get; set; } = ""; + + public string Host { get; set; } = ""; + + public int Port { get; set; } = 443; + + public string User { get; set; } = ""; + + /// Unencrypted PKCS#8 PEM file for key-pair (JWT) authentication. + public string PrivateKeyPath { get; set; } = ""; + + public string Role { get; set; } = ""; + + public string Warehouse { get; set; } = ""; + + public string Database { get; set; } = ""; + + public string Schema { get; set; } = ""; + + /// Shown in every query tag, so Snowflake's query history says which app ran a query. + public string Application { get; set; } = "fleet-telemetry"; + + public string ToConnectionString() + { + var builder = new DbConnectionStringBuilder + { + ["account"] = Account, + ["host"] = Host, + ["port"] = Port.ToString(CultureInfo.InvariantCulture), + ["scheme"] = "https", + ["user"] = User, + ["authenticator"] = "snowflake_jwt", + ["private_key_file"] = Path.GetFullPath(PrivateKeyPath), + ["role"] = Role, + }; + foreach (var (key, value) in new[] { ("warehouse", Warehouse), ("db", Database), ("schema", Schema) }) + { + if (!string.IsNullOrEmpty(value)) + { + builder[key] = value; + } + } + + return builder.ConnectionString; + } +} + +internal sealed class SnowflakeOptionsValidator : IValidateOptions +{ + public ValidateOptionsResult Validate(string? name, SnowflakeOptions options) + { + var missing = new (string Key, string Value)[] + { + (nameof(options.Account), options.Account), + (nameof(options.Host), options.Host), + (nameof(options.User), options.User), + (nameof(options.PrivateKeyPath), options.PrivateKeyPath), + (nameof(options.Role), options.Role), + (nameof(options.Warehouse), options.Warehouse), + (nameof(options.Database), options.Database), + (nameof(options.Schema), options.Schema), + } + .Where(s => string.IsNullOrWhiteSpace(s.Value)) + .Select(s => $"{SnowflakeOptions.Section}:{s.Key}") + .ToList(); + if (missing.Count > 0) + { + return ValidateOptionsResult.Fail($"Missing Snowflake settings: {string.Join(", ", missing)}."); + } + + return File.Exists(options.PrivateKeyPath) + ? ValidateOptionsResult.Success + : ValidateOptionsResult.Fail( + $"Private key '{Path.GetFullPath(options.PrivateKeyPath)}' not found. Run `make up` to generate a development key."); + } +} + +public static class SnowflakeConfigurationExtensions +{ + private static readonly (string Variable, string Key)[] EnvironmentMapping = + [ + ("SNOWFLAKE_ACCOUNT", nameof(SnowflakeOptions.Account)), + ("SNOWFLAKE_HOST", nameof(SnowflakeOptions.Host)), + ("SNOWFLAKE_PORT", nameof(SnowflakeOptions.Port)), + ("SNOWFLAKE_USER", nameof(SnowflakeOptions.User)), + ("SNOWFLAKE_PRIVATE_KEY_PATH", nameof(SnowflakeOptions.PrivateKeyPath)), + ("SNOWFLAKE_ROLE", nameof(SnowflakeOptions.Role)), + ("SNOWFLAKE_WAREHOUSE", nameof(SnowflakeOptions.Warehouse)), + ("SNOWFLAKE_DATABASE", nameof(SnowflakeOptions.Database)), + ("SNOWFLAKE_SCHEMA", nameof(SnowflakeOptions.Schema)), + ]; + + /// + /// Lets the conventional SNOWFLAKE_* variables (the ones SnowSQL and the other + /// Snowflake drivers use) override appsettings, next to the .NET form Snowflake__Port. + /// + public static IConfigurationBuilder AddSnowflakeEnvironmentVariables(this IConfigurationBuilder builder) + { + var values = EnvironmentMapping + .Select(m => (m.Key, Value: Environment.GetEnvironmentVariable(m.Variable))) + .Where(m => !string.IsNullOrEmpty(m.Value)) + .ToDictionary(m => $"{SnowflakeOptions.Section}:{m.Key}", m => m.Value); + return builder.AddInMemoryCollection(values); + } +} diff --git a/fleet-telemetry/src/FleetTelemetry.Data/TelemetryBackfill.cs b/fleet-telemetry/src/FleetTelemetry.Data/TelemetryBackfill.cs new file mode 100644 index 0000000..bc298fb --- /dev/null +++ b/fleet-telemetry/src/FleetTelemetry.Data/TelemetryBackfill.cs @@ -0,0 +1,138 @@ +using System.Data.Common; +using System.Globalization; +using Microsoft.Extensions.Logging; + +namespace FleetTelemetry.Data; + +/// Result of uploading one local file with PUT. +public sealed record UploadedFile(string File, string Status); + +/// COPY INTO result for one staged file. +public sealed record LoadedFile( + string File, + string Status, + long RowsParsed, + long RowsLoaded, + long ErrorsSeen, + string? FirstError, + long? FirstErrorLine); + +public sealed record BackfillReport(IReadOnlyList Uploaded, IReadOnlyList Loaded) +{ + public long RowsLoaded => Loaded.Sum(f => f.RowsLoaded); + + public long RowsFailed => Loaded.Sum(f => f.ErrorsSeen); +} + +/// +/// Bulk-loads NDJSON exports: PUT uploads them to the internal stage (compressed), COPY INTO +/// loads every file not loaded before, skipping bad lines (ON_ERROR = CONTINUE) and +/// reporting them per file. Snowflake's load history makes a re-run load nothing twice. +/// +public sealed partial class TelemetryBackfill(FleetDatabase database, FleetAnalytics analytics, ILogger logger) +{ + public const string StagePath = "@TELEMETRY_STAGE/backfill/"; + + public async Task RunAsync(string directory, CancellationToken cancellationToken) + { + var fullPath = Path.GetFullPath(directory); + if (!Directory.EnumerateFiles(fullPath, "*.ndjson").Any()) + { + throw new FileNotFoundException($"No *.ndjson files in {fullPath}."); + } + + await using var connection = await database.OpenAsync(cancellationToken); + + // The driver runs PUT synchronously: it uploads the files itself. + var uploaded = await Task.Run( + () => ReadRows( + connection, + $"PUT 'file://{fullPath.Replace('\\', '/')}/*.ndjson' {StagePath} AUTO_COMPRESS = TRUE", + r => new UploadedFile(r.GetString(r.GetOrdinal("SourceFileName")), r.GetString(r.GetOrdinal("ResultStatus")))), + cancellationToken); + foreach (var file in uploaded) + { + LogUploaded(file.File, file.Status); + } + + var loaded = await Task.Run( + () => ReadRows( + connection, + $""" + COPY INTO TELEMETRY_RAW + FROM {StagePath} + FILE_FORMAT = (FORMAT_NAME = 'TELEMETRY_NDJSON') + ON_ERROR = CONTINUE + """, + ReadCopyResult), + cancellationToken); + var report = new BackfillReport(uploaded, loaded.OfType().ToList()); + foreach (var file in report.Loaded) + { + if (file.ErrorsSeen > 0) + { + LogLoadedWithErrors(file.File, file.Status, file.RowsLoaded, file.RowsParsed, file.FirstErrorLine, file.FirstError); + } + else + { + LogLoaded(file.File, file.Status, file.RowsLoaded); + } + } + + if (report.Loaded.Count == 0) + { + LogNothingToLoad(); + } + + await analytics.RefreshAsync(cancellationToken); + return report; + } + + [LoggerMessage(Level = LogLevel.Information, Message = "PUT {File}: {Status}")] + private partial void LogUploaded(string file, string status); + + [LoggerMessage(Level = LogLevel.Information, Message = "COPY {File}: {Status}, {RowsLoaded} rows loaded")] + private partial void LogLoaded(string file, string status, long rowsLoaded); + + [LoggerMessage( + Level = LogLevel.Warning, + Message = "COPY {File}: {Status}, {RowsLoaded} of {RowsParsed} rows loaded, first error at line {Line}: {Error}")] + private partial void LogLoadedWithErrors(string file, string status, long rowsLoaded, long rowsParsed, long? line, string? error); + + [LoggerMessage(Level = LogLevel.Information, Message = "COPY: every staged file was already loaded")] + private partial void LogNothingToLoad(); + + /// One row per file, or a single status row ("0 files processed") when there was nothing new. + private static LoadedFile? ReadCopyResult(DbDataReader r) + { + if (r.FieldCount == 1) + { + return null; + } + + return new LoadedFile( + File: Path.GetFileName(r.GetString(r.GetOrdinal("file"))), + Status: r.GetString(r.GetOrdinal("status")), + RowsParsed: Convert.ToInt64(r.GetValue(r.GetOrdinal("rows_parsed")), CultureInfo.InvariantCulture), + RowsLoaded: Convert.ToInt64(r.GetValue(r.GetOrdinal("rows_loaded")), CultureInfo.InvariantCulture), + ErrorsSeen: Convert.ToInt64(r.GetValue(r.GetOrdinal("errors_seen")), CultureInfo.InvariantCulture), + FirstError: r.IsDBNull(r.GetOrdinal("first_error")) ? null : r.GetString(r.GetOrdinal("first_error")), + FirstErrorLine: r.IsDBNull(r.GetOrdinal("first_error_line")) + ? null + : Convert.ToInt64(r.GetValue(r.GetOrdinal("first_error_line")), CultureInfo.InvariantCulture)); + } + + private static List ReadRows(DbConnection connection, string sql, Func map) + { + using var command = connection.CreateCommand(); + command.CommandText = sql; + using var reader = command.ExecuteReader(); + var rows = new List(); + while (reader.Read()) + { + rows.Add(map(reader)); + } + + return rows; + } +} diff --git a/fleet-telemetry/src/FleetTelemetry.Data/TelemetryWriter.cs b/fleet-telemetry/src/FleetTelemetry.Data/TelemetryWriter.cs new file mode 100644 index 0000000..b922975 --- /dev/null +++ b/fleet-telemetry/src/FleetTelemetry.Data/TelemetryWriter.cs @@ -0,0 +1,23 @@ +using Dapper; + +namespace FleetTelemetry.Data; + +/// Writes pings received over HTTP into the landing table. +public sealed class TelemetryWriter(FleetDatabase database) +{ + /// + /// Inserts a validated JSON array of pings in one statement: the whole batch is one bound + /// parameter, parsed and split server-side, so a batch lands completely or not at all. + /// + public async Task InsertBatchAsync(string pingsJsonArray, CancellationToken cancellationToken) + { + await using var connection = await database.OpenAsync(cancellationToken); + return await connection.ExecuteAsync(new CommandDefinition( + """ + INSERT INTO TELEMETRY_RAW (PAYLOAD) + SELECT p.value FROM TABLE(FLATTEN(INPUT => PARSE_JSON(:pings))) p + """, + new { pings = pingsJsonArray }, + cancellationToken: cancellationToken)); + } +} diff --git a/fleet-telemetry/src/FleetTelemetry.Data/VehicleAnalytics.cs b/fleet-telemetry/src/FleetTelemetry.Data/VehicleAnalytics.cs new file mode 100644 index 0000000..941ea3a --- /dev/null +++ b/fleet-telemetry/src/FleetTelemetry.Data/VehicleAnalytics.cs @@ -0,0 +1,156 @@ +using Dapper; +using FleetTelemetry.Rules; + +namespace FleetTelemetry.Data; + +/// Per-vehicle analytics, computed in Snowflake from the TELEMETRY view. +public sealed class VehicleAnalytics(FleetDatabase database) +{ + /// + /// Trips on one UTC day. A gap of more than between + /// consecutive pings starts a new trip: LAG finds the gaps, a running SUM numbers the trips. + /// + public async Task> GetTripsAsync(string vehicleId, DateOnly day, CancellationToken cancellationToken) + { + const string sql = """ + WITH pings AS ( + SELECT TS, SPEED_KMH, ODOMETER_KM, FUEL_USED_L, + DATEDIFF('second', LAG(TS) OVER (ORDER BY TS), TS) AS GAP_S, + LAG(SPEED_KMH) OVER (ORDER BY TS) AS PREV_SPEED_KMH + FROM TELEMETRY + WHERE VEHICLE_ID = :vehicleId AND TS >= :dayStart AND TS < :dayEnd + ), + numbered AS ( + SELECT *, + SUM(IFF(GAP_S IS NULL OR GAP_S > :maxGapS, 1, 0)) + OVER (ORDER BY TS ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS TRIP_NO + FROM pings + ) + SELECT TRIP_NO, + MIN(TS) AS STARTED_AT, + MAX(TS) AS ENDED_AT, + DATEDIFF('second', MIN(TS), MAX(TS)) AS DURATION_S, + MAX(ODOMETER_KM) - MIN(ODOMETER_KM) AS DISTANCE_KM, + MAX(FUEL_USED_L) - MIN(FUEL_USED_L) AS FUEL_USED_L, + MAX(SPEED_KMH) AS MAX_SPEED_KMH, + SUM(IFF(GAP_S <= :maxGapS AND PREV_SPEED_KMH = 0 AND SPEED_KMH = 0, GAP_S, 0)) AS IDLE_S, + COUNT(*) AS SAMPLES + FROM numbered + GROUP BY TRIP_NO + ORDER BY TRIP_NO + """; + var dayStart = day.ToDateTime(TimeOnly.MinValue); + await using var connection = await database.OpenAsync(cancellationToken); + var rows = await connection.QueryAsync(new CommandDefinition( + sql, + new { vehicleId, dayStart, dayEnd = dayStart.AddDays(1), maxGapS = (long)TripRules.MaxPingGap.TotalSeconds }, + cancellationToken: cancellationToken)); + return rows.ToList(); + } + + /// Consecutive pings whose speed dropped by at least . + public async Task> GetHarshBrakingAsync( + string vehicleId, DateTime from, DateTime to, CancellationToken cancellationToken) + { + const string sql = """ + WITH pings AS ( + SELECT TS, LAT, LON, SPEED_KMH, + LAG(SPEED_KMH) OVER (ORDER BY TS) AS PREV_SPEED_KMH, + DATEDIFF('second', LAG(TS) OVER (ORDER BY TS), TS) AS GAP_S + FROM TELEMETRY + WHERE VEHICLE_ID = :vehicleId AND TS >= :fromTs AND TS < :toTs + ) + SELECT TS AS OCCURRED_AT, LAT, LON, + PREV_SPEED_KMH AS FROM_SPEED_KMH, + SPEED_KMH AS TO_SPEED_KMH, + (PREV_SPEED_KMH - SPEED_KMH) / 3.6 / GAP_S AS DECELERATION_MS2 + FROM pings + WHERE GAP_S > 0 + AND (PREV_SPEED_KMH - SPEED_KMH) / 3.6 / GAP_S >= :thresholdMs2 + ORDER BY TS + """; + await using var connection = await database.OpenAsync(cancellationToken); + var rows = await connection.QueryAsync(new CommandDefinition( + sql, + new + { + vehicleId, + fromTs = from, + toTs = to, + thresholdMs2 = DrivingEventRules.HarshBrakingMs2, + }, + cancellationToken: cancellationToken)); + return rows.ToList(); + } + + /// + /// Stretches of at least standing still with the + /// engine running. Consecutive stationary pings form an island (gaps and islands). + /// + public async Task> GetIdlingAsync( + string vehicleId, DateTime from, DateTime to, CancellationToken cancellationToken) + { + const string sql = """ + WITH pings AS ( + SELECT TS, LAT, LON, SPEED_KMH, + LAG(SPEED_KMH) OVER (ORDER BY TS) AS PREV_SPEED_KMH, + DATEDIFF('second', LAG(TS) OVER (ORDER BY TS), TS) AS GAP_S + FROM TELEMETRY + WHERE VEHICLE_ID = :vehicleId AND TS >= :fromTs AND TS < :toTs + ), + islands AS ( + SELECT *, + SUM(IFF(SPEED_KMH = 0 AND PREV_SPEED_KMH = 0 AND GAP_S <= :maxGapS, 0, 1)) + OVER (ORDER BY TS ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS ISLAND + FROM pings + ) + SELECT MIN(TS) AS STARTED_AT, + MAX(TS) AS ENDED_AT, + DATEDIFF('second', MIN(TS), MAX(TS)) AS DURATION_S, + ANY_VALUE(LAT) AS LAT, + ANY_VALUE(LON) AS LON + FROM islands + WHERE SPEED_KMH = 0 + GROUP BY ISLAND + HAVING DATEDIFF('second', MIN(TS), MAX(TS)) >= :minIdleS + ORDER BY STARTED_AT + """; + await using var connection = await database.OpenAsync(cancellationToken); + var rows = await connection.QueryAsync(new CommandDefinition( + sql, + new + { + vehicleId, + fromTs = from, + toTs = to, + maxGapS = (long)TripRules.MaxPingGap.TotalSeconds, + minIdleS = (long)DrivingEventRules.MinIdling.TotalSeconds, + }, + cancellationToken: cancellationToken)); + return rows.ToList(); + } + + /// Active fault codes, unnested from each ping's engine.dtc array by the TELEMETRY_FAULTS view. + public async Task> GetFaultsAsync( + string vehicleId, DateTime from, DateTime to, CancellationToken cancellationToken) + { + const string sql = """ + SELECT CODE, + ANY_VALUE(SEVERITY) AS SEVERITY, + COUNT(*) AS OCCURRENCES, + MIN(TS) AS FIRST_SEEN, + MAX(TS) AS LAST_SEEN, + MAX(COOLANT_C) AS MAX_COOLANT_C + FROM TELEMETRY_FAULTS + WHERE VEHICLE_ID = :vehicleId AND TS >= :fromTs AND TS < :toTs + GROUP BY CODE + ORDER BY CODE + """; + await using var connection = await database.OpenAsync(cancellationToken); + var rows = await connection.QueryAsync(new CommandDefinition( + sql, + new { vehicleId, fromTs = from, toTs = to }, + cancellationToken: cancellationToken)); + return rows.ToList(); + } +} diff --git a/fleet-telemetry/src/FleetTelemetry.Data/packages.lock.json b/fleet-telemetry/src/FleetTelemetry.Data/packages.lock.json new file mode 100644 index 0000000..a2fd01a --- /dev/null +++ b/fleet-telemetry/src/FleetTelemetry.Data/packages.lock.json @@ -0,0 +1,402 @@ +{ + "version": 2, + "dependencies": { + "net10.0": { + "Dapper": { + "type": "Direct", + "requested": "[2.1.89, )", + "resolved": "2.1.89", + "contentHash": "1VjFE664wnxvuhlgP8kS4PeFFF4jqD3VkbIHW6Zlc52zuibgcz5ICnIHcQXlTp6pURpPNGLV0ZpNe5x9KePJnA==" + }, + "Microsoft.Extensions.Configuration": { + "type": "Direct", + "requested": "[10.0.13, )", + "resolved": "10.0.13", + "contentHash": "inhujYI/nikhmLxYI6ZEcaCgZ28ipJsKMPPxxr2XWf3vM0Q2P+E9R3hDHABZOLBxzD3gjpgcbSH/+hd2+gvI2A==", + "dependencies": { + "Microsoft.Extensions.Configuration.Abstractions": "10.0.13", + "Microsoft.Extensions.Primitives": "10.0.13" + } + }, + "Microsoft.Extensions.Logging.Abstractions": { + "type": "Direct", + "requested": "[10.0.13, )", + "resolved": "10.0.13", + "contentHash": "aSZEhEig4fvNiw2zs8Z6NyDlniKtq6YjcF3DyHuqTEHcZ2VKA/6itMPZ+dW+yWABWExlDcQwwdbqX1Rcb7W8WQ==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.13" + } + }, + "Microsoft.Extensions.Options.ConfigurationExtensions": { + "type": "Direct", + "requested": "[10.0.13, )", + "resolved": "10.0.13", + "contentHash": "xVO5HhATc09WXuS/RaDmx8ZeWS0mTpT1L0tkEDMSBHtAnNpNsDr0ge0wIO2jinCWWqpq5dL1tRTzpYNVlvyqLw==", + "dependencies": { + "Microsoft.Extensions.Configuration.Abstractions": "10.0.13", + "Microsoft.Extensions.Configuration.Binder": "10.0.13", + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.13", + "Microsoft.Extensions.Options": "10.0.13", + "Microsoft.Extensions.Primitives": "10.0.13" + } + }, + "Snowflake.Data": { + "type": "Direct", + "requested": "[6.4.0, )", + "resolved": "6.4.0", + "contentHash": "9zGrm/bD5h+DVha41TKy7soNDqw1bObAXCz9fQdhMJ0wKbj9IRuZvE+ZUvflBMq2mnjkgbNdiGeKz7qixgqbJA==", + "dependencies": { + "AWSSDK.S3": "4.0.101.4", + "Apache.Arrow": "23.0.0", + "Azure.Storage.Blobs": "12.29.1", + "BouncyCastle.Cryptography": "2.7.0", + "Google.Cloud.Storage.V1": "4.15.0", + "Microsoft.Extensions.Logging": "9.0.5", + "Mono.Unix": "7.1.0-final.1.21458.1", + "Newtonsoft.Json": "13.0.4", + "System.IdentityModel.Tokens.Jwt": "8.21.0", + "TimeZoneConverter": "7.2.0", + "Tomlyn.Signed": "2.10.1" + } + }, + "Apache.Arrow": { + "type": "Transitive", + "resolved": "23.0.0", + "contentHash": "cU4Zm7byFwnb82Ea9ZRvVnq2nJNf/TF9j+rj/MHlvu9RBYMqYtu4sJaR1JmMh3MmSIuwMafVg7PfJxPAMv9dGw==", + "dependencies": { + "Apache.Arrow.Scalars": "23.0.0" + } + }, + "Apache.Arrow.Scalars": { + "type": "Transitive", + "resolved": "23.0.0", + "contentHash": "i53VmQ505ASXsStf7y29Z5qvhN6PGgFGNb6UUU7z1WmbzEgSjKQuIJkKKVbh+K5rtXRiKyr1Ah3Rmvbeb2BEHQ==" + }, + "AWSSDK.Core": { + "type": "Transitive", + "resolved": "4.0.100.8", + "contentHash": "xnuBVLQBmYQXsDZJ9mq2UDSFZm3xgO5oUb4/UR8p0UO7tG6heDhsLLI1NzZhIVAKyfW0tXg9hn9a/ek018TOVw==" + }, + "AWSSDK.S3": { + "type": "Transitive", + "resolved": "4.0.101.4", + "contentHash": "TYFuatWECzCbj/Lu1SsANucgpU/Br5YJ5Padl3YEdNJGfZqAcx3QemJw2BUopAsYTJ3e7TFpWhpNWha3dB9/dw==", + "dependencies": { + "AWSSDK.Core": "[4.0.100.8, 5.0.0)" + } + }, + "Azure.Core": { + "type": "Transitive", + "resolved": "1.55.0", + "contentHash": "c7femvYS/xEUrLP1sNZN+DoQZmx3X+G3ZXtOOz0RL/7cGMCM3JVqepVmRd3AwM4IK8AtTGS4GOigCO+d/zaSsQ==", + "dependencies": { + "Microsoft.Bcl.AsyncInterfaces": "10.0.3", + "Microsoft.Extensions.Configuration.Abstractions": "10.0.3", + "Microsoft.Extensions.Hosting.Abstractions": "10.0.3", + "Microsoft.Identity.Client": "4.83.1", + "Microsoft.Identity.Client.Extensions.Msal": "4.83.1", + "System.ClientModel": "1.11.0", + "System.Memory.Data": "10.0.3" + } + }, + "Azure.Storage.Blobs": { + "type": "Transitive", + "resolved": "12.29.1", + "contentHash": "pWh7xZBto8YcwiO853AB3uijTfVqs9PDte0Bu9/Zd8O9nwKiitWm0Jej1vsNkmYUzeG1552f8vJPjmtW30pBUQ==", + "dependencies": { + "Azure.Core": "1.55.0", + "Azure.Storage.Common": "12.28.0" + } + }, + "Azure.Storage.Common": { + "type": "Transitive", + "resolved": "12.28.0", + "contentHash": "5l8YhNrks38zKLGFW2BBFLwieyamH/xauABSASsdpZv79G0f+n2n22IjkRmUqCmALSupkwRHh2LOhbW3yj5Qgw==", + "dependencies": { + "Azure.Core": "1.55.0", + "System.IO.Hashing": "10.0.3" + } + }, + "BouncyCastle.Cryptography": { + "type": "Transitive", + "resolved": "2.7.0", + "contentHash": "U+12df8UEWHgBi04YVf/Lgi2dy3SItlIYvHjjEVa/BngCQIzDCDRBk50DDByCfDvSbe5pRNFr3b7UrVK2kMcLw==" + }, + "Google.Api.Gax": { + "type": "Transitive", + "resolved": "4.13.1", + "contentHash": "ujDpZk7O2VqAEIqcSlhH0FKzVpGZWly/qcNjHxpNUrL445Tei9PHnu4V1pwW2WDiMDvo3TpL1Bi4DeU525Afrg==", + "dependencies": { + "Microsoft.Bcl.AsyncInterfaces": "6.0.0", + "Newtonsoft.Json": "13.0.4" + } + }, + "Google.Api.Gax.Rest": { + "type": "Transitive", + "resolved": "4.13.1", + "contentHash": "wDfFZXKoHC6sy+Yfub/4cLQOb5ozBVPqDOrxCUAeTfku4d96ILqgzulJSdkPHbiT3oWB3hlYTdJgUwNSJjyo+A==", + "dependencies": { + "Google.Api.Gax": "4.13.1", + "Google.Apis.Auth": "1.73.0", + "Microsoft.Extensions.DependencyInjection.Abstractions": "6.0.0" + } + }, + "Google.Apis": { + "type": "Transitive", + "resolved": "1.74.0", + "contentHash": "Z6hSkmRD6V8/raSBqn/EsPpNop6suaLaGE9DbSY3yRE5G47l4zYqrJgapcLARk2Z3TsQUDmWz+w/tob3wuVt3g==", + "dependencies": { + "Google.Apis.Core": "1.74.0" + } + }, + "Google.Apis.Auth": { + "type": "Transitive", + "resolved": "1.74.0", + "contentHash": "43H4foh3zfWgboK8QnJjIfPQuBiS794TxqQLDfeE3erQSluKmEN0qeDuwxLTpSr9Mtwj76S4LWxPneuSbC6fNg==", + "dependencies": { + "Google.Apis": "1.74.0", + "Google.Apis.Core": "1.74.0", + "System.Management": "7.0.2" + } + }, + "Google.Apis.Core": { + "type": "Transitive", + "resolved": "1.74.0", + "contentHash": "gWOIiks6yo2TRliIyy10V18BJOO2BfUSPJFiTW6Tfap4tm3hFj7nvYU3XIfwzAxwoIy6nsBoqVDt9iBDSXrvYQ==", + "dependencies": { + "Newtonsoft.Json": "13.0.4" + } + }, + "Google.Apis.Storage.v1": { + "type": "Transitive", + "resolved": "1.74.0.4115", + "contentHash": "9J3YPPSG53xwQLzdQ304dI+DMtVxJDpEazA2QQ2LA8IJoM4VUu4DpIxyhTGxpnAK4Db7s4LwzeNi6NkPKM3HzQ==", + "dependencies": { + "Google.Apis": "1.74.0", + "Google.Apis.Auth": "1.74.0" + } + }, + "Google.Cloud.Storage.V1": { + "type": "Transitive", + "resolved": "4.15.0", + "contentHash": "TVlJsaIkMDwHbZPCcLFFKduFcFD0yJL1jsZmV/V/FMjv5iok/eoC0oy8/VHjxfmWmNGkJLPLMAe5MrFLtY7hzg==", + "dependencies": { + "Google.Api.Gax.Rest": "[4.13.1, 5.0.0)", + "Google.Apis.Storage.v1": "[1.74.0.4115, 2.0.0)" + } + }, + "Microsoft.Bcl.AsyncInterfaces": { + "type": "Transitive", + "resolved": "10.0.3", + "contentHash": "TV62UsrJZPX6gbt3c4WrtXh7bmaDIcMqf9uft1cc4L6gJXOU07hDGEh+bFQh/L2Az0R1WVOkiT66lFqS6G2NmA==" + }, + "Microsoft.Bcl.Cryptography": { + "type": "Transitive", + "resolved": "10.0.2", + "contentHash": "LG9Yll3B5aNpxv0+D47g6LiOiKBIlodhcHdQwcYzo8VeexFLGqx5ymetmA2aBRyo9cCcWsQWrFsdbsr8LvmWDw==" + }, + "Microsoft.Extensions.Configuration.Abstractions": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "DY4C2qzTt2lOIY3VoscBH/TCwTolI5DKOfgJLuI+k1QUvRNT5vTMS58AOerNiihdVRStO0wEpLeRWA8jytzHBQ==", + "dependencies": { + "Microsoft.Extensions.Primitives": "10.0.13" + } + }, + "Microsoft.Extensions.Configuration.Binder": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "yJJl/b0+W+GEWbu+7viw6C3h2BoNkB69Vlz7s/4e+hm4mYbJqfhdcDSi9sCXJw47blsBaiQd3UDPgDtop92+ug==", + "dependencies": { + "Microsoft.Extensions.Configuration": "10.0.13", + "Microsoft.Extensions.Configuration.Abstractions": "10.0.13" + } + }, + "Microsoft.Extensions.DependencyInjection": { + "type": "Transitive", + "resolved": "9.0.5", + "contentHash": "N1Mn0T/tUBPoLL+Fzsp+VCEtneUhhxc1//Dx3BeuQ8AX+XrMlYCfnp2zgpEXnTCB7053CLdiqVWPZ7mEX6MPjg==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "9.0.5" + } + }, + "Microsoft.Extensions.DependencyInjection.Abstractions": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "G2GrGWNAPZBsii2F6AU8n+JT2tC1/I0Ml/e9ZyNCgPjf3hZqronP7o0X91aAynCmVeae8KkSIUNXXeXCNRmdVQ==" + }, + "Microsoft.Extensions.Diagnostics.Abstractions": { + "type": "Transitive", + "resolved": "10.0.3", + "contentHash": "mQiTzAj7PIJ2A9YXR5QhgulS1fTWhmQc3ckd1Mrf3hKW07d03fBDqx8vVaFw+cRTebDOeB6pNqdWdnRxsi1hBA==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.3", + "Microsoft.Extensions.Options": "10.0.3" + } + }, + "Microsoft.Extensions.FileProviders.Abstractions": { + "type": "Transitive", + "resolved": "10.0.3", + "contentHash": "4TD9AXDRsipTmaemwnjt/DM5Ri0de2JzHQhvZ4woBTjUtL4XrPNsMrOk5oiLJAx1gTrE6pOIhxv+lEde5F6CZA==", + "dependencies": { + "Microsoft.Extensions.Primitives": "10.0.3" + } + }, + "Microsoft.Extensions.Hosting.Abstractions": { + "type": "Transitive", + "resolved": "10.0.3", + "contentHash": "GdMpC10Jf6poxSvUJ4lgYpJ5F/kJeaAoJmrPufjBoPYyCTKKY5Dyl0rZA+LBNvFqTq1cZa/lhlptlUhNvU6xrg==", + "dependencies": { + "Microsoft.Extensions.Configuration.Abstractions": "10.0.3", + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.3", + "Microsoft.Extensions.Diagnostics.Abstractions": "10.0.3", + "Microsoft.Extensions.FileProviders.Abstractions": "10.0.3", + "Microsoft.Extensions.Logging.Abstractions": "10.0.3" + } + }, + "Microsoft.Extensions.Logging": { + "type": "Transitive", + "resolved": "9.0.5", + "contentHash": "rQU61lrgvpE/UgcAd4E56HPxUIkX/VUQCxWmwDTLLVeuwRDYTL0q/FLGfAW17cGTKyCh7ywYAEnY3sTEvURsfg==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection": "9.0.5", + "Microsoft.Extensions.Logging.Abstractions": "9.0.5", + "Microsoft.Extensions.Options": "9.0.5" + } + }, + "Microsoft.Extensions.Options": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "tmMJgdrE4uhf1JcNs9CV9moAHW+hmDuvgMiPpnQxarKaUxqI2eN33sYK5S4l9Uw+rG9PJvlm4qPxMhvolTDTaQ==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.13", + "Microsoft.Extensions.Primitives": "10.0.13" + } + }, + "Microsoft.Extensions.Primitives": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "ZjTwxzhiw2TMEzIbUDkMF0dfDB2I4NQgIfKD0ELu9/5Mk+SRGdKZcD0/lvBrRCAPo6KN2r0BW86MRancoyJkoA==" + }, + "Microsoft.Identity.Client": { + "type": "Transitive", + "resolved": "4.83.1", + "contentHash": "jOLIrZ3cynoqHLLO1cXplFFabrhrMEYs/EuKHvmCyrOm1axqiVFT6nCSnHxk7w5+d2BeQfCdM12Yf/0X7OeS1g==", + "dependencies": { + "Microsoft.IdentityModel.Abstractions": "8.14.0" + } + }, + "Microsoft.Identity.Client.Extensions.Msal": { + "type": "Transitive", + "resolved": "4.83.1", + "contentHash": "I3k4J4Hj4KbLEFanjeUzzDOVecukETaTgEkJ7h2pP/Yazs6SLp6TVUTo/Eo+ptPXMwvc+iX7rBFtMSUrA7R+Mg==", + "dependencies": { + "Microsoft.Identity.Client": "4.83.1", + "System.Security.Cryptography.ProtectedData": "4.5.0" + } + }, + "Microsoft.IdentityModel.Abstractions": { + "type": "Transitive", + "resolved": "8.21.0", + "contentHash": "XYgEmYpUvng0oYIagDJ+uebboQSnWtCV7q9zMyXlBw6ZqLLCHcmGUb7TpXvQ3HWY4OzbOCUQm2Ac0saPSnwqSw==" + }, + "Microsoft.IdentityModel.JsonWebTokens": { + "type": "Transitive", + "resolved": "8.21.0", + "contentHash": "iiA5uimwlRe4Drd8YKOaB8IQkgGOjxGjO6mu5zCGFjaEBpB6yNOluZ33zJFXiW5MQYRC6ypMnOXsiR5SPrO+tw==", + "dependencies": { + "Microsoft.IdentityModel.Tokens": "8.21.0" + } + }, + "Microsoft.IdentityModel.Logging": { + "type": "Transitive", + "resolved": "8.21.0", + "contentHash": "LgIWUUX386xCbvjq2ex8iXGIu2raDWrrIQ4OyYZKaNWMhzvN8YOTbAYMFJC+iHmKnkNBM8g7mLak9KQodldH3g==", + "dependencies": { + "Microsoft.IdentityModel.Abstractions": "8.21.0" + } + }, + "Microsoft.IdentityModel.Tokens": { + "type": "Transitive", + "resolved": "8.21.0", + "contentHash": "xFnFEl4VrhAJEbqtRrpgfAACgkm6NYo7cWYKYROH00N9/M5OYrRwTP0lHsginnLrqPrseLcSqAueQNRCyGNsdw==", + "dependencies": { + "Microsoft.Bcl.Cryptography": "10.0.2", + "Microsoft.Extensions.Logging.Abstractions": "8.0.0", + "Microsoft.IdentityModel.Logging": "8.21.0" + } + }, + "Mono.Unix": { + "type": "Transitive", + "resolved": "7.1.0-final.1.21458.1", + "contentHash": "Rhxz4A7By8Q0wEgDqR+mioDsYXGrcYMYPiWE9bSaUKMpG8yAGArhetEQV5Ms6KhKCLdQTlPYLBKPZYoKbAvT/g==" + }, + "Newtonsoft.Json": { + "type": "Transitive", + "resolved": "13.0.4", + "contentHash": "pdgNNMai3zv51W5aq268sujXUyx7SNdE2bj1wZcWjAQrKMFZV260lbqYop1d2GM67JI1huLRwxo9ZqnfF/lC6A==" + }, + "System.ClientModel": { + "type": "Transitive", + "resolved": "1.11.0", + "contentHash": "1Wl32zh7TbvN+HAO8NqDuaN0Ao2Qu/0j0NJSrXGDtpUQsTXQYJ3C6hd9/Ds2IrgP4agMQYFoXB35GZfC21ByHw==", + "dependencies": { + "Microsoft.Extensions.Configuration.Abstractions": "10.0.3", + "Microsoft.Extensions.Hosting.Abstractions": "10.0.3", + "Microsoft.Extensions.Logging.Abstractions": "10.0.3", + "System.Memory.Data": "10.0.3" + } + }, + "System.CodeDom": { + "type": "Transitive", + "resolved": "7.0.0", + "contentHash": "GLltyqEsE5/3IE+zYRP5sNa1l44qKl9v+bfdMcwg+M9qnQf47wK3H0SUR/T+3N4JEQXF3vV4CSuuo0rsg+nq2A==" + }, + "System.IdentityModel.Tokens.Jwt": { + "type": "Transitive", + "resolved": "8.21.0", + "contentHash": "dKsTkmlpUf/I6g1Aw9joWaZAahXvC0gWGQvkXW1IjZLSefC/4LhQI0tL0VIs64/NZQbl4XnQvLQyu6c7z1pMxg==", + "dependencies": { + "Microsoft.IdentityModel.JsonWebTokens": "8.21.0", + "Microsoft.IdentityModel.Tokens": "8.21.0" + } + }, + "System.IO.Hashing": { + "type": "Transitive", + "resolved": "10.0.3", + "contentHash": "La6ICwsdTKhVX+LKN+pvFjQRR3LhLwq3uKdi2knjLzRyPYBSydF4cjXidYxIiTcDD6XVYdsBWQEI8ZxiZ/OdIg==" + }, + "System.Management": { + "type": "Transitive", + "resolved": "7.0.2", + "contentHash": "/qEUN91mP/MUQmJnM5y5BdT7ZoPuVrtxnFlbJ8a3kBJGhe2wCzBfnPFtK2wTtEEcf3DMGR9J00GZZfg6HRI6yA==", + "dependencies": { + "System.CodeDom": "7.0.0" + } + }, + "System.Memory.Data": { + "type": "Transitive", + "resolved": "10.0.3", + "contentHash": "MaGhRfGunmrj/nHjtsi9XkhlYJ/ERGWrbA+BiSKNtGnAjc9XlG5EhAvak6VRcX5LYzPF6pBO8nJ613dTgzabig==" + }, + "System.Security.Cryptography.ProtectedData": { + "type": "Transitive", + "resolved": "4.5.0", + "contentHash": "wLBKzFnDCxP12VL9ANydSYhk59fC4cvOr9ypYQLPnAj48NQIhqnjdD2yhP8yEKyBJEjERWS9DisKL7rX5eU25Q==" + }, + "TimeZoneConverter": { + "type": "Transitive", + "resolved": "7.2.0", + "contentHash": "4Poqu79dTmacVETqUJ53g3x4U21V/cmWPSlehdsVE4Q00akavvlRq+SRjcg/CmfO9ibOCjRl6TeUV7hkC5RC4w==" + }, + "Tomlyn.Signed": { + "type": "Transitive", + "resolved": "2.10.1", + "contentHash": "+BHtb+YzcZ8uV9KVxjQE2uq7CJoYfd1q9gDuU2lx+wrrUppsj41lm2/uYQKTchz5Cs0pzvhf8t71bScDKVi8zA==" + }, + "fleettelemetry.core": { + "type": "Project" + } + } + } +} \ No newline at end of file diff --git a/fleet-telemetry/src/FleetTelemetry.Ingest/Commands.cs b/fleet-telemetry/src/FleetTelemetry.Ingest/Commands.cs new file mode 100644 index 0000000..9217ca0 --- /dev/null +++ b/fleet-telemetry/src/FleetTelemetry.Ingest/Commands.cs @@ -0,0 +1,89 @@ +using System.Globalization; +using FleetTelemetry.Data; +using FleetTelemetry.Simulation; +using Microsoft.Extensions.DependencyInjection; + +namespace FleetTelemetry.Ingest; + +internal sealed record GenerateOptions(int Seed, DateOnly Start, int Days) +{ + public static GenerateOptions Parse(string[] args) + { + var options = new GenerateOptions(42, new DateOnly(2026, 9, 7), 3); + for (var i = 0; i + 1 < args.Length; i += 2) + { + options = args[i] switch + { + "--seed" => options with { Seed = int.Parse(args[i + 1], CultureInfo.InvariantCulture) }, + "--start" => options with { Start = DateOnly.ParseExact(args[i + 1], "yyyy-MM-dd", CultureInfo.InvariantCulture) }, + "--days" => options with { Days = int.Parse(args[i + 1], CultureInfo.InvariantCulture) }, + _ => throw new ArgumentException($"Unknown option {args[i]}"), + }; + } + + return options; + } +} + +internal static class Commands +{ + /// + /// Simulates the registry's vehicles and writes one NDJSON file per vehicle-day, plus the next + /// day's first trip of the first vehicle as a JSON array for POST /api/telemetry. + /// + public static async Task GenerateAsync( + IServiceProvider services, string directory, GenerateOptions options, CancellationToken cancellationToken) + { + var vehicles = await services.GetRequiredService().ListVehiclesAsync(cancellationToken); + var simulated = new FleetSimulator(options.Seed).Simulate(vehicles, options.Start, options.Days + 1); + var liveDay = options.Start.AddDays(options.Days); + + var files = TelemetryExport.WriteNdjson(directory, simulated.Where(d => d.Date < liveDay)); + Console.WriteLine($"wrote {files.Count} files to {Path.GetRelativePath(Environment.CurrentDirectory, directory)}/, " + + $"for example {Path.GetFileName(files[0].Path)} ({files[0].Lines:N0} pings)"); + + var live = simulated.First(d => d.Date == liveDay); + var trip = live.Trips[0]; + var livePath = Path.Combine(directory, "live", $"{live.VehicleId}-{liveDay:yyyy-MM-dd}.json"); + Directory.CreateDirectory(Path.GetDirectoryName(livePath)!); + var pings = live.Pings.Where(p => p.Ts <= trip.EndedAt).ToList(); + await File.WriteAllTextAsync(livePath, TelemetryExport.ToJsonArray(pings), cancellationToken); + Console.WriteLine($"wrote {Path.GetRelativePath(Environment.CurrentDirectory, livePath)} ({pings.Count} pings, one trip for POST /api/telemetry)"); + Console.WriteLine($"{vehicles.Count} vehicles, {files.Sum(f => f.Lines):N0} pings over {options.Days} days from {options.Start:yyyy-MM-dd}, seed {options.Seed}"); + return 0; + } + + public static async Task BackfillAsync(IServiceProvider services, string directory, CancellationToken cancellationToken) + { + var report = await services.GetRequiredService().RunAsync(directory, cancellationToken); + + // List every file with errors, and the clean ones too when there are only a few. + var files = report.Loaded.OrderBy(f => f.File, StringComparer.Ordinal).ToList(); + var shown = files.Count <= 10 ? files : files.Where(f => f.ErrorsSeen > 0).ToList(); + Console.WriteLine(); + if (shown.Count > 0) + { + Console.WriteLine($"{"file",-30} {"status",-17} {"parsed",8} {"loaded",8} {"errors",7} first error"); + foreach (var file in shown) + { + var firstError = file.FirstErrorLine is { } line ? $"line {line}: {file.FirstError}" : ""; + Console.WriteLine($"{file.File,-30} {file.Status,-17} {file.RowsParsed,8:N0} {file.RowsLoaded,8:N0} {file.ErrorsSeen,7:N0} {firstError}"); + } + } + + if (files.Count > shown.Count) + { + Console.WriteLine($"{files.Count - shown.Count} files loaded without errors"); + } + + if (files.Count == 0) + { + Console.WriteLine("No new files: every staged file was loaded by an earlier run."); + } + + Console.WriteLine( + $"Uploaded {report.Uploaded.Count(u => u.Status == "UPLOADED")} of {report.Uploaded.Count} files, " + + $"loaded {report.RowsLoaded:N0} rows from {files.Count} files, {report.RowsFailed:N0} rows failed. FLEET_DAILY_KPIS refreshed."); + return 0; + } +} diff --git a/fleet-telemetry/src/FleetTelemetry.Ingest/FleetTelemetry.Ingest.csproj b/fleet-telemetry/src/FleetTelemetry.Ingest/FleetTelemetry.Ingest.csproj new file mode 100644 index 0000000..9a759ee --- /dev/null +++ b/fleet-telemetry/src/FleetTelemetry.Ingest/FleetTelemetry.Ingest.csproj @@ -0,0 +1,20 @@ + + + + Exe + FleetTelemetry.Ingest + + + + + + + + + + + + + + + diff --git a/fleet-telemetry/src/FleetTelemetry.Ingest/Program.cs b/fleet-telemetry/src/FleetTelemetry.Ingest/Program.cs new file mode 100644 index 0000000..68336b4 --- /dev/null +++ b/fleet-telemetry/src/FleetTelemetry.Ingest/Program.cs @@ -0,0 +1,31 @@ +using FleetTelemetry.Data; +using FleetTelemetry.Ingest; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; + +// Command-line tool for bulk telemetry: +// generate [--seed 42] [--start 2026-09-07] [--days 3] write synthetic NDJSON exports +// backfill PUT + COPY INTO, then refresh KPIs +if (args is not [var command, var directory, .. var options] || command is not ("generate" or "backfill")) +{ + Console.Error.WriteLine("Usage: FleetTelemetry.Ingest generate [--seed N] [--start yyyy-MM-dd] [--days N]"); + Console.Error.WriteLine(" FleetTelemetry.Ingest backfill "); + return 64; +} + +var builder = Host.CreateApplicationBuilder(new HostApplicationBuilderSettings +{ + ContentRootPath = AppContext.BaseDirectory, +}); +builder.Configuration.AddSnowflakeEnvironmentVariables(); +builder.Services.AddFleetSnowflake(); +builder.Services.AddSingleton(new FixedQueryTag($"ingest {command}")); + +using var host = builder.Build(); +await host.StartAsync(); +using var scope = host.Services.CreateScope(); +var cancellationToken = host.Services.GetRequiredService().ApplicationStopping; + +return command == "generate" + ? await Commands.GenerateAsync(scope.ServiceProvider, directory, GenerateOptions.Parse(options), cancellationToken) + : await Commands.BackfillAsync(scope.ServiceProvider, directory, cancellationToken); diff --git a/fleet-telemetry/src/FleetTelemetry.Ingest/appsettings.Development.json b/fleet-telemetry/src/FleetTelemetry.Ingest/appsettings.Development.json new file mode 100644 index 0000000..4d9391d --- /dev/null +++ b/fleet-telemetry/src/FleetTelemetry.Ingest/appsettings.Development.json @@ -0,0 +1,8 @@ +{ + "Snowflake": { + "Account": "snowflake", + "Host": "snowflake.localhost.localstack.cloud", + "Port": 4566, + "PrivateKeyPath": ".secrets/rsa_key.p8" + } +} diff --git a/fleet-telemetry/src/FleetTelemetry.Ingest/appsettings.json b/fleet-telemetry/src/FleetTelemetry.Ingest/appsettings.json new file mode 100644 index 0000000..98c21c0 --- /dev/null +++ b/fleet-telemetry/src/FleetTelemetry.Ingest/appsettings.json @@ -0,0 +1,26 @@ +{ + "Snowflake": { + "Account": "myorg-fleet", + "Host": "myorg-fleet.snowflakecomputing.com", + "Port": 443, + "User": "FLEET_SVC", + "PrivateKeyPath": "/run/secrets/snowflake_rsa_key.p8", + "Role": "FLEET_APP", + "Warehouse": "FLEET_WH", + "Database": "FLEET", + "Schema": "TELEMETRY", + "Application": "fleet-telemetry-ingest" + }, + "Logging": { + "LogLevel": { + "Default": "Information", + "Microsoft.Hosting": "Warning" + }, + "Console": { + "FormatterName": "simple", + "FormatterOptions": { + "SingleLine": true + } + } + } +} diff --git a/fleet-telemetry/src/FleetTelemetry.Ingest/packages.lock.json b/fleet-telemetry/src/FleetTelemetry.Ingest/packages.lock.json new file mode 100644 index 0000000..f5c2732 --- /dev/null +++ b/fleet-telemetry/src/FleetTelemetry.Ingest/packages.lock.json @@ -0,0 +1,586 @@ +{ + "version": 2, + "dependencies": { + "net10.0": { + "Microsoft.Extensions.Hosting": { + "type": "Direct", + "requested": "[10.0.13, )", + "resolved": "10.0.13", + "contentHash": "Q+hmPoDqeHipHyJcY7mDCQiLDUcn9jIgjoCThcgaOgjLTTT3ApTe8p6M2+twsDyfKuuCsKhSNlnNjaaggtXxRA==", + "dependencies": { + "Microsoft.Extensions.Configuration": "10.0.13", + "Microsoft.Extensions.Configuration.Abstractions": "10.0.13", + "Microsoft.Extensions.Configuration.Binder": "10.0.13", + "Microsoft.Extensions.Configuration.CommandLine": "10.0.13", + "Microsoft.Extensions.Configuration.EnvironmentVariables": "10.0.13", + "Microsoft.Extensions.Configuration.FileExtensions": "10.0.13", + "Microsoft.Extensions.Configuration.Json": "10.0.13", + "Microsoft.Extensions.Configuration.UserSecrets": "10.0.13", + "Microsoft.Extensions.DependencyInjection": "10.0.13", + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.13", + "Microsoft.Extensions.Diagnostics": "10.0.13", + "Microsoft.Extensions.FileProviders.Abstractions": "10.0.13", + "Microsoft.Extensions.FileProviders.Physical": "10.0.13", + "Microsoft.Extensions.Hosting.Abstractions": "10.0.13", + "Microsoft.Extensions.Logging": "10.0.13", + "Microsoft.Extensions.Logging.Abstractions": "10.0.13", + "Microsoft.Extensions.Logging.Configuration": "10.0.13", + "Microsoft.Extensions.Logging.Console": "10.0.13", + "Microsoft.Extensions.Logging.Debug": "10.0.13", + "Microsoft.Extensions.Logging.EventLog": "10.0.13", + "Microsoft.Extensions.Logging.EventSource": "10.0.13", + "Microsoft.Extensions.Options": "10.0.13" + } + }, + "Apache.Arrow": { + "type": "Transitive", + "resolved": "23.0.0", + "contentHash": "cU4Zm7byFwnb82Ea9ZRvVnq2nJNf/TF9j+rj/MHlvu9RBYMqYtu4sJaR1JmMh3MmSIuwMafVg7PfJxPAMv9dGw==", + "dependencies": { + "Apache.Arrow.Scalars": "23.0.0" + } + }, + "Apache.Arrow.Scalars": { + "type": "Transitive", + "resolved": "23.0.0", + "contentHash": "i53VmQ505ASXsStf7y29Z5qvhN6PGgFGNb6UUU7z1WmbzEgSjKQuIJkKKVbh+K5rtXRiKyr1Ah3Rmvbeb2BEHQ==" + }, + "AWSSDK.Core": { + "type": "Transitive", + "resolved": "4.0.100.8", + "contentHash": "xnuBVLQBmYQXsDZJ9mq2UDSFZm3xgO5oUb4/UR8p0UO7tG6heDhsLLI1NzZhIVAKyfW0tXg9hn9a/ek018TOVw==" + }, + "AWSSDK.S3": { + "type": "Transitive", + "resolved": "4.0.101.4", + "contentHash": "TYFuatWECzCbj/Lu1SsANucgpU/Br5YJ5Padl3YEdNJGfZqAcx3QemJw2BUopAsYTJ3e7TFpWhpNWha3dB9/dw==", + "dependencies": { + "AWSSDK.Core": "[4.0.100.8, 5.0.0)" + } + }, + "Azure.Core": { + "type": "Transitive", + "resolved": "1.55.0", + "contentHash": "c7femvYS/xEUrLP1sNZN+DoQZmx3X+G3ZXtOOz0RL/7cGMCM3JVqepVmRd3AwM4IK8AtTGS4GOigCO+d/zaSsQ==", + "dependencies": { + "Microsoft.Bcl.AsyncInterfaces": "10.0.3", + "Microsoft.Extensions.Configuration.Abstractions": "10.0.3", + "Microsoft.Extensions.Hosting.Abstractions": "10.0.3", + "Microsoft.Identity.Client": "4.83.1", + "Microsoft.Identity.Client.Extensions.Msal": "4.83.1", + "System.ClientModel": "1.11.0", + "System.Memory.Data": "10.0.3" + } + }, + "Azure.Storage.Blobs": { + "type": "Transitive", + "resolved": "12.29.1", + "contentHash": "pWh7xZBto8YcwiO853AB3uijTfVqs9PDte0Bu9/Zd8O9nwKiitWm0Jej1vsNkmYUzeG1552f8vJPjmtW30pBUQ==", + "dependencies": { + "Azure.Core": "1.55.0", + "Azure.Storage.Common": "12.28.0" + } + }, + "Azure.Storage.Common": { + "type": "Transitive", + "resolved": "12.28.0", + "contentHash": "5l8YhNrks38zKLGFW2BBFLwieyamH/xauABSASsdpZv79G0f+n2n22IjkRmUqCmALSupkwRHh2LOhbW3yj5Qgw==", + "dependencies": { + "Azure.Core": "1.55.0", + "System.IO.Hashing": "10.0.3" + } + }, + "BouncyCastle.Cryptography": { + "type": "Transitive", + "resolved": "2.7.0", + "contentHash": "U+12df8UEWHgBi04YVf/Lgi2dy3SItlIYvHjjEVa/BngCQIzDCDRBk50DDByCfDvSbe5pRNFr3b7UrVK2kMcLw==" + }, + "Google.Api.Gax": { + "type": "Transitive", + "resolved": "4.13.1", + "contentHash": "ujDpZk7O2VqAEIqcSlhH0FKzVpGZWly/qcNjHxpNUrL445Tei9PHnu4V1pwW2WDiMDvo3TpL1Bi4DeU525Afrg==", + "dependencies": { + "Microsoft.Bcl.AsyncInterfaces": "6.0.0", + "Newtonsoft.Json": "13.0.4" + } + }, + "Google.Api.Gax.Rest": { + "type": "Transitive", + "resolved": "4.13.1", + "contentHash": "wDfFZXKoHC6sy+Yfub/4cLQOb5ozBVPqDOrxCUAeTfku4d96ILqgzulJSdkPHbiT3oWB3hlYTdJgUwNSJjyo+A==", + "dependencies": { + "Google.Api.Gax": "4.13.1", + "Google.Apis.Auth": "1.73.0", + "Microsoft.Extensions.DependencyInjection.Abstractions": "6.0.0" + } + }, + "Google.Apis": { + "type": "Transitive", + "resolved": "1.74.0", + "contentHash": "Z6hSkmRD6V8/raSBqn/EsPpNop6suaLaGE9DbSY3yRE5G47l4zYqrJgapcLARk2Z3TsQUDmWz+w/tob3wuVt3g==", + "dependencies": { + "Google.Apis.Core": "1.74.0" + } + }, + "Google.Apis.Auth": { + "type": "Transitive", + "resolved": "1.74.0", + "contentHash": "43H4foh3zfWgboK8QnJjIfPQuBiS794TxqQLDfeE3erQSluKmEN0qeDuwxLTpSr9Mtwj76S4LWxPneuSbC6fNg==", + "dependencies": { + "Google.Apis": "1.74.0", + "Google.Apis.Core": "1.74.0", + "System.Management": "7.0.2" + } + }, + "Google.Apis.Core": { + "type": "Transitive", + "resolved": "1.74.0", + "contentHash": "gWOIiks6yo2TRliIyy10V18BJOO2BfUSPJFiTW6Tfap4tm3hFj7nvYU3XIfwzAxwoIy6nsBoqVDt9iBDSXrvYQ==", + "dependencies": { + "Newtonsoft.Json": "13.0.4" + } + }, + "Google.Apis.Storage.v1": { + "type": "Transitive", + "resolved": "1.74.0.4115", + "contentHash": "9J3YPPSG53xwQLzdQ304dI+DMtVxJDpEazA2QQ2LA8IJoM4VUu4DpIxyhTGxpnAK4Db7s4LwzeNi6NkPKM3HzQ==", + "dependencies": { + "Google.Apis": "1.74.0", + "Google.Apis.Auth": "1.74.0" + } + }, + "Google.Cloud.Storage.V1": { + "type": "Transitive", + "resolved": "4.15.0", + "contentHash": "TVlJsaIkMDwHbZPCcLFFKduFcFD0yJL1jsZmV/V/FMjv5iok/eoC0oy8/VHjxfmWmNGkJLPLMAe5MrFLtY7hzg==", + "dependencies": { + "Google.Api.Gax.Rest": "[4.13.1, 5.0.0)", + "Google.Apis.Storage.v1": "[1.74.0.4115, 2.0.0)" + } + }, + "Microsoft.Bcl.AsyncInterfaces": { + "type": "Transitive", + "resolved": "10.0.3", + "contentHash": "TV62UsrJZPX6gbt3c4WrtXh7bmaDIcMqf9uft1cc4L6gJXOU07hDGEh+bFQh/L2Az0R1WVOkiT66lFqS6G2NmA==" + }, + "Microsoft.Bcl.Cryptography": { + "type": "Transitive", + "resolved": "10.0.2", + "contentHash": "LG9Yll3B5aNpxv0+D47g6LiOiKBIlodhcHdQwcYzo8VeexFLGqx5ymetmA2aBRyo9cCcWsQWrFsdbsr8LvmWDw==" + }, + "Microsoft.Extensions.Configuration.Abstractions": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "DY4C2qzTt2lOIY3VoscBH/TCwTolI5DKOfgJLuI+k1QUvRNT5vTMS58AOerNiihdVRStO0wEpLeRWA8jytzHBQ==", + "dependencies": { + "Microsoft.Extensions.Primitives": "10.0.13" + } + }, + "Microsoft.Extensions.Configuration.Binder": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "yJJl/b0+W+GEWbu+7viw6C3h2BoNkB69Vlz7s/4e+hm4mYbJqfhdcDSi9sCXJw47blsBaiQd3UDPgDtop92+ug==", + "dependencies": { + "Microsoft.Extensions.Configuration": "10.0.13", + "Microsoft.Extensions.Configuration.Abstractions": "10.0.13" + } + }, + "Microsoft.Extensions.Configuration.CommandLine": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "RwdNZ9JdGs8eZmeL5qRe7ugp0Irk3KVkWpC/w1c8kjJwu1R4HvCXYz9kwHZxn45WiaRBG5lphQKtqnBaJOQC+Q==", + "dependencies": { + "Microsoft.Extensions.Configuration": "10.0.13", + "Microsoft.Extensions.Configuration.Abstractions": "10.0.13" + } + }, + "Microsoft.Extensions.Configuration.EnvironmentVariables": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "qJK2dm9/AxyPVfqqBaZtjkUFl2hnBR/CQnMJtSO5LdEoyATMqvZjeQVY+8l0oa6PM6qlVe28iMEyXIRQ0wyZjA==", + "dependencies": { + "Microsoft.Extensions.Configuration": "10.0.13", + "Microsoft.Extensions.Configuration.Abstractions": "10.0.13" + } + }, + "Microsoft.Extensions.Configuration.FileExtensions": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "kwC1DY55CsqPmDtJAz+7alD3G+gJcM+XwHSZrcRO8mJGYB9hRq1qC6ZH8DL0SPW9WtJ/Bae5ikzPic8SxSv6Vw==", + "dependencies": { + "Microsoft.Extensions.Configuration": "10.0.13", + "Microsoft.Extensions.Configuration.Abstractions": "10.0.13", + "Microsoft.Extensions.FileProviders.Abstractions": "10.0.13", + "Microsoft.Extensions.FileProviders.Physical": "10.0.13", + "Microsoft.Extensions.Primitives": "10.0.13" + } + }, + "Microsoft.Extensions.Configuration.Json": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "acCWM2+WR67kBAz6gAhD2zG+IZH9Q2VpKv5le7Msiu0vAMsnnLXpnV3BAzxd8/i+uOEo/dfb2XRYTynRVTDvEg==", + "dependencies": { + "Microsoft.Extensions.Configuration": "10.0.13", + "Microsoft.Extensions.Configuration.Abstractions": "10.0.13", + "Microsoft.Extensions.Configuration.FileExtensions": "10.0.13", + "Microsoft.Extensions.FileProviders.Abstractions": "10.0.13" + } + }, + "Microsoft.Extensions.Configuration.UserSecrets": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "kQ/Tx972unP+ueQL8SSninthIj6w6XP2o0VnJPFC5HQIimPMwEN704MOspEgDkmm4FUL5SuxLEU0bNiW4HEZFA==", + "dependencies": { + "Microsoft.Extensions.Configuration.Abstractions": "10.0.13", + "Microsoft.Extensions.Configuration.Json": "10.0.13", + "Microsoft.Extensions.FileProviders.Abstractions": "10.0.13", + "Microsoft.Extensions.FileProviders.Physical": "10.0.13" + } + }, + "Microsoft.Extensions.DependencyInjection": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "4tnWU9vrU/A8NwYe9Z6wvieZ0EBb/uM82LAT+edNhdepOBWREZ7M7mzb941EIDPJaeOFdbVrDd5+7s9e89U6bQ==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.13" + } + }, + "Microsoft.Extensions.DependencyInjection.Abstractions": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "G2GrGWNAPZBsii2F6AU8n+JT2tC1/I0Ml/e9ZyNCgPjf3hZqronP7o0X91aAynCmVeae8KkSIUNXXeXCNRmdVQ==" + }, + "Microsoft.Extensions.Diagnostics": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "J9ODOrffj1mK3Et69elNsLGRguesPxvThTdVpFmQX7fLNPhIOgqHQfLdGnTBEwNAV3yf8yiyVlpDZNw2qN87Nw==", + "dependencies": { + "Microsoft.Extensions.Configuration": "10.0.13", + "Microsoft.Extensions.Diagnostics.Abstractions": "10.0.13", + "Microsoft.Extensions.Options.ConfigurationExtensions": "10.0.13" + } + }, + "Microsoft.Extensions.Diagnostics.Abstractions": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "oZzqZze8ed1fLS20HtIrkPxQXGRG/koq09srIw2ptnDHm9RHfwB++d35hosB/cXc55TH7Pshv85O0txongjCIg==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.13", + "Microsoft.Extensions.Options": "10.0.13" + } + }, + "Microsoft.Extensions.FileProviders.Abstractions": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "BZDwtN3uoWRghDi5lRRZPDjMMCg2Ds7J6YEYjqka/TLjqnzquEpXyvchrIMeoubwT1ns23ow6PPeAoBHJO81Rw==", + "dependencies": { + "Microsoft.Extensions.Primitives": "10.0.13" + } + }, + "Microsoft.Extensions.FileProviders.Physical": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "U7HkgGCNPICN5k7NVK6dmLsSPSPeO/EbNcuxOl/5qdC9Ahjp9UZWKOvlnUC+rYurmoxIlwyv9e5j0BaQlTOELA==", + "dependencies": { + "Microsoft.Extensions.FileProviders.Abstractions": "10.0.13", + "Microsoft.Extensions.FileSystemGlobbing": "10.0.13", + "Microsoft.Extensions.Primitives": "10.0.13" + } + }, + "Microsoft.Extensions.FileSystemGlobbing": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "vQVzmAbnzlhXSXA2caYnlxtG9GXxEwtj5bODiWLu06A6iqeJmfBtlfKjuOSJGRy0/CFaNUrm7u9nxEUlljc2Rg==" + }, + "Microsoft.Extensions.Hosting.Abstractions": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "zIrbe9vkIFXxW3mOApIOWXw+xMwX7Cgsptr3ZSlglJt4hW2qp4fB/JU1NHgAevteBNH4PicvU9DsFPHxlnnloA==", + "dependencies": { + "Microsoft.Extensions.Configuration.Abstractions": "10.0.13", + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.13", + "Microsoft.Extensions.Diagnostics.Abstractions": "10.0.13", + "Microsoft.Extensions.FileProviders.Abstractions": "10.0.13", + "Microsoft.Extensions.Logging.Abstractions": "10.0.13" + } + }, + "Microsoft.Extensions.Logging": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "fqbiPy4nUp4H2XHKAVQXKN2y3T2h4cFIx1DAAgE6iY9OHkN/OA1cKFi+geJKRWGuX6pS5SFbsjIPWHJqrxE0eg==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection": "10.0.13", + "Microsoft.Extensions.Logging.Abstractions": "10.0.13", + "Microsoft.Extensions.Options": "10.0.13" + } + }, + "Microsoft.Extensions.Logging.Configuration": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "OyYJOaE9WbBx1Hp+Dd2zrqeFM0JwH5M/x0VL6tNUv3V18vIAVpix8UjhvY9eW/iXpguEHy68JO/Z9YkBOwftIw==", + "dependencies": { + "Microsoft.Extensions.Configuration": "10.0.13", + "Microsoft.Extensions.Configuration.Abstractions": "10.0.13", + "Microsoft.Extensions.Configuration.Binder": "10.0.13", + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.13", + "Microsoft.Extensions.Logging": "10.0.13", + "Microsoft.Extensions.Logging.Abstractions": "10.0.13", + "Microsoft.Extensions.Options": "10.0.13", + "Microsoft.Extensions.Options.ConfigurationExtensions": "10.0.13" + } + }, + "Microsoft.Extensions.Logging.Console": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "i9rn0rHkiCxfNVbf4DRG7LOFcKPy/BioAxS07w6daWvIpLNRjskqDJcGGaYVFWndsq0iTRaS8Msry0t5fRYQug==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.13", + "Microsoft.Extensions.Logging": "10.0.13", + "Microsoft.Extensions.Logging.Abstractions": "10.0.13", + "Microsoft.Extensions.Logging.Configuration": "10.0.13", + "Microsoft.Extensions.Options": "10.0.13" + } + }, + "Microsoft.Extensions.Logging.Debug": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "yBig0DNMq+3UgliwxYlUC48dg0fPqvqI5B3ljzmLTrpR6maoKTOO5i9QnPgQUmQoaQWZMgu74fuF6oSDihb6pg==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.13", + "Microsoft.Extensions.Logging": "10.0.13", + "Microsoft.Extensions.Logging.Abstractions": "10.0.13" + } + }, + "Microsoft.Extensions.Logging.EventLog": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "OAK40lXhgMuvyNwvzW3fxLvvucN8yr5XS7dQRcPmtnPf+Pt1lpbMgao5T4Wa6VdFagtIeDZNgPtV5Ig4xstziQ==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.13", + "Microsoft.Extensions.Logging": "10.0.13", + "Microsoft.Extensions.Logging.Abstractions": "10.0.13", + "Microsoft.Extensions.Options": "10.0.13", + "System.Diagnostics.EventLog": "10.0.13" + } + }, + "Microsoft.Extensions.Logging.EventSource": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "SxJu3oczk+TJtHvpgAmCVzEkzx69/obA6GZCbGrbqYqc981YOAyfEprySBdFX7Yc5qatHJKHNTlpGeeBAQkZcA==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.13", + "Microsoft.Extensions.Logging": "10.0.13", + "Microsoft.Extensions.Logging.Abstractions": "10.0.13", + "Microsoft.Extensions.Options": "10.0.13", + "Microsoft.Extensions.Primitives": "10.0.13" + } + }, + "Microsoft.Extensions.Options": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "tmMJgdrE4uhf1JcNs9CV9moAHW+hmDuvgMiPpnQxarKaUxqI2eN33sYK5S4l9Uw+rG9PJvlm4qPxMhvolTDTaQ==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.13", + "Microsoft.Extensions.Primitives": "10.0.13" + } + }, + "Microsoft.Extensions.Primitives": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "ZjTwxzhiw2TMEzIbUDkMF0dfDB2I4NQgIfKD0ELu9/5Mk+SRGdKZcD0/lvBrRCAPo6KN2r0BW86MRancoyJkoA==" + }, + "Microsoft.Identity.Client": { + "type": "Transitive", + "resolved": "4.83.1", + "contentHash": "jOLIrZ3cynoqHLLO1cXplFFabrhrMEYs/EuKHvmCyrOm1axqiVFT6nCSnHxk7w5+d2BeQfCdM12Yf/0X7OeS1g==", + "dependencies": { + "Microsoft.IdentityModel.Abstractions": "8.14.0" + } + }, + "Microsoft.Identity.Client.Extensions.Msal": { + "type": "Transitive", + "resolved": "4.83.1", + "contentHash": "I3k4J4Hj4KbLEFanjeUzzDOVecukETaTgEkJ7h2pP/Yazs6SLp6TVUTo/Eo+ptPXMwvc+iX7rBFtMSUrA7R+Mg==", + "dependencies": { + "Microsoft.Identity.Client": "4.83.1", + "System.Security.Cryptography.ProtectedData": "4.5.0" + } + }, + "Microsoft.IdentityModel.Abstractions": { + "type": "Transitive", + "resolved": "8.21.0", + "contentHash": "XYgEmYpUvng0oYIagDJ+uebboQSnWtCV7q9zMyXlBw6ZqLLCHcmGUb7TpXvQ3HWY4OzbOCUQm2Ac0saPSnwqSw==" + }, + "Microsoft.IdentityModel.JsonWebTokens": { + "type": "Transitive", + "resolved": "8.21.0", + "contentHash": "iiA5uimwlRe4Drd8YKOaB8IQkgGOjxGjO6mu5zCGFjaEBpB6yNOluZ33zJFXiW5MQYRC6ypMnOXsiR5SPrO+tw==", + "dependencies": { + "Microsoft.IdentityModel.Tokens": "8.21.0" + } + }, + "Microsoft.IdentityModel.Logging": { + "type": "Transitive", + "resolved": "8.21.0", + "contentHash": "LgIWUUX386xCbvjq2ex8iXGIu2raDWrrIQ4OyYZKaNWMhzvN8YOTbAYMFJC+iHmKnkNBM8g7mLak9KQodldH3g==", + "dependencies": { + "Microsoft.IdentityModel.Abstractions": "8.21.0" + } + }, + "Microsoft.IdentityModel.Tokens": { + "type": "Transitive", + "resolved": "8.21.0", + "contentHash": "xFnFEl4VrhAJEbqtRrpgfAACgkm6NYo7cWYKYROH00N9/M5OYrRwTP0lHsginnLrqPrseLcSqAueQNRCyGNsdw==", + "dependencies": { + "Microsoft.Bcl.Cryptography": "10.0.2", + "Microsoft.Extensions.Logging.Abstractions": "8.0.0", + "Microsoft.IdentityModel.Logging": "8.21.0" + } + }, + "Mono.Unix": { + "type": "Transitive", + "resolved": "7.1.0-final.1.21458.1", + "contentHash": "Rhxz4A7By8Q0wEgDqR+mioDsYXGrcYMYPiWE9bSaUKMpG8yAGArhetEQV5Ms6KhKCLdQTlPYLBKPZYoKbAvT/g==" + }, + "Newtonsoft.Json": { + "type": "Transitive", + "resolved": "13.0.4", + "contentHash": "pdgNNMai3zv51W5aq268sujXUyx7SNdE2bj1wZcWjAQrKMFZV260lbqYop1d2GM67JI1huLRwxo9ZqnfF/lC6A==" + }, + "System.ClientModel": { + "type": "Transitive", + "resolved": "1.11.0", + "contentHash": "1Wl32zh7TbvN+HAO8NqDuaN0Ao2Qu/0j0NJSrXGDtpUQsTXQYJ3C6hd9/Ds2IrgP4agMQYFoXB35GZfC21ByHw==", + "dependencies": { + "Microsoft.Extensions.Configuration.Abstractions": "10.0.3", + "Microsoft.Extensions.Hosting.Abstractions": "10.0.3", + "Microsoft.Extensions.Logging.Abstractions": "10.0.3", + "System.Memory.Data": "10.0.3" + } + }, + "System.CodeDom": { + "type": "Transitive", + "resolved": "7.0.0", + "contentHash": "GLltyqEsE5/3IE+zYRP5sNa1l44qKl9v+bfdMcwg+M9qnQf47wK3H0SUR/T+3N4JEQXF3vV4CSuuo0rsg+nq2A==" + }, + "System.Diagnostics.EventLog": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "DkNjtQyPR5Ud/zHnWHdsOuKbFRL9nIC92Q76z0NmHVDoQm40toLtsGFTIWq7toBoVa6fo/DInL3PLxY3Asixyw==" + }, + "System.IdentityModel.Tokens.Jwt": { + "type": "Transitive", + "resolved": "8.21.0", + "contentHash": "dKsTkmlpUf/I6g1Aw9joWaZAahXvC0gWGQvkXW1IjZLSefC/4LhQI0tL0VIs64/NZQbl4XnQvLQyu6c7z1pMxg==", + "dependencies": { + "Microsoft.IdentityModel.JsonWebTokens": "8.21.0", + "Microsoft.IdentityModel.Tokens": "8.21.0" + } + }, + "System.IO.Hashing": { + "type": "Transitive", + "resolved": "10.0.3", + "contentHash": "La6ICwsdTKhVX+LKN+pvFjQRR3LhLwq3uKdi2knjLzRyPYBSydF4cjXidYxIiTcDD6XVYdsBWQEI8ZxiZ/OdIg==" + }, + "System.Management": { + "type": "Transitive", + "resolved": "7.0.2", + "contentHash": "/qEUN91mP/MUQmJnM5y5BdT7ZoPuVrtxnFlbJ8a3kBJGhe2wCzBfnPFtK2wTtEEcf3DMGR9J00GZZfg6HRI6yA==", + "dependencies": { + "System.CodeDom": "7.0.0" + } + }, + "System.Memory.Data": { + "type": "Transitive", + "resolved": "10.0.3", + "contentHash": "MaGhRfGunmrj/nHjtsi9XkhlYJ/ERGWrbA+BiSKNtGnAjc9XlG5EhAvak6VRcX5LYzPF6pBO8nJ613dTgzabig==" + }, + "System.Security.Cryptography.ProtectedData": { + "type": "Transitive", + "resolved": "4.5.0", + "contentHash": "wLBKzFnDCxP12VL9ANydSYhk59fC4cvOr9ypYQLPnAj48NQIhqnjdD2yhP8yEKyBJEjERWS9DisKL7rX5eU25Q==" + }, + "TimeZoneConverter": { + "type": "Transitive", + "resolved": "7.2.0", + "contentHash": "4Poqu79dTmacVETqUJ53g3x4U21V/cmWPSlehdsVE4Q00akavvlRq+SRjcg/CmfO9ibOCjRl6TeUV7hkC5RC4w==" + }, + "Tomlyn.Signed": { + "type": "Transitive", + "resolved": "2.10.1", + "contentHash": "+BHtb+YzcZ8uV9KVxjQE2uq7CJoYfd1q9gDuU2lx+wrrUppsj41lm2/uYQKTchz5Cs0pzvhf8t71bScDKVi8zA==" + }, + "fleettelemetry.core": { + "type": "Project" + }, + "fleettelemetry.data": { + "type": "Project", + "dependencies": { + "Dapper": "[2.1.89, )", + "FleetTelemetry.Core": "[1.0.0, )", + "Microsoft.Extensions.Configuration": "[10.0.13, )", + "Microsoft.Extensions.Logging.Abstractions": "[10.0.13, )", + "Microsoft.Extensions.Options.ConfigurationExtensions": "[10.0.13, )", + "Snowflake.Data": "[6.4.0, )" + } + }, + "Dapper": { + "type": "CentralTransitive", + "requested": "[2.1.89, )", + "resolved": "2.1.89", + "contentHash": "1VjFE664wnxvuhlgP8kS4PeFFF4jqD3VkbIHW6Zlc52zuibgcz5ICnIHcQXlTp6pURpPNGLV0ZpNe5x9KePJnA==" + }, + "Microsoft.Extensions.Configuration": { + "type": "CentralTransitive", + "requested": "[10.0.13, )", + "resolved": "10.0.13", + "contentHash": "inhujYI/nikhmLxYI6ZEcaCgZ28ipJsKMPPxxr2XWf3vM0Q2P+E9R3hDHABZOLBxzD3gjpgcbSH/+hd2+gvI2A==", + "dependencies": { + "Microsoft.Extensions.Configuration.Abstractions": "10.0.13", + "Microsoft.Extensions.Primitives": "10.0.13" + } + }, + "Microsoft.Extensions.Logging.Abstractions": { + "type": "CentralTransitive", + "requested": "[10.0.13, )", + "resolved": "10.0.13", + "contentHash": "aSZEhEig4fvNiw2zs8Z6NyDlniKtq6YjcF3DyHuqTEHcZ2VKA/6itMPZ+dW+yWABWExlDcQwwdbqX1Rcb7W8WQ==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.13" + } + }, + "Microsoft.Extensions.Options.ConfigurationExtensions": { + "type": "CentralTransitive", + "requested": "[10.0.13, )", + "resolved": "10.0.13", + "contentHash": "xVO5HhATc09WXuS/RaDmx8ZeWS0mTpT1L0tkEDMSBHtAnNpNsDr0ge0wIO2jinCWWqpq5dL1tRTzpYNVlvyqLw==", + "dependencies": { + "Microsoft.Extensions.Configuration.Abstractions": "10.0.13", + "Microsoft.Extensions.Configuration.Binder": "10.0.13", + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.13", + "Microsoft.Extensions.Options": "10.0.13", + "Microsoft.Extensions.Primitives": "10.0.13" + } + }, + "Snowflake.Data": { + "type": "CentralTransitive", + "requested": "[6.4.0, )", + "resolved": "6.4.0", + "contentHash": "9zGrm/bD5h+DVha41TKy7soNDqw1bObAXCz9fQdhMJ0wKbj9IRuZvE+ZUvflBMq2mnjkgbNdiGeKz7qixgqbJA==", + "dependencies": { + "AWSSDK.S3": "4.0.101.4", + "Apache.Arrow": "23.0.0", + "Azure.Storage.Blobs": "12.29.1", + "BouncyCastle.Cryptography": "2.7.0", + "Google.Cloud.Storage.V1": "4.15.0", + "Microsoft.Extensions.Logging": "9.0.5", + "Mono.Unix": "7.1.0-final.1.21458.1", + "Newtonsoft.Json": "13.0.4", + "System.IdentityModel.Tokens.Jwt": "8.21.0", + "TimeZoneConverter": "7.2.0", + "Tomlyn.Signed": "2.10.1" + } + } + } + } +} \ No newline at end of file diff --git a/fleet-telemetry/tests/FleetTelemetry.IntegrationTests/BackfillTests.cs b/fleet-telemetry/tests/FleetTelemetry.IntegrationTests/BackfillTests.cs new file mode 100644 index 0000000..d6be3ae --- /dev/null +++ b/fleet-telemetry/tests/FleetTelemetry.IntegrationTests/BackfillTests.cs @@ -0,0 +1,94 @@ +using FleetTelemetry.Data; +using FleetTelemetry.IntegrationTests.Infrastructure; +using FleetTelemetry.Simulation; +using Microsoft.Extensions.DependencyInjection; + +namespace FleetTelemetry.IntegrationTests; + +/// The backfill path: PUT NDJSON exports to the stage, COPY INTO the landing table. +public sealed class BackfillTests(BackfillTests.Fixture fixture) : IClassFixture +{ + [Fact] + public void EveryFileIsLoadedWithOneRowPerPing() + { + Assert.Equal(fixture.Files.Count, fixture.FirstRun.Uploaded.Count); + Assert.All(fixture.FirstRun.Uploaded, u => Assert.Equal("UPLOADED", u.Status)); + + var loaded = fixture.FirstRun.Loaded.ToDictionary(f => f.File); + Assert.Equal(fixture.Files.Select(f => Path.GetFileName(f.Path) + ".gz").Order(), loaded.Keys.Order()); + foreach (var file in fixture.Files.Where(f => f.Path != fixture.CorruptFile)) + { + var result = loaded[Path.GetFileName(file.Path) + ".gz"]; + Assert.Equal(("LOADED", file.Lines, file.Lines, 0L), (result.Status, result.RowsParsed, result.RowsLoaded, result.ErrorsSeen)); + } + + Assert.Equal(fixture.Pings, fixture.FirstRun.RowsLoaded); + } + + [Fact] + public void ACorruptLineIsSkippedAndReportedWithItsLineNumber() + { + var expectedPings = fixture.Files.Single(f => f.Path == fixture.CorruptFile).Lines; + var result = fixture.FirstRun.Loaded.Single(f => f.File == Path.GetFileName(fixture.CorruptFile) + ".gz"); + + Assert.Equal("PARTIALLY_LOADED", result.Status); + Assert.Equal(expectedPings + 1, result.RowsParsed); + Assert.Equal(expectedPings, result.RowsLoaded); + Assert.Equal(1, result.ErrorsSeen); + Assert.Equal(Fixture.CorruptLine, result.FirstErrorLine); + Assert.False(string.IsNullOrEmpty(result.FirstError)); + Assert.Equal(1, fixture.FirstRun.RowsFailed); + } + + [Fact] + public async Task TheLandingTableAndTheTypedViewHoldEveryPing() + { + Assert.Equal(fixture.Pings, await fixture.QuerySingleAsync("SELECT COUNT(*) FROM TELEMETRY_RAW")); + Assert.Equal(fixture.Pings, await fixture.QuerySingleAsync("SELECT COUNT(*) FROM TELEMETRY")); + } + + [Fact] + public async Task RunningTheBackfillAgainLoadsNothingTwice() + { + await using var scope = fixture.CreateScope(); + var again = await scope.ServiceProvider.GetRequiredService() + .RunAsync(fixture.Directory, TestContext.Current.CancellationToken); + + Assert.All(again.Uploaded, u => Assert.Equal("SKIPPED", u.Status)); + Assert.Empty(again.Loaded); + Assert.Equal(fixture.Pings, await fixture.QuerySingleAsync("SELECT COUNT(*) FROM TELEMETRY_RAW")); + } + + public sealed class Fixture(SnowflakeTarget target) : FleetDatabaseFixture(target) + { + /// 1-based line of the corrupt record (a truncated copy of the line before it). + public const long CorruptLine = 42; + + public string Directory { get; } = Path.Combine(Path.GetTempPath(), $"fleet-backfill-{Guid.NewGuid():N}"); + + public IReadOnlyList Files { get; private set; } = []; + + public string CorruptFile { get; private set; } = ""; + + public long Pings { get; private set; } + + public BackfillReport FirstRun { get; private set; } = new([], []); + + protected override async Task LoadAsync(CancellationToken cancellationToken) + { + var fleet = new FleetSimulator(seed: 11).Simulate(await VehiclesAsync(3, cancellationToken), new DateOnly(2026, 9, 7), days: 2); + Files = TelemetryExport.WriteNdjson(Directory, fleet); + Pings = Files.Sum(f => f.Lines); + + // A gateway that crashed mid-write leaves a truncated record behind. + CorruptFile = Files[4].Path; + var lines = (await File.ReadAllLinesAsync(CorruptFile, cancellationToken)).ToList(); + var previous = lines[(int)CorruptLine - 2]; + lines.Insert((int)CorruptLine - 1, previous[..(previous.Length / 2)]); + await File.WriteAllTextAsync(CorruptFile, string.Join('\n', lines) + "\n", cancellationToken); + + await using var scope = CreateScope(); + FirstRun = await scope.ServiceProvider.GetRequiredService().RunAsync(Directory, cancellationToken); + } + } +} diff --git a/fleet-telemetry/tests/FleetTelemetry.IntegrationTests/FleetKpiTests.cs b/fleet-telemetry/tests/FleetTelemetry.IntegrationTests/FleetKpiTests.cs new file mode 100644 index 0000000..5cfbb25 --- /dev/null +++ b/fleet-telemetry/tests/FleetTelemetry.IntegrationTests/FleetKpiTests.cs @@ -0,0 +1,85 @@ +using System.Net.Http.Json; +using System.Text; +using FleetTelemetry.Api.Endpoints; +using FleetTelemetry.Data; +using FleetTelemetry.IntegrationTests.Infrastructure; +using FleetTelemetry.Simulation; +using Microsoft.Extensions.DependencyInjection; + +namespace FleetTelemetry.IntegrationTests; + +/// Daily fleet KPIs, served from the FLEET_DAILY_KPIS dynamic table. +public sealed class FleetKpiTests(FleetKpiTests.Fixture fixture) : IClassFixture +{ + private static readonly DateOnly FirstDay = new(2026, 9, 7); + + [Fact] + public async Task DailyKpisAreComputedFromTheBackfilledTelemetry() + { + var response = await fixture.Api.GetFromJsonAsync( + "/api/fleet/daily?from=2026-09-07&to=2026-09-09", TestContext.Current.CancellationToken); + + Assert.NotNull(response); + Assert.Equal(Expected.FleetDays(fixture.Backfilled), response.Days.Select(AsExpected)); + } + + [Fact] + public async Task TheRangeIsInclusive() + { + var response = await fixture.Api.GetFromJsonAsync( + "/api/fleet/daily?from=2026-09-08&to=2026-09-08", TestContext.Current.CancellationToken); + + Assert.NotNull(response); + Assert.Equal([FirstDay.AddDays(1)], response.Days.Select(d => d.Day)); + } + + [Fact] + public async Task LiveTelemetryShowsUpOnceTheDynamicTableRefreshes() + { + var cancellationToken = TestContext.Current.CancellationToken; + var batch = TelemetryExport.ToJsonArray(fixture.Live.Pings); + using (var post = await fixture.Api.PostAsync( + "/api/telemetry", new StringContent(batch, Encoding.UTF8, "application/json"), cancellationToken)) + { + post.EnsureSuccessStatusCode(); + } + + await using (var scope = fixture.CreateScope()) + { + await scope.ServiceProvider.GetRequiredService().RefreshAsync(cancellationToken); + } + + var day = fixture.Live.Date; + var response = await fixture.Api.GetFromJsonAsync( + $"/api/fleet/daily?from={day:yyyy-MM-dd}&to={day:yyyy-MM-dd}", cancellationToken); + + Assert.NotNull(response); + Assert.Equal(Expected.FleetDays([fixture.Live]), response.Days.Select(AsExpected)); + } + + private static Expected.FleetDay AsExpected(FleetTelemetry.Kpis.FleetDayKpi d) => new( + d.Day, d.ActiveVehicles, d.DistanceKm, d.FuelUsedL, d.FuelEconomyLPer100Km ?? double.NaN, d.IdleShare, d.MaxSpeedKmh, d.VehiclesWithFaults); + + public sealed class Fixture(SnowflakeTarget target) : FleetDatabaseFixture(target) + { + /// Four vehicles, 7 to 9 September, bulk-loaded. + public IReadOnlyList Backfilled { get; private set; } = []; + + /// One vehicle's first trip on 10 September, posted by a test. + public VehicleDay Live { get; private set; } = null!; + + protected override async Task LoadAsync(CancellationToken cancellationToken) + { + var fleet = new FleetSimulator(seed: 21).Simulate(await VehiclesAsync(4, cancellationToken), FirstDay, days: 4); + Backfilled = fleet.Where(d => d.Date < FirstDay.AddDays(3)).ToList(); + var live = fleet.First(d => d.Date == FirstDay.AddDays(3)); + Live = live with { Pings = live.Pings.Where(p => p.Ts <= live.Trips[0].EndedAt).ToList() }; + + var directory = Path.Combine(Path.GetTempPath(), $"fleet-kpis-{Guid.NewGuid():N}"); + TelemetryExport.WriteNdjson(directory, Backfilled); + await using var scope = CreateScope(); + await scope.ServiceProvider.GetRequiredService().RunAsync(directory, cancellationToken); + Directory.Delete(directory, recursive: true); + } + } +} diff --git a/fleet-telemetry/tests/FleetTelemetry.IntegrationTests/FleetTelemetry.IntegrationTests.csproj b/fleet-telemetry/tests/FleetTelemetry.IntegrationTests/FleetTelemetry.IntegrationTests.csproj new file mode 100644 index 0000000..47ae664 --- /dev/null +++ b/fleet-telemetry/tests/FleetTelemetry.IntegrationTests/FleetTelemetry.IntegrationTests.csproj @@ -0,0 +1,26 @@ + + + + Exe + false + FleetTelemetry.IntegrationTests + + + + + + + + + + + + + + + + + + + + diff --git a/fleet-telemetry/tests/FleetTelemetry.IntegrationTests/Infrastructure/Expected.cs b/fleet-telemetry/tests/FleetTelemetry.IntegrationTests/Infrastructure/Expected.cs new file mode 100644 index 0000000..7c26181 --- /dev/null +++ b/fleet-telemetry/tests/FleetTelemetry.IntegrationTests/Infrastructure/Expected.cs @@ -0,0 +1,62 @@ +using FleetTelemetry.Simulation; +using FleetTelemetry.Telemetry; + +namespace FleetTelemetry.IntegrationTests.Infrastructure; + +/// +/// Expected values computed directly from the simulated pings with LINQ, independently of +/// the SQL under test. +/// +internal static class Expected +{ + public sealed record Fault(string Code, string Severity, int Occurrences, DateTime FirstSeen, DateTime LastSeen, double MaxCoolantC); + + public sealed record FleetDay( + DateOnly Day, + int ActiveVehicles, + double DistanceKm, + double FuelUsedL, + double FuelEconomyLPer100Km, + double IdleShare, + double MaxSpeedKmh, + int VehiclesWithFaults); + + public static IReadOnlyList Faults(IEnumerable pings) => + pings + .SelectMany(p => p.Engine.Dtc.Select(code => (Ping: p, Code: code))) + .GroupBy(x => x.Code.Code) + .OrderBy(g => g.Key, StringComparer.Ordinal) + .Select(g => new Fault( + g.Key, + g.First().Code.Severity, + g.Count(), + g.Min(x => x.Ping.Ts).UtcDateTime, + g.Max(x => x.Ping.Ts).UtcDateTime, + g.Max(x => x.Ping.Engine.CoolantC))) + .ToList(); + + /// + /// Per day: distance and fuel are each vehicle's last minus first reading, summed; fuel + /// economy is total fuel over total distance; idle share is stationary pings over all pings. + /// + public static IReadOnlyList FleetDays(IEnumerable vehicleDays) => + vehicleDays + .GroupBy(d => d.Date) + .OrderBy(g => g.Key) + .Select(day => + { + var distance = day.Sum(v => v.Pings.Max(p => p.OdometerKm) - v.Pings.Min(p => p.OdometerKm)); + var fuel = day.Sum(v => v.Pings.Max(p => p.Fuel.UsedL) - v.Pings.Min(p => p.Fuel.UsedL)); + var pings = day.SelectMany(v => v.Pings).ToList(); + return new FleetDay( + day.Key, + day.Count(), + Math.Round(distance, 1), + Math.Round(fuel, 1), + Math.Round(fuel / distance * 100, 1), + Math.Round(pings.Count(p => p.SpeedKmh == 0) / (double)pings.Count, 3), + pings.Max(p => p.SpeedKmh), + day.Count(v => v.Pings.Any(p => p.Engine.Dtc.Count > 0))); + }) + .ToList(); +} diff --git a/fleet-telemetry/tests/FleetTelemetry.IntegrationTests/Infrastructure/FleetDatabaseFixture.cs b/fleet-telemetry/tests/FleetTelemetry.IntegrationTests/Infrastructure/FleetDatabaseFixture.cs new file mode 100644 index 0000000..6eb0fb6 --- /dev/null +++ b/fleet-telemetry/tests/FleetTelemetry.IntegrationTests/Infrastructure/FleetDatabaseFixture.cs @@ -0,0 +1,113 @@ +using System.Globalization; +using Dapper; +using FleetTelemetry.Data; +using FleetTelemetry.Simulation; +using Microsoft.AspNetCore.Hosting; +using Microsoft.AspNetCore.Mvc.Testing; +using Microsoft.Extensions.Configuration; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; + +namespace FleetTelemetry.IntegrationTests.Infrastructure; + +/// +/// A private database per test class: created with the init scripts, loaded by the +/// subclass, dropped afterwards. Exposes the real API (WebApplicationFactory) and the +/// data-access services (for the backfill) wired to that database. +/// +public abstract class FleetDatabaseFixture(SnowflakeTarget target) : IAsyncLifetime +{ + private WebApplicationFactory? _api; + private ServiceProvider? _services; + + public SnowflakeOptions Settings { get; private set; } = new(); + + public HttpClient Api { get; private set; } = new(); + + public async ValueTask InitializeAsync() + { + var cancellationToken = TestContext.Current.CancellationToken; + var database = string.Create( + CultureInfo.InvariantCulture, + $"FLEET_IT_{GetType().Name.Replace("Fixture", "", StringComparison.Ordinal).ToUpperInvariant()}_{Random.Shared.Next(0x100000, 0xFFFFFF):X6}"); + + await using (var connection = await OpenAsync(target.Settings, cancellationToken)) + { + await connection.ExecuteAsync(new CommandDefinition($"CREATE DATABASE {database}", cancellationToken: cancellationToken)); + await connection.ExecuteAsync(new CommandDefinition($"CREATE SCHEMA {database}.TELEMETRY", cancellationToken: cancellationToken)); + } + + Settings = target.Settings.With(database: database, schema: "TELEMETRY"); + await SqlScripts.RunAsync(Settings, SqlScripts.Schema, cancellationToken); + + _services = new ServiceCollection() + .AddSingleton(new ConfigurationBuilder().Build()) + .AddLogging(l => l.AddSimpleConsole().SetMinimumLevel(LogLevel.Warning)) + .AddFleetSnowflake() + .AddSingleton(new FixedQueryTag($"integration-tests {GetType().Name}")) + .PostConfigure(o => Settings.CopyTo(o)) + .BuildServiceProvider(); + + _api = new WebApplicationFactory().WithWebHostBuilder(builder => + { + builder.UseEnvironment("Development"); + builder.UseSetting("Logging:LogLevel:Default", "Warning"); + builder.ConfigureServices(s => s.PostConfigure(o => Settings.CopyTo(o))); + }); + Api = _api.CreateClient(); + + await LoadAsync(cancellationToken); + } + + public async ValueTask DisposeAsync() + { + Api.Dispose(); + if (_api is not null) + { + await _api.DisposeAsync(); + } + + if (_services is not null) + { + await _services.DisposeAsync(); + } + + if (Settings.Database.Length > 0) + { + await using var connection = await OpenAsync(target.Settings, CancellationToken.None); + await connection.ExecuteAsync($"DROP DATABASE IF EXISTS {Settings.Database}"); + } + + GC.SuppressFinalize(this); + } + + /// Loads the class's test data; the database already has the schema and the registry. + protected virtual Task LoadAsync(CancellationToken cancellationToken) => Task.CompletedTask; + + /// A scope with the data-access services, as the ingest tool uses them. + public AsyncServiceScope CreateScope() => _services!.CreateAsyncScope(); + + public async Task QuerySingleAsync(string sql, object? parameters = null) => + (await QueryAsync(sql, parameters)).Single(); + + public async Task> QueryAsync(string sql, object? parameters = null) + { + await using var connection = await OpenAsync(Settings, TestContext.Current.CancellationToken); + var rows = await connection.QueryAsync(new CommandDefinition( + sql, parameters, cancellationToken: TestContext.Current.CancellationToken)); + return rows.ToList(); + } + + /// The first registry vehicles, ready for the simulator. + protected async Task> VehiclesAsync(int count, CancellationToken cancellationToken) + { + await using var scope = CreateScope(); + var all = await scope.ServiceProvider.GetRequiredService().ListVehiclesAsync(cancellationToken); + return all.Take(count).ToList(); + } + + private static Task OpenAsync(SnowflakeOptions settings, CancellationToken cancellationToken) => + new FleetTelemetry.Data.FleetDatabase(Options.Create(settings), new FixedQueryTag("integration-tests setup")) + .OpenAsync(cancellationToken); +} diff --git a/fleet-telemetry/tests/FleetTelemetry.IntegrationTests/Infrastructure/SnowflakeOptionsExtensions.cs b/fleet-telemetry/tests/FleetTelemetry.IntegrationTests/Infrastructure/SnowflakeOptionsExtensions.cs new file mode 100644 index 0000000..c49ecd7 --- /dev/null +++ b/fleet-telemetry/tests/FleetTelemetry.IntegrationTests/Infrastructure/SnowflakeOptionsExtensions.cs @@ -0,0 +1,34 @@ +using FleetTelemetry.Data; + +namespace FleetTelemetry.IntegrationTests.Infrastructure; + +internal static class SnowflakeOptionsExtensions +{ + public static SnowflakeOptions With(this SnowflakeOptions o, string database, string schema) => new() + { + Account = o.Account, + Host = o.Host, + Port = o.Port, + User = o.User, + PrivateKeyPath = o.PrivateKeyPath, + Role = o.Role, + Warehouse = o.Warehouse, + Database = database, + Schema = schema, + Application = o.Application, + }; + + public static void CopyTo(this SnowflakeOptions source, SnowflakeOptions target) + { + target.Account = source.Account; + target.Host = source.Host; + target.Port = source.Port; + target.User = source.User; + target.PrivateKeyPath = source.PrivateKeyPath; + target.Role = source.Role; + target.Warehouse = source.Warehouse; + target.Database = source.Database; + target.Schema = source.Schema; + target.Application = source.Application; + } +} diff --git a/fleet-telemetry/tests/FleetTelemetry.IntegrationTests/Infrastructure/SnowflakeTarget.cs b/fleet-telemetry/tests/FleetTelemetry.IntegrationTests/Infrastructure/SnowflakeTarget.cs new file mode 100644 index 0000000..940bdf5 --- /dev/null +++ b/fleet-telemetry/tests/FleetTelemetry.IntegrationTests/Infrastructure/SnowflakeTarget.cs @@ -0,0 +1,172 @@ +using System.Net; +using System.Security.Cryptography; +using System.Text; +using DotNet.Testcontainers.Builders; +using DotNet.Testcontainers.Configurations; +using DotNet.Testcontainers.Containers; +using FleetTelemetry.Data; + +[assembly: AssemblyFixture(typeof(FleetTelemetry.IntegrationTests.Infrastructure.SnowflakeTarget))] + +namespace FleetTelemetry.IntegrationTests.Infrastructure; + +/// +/// Where the suite runs, chosen by TEST_TARGET: +/// +/// EMULATOR (default): one Snowflake-Next container for the whole run, on a +/// random host port, running the init/*.sf.sql scripts as its init hooks, just like +/// docker compose; +/// SNOWFLAKE_CLOUD: the account in the SNOWFLAKE_* variables, provisioned +/// once by an administrator with the same scripts. +/// +/// Either way the tests connect as FLEET_SVC with role FLEET_APP and key-pair authentication, +/// starting in the FLEET database. +/// +public sealed class SnowflakeTarget : IAsyncLifetime +{ + public const string EmulatorHost = "snowflake.localhost.localstack.cloud"; + private const ushort EmulatorPort = 4566; + private static readonly TimeSpan ProvisioningTimeout = TimeSpan.FromMinutes(2); + + private IContainer? _container; + private string? _throwawayKey; + + public static bool IsCloud => + string.Equals(Environment.GetEnvironmentVariable("TEST_TARGET"), "SNOWFLAKE_CLOUD", StringComparison.OrdinalIgnoreCase); + + /// Connection settings for the FLEET database; each test class creates its own from there. + public SnowflakeOptions Settings { get; private set; } = new(); + + public async ValueTask InitializeAsync() + { + if (IsCloud) + { + var account = RequireEnv("SNOWFLAKE_ACCOUNT"); + Settings = new SnowflakeOptions + { + Account = account, + Host = Environment.GetEnvironmentVariable("SNOWFLAKE_HOST") ?? $"{account}.snowflakecomputing.com", + Port = 443, + User = Env("SNOWFLAKE_USER", "FLEET_SVC"), + PrivateKeyPath = RequireEnv("SNOWFLAKE_PRIVATE_KEY_PATH"), + Role = Env("SNOWFLAKE_ROLE", "FLEET_APP"), + Warehouse = Env("SNOWFLAKE_WAREHOUSE", "FLEET_WH"), + Database = "FLEET", + Schema = "TELEMETRY", + Application = "fleet-telemetry-tests", + }; + return; + } + + _container = new ContainerBuilder(Env("SNOWFLAKE_IMAGE", "localstack/snowflake-next:latest")) + .WithPortBinding(EmulatorPort, assignRandomHostPort: true) + .WithEnvironment("LOCALSTACK_AUTH_TOKEN", RequireEnv("LOCALSTACK_AUTH_TOKEN")) + .WithResourceMapping(new DirectoryInfo(SqlScripts.Directory), "/etc/localstack/init/ready.d/") + .WithWaitStrategy(Wait.ForUnixContainer().AddCustomWaitStrategy( + new AnswersSql(), o => o.WithTimeout(TimeSpan.FromMinutes(3)))) + .Build(); + await _container.StartAsync(); + + _throwawayKey = Path.Combine(Path.GetTempPath(), $"fleet-it-{Guid.NewGuid():N}.p8"); + using (var rsa = RSA.Create(2048)) + { + await File.WriteAllTextAsync(_throwawayKey, rsa.ExportPkcs8PrivateKeyPem()); + } + + Settings = new SnowflakeOptions + { + Account = "snowflake", + Host = EmulatorHost, + Port = _container.GetMappedPublicPort(EmulatorPort), + User = "FLEET_SVC", + PrivateKeyPath = _throwawayKey, + Role = "FLEET_APP", + Warehouse = "FLEET_WH", + Database = "FLEET", + Schema = "TELEMETRY", + Application = "fleet-telemetry-tests", + }; + + await WaitForProvisioningAsync(TestContext.Current.CancellationToken); + } + + /// The init hooks run once the emulator answers SQL; the last one creates FLEET_DAILY_KPIS. + private async Task WaitForProvisioningAsync(CancellationToken cancellationToken) + { + var deadline = DateTime.UtcNow + ProvisioningTimeout; + while (true) + { + try + { + await using var connection = await new FleetTelemetry.Data.FleetDatabase( + Microsoft.Extensions.Options.Options.Create(Settings), new FixedQueryTag("integration-tests setup")) + .OpenAsync(cancellationToken); + await Dapper.SqlMapper.ExecuteScalarAsync(connection, "SELECT COUNT(*) FROM FLEET.TELEMETRY.FLEET_DAILY_KPIS"); + return; + } + catch (Snowflake.Data.Client.SnowflakeDbException) when (DateTime.UtcNow < deadline) + { + await Task.Delay(TimeSpan.FromSeconds(1), cancellationToken); + } + } + } + + public async ValueTask DisposeAsync() + { + if (_container is not null) + { + await _container.DisposeAsync(); + } + + if (_throwawayKey is not null) + { + File.Delete(_throwawayKey); + } + } + + private static string Env(string name, string fallback) => + Environment.GetEnvironmentVariable(name) is { Length: > 0 } value ? value : fallback; + + private static string RequireEnv(string name) => + Environment.GetEnvironmentVariable(name) is { Length: > 0 } value + ? value + : throw new InvalidOperationException(IsCloud + ? $"TEST_TARGET=SNOWFLAKE_CLOUD needs {name}." + : $"{name} is not set. Export it, or run `make test`, which reads it from ~/.localstack/default.env."); + + /// Ready once the emulator answers SELECT 1, checked from the host like scripts/wait-for-emulator.sh. + private sealed class AnswersSql : IWaitUntil + { + private static readonly HttpClient Http = new(new HttpClientHandler + { + AutomaticDecompression = DecompressionMethods.All, + }) + { + Timeout = TimeSpan.FromSeconds(10), + }; + + public async Task UntilAsync(IContainer container) + { + var url = $"https://{EmulatorHost}:{container.GetMappedPublicPort(EmulatorPort)}/api/v2/statements"; + using var request = new HttpRequestMessage(HttpMethod.Post, url) + { + Content = new StringContent("""{"statement":"SELECT 1"}""", Encoding.UTF8, "application/json"), + }; + request.Headers.Add("Authorization", "Bearer local"); + request.Headers.Add("Accept", "application/json"); + try + { + using var response = await Http.SendAsync(request); + return response.IsSuccessStatusCode; + } + catch (HttpRequestException) + { + return false; + } + catch (TaskCanceledException) + { + return false; + } + } + } +} diff --git a/fleet-telemetry/tests/FleetTelemetry.IntegrationTests/Infrastructure/SqlScripts.cs b/fleet-telemetry/tests/FleetTelemetry.IntegrationTests/Infrastructure/SqlScripts.cs new file mode 100644 index 0000000..6d04630 --- /dev/null +++ b/fleet-telemetry/tests/FleetTelemetry.IntegrationTests/Infrastructure/SqlScripts.cs @@ -0,0 +1,60 @@ +using Dapper; +using FleetTelemetry.Data; +using Microsoft.Extensions.Options; + +namespace FleetTelemetry.IntegrationTests.Infrastructure; + +/// The repository's init/NN_name.sf.sql scripts, copied next to the test assembly. +internal static class SqlScripts +{ + public static string Directory { get; } = Path.Combine(AppContext.BaseDirectory, "init"); + + /// + /// Database-scoped objects (02 onwards: landing table, registry, views, dynamic table), + /// created in whatever database the session uses. 01 is account-level provisioning. + /// + public static IReadOnlyList Schema { get; } = System.IO.Directory + .GetFiles(Directory, "*.sf.sql") + .Where(f => !Path.GetFileName(f).StartsWith("01_", StringComparison.Ordinal)) + .Order(StringComparer.Ordinal) + .ToList(); + + /// + /// Runs each statement in order. The scripts start with USE statements that point at + /// FLEET.TELEMETRY; skipping them runs the scripts in the session's own database + /// instead, which is how each test class gets a private copy. + /// + public static async Task RunAsync(SnowflakeOptions settings, IEnumerable scripts, CancellationToken cancellationToken) + { + await using var connection = await new FleetTelemetry.Data.FleetDatabase( + Options.Create(settings), new FixedQueryTag("integration-tests setup")).OpenAsync(cancellationToken); + foreach (var script in scripts) + { + foreach (var statement in Statements(await File.ReadAllTextAsync(script, cancellationToken))) + { + if (statement.StartsWith("USE ", StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + await connection.ExecuteAsync(new CommandDefinition(statement, cancellationToken: cancellationToken)); + } + } + } + + /// Statements end with ';' at the end of a line; whole-line '--' comments are dropped. + private static IEnumerable Statements(string sql) + { + var lines = sql.Split('\n').Where(l => !l.TrimStart().StartsWith("--", StringComparison.Ordinal)); + var current = new List(); + foreach (var line in lines) + { + current.Add(line); + if (line.TrimEnd().EndsWith(';')) + { + yield return string.Join('\n', current).Trim().TrimEnd(';'); + current.Clear(); + } + } + } +} diff --git a/fleet-telemetry/tests/FleetTelemetry.IntegrationTests/TelemetryIngestTests.cs b/fleet-telemetry/tests/FleetTelemetry.IntegrationTests/TelemetryIngestTests.cs new file mode 100644 index 0000000..17c2606 --- /dev/null +++ b/fleet-telemetry/tests/FleetTelemetry.IntegrationTests/TelemetryIngestTests.cs @@ -0,0 +1,139 @@ +using System.Net; +using System.Net.Http.Json; +using System.Text; +using System.Text.Json.Nodes; +using FleetTelemetry.Api.Endpoints; +using FleetTelemetry.IntegrationTests.Infrastructure; +using FleetTelemetry.Simulation; +using FleetTelemetry.Telemetry; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; + +namespace FleetTelemetry.IntegrationTests; + +/// POST /api/telemetry against an empty database. Each test uses its own vehicle-day. +public sealed class TelemetryIngestTests(TelemetryIngestTests.Fixture fixture) : IClassFixture +{ + [Fact] + public async Task AnAcceptedBatchIsQueryableAsATrip() + { + var day = fixture.Day(vehicle: 0, dayOffset: 0); + var trip = day.Trips[0]; + var pings = day.Pings.Where(p => p.Ts <= trip.EndedAt).ToList(); + + var accepted = await PostAsync(TelemetryExport.ToJsonArray(pings)); + + Assert.Equal(pings.Count, accepted); + var trips = await GetTripsAsync(day.VehicleId, day.Date); + var only = Assert.Single(trips); + Assert.Equal((trip.StartedAt.UtcDateTime, trip.EndedAt.UtcDateTime, (long)trip.Samples), (only.StartedAt, only.EndedAt, only.Samples)); + Assert.Equal(trip.DistanceKm, only.DistanceKm, 3); + } + + [Fact] + public async Task ARetriedBatchIsCountedOnce() + { + var day = fixture.Day(vehicle: 1, dayOffset: 0); + var pings = day.Pings.Where(p => p.Ts <= day.Trips[0].EndedAt).ToList(); + var batch = TelemetryExport.ToJsonArray(pings); + + await PostAsync(batch); + await PostAsync(batch); // the device did not see the first 202 and sent it again + + Assert.Equal(2L * pings.Count, await RawCountAsync(day.VehicleId)); + Assert.Equal(pings.Count, Assert.Single(await GetTripsAsync(day.VehicleId, day.Date)).Samples); + } + + [Fact] + public async Task APauseOfExactlyTenMinutesKeepsTheTripOneSecondMoreStartsANewOne() + { + // Business rule: more than 10 minutes without a ping ends a trip. + var start = new DateTimeOffset(2026, 9, 14, 7, 0, 0, TimeSpan.Zero); + var times = new[] + { + start, + start.AddSeconds(5), + start.AddSeconds(605), // 600 s gap: same trip + start.AddSeconds(610), + start.AddSeconds(1211), // 601 s gap: new trip + start.AddSeconds(1216), + }; + + await PostAsync(TelemetryExport.ToJsonArray(times.Select((t, i) => Ping("TRK-105", t, odometerKm: 1000 + i)))); + + var trips = await GetTripsAsync("TRK-105", DateOnly.FromDateTime(start.UtcDateTime)); + Assert.Equal( + [(times[0].UtcDateTime, times[3].UtcDateTime, 610L, 4L), (times[4].UtcDateTime, times[5].UtcDateTime, 5L, 2L)], + trips.Select(t => (t.StartedAt, t.EndedAt, t.DurationS, t.Samples))); + } + + [Fact] + public async Task MalformedJsonIsA400ProblemAndWritesNothing() + { + var day = fixture.Day(vehicle: 2, dayOffset: 0); + var batch = TelemetryExport.ToJsonArray(day.Pings.Take(10)); + + using var response = await fixture.Api.PostAsync( + "/api/telemetry", Json(batch[..^20]), TestContext.Current.CancellationToken); + + Assert.Equal(HttpStatusCode.BadRequest, response.StatusCode); + var problem = await response.Content.ReadFromJsonAsync(TestContext.Current.CancellationToken); + Assert.Equal("urn:fleet-telemetry:problem:malformed-telemetry", problem?.Type); + Assert.Equal(0, await RawCountAsync(day.VehicleId)); + } + + [Fact] + public async Task OneInvalidPingRejectsTheWholeBatchWithFieldErrors() + { + var day = fixture.Day(vehicle: 3, dayOffset: 0); + var pings = JsonNode.Parse(TelemetryExport.ToJsonArray(day.Pings.Take(10)))!.AsArray(); + pings[3]!["speed_kmh"] = 412; + pings[7]!["gps"]!["lon"] = -190; + + using var response = await fixture.Api.PostAsync( + "/api/telemetry", Json(pings.ToJsonString()), TestContext.Current.CancellationToken); + + Assert.Equal(HttpStatusCode.BadRequest, response.StatusCode); + var problem = await response.Content.ReadFromJsonAsync(TestContext.Current.CancellationToken); + Assert.Equal(["[3].speed_kmh", "[7].gps.lon"], problem!.Errors.Keys.Order(StringComparer.Ordinal)); + Assert.Equal(0, await RawCountAsync(day.VehicleId)); + } + + private async Task PostAsync(string batch) + { + using var response = await fixture.Api.PostAsync("/api/telemetry", Json(batch), TestContext.Current.CancellationToken); + Assert.Equal(HttpStatusCode.Accepted, response.StatusCode); + return (await response.Content.ReadFromJsonAsync(TestContext.Current.CancellationToken))!.Accepted; + } + + private async Task> GetTripsAsync(string vehicleId, DateOnly date) => + (await fixture.Api.GetFromJsonAsync( + $"/api/vehicles/{vehicleId}/trips?date={date:yyyy-MM-dd}", TestContext.Current.CancellationToken))!.Trips; + + private Task RawCountAsync(string vehicleId) => fixture.QuerySingleAsync( + "SELECT COUNT(*) FROM TELEMETRY_RAW WHERE PAYLOAD:vehicle_id::VARCHAR = :vehicleId", new { vehicleId }); + + private static StringContent Json(string body) => new(body, Encoding.UTF8, "application/json"); + + private static TelemetryPing Ping(string vehicleId, DateTimeOffset ts, double odometerKm) => new( + $"{vehicleId}-{ts:yyyyMMddHHmmss}", + vehicleId, + ts, + new GpsFix(51.9, 4.4, 90), + SpeedKmh: 30, + odometerKm, + new FuelReading(80, 1000), + new EngineReading(1300, 88, [])); + + public sealed class Fixture(SnowflakeTarget target) : FleetDatabaseFixture(target) + { + private IReadOnlyList _fleet = []; + + public VehicleDay Day(int vehicle, int dayOffset) => _fleet + .Where(d => d.VehicleId == _fleet.Select(f => f.VehicleId).Distinct().ElementAt(vehicle)) + .ElementAt(dayOffset); + + protected override async Task LoadAsync(CancellationToken cancellationToken) => + _fleet = new FleetSimulator(seed: 5).Simulate(await VehiclesAsync(4, cancellationToken), new DateOnly(2026, 9, 7), days: 1); + } +} diff --git a/fleet-telemetry/tests/FleetTelemetry.IntegrationTests/VehicleAnalyticsTests.cs b/fleet-telemetry/tests/FleetTelemetry.IntegrationTests/VehicleAnalyticsTests.cs new file mode 100644 index 0000000..fbddc7d --- /dev/null +++ b/fleet-telemetry/tests/FleetTelemetry.IntegrationTests/VehicleAnalyticsTests.cs @@ -0,0 +1,193 @@ +using System.Diagnostics; +using System.Net; +using System.Net.Http.Json; +using System.Text.Json; +using FleetTelemetry.Api.Endpoints; +using FleetTelemetry.Data; +using FleetTelemetry.IntegrationTests.Infrastructure; +using FleetTelemetry.Simulation; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Mvc; +using Microsoft.Extensions.DependencyInjection; + +namespace FleetTelemetry.IntegrationTests; + +/// +/// Trips, driving events and faults over HTTP, compared with what the simulator drove. +/// Four vehicles over two days are bulk-loaded once for the class. +/// +public sealed class VehicleAnalyticsTests(VehicleAnalyticsTests.Fixture fixture) : IClassFixture +{ + private static readonly DateOnly FirstDay = new(2026, 9, 7); + + [Fact] + public async Task TripsAreTheDrivesBetweenIgnitionOffPeriods() + { + foreach (var day in fixture.Fleet) + { + var response = await fixture.Api.GetFromJsonAsync( + $"/api/vehicles/{day.VehicleId}/trips?date={day.Date:yyyy-MM-dd}", TestContext.Current.CancellationToken); + + Assert.NotNull(response); + Assert.Equal(day.Trips.Count, response.Trips.Count); + foreach (var (expected, actual) in day.Trips.Zip(response.Trips)) + { + Assert.Equal(expected.StartedAt.UtcDateTime, actual.StartedAt); + Assert.Equal(expected.EndedAt.UtcDateTime, actual.EndedAt); + Assert.Equal((long)(expected.EndedAt - expected.StartedAt).TotalSeconds, actual.DurationS); + Assert.Equal(expected.DistanceKm, actual.DistanceKm, 3); + Assert.Equal(expected.MaxSpeedKmh, actual.MaxSpeedKmh); + Assert.Equal(expected.Samples, actual.Samples); + } + } + } + + [Fact] + public async Task HarshBrakingEventsAreTheHardStops() + { + Assert.NotEmpty(fixture.Fleet.SelectMany(d => d.HarshBrakes)); // the data must contain some + foreach (var vehicle in fixture.Fleet.GroupBy(d => d.VehicleId)) + { + var response = await fixture.Api.GetFromJsonAsync>( + $"/api/vehicles/{vehicle.Key}/events?type=harsh_braking", TestContext.Current.CancellationToken); + + var expected = vehicle.SelectMany(d => d.HarshBrakes).ToList(); + Assert.NotNull(response); + Assert.Equal(expected.Select(e => e.At.UtcDateTime), response.Events.Select(e => e.OccurredAt)); + Assert.Equal(expected.Select(e => (e.FromKmh, e.ToKmh)), response.Events.Select(e => (e.FromSpeedKmh, e.ToSpeedKmh))); + // Pings are 5 s apart: deceleration = speed drop in m/s over 5 s. + Assert.Equal( + expected.Select(e => Math.Round((e.FromKmh - e.ToKmh) / 3.6 / 5, 2)), + response.Events.Select(e => e.DecelerationMs2)); + } + } + + [Fact] + public async Task IdlingEventsAreTheLoadingDockWaits() + { + Assert.NotEmpty(fixture.Fleet.SelectMany(d => d.Idling)); + foreach (var vehicle in fixture.Fleet.GroupBy(d => d.VehicleId)) + { + var response = await fixture.Api.GetFromJsonAsync>( + $"/api/vehicles/{vehicle.Key}/events?type=idling", TestContext.Current.CancellationToken); + + var expected = vehicle.SelectMany(d => d.Idling).ToList(); + Assert.NotNull(response); + Assert.Equal( + expected.Select(e => (e.StartedAt.UtcDateTime, e.EndedAt.UtcDateTime, (long)(e.EndedAt - e.StartedAt).TotalSeconds)), + response.Events.Select(e => (e.StartedAt, e.EndedAt, e.DurationS))); + } + } + + [Fact] + public async Task EventsCanBeLimitedToOneDay() + { + var vehicle = fixture.Fleet.Where(d => d.Date == FirstDay && d.HarshBrakes.Count > 0).Select(d => d.VehicleId).First(); + var response = await fixture.Api.GetFromJsonAsync>( + $"/api/vehicles/{vehicle}/events?type=harsh_braking&from={FirstDay:yyyy-MM-dd}&to={FirstDay:yyyy-MM-dd}", + TestContext.Current.CancellationToken); + + var expected = fixture.Fleet.Single(d => d.VehicleId == vehicle && d.Date == FirstDay).HarshBrakes; + Assert.NotNull(response); + Assert.Equal(expected.Select(e => e.At.UtcDateTime), response.Events.Select(e => e.OccurredAt)); + } + + [Fact] + public async Task FaultCodesAreUnnestedFromEachPingsDtcArray() + { + var byVehicle = fixture.Fleet.GroupBy(d => d.VehicleId).ToList(); + Assert.Contains(byVehicle, v => v.Any(d => d.Pings.Any(p => p.Engine.Dtc.Count > 0))); + Assert.Contains(byVehicle, v => v.All(d => d.Pings.All(p => p.Engine.Dtc.Count == 0))); + + foreach (var vehicle in byVehicle) + { + var response = await fixture.Api.GetFromJsonAsync( + $"/api/vehicles/{vehicle.Key}/faults", TestContext.Current.CancellationToken); + + Assert.NotNull(response); + Assert.Equal( + Expected.Faults(vehicle.SelectMany(d => d.Pings)), + response.Faults.Select(f => new Expected.Fault(f.Code, f.Severity, (int)f.Occurrences, f.FirstSeen, f.LastSeen, f.MaxCoolantC))); + } + } + + [Fact] + public async Task EveryQueryIsTaggedWithTheRouteAndTheCallersTraceId() + { + var traceId = ActivityTraceId.CreateRandom().ToHexString(); + using var request = new HttpRequestMessage(HttpMethod.Get, "/api/vehicles/TRK-101/trips?date=2026-09-07"); + request.Headers.Add("traceparent", $"00-{traceId}-{ActivitySpanId.CreateRandom().ToHexString()}-01"); + using (var response = await fixture.Api.SendAsync(request, TestContext.Current.CancellationToken)) + { + response.EnsureSuccessStatusCode(); + } + + var queries = await fixture.QueryAsync<(string Text, string Tag)>( + """ + SELECT QUERY_TEXT, QUERY_TAG + FROM TABLE(INFORMATION_SCHEMA.QUERY_HISTORY()) + WHERE QUERY_TAG LIKE :pattern AND QUERY_TEXT NOT ILIKE 'ALTER SESSION%' + """, + new { pattern = $"%{traceId}%" }); + + // The registry lookup and the trips query, both from this one request. + Assert.Contains(queries, q => q.Text.Contains("FROM VEHICLES", StringComparison.Ordinal)); + Assert.Contains(queries, q => q.Text.Contains("TRIP_NO", StringComparison.Ordinal)); + Assert.All(queries, q => + { + using var tag = JsonDocument.Parse(q.Tag); + Assert.Equal("GET /api/vehicles/{vehicleId}/trips", tag.RootElement.GetProperty("op").GetString()); + Assert.Equal(traceId, tag.RootElement.GetProperty("id").GetString()); + }); + } + + [Theory] + [InlineData("/api/vehicles/TRK-999/trips?date=2026-09-07")] + [InlineData("/api/vehicles/TRK-999/events?type=harsh_braking")] + [InlineData("/api/vehicles/TRK-999/faults")] + public async Task AnUnknownVehicleIsA404Problem(string url) + { + using var response = await fixture.Api.GetAsync(url, TestContext.Current.CancellationToken); + + Assert.Equal(HttpStatusCode.NotFound, response.StatusCode); + Assert.Equal("application/problem+json", response.Content.Headers.ContentType?.MediaType); + var problem = await response.Content.ReadFromJsonAsync(TestContext.Current.CancellationToken); + Assert.Equal("urn:fleet-telemetry:problem:vehicle-not-found", problem?.Type); + Assert.Equal("No vehicle 'TRK-999' in the fleet registry.", problem?.Detail); + } + + [Theory] + [InlineData("/api/vehicles/TRK-101/trips", "date")] // required + [InlineData("/api/vehicles/TRK-101/trips?date=yesterday", "date")] + [InlineData("/api/vehicles/TRK-101/events?type=speeding", "type")] + [InlineData("/api/vehicles/TRK-101/faults?from=2026-09-08&to=2026-09-07", "to")] + [InlineData("/api/fleet/daily?from=2026-09-08&to=2026-09-07", "to")] + public async Task InvalidQueryParametersAreA400Problem(string url, string parameter) + { + using var response = await fixture.Api.GetAsync(url, TestContext.Current.CancellationToken); + + Assert.Equal(HttpStatusCode.BadRequest, response.StatusCode); + Assert.Equal("application/problem+json", response.Content.Headers.ContentType?.MediaType); + var problem = await response.Content.ReadFromJsonAsync(TestContext.Current.CancellationToken); + Assert.NotNull(problem); + Assert.Equal("urn:fleet-telemetry:problem:validation", problem.Type); + Assert.True( + problem.Errors.ContainsKey(parameter) || (problem.Detail?.Contains(parameter, StringComparison.Ordinal) ?? false), + $"Expected the problem to name '{parameter}', got errors [{string.Join(", ", problem.Errors.Keys)}], detail '{problem.Detail}'."); + } + + public sealed class Fixture(SnowflakeTarget target) : FleetDatabaseFixture(target) + { + public IReadOnlyList Fleet { get; private set; } = []; + + protected override async Task LoadAsync(CancellationToken cancellationToken) + { + Fleet = new FleetSimulator(seed: 3).Simulate(await VehiclesAsync(4, cancellationToken), FirstDay, days: 2); + var directory = Path.Combine(Path.GetTempPath(), $"fleet-analytics-{Guid.NewGuid():N}"); + TelemetryExport.WriteNdjson(directory, Fleet); + await using var scope = CreateScope(); + await scope.ServiceProvider.GetRequiredService().RunAsync(directory, cancellationToken); + Directory.Delete(directory, recursive: true); + } + } +} diff --git a/fleet-telemetry/tests/FleetTelemetry.IntegrationTests/packages.lock.json b/fleet-telemetry/tests/FleetTelemetry.IntegrationTests/packages.lock.json new file mode 100644 index 0000000..6de4786 --- /dev/null +++ b/fleet-telemetry/tests/FleetTelemetry.IntegrationTests/packages.lock.json @@ -0,0 +1,823 @@ +{ + "version": 2, + "dependencies": { + "net10.0": { + "Microsoft.AspNetCore.Mvc.Testing": { + "type": "Direct", + "requested": "[10.0.13, )", + "resolved": "10.0.13", + "contentHash": "AiQkwLuYCtZoWD1lm6ll/tqVf+u1GVdVJw+EEBQnCOgdofcwpMCnZ6h8HEwD04rOdmF1v+pVfGzr9HFCmq3zdw==", + "dependencies": { + "Microsoft.AspNetCore.TestHost": "10.0.13", + "Microsoft.Extensions.DependencyModel": "10.0.13", + "Microsoft.Extensions.Hosting": "10.0.13" + } + }, + "Testcontainers": { + "type": "Direct", + "requested": "[4.16.0, )", + "resolved": "4.16.0", + "contentHash": "JmMUZk6ZXQovbGQjaMB+YxOR0PbJTdYjOYjRcrm5LrA2dhLVMuCbhVGGunzCGIQfFMUB5Lno2LPOHMMuaGmuIg==", + "dependencies": { + "Docker.DotNet.Enhanced": "4.3.3", + "Docker.DotNet.Enhanced.X509": "4.3.3", + "Microsoft.Extensions.Logging.Abstractions": "8.0.3", + "SSH.NET": "2026.0.0", + "SharpZipLib": "1.4.2" + } + }, + "xunit.v3": { + "type": "Direct", + "requested": "[4.0.2, )", + "resolved": "4.0.2", + "contentHash": "TdjM2OAx7w/0E88E6PgaigLoTbQaxbSR3HF/8p92plchTjf29jPHW27ylvEOz+Nx8CvPwWKGQlOR1fDv6/Qh5A==", + "dependencies": { + "xunit.v3.mtp-v2": "[4.0.2]" + } + }, + "Apache.Arrow": { + "type": "Transitive", + "resolved": "23.0.0", + "contentHash": "cU4Zm7byFwnb82Ea9ZRvVnq2nJNf/TF9j+rj/MHlvu9RBYMqYtu4sJaR1JmMh3MmSIuwMafVg7PfJxPAMv9dGw==", + "dependencies": { + "Apache.Arrow.Scalars": "23.0.0" + } + }, + "Apache.Arrow.Scalars": { + "type": "Transitive", + "resolved": "23.0.0", + "contentHash": "i53VmQ505ASXsStf7y29Z5qvhN6PGgFGNb6UUU7z1WmbzEgSjKQuIJkKKVbh+K5rtXRiKyr1Ah3Rmvbeb2BEHQ==" + }, + "AWSSDK.Core": { + "type": "Transitive", + "resolved": "4.0.100.8", + "contentHash": "xnuBVLQBmYQXsDZJ9mq2UDSFZm3xgO5oUb4/UR8p0UO7tG6heDhsLLI1NzZhIVAKyfW0tXg9hn9a/ek018TOVw==" + }, + "AWSSDK.S3": { + "type": "Transitive", + "resolved": "4.0.101.4", + "contentHash": "TYFuatWECzCbj/Lu1SsANucgpU/Br5YJ5Padl3YEdNJGfZqAcx3QemJw2BUopAsYTJ3e7TFpWhpNWha3dB9/dw==", + "dependencies": { + "AWSSDK.Core": "[4.0.100.8, 5.0.0)" + } + }, + "Azure.Core": { + "type": "Transitive", + "resolved": "1.55.0", + "contentHash": "c7femvYS/xEUrLP1sNZN+DoQZmx3X+G3ZXtOOz0RL/7cGMCM3JVqepVmRd3AwM4IK8AtTGS4GOigCO+d/zaSsQ==", + "dependencies": { + "Microsoft.Bcl.AsyncInterfaces": "10.0.3", + "Microsoft.Extensions.Configuration.Abstractions": "10.0.3", + "Microsoft.Extensions.Hosting.Abstractions": "10.0.3", + "Microsoft.Identity.Client": "4.83.1", + "Microsoft.Identity.Client.Extensions.Msal": "4.83.1", + "System.ClientModel": "1.11.0", + "System.Memory.Data": "10.0.3" + } + }, + "Azure.Storage.Blobs": { + "type": "Transitive", + "resolved": "12.29.1", + "contentHash": "pWh7xZBto8YcwiO853AB3uijTfVqs9PDte0Bu9/Zd8O9nwKiitWm0Jej1vsNkmYUzeG1552f8vJPjmtW30pBUQ==", + "dependencies": { + "Azure.Core": "1.55.0", + "Azure.Storage.Common": "12.28.0" + } + }, + "Azure.Storage.Common": { + "type": "Transitive", + "resolved": "12.28.0", + "contentHash": "5l8YhNrks38zKLGFW2BBFLwieyamH/xauABSASsdpZv79G0f+n2n22IjkRmUqCmALSupkwRHh2LOhbW3yj5Qgw==", + "dependencies": { + "Azure.Core": "1.55.0", + "System.IO.Hashing": "10.0.3" + } + }, + "BouncyCastle.Cryptography": { + "type": "Transitive", + "resolved": "2.7.0", + "contentHash": "U+12df8UEWHgBi04YVf/Lgi2dy3SItlIYvHjjEVa/BngCQIzDCDRBk50DDByCfDvSbe5pRNFr3b7UrVK2kMcLw==" + }, + "Docker.DotNet.Enhanced": { + "type": "Transitive", + "resolved": "4.3.3", + "contentHash": "nGicLwvd42FhRk+khY5uS6cx49ErNdwYKnYBg0F4m4BDKLp/R77AVmmN9xAiqI3W/wN5ZCHkdUhgxf5ORkZuFQ==", + "dependencies": { + "Docker.DotNet.Enhanced.Handler.Abstractions": "4.3.3", + "Docker.DotNet.Enhanced.LegacyHttp": "4.3.3", + "Docker.DotNet.Enhanced.NPipe": "4.3.3", + "Docker.DotNet.Enhanced.NativeHttp": "4.3.3", + "Docker.DotNet.Enhanced.Unix": "4.3.3", + "Microsoft.Extensions.Logging.Abstractions": "8.0.3" + } + }, + "Docker.DotNet.Enhanced.Handler.Abstractions": { + "type": "Transitive", + "resolved": "4.3.3", + "contentHash": "9Cp8hOgtynixcDoAs9lnEaQosluojSYmiW3fsLsLIVfZjlq/fznSIZNUhnmyT4Xo1Iyuok/y49WL/25O47u0Pw==", + "dependencies": { + "Microsoft.Extensions.Logging.Abstractions": "8.0.3" + } + }, + "Docker.DotNet.Enhanced.LegacyHttp": { + "type": "Transitive", + "resolved": "4.3.3", + "contentHash": "7j3M16emv9PAQN7VwFn23xLYNj8GJmwPOcogveHkaWnOCqiC+anRaNKQwqIBNApM1AuwZKivehTKTPmmrjUUnw==", + "dependencies": { + "Docker.DotNet.Enhanced.Handler.Abstractions": "4.3.3" + } + }, + "Docker.DotNet.Enhanced.NativeHttp": { + "type": "Transitive", + "resolved": "4.3.3", + "contentHash": "iNzK+jRFeEMobSA7l/h4ARwCKOOefOWtVN5/RB0ft6/6H6IQXvVUuOgGyZAjYLBT7TsyClRYno2B904f3dtBuQ==", + "dependencies": { + "Docker.DotNet.Enhanced.Handler.Abstractions": "4.3.3" + } + }, + "Docker.DotNet.Enhanced.NPipe": { + "type": "Transitive", + "resolved": "4.3.3", + "contentHash": "ZTLYufuEfY0e6qLOgeH9QgXx2KYuoABRVaY5A8rsggyLgYqbDj9rCRfVAhHPCUv83S7pVxDHy+Tvm/BnxjWVpg==", + "dependencies": { + "Docker.DotNet.Enhanced.Handler.Abstractions": "4.3.3" + } + }, + "Docker.DotNet.Enhanced.Unix": { + "type": "Transitive", + "resolved": "4.3.3", + "contentHash": "ypo8qNbmvHw1t9VfpRTMogCw2vht6VjkXzlGYUUeP2H2bf83USURdla1maW1njn2oq2rfLUFOGMfmt+A37QU2w==", + "dependencies": { + "Docker.DotNet.Enhanced.Handler.Abstractions": "4.3.3" + } + }, + "Docker.DotNet.Enhanced.X509": { + "type": "Transitive", + "resolved": "4.3.3", + "contentHash": "oBDibWezEv4hgj3RIQxI3DVcxkNV1MdrD0d/jhjUu+h3DL+qc0wlkQva15kkwMatXmC/hWp1VP0DMoFXe+BmEw==", + "dependencies": { + "Docker.DotNet.Enhanced.Handler.Abstractions": "4.3.3" + } + }, + "Google.Api.Gax": { + "type": "Transitive", + "resolved": "4.13.1", + "contentHash": "ujDpZk7O2VqAEIqcSlhH0FKzVpGZWly/qcNjHxpNUrL445Tei9PHnu4V1pwW2WDiMDvo3TpL1Bi4DeU525Afrg==", + "dependencies": { + "Microsoft.Bcl.AsyncInterfaces": "6.0.0", + "Newtonsoft.Json": "13.0.4" + } + }, + "Google.Api.Gax.Rest": { + "type": "Transitive", + "resolved": "4.13.1", + "contentHash": "wDfFZXKoHC6sy+Yfub/4cLQOb5ozBVPqDOrxCUAeTfku4d96ILqgzulJSdkPHbiT3oWB3hlYTdJgUwNSJjyo+A==", + "dependencies": { + "Google.Api.Gax": "4.13.1", + "Google.Apis.Auth": "1.73.0", + "Microsoft.Extensions.DependencyInjection.Abstractions": "6.0.0" + } + }, + "Google.Apis": { + "type": "Transitive", + "resolved": "1.74.0", + "contentHash": "Z6hSkmRD6V8/raSBqn/EsPpNop6suaLaGE9DbSY3yRE5G47l4zYqrJgapcLARk2Z3TsQUDmWz+w/tob3wuVt3g==", + "dependencies": { + "Google.Apis.Core": "1.74.0" + } + }, + "Google.Apis.Auth": { + "type": "Transitive", + "resolved": "1.74.0", + "contentHash": "43H4foh3zfWgboK8QnJjIfPQuBiS794TxqQLDfeE3erQSluKmEN0qeDuwxLTpSr9Mtwj76S4LWxPneuSbC6fNg==", + "dependencies": { + "Google.Apis": "1.74.0", + "Google.Apis.Core": "1.74.0", + "System.Management": "7.0.2" + } + }, + "Google.Apis.Core": { + "type": "Transitive", + "resolved": "1.74.0", + "contentHash": "gWOIiks6yo2TRliIyy10V18BJOO2BfUSPJFiTW6Tfap4tm3hFj7nvYU3XIfwzAxwoIy6nsBoqVDt9iBDSXrvYQ==", + "dependencies": { + "Newtonsoft.Json": "13.0.4" + } + }, + "Google.Apis.Storage.v1": { + "type": "Transitive", + "resolved": "1.74.0.4115", + "contentHash": "9J3YPPSG53xwQLzdQ304dI+DMtVxJDpEazA2QQ2LA8IJoM4VUu4DpIxyhTGxpnAK4Db7s4LwzeNi6NkPKM3HzQ==", + "dependencies": { + "Google.Apis": "1.74.0", + "Google.Apis.Auth": "1.74.0" + } + }, + "Google.Cloud.Storage.V1": { + "type": "Transitive", + "resolved": "4.15.0", + "contentHash": "TVlJsaIkMDwHbZPCcLFFKduFcFD0yJL1jsZmV/V/FMjv5iok/eoC0oy8/VHjxfmWmNGkJLPLMAe5MrFLtY7hzg==", + "dependencies": { + "Google.Api.Gax.Rest": "[4.13.1, 5.0.0)", + "Google.Apis.Storage.v1": "[1.74.0.4115, 2.0.0)" + } + }, + "Microsoft.ApplicationInsights": { + "type": "Transitive", + "resolved": "2.23.0", + "contentHash": "nWArUZTdU7iqZLycLKWe0TDms48KKGE6pONH2terYNa8REXiqixrMOkf1sk5DHGMaUTqONU2YkS4SAXBhLStgw==" + }, + "Microsoft.AspNetCore.TestHost": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "8xzbDfEOWg5GRp5byA42Axr4TbockbxCvdHTYk27ZIdst79DO0CLBp/Zw5sduK2goRii7of4EeuTjG/yPH4tsg==" + }, + "Microsoft.Bcl.AsyncInterfaces": { + "type": "Transitive", + "resolved": "10.0.3", + "contentHash": "TV62UsrJZPX6gbt3c4WrtXh7bmaDIcMqf9uft1cc4L6gJXOU07hDGEh+bFQh/L2Az0R1WVOkiT66lFqS6G2NmA==" + }, + "Microsoft.Bcl.Cryptography": { + "type": "Transitive", + "resolved": "10.0.2", + "contentHash": "LG9Yll3B5aNpxv0+D47g6LiOiKBIlodhcHdQwcYzo8VeexFLGqx5ymetmA2aBRyo9cCcWsQWrFsdbsr8LvmWDw==" + }, + "Microsoft.Extensions.Configuration.Abstractions": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "DY4C2qzTt2lOIY3VoscBH/TCwTolI5DKOfgJLuI+k1QUvRNT5vTMS58AOerNiihdVRStO0wEpLeRWA8jytzHBQ==", + "dependencies": { + "Microsoft.Extensions.Primitives": "10.0.13" + } + }, + "Microsoft.Extensions.Configuration.Binder": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "yJJl/b0+W+GEWbu+7viw6C3h2BoNkB69Vlz7s/4e+hm4mYbJqfhdcDSi9sCXJw47blsBaiQd3UDPgDtop92+ug==", + "dependencies": { + "Microsoft.Extensions.Configuration": "10.0.13", + "Microsoft.Extensions.Configuration.Abstractions": "10.0.13" + } + }, + "Microsoft.Extensions.Configuration.CommandLine": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "RwdNZ9JdGs8eZmeL5qRe7ugp0Irk3KVkWpC/w1c8kjJwu1R4HvCXYz9kwHZxn45WiaRBG5lphQKtqnBaJOQC+Q==", + "dependencies": { + "Microsoft.Extensions.Configuration": "10.0.13", + "Microsoft.Extensions.Configuration.Abstractions": "10.0.13" + } + }, + "Microsoft.Extensions.Configuration.EnvironmentVariables": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "qJK2dm9/AxyPVfqqBaZtjkUFl2hnBR/CQnMJtSO5LdEoyATMqvZjeQVY+8l0oa6PM6qlVe28iMEyXIRQ0wyZjA==", + "dependencies": { + "Microsoft.Extensions.Configuration": "10.0.13", + "Microsoft.Extensions.Configuration.Abstractions": "10.0.13" + } + }, + "Microsoft.Extensions.Configuration.FileExtensions": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "kwC1DY55CsqPmDtJAz+7alD3G+gJcM+XwHSZrcRO8mJGYB9hRq1qC6ZH8DL0SPW9WtJ/Bae5ikzPic8SxSv6Vw==", + "dependencies": { + "Microsoft.Extensions.Configuration": "10.0.13", + "Microsoft.Extensions.Configuration.Abstractions": "10.0.13", + "Microsoft.Extensions.FileProviders.Abstractions": "10.0.13", + "Microsoft.Extensions.FileProviders.Physical": "10.0.13", + "Microsoft.Extensions.Primitives": "10.0.13" + } + }, + "Microsoft.Extensions.Configuration.Json": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "acCWM2+WR67kBAz6gAhD2zG+IZH9Q2VpKv5le7Msiu0vAMsnnLXpnV3BAzxd8/i+uOEo/dfb2XRYTynRVTDvEg==", + "dependencies": { + "Microsoft.Extensions.Configuration": "10.0.13", + "Microsoft.Extensions.Configuration.Abstractions": "10.0.13", + "Microsoft.Extensions.Configuration.FileExtensions": "10.0.13", + "Microsoft.Extensions.FileProviders.Abstractions": "10.0.13" + } + }, + "Microsoft.Extensions.Configuration.UserSecrets": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "kQ/Tx972unP+ueQL8SSninthIj6w6XP2o0VnJPFC5HQIimPMwEN704MOspEgDkmm4FUL5SuxLEU0bNiW4HEZFA==", + "dependencies": { + "Microsoft.Extensions.Configuration.Abstractions": "10.0.13", + "Microsoft.Extensions.Configuration.Json": "10.0.13", + "Microsoft.Extensions.FileProviders.Abstractions": "10.0.13", + "Microsoft.Extensions.FileProviders.Physical": "10.0.13" + } + }, + "Microsoft.Extensions.DependencyInjection": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "4tnWU9vrU/A8NwYe9Z6wvieZ0EBb/uM82LAT+edNhdepOBWREZ7M7mzb941EIDPJaeOFdbVrDd5+7s9e89U6bQ==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.13" + } + }, + "Microsoft.Extensions.DependencyInjection.Abstractions": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "G2GrGWNAPZBsii2F6AU8n+JT2tC1/I0Ml/e9ZyNCgPjf3hZqronP7o0X91aAynCmVeae8KkSIUNXXeXCNRmdVQ==" + }, + "Microsoft.Extensions.DependencyModel": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "6TyllZsToy3S/Ziu67GddljRTttuVJ8bbrbVUrsdHqH5678Oax5MpaEubxLoPoWLvgUBYo4KfcAig0mkBtktXQ==" + }, + "Microsoft.Extensions.Diagnostics": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "J9ODOrffj1mK3Et69elNsLGRguesPxvThTdVpFmQX7fLNPhIOgqHQfLdGnTBEwNAV3yf8yiyVlpDZNw2qN87Nw==", + "dependencies": { + "Microsoft.Extensions.Configuration": "10.0.13", + "Microsoft.Extensions.Diagnostics.Abstractions": "10.0.13", + "Microsoft.Extensions.Options.ConfigurationExtensions": "10.0.13" + } + }, + "Microsoft.Extensions.Diagnostics.Abstractions": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "oZzqZze8ed1fLS20HtIrkPxQXGRG/koq09srIw2ptnDHm9RHfwB++d35hosB/cXc55TH7Pshv85O0txongjCIg==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.13", + "Microsoft.Extensions.Options": "10.0.13" + } + }, + "Microsoft.Extensions.FileProviders.Abstractions": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "BZDwtN3uoWRghDi5lRRZPDjMMCg2Ds7J6YEYjqka/TLjqnzquEpXyvchrIMeoubwT1ns23ow6PPeAoBHJO81Rw==", + "dependencies": { + "Microsoft.Extensions.Primitives": "10.0.13" + } + }, + "Microsoft.Extensions.FileProviders.Physical": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "U7HkgGCNPICN5k7NVK6dmLsSPSPeO/EbNcuxOl/5qdC9Ahjp9UZWKOvlnUC+rYurmoxIlwyv9e5j0BaQlTOELA==", + "dependencies": { + "Microsoft.Extensions.FileProviders.Abstractions": "10.0.13", + "Microsoft.Extensions.FileSystemGlobbing": "10.0.13", + "Microsoft.Extensions.Primitives": "10.0.13" + } + }, + "Microsoft.Extensions.FileSystemGlobbing": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "vQVzmAbnzlhXSXA2caYnlxtG9GXxEwtj5bODiWLu06A6iqeJmfBtlfKjuOSJGRy0/CFaNUrm7u9nxEUlljc2Rg==" + }, + "Microsoft.Extensions.Hosting.Abstractions": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "zIrbe9vkIFXxW3mOApIOWXw+xMwX7Cgsptr3ZSlglJt4hW2qp4fB/JU1NHgAevteBNH4PicvU9DsFPHxlnnloA==", + "dependencies": { + "Microsoft.Extensions.Configuration.Abstractions": "10.0.13", + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.13", + "Microsoft.Extensions.Diagnostics.Abstractions": "10.0.13", + "Microsoft.Extensions.FileProviders.Abstractions": "10.0.13", + "Microsoft.Extensions.Logging.Abstractions": "10.0.13" + } + }, + "Microsoft.Extensions.Logging": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "fqbiPy4nUp4H2XHKAVQXKN2y3T2h4cFIx1DAAgE6iY9OHkN/OA1cKFi+geJKRWGuX6pS5SFbsjIPWHJqrxE0eg==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection": "10.0.13", + "Microsoft.Extensions.Logging.Abstractions": "10.0.13", + "Microsoft.Extensions.Options": "10.0.13" + } + }, + "Microsoft.Extensions.Logging.Configuration": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "OyYJOaE9WbBx1Hp+Dd2zrqeFM0JwH5M/x0VL6tNUv3V18vIAVpix8UjhvY9eW/iXpguEHy68JO/Z9YkBOwftIw==", + "dependencies": { + "Microsoft.Extensions.Configuration": "10.0.13", + "Microsoft.Extensions.Configuration.Abstractions": "10.0.13", + "Microsoft.Extensions.Configuration.Binder": "10.0.13", + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.13", + "Microsoft.Extensions.Logging": "10.0.13", + "Microsoft.Extensions.Logging.Abstractions": "10.0.13", + "Microsoft.Extensions.Options": "10.0.13", + "Microsoft.Extensions.Options.ConfigurationExtensions": "10.0.13" + } + }, + "Microsoft.Extensions.Logging.Console": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "i9rn0rHkiCxfNVbf4DRG7LOFcKPy/BioAxS07w6daWvIpLNRjskqDJcGGaYVFWndsq0iTRaS8Msry0t5fRYQug==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.13", + "Microsoft.Extensions.Logging": "10.0.13", + "Microsoft.Extensions.Logging.Abstractions": "10.0.13", + "Microsoft.Extensions.Logging.Configuration": "10.0.13", + "Microsoft.Extensions.Options": "10.0.13" + } + }, + "Microsoft.Extensions.Logging.Debug": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "yBig0DNMq+3UgliwxYlUC48dg0fPqvqI5B3ljzmLTrpR6maoKTOO5i9QnPgQUmQoaQWZMgu74fuF6oSDihb6pg==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.13", + "Microsoft.Extensions.Logging": "10.0.13", + "Microsoft.Extensions.Logging.Abstractions": "10.0.13" + } + }, + "Microsoft.Extensions.Logging.EventLog": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "OAK40lXhgMuvyNwvzW3fxLvvucN8yr5XS7dQRcPmtnPf+Pt1lpbMgao5T4Wa6VdFagtIeDZNgPtV5Ig4xstziQ==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.13", + "Microsoft.Extensions.Logging": "10.0.13", + "Microsoft.Extensions.Logging.Abstractions": "10.0.13", + "Microsoft.Extensions.Options": "10.0.13", + "System.Diagnostics.EventLog": "10.0.13" + } + }, + "Microsoft.Extensions.Logging.EventSource": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "SxJu3oczk+TJtHvpgAmCVzEkzx69/obA6GZCbGrbqYqc981YOAyfEprySBdFX7Yc5qatHJKHNTlpGeeBAQkZcA==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.13", + "Microsoft.Extensions.Logging": "10.0.13", + "Microsoft.Extensions.Logging.Abstractions": "10.0.13", + "Microsoft.Extensions.Options": "10.0.13", + "Microsoft.Extensions.Primitives": "10.0.13" + } + }, + "Microsoft.Extensions.Options": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "tmMJgdrE4uhf1JcNs9CV9moAHW+hmDuvgMiPpnQxarKaUxqI2eN33sYK5S4l9Uw+rG9PJvlm4qPxMhvolTDTaQ==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.13", + "Microsoft.Extensions.Primitives": "10.0.13" + } + }, + "Microsoft.Extensions.Primitives": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "ZjTwxzhiw2TMEzIbUDkMF0dfDB2I4NQgIfKD0ELu9/5Mk+SRGdKZcD0/lvBrRCAPo6KN2r0BW86MRancoyJkoA==" + }, + "Microsoft.Identity.Client": { + "type": "Transitive", + "resolved": "4.83.1", + "contentHash": "jOLIrZ3cynoqHLLO1cXplFFabrhrMEYs/EuKHvmCyrOm1axqiVFT6nCSnHxk7w5+d2BeQfCdM12Yf/0X7OeS1g==", + "dependencies": { + "Microsoft.IdentityModel.Abstractions": "8.14.0" + } + }, + "Microsoft.Identity.Client.Extensions.Msal": { + "type": "Transitive", + "resolved": "4.83.1", + "contentHash": "I3k4J4Hj4KbLEFanjeUzzDOVecukETaTgEkJ7h2pP/Yazs6SLp6TVUTo/Eo+ptPXMwvc+iX7rBFtMSUrA7R+Mg==", + "dependencies": { + "Microsoft.Identity.Client": "4.83.1", + "System.Security.Cryptography.ProtectedData": "4.5.0" + } + }, + "Microsoft.IdentityModel.Abstractions": { + "type": "Transitive", + "resolved": "8.21.0", + "contentHash": "XYgEmYpUvng0oYIagDJ+uebboQSnWtCV7q9zMyXlBw6ZqLLCHcmGUb7TpXvQ3HWY4OzbOCUQm2Ac0saPSnwqSw==" + }, + "Microsoft.IdentityModel.JsonWebTokens": { + "type": "Transitive", + "resolved": "8.21.0", + "contentHash": "iiA5uimwlRe4Drd8YKOaB8IQkgGOjxGjO6mu5zCGFjaEBpB6yNOluZ33zJFXiW5MQYRC6ypMnOXsiR5SPrO+tw==", + "dependencies": { + "Microsoft.IdentityModel.Tokens": "8.21.0" + } + }, + "Microsoft.IdentityModel.Logging": { + "type": "Transitive", + "resolved": "8.21.0", + "contentHash": "LgIWUUX386xCbvjq2ex8iXGIu2raDWrrIQ4OyYZKaNWMhzvN8YOTbAYMFJC+iHmKnkNBM8g7mLak9KQodldH3g==", + "dependencies": { + "Microsoft.IdentityModel.Abstractions": "8.21.0" + } + }, + "Microsoft.IdentityModel.Tokens": { + "type": "Transitive", + "resolved": "8.21.0", + "contentHash": "xFnFEl4VrhAJEbqtRrpgfAACgkm6NYo7cWYKYROH00N9/M5OYrRwTP0lHsginnLrqPrseLcSqAueQNRCyGNsdw==", + "dependencies": { + "Microsoft.Bcl.Cryptography": "10.0.2", + "Microsoft.Extensions.Logging.Abstractions": "8.0.0", + "Microsoft.IdentityModel.Logging": "8.21.0" + } + }, + "Microsoft.Testing.Extensions.Telemetry": { + "type": "Transitive", + "resolved": "2.5.1", + "contentHash": "mu2iyoHovwl0q1mGeeRU0h8+LALXC7HqbxniLa1shTKYdJGl+KPoCEgjWIQ8MLzk1tlTGPOfg1JsxdH3RxhMCA==", + "dependencies": { + "Microsoft.ApplicationInsights": "2.23.0", + "Microsoft.Testing.Platform": "[2.5.1, 3.0.0)" + } + }, + "Microsoft.Testing.Extensions.TrxReport.Abstractions": { + "type": "Transitive", + "resolved": "2.5.1", + "contentHash": "DYTZAguaHz0wNJdHDTTGYOXtnlDoLoIbTMDKSApNNcytm55DJcnwpwELfQGLk5DB5QlgaEjNhnGo1Pu5QvL5VA==", + "dependencies": { + "Microsoft.Testing.Platform": "[2.5.1, 3.0.0)" + } + }, + "Microsoft.Testing.Platform": { + "type": "Transitive", + "resolved": "2.5.1", + "contentHash": "FAMaFFWnULW+hDHMb40Qzhcgz9cAU2/EtgRCBNX/DdQoKo0sPG2EiY+w7bQdl/BgHnf/DbpXLbf5Xy0o15M8NA==" + }, + "Microsoft.Testing.Platform.MSBuild": { + "type": "Transitive", + "resolved": "2.5.1", + "contentHash": "lYawcAwY+h76rQmpma2n0HaL8zkOFbbKapso0Eb+uOAUQn7333pIByLftEuiO67csAoZufHNMwi6PqhNemWQXA==", + "dependencies": { + "Microsoft.Testing.Platform": "[2.5.1, 3.0.0)" + } + }, + "Microsoft.Win32.Registry": { + "type": "Transitive", + "resolved": "5.0.0", + "contentHash": "dDoKi0PnDz31yAyETfRntsLArTlVAVzUzCIvvEDsDsucrl33Dl8pIJG06ePTJTI3tGpeyHS9Cq7Foc/s4EeKcg==" + }, + "Mono.Unix": { + "type": "Transitive", + "resolved": "7.1.0-final.1.21458.1", + "contentHash": "Rhxz4A7By8Q0wEgDqR+mioDsYXGrcYMYPiWE9bSaUKMpG8yAGArhetEQV5Ms6KhKCLdQTlPYLBKPZYoKbAvT/g==" + }, + "Newtonsoft.Json": { + "type": "Transitive", + "resolved": "13.0.4", + "contentHash": "pdgNNMai3zv51W5aq268sujXUyx7SNdE2bj1wZcWjAQrKMFZV260lbqYop1d2GM67JI1huLRwxo9ZqnfF/lC6A==" + }, + "SharpZipLib": { + "type": "Transitive", + "resolved": "1.4.2", + "contentHash": "yjj+3zgz8zgXpiiC3ZdF/iyTBbz2fFvMxZFEBPUcwZjIvXOf37Ylm+K58hqMfIBt5JgU/Z2uoUS67JmTLe973A==" + }, + "SSH.NET": { + "type": "Transitive", + "resolved": "2026.0.0", + "contentHash": "Yu9dirPq8l3oaat0+OQ7K0nUf5MmYltpia5UGqsApTG4zTPvBC1cxbNnC3NERij26dUST0A3Ef1QdHSn5ArbWQ==", + "dependencies": { + "BouncyCastle.Cryptography": "2.7.0", + "Microsoft.Extensions.Logging.Abstractions": "8.0.3" + } + }, + "System.ClientModel": { + "type": "Transitive", + "resolved": "1.11.0", + "contentHash": "1Wl32zh7TbvN+HAO8NqDuaN0Ao2Qu/0j0NJSrXGDtpUQsTXQYJ3C6hd9/Ds2IrgP4agMQYFoXB35GZfC21ByHw==", + "dependencies": { + "Microsoft.Extensions.Configuration.Abstractions": "10.0.3", + "Microsoft.Extensions.Hosting.Abstractions": "10.0.3", + "Microsoft.Extensions.Logging.Abstractions": "10.0.3", + "System.Memory.Data": "10.0.3" + } + }, + "System.CodeDom": { + "type": "Transitive", + "resolved": "7.0.0", + "contentHash": "GLltyqEsE5/3IE+zYRP5sNa1l44qKl9v+bfdMcwg+M9qnQf47wK3H0SUR/T+3N4JEQXF3vV4CSuuo0rsg+nq2A==" + }, + "System.Diagnostics.EventLog": { + "type": "Transitive", + "resolved": "10.0.13", + "contentHash": "DkNjtQyPR5Ud/zHnWHdsOuKbFRL9nIC92Q76z0NmHVDoQm40toLtsGFTIWq7toBoVa6fo/DInL3PLxY3Asixyw==" + }, + "System.IdentityModel.Tokens.Jwt": { + "type": "Transitive", + "resolved": "8.21.0", + "contentHash": "dKsTkmlpUf/I6g1Aw9joWaZAahXvC0gWGQvkXW1IjZLSefC/4LhQI0tL0VIs64/NZQbl4XnQvLQyu6c7z1pMxg==", + "dependencies": { + "Microsoft.IdentityModel.JsonWebTokens": "8.21.0", + "Microsoft.IdentityModel.Tokens": "8.21.0" + } + }, + "System.IO.Hashing": { + "type": "Transitive", + "resolved": "10.0.3", + "contentHash": "La6ICwsdTKhVX+LKN+pvFjQRR3LhLwq3uKdi2knjLzRyPYBSydF4cjXidYxIiTcDD6XVYdsBWQEI8ZxiZ/OdIg==" + }, + "System.Management": { + "type": "Transitive", + "resolved": "7.0.2", + "contentHash": "/qEUN91mP/MUQmJnM5y5BdT7ZoPuVrtxnFlbJ8a3kBJGhe2wCzBfnPFtK2wTtEEcf3DMGR9J00GZZfg6HRI6yA==", + "dependencies": { + "System.CodeDom": "7.0.0" + } + }, + "System.Memory.Data": { + "type": "Transitive", + "resolved": "10.0.3", + "contentHash": "MaGhRfGunmrj/nHjtsi9XkhlYJ/ERGWrbA+BiSKNtGnAjc9XlG5EhAvak6VRcX5LYzPF6pBO8nJ613dTgzabig==" + }, + "System.Security.AccessControl": { + "type": "Transitive", + "resolved": "6.0.1", + "contentHash": "IQ4NXP/B3Ayzvw0rDQzVTYsCKyy0Jp9KI6aYcK7UnGVlR9+Awz++TIPCQtPYfLJfOpm8ajowMR09V7quD3sEHw==" + }, + "System.Security.Cryptography.ProtectedData": { + "type": "Transitive", + "resolved": "4.5.0", + "contentHash": "wLBKzFnDCxP12VL9ANydSYhk59fC4cvOr9ypYQLPnAj48NQIhqnjdD2yhP8yEKyBJEjERWS9DisKL7rX5eU25Q==" + }, + "TimeZoneConverter": { + "type": "Transitive", + "resolved": "7.2.0", + "contentHash": "4Poqu79dTmacVETqUJ53g3x4U21V/cmWPSlehdsVE4Q00akavvlRq+SRjcg/CmfO9ibOCjRl6TeUV7hkC5RC4w==" + }, + "Tomlyn.Signed": { + "type": "Transitive", + "resolved": "2.10.1", + "contentHash": "+BHtb+YzcZ8uV9KVxjQE2uq7CJoYfd1q9gDuU2lx+wrrUppsj41lm2/uYQKTchz5Cs0pzvhf8t71bScDKVi8zA==" + }, + "xunit.analyzers": { + "type": "Transitive", + "resolved": "2.2.0", + "contentHash": "dXMIIztipcAyL3a19gtUrih/RLrE33/4VJdICoinYqIXXo1KLtsMPsaQ9dO2pE8xajDguwE2kqVbBqE5/EoJ0Q==" + }, + "xunit.v3.assert": { + "type": "Transitive", + "resolved": "4.0.2", + "contentHash": "5PKRexYlvufWFdXenRcRGKqvaJBBZFJZpqUGlUs9p13eKEgVdOrLpkPknjzUvLY5kEUicK9871DIUDcCPA6AXA==" + }, + "xunit.v3.common": { + "type": "Transitive", + "resolved": "4.0.2", + "contentHash": "++CtT8daIf6qfieQev/8V88U3zViIDWou5cj+oWps1/Hf73UoaY/L0hKyi3v7Zlt3BmbU0pUc73zCB1GiB4nYA==", + "dependencies": { + "Microsoft.Bcl.AsyncInterfaces": "6.0.0" + } + }, + "xunit.v3.core.mtp-v2": { + "type": "Transitive", + "resolved": "4.0.2", + "contentHash": "XyPMcQRsnUQLhmBBRRgYWsuPc8TyiANzcXtqfzyWfDpPuyh3Q0K4oWozZYQKZRNGnqJ97ifD6qUDWhsEEe32uw==", + "dependencies": { + "Microsoft.Testing.Extensions.Telemetry": "2.5.1", + "Microsoft.Testing.Extensions.TrxReport.Abstractions": "2.5.1", + "Microsoft.Testing.Platform": "2.5.1", + "Microsoft.Testing.Platform.MSBuild": "2.5.1", + "xunit.v3.extensibility.core": "[4.0.2]", + "xunit.v3.runner.inproc.console": "[4.0.2]" + } + }, + "xunit.v3.extensibility.core": { + "type": "Transitive", + "resolved": "4.0.2", + "contentHash": "ARTt+pGyd86mdtiFBkKpyBQvRC3u0rr8oidBlII2GsAg7D6TfOsDFCsoQtFij83V5pE57m3whUIqT4xjiARXbQ==", + "dependencies": { + "xunit.v3.common": "[4.0.2]" + } + }, + "xunit.v3.mtp-v2": { + "type": "Transitive", + "resolved": "4.0.2", + "contentHash": "XLV32FhkuvheBStvX0DvjfPEtI+4SOVcIhnlTS+549r8VfVwsjpWdkJtQmQ7D23PA+Zoaqo+sTjBkRSkidC6Tw==", + "dependencies": { + "xunit.analyzers": "2.2.0", + "xunit.v3.assert": "[4.0.2]", + "xunit.v3.core.mtp-v2": "[4.0.2]" + } + }, + "xunit.v3.runner.common": { + "type": "Transitive", + "resolved": "4.0.2", + "contentHash": "YlYKVhTJVcwEgJ6rm59Pjf71lQqKwKxEHqVRxWUuwrwm/nxB/mNFP/lP4vlntDO3GclNnLT9BjYOpJH9gViGQg==", + "dependencies": { + "Microsoft.Win32.Registry": "[5.0.0]", + "System.Security.AccessControl": "[6.0.1]", + "xunit.v3.common": "[4.0.2]" + } + }, + "xunit.v3.runner.inproc.console": { + "type": "Transitive", + "resolved": "4.0.2", + "contentHash": "cg4XNpCN4Ry9nswutknpreN3ckSUBwtjMRmEQSNK/sIKPo6KN5tDGGGCaHDm5HGA4yFVVtTJX65tLCsZArUNfQ==", + "dependencies": { + "xunit.v3.extensibility.core": "[4.0.2]", + "xunit.v3.runner.common": "[4.0.2]" + } + }, + "fleettelemetry.api": { + "type": "Project", + "dependencies": { + "FleetTelemetry.Data": "[1.0.0, )" + } + }, + "fleettelemetry.core": { + "type": "Project" + }, + "fleettelemetry.data": { + "type": "Project", + "dependencies": { + "Dapper": "[2.1.89, )", + "FleetTelemetry.Core": "[1.0.0, )", + "Microsoft.Extensions.Configuration": "[10.0.13, )", + "Microsoft.Extensions.Logging.Abstractions": "[10.0.13, )", + "Microsoft.Extensions.Options.ConfigurationExtensions": "[10.0.13, )", + "Snowflake.Data": "[6.4.0, )" + } + }, + "Dapper": { + "type": "CentralTransitive", + "requested": "[2.1.89, )", + "resolved": "2.1.89", + "contentHash": "1VjFE664wnxvuhlgP8kS4PeFFF4jqD3VkbIHW6Zlc52zuibgcz5ICnIHcQXlTp6pURpPNGLV0ZpNe5x9KePJnA==" + }, + "Microsoft.Extensions.Configuration": { + "type": "CentralTransitive", + "requested": "[10.0.13, )", + "resolved": "10.0.13", + "contentHash": "inhujYI/nikhmLxYI6ZEcaCgZ28ipJsKMPPxxr2XWf3vM0Q2P+E9R3hDHABZOLBxzD3gjpgcbSH/+hd2+gvI2A==", + "dependencies": { + "Microsoft.Extensions.Configuration.Abstractions": "10.0.13", + "Microsoft.Extensions.Primitives": "10.0.13" + } + }, + "Microsoft.Extensions.Hosting": { + "type": "CentralTransitive", + "requested": "[10.0.13, )", + "resolved": "10.0.13", + "contentHash": "Q+hmPoDqeHipHyJcY7mDCQiLDUcn9jIgjoCThcgaOgjLTTT3ApTe8p6M2+twsDyfKuuCsKhSNlnNjaaggtXxRA==", + "dependencies": { + "Microsoft.Extensions.Configuration": "10.0.13", + "Microsoft.Extensions.Configuration.Abstractions": "10.0.13", + "Microsoft.Extensions.Configuration.Binder": "10.0.13", + "Microsoft.Extensions.Configuration.CommandLine": "10.0.13", + "Microsoft.Extensions.Configuration.EnvironmentVariables": "10.0.13", + "Microsoft.Extensions.Configuration.FileExtensions": "10.0.13", + "Microsoft.Extensions.Configuration.Json": "10.0.13", + "Microsoft.Extensions.Configuration.UserSecrets": "10.0.13", + "Microsoft.Extensions.DependencyInjection": "10.0.13", + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.13", + "Microsoft.Extensions.Diagnostics": "10.0.13", + "Microsoft.Extensions.FileProviders.Abstractions": "10.0.13", + "Microsoft.Extensions.FileProviders.Physical": "10.0.13", + "Microsoft.Extensions.Hosting.Abstractions": "10.0.13", + "Microsoft.Extensions.Logging": "10.0.13", + "Microsoft.Extensions.Logging.Abstractions": "10.0.13", + "Microsoft.Extensions.Logging.Configuration": "10.0.13", + "Microsoft.Extensions.Logging.Console": "10.0.13", + "Microsoft.Extensions.Logging.Debug": "10.0.13", + "Microsoft.Extensions.Logging.EventLog": "10.0.13", + "Microsoft.Extensions.Logging.EventSource": "10.0.13", + "Microsoft.Extensions.Options": "10.0.13" + } + }, + "Microsoft.Extensions.Logging.Abstractions": { + "type": "CentralTransitive", + "requested": "[10.0.13, )", + "resolved": "10.0.13", + "contentHash": "aSZEhEig4fvNiw2zs8Z6NyDlniKtq6YjcF3DyHuqTEHcZ2VKA/6itMPZ+dW+yWABWExlDcQwwdbqX1Rcb7W8WQ==", + "dependencies": { + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.13" + } + }, + "Microsoft.Extensions.Options.ConfigurationExtensions": { + "type": "CentralTransitive", + "requested": "[10.0.13, )", + "resolved": "10.0.13", + "contentHash": "xVO5HhATc09WXuS/RaDmx8ZeWS0mTpT1L0tkEDMSBHtAnNpNsDr0ge0wIO2jinCWWqpq5dL1tRTzpYNVlvyqLw==", + "dependencies": { + "Microsoft.Extensions.Configuration.Abstractions": "10.0.13", + "Microsoft.Extensions.Configuration.Binder": "10.0.13", + "Microsoft.Extensions.DependencyInjection.Abstractions": "10.0.13", + "Microsoft.Extensions.Options": "10.0.13", + "Microsoft.Extensions.Primitives": "10.0.13" + } + }, + "Snowflake.Data": { + "type": "CentralTransitive", + "requested": "[6.4.0, )", + "resolved": "6.4.0", + "contentHash": "9zGrm/bD5h+DVha41TKy7soNDqw1bObAXCz9fQdhMJ0wKbj9IRuZvE+ZUvflBMq2mnjkgbNdiGeKz7qixgqbJA==", + "dependencies": { + "AWSSDK.S3": "4.0.101.4", + "Apache.Arrow": "23.0.0", + "Azure.Storage.Blobs": "12.29.1", + "BouncyCastle.Cryptography": "2.7.0", + "Google.Cloud.Storage.V1": "4.15.0", + "Microsoft.Extensions.Logging": "9.0.5", + "Mono.Unix": "7.1.0-final.1.21458.1", + "Newtonsoft.Json": "13.0.4", + "System.IdentityModel.Tokens.Jwt": "8.21.0", + "TimeZoneConverter": "7.2.0", + "Tomlyn.Signed": "2.10.1" + } + } + } + } +} \ No newline at end of file diff --git a/fleet-telemetry/tests/FleetTelemetry.IntegrationTests/xunit.runner.json b/fleet-telemetry/tests/FleetTelemetry.IntegrationTests/xunit.runner.json new file mode 100644 index 0000000..1270e8b --- /dev/null +++ b/fleet-telemetry/tests/FleetTelemetry.IntegrationTests/xunit.runner.json @@ -0,0 +1,4 @@ +{ + "$schema": "https://xunit.net/schema/current/xunit.runner.schema.json", + "maxParallelThreads": 2 +} diff --git a/fleet-telemetry/tests/FleetTelemetry.UnitTests/DrivingEventRulesTests.cs b/fleet-telemetry/tests/FleetTelemetry.UnitTests/DrivingEventRulesTests.cs new file mode 100644 index 0000000..9ac4f8f --- /dev/null +++ b/fleet-telemetry/tests/FleetTelemetry.UnitTests/DrivingEventRulesTests.cs @@ -0,0 +1,33 @@ +using FleetTelemetry.Rules; + +namespace FleetTelemetry.UnitTests; + +public sealed class DrivingEventRulesTests +{ + private static readonly TimeSpan FiveSeconds = TimeSpan.FromSeconds(5); + + [Fact] + public void DecelerationIsTheSpeedDropInMetresPerSecondPerSecond() + { + // 72 km/h is 20 m/s; losing it in 5 s is 4 m/s². + Assert.Equal(4.0, DrivingEventRules.DecelerationMs2(72, 0, FiveSeconds), 9); + Assert.Equal(-1.0, DrivingEventRules.DecelerationMs2(0, 18, FiveSeconds), 9); + } + + [Theory] + [InlineData(45, 0, true)] // 12.5 m/s in 5 s = 2.5 m/s², exactly the threshold + [InlineData(80, 20, true)] // 3.33 m/s² + [InlineData(44.9, 0, false)] // just below + [InlineData(80, 66, false)] // ordinary braking, 0.78 m/s² + [InlineData(0, 50, false)] // accelerating + public void HarshBrakingStartsAtTheThreshold(double fromKmh, double toKmh, bool harsh) => + Assert.Equal(harsh, DrivingEventRules.IsHarshBraking(fromKmh, toKmh, FiveSeconds)); + + [Fact] + public void TheSameDropOverALongerIntervalIsNotHarsh() => + Assert.False(DrivingEventRules.IsHarshBraking(45, 0, TimeSpan.FromSeconds(6))); + + [Fact] + public void PingsOutOfOrderAreRejected() => + Assert.Throws(() => DrivingEventRules.DecelerationMs2(50, 0, TimeSpan.Zero)); +} diff --git a/fleet-telemetry/tests/FleetTelemetry.UnitTests/FleetDailyKpisTests.cs b/fleet-telemetry/tests/FleetTelemetry.UnitTests/FleetDailyKpisTests.cs new file mode 100644 index 0000000..12fcba7 --- /dev/null +++ b/fleet-telemetry/tests/FleetTelemetry.UnitTests/FleetDailyKpisTests.cs @@ -0,0 +1,65 @@ +using FleetTelemetry.Kpis; + +namespace FleetTelemetry.UnitTests; + +public sealed class FleetDailyKpisTests +{ + private static readonly DateOnly Monday = new(2026, 9, 7); + + [Fact] + public void FleetFuelEconomyIsTotalFuelOverTotalDistance() + { + var days = FleetDailyKpis.Summarize( + [ + Day(Monday, "TRK-101", distanceKm: 400, fuelL: 120), // 30 L/100 km + Day(Monday, "VAN-106", distanceKm: 2, fuelL: 1), // 50 L/100 km, but only 2 km + ]); + + // (120 + 1) / (400 + 2) * 100 = 30.1, not the 40.0 a mean of the two ratios would give. + var monday = Assert.Single(days); + Assert.Equal(30.1, monday.FuelEconomyLPer100Km); + Assert.Equal(402.0, monday.DistanceKm); + Assert.Equal(121.0, monday.FuelUsedL); + } + + [Fact] + public void RollsUpVehiclesPerDayInDateOrder() + { + var tuesday = Monday.AddDays(1); + var days = FleetDailyKpis.Summarize( + [ + Day(tuesday, "TRK-101", samples: 100, idleSamples: 10, maxSpeedKmh: 81, faultSamples: 0), + Day(Monday, "TRK-101", samples: 300, idleSamples: 30, maxSpeedKmh: 80, faultSamples: 12), + Day(Monday, "VAN-106", samples: 100, idleSamples: 50, maxSpeedKmh: 62.5, faultSamples: 0), + ]); + + Assert.Equal([Monday, tuesday], days.Select(d => d.Day)); + Assert.Equal(2, days[0].ActiveVehicles); + Assert.Equal(0.2, days[0].IdleShare); // (30 + 50) / (300 + 100) + Assert.Equal(80, days[0].MaxSpeedKmh); + Assert.Equal(1, days[0].VehiclesWithFaults); + Assert.Equal(1, days[1].ActiveVehicles); + Assert.Equal(0, days[1].VehiclesWithFaults); + } + + [Fact] + public void ADayWithoutDistanceHasNoFuelEconomy() + { + var day = Assert.Single(FleetDailyKpis.Summarize([Day(Monday, "TRK-101", distanceKm: 0.4, fuelL: 0.3)])); + Assert.Null(day.FuelEconomyLPer100Km); + } + + [Fact] + public void NoVehicleDaysGiveNoFleetDays() => Assert.Empty(FleetDailyKpis.Summarize([])); + + private static VehicleDayKpi Day( + DateOnly day, + string vehicleId, + double distanceKm = 100, + double fuelL = 30, + long samples = 1000, + long idleSamples = 100, + double maxSpeedKmh = 80, + long faultSamples = 0) => + new(day, vehicleId, samples, idleSamples, distanceKm, fuelL, maxSpeedKmh, faultSamples); +} diff --git a/fleet-telemetry/tests/FleetTelemetry.UnitTests/FleetSimulatorTests.cs b/fleet-telemetry/tests/FleetTelemetry.UnitTests/FleetSimulatorTests.cs new file mode 100644 index 0000000..19fad06 --- /dev/null +++ b/fleet-telemetry/tests/FleetTelemetry.UnitTests/FleetSimulatorTests.cs @@ -0,0 +1,144 @@ +using FleetTelemetry.Rules; +using FleetTelemetry.Simulation; + +namespace FleetTelemetry.UnitTests; + +/// +/// The integration tests compare Snowflake's answers with the simulator's ground truth, +/// so the simulator must be deterministic and its truth must follow the business rules. +/// +public sealed class FleetSimulatorTests +{ + private static readonly SimulatedVehicle[] Vehicles = + [ + new("TRK-101", "tractor", 845, 51.8890, 4.4310), + new("TRK-204", "rigid", 400, 51.2600, 4.3600), + new("VAN-306", "van", 75, 51.3870, 6.6900), + ]; + + private static readonly string[] KnownCodes = ["P0420", "P0301", "P0087", "U0100", "P0217"]; + + private static readonly IReadOnlyList Fleet = + new FleetSimulator(seed: 7).Simulate(Vehicles, new DateOnly(2026, 9, 7), days: 5); + + [Fact] + public void TheSameSeedProducesTheSamePings() + { + var again = new FleetSimulator(seed: 7).Simulate(Vehicles, new DateOnly(2026, 9, 7), days: 5); + Assert.Equal(Fleet.SelectMany(d => d.Pings), again.SelectMany(d => d.Pings), PingComparer.Instance); + + var other = new FleetSimulator(seed: 8).Simulate(Vehicles, new DateOnly(2026, 9, 7), days: 5); + Assert.NotEqual(Fleet.SelectMany(d => d.Pings), other.SelectMany(d => d.Pings), PingComparer.Instance); + } + + [Fact] + public void PingsAreFiveSecondsApartWithinATripAndStayOnTheirDay() + { + foreach (var day in Fleet) + { + Assert.All(day.Pings, p => Assert.Equal(day.Date, DateOnly.FromDateTime(p.Ts.UtcDateTime))); + Assert.Equal(day.Pings.Count, day.Pings.Select(p => p.MessageId).Distinct().Count()); + foreach (var trip in day.Trips) + { + var pings = day.Pings.Where(p => p.Ts >= trip.StartedAt && p.Ts <= trip.EndedAt).ToList(); + Assert.Equal(trip.Samples, pings.Count); + Assert.All(pings.Zip(pings.Skip(1)), pair => Assert.Equal(FleetSimulator.PingInterval, pair.Second.Ts - pair.First.Ts)); + } + } + } + + [Fact] + public void SegmentingThePingsWithTheTripRuleGivesTheSimulatedTrips() + { + foreach (var day in Fleet) + { + var starts = day.Pings + .Select((p, i) => (Ping: p, Gap: i == 0 ? (TimeSpan?)null : p.Ts - day.Pings[i - 1].Ts)) + .Where(x => TripRules.StartsNewTrip(x.Gap)) + .Select(x => x.Ping.Ts); + Assert.Equal(day.Trips.Select(t => t.StartedAt), starts); + } + } + + [Fact] + public void OnlyTheSimulatedHardStopsAreHarshBraking() + { + foreach (var day in Fleet) + { + var harsh = day.Pings.Zip(day.Pings.Skip(1)) + .Where(pair => pair.Second.Ts - pair.First.Ts <= TripRules.MaxPingGap) + .Where(pair => DrivingEventRules.IsHarshBraking(pair.First.SpeedKmh, pair.Second.SpeedKmh, pair.Second.Ts - pair.First.Ts)) + .Select(pair => new HarshBrakeTruth(pair.Second.Ts, pair.First.SpeedKmh, pair.Second.SpeedKmh)); + Assert.Equal(day.HarshBrakes, harsh); + } + + Assert.NotEmpty(Fleet.SelectMany(d => d.HarshBrakes)); + } + + [Fact] + public void OnlyTheSimulatedLoadingDockWaitsAreIdling() + { + foreach (var day in Fleet) + { + Assert.Equal(day.Idling, StationaryRuns(day).Where(r => r.EndedAt - r.StartedAt >= DrivingEventRules.MinIdling)); + } + + Assert.NotEmpty(Fleet.SelectMany(d => d.Idling)); + } + + [Fact] + public void FaultCodesComeFromTheCatalogAndStayActiveOnceRaised() + { + foreach (var day in Fleet) + { + var firstFault = day.Pings.ToList().FindIndex(p => p.Engine.Dtc.Count > 0); + if (firstFault >= 0) + { + Assert.All(day.Pings.Skip(firstFault), p => Assert.NotEmpty(p.Engine.Dtc)); + } + } + + var codes = Fleet.SelectMany(d => d.Pings).SelectMany(p => p.Engine.Dtc).Select(c => c.Code).Distinct(); + Assert.NotEmpty(codes); + Assert.All(codes, c => Assert.Contains(c, KnownCodes)); + } + + private static IEnumerable StationaryRuns(VehicleDay day) + { + DateTimeOffset? start = null; + DateTimeOffset last = default; + foreach (var ping in day.Pings) + { + var continues = start is not null && ping.SpeedKmh == 0 && ping.Ts - last <= TripRules.MaxPingGap; + if (!continues && start is { } s) + { + yield return new IdlingTruth(s, last); + start = null; + } + + if (ping.SpeedKmh == 0) + { + start ??= ping.Ts; + } + + last = ping.Ts; + } + + if (start is { } open) + { + yield return new IdlingTruth(open, last); + } + } + + private sealed class PingComparer : IEqualityComparer + { + public static readonly PingComparer Instance = new(); + + public bool Equals(Telemetry.TelemetryPing? x, Telemetry.TelemetryPing? y) => + x is not null && y is not null + && x with { Engine = x.Engine with { Dtc = [] } } == y with { Engine = y.Engine with { Dtc = [] } } + && x.Engine.Dtc.SequenceEqual(y.Engine.Dtc); + + public int GetHashCode(Telemetry.TelemetryPing obj) => obj.MessageId.GetHashCode(StringComparison.Ordinal); + } +} diff --git a/fleet-telemetry/tests/FleetTelemetry.UnitTests/FleetTelemetry.UnitTests.csproj b/fleet-telemetry/tests/FleetTelemetry.UnitTests/FleetTelemetry.UnitTests.csproj new file mode 100644 index 0000000..19abcdb --- /dev/null +++ b/fleet-telemetry/tests/FleetTelemetry.UnitTests/FleetTelemetry.UnitTests.csproj @@ -0,0 +1,18 @@ + + + + Exe + false + FleetTelemetry.UnitTests + + + + + + + + + + + + diff --git a/fleet-telemetry/tests/FleetTelemetry.UnitTests/KpiMathTests.cs b/fleet-telemetry/tests/FleetTelemetry.UnitTests/KpiMathTests.cs new file mode 100644 index 0000000..7894165 --- /dev/null +++ b/fleet-telemetry/tests/FleetTelemetry.UnitTests/KpiMathTests.cs @@ -0,0 +1,39 @@ +using FleetTelemetry.Kpis; + +namespace FleetTelemetry.UnitTests; + +public sealed class KpiMathTests +{ + [Theory] + [InlineData(31.0, 100.0, 31.0)] + [InlineData(7.081, 22.669, 31.2)] // 31.236 rounds to one decimal + [InlineData(0.2, 1.0, 20.0)] // exactly the minimum distance + public void FuelEconomyIsLitresPer100Km(double litres, double km, double expected) => + Assert.Equal(expected, KpiMath.FuelEconomyLPer100Km(litres, km)); + + [Theory] + [InlineData(0.05, 0.0)] + [InlineData(0.05, 0.999)] + public void FuelEconomyIsUnknownBelowOneKilometre(double litres, double km) => + Assert.Null(KpiMath.FuelEconomyLPer100Km(litres, km)); + + [Theory] + [InlineData(145, 1000, 0.145)] + [InlineData(1, 3, 0.333)] + [InlineData(0, 0, 0.0)] // no samples: nothing was idle + [InlineData(12, 12, 1.0)] + public void IdleShareIsTheStationaryFractionOfSamples(long idle, long samples, double expected) => + Assert.Equal(expected, KpiMath.IdleShare(idle, samples)); + + [Fact] + public void IdleShareRejectsMoreIdleSamplesThanSamples() => + Assert.Throws(() => KpiMath.IdleShare(11, 10)); + + [Fact] + public void AverageSpeedIncludesStops() + { + // 25.754 km in 1,550 s (25 min 50 s) is 59.82 km/h. + Assert.Equal(59.8, KpiMath.AverageSpeedKmh(25.754, 1_550)); + Assert.Null(KpiMath.AverageSpeedKmh(0, 0)); + } +} diff --git a/fleet-telemetry/tests/FleetTelemetry.UnitTests/TelemetryBatchParserTests.cs b/fleet-telemetry/tests/FleetTelemetry.UnitTests/TelemetryBatchParserTests.cs new file mode 100644 index 0000000..ab3eb0e --- /dev/null +++ b/fleet-telemetry/tests/FleetTelemetry.UnitTests/TelemetryBatchParserTests.cs @@ -0,0 +1,117 @@ +using System.Text.Json.Nodes; +using FleetTelemetry.Telemetry; + +namespace FleetTelemetry.UnitTests; + +public sealed class TelemetryBatchParserTests +{ + private static readonly DateTimeOffset ReceivedAt = new(2026, 9, 10, 8, 0, 0, TimeSpan.Zero); + + [Fact] + public void AcceptsAnArrayOfValidPings() + { + var result = TelemetryBatchParser.Parse(Batch(Ping(), Ping("TRK-101-2"), Ping("TRK-101-3")), ReceivedAt); + + var valid = Assert.IsType(result); + Assert.Equal(["TRK-101-1", "TRK-101-2", "TRK-101-3"], valid.Pings.Select(p => p.MessageId)); + Assert.Equal("P0301", valid.Pings[0].Engine.Dtc[0].Code); + } + + [Fact] + public void ConvertsAnyUtcOffset() + { + var ping = Ping(); + ping["ts"] = "2026-09-10T09:30:00+02:00"; + + var valid = Assert.IsType(TelemetryBatchParser.Parse(Batch(ping), ReceivedAt)); + Assert.Equal(new DateTimeOffset(2026, 9, 10, 7, 30, 0, TimeSpan.Zero), valid.Pings[0].Ts); + } + + [Theory] + [InlineData("[{\"message_id\": ")] + [InlineData("not json")] + [InlineData("{\"message_id\": \"x\"}")] // an object, not an array + public void RejectsABodyThatIsNotAJsonArray(string body) => + Assert.IsType(TelemetryBatchParser.Parse(body, ReceivedAt)); + + [Fact] + public void RejectsAnEmptyBatch() + { + var invalid = Assert.IsType(TelemetryBatchParser.Parse("[]", ReceivedAt)); + Assert.Equal(["$"], invalid.Errors.Keys); + } + + [Fact] + public void RejectsABatchOverTheLimit() + { + var pings = Enumerable.Range(0, TelemetryBatchParser.MaxBatchSize + 1).Select(i => Ping($"m-{i}")).ToArray(); + var invalid = Assert.IsType(TelemetryBatchParser.Parse(Batch(pings), ReceivedAt)); + Assert.Equal(["$"], invalid.Errors.Keys); + } + + [Fact] + public void ReportsEveryBrokenFieldByPingIndex() + { + var bad = Ping(); + bad["vehicle_id"] = "truck 7"; + bad["speed_kmh"] = -3; + bad["gps"]!["lat"] = 91; + bad["fuel"]!["level_pct"] = 101; + bad["engine"]!["dtc"]![0]!["code"] = "X123"; + + var invalid = Assert.IsType(TelemetryBatchParser.Parse(Batch(Ping(), bad), ReceivedAt)); + + Assert.Equal( + ["[1].engine.dtc[0].code", "[1].fuel.level_pct", "[1].gps.lat", "[1].speed_kmh", "[1].vehicle_id"], + invalid.Errors.Keys.Order(StringComparer.Ordinal)); + } + + [Fact] + public void ReportsMissingRequiredFields() + { + var incomplete = Ping(); + incomplete.Remove("odometer_km"); + + var invalid = Assert.IsType(TelemetryBatchParser.Parse(Batch(incomplete), ReceivedAt)); + Assert.Contains("odometer_km", Assert.Single(invalid.Errors["[0]"]), StringComparison.Ordinal); + } + + [Theory] + [InlineData(300, true)] // five minutes ahead of our clock is tolerated + [InlineData(301, false)] + public void ToleratesSmallDeviceClockSkew(int secondsAhead, bool accepted) + { + var ping = Ping(); + ping["ts"] = ReceivedAt.AddSeconds(secondsAhead).ToString("O"); + + var result = TelemetryBatchParser.Parse(Batch(ping), ReceivedAt); + + if (accepted) + { + Assert.IsType(result); + } + else + { + Assert.Equal(["[0].ts"], Assert.IsType(result).Errors.Keys); + } + } + + private static JsonObject Ping(string messageId = "TRK-101-1") => new() + { + ["message_id"] = messageId, + ["vehicle_id"] = "TRK-101", + ["ts"] = "2026-09-10T07:00:00Z", + ["gps"] = new JsonObject { ["lat"] = 51.889, ["lon"] = 4.431, ["heading_deg"] = 87 }, + ["speed_kmh"] = 42.5, + ["odometer_km"] = 309_199.518, + ["fuel"] = new JsonObject { ["level_pct"] = 63.2, ["used_l"] = 75_416.832 }, + ["engine"] = new JsonObject + { + ["rpm"] = 1450, + ["coolant_c"] = 88.5, + ["dtc"] = new JsonArray(new JsonObject { ["code"] = "P0301", ["severity"] = "warning" }), + }, + }; + + private static string Batch(params JsonObject[] pings) => new JsonArray(pings).ToJsonString(); +} diff --git a/fleet-telemetry/tests/FleetTelemetry.UnitTests/TripRulesTests.cs b/fleet-telemetry/tests/FleetTelemetry.UnitTests/TripRulesTests.cs new file mode 100644 index 0000000..3f1a18b --- /dev/null +++ b/fleet-telemetry/tests/FleetTelemetry.UnitTests/TripRulesTests.cs @@ -0,0 +1,22 @@ +using FleetTelemetry.Rules; + +namespace FleetTelemetry.UnitTests; + +public sealed class TripRulesTests +{ + [Fact] + public void FirstPingOfAVehicleStartsATrip() => Assert.True(TripRules.StartsNewTrip(null)); + + [Theory] + [InlineData(5)] // the regular ping interval + [InlineData(120)] // a traffic jam with the ignition on + [InlineData(600)] // exactly ten minutes: still the same trip + public void GapsUpToTenMinutesContinueTheTrip(int seconds) => + Assert.False(TripRules.StartsNewTrip(TimeSpan.FromSeconds(seconds))); + + [Theory] + [InlineData(601)] + [InlineData(3_600)] + public void GapsLongerThanTenMinutesStartANewTrip(int seconds) => + Assert.True(TripRules.StartsNewTrip(TimeSpan.FromSeconds(seconds))); +} diff --git a/fleet-telemetry/tests/FleetTelemetry.UnitTests/packages.lock.json b/fleet-telemetry/tests/FleetTelemetry.UnitTests/packages.lock.json new file mode 100644 index 0000000..31c77e2 --- /dev/null +++ b/fleet-telemetry/tests/FleetTelemetry.UnitTests/packages.lock.json @@ -0,0 +1,137 @@ +{ + "version": 2, + "dependencies": { + "net10.0": { + "xunit.v3": { + "type": "Direct", + "requested": "[4.0.2, )", + "resolved": "4.0.2", + "contentHash": "TdjM2OAx7w/0E88E6PgaigLoTbQaxbSR3HF/8p92plchTjf29jPHW27ylvEOz+Nx8CvPwWKGQlOR1fDv6/Qh5A==", + "dependencies": { + "xunit.v3.mtp-v2": "[4.0.2]" + } + }, + "Microsoft.ApplicationInsights": { + "type": "Transitive", + "resolved": "2.23.0", + "contentHash": "nWArUZTdU7iqZLycLKWe0TDms48KKGE6pONH2terYNa8REXiqixrMOkf1sk5DHGMaUTqONU2YkS4SAXBhLStgw==" + }, + "Microsoft.Bcl.AsyncInterfaces": { + "type": "Transitive", + "resolved": "6.0.0", + "contentHash": "UcSjPsst+DfAdJGVDsu346FX0ci0ah+lw3WRtn18NUwEqRt70HaOQ7lI72vy3+1LxtqI3T5GWwV39rQSrCzAeg==" + }, + "Microsoft.Testing.Extensions.Telemetry": { + "type": "Transitive", + "resolved": "2.5.1", + "contentHash": "mu2iyoHovwl0q1mGeeRU0h8+LALXC7HqbxniLa1shTKYdJGl+KPoCEgjWIQ8MLzk1tlTGPOfg1JsxdH3RxhMCA==", + "dependencies": { + "Microsoft.ApplicationInsights": "2.23.0", + "Microsoft.Testing.Platform": "[2.5.1, 3.0.0)" + } + }, + "Microsoft.Testing.Extensions.TrxReport.Abstractions": { + "type": "Transitive", + "resolved": "2.5.1", + "contentHash": "DYTZAguaHz0wNJdHDTTGYOXtnlDoLoIbTMDKSApNNcytm55DJcnwpwELfQGLk5DB5QlgaEjNhnGo1Pu5QvL5VA==", + "dependencies": { + "Microsoft.Testing.Platform": "[2.5.1, 3.0.0)" + } + }, + "Microsoft.Testing.Platform": { + "type": "Transitive", + "resolved": "2.5.1", + "contentHash": "FAMaFFWnULW+hDHMb40Qzhcgz9cAU2/EtgRCBNX/DdQoKo0sPG2EiY+w7bQdl/BgHnf/DbpXLbf5Xy0o15M8NA==" + }, + "Microsoft.Testing.Platform.MSBuild": { + "type": "Transitive", + "resolved": "2.5.1", + "contentHash": "lYawcAwY+h76rQmpma2n0HaL8zkOFbbKapso0Eb+uOAUQn7333pIByLftEuiO67csAoZufHNMwi6PqhNemWQXA==", + "dependencies": { + "Microsoft.Testing.Platform": "[2.5.1, 3.0.0)" + } + }, + "Microsoft.Win32.Registry": { + "type": "Transitive", + "resolved": "5.0.0", + "contentHash": "dDoKi0PnDz31yAyETfRntsLArTlVAVzUzCIvvEDsDsucrl33Dl8pIJG06ePTJTI3tGpeyHS9Cq7Foc/s4EeKcg==" + }, + "System.Security.AccessControl": { + "type": "Transitive", + "resolved": "6.0.1", + "contentHash": "IQ4NXP/B3Ayzvw0rDQzVTYsCKyy0Jp9KI6aYcK7UnGVlR9+Awz++TIPCQtPYfLJfOpm8ajowMR09V7quD3sEHw==" + }, + "xunit.analyzers": { + "type": "Transitive", + "resolved": "2.2.0", + "contentHash": "dXMIIztipcAyL3a19gtUrih/RLrE33/4VJdICoinYqIXXo1KLtsMPsaQ9dO2pE8xajDguwE2kqVbBqE5/EoJ0Q==" + }, + "xunit.v3.assert": { + "type": "Transitive", + "resolved": "4.0.2", + "contentHash": "5PKRexYlvufWFdXenRcRGKqvaJBBZFJZpqUGlUs9p13eKEgVdOrLpkPknjzUvLY5kEUicK9871DIUDcCPA6AXA==" + }, + "xunit.v3.common": { + "type": "Transitive", + "resolved": "4.0.2", + "contentHash": "++CtT8daIf6qfieQev/8V88U3zViIDWou5cj+oWps1/Hf73UoaY/L0hKyi3v7Zlt3BmbU0pUc73zCB1GiB4nYA==", + "dependencies": { + "Microsoft.Bcl.AsyncInterfaces": "6.0.0" + } + }, + "xunit.v3.core.mtp-v2": { + "type": "Transitive", + "resolved": "4.0.2", + "contentHash": "XyPMcQRsnUQLhmBBRRgYWsuPc8TyiANzcXtqfzyWfDpPuyh3Q0K4oWozZYQKZRNGnqJ97ifD6qUDWhsEEe32uw==", + "dependencies": { + "Microsoft.Testing.Extensions.Telemetry": "2.5.1", + "Microsoft.Testing.Extensions.TrxReport.Abstractions": "2.5.1", + "Microsoft.Testing.Platform": "2.5.1", + "Microsoft.Testing.Platform.MSBuild": "2.5.1", + "xunit.v3.extensibility.core": "[4.0.2]", + "xunit.v3.runner.inproc.console": "[4.0.2]" + } + }, + "xunit.v3.extensibility.core": { + "type": "Transitive", + "resolved": "4.0.2", + "contentHash": "ARTt+pGyd86mdtiFBkKpyBQvRC3u0rr8oidBlII2GsAg7D6TfOsDFCsoQtFij83V5pE57m3whUIqT4xjiARXbQ==", + "dependencies": { + "xunit.v3.common": "[4.0.2]" + } + }, + "xunit.v3.mtp-v2": { + "type": "Transitive", + "resolved": "4.0.2", + "contentHash": "XLV32FhkuvheBStvX0DvjfPEtI+4SOVcIhnlTS+549r8VfVwsjpWdkJtQmQ7D23PA+Zoaqo+sTjBkRSkidC6Tw==", + "dependencies": { + "xunit.analyzers": "2.2.0", + "xunit.v3.assert": "[4.0.2]", + "xunit.v3.core.mtp-v2": "[4.0.2]" + } + }, + "xunit.v3.runner.common": { + "type": "Transitive", + "resolved": "4.0.2", + "contentHash": "YlYKVhTJVcwEgJ6rm59Pjf71lQqKwKxEHqVRxWUuwrwm/nxB/mNFP/lP4vlntDO3GclNnLT9BjYOpJH9gViGQg==", + "dependencies": { + "Microsoft.Win32.Registry": "[5.0.0]", + "System.Security.AccessControl": "[6.0.1]", + "xunit.v3.common": "[4.0.2]" + } + }, + "xunit.v3.runner.inproc.console": { + "type": "Transitive", + "resolved": "4.0.2", + "contentHash": "cg4XNpCN4Ry9nswutknpreN3ckSUBwtjMRmEQSNK/sIKPo6KN5tDGGGCaHDm5HGA4yFVVtTJX65tLCsZArUNfQ==", + "dependencies": { + "xunit.v3.extensibility.core": "[4.0.2]", + "xunit.v3.runner.common": "[4.0.2]" + } + }, + "fleettelemetry.core": { + "type": "Project" + } + } + } +} \ No newline at end of file diff --git a/metering-api/.env.example b/metering-api/.env.example new file mode 100644 index 0000000..89c7aeb --- /dev/null +++ b/metering-api/.env.example @@ -0,0 +1,33 @@ +# Every variable the metering API, the seed and the tests read, with its default. +# Nothing here is needed for the local workflow except the LocalStack token. +# The Makefile does not load this file: export what you change (direnv, or +# `set -a; . ./.env; set +a`). + +# --- Local Snowflake (make up) ---------------------------------------------- +# LocalStack auth token. The Makefile also reads ~/.localstack/default.env. +# LOCALSTACK_AUTH_TOKEN= +SNOWFLAKE_IMAGE=localstack/snowflake-next:latest +# Host port of the local Snowflake. The integration tests pick their own. +SNOWFLAKE_PORT=4566 + +# --- Connection (the same shape locally and in production) ------------------- +# EMULATOR (default) or SNOWFLAKE_CLOUD: picks the defaults below and the target +# of the integration tests. +TEST_TARGET=EMULATOR +# Emulator defaults: account snowflake on snowflake.localhost.localstack.cloud. +# On a real account set SNOWFLAKE_ACCOUNT; SNOWFLAKE_HOST defaults to +# .snowflakecomputing.com and the port to 443. +SNOWFLAKE_ACCOUNT= +SNOWFLAKE_HOST= +SNOWFLAKE_USER=METERING_SVC +SNOWFLAKE_ROLE=METERING_APP +SNOWFLAKE_WAREHOUSE=METERING_WH +SNOWFLAKE_DATABASE=METERING +SNOWFLAKE_SCHEMA=BILLING +# Unencrypted PKCS#8 key for SNOWFLAKE_JWT auth. `make up` generates a throwaway one. +SNOWFLAKE_PRIVATE_KEY_PATH=.secrets/rsa_key.p8 + +# --- API --------------------------------------------------------------------- +HTTP_ADDR=127.0.0.1:8080 +# DEBUG, INFO, WARN or ERROR +LOG_LEVEL=INFO diff --git a/metering-api/.gitignore b/metering-api/.gitignore new file mode 100644 index 0000000..4f58494 --- /dev/null +++ b/metering-api/.gitignore @@ -0,0 +1,6 @@ +# Throwaway key pair generated by `make up` +.secrets/ +.env + +# Binaries built by `make demo` +bin/ diff --git a/metering-api/Makefile b/metering-api/Makefile new file mode 100644 index 0000000..a8afb4f --- /dev/null +++ b/metering-api/Makefile @@ -0,0 +1,72 @@ +# Metering API: a Go usage-metering and invoicing service on Snowflake. +# Run `make help` for the targets. + +-include $(HOME)/.localstack/default.env +export LOCALSTACK_AUTH_TOKEN +SNOWFLAKE_PORT ?= 4566 +SNOWFLAKE_IMAGE ?= localstack/snowflake-next:latest +SNOWFLAKE_PRIVATE_KEY_PATH ?= $(CURDIR)/.secrets/rsa_key.p8 +HTTP_ADDR ?= 127.0.0.1:8080 +export SNOWFLAKE_PORT SNOWFLAKE_IMAGE SNOWFLAKE_PRIVATE_KEY_PATH HTTP_ADDR + +COMPOSE := docker compose -p snfs-metering-api +GO_TEST_INTEGRATION := go test -tags integration -count=1 -timeout 30m + +.DEFAULT_GOAL := help +.PHONY: help keys up seed run demo test test-unit test-integration test-cloud lint down logs install start stop + +help: ## List targets + @grep -hE '^[a-z-]+:.*## ' $(firstword $(MAKEFILE_LIST)) | awk -F':.*## ' '{printf " %-17s %s\n", $$1, $$2}' + +keys: .secrets/rsa_key.p8 ## Generate the throwaway key pair used for SNOWFLAKE_JWT auth + +.secrets/rsa_key.p8: + @mkdir -p .secrets && chmod 700 .secrets + @openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 -out $@ 2>/dev/null + @openssl rsa -in $@ -pubout -out .secrets/rsa_key.pub 2>/dev/null + @chmod 600 $@ + @echo "Generated .secrets/rsa_key.p8 (throwaway, for local development)" + +up: keys ## Pull and start the local Snowflake, wait until it answers SQL + $(COMPOSE) pull --quiet + $(COMPOSE) up -d + ./scripts/wait-for-emulator.sh 180 + go mod download + +seed: ## Load ~200 synthetic customers and three months of usage (idempotent) + go run ./cmd/seed + +run: ## Run the API on http://$(HTTP_ADDR) against the local Snowflake + go run ./cmd/api + +demo: ## Walk through ingest, duplicate re-ingest, invoice close, async rollup and export + ./scripts/demo.sh + +test: test-unit test-integration ## Unit tests, then integration tests on a testcontainers emulator + +test-unit: ## Pricing, validation and handler tests (no database) + go test -count=1 ./... + +test-integration: ## Integration tests: own emulator via testcontainers-go, one database per test + $(GO_TEST_INTEGRATION) ./internal/metering/ + +test-cloud: ## The integration tests against real Snowflake (billable; needs SNOWFLAKE_* env) + TEST_TARGET=SNOWFLAKE_CLOUD $(GO_TEST_INTEGRATION) ./internal/metering/ + +lint: ## go vet and gofmt + go vet ./... && go vet -tags integration ./... + @test -z "$$(gofmt -l .)" || { gofmt -l .; exit 1; } + +down: ## Stop and remove the local Snowflake + $(COMPOSE) down -v --remove-orphans + +logs: ## Tail the emulator log (init hook results included) + $(COMPOSE) logs --tail=200 snowflake + +# The targets the repository CI runs for every sample (see the root README) +install: ## Nothing to install up front: make up and make test fetch the dependencies + @echo "Nothing to install, make up and make test fetch the dependencies" + +start: up ## Same as make up + +stop: down ## Same as make down diff --git a/metering-api/README.md b/metering-api/README.md new file mode 100644 index 0000000..a3cc908 --- /dev/null +++ b/metering-api/README.md @@ -0,0 +1,142 @@ +# Metering API + +A Go service that meters API usage and bills it: customers on Free, Pro and Enterprise plans, idempotent usage ingestion, monthly invoices priced on graduated tiers, async rollups and a streamed CSV export, all on Snowflake through `database/sql` and gosnowflake. The Go app runs on your machine with `go run`, against a local Snowflake in Docker, the way you would develop it day to day. + +## Architecture + +```mermaid +flowchart LR + gw["API gateway
(curl, make demo)"] -- "JSON over HTTP" --> api["cmd/api
net/http + slog"] + subgraph svc["Go service (go run ./cmd/api)"] + api --> m["internal/metering
ingest, usage, invoices, rollups, export"] + m --> p["internal/pricing
graduated tiers"] + end + m -- "gosnowflake v2, key-pair JWT
QUERY_TAG per endpoint" --> sf + subgraph sf["Snowflake: METERING.BILLING"] + stage[("USAGE_EVENTS_STAGING")] -- MERGE --> events[("USAGE_EVENTS")] + plans[("PRICE_PLANS")] + cust[("CUSTOMERS")] + inv[("INVOICES, INVOICE_LINES")] + end + init["init/*.sf.sql"] -. "run at startup" .-> sf + tests["go test -tags integration"] -. "testcontainers-go,
one database per test" .-> sf +``` + +## How it works + +1. **Ingest** (`POST /v1/events`). The gateway posts batches of up to 1,000 usage reports. One `INSERT` with array binds lands the batch in a staging table, a `MERGE` keyed by `event_id` adds only the events not stored yet, and the staging rows are deleted, in one transaction. Retried batches change nothing. +2. **Usage** (`GET /v1/customers/{id}/usage?from=&to=`) aggregates a customer's calls per day. +3. **Invoice close** (`POST /v1/invoices/{month}/close`). One transaction rates every customer's uninvoiced usage against their plan's tiers (in Go, `internal/pricing`), inserts the invoices and their lines, and marks the usage invoiced. It commits all of it or none of it, and a month is closed only once. +4. **Rollup** (`POST /v1/rollups/{month}`) submits the monthly per-plan, per-day aggregation as an asynchronous Snowflake query and returns `202` with its query id. `GET /v1/rollups/{queryId}` reports the status and, once done, fetches the result by query id, from any instance of the service. +5. **Export** (`GET /v1/invoices/{month}/export.csv`) streams the invoice lines to the client as rows arrive. + +Errors are RFC 9457 problem details (`application/problem+json`). + +```mermaid +sequenceDiagram + autonumber + participant G as Gateway + participant A as Metering API + participant S as Snowflake + G->>A: POST /v1/events (batch of 1,000) + A->>S: BEGIN, INSERT staging (array binds), MERGE on EVENT_ID, DELETE staging, COMMIT + A-->>G: 200 received 1000, inserted 1000, duplicates 0 + G->>A: POST /v1/events (same batch, retried) + A-->>G: 200 received 1000, inserted 0, duplicates 1000 + G->>A: POST /v1/invoices/2026-09/close + A->>S: BEGIN, read usage, INSERT INVOICES and INVOICE_LINES, UPDATE USAGE_EVENTS, COMMIT + A-->>G: 201 invoices, total, events invoiced + G->>A: POST /v1/rollups/2026-09 + A->>S: rollup query in async mode + A-->>G: 202 query_id + G->>A: GET /v1/rollups/{query_id} + A->>S: query status, then fetch result by query id + A-->>G: 200 succeeded, rows +``` + +Design notes (data model, idempotency, transactions, async queries) are in [docs/architecture.md](docs/architecture.md). + +## Prerequisites + +- Docker with Compose v2 +- Go 1.26 or newer (`go.mod` pins the 1.27.3 toolchain) +- `make`, `openssl` and `curl` +- A LocalStack auth token, exported as `LOCALSTACK_AUTH_TOKEN` or stored in `~/.localstack/default.env` + +## Quickstart + +```sh +make up # throwaway key pair in .secrets/, start the local Snowflake, wait until it answers SQL +make seed # 200 customers (120 Free, 60 Pro, 20 Enterprise), ~30,000 usage events over 3 months +make run # the API on http://127.0.0.1:8080 (go run ./cmd/api) +make demo # scripted walkthrough; starts the API itself if it is not running +make test # unit tests, then integration tests on their own emulator +make down # stop and remove the local Snowflake +``` + +If port 4566 is taken, pick another for every target: `SNOWFLAKE_PORT=14566 make up seed demo`. + +With `make run` going: + +```sh +curl -s -X POST localhost:8080/v1/events -d '{"events": [ + {"event_id": "evt_1", "customer_id": "cus_0007", "quantity": 1200, "occurred_at": "2026-09-30T10:00:00Z"}]}' +# {"received":1,"inserted":1,"duplicates":0} (again: "inserted":0,"duplicates":1) +curl -s "localhost:8080/v1/customers/cus_0007/usage?from=2026-09-01&to=2026-10-01" +curl -s -X POST localhost:8080/v1/invoices/2026-09/close +# {"month":"2026-09","invoices":200,"total_cents":5828821,"events_invoiced":9700} +curl -s -X POST localhost:8080/v1/rollups/2026-09 # {"query_id":"...","status_url":"/v1/rollups/..."} +curl -s localhost:8080/v1/invoices/2026-09/export.csv | head -3 +``` + +The seed covers the three months before the current one; use the last of them in these examples. The connection is configured only through environment variables (`.env.example` lists them all). Against a real account the same variables carry the account, user and key instead of the emulator defaults. + +## Tests + +`make test` runs both suites: + +- **Unit tests** (`go test ./...`): tier pricing and rounding against hand-computed amounts, request validation, handler error responses, configuration and the data generator. No database. +- **Integration tests** (`go test -tags integration ./internal/metering/`): `TestMain` starts `localstack/snowflake-next` with testcontainers-go (`GenericContainer`, `LOCALSTACK_AUTH_TOKEN` passed through, readiness checked from the host over HTTPS on the mapped port) and provisions it with `init/01_platform.sf.sql`. Each test creates its own database from `init/02` and `init/03`, seeds a generated dataset through the API, runs in parallel with the others and drops its database at the end. + +The integration tests check concrete values computed independently from the generated events and the published price list: + +| Test | Asserts | +| --- | --- | +| `TestIngestIsIdempotent` | Re-posting every event inserts 0 rows and leaves counts, sums and per-customer usage unchanged; a batch with a repeated new event inserts it once | +| `TestCloseMonthInvoicesAtTierPrices` | Every invoice total equals tier pricing by hand, including a Pro customer with no usage (the $49 fee); only the closed month is invoiced; a second close is `409` | +| `TestCloseMonthIsAtomic` | A failure injected after the invoices are written leaves no invoices, no lines and no invoiced usage; the retried close succeeds | +| `TestRollupRunsAsync` | The rollup returns `202`, reaches `succeeded` within a bounded poll, and returns exactly the expected per-plan, per-day rows | +| `TestInvoiceExportCSV` | Row count and SHA-256 of the export match a CSV built from the dataset; `404` before the month is closed | +| `TestErrorPaths` | Unknown customer `404`, malformed JSON `400`, invalid events `422`, unknown customer in a batch `422` with nothing stored, unknown rollup `404` | + +To run the integration tests against Snowflake, set `SNOWFLAKE_ACCOUNT`, `SNOWFLAKE_USER`, `SNOWFLAKE_PRIVATE_KEY_PATH`, `SNOWFLAKE_ROLE` (with `CREATE DATABASE`) and `SNOWFLAKE_WAREHOUSE` (X-Small is plenty), then run `make test-cloud`. Each test creates and drops a `METERING_IT_` database; this consumes credits. + +## Project structure + +``` +metering-api/ + Makefile, docker-compose.yml, .env.example, go.mod, go.sum + init/ + 01_platform.sf.sql warehouse, role, database, service user + 02_schema.sf.sql tables (database-relative) + 03_price_plans.sf.sql Free, Pro and Enterprise tiers + cmd/ + api/ HTTP server, graceful shutdown + seed/ loads the synthetic dataset (idempotent) + demo/ the make demo walkthrough + internal/ + config/ environment settings, key-pair auth, connection pool + pricing/ graduated tier rating + metering/ service, handlers, problem details, integration tests + seed/ deterministic data generator + scripts/ wait-for-emulator.sh, demo.sh + docs/architecture.md data model and design notes +``` + +## Learn more + +- [Go Snowflake driver](https://docs.snowflake.com/en/developer-guide/golang/go-driver) and [gosnowflake on pkg.go.dev](https://pkg.go.dev/github.com/snowflakedb/gosnowflake/v2) +- [Key-pair authentication](https://docs.snowflake.com/en/user-guide/key-pair-auth) +- [`MERGE`](https://docs.snowflake.com/en/sql-reference/sql/merge) and [transactions](https://docs.snowflake.com/en/sql-reference/transactions) +- [Asynchronous queries in gosnowflake](https://pkg.go.dev/github.com/snowflakedb/gosnowflake/v2#hdr-Asynchronous_Queries), [persisted query results](https://docs.snowflake.com/en/user-guide/querying-persisted-results) and [`QUERY_TAG`](https://docs.snowflake.com/en/sql-reference/parameters#query-tag) +- [testcontainers-go](https://golang.testcontainers.org/) and [RFC 9457 problem details](https://www.rfc-editor.org/rfc/rfc9457) diff --git a/metering-api/cmd/api/main.go b/metering-api/cmd/api/main.go new file mode 100644 index 0000000..4802d12 --- /dev/null +++ b/metering-api/cmd/api/main.go @@ -0,0 +1,84 @@ +// Command api serves the metering API. +// +// Configuration comes from the environment (see .env.example): SNOWFLAKE_* for +// the connection, HTTP_ADDR for the listen address, LOG_LEVEL for logging. +package main + +import ( + "context" + "errors" + "log/slog" + "net/http" + "os" + "os/signal" + "syscall" + "time" + + "github.com/localstack-samples/localstack-snowflake-samples/metering-api/internal/config" + "github.com/localstack-samples/localstack-snowflake-samples/metering-api/internal/metering" +) + +func main() { + if err := run(); err != nil { + slog.Error("metering-api stopped", "err", err) + os.Exit(1) + } +} + +func run() error { + var level slog.Level + if err := level.UnmarshalText([]byte(getenv("LOG_LEVEL", "INFO"))); err != nil { + return err + } + log := slog.New(slog.NewJSONHandler(os.Stdout, &slog.HandlerOptions{Level: level})) + slog.SetDefault(log) + + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + + sfCfg, err := config.SnowflakeFromEnv() + if err != nil { + return err + } + db, err := config.Open(ctx, sfCfg) + if err != nil { + return err + } + defer db.Close() + + srv := &http.Server{ + Addr: getenv("HTTP_ADDR", "127.0.0.1:8080"), + Handler: metering.NewHandler(metering.NewService(db, log), log), + ReadHeaderTimeout: 5 * time.Second, + ReadTimeout: 30 * time.Second, + IdleTimeout: 2 * time.Minute, + } + errCh := make(chan error, 1) + go func() { + log.Info("listening", "addr", srv.Addr, "snowflake", sfCfg.Host, "database", sfCfg.Database, "target", string(sfCfg.Target)) + errCh <- srv.ListenAndServe() + }() + + select { + case err := <-errCh: + return err + case <-ctx.Done(): + } + log.Info("shutting down") + shutdownCtx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + defer cancel() + if err := srv.Shutdown(shutdownCtx); err != nil { + return err + } + if err := <-errCh; !errors.Is(err, http.ErrServerClosed) { + return err + } + return nil +} + +func getenv(name, fallback string) string { + if v := os.Getenv(name); v != "" { + return v + } + return fallback +} diff --git a/metering-api/cmd/demo/main.go b/metering-api/cmd/demo/main.go new file mode 100644 index 0000000..03ab409 --- /dev/null +++ b/metering-api/cmd/demo/main.go @@ -0,0 +1,255 @@ +// Command demo walks through the metering API over HTTP: ingest, a retried +// (duplicate) batch, invoice close, an async rollup and the CSV export. +// It expects a seeded database (`make seed`) and a running API (`make run`, or +// let `make demo` start one). +package main + +import ( + "bufio" + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "flag" + "fmt" + "io" + "net/http" + "os" + "strings" + "time" + + "github.com/localstack-samples/localstack-snowflake-samples/metering-api/internal/metering" +) + +const customer = "cus_0007" + +var ( + base string + client = &http.Client{Timeout: 2 * time.Minute} +) + +func main() { + flag.StringVar(&base, "addr", "http://127.0.0.1:8080", "API base URL") + flag.Parse() + if err := run(); err != nil { + fmt.Fprintln(os.Stderr, "\ndemo:", err) + os.Exit(1) + } +} + +func run() error { + month := metering.MonthOf(time.Now()).Prev() + usagePath := fmt.Sprintf("/v1/customers/%s/usage?from=%s&to=%s", customer, + month.Start().Format(time.DateOnly), month.End().Format(time.DateOnly)) + fmt.Printf("Metering API demo against %s, billing month %s\n", base, month) + + var before metering.Usage + if status, err := call("GET", usagePath, nil, &before); err != nil || status != http.StatusOK { + return fmt.Errorf("customer %s not found (status %d, %v): run `make seed` first", customer, status, err) + } + + // Hourly gateway reports for the last day of the month, with stable ids. + events := make([]metering.Event, 24) + lastDay := month.End().AddDate(0, 0, -1) + for h := range events { + events[h] = metering.Event{ + EventID: fmt.Sprintf("evt_demo_%s_%s_%02d", customer, month.Start().Format("200601"), h), + CustomerID: customer, + Quantity: 1000, + OccurredAt: lastDay.Add(time.Duration(h)*time.Hour + 59*time.Minute), + } + } + body := map[string]any{"events": events} + + step(1, "Ingest a batch of usage reports") + var res metering.IngestResult + status, err := call("POST", "/v1/events", body, &res) + if err != nil { + return err + } + show("POST /v1/events", status, fmt.Sprintf("received %d, inserted %d, duplicates %d", res.Received, res.Inserted, res.Duplicates)) + + step(2, "The gateway retries the same batch (same event ids)") + if status, err = call("POST", "/v1/events", body, &res); err != nil { + return err + } + show("POST /v1/events", status, fmt.Sprintf("received %d, inserted %d, duplicates %d", res.Received, res.Inserted, res.Duplicates)) + var after metering.Usage + if _, err := call("GET", usagePath, nil, &after); err != nil { + return err + } + show("GET "+usagePath, http.StatusOK, fmt.Sprintf("%s calls in %d events (%s plan)", thousands(after.TotalCalls), after.Events, after.PlanID)) + note("before step 1: %s calls; step 1 added %s, the retry nothing", + thousands(before.TotalCalls), thousands(after.TotalCalls-before.TotalCalls)) + + step(3, "Close the month: rate usage, write invoices, mark usage invoiced (one transaction)") + var closed metering.CloseResult + path := "/v1/invoices/" + month.String() + "/close" + if status, err = call("POST", path, nil, &closed); err != nil { + return err + } + switch status { + case http.StatusCreated: + show("POST "+path, status, fmt.Sprintf("%d invoices, %s total, %s events invoiced", + closed.Invoices, dollars(closed.TotalCents), thousands(closed.EventsInvoiced))) + case http.StatusConflict: + show("POST "+path, status, "already closed by an earlier run (invoices are issued once)") + default: + return fmt.Errorf("close: unexpected status %d", status) + } + + step(4, "Start the monthly rollup as an async query and poll it") + var started struct { + QueryID string `json:"query_id"` + StatusURL string `json:"status_url"` + } + path = "/v1/rollups/" + month.String() + if status, err = call("POST", path, nil, &started); err != nil { + return err + } + show("POST "+path, status, "query "+started.QueryID) + rollup, polls, err := poll(started.StatusURL) + if err != nil { + return err + } + show("GET "+started.StatusURL, http.StatusOK, fmt.Sprintf("%s after %d poll(s), %d rows", rollup.Status, polls, len(rollup.Rows))) + printRollup(rollup.Rows) + + step(5, "Export the month's invoice lines as CSV (streamed)") + path = "/v1/invoices/" + month.String() + "/export.csv" + lines, sum, head, err := download(path) + if err != nil { + return err + } + show("GET "+path, http.StatusOK, fmt.Sprintf("%d rows plus header, sha256 %s...", lines-1, sum[:16])) + for _, l := range head { + fmt.Println(" " + l) + } + + step(6, "Errors are problem details") + var problem metering.Problem + path = "/v1/customers/cus_9999/usage" + status, _ = call("GET", path, nil, &problem) + show("GET "+path, status, problem.Type+": "+problem.Detail) + problem = metering.Problem{} + status, _ = call("POST", "/v1/events", map[string]any{"events": []map[string]any{{"event_id": "e1", "customer_id": customer, "quantity": -5}}}, &problem) + detail := problem.Type + for _, e := range problem.Errors { + detail += fmt.Sprintf("\n %s %s", e.Field, e.Detail) + } + show("POST /v1/events (quantity -5, no occurred_at)", status, detail) + fmt.Println("\nDone.") + return nil +} + +func call(method, path string, body, out any) (int, error) { + var reader io.Reader + if body != nil { + b, err := json.Marshal(body) + if err != nil { + return 0, err + } + reader = bytes.NewReader(b) + } + req, err := http.NewRequest(method, base+path, reader) + if err != nil { + return 0, err + } + req.Header.Set("Content-Type", "application/json") + resp, err := client.Do(req) + if err != nil { + return 0, err + } + defer resp.Body.Close() + if out != nil { + if err := json.NewDecoder(resp.Body).Decode(out); err != nil { + return resp.StatusCode, fmt.Errorf("%s %s: decode response: %w", method, path, err) + } + } + return resp.StatusCode, nil +} + +func poll(path string) (metering.Rollup, int, error) { + deadline := time.Now().Add(2 * time.Minute) + for n := 1; ; n++ { + var r metering.Rollup + if _, err := call("GET", path, nil, &r); err != nil { + return r, n, err + } + switch r.Status { + case metering.RollupSucceeded: + return r, n, nil + case metering.RollupFailed: + return r, n, fmt.Errorf("rollup failed: %s", r.Error) + } + if time.Now().After(deadline) { + return r, n, fmt.Errorf("rollup still %s after 2 minutes", r.Status) + } + time.Sleep(500 * time.Millisecond) + } +} + +func download(path string) (lines int, sum string, head []string, err error) { + resp, err := client.Get(base + path) + if err != nil { + return 0, "", nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return 0, "", nil, fmt.Errorf("export: status %d", resp.StatusCode) + } + h := sha256.New() + scanner := bufio.NewScanner(io.TeeReader(resp.Body, h)) + for scanner.Scan() { + if lines < 4 { + head = append(head, scanner.Text()) + } + lines++ + } + return lines, hex.EncodeToString(h.Sum(nil)), head, scanner.Err() +} + +func printRollup(rows []metering.RollupRow) { + type total struct{ calls, events, peak int64 } + totals := map[string]*total{} + for _, r := range rows { + t := totals[r.PlanID] + if t == nil { + t = &total{} + totals[r.PlanID] = t + } + t.calls += r.Calls + t.events += r.Events + t.peak = max(t.peak, r.ActiveCustomers) + } + fmt.Printf(" %-11s %14s %8s %22s\n", "plan", "calls", "events", "peak daily customers") + for _, plan := range []string{"FREE", "PRO", "ENTERPRISE"} { + if t := totals[plan]; t != nil { + fmt.Printf(" %-11s %14s %8s %22d\n", plan, thousands(t.calls), thousands(t.events), t.peak) + } + } +} + +func step(n int, title string) { fmt.Printf("\n[%d/6] %s\n", n, title) } + +func show(request string, status int, result string) { + fmt.Printf(" %s\n -> %d %s\n", request, status, result) +} + +func note(format string, args ...any) { fmt.Printf(" (%s)\n", fmt.Sprintf(format, args...)) } + +func dollars(cents int64) string { + return fmt.Sprintf("$%s.%02d", thousands(cents/100), cents%100) +} + +func thousands(n int64) string { + s := fmt.Sprint(n) + var b strings.Builder + for i, r := range s { + if i > 0 && (len(s)-i)%3 == 0 { + b.WriteByte(',') + } + b.WriteRune(r) + } + return b.String() +} diff --git a/metering-api/cmd/seed/main.go b/metering-api/cmd/seed/main.go new file mode 100644 index 0000000..2c66c7a --- /dev/null +++ b/metering-api/cmd/seed/main.go @@ -0,0 +1,73 @@ +// Command seed loads the synthetic dataset: ~200 customers and the last three +// complete months of their usage. Events go through the same idempotent ingest +// path as POST /v1/events, so running it again inserts nothing. +package main + +import ( + "context" + "flag" + "fmt" + "log/slog" + "os" + "time" + + "github.com/localstack-samples/localstack-snowflake-samples/metering-api/internal/config" + "github.com/localstack-samples/localstack-snowflake-samples/metering-api/internal/metering" + "github.com/localstack-samples/localstack-snowflake-samples/metering-api/internal/seed" +) + +func main() { + if err := run(); err != nil { + fmt.Fprintln(os.Stderr, "seed:", err) + os.Exit(1) + } +} + +func run() error { + customers := flag.Int("customers", 200, "number of customers") + months := flag.Int("months", 3, "complete months of usage before the current one") + seedValue := flag.Uint64("seed", 42, "random seed") + flag.Parse() + + ctx := metering.WithQueryTag(context.Background(), "seed") + cfg, err := config.SnowflakeFromEnv() + if err != nil { + return err + } + db, err := config.Open(ctx, cfg) + if err != nil { + return err + } + defer db.Close() + svc := metering.NewService(db, slog.New(slog.NewTextHandler(os.Stderr, nil))) + + window := seed.LastCompleteMonths(time.Now(), *months) + data := seed.Generate(seed.Options{Seed: *seedValue, Customers: *customers, Months: window}) + plans := map[string]int{} + for _, c := range data.Customers { + plans[c.PlanID]++ + } + fmt.Printf("Seeding %s (%s to %s, seed %d)\n", cfg.Database, window[0], window[len(window)-1], *seedValue) + + start := time.Now() + if err := svc.UpsertCustomers(ctx, data.Customers); err != nil { + return err + } + fmt.Printf(" customers %d (FREE %d, PRO %d, ENTERPRISE %d)\n", + len(data.Customers), plans["FREE"], plans["PRO"], plans["ENTERPRISE"]) + + var total metering.IngestResult + for i := 0; i < len(data.Events); i += metering.MaxBatch { + batch := data.Events[i:min(i+metering.MaxBatch, len(data.Events))] + res, err := svc.IngestEvents(ctx, batch) + if err != nil { + return fmt.Errorf("events %d-%d: %w", i, i+len(batch)-1, err) + } + total.Received += res.Received + total.Inserted += res.Inserted + total.Duplicates += res.Duplicates + } + fmt.Printf(" events %d received, %d inserted, %d already present\n", total.Received, total.Inserted, total.Duplicates) + fmt.Printf("Done in %s\n", time.Since(start).Round(time.Second)) + return nil +} diff --git a/metering-api/docker-compose.yml b/metering-api/docker-compose.yml new file mode 100644 index 0000000..b3587f7 --- /dev/null +++ b/metering-api/docker-compose.yml @@ -0,0 +1,21 @@ +# The local Snowflake for the dev loop (make up / seed / run / demo). The API runs on +# the host with `go run`; the integration tests start their own emulator with +# testcontainers-go. +name: snfs-metering-api + +services: + snowflake: + image: ${SNOWFLAKE_IMAGE:-localstack/snowflake-next:latest} + pull_policy: always + ports: + - "127.0.0.1:${SNOWFLAKE_PORT:-4566}:4566" + environment: + LOCALSTACK_AUTH_TOKEN: ${LOCALSTACK_AUTH_TOKEN:?set LOCALSTACK_AUTH_TOKEN} + volumes: + - ./init:/etc/localstack/init/ready.d:ro + networks: + default: + aliases: + - snowflake.localhost.localstack.cloud + - stages.s3.localhost.localstack.cloud + - stages-enc.s3.localhost.localstack.cloud diff --git a/metering-api/docs/architecture.md b/metering-api/docs/architecture.md new file mode 100644 index 0000000..0cf01df --- /dev/null +++ b/metering-api/docs/architecture.md @@ -0,0 +1,114 @@ +# Metering API: design notes + +How the [Metering API](../README.md) stores usage, prices it and talks to Snowflake. + +## Data model + +```mermaid +erDiagram + PRICE_PLANS ||--o{ CUSTOMERS : "prices" + CUSTOMERS ||--o{ USAGE_EVENTS : "reports" + CUSTOMERS ||--o{ INVOICES : "is billed" + INVOICES ||--|{ INVOICE_LINES : "has" + INVOICES ||--o{ USAGE_EVENTS : "bills" + PRICE_PLANS { + varchar PLAN_ID PK + number TIER_NO PK + number UP_TO_CALLS + number UNIT_PRICE_MICROS + number FLAT_FEE_CENTS + } + CUSTOMERS { + varchar CUSTOMER_ID PK + varchar NAME + varchar PLAN_ID + timestamp_ntz CREATED_AT + } + USAGE_EVENTS { + varchar EVENT_ID PK + varchar CUSTOMER_ID + number QUANTITY + timestamp_ntz OCCURRED_AT + timestamp_ntz RECEIVED_AT + varchar INVOICE_ID + } + INVOICES { + varchar INVOICE_ID PK + varchar CUSTOMER_ID + date BILLING_MONTH + varchar PLAN_ID + number TOTAL_CALLS + number TOTAL_CENTS + } + INVOICE_LINES { + varchar INVOICE_ID PK + number LINE_NO PK + number TIER_NO + number QUANTITY + number AMOUNT_CENTS + } +``` + +- A usage event is one report from the API gateway: `QUANTITY` calls by one customer in one reporting window. `EVENT_ID` is the gateway's idempotency key. +- Money is integer throughout: unit prices in micro-dollars per call, amounts in cents. Nothing is ever a float. +- `USAGE_EVENTS_STAGING` (transient, not shown) holds an ingest batch for the length of one transaction. +- Snowflake records primary keys but does not enforce them; the write paths below keep the keys unique. + +## Pricing + +Plans are graduated tiers, one `PRICE_PLANS` row per tier. Calls fill the tiers from the bottom: each tier bills the calls between the previous tier's limit and its own at its unit price, plus its flat fee once usage reaches it. Tier 1 is always reached, so its flat fee is the monthly fee. + +| Plan | Tier 1 | Tier 2 | Tier 3 | +| --- | --- | --- | --- | +| FREE | first 10,000 calls free | $2.00 per 1,000 | | +| PRO | $49, includes 100,000 calls | $0.80 per 1,000 up to 1M | $0.50 per 1,000 | +| ENTERPRISE | $499, includes 1M calls | $0.40 per 1,000 up to 10M | $0.25 per 1,000 | + +Example: a Pro customer with 250,000 calls pays $49 + 150,000 x $0.80 / 1,000 = $169.00. Each line is rounded half up to the cent. Rating runs in Go (`internal/pricing`), so it is unit-tested without a database; the tiers themselves are data. + +## Idempotent ingestion + +`POST /v1/events` accepts up to 1,000 events and runs one transaction: + +1. `INSERT INTO USAGE_EVENTS_STAGING ... VALUES (?, ?, ?, ?, ?)` with one array bind per column (`gosnowflake.Array`), tagged with a fresh `BATCH_ID`. The whole batch is one round trip. +2. A `LEFT JOIN` against `CUSTOMERS` finds unknown customer ids; if there are any, the transaction rolls back and the API answers `422` with the ids. +3. `MERGE INTO USAGE_EVENTS ... ON t.EVENT_ID = s.EVENT_ID WHEN NOT MATCHED THEN INSERT`, over the batch deduplicated with `QUALIFY ROW_NUMBER() OVER (PARTITION BY EVENT_ID ...) = 1`. The `MERGE`'s row count is the number of new events. +4. `DELETE` of the batch's staging rows, then `COMMIT`. + +Snowflake serializes concurrent `MERGE` statements on a table, so two gateways retrying the same event at once still store it once. The 1,000-event cap keeps each array bind well inside a single request. + +## Transactional invoice close + +`POST /v1/invoices/{month}/close` refuses a month that has not ended (`409`) and runs one `database/sql` transaction: + +1. If `INVOICES` already has rows for the month, roll back with `409`: invoices are issued once. +2. Read `PRICE_PLANS`, and every customer's uninvoiced calls in the month (`LEFT JOIN`, so customers without usage are billed their monthly fee). +3. Rate each customer in Go, then insert all invoices and all invoice lines with two array-bound `INSERT`s. +4. `UPDATE USAGE_EVENTS ... FROM INVOICES` sets `INVOICE_ID` on the month's events. +5. `COMMIT`. + +Any error rolls everything back, so a month is either fully invoiced or untouched and can simply be closed again. The integration suite proves this by failing the transaction between steps 3 and 4. Invoice ids are deterministic (`inv_202609_cus_0007`), which keeps the export stable and makes retries easy to reason about. + +## Async rollups + +The monthly rollup (calls, events and active customers per plan and day) is a query a dashboard asks for and polls, rather than a request that holds a connection open: + +- `POST /v1/rollups/{month}` runs the query with `gosnowflake.WithAsyncMode` and returns `202` with the Snowflake query id as soon as Snowflake accepts it. The request's context is detached (`context.WithoutCancel` plus a timeout) so the query is not canceled when the response is sent. +- `GET /v1/rollups/{queryId}` calls `GetQueryStatus` on a pooled connection: running, failed, unknown (`404`) or succeeded. On success it reads the rows with `WithFetchResultByID`. + +Results are kept by Snowflake, not by the service, so any replica can answer the poll, and a restart loses nothing. + +## Streaming export + +`GET /v1/invoices/{month}/export.csv` scans rows from `database/sql` and writes them straight to a `csv.Writer` on the response, flushing every 500 rows. Memory stays flat whatever the month's size. If a read fails after the first byte has been sent, the handler aborts the connection so the client sees a broken transfer rather than a short file. + +## Connection and configuration + +- All settings come from the environment (`internal/config`), with `TEST_TARGET` choosing emulator or account defaults. The production shape is the same: only `SNOWFLAKE_ACCOUNT`, `SNOWFLAKE_HOST` and the key change. +- Authentication is key-pair only (`SNOWFLAKE_JWT`) with a PKCS#8 key; `make up` generates a throwaway pair in `.secrets/`. On a real account, register the public key with `ALTER USER METERING_SVC SET RSA_PUBLIC_KEY = '...'`. +- One `*sql.DB` pool per process. Every route sets a `QUERY_TAG` through the context (`metering-api:invoices.close`, `metering-api:events.ingest`, ...), so `QUERY_HISTORY` shows which endpoint ran each statement. +- The API shuts down gracefully on `SIGINT`/`SIGTERM`: it stops accepting connections and waits for in-flight requests. + +## Test isolation + +`TestMain` provisions the account once (`init/01_platform.sf.sql`). Each test then creates a `METERING_IT_` database, hands it to the service role, applies `init/02_schema.sf.sql` and `init/03_price_plans.sf.sql` to it, starts the API on `httptest` and drops the database in `t.Cleanup`. Tests share nothing, so they run with `t.Parallel()`, and the same code runs unchanged against a real account with `TEST_TARGET=SNOWFLAKE_CLOUD`. diff --git a/metering-api/go.mod b/metering-api/go.mod new file mode 100644 index 0000000..1c64068 --- /dev/null +++ b/metering-api/go.mod @@ -0,0 +1,112 @@ +module github.com/localstack-samples/localstack-snowflake-samples/metering-api + +go 1.26.0 + +toolchain go1.27.3 + +require ( + github.com/snowflakedb/gosnowflake/v2 v2.3.0 + github.com/testcontainers/testcontainers-go v0.44.0 +) + +require ( + dario.cat/mergo v1.0.2 // indirect + github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect + github.com/99designs/keyring v1.2.2 // indirect + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 // indirect + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 // indirect + github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c // indirect + github.com/BurntSushi/toml v1.4.0 // indirect + github.com/Microsoft/go-winio v0.6.2 // indirect + github.com/apache/arrow-go/v18 v18.4.0 // indirect + github.com/aws/aws-sdk-go-v2 v1.43.0 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.14 // indirect + github.com/aws/aws-sdk-go-v2/config v1.32.31 // indirect + github.com/aws/aws-sdk-go-v2/credentials v1.19.30 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.31 // indirect + github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.3.5 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.31 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.31 // indirect + github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.32 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.13 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.24 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.31 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.32 // indirect + github.com/aws/aws-sdk-go-v2/service/s3 v1.106.0 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.5.0 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.33.0 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.38.0 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.45.0 // indirect + github.com/aws/smithy-go v1.27.4 // indirect + github.com/cenkalti/backoff/v4 v4.3.0 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/containerd/errdefs v1.0.0 // indirect + github.com/containerd/errdefs/pkg v0.3.0 // indirect + github.com/containerd/log v0.1.0 // indirect + github.com/containerd/platforms v0.2.1 // indirect + github.com/cpuguy83/dockercfg v0.3.2 // indirect + github.com/danieljoos/wincred v1.2.2 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/distribution/reference v0.6.0 // indirect + github.com/docker/go-connections v0.7.0 // indirect + github.com/docker/go-units v0.5.0 // indirect + github.com/dvsekhvalnov/jose2go v1.7.0 // indirect + github.com/ebitengine/purego v0.10.1 // indirect + github.com/felixge/httpsnoop v1.1.0 // indirect + github.com/gabriel-vasile/mimetype v1.4.7 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-ole/go-ole v1.3.0 // indirect + github.com/goccy/go-json v0.10.5 // indirect + github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 // indirect + github.com/golang-jwt/jwt/v5 v5.2.2 // indirect + github.com/google/flatbuffers v25.2.10+incompatible // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect + github.com/klauspost/compress v1.18.6 // indirect + github.com/klauspost/cpuid/v2 v2.2.11 // indirect + github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e // indirect + github.com/magiconair/properties v1.8.10 // indirect + github.com/moby/docker-image-spec v1.3.1 // indirect + github.com/moby/go-archive v0.2.0 // indirect + github.com/moby/moby/api v1.55.0 // indirect + github.com/moby/moby/client v0.5.0 // indirect + github.com/moby/patternmatcher v0.6.1 // indirect + github.com/moby/sys/sequential v0.7.0 // indirect + github.com/moby/sys/user v0.4.0 // indirect + github.com/moby/sys/userns v0.1.0 // indirect + github.com/moby/term v0.5.2 // indirect + github.com/mtibben/percent v0.2.1 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/opencontainers/image-spec v1.1.1 // indirect + github.com/pierrec/lz4/v4 v4.1.22 // indirect + github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect + github.com/shirou/gopsutil/v4 v4.26.6 // indirect + github.com/sirupsen/logrus v1.9.4 // indirect + github.com/stretchr/testify v1.11.1 // indirect + github.com/tklauser/go-sysconf v0.4.0 // indirect + github.com/tklauser/numcpus v0.12.0 // indirect + github.com/yusufpapurcu/wmi v1.2.4 // indirect + github.com/zeebo/xxh3 v1.0.2 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 // indirect + go.opentelemetry.io/otel v1.44.0 // indirect + go.opentelemetry.io/otel/metric v1.44.0 // indirect + go.opentelemetry.io/otel/trace v1.44.0 // indirect + golang.org/x/crypto v0.54.0 // indirect + golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0 // indirect + golang.org/x/mod v0.37.0 // indirect + golang.org/x/net v0.56.0 // indirect + golang.org/x/oauth2 v0.34.0 // indirect + golang.org/x/sync v0.22.0 // indirect + golang.org/x/sys v0.47.0 // indirect + golang.org/x/telemetry v0.0.0-20260625142307-59b4966ccb57 // indirect + golang.org/x/term v0.45.0 // indirect + golang.org/x/text v0.40.0 // indirect + golang.org/x/tools v0.47.0 // indirect + golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/metering-api/go.sum b/metering-api/go.sum new file mode 100644 index 0000000..bc0b5bc --- /dev/null +++ b/metering-api/go.sum @@ -0,0 +1,266 @@ +dario.cat/mergo v1.0.2 h1:85+piFYR1tMbRrLcDwR18y4UKJ3aH1Tbzi24VRW1TK8= +dario.cat/mergo v1.0.2/go.mod h1:E/hbnu0NxMFBjpMIE34DRGLWqDy0g5FuKDhCb31ngxA= +github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 h1:/vQbFIOMbk2FiG/kXiLl8BRyzTWDw7gX/Hz7Dd5eDMs= +github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4/go.mod h1:hN7oaIRCjzsZ2dE+yG5k+rsdt3qcwykqK6HVGcKwsw4= +github.com/99designs/keyring v1.2.2 h1:pZd3neh/EmUzWONb35LxQfvuY7kiSXAq3HQd97+XBn0= +github.com/99designs/keyring v1.2.2/go.mod h1:wes/FrByc8j7lFOAGLGSNEg8f/PaI3cgTBqhFkHUrPk= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6 h1:He8afgbRMd7mFxO99hRNu+6tazq8nFF9lIwo9JFroBk= +github.com/AdaLogics/go-fuzz-headers v0.0.0-20240806141605-e8a1dd7889d6/go.mod h1:8o94RPi1/7XTJvwPpRSzSUedZrtlirdB3r9Z20bi2f8= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0 h1:rTnT/Jrcm+figWlYz4Ixzt0SJVR2cMC8lvZcimipiEY= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.4.0/go.mod h1:ON4tFdPTwRcgWEaVDrN3584Ef+b7GgSJaXxe5fW9t4M= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0 h1:QkAcEIAKbNL4KoFr4SathZPhDhF4mVwpBMFlYjyAqy8= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.1.0/go.mod h1:bhXu1AjYL+wutSL/kpSq6s7733q2Rb0yuot9Zgfqa/0= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2 h1:+5VZ72z0Qan5Bog5C+ZkgSqUbeVUd9wgtHOrIKuc5b8= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.1.2/go.mod h1:eWRD7oawr1Mu1sLCawqVc0CUiF43ia3qQMxLscsKQ9w= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0 h1:u/LLAOFgsMv7HmNL4Qufg58y+qElGOt5qv0z1mURkRY= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.0.0/go.mod h1:2e8rMJtl2+2j+HXbTBwnyGpm5Nou7KhvSfxOq8JpTag= +github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c h1:udKWzYgxTojEKWjV8V+WSxDXJ4NFATAsZjh8iIbsQIg= +github.com/Azure/go-ansiterm v0.0.0-20250102033503-faa5f7b0171c/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/AzureAD/microsoft-authentication-library-for-go v0.5.1 h1:BWe8a+f/t+7KY7zH2mqygeUD0t8hNFXe08p1Pb3/jKE= +github.com/AzureAD/microsoft-authentication-library-for-go v0.5.1/go.mod h1:Vt9sXTKwMyGcOxSmLDMnGPgqsUg7m8pe215qMLrDXw4= +github.com/BurntSushi/toml v1.4.0 h1:kuoIxZQy2WRRk1pttg9asf+WVv6tWQuBNVmK8+nqPr0= +github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/andybalholm/brotli v1.2.0 h1:ukwgCxwYrmACq68yiUqwIWnGY0cTPox/M94sVwToPjQ= +github.com/andybalholm/brotli v1.2.0/go.mod h1:rzTDkvFWvIrjDXZHkuS16NPggd91W3kUSvPlQ1pLaKY= +github.com/apache/arrow-go/v18 v18.4.0 h1:/RvkGqH517iY8bZKc4FD5/kkdwXJGjxf28JIXbJ/oB0= +github.com/apache/arrow-go/v18 v18.4.0/go.mod h1:Aawvwhj8x2jURIzD9Moy72cF0FyJXOpkYpdmGRHcw14= +github.com/apache/thrift v0.22.0 h1:r7mTJdj51TMDe6RtcmNdQxgn9XcyfGDOzegMDRg47uc= +github.com/apache/thrift v0.22.0/go.mod h1:1e7J/O1Ae6ZQMTYdy9xa3w9k+XHWPfRvdPyJeynQ+/g= +github.com/aws/aws-sdk-go-v2 v1.43.0 h1:fharf/WhbRAVZ1du0QL7roNFxZ6T/sWr+4Ni617bwSI= +github.com/aws/aws-sdk-go-v2 v1.43.0/go.mod h1:5pKeft2eJj+gElQ38Jqg4ibCqh+/AK33/0X3hip7IjM= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.14 h1:3IZY0XAJquT3aHzbkHfPzy4ACPcEjVG0x87KOwtpqGY= +github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.14/go.mod h1:zwM6veDkhGgQFqkBy+uT28AAYpLu+uFMlPl+rCg/73E= +github.com/aws/aws-sdk-go-v2/config v1.32.31 h1:n4nY9O3QKoHIkL85EX+V8RcMFtOhlpTFhGArg915PXk= +github.com/aws/aws-sdk-go-v2/config v1.32.31/go.mod h1:PN0NYDCCoOpGGsZ2+elDUidmHfQBPyYzN2GCgl8HEBs= +github.com/aws/aws-sdk-go-v2/credentials v1.19.30 h1:TTCvvzFU6gXa4iJecNG/0F/B0oYTiazoRECr2XyLHrY= +github.com/aws/aws-sdk-go-v2/credentials v1.19.30/go.mod h1:jKxAp2AEncnliinzpgOSZDFv6+VjvWhjw/AtbfsWT9U= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.31 h1:kfVL5wAunCJycL6MOQ6aNh6PlAYEymflcjuKmrWUA0o= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.31/go.mod h1:nWfRNDAppujCQgOUd43lKT4yeLv9z3nJ3bw1G3BgQKo= +github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.3.5 h1:7ZFdtE1XEHH+58GZU4Mbhq6SO/UbColleDApOtlv3vo= +github.com/aws/aws-sdk-go-v2/feature/s3/transfermanager v0.3.5/go.mod h1:M/qt7xBBXqilBwNmrO1yiu0cywZwQx5aqtKpdJN9J2A= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.31 h1:Z8F3hfCY33IGpJjFAnv0wvtv1FIKj1GHmRDEYqy64tw= +github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.31/go.mod h1:aVyUoytEyOViR6jhq6jula0xkc5NfBE2hgeF6BvOrao= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.31 h1:hyOxUyXdh3AyjE93gBgsfziJag9ACwcs+ZpDBLzi8mw= +github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.31/go.mod h1:OERqI9k0draSLB8O8woxY3q25ZWTELRK4RRoLMuMZFo= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.32 h1:0MrUL35H/Y4kdFfItoR5jCgtDQ4Z/8LudAoIHRfA4hE= +github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.32/go.mod h1:2tNZkuWz54arj8mHVf+8Y7cKkcD8Wr/fBpENgEXpjLc= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.13 h1:mbRIur/BiHK6SKPjoBIXSE/hJ6g6JGRLuxQy1jGjlN4= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.13/go.mod h1:ITg9em2KbJx1s0y4aqRX5OYWG6HBZ5TVR//OdpEZ2CQ= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.24 h1:mdPwDQPqxlw9Sc62Nt15yjEcARaDbPXkjRYtXsUripo= +github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.24/go.mod h1:ls5ytnwLTcQaUu32fMYXFI3MjpKuTwL840PAm9iqyEg= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.31 h1:w2SIhW92DZPFrSL4ksVCr8IYff5OZwIcxg8+95tzvAI= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.31/go.mod h1:wAhpCQbkov+IcvjozJbd2xRCoZybUEHNkcFunssNACg= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.32 h1:jWXtZdCnhXa9sGFixRaU2AxT4DIVse9HS4E2f+/KwV0= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.32/go.mod h1:9JS1UpfVvyD/ZPX8GsKb/Pq8scEM+7GP5fqh9SwH7po= +github.com/aws/aws-sdk-go-v2/service/s3 v1.106.0 h1:7QZWVJZWzHivHWIa+5TELLaBBkbuoj0GPwQtMlJ0sqk= +github.com/aws/aws-sdk-go-v2/service/s3 v1.106.0/go.mod h1:fcvq5L7dK+5cQFicEJwpI6e6Wn8NY2i6yT5wRLYVc7s= +github.com/aws/aws-sdk-go-v2/service/signin v1.5.0 h1:OHH5iTQvVGmfHjX/5Q+vFuA/Rf2x6/95aJ/75QCQSm4= +github.com/aws/aws-sdk-go-v2/service/signin v1.5.0/go.mod h1:mCF3AK9PpL49oOrhniUXWAfhVBVQ/XbytoE5eccZUIs= +github.com/aws/aws-sdk-go-v2/service/sso v1.33.0 h1:CaJyYhxBE0M/HJX/YvSaSmQlsI91VHB0lKU8LtLxL3A= +github.com/aws/aws-sdk-go-v2/service/sso v1.33.0/go.mod h1:+e6BMRMPjBQoCw/WovYR9GLy2IU0z4Q77smOB1DraSg= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.38.0 h1:tC323YV77QdafeBr6LUhLDTsboyuyHLNRwAyCP44kGU= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.38.0/go.mod h1:SfLK1sgviHmbI+MozR9iDwDjL4cdCVZtahsjoR+z7wg= +github.com/aws/aws-sdk-go-v2/service/sts v1.45.0 h1:Pd6PNlp4t8PTXxqzstICl52Wsy78vpjFZ7PRUj44mJc= +github.com/aws/aws-sdk-go-v2/service/sts v1.45.0/go.mod h1:rmQ0TnHzuLPmabgjPcsywhsSOmaBDgzR4zvDxSPsGdg= +github.com/aws/smithy-go v1.27.4 h1:JQcphmBN4f0q/sPqXqROIItRNV/hy10cgu7CsFy616M= +github.com/aws/smithy-go v1.27.4/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= +github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= +github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= +github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= +github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= +github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= +github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I= +github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo= +github.com/containerd/platforms v0.2.1 h1:zvwtM3rz2YHPQsF2CHYM8+KtB5dvhISiXh5ZpSBQv6A= +github.com/containerd/platforms v0.2.1/go.mod h1:XHCb+2/hzowdiut9rkudds9bE5yJ7npe7dG/wG+uFPw= +github.com/cpuguy83/dockercfg v0.3.2 h1:DlJTyZGBDlXqUZ2Dk2Q3xHs/FtnooJJVaad2S9GKorA= +github.com/cpuguy83/dockercfg v0.3.2/go.mod h1:sugsbF4//dDlL/i+S+rtpIWp+5h0BHJHfjj5/jFyUJc= +github.com/creack/pty v1.1.24 h1:bJrF4RRfyJnbTJqzRLHzcGaZK1NeM5kTC9jGgovnR1s= +github.com/creack/pty v1.1.24/go.mod h1:08sCNb52WyoAwi2QDyzUCTgcvVFhUzewun7wtTfvcwE= +github.com/danieljoos/wincred v1.2.2 h1:774zMFJrqaeYCK2W57BgAem/MLi6mtSE47MB6BOJ0i0= +github.com/danieljoos/wincred v1.2.2/go.mod h1:w7w4Utbrz8lqeMbDAK0lkNJUv5sAOkFi7nd/ogr0Uh8= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/dnaeon/go-vcr v1.1.0 h1:ReYa/UBrRyQdant9B4fNHGoCNKw6qh6P0fsdGmZpR7c= +github.com/dnaeon/go-vcr v1.1.0/go.mod h1:M7tiix8f0r6mKKJ3Yq/kqU1OYf3MnfmBWVbPx/yU9ko= +github.com/docker/go-connections v0.7.0 h1:6SsRfJddP22WMrCkj19x9WKjEDTB+ahsdiGYf0mN39c= +github.com/docker/go-connections v0.7.0/go.mod h1:no1qkHdjq7kLMGUXYAduOhYPSJxxvgWBh7ogVvptn3Q= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/dvsekhvalnov/jose2go v1.7.0 h1:bnQc8+GMnidJZA8zc6lLEAb4xNrIqHwO+9TzqvtQZPo= +github.com/dvsekhvalnov/jose2go v1.7.0/go.mod h1:QsHjhyTlD/lAVqn/NSbVZmSCGeDehTB/mPZadG+mhXU= +github.com/ebitengine/purego v0.10.1 h1:dewVBCBT2GaMu1SrNTYxQhgQBethzfhiwvZiLGP/qyY= +github.com/ebitengine/purego v0.10.1/go.mod h1:iIjxzd6CiRiOG0UyXP+V1+jWqUXVjPKLAI0mRfJZTmQ= +github.com/felixge/httpsnoop v1.1.0 h1:3YtUj32ZZkqZtt3sZZsClsymw/QDuVfpNhoA31zeORc= +github.com/felixge/httpsnoop v1.1.0/go.mod h1:Zqxgdd+1Rkcz8euOqdr7lqgCRJztwr5hp9vDSi5UZCE= +github.com/gabriel-vasile/mimetype v1.4.7 h1:SKFKl7kD0RiPdbht0s7hFtjl489WcQ1VyPW8ZzUMYCA= +github.com/gabriel-vasile/mimetype v1.4.7/go.mod h1:GDlAgAyIRT27BhFl53XNAFtfjzOkLaF35JdEG0P7LtU= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= +github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= +github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= +github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 h1:ZpnhV/YsD2/4cESfV5+Hoeu/iUR3ruzNvZ+yQfO03a0= +github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= +github.com/golang-jwt/jwt v3.2.1+incompatible h1:73Z+4BJcrTC+KczS6WvTPvRGOp1WmfEP4Q1lOd9Z/+c= +github.com/golang-jwt/jwt v3.2.1+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= +github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8= +github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= +github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= +github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/flatbuffers v25.2.10+incompatible h1:F3vclr7C3HpB1k9mxCGRMXq6FdUalZ6H/pNX4FP1v0Q= +github.com/google/flatbuffers v25.2.10+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c h1:6rhixN/i8ZofjG1Y75iExal34USq5p+wiN1tpie8IrU= +github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c/go.mod h1:NMPJylDgVpX0MLRlPy15sqSwOFv/U1GZ2m21JhFfek0= +github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4= +github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= +github.com/klauspost/compress v1.18.6 h1:2jupLlAwFm95+YDR+NwD2MEfFO9d4z4Prjl1XXDjuao= +github.com/klauspost/compress v1.18.6/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/cpuid/v2 v2.2.11 h1:0OwqZRYI2rFrjS4kvkDnqJkKHdHaRnCm68/DY4OxRzU= +github.com/klauspost/cpuid/v2 v2.2.11/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e h1:Q6MvJtQK/iRcRtzAscm/zF23XxJlbECiGPyRicsX+Ak= +github.com/lufia/plan9stats v0.0.0-20260330125221-c963978e514e/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg= +github.com/magiconair/properties v1.8.10 h1:s31yESBquKXCV9a/ScB3ESkOjUYYv+X0rg8SYxI99mE= +github.com/magiconair/properties v1.8.10/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs= +github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8/go.mod h1:mC1jAcsrzbxHt8iiaC+zU4b1ylILSosueou12R++wfY= +github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 h1:+n/aFZefKZp7spd8DFdX7uMikMLXX4oubIzJF4kv/wI= +github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3/go.mod h1:RagcQ7I8IeTMnF8JTXieKnO4Z6JCsikNEzj0DwauVzE= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/go-archive v0.2.0 h1:zg5QDUM2mi0JIM9fdQZWC7U8+2ZfixfTYoHL7rWUcP8= +github.com/moby/go-archive v0.2.0/go.mod h1:mNeivT14o8xU+5q1YnNrkQVpK+dnNe/K6fHqnTg4qPU= +github.com/moby/moby/api v1.55.0 h1:2/sexvQyqIWS8pRSCFddBfpW2qE7vR7FCL+vN8pxwMc= +github.com/moby/moby/api v1.55.0/go.mod h1:+RQ6wluLwtYaTd1WnPLykIDPekkuyD/ROWQClE83pzs= +github.com/moby/moby/client v0.5.0 h1:5XhyPk2fuOWf6RlSFa3MkIIgDZkF25xToXW8Q/BH7cc= +github.com/moby/moby/client v0.5.0/go.mod h1:rcVpF8ncl9vo5gaIBdol6CnbEtSj1uxMvEV/UrykF/s= +github.com/moby/patternmatcher v0.6.1 h1:qlhtafmr6kgMIJjKJMDmMWq7WLkKIo23hsrpR3x084U= +github.com/moby/patternmatcher v0.6.1/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc= +github.com/moby/sys/sequential v0.7.0 h1:ASQNGNROJSuOO6LL6bPHbKvuZu6NU8P4ldPWk31zj/8= +github.com/moby/sys/sequential v0.7.0/go.mod h1:NfSTAp6V3fw4tmkD62PEcOKeZKquXT8VKCkf7aVR79o= +github.com/moby/sys/user v0.4.0 h1:jhcMKit7SA80hivmFJcbB1vqmw//wU61Zdui2eQXuMs= +github.com/moby/sys/user v0.4.0/go.mod h1:bG+tYYYJgaMtRKgEmuueC0hJEAZWwtIbZTB+85uoHjs= +github.com/moby/sys/userns v0.1.0 h1:tVLXkFOxVu9A64/yh59slHVv9ahO9UIev4JZusOLG/g= +github.com/moby/sys/userns v0.1.0/go.mod h1:IHUYgu/kao6N8YZlp9Cf444ySSvCmDlmzUcYfDHOl28= +github.com/moby/term v0.5.2 h1:6qk3FJAFDs6i/q3W/pQ97SX192qKfZgGjCQqfCJkgzQ= +github.com/moby/term v0.5.2/go.mod h1:d3djjFCrjnB+fl8NJux+EJzu0msscUP+f8it8hPkFLc= +github.com/mtibben/percent v0.2.1 h1:5gssi8Nqo8QU/r2pynCm+hBQHpkB/uNK7BJCFogWdzs= +github.com/mtibben/percent v0.2.1/go.mod h1:KG9uO+SZkUp+VkRHsCdYQV3XSZrrSpR3O9ibNBTZrns= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= +github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= +github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU= +github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= +github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= +github.com/shirou/gopsutil/v4 v4.26.6 h1:Mzr/npDtQC/xpeEuQKHZt8Zo9CmPvhTj8nkR8w5TLDs= +github.com/shirou/gopsutil/v4 v4.26.6/go.mod h1:LZ6ewCSkBqUpvSOf+LsTGnRinC6iaNUNMGBtDkJBaLQ= +github.com/sirupsen/logrus v1.9.4 h1:TsZE7l11zFCLZnZ+teH4Umoq5BhEIfIzfRDZ1Uzql2w= +github.com/sirupsen/logrus v1.9.4/go.mod h1:ftWc9WdOfJ0a92nsE2jF5u5ZwH8Bv2zdeOC42RjbV2g= +github.com/snowflakedb/gosnowflake/v2 v2.3.0 h1:uYELDDyjm2A/hrv+Lfc/ZPbohgcN5k8o72V1qtlCoVc= +github.com/snowflakedb/gosnowflake/v2 v2.3.0/go.mod h1:poUMAfesYFh8piXMqKVPW304tekACzv1cfskxMdpikY= +github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4= +github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/testcontainers/testcontainers-go v0.44.0 h1:/Fwh6HY1mIikhnm9e7HwoxGycx0lzRAE0f5VQpjFxzI= +github.com/testcontainers/testcontainers-go v0.44.0/go.mod h1:IcnwQrYTO86xHXu5bvMaBH7ATlbS3Qn1M1QWW3c66rE= +github.com/tklauser/go-sysconf v0.4.0 h1:7H0uAN+7RkwWRaxhYXDLqa5V3LPrJeV8wmD9dRUgPQU= +github.com/tklauser/go-sysconf v0.4.0/go.mod h1:8mTNWyog7H+MpKijp4VmKJAd2bbYQ2zuUwkYRbUArPI= +github.com/tklauser/numcpus v0.12.0 h1:NR85qdvHA9pFse3x3weVZ0r0ST8R6l5RHbZrlRaqob4= +github.com/tklauser/numcpus v0.12.0/go.mod h1:ABHeXzJnr/qqwguhClkZKT1/8VABcYrsyUiUGobwWJg= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= +github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 h1:8tvICD4vSTOOsNrsI4Ljf6C+6UKvpTEH5XY3JMoyPoo= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0/go.mod h1:z9+yiacE0IHRqM4qFfkbt/JYlmYXgss8GY/jXoNuPJI= +go.opentelemetry.io/otel v1.44.0 h1:JjwHmHpA4iZ3wBxluu2fbbE7j4kqlE8jXyAyPXH7HqU= +go.opentelemetry.io/otel v1.44.0/go.mod h1:BMgjTHL9WPRlRjL2oZCBTL4whCGtXch2H4BhOPIAyYc= +go.opentelemetry.io/otel/metric v1.44.0 h1:1w0gILTcHdr3YI+ixLyjemwrVnsMURbTZFrSYCdDdmc= +go.opentelemetry.io/otel/metric v1.44.0/go.mod h1:8O7hanEPBNgEMmybD3s2VBKcgWOCsA6tzHBPODAiquo= +go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= +go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= +go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= +go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= +go.opentelemetry.io/otel/trace v1.44.0 h1:jxF5CsGYCe74MCRx2X4g7WsY/VBKRqqpNvXlX/6gtIk= +go.opentelemetry.io/otel/trace v1.44.0/go.mod h1:oLl1jrMQAVo6v3GAggN+1VH9VIz9iUSvW53sW1Q8PIE= +golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= +golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= +golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0 h1:R84qjqJb5nVJMxqWYb3np9L5ZsaDtB+a39EqjV0JSUM= +golang.org/x/exp v0.0.0-20250408133849-7e4ce0ab07d0/go.mod h1:S9Xr4PYopiDyqSyp5NjCrhFrqg6A5zA2E/iPHPhqnS8= +golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ= +golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= +golang.org/x/net v0.56.0 h1:Rw8j/hFzGvJUZwNBXnAtf5sVDVt+65SK2C7IxCxZt5o= +golang.org/x/net v0.56.0/go.mod h1:D3Ku6r+V6JROoZK144D2XfMHFcMq/0zSfLelVTCFKec= +golang.org/x/oauth2 v0.34.0 h1:hqK/t4AKgbqWkdkcAeI8XLmbK+4m4G5YeQRrmiotGlw= +golang.org/x/oauth2 v0.34.0/go.mod h1:lzm5WQJQwKZ3nwavOZ3IS5Aulzxi68dUSgRHujetwEA= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/telemetry v0.0.0-20260625142307-59b4966ccb57 h1:nwGZBCt+FnXUrGsj5vjzAsEmkcaFvd82BbOjECiFYZc= +golang.org/x/telemetry v0.0.0-20260625142307-59b4966ccb57/go.mod h1:3AWMyWHS+caVoiEXpiq6+tzKA40J4vQT3MYr80ZtQpc= +golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0= +golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w= +golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= +golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= +golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q= +golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= +golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY= +golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= +gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk= +gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools/v3 v3.5.2 h1:7koQfIKdy+I8UTetycgUqXWSDwpgv193Ka+qRsmBY8Q= +gotest.tools/v3 v3.5.2/go.mod h1:LtdLGcnqToBH83WByAAi/wiwSFCArdFIUV/xxN4pcjA= +pgregory.net/rapid v1.2.0 h1:keKAYRcjm+e1F0oAuU5F5+YPAWcyxNNRK2wud503Gnk= +pgregory.net/rapid v1.2.0/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04= diff --git a/metering-api/init/01_platform.sf.sql b/metering-api/init/01_platform.sf.sql new file mode 100644 index 0000000..f6d8c18 --- /dev/null +++ b/metering-api/init/01_platform.sf.sql @@ -0,0 +1,31 @@ +-- Account objects for the metering API: what a Snowflake administrator runs once +-- per environment (in production usually Terraform or a platform repository). +-- +-- The local emulator runs every init/*.sf.sql file at startup. The integration +-- tests run this file over the driver, then apply 02 and 03 to a fresh database +-- per test. Keep it to plain statements, each ending with ';' at the end of a line. + +USE ROLE ACCOUNTADMIN; + +CREATE WAREHOUSE IF NOT EXISTS METERING_WH + WAREHOUSE_SIZE = XSMALL + AUTO_SUSPEND = 60 + AUTO_RESUME = TRUE + INITIALLY_SUSPENDED = TRUE; + +-- The service's role. It owns the METERING database and everything in it. +CREATE ROLE IF NOT EXISTS METERING_APP; +GRANT ROLE METERING_APP TO ROLE SYSADMIN; +GRANT USAGE ON WAREHOUSE METERING_WH TO ROLE METERING_APP; + +CREATE DATABASE IF NOT EXISTS METERING; +GRANT OWNERSHIP ON DATABASE METERING TO ROLE METERING_APP COPY CURRENT GRANTS; + +-- Service user with key-pair auth only. On a real account, register the key with: +-- ALTER USER METERING_SVC SET RSA_PUBLIC_KEY = '<.secrets/rsa_key.pub without the PEM lines>'; +CREATE USER IF NOT EXISTS METERING_SVC + TYPE = SERVICE + DEFAULT_ROLE = METERING_APP + DEFAULT_WAREHOUSE = METERING_WH + COMMENT = 'Metering API service user (key-pair auth)'; +GRANT ROLE METERING_APP TO USER METERING_SVC; diff --git a/metering-api/init/02_schema.sf.sql b/metering-api/init/02_schema.sf.sql new file mode 100644 index 0000000..49e63d7 --- /dev/null +++ b/metering-api/init/02_schema.sf.sql @@ -0,0 +1,74 @@ +-- Tables of the metering API, created as the service role so it owns them. +-- +-- Everything after the USE DATABASE line is database-relative: the integration +-- tests apply this file to their own database (they skip the USE ROLE and +-- USE DATABASE lines and run the rest in a session on that database). + +USE ROLE METERING_APP; +USE DATABASE METERING; + +CREATE SCHEMA IF NOT EXISTS BILLING; +USE SCHEMA BILLING; + +-- Graduated price tiers, one row per tier. A tier's FLAT_FEE_CENTS is charged once +-- the usage reaches it; tier 1 is always reached, so its flat fee is the monthly fee. +-- UP_TO_CALLS is the last call of the tier (NULL for the open-ended top tier). +CREATE TABLE IF NOT EXISTS PRICE_PLANS ( + PLAN_ID VARCHAR(16) NOT NULL, + TIER_NO NUMBER(2, 0) NOT NULL, + UP_TO_CALLS NUMBER(18, 0), + UNIT_PRICE_MICROS NUMBER(18, 0) NOT NULL, + FLAT_FEE_CENTS NUMBER(18, 0) NOT NULL, + PRIMARY KEY (PLAN_ID, TIER_NO) +); + +CREATE TABLE IF NOT EXISTS CUSTOMERS ( + CUSTOMER_ID VARCHAR(32) NOT NULL PRIMARY KEY, + NAME VARCHAR(200) NOT NULL, + PLAN_ID VARCHAR(16) NOT NULL, + CREATED_AT TIMESTAMP_NTZ NOT NULL +); + +-- One row per usage report from the API gateway: QUANTITY calls by one customer in +-- one reporting window. EVENT_ID is the idempotency key; INVOICE_ID is set when the +-- event is billed. +CREATE TABLE IF NOT EXISTS USAGE_EVENTS ( + EVENT_ID VARCHAR(64) NOT NULL PRIMARY KEY, + CUSTOMER_ID VARCHAR(32) NOT NULL, + QUANTITY NUMBER(18, 0) NOT NULL, + OCCURRED_AT TIMESTAMP_NTZ NOT NULL, + RECEIVED_AT TIMESTAMP_NTZ NOT NULL, + INVOICE_ID VARCHAR(64) +); + +-- Landing table for ingest batches: each POST /v1/events writes its rows here under +-- a fresh BATCH_ID with one array-bound INSERT, MERGEs them into USAGE_EVENTS and +-- deletes them again, all in one transaction. +CREATE TRANSIENT TABLE IF NOT EXISTS USAGE_EVENTS_STAGING ( + BATCH_ID VARCHAR(36) NOT NULL, + EVENT_ID VARCHAR(64) NOT NULL, + CUSTOMER_ID VARCHAR(32) NOT NULL, + QUANTITY NUMBER(18, 0) NOT NULL, + OCCURRED_AT TIMESTAMP_NTZ NOT NULL +); + +CREATE TABLE IF NOT EXISTS INVOICES ( + INVOICE_ID VARCHAR(64) NOT NULL PRIMARY KEY, + CUSTOMER_ID VARCHAR(32) NOT NULL, + BILLING_MONTH DATE NOT NULL, + PLAN_ID VARCHAR(16) NOT NULL, + TOTAL_CALLS NUMBER(18, 0) NOT NULL, + TOTAL_CENTS NUMBER(18, 0) NOT NULL, + CLOSED_AT TIMESTAMP_NTZ NOT NULL +); + +CREATE TABLE IF NOT EXISTS INVOICE_LINES ( + INVOICE_ID VARCHAR(64) NOT NULL, + LINE_NO NUMBER(4, 0) NOT NULL, + TIER_NO NUMBER(2, 0) NOT NULL, + QUANTITY NUMBER(18, 0) NOT NULL, + UNIT_PRICE_MICROS NUMBER(18, 0) NOT NULL, + FLAT_FEE_CENTS NUMBER(18, 0) NOT NULL, + AMOUNT_CENTS NUMBER(18, 0) NOT NULL, + PRIMARY KEY (INVOICE_ID, LINE_NO) +); diff --git a/metering-api/init/03_price_plans.sf.sql b/metering-api/init/03_price_plans.sf.sql new file mode 100644 index 0000000..db0cf7b --- /dev/null +++ b/metering-api/init/03_price_plans.sf.sql @@ -0,0 +1,32 @@ +-- Reference data: the three public plans, as graduated tiers (idempotent). +-- +-- FREE first 10,000 calls free, then $2.00 per 1,000 +-- PRO $49 per month incl. 100,000 calls, $0.80 per 1,000 up to 1M, then $0.50 +-- ENTERPRISE $499 per month incl. 1M calls, $0.40 per 1,000 up to 10M, then $0.25 +-- +-- Unit prices are in micro-dollars per call ($0.80 per 1,000 calls = 800). + +USE ROLE METERING_APP; +USE DATABASE METERING; +USE SCHEMA BILLING; + +MERGE INTO PRICE_PLANS t +USING ( + SELECT * FROM VALUES + ('FREE', 1, 10000, 0, 0), + ('FREE', 2, NULL, 2000, 0), + ('PRO', 1, 100000, 0, 4900), + ('PRO', 2, 1000000, 800, 0), + ('PRO', 3, NULL, 500, 0), + ('ENTERPRISE', 1, 1000000, 0, 49900), + ('ENTERPRISE', 2, 10000000, 400, 0), + ('ENTERPRISE', 3, NULL, 250, 0) + AS v (PLAN_ID, TIER_NO, UP_TO_CALLS, UNIT_PRICE_MICROS, FLAT_FEE_CENTS) +) s +ON t.PLAN_ID = s.PLAN_ID AND t.TIER_NO = s.TIER_NO +WHEN MATCHED THEN UPDATE SET + UP_TO_CALLS = s.UP_TO_CALLS, + UNIT_PRICE_MICROS = s.UNIT_PRICE_MICROS, + FLAT_FEE_CENTS = s.FLAT_FEE_CENTS +WHEN NOT MATCHED THEN INSERT (PLAN_ID, TIER_NO, UP_TO_CALLS, UNIT_PRICE_MICROS, FLAT_FEE_CENTS) + VALUES (s.PLAN_ID, s.TIER_NO, s.UP_TO_CALLS, s.UNIT_PRICE_MICROS, s.FLAT_FEE_CENTS); diff --git a/metering-api/internal/config/config.go b/metering-api/internal/config/config.go new file mode 100644 index 0000000..c2ba5f6 --- /dev/null +++ b/metering-api/internal/config/config.go @@ -0,0 +1,171 @@ +// Package config reads the service's settings from the environment (12-factor) +// and turns the Snowflake settings into a database/sql connection pool. +// +// The same variables describe the local emulator and a real Snowflake account; +// TEST_TARGET picks which defaults apply. Only the account, host and key differ. +package config + +import ( + "context" + "crypto/rsa" + "crypto/x509" + "database/sql" + "encoding/pem" + "errors" + "fmt" + "os" + "strconv" + "strings" + "time" + + sf "github.com/snowflakedb/gosnowflake/v2" +) + +// Target selects the Snowflake the process talks to. +type Target string + +const ( + // Emulator is the local Snowflake started by `make up` (the default). + Emulator Target = "EMULATOR" + // Cloud is a real Snowflake account. + Cloud Target = "SNOWFLAKE_CLOUD" +) + +const emulatorHost = "snowflake.localhost.localstack.cloud" + +// Snowflake holds the connection settings. PrivateKey is loaded from +// SNOWFLAKE_PRIVATE_KEY_PATH by FromEnv; tests may set it directly. +type Snowflake struct { + Target Target + Account string + Host string + Port int + Protocol string + User string + Role string + Warehouse string + Database string + Schema string + PrivateKey *rsa.PrivateKey +} + +// TargetFromEnv reads TEST_TARGET (EMULATOR or SNOWFLAKE_CLOUD). +func TargetFromEnv() (Target, error) { + switch t := Target(strings.ToUpper(env("TEST_TARGET", string(Emulator)))); t { + case Emulator, Cloud: + return t, nil + default: + return "", fmt.Errorf("TEST_TARGET must be %s or %s, got %q", Emulator, Cloud, t) + } +} + +// SnowflakeFromEnv reads the SNOWFLAKE_* variables. On the emulator every value +// has a working default; on a real account the account, user, role, warehouse +// and key path are required. +func SnowflakeFromEnv() (Snowflake, error) { + target, err := TargetFromEnv() + if err != nil { + return Snowflake{}, err + } + c := Snowflake{ + Target: target, + Protocol: "https", + Database: env("SNOWFLAKE_DATABASE", "METERING"), + Schema: env("SNOWFLAKE_SCHEMA", "BILLING"), + Role: env("SNOWFLAKE_ROLE", "METERING_APP"), + Warehouse: env("SNOWFLAKE_WAREHOUSE", "METERING_WH"), + } + keyPath := env("SNOWFLAKE_PRIVATE_KEY_PATH", ".secrets/rsa_key.p8") + switch target { + case Emulator: + c.Account = env("SNOWFLAKE_ACCOUNT", "snowflake") + c.Host = env("SNOWFLAKE_HOST", emulatorHost) + c.User = env("SNOWFLAKE_USER", "METERING_SVC") + if c.Port, err = strconv.Atoi(env("SNOWFLAKE_PORT", "4566")); err != nil { + return Snowflake{}, fmt.Errorf("SNOWFLAKE_PORT: %w", err) + } + case Cloud: + var missing []string + for _, name := range []string{"SNOWFLAKE_ACCOUNT", "SNOWFLAKE_USER", "SNOWFLAKE_ROLE", "SNOWFLAKE_WAREHOUSE", "SNOWFLAKE_PRIVATE_KEY_PATH"} { + if os.Getenv(name) == "" { + missing = append(missing, name) + } + } + if len(missing) > 0 { + return Snowflake{}, fmt.Errorf("TEST_TARGET=%s needs %s", Cloud, strings.Join(missing, ", ")) + } + c.Account = os.Getenv("SNOWFLAKE_ACCOUNT") + c.Host = env("SNOWFLAKE_HOST", c.Account+".snowflakecomputing.com") + c.User = os.Getenv("SNOWFLAKE_USER") + c.Port = 443 + } + if c.PrivateKey, err = ReadPrivateKey(keyPath); err != nil { + return Snowflake{}, err + } + return c, nil +} + +// ReadPrivateKey loads an unencrypted PKCS#8 RSA key in PEM form, as written by +// `openssl genpkey` (see `make keys`). +func ReadPrivateKey(path string) (*rsa.PrivateKey, error) { + data, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("read private key (run `make keys`?): %w", err) + } + block, _ := pem.Decode(data) + if block == nil || block.Type != "PRIVATE KEY" { + return nil, fmt.Errorf("%s: expected a PEM \"PRIVATE KEY\" (PKCS#8) block", path) + } + key, err := x509.ParsePKCS8PrivateKey(block.Bytes) + if err != nil { + return nil, fmt.Errorf("%s: %w", path, err) + } + rsaKey, ok := key.(*rsa.PrivateKey) + if !ok { + return nil, fmt.Errorf("%s: key is %T, want RSA", path, key) + } + return rsaKey, nil +} + +// DriverConfig is the gosnowflake configuration for these settings. +func (c Snowflake) DriverConfig() sf.Config { + return sf.Config{ + Account: c.Account, + User: c.User, + Host: c.Host, + Port: c.Port, + Protocol: c.Protocol, + Authenticator: sf.AuthTypeJwt, + PrivateKey: c.PrivateKey, + Role: c.Role, + Warehouse: c.Warehouse, + Database: c.Database, + Schema: c.Schema, + Application: "metering-api", + } +} + +// Open returns a connection pool and checks that Snowflake answers. +func Open(ctx context.Context, c Snowflake) (*sql.DB, error) { + if c.PrivateKey == nil { + return nil, errors.New("snowflake: no private key configured") + } + db := sql.OpenDB(sf.NewConnector(sf.SnowflakeDriver{}, c.DriverConfig())) + db.SetMaxOpenConns(16) + db.SetMaxIdleConns(4) + db.SetConnMaxIdleTime(5 * time.Minute) + pingCtx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + if err := db.PingContext(pingCtx); err != nil { + _ = db.Close() + return nil, fmt.Errorf("connect to Snowflake at %s:%d as %s: %w", c.Host, c.Port, c.User, err) + } + return db, nil +} + +func env(name, fallback string) string { + if v := os.Getenv(name); v != "" { + return v + } + return fallback +} diff --git a/metering-api/internal/config/config_test.go b/metering-api/internal/config/config_test.go new file mode 100644 index 0000000..8506101 --- /dev/null +++ b/metering-api/internal/config/config_test.go @@ -0,0 +1,127 @@ +package config + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/pem" + "os" + "path/filepath" + "strings" + "testing" +) + +var allVars = []string{ + "TEST_TARGET", "SNOWFLAKE_ACCOUNT", "SNOWFLAKE_HOST", "SNOWFLAKE_PORT", "SNOWFLAKE_USER", + "SNOWFLAKE_ROLE", "SNOWFLAKE_WAREHOUSE", "SNOWFLAKE_DATABASE", "SNOWFLAKE_SCHEMA", "SNOWFLAKE_PRIVATE_KEY_PATH", +} + +// cleanEnv clears every variable the package reads, for the test's duration. +func cleanEnv(t *testing.T) { + t.Helper() + for _, name := range allVars { + t.Setenv(name, "") + } +} + +func writeKey(t *testing.T) string { + t.Helper() + key, err := rsa.GenerateKey(rand.Reader, 2048) + if err != nil { + t.Fatal(err) + } + der, err := x509.MarshalPKCS8PrivateKey(key) + if err != nil { + t.Fatal(err) + } + path := filepath.Join(t.TempDir(), "rsa_key.p8") + if err := os.WriteFile(path, pem.EncodeToMemory(&pem.Block{Type: "PRIVATE KEY", Bytes: der}), 0o600); err != nil { + t.Fatal(err) + } + return path +} + +func TestEmulatorDefaults(t *testing.T) { + cleanEnv(t) + t.Setenv("SNOWFLAKE_PRIVATE_KEY_PATH", writeKey(t)) + t.Setenv("SNOWFLAKE_PORT", "34566") + c, err := SnowflakeFromEnv() + if err != nil { + t.Fatal(err) + } + got := []any{c.Target, c.Account, c.Host, c.Port, c.Protocol, c.User, c.Role, c.Warehouse, c.Database, c.Schema} + want := []any{Emulator, "snowflake", "snowflake.localhost.localstack.cloud", 34566, "https", "METERING_SVC", + "METERING_APP", "METERING_WH", "METERING", "BILLING"} + for i := range want { + if got[i] != want[i] { + t.Errorf("field %d = %v, want %v", i, got[i], want[i]) + } + } + d := c.DriverConfig() + if d.PrivateKey == nil || d.Authenticator.String() != "SNOWFLAKE_JWT" { + t.Errorf("driver config is not key-pair auth: %v", d.Authenticator) + } +} + +func TestCloudNeedsAccountSettings(t *testing.T) { + cleanEnv(t) + t.Setenv("TEST_TARGET", "snowflake_cloud") + t.Setenv("SNOWFLAKE_USER", "ci_user") + _, err := SnowflakeFromEnv() + if err == nil { + t.Fatal("accepted a cloud target without an account") + } + for _, name := range []string{"SNOWFLAKE_ACCOUNT", "SNOWFLAKE_ROLE", "SNOWFLAKE_WAREHOUSE", "SNOWFLAKE_PRIVATE_KEY_PATH"} { + if !strings.Contains(err.Error(), name) { + t.Errorf("error %q does not name %s", err, name) + } + } + if strings.Contains(err.Error(), "SNOWFLAKE_USER") { + t.Errorf("error %q names a variable that is set", err) + } +} + +func TestCloudDerivesHostAndIgnoresEmulatorPort(t *testing.T) { + cleanEnv(t) + t.Setenv("TEST_TARGET", "SNOWFLAKE_CLOUD") + t.Setenv("SNOWFLAKE_ACCOUNT", "myorg-prod") + t.Setenv("SNOWFLAKE_USER", "ci_user") + t.Setenv("SNOWFLAKE_ROLE", "CI_ROLE") + t.Setenv("SNOWFLAKE_WAREHOUSE", "CI_WH") + t.Setenv("SNOWFLAKE_PORT", "34566") + t.Setenv("SNOWFLAKE_PRIVATE_KEY_PATH", writeKey(t)) + c, err := SnowflakeFromEnv() + if err != nil { + t.Fatal(err) + } + if c.Host != "myorg-prod.snowflakecomputing.com" || c.Port != 443 || c.User != "ci_user" || c.Role != "CI_ROLE" { + t.Errorf("cloud config = %+v", c) + } +} + +func TestInvalidSettings(t *testing.T) { + cleanEnv(t) + t.Setenv("TEST_TARGET", "staging") + if _, err := SnowflakeFromEnv(); err == nil { + t.Error("accepted TEST_TARGET=staging") + } + t.Setenv("TEST_TARGET", "") + t.Setenv("SNOWFLAKE_PORT", "https") + if _, err := SnowflakeFromEnv(); err == nil { + t.Error("accepted a non-numeric port") + } +} + +func TestReadPrivateKeyErrors(t *testing.T) { + dir := t.TempDir() + notPEM := filepath.Join(dir, "plain.txt") + pkcs1 := filepath.Join(dir, "pkcs1.pem") + _ = os.WriteFile(notPEM, []byte("not a key"), 0o600) + key, _ := rsa.GenerateKey(rand.Reader, 2048) + _ = os.WriteFile(pkcs1, pem.EncodeToMemory(&pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(key)}), 0o600) + for _, path := range []string{filepath.Join(dir, "missing.p8"), notPEM, pkcs1} { + if _, err := ReadPrivateKey(path); err == nil { + t.Errorf("ReadPrivateKey(%s) accepted", filepath.Base(path)) + } + } +} diff --git a/metering-api/internal/metering/dates.go b/metering-api/internal/metering/dates.go new file mode 100644 index 0000000..7dd1e50 --- /dev/null +++ b/metering-api/internal/metering/dates.go @@ -0,0 +1,87 @@ +package metering + +import ( + "fmt" + "time" +) + +// Date is a calendar day in UTC, written as 2006-01-02. +type Date struct{ t time.Time } + +// ParseDate parses YYYY-MM-DD. +func ParseDate(s string) (Date, error) { + t, err := time.Parse(time.DateOnly, s) + if err != nil { + return Date{}, fmt.Errorf("%q is not a date (want YYYY-MM-DD)", s) + } + return Date{t}, nil +} + +// DateOf returns the UTC calendar day of t. +func DateOf(t time.Time) Date { + y, m, d := t.UTC().Date() + return Date{time.Date(y, m, d, 0, 0, 0, 0, time.UTC)} +} + +// Time is midnight UTC at the start of the day. +func (d Date) Time() time.Time { return d.t } + +// AddDays returns the date n days later. +func (d Date) AddDays(n int) Date { return Date{d.t.AddDate(0, 0, n)} } + +func (d Date) String() string { return d.t.Format(time.DateOnly) } + +// MarshalText implements encoding.TextMarshaler. +func (d Date) MarshalText() ([]byte, error) { return []byte(d.String()), nil } + +// UnmarshalText implements encoding.TextUnmarshaler. +func (d *Date) UnmarshalText(b []byte) error { + parsed, err := ParseDate(string(b)) + if err != nil { + return err + } + *d = parsed + return nil +} + +// Month is a billing month, written as 2006-01. +type Month struct{ start time.Time } + +// ParseMonth parses YYYY-MM. +func ParseMonth(s string) (Month, error) { + t, err := time.Parse("2006-01", s) + if err != nil { + return Month{}, fmt.Errorf("%q is not a month (want YYYY-MM)", s) + } + return Month{t}, nil +} + +// MonthOf returns the month containing t (in UTC). +func MonthOf(t time.Time) Month { + y, m, _ := t.UTC().Date() + return Month{time.Date(y, m, 1, 0, 0, 0, 0, time.UTC)} +} + +// Start is the first instant of the month. +func (m Month) Start() time.Time { return m.start } + +// End is the first instant of the next month. +func (m Month) End() time.Time { return m.start.AddDate(0, 1, 0) } + +// Prev is the month before. +func (m Month) Prev() Month { return Month{m.start.AddDate(0, -1, 0)} } + +func (m Month) String() string { return m.start.Format("2006-01") } + +// MarshalText implements encoding.TextMarshaler. +func (m Month) MarshalText() ([]byte, error) { return []byte(m.String()), nil } + +// UnmarshalText implements encoding.TextUnmarshaler. +func (m *Month) UnmarshalText(b []byte) error { + parsed, err := ParseMonth(string(b)) + if err != nil { + return err + } + *m = parsed + return nil +} diff --git a/metering-api/internal/metering/events.go b/metering-api/internal/metering/events.go new file mode 100644 index 0000000..85d32c3 --- /dev/null +++ b/metering-api/internal/metering/events.go @@ -0,0 +1,162 @@ +package metering + +import ( + "context" + "crypto/rand" + "database/sql" + "fmt" + "time" + + sf "github.com/snowflakedb/gosnowflake/v2" +) + +// Event is one usage report from the API gateway: Quantity calls made by a +// customer in the window ending at OccurredAt. EventID is the idempotency key: +// the gateway retries with the same id, and a retried event is never counted twice. +type Event struct { + EventID string `json:"event_id"` + CustomerID string `json:"customer_id"` + Quantity int64 `json:"quantity"` + OccurredAt time.Time `json:"occurred_at"` +} + +// IngestResult reports how many events of a batch were new. +type IngestResult struct { + Received int `json:"received"` + Inserted int `json:"inserted"` + Duplicates int `json:"duplicates"` +} + +// MaxBatch is the largest ingest batch. It keeps one array-bound INSERT well +// below the driver's stage-binding threshold. +const MaxBatch = 1000 + +const mergeStagedEvents = `MERGE INTO USAGE_EVENTS t +USING ( + SELECT EVENT_ID, CUSTOMER_ID, QUANTITY, OCCURRED_AT + FROM USAGE_EVENTS_STAGING + WHERE BATCH_ID = ? + QUALIFY ROW_NUMBER() OVER (PARTITION BY EVENT_ID ORDER BY OCCURRED_AT) = 1 +) s +ON t.EVENT_ID = s.EVENT_ID +WHEN NOT MATCHED THEN INSERT (EVENT_ID, CUSTOMER_ID, QUANTITY, OCCURRED_AT, RECEIVED_AT) + VALUES (s.EVENT_ID, s.CUSTOMER_ID, s.QUANTITY, s.OCCURRED_AT, SYSDATE())` + +// IngestEvents stores a batch idempotently, in one transaction: +// +// 1. one INSERT with array binds lands the batch in USAGE_EVENTS_STAGING; +// 2. the batch is rejected if it names an unknown customer; +// 3. a MERGE keyed by EVENT_ID inserts the events not stored yet; +// 4. the staging rows are deleted. +// +// Events must already be validated (see ValidateEvents). +func (s *Service) IngestEvents(ctx context.Context, events []Event) (IngestResult, error) { + result := IngestResult{Received: len(events)} + if len(events) == 0 { + return result, nil + } + n := len(events) + batch := make([]string, n) + ids := make([]string, n) + customers := make([]string, n) + quantities := make([]int64, n) + occurred := make([]time.Time, n) + batchID := rand.Text() + for i, e := range events { + batch[i], ids[i], customers[i], quantities[i], occurred[i] = batchID, e.EventID, e.CustomerID, e.Quantity, e.OccurredAt.UTC() + } + binds, err := arrays(batch, ids, customers, quantities) + if err != nil { + return result, err + } + occurredBind, err := sf.Array(occurred, sf.TimestampNTZType) + if err != nil { + return result, err + } + binds = append(binds, occurredBind) + + err = s.inTx(ctx, func(tx *sql.Tx) error { + if _, err := tx.ExecContext(ctx, `INSERT INTO USAGE_EVENTS_STAGING + (BATCH_ID, EVENT_ID, CUSTOMER_ID, QUANTITY, OCCURRED_AT) VALUES (?, ?, ?, ?, ?)`, binds...); err != nil { + return fmt.Errorf("stage events: %w", err) + } + unknown, err := unknownCustomers(ctx, tx, batchID) + if err != nil { + return err + } + if len(unknown) > 0 { + return &UnknownCustomersError{IDs: unknown} + } + res, err := tx.ExecContext(ctx, mergeStagedEvents, batchID) + if err != nil { + return fmt.Errorf("merge events: %w", err) + } + inserted, err := res.RowsAffected() + if err != nil { + return err + } + result.Inserted = int(inserted) + if _, err := tx.ExecContext(ctx, `DELETE FROM USAGE_EVENTS_STAGING WHERE BATCH_ID = ?`, batchID); err != nil { + return fmt.Errorf("clear staging: %w", err) + } + return nil + }) + if err != nil { + return IngestResult{}, err + } + result.Duplicates = result.Received - result.Inserted + return result, nil +} + +func unknownCustomers(ctx context.Context, tx *sql.Tx, batchID string) ([]string, error) { + rows, err := tx.QueryContext(ctx, `SELECT DISTINCT s.CUSTOMER_ID +FROM USAGE_EVENTS_STAGING s +LEFT JOIN CUSTOMERS c ON c.CUSTOMER_ID = s.CUSTOMER_ID +WHERE s.BATCH_ID = ? AND c.CUSTOMER_ID IS NULL +ORDER BY 1`, batchID) + if err != nil { + return nil, fmt.Errorf("check customers: %w", err) + } + defer rows.Close() + var ids []string + for rows.Next() { + var id string + if err := rows.Scan(&id); err != nil { + return nil, err + } + ids = append(ids, id) + } + return ids, rows.Err() +} + +// arrays wraps each column slice for gosnowflake array binding. +func arrays(columns ...any) ([]any, error) { + binds := make([]any, len(columns)) + for i, c := range columns { + b, err := sf.Array(c) + if err != nil { + return nil, err + } + binds[i] = b + } + return binds, nil +} + +// inTx runs fn in a transaction, committing if it returns nil and rolling back +// otherwise. +func (s *Service) inTx(ctx context.Context, fn func(*sql.Tx) error) error { + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return fmt.Errorf("begin: %w", err) + } + if err := fn(tx); err != nil { + if rbErr := tx.Rollback(); rbErr != nil { + s.log.ErrorContext(ctx, "rollback failed", "err", rbErr) + } + return err + } + if err := tx.Commit(); err != nil { + return fmt.Errorf("commit: %w", err) + } + return nil +} diff --git a/metering-api/internal/metering/export_integration_test.go b/metering-api/internal/metering/export_integration_test.go new file mode 100644 index 0000000..446228b --- /dev/null +++ b/metering-api/internal/metering/export_integration_test.go @@ -0,0 +1,14 @@ +//go:build integration + +package metering + +import ( + "context" + "database/sql" +) + +// SetBeforeMarkInvoiced installs a hook that runs inside the invoice-close +// transaction between writing the invoices and marking usage invoiced. +func SetBeforeMarkInvoiced(s *Service, fn func(context.Context, *sql.Tx) error) { + s.beforeMarkInvoiced = fn +} diff --git a/metering-api/internal/metering/harness_integration_test.go b/metering-api/internal/metering/harness_integration_test.go new file mode 100644 index 0000000..8f3f1b4 --- /dev/null +++ b/metering-api/internal/metering/harness_integration_test.go @@ -0,0 +1,344 @@ +//go:build integration + +package metering_test + +import ( + "bytes" + "context" + "crypto/rand" + "crypto/rsa" + "crypto/tls" + "database/sql" + "encoding/json" + "fmt" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/testcontainers/testcontainers-go" + "github.com/testcontainers/testcontainers-go/wait" + + "github.com/localstack-samples/localstack-snowflake-samples/metering-api/internal/config" + "github.com/localstack-samples/localstack-snowflake-samples/metering-api/internal/metering" + "github.com/localstack-samples/localstack-snowflake-samples/metering-api/internal/seed" +) + +// The integration suite runs against one of two targets, chosen by TEST_TARGET: +// +// - EMULATOR (default): a fresh localstack/snowflake-next container started +// with testcontainers-go; init/01_platform.sf.sql provisions it. +// - SNOWFLAKE_CLOUD: the account in the SNOWFLAKE_* variables. +// +// Either way every test gets its own database, built from init/02 and init/03, +// and its own API instance on httptest. Nothing else differs between targets. + +const initDir = "../../init" + +var suite struct { + app config.Snowflake // the service's connection settings, minus the database + admin *sql.DB // creates and drops the per-test databases + // grantTo is the role that must own each test database, when the admin + // role is a different one. + grantTo string +} + +func TestMain(m *testing.M) { + code, err := runSuite(m) + if err != nil { + fmt.Fprintln(os.Stderr, "integration suite:", err) + code = 1 + } + os.Exit(code) +} + +func runSuite(m *testing.M) (int, error) { + ctx := context.Background() + target, err := config.TargetFromEnv() + if err != nil { + return 1, err + } + var stop func() + switch target { + case config.Emulator: + stop, err = startEmulator(ctx) + case config.Cloud: + stop, err = useCloud(ctx) + } + if err != nil { + return 1, err + } + defer stop() + return m.Run(), nil +} + +// startEmulator starts the emulator container, waits until it serves HTTPS on +// the mapped port (checked from the host), and provisions the account. +func startEmulator(ctx context.Context) (func(), error) { + token := os.Getenv("LOCALSTACK_AUTH_TOKEN") + if token == "" { + return nil, fmt.Errorf("LOCALSTACK_AUTH_TOKEN is not set (export it, or run `make test`, which reads ~/.localstack/default.env)") + } + image := os.Getenv("SNOWFLAKE_IMAGE") + if image == "" { + image = "localstack/snowflake-next:latest" + } + const host = "snowflake.localhost.localstack.cloud" // resolves to 127.0.0.1 + ctr, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{ + ContainerRequest: testcontainers.ContainerRequest{ + Image: image, + ExposedPorts: []string{"4566/tcp"}, + Env: map[string]string{"LOCALSTACK_AUTH_TOKEN": token}, + WaitingFor: wait.ForHTTP("/"). + WithPort("4566/tcp"). + WithTLS(true, &tls.Config{ServerName: host}). + WithStatusCodeMatcher(func(status int) bool { return status < 500 }). + WithStartupTimeout(3 * time.Minute), + }, + Started: true, + }) + if err != nil { + return nil, fmt.Errorf("start emulator: %w", err) + } + stop := func() { _ = testcontainers.TerminateContainer(ctr) } + port, err := ctr.MappedPort(ctx, "4566/tcp") + if err != nil { + stop() + return nil, err + } + key, err := rsa.GenerateKey(rand.Reader, 2048) // throwaway key for this run + if err != nil { + stop() + return nil, err + } + suite.app = config.Snowflake{ + Target: config.Emulator, Account: "snowflake", Host: host, Port: int(port.Num()), Protocol: "https", + User: "METERING_SVC", Role: "METERING_APP", Warehouse: "METERING_WH", PrivateKey: key, + } + // The admin session names no warehouse: 01_platform creates it. + adminCfg := suite.app + adminCfg.Role, adminCfg.Warehouse = "ACCOUNTADMIN", "" + if suite.admin, err = openWhenReady(ctx, adminCfg); err != nil { + stop() + return nil, err + } + if err := applyScript(ctx, suite.admin, filepath.Join(initDir, "01_platform.sf.sql"), false); err != nil { + stop() + return nil, fmt.Errorf("provision emulator: %w", err) + } + suite.grantTo = "METERING_APP" + return func() { _ = suite.admin.Close(); stop() }, nil +} + +// useCloud connects to a real account. The role needs CREATE DATABASE. +func useCloud(ctx context.Context) (func(), error) { + cfg, err := config.SnowflakeFromEnv() + if err != nil { + return nil, err + } + cfg.Database, cfg.Schema = "", "" + suite.app = cfg + if suite.admin, err = config.Open(ctx, cfg); err != nil { + return nil, err + } + return func() { _ = suite.admin.Close() }, nil +} + +// openWhenReady connects, retrying until the target answers SELECT 1. +func openWhenReady(ctx context.Context, cfg config.Snowflake) (*sql.DB, error) { + deadline := time.Now().Add(90 * time.Second) + for { + db, err := config.Open(ctx, cfg) + if err == nil { + return db, nil + } + if time.Now().After(deadline) { + return nil, fmt.Errorf("no SQL answer within 90s: %w", err) + } + time.Sleep(2 * time.Second) + } +} + +// applyScript runs an init/*.sf.sql file statement by statement on one session. +// With databaseRelative set it skips the file's USE ROLE and USE DATABASE lines, +// so the objects land in the session's current database. +func applyScript(ctx context.Context, db *sql.DB, path string, databaseRelative bool) error { + raw, err := os.ReadFile(path) + if err != nil { + return err + } + var lines []string + for _, line := range strings.Split(string(raw), "\n") { + if !strings.HasPrefix(strings.TrimSpace(line), "--") { + lines = append(lines, line) + } + } + conn, err := db.Conn(ctx) + if err != nil { + return err + } + defer conn.Close() + for _, stmt := range strings.SplitAfter(strings.Join(lines, "\n"), ";\n") { + stmt = strings.TrimSuffix(strings.TrimSpace(stmt), ";") + upper := strings.ToUpper(stmt) + if stmt == "" || databaseRelative && (strings.HasPrefix(upper, "USE ROLE") || strings.HasPrefix(upper, "USE DATABASE")) { + continue + } + if _, err := conn.ExecContext(ctx, stmt); err != nil { + return fmt.Errorf("%s: %w\n%s", filepath.Base(path), err, stmt) + } + } + return nil +} + +// env is one test's database, service and API. +type env struct { + t *testing.T + ctx context.Context + db *sql.DB + svc *metering.Service + srv *httptest.Server +} + +// newEnv creates a database for the test, applies the schema and reference +// data, and starts the API on it. Everything is dropped when the test ends. +func newEnv(t *testing.T) *env { + t.Helper() + ctx := context.Background() + name := "METERING_IT_" + strings.ToUpper(rand.Text()[:12]) + if _, err := suite.admin.ExecContext(ctx, "CREATE DATABASE "+name); err != nil { + t.Fatalf("create database: %v", err) + } + t.Cleanup(func() { + if _, err := suite.admin.ExecContext(context.Background(), "DROP DATABASE IF EXISTS "+name); err != nil { + t.Logf("drop database %s: %v", name, err) + } + }) + if suite.grantTo != "" { + if _, err := suite.admin.ExecContext(ctx, "GRANT OWNERSHIP ON DATABASE "+name+" TO ROLE "+suite.grantTo+" COPY CURRENT GRANTS"); err != nil { + t.Fatalf("grant database: %v", err) + } + } + + cfg := suite.app + cfg.Database = name + ddl, err := config.Open(ctx, cfg) + if err != nil { + t.Fatal(err) + } + for _, file := range []string{"02_schema.sf.sql", "03_price_plans.sf.sql"} { + if err := applyScript(ctx, ddl, filepath.Join(initDir, file), true); err != nil { + _ = ddl.Close() + t.Fatal(err) + } + } + _ = ddl.Close() + + cfg.Schema = "BILLING" + db, err := config.Open(ctx, cfg) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = db.Close() }) + log := slog.New(slog.NewTextHandler(testWriter{t}, &slog.HandlerOptions{Level: slog.LevelWarn})) + svc := metering.NewService(db, log) + srv := httptest.NewServer(metering.NewHandler(svc, log)) + t.Cleanup(srv.Close) + return &env{t: t, ctx: metering.WithQueryTag(ctx, "integration-test"), db: db, svc: svc, srv: srv} +} + +type testWriter struct{ t *testing.T } + +func (w testWriter) Write(p []byte) (int, error) { + w.t.Log(strings.TrimSpace(string(p))) + return len(p), nil +} + +// seed loads a generated dataset: customers directly, events through the API. +func (e *env) seed(o seed.Options) seed.Dataset { + e.t.Helper() + d := seed.Generate(o) + if err := e.svc.UpsertCustomers(e.ctx, d.Customers); err != nil { + e.t.Fatal(err) + } + res := e.ingest(d.Events, 400) + if res.Inserted != len(d.Events) || res.Duplicates != 0 { + e.t.Fatalf("seeding %d events: %+v", len(d.Events), res) + } + return d +} + +// ingest posts events in batches and sums the results. +func (e *env) ingest(events []metering.Event, batch int) metering.IngestResult { + e.t.Helper() + var total metering.IngestResult + for i := 0; i < len(events); i += batch { + var res metering.IngestResult + e.call("POST", "/v1/events", map[string]any{"events": events[i:min(i+batch, len(events))]}, http.StatusOK, &res) + total.Received += res.Received + total.Inserted += res.Inserted + total.Duplicates += res.Duplicates + } + return total +} + +// call sends a JSON request, checks the status and decodes the response into out. +func (e *env) call(method, path string, body any, wantStatus int, out any) http.Header { + e.t.Helper() + resp, data := e.do(method, path, body) + if resp.StatusCode != wantStatus { + e.t.Fatalf("%s %s: status %d, want %d; body %s", method, path, resp.StatusCode, wantStatus, data) + } + if out != nil { + if err := json.Unmarshal(data, out); err != nil { + e.t.Fatalf("%s %s: decode %s: %v", method, path, data, err) + } + } + return resp.Header +} + +func (e *env) do(method, path string, body any) (*http.Response, []byte) { + e.t.Helper() + var reader io.Reader + switch b := body.(type) { + case nil: + case string: + reader = strings.NewReader(b) + default: + raw, err := json.Marshal(b) + if err != nil { + e.t.Fatal(err) + } + reader = bytes.NewReader(raw) + } + req, err := http.NewRequest(method, e.srv.URL+path, reader) + if err != nil { + e.t.Fatal(err) + } + req.Header.Set("Content-Type", "application/json") + resp, err := e.srv.Client().Do(req) + if err != nil { + e.t.Fatal(err) + } + defer resp.Body.Close() + data, err := io.ReadAll(resp.Body) + if err != nil { + e.t.Fatal(err) + } + return resp, data +} + +// scalar runs a query that returns one integer. +func (e *env) scalar(query string, args ...any) int64 { + e.t.Helper() + var n sql.NullInt64 + if err := e.db.QueryRowContext(e.ctx, query, args...).Scan(&n); err != nil { + e.t.Fatalf("%s: %v", query, err) + } + return n.Int64 +} diff --git a/metering-api/internal/metering/http.go b/metering-api/internal/metering/http.go new file mode 100644 index 0000000..03c4f0e --- /dev/null +++ b/metering-api/internal/metering/http.go @@ -0,0 +1,265 @@ +package metering + +import ( + "context" + "encoding/csv" + "errors" + "fmt" + "log/slog" + "net/http" + "strconv" + "time" +) + +// maxBodyBytes caps an ingest request body. +const maxBodyBytes = 1 << 20 + +// CSVHeader is the header row of the invoice export. +var CSVHeader = []string{ + "invoice_id", "customer_id", "plan_id", "billing_month", "line_no", "tier_no", + "quantity", "unit_price_micros", "flat_fee_cents", "amount_cents", +} + +type api struct { + svc *Service + log *slog.Logger +} + +// NewHandler returns the HTTP API. Every route runs its Snowflake statements +// under its own query tag. +func NewHandler(svc *Service, log *slog.Logger) http.Handler { + a := &api{svc: svc, log: log} + mux := http.NewServeMux() + route := func(pattern, tag string, h http.HandlerFunc) { + mux.Handle(pattern, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + h(w, r.WithContext(WithQueryTag(r.Context(), tag))) + })) + } + route("GET /healthz", "healthz", a.health) + route("POST /v1/events", "events.ingest", a.ingestEvents) + route("GET /v1/customers/{id}/usage", "customers.usage", a.customerUsage) + route("POST /v1/invoices/{month}/close", "invoices.close", a.closeMonth) + route("GET /v1/invoices/{month}/export.csv", "invoices.export", a.exportInvoices) + route("POST /v1/rollups/{month}", "rollups.start", a.startRollup) + route("GET /v1/rollups/{queryId}", "rollups.get", a.getRollup) + return a.logRequests(mux) +} + +func (a *api) health(w http.ResponseWriter, r *http.Request) { + ctx, cancel := context.WithTimeout(r.Context(), 5*time.Second) + defer cancel() + if err := a.svc.Ping(ctx); err != nil { + a.log.WarnContext(ctx, "health check failed", "err", err) + writeJSON(w, http.StatusServiceUnavailable, map[string]string{"status": "unavailable"}) + return + } + writeJSON(w, http.StatusOK, map[string]string{"status": "ok"}) +} + +func (a *api) ingestEvents(w http.ResponseWriter, r *http.Request) { + events, err := decodeEvents(http.MaxBytesReader(w, r.Body, maxBodyBytes)) + if err != nil { + writeProblem(w, r, Problem{Type: problemMalformed, Title: "Malformed request body", Status: http.StatusBadRequest, + Detail: err.Error()}) + return + } + if errs := ValidateEvents(events, a.svc.now()); len(errs) > 0 { + writeProblem(w, r, Problem{Type: problemInvalid, Title: "Invalid events", Status: http.StatusUnprocessableEntity, + Detail: fmt.Sprintf("%d invalid field(s); nothing was stored", len(errs)), Errors: errs}) + return + } + res, err := a.svc.IngestEvents(r.Context(), events) + var unknown *UnknownCustomersError + switch { + case errors.As(err, &unknown): + writeProblem(w, r, Problem{Type: problemUnknownCustomer, Title: "Unknown customer", Status: http.StatusUnprocessableEntity, + Detail: "the batch names customers that do not exist; nothing was stored", UnknownCustomers: unknown.IDs}) + case err != nil: + a.internalError(w, r, err) + default: + writeJSON(w, http.StatusOK, res) + } +} + +func (a *api) customerUsage(w http.ResponseWriter, r *http.Request) { + q := r.URL.Query() + from, to, err := parseUsageRange(q.Get("from"), q.Get("to"), a.svc.now()) + if err != nil { + writeProblem(w, r, Problem{Type: problemMalformed, Title: "Invalid date range", Status: http.StatusBadRequest, Detail: err.Error()}) + return + } + id := r.PathValue("id") + usage, err := a.svc.CustomerUsage(r.Context(), id, from, to) + switch { + case errors.Is(err, ErrCustomerNotFound): + writeProblem(w, r, Problem{Type: problemNotFound, Title: "Customer not found", Status: http.StatusNotFound, + Detail: fmt.Sprintf("no customer with id %q", id)}) + case err != nil: + a.internalError(w, r, err) + default: + writeJSON(w, http.StatusOK, usage) + } +} + +func (a *api) closeMonth(w http.ResponseWriter, r *http.Request) { + m, ok := a.month(w, r) + if !ok { + return + } + res, err := a.svc.CloseMonth(r.Context(), m) + switch { + case errors.Is(err, ErrMonthClosed): + writeProblem(w, r, Problem{Type: problemConflict, Title: "Month already closed", Status: http.StatusConflict, + Detail: fmt.Sprintf("invoices for %s were already issued", m)}) + case errors.Is(err, ErrMonthNotOver): + writeProblem(w, r, Problem{Type: problemConflict, Title: "Month not over", Status: http.StatusConflict, + Detail: fmt.Sprintf("%s can be closed from %s", m, m.End().Format(time.DateOnly))}) + case err != nil: + a.internalError(w, r, err) + default: + writeJSON(w, http.StatusCreated, res) + } +} + +// exportInvoices streams a month's invoice lines as CSV. Rows go to the client +// as they arrive from Snowflake; the response is never held in memory. +func (a *api) exportInvoices(w http.ResponseWriter, r *http.Request) { + m, ok := a.month(w, r) + if !ok { + return + } + cw := csv.NewWriter(w) + flusher, _ := w.(http.Flusher) + rows := 0 + err := a.svc.EachInvoiceLine(r.Context(), m, func(l InvoiceLine) error { + if rows == 0 { + w.Header().Set("Content-Type", "text/csv; charset=utf-8") + w.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="invoice-lines-%s.csv"`, m)) + if err := cw.Write(CSVHeader); err != nil { + return err + } + } + rows++ + if err := cw.Write(l.Record()); err != nil { + return err + } + if rows%500 == 0 { + cw.Flush() + if flusher != nil { + flusher.Flush() + } + } + return cw.Error() + }) + switch { + case errors.Is(err, ErrNoInvoices): + writeProblem(w, r, Problem{Type: problemNotFound, Title: "No invoices", Status: http.StatusNotFound, + Detail: fmt.Sprintf("%s has not been closed", m)}) + case err != nil && rows == 0: + a.internalError(w, r, err) + case err != nil: + // Headers are already sent: abort the response so the client sees a + // truncated transfer instead of a short but well-formed file. + a.log.ErrorContext(r.Context(), "export interrupted", "month", m.String(), "rows", rows, "err", err) + panic(http.ErrAbortHandler) + default: + cw.Flush() + } +} + +// Record is the CSV row of the export, in CSVHeader order. +func (l InvoiceLine) Record() []string { + return []string{ + l.InvoiceID, l.CustomerID, l.PlanID, l.Month.String(), + strconv.FormatInt(l.LineNo, 10), strconv.FormatInt(l.TierNo, 10), + strconv.FormatInt(l.Quantity, 10), strconv.FormatInt(l.UnitPriceMicros, 10), + strconv.FormatInt(l.FlatFeeCents, 10), strconv.FormatInt(l.AmountCents, 10), + } +} + +func (a *api) startRollup(w http.ResponseWriter, r *http.Request) { + m, ok := a.month(w, r) + if !ok { + return + } + queryID, err := a.svc.StartRollup(r.Context(), m) + if err != nil { + a.internalError(w, r, err) + return + } + location := "/v1/rollups/" + queryID + w.Header().Set("Location", location) + writeJSON(w, http.StatusAccepted, map[string]string{"query_id": queryID, "status_url": location}) +} + +func (a *api) getRollup(w http.ResponseWriter, r *http.Request) { + id := r.PathValue("queryId") + if !ValidQueryID(id) { + writeProblem(w, r, Problem{Type: problemMalformed, Title: "Invalid query id", Status: http.StatusBadRequest, + Detail: fmt.Sprintf("%q is not a Snowflake query id", id)}) + return + } + rollup, err := a.svc.GetRollup(r.Context(), id) + switch { + case errors.Is(err, ErrQueryNotFound): + writeProblem(w, r, Problem{Type: problemNotFound, Title: "Rollup not found", Status: http.StatusNotFound, + Detail: fmt.Sprintf("no query with id %s", id)}) + case err != nil: + a.internalError(w, r, err) + default: + writeJSON(w, http.StatusOK, rollup) + } +} + +func (a *api) month(w http.ResponseWriter, r *http.Request) (Month, bool) { + m, err := ParseMonth(r.PathValue("month")) + if err != nil { + writeProblem(w, r, Problem{Type: problemMalformed, Title: "Invalid month", Status: http.StatusBadRequest, Detail: err.Error()}) + return Month{}, false + } + return m, true +} + +func (a *api) internalError(w http.ResponseWriter, r *http.Request, err error) { + a.log.ErrorContext(r.Context(), "request failed", "method", r.Method, "path", r.URL.Path, "err", err) + writeProblem(w, r, Problem{Type: problemInternal, Title: "Internal error", Status: http.StatusInternalServerError}) +} + +type statusRecorder struct { + http.ResponseWriter + status int +} + +func (s *statusRecorder) WriteHeader(code int) { + if s.status == 0 { + s.status = code + } + s.ResponseWriter.WriteHeader(code) +} + +func (s *statusRecorder) Write(b []byte) (int, error) { + if s.status == 0 { + s.status = http.StatusOK + } + return s.ResponseWriter.Write(b) +} + +// Unwrap lets http.ResponseController reach the underlying writer. +func (s *statusRecorder) Unwrap() http.ResponseWriter { return s.ResponseWriter } + +// Flush keeps streaming responses streaming through the recorder. +func (s *statusRecorder) Flush() { + if f, ok := s.ResponseWriter.(http.Flusher); ok { + f.Flush() + } +} + +func (a *api) logRequests(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + start := time.Now() + rec := &statusRecorder{ResponseWriter: w} + next.ServeHTTP(rec, r) + a.log.InfoContext(r.Context(), "request", "method", r.Method, "path", r.URL.Path, + "status", rec.status, "duration_ms", time.Since(start).Milliseconds()) + }) +} diff --git a/metering-api/internal/metering/http_test.go b/metering-api/internal/metering/http_test.go new file mode 100644 index 0000000..9b428bc --- /dev/null +++ b/metering-api/internal/metering/http_test.go @@ -0,0 +1,80 @@ +package metering + +import ( + "encoding/json" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +// Requests rejected before any Snowflake call, so the service has no database. +func TestHandlerRejectsBadRequests(t *testing.T) { + svc := NewService(nil, slog.New(slog.NewTextHandler(io.Discard, nil))) + svc.now = func() time.Time { return now } + h := NewHandler(svc, svc.log) + + tests := []struct { + name, method, path, body string + status int + problemType string + }{ + {"truncated JSON", "POST", "/v1/events", `{"events": [`, 400, problemMalformed}, + {"quantity as a string", "POST", "/v1/events", `{"events": [{"event_id": "e1", "customer_id": "c1", "quantity": "10", "occurred_at": "2026-10-01T00:00:00Z"}]}`, 400, problemMalformed}, + {"unknown field", "POST", "/v1/events", `{"events": [], "tenant": "x"}`, 400, problemMalformed}, + {"two JSON documents", "POST", "/v1/events", `{"events": []} {}`, 400, problemMalformed}, + {"empty batch", "POST", "/v1/events", `{"events": []}`, 422, problemInvalid}, + {"invalid event", "POST", "/v1/events", `{"events": [{"event_id": "e1", "customer_id": "c1", "quantity": 0, "occurred_at": "2026-10-01T00:00:00Z"}]}`, 422, problemInvalid}, + {"bad usage range", "GET", "/v1/customers/c1/usage?from=2026-10-05&to=2026-10-01", "", 400, problemMalformed}, + {"bad month on close", "POST", "/v1/invoices/2026-13/close", "", 400, problemMalformed}, + {"current month cannot be closed", "POST", "/v1/invoices/2026-10/close", "", 409, problemConflict}, + {"bad month on export", "GET", "/v1/invoices/sept/export.csv", "", 400, problemMalformed}, + {"bad month on rollup", "POST", "/v1/rollups/2026-9", "", 400, problemMalformed}, + {"bad query id", "GET", "/v1/rollups/not-a-query-id", "", 400, problemMalformed}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + rec := httptest.NewRecorder() + h.ServeHTTP(rec, httptest.NewRequest(tt.method, tt.path, strings.NewReader(tt.body))) + if rec.Code != tt.status { + t.Fatalf("status = %d, want %d; body %s", rec.Code, tt.status, rec.Body) + } + if ct := rec.Header().Get("Content-Type"); ct != "application/problem+json" { + t.Errorf("Content-Type = %q", ct) + } + var p Problem + if err := json.Unmarshal(rec.Body.Bytes(), &p); err != nil { + t.Fatalf("body is not JSON: %v", err) + } + if p.Type != tt.problemType || p.Status != tt.status || p.Instance != strings.SplitN(tt.path, "?", 2)[0] { + t.Errorf("problem = %+v", p) + } + }) + } +} + +func TestHandlerReportsEveryInvalidField(t *testing.T) { + svc := NewService(nil, slog.New(slog.NewTextHandler(io.Discard, nil))) + svc.now = func() time.Time { return now } + body := `{"events": [ + {"event_id": "e1", "customer_id": "c1", "quantity": 5, "occurred_at": "2026-10-01T00:00:00Z"}, + {"event_id": "", "customer_id": "c1", "quantity": -1} + ]}` + rec := httptest.NewRecorder() + NewHandler(svc, svc.log).ServeHTTP(rec, httptest.NewRequest("POST", "/v1/events", strings.NewReader(body))) + var p Problem + if err := json.Unmarshal(rec.Body.Bytes(), &p); err != nil { + t.Fatal(err) + } + var fields []string + for _, e := range p.Errors { + fields = append(fields, e.Field) + } + want := "events[1].event_id events[1].quantity events[1].occurred_at" + if rec.Code != http.StatusUnprocessableEntity || strings.Join(fields, " ") != want { + t.Errorf("status %d, fields %v; want 422 and %s", rec.Code, fields, want) + } +} diff --git a/metering-api/internal/metering/invoices.go b/metering-api/internal/metering/invoices.go new file mode 100644 index 0000000..7aa6ce9 --- /dev/null +++ b/metering-api/internal/metering/invoices.go @@ -0,0 +1,237 @@ +package metering + +import ( + "context" + "database/sql" + "fmt" + "time" + + sf "github.com/snowflakedb/gosnowflake/v2" +) + +// CloseResult summarizes a closed billing month. +type CloseResult struct { + Month Month `json:"month"` + Invoices int `json:"invoices"` + TotalCents int64 `json:"total_cents"` + EventsInvoiced int64 `json:"events_invoiced"` +} + +// InvoiceID is the deterministic id of a customer's invoice for a month. +func InvoiceID(m Month, customerID string) string { + return "inv_" + m.Start().Format("200601") + "_" + customerID +} + +const usageToInvoice = `SELECT c.CUSTOMER_ID, c.PLAN_ID, COALESCE(SUM(e.QUANTITY), 0) AS CALLS +FROM CUSTOMERS c +LEFT JOIN USAGE_EVENTS e + ON e.CUSTOMER_ID = c.CUSTOMER_ID + AND e.OCCURRED_AT >= ? AND e.OCCURRED_AT < ? + AND e.INVOICE_ID IS NULL +WHERE c.CREATED_AT < ? +GROUP BY c.CUSTOMER_ID, c.PLAN_ID +ORDER BY c.CUSTOMER_ID` + +const markUsageInvoiced = `UPDATE USAGE_EVENTS e +SET INVOICE_ID = i.INVOICE_ID +FROM INVOICES i +WHERE i.CUSTOMER_ID = e.CUSTOMER_ID + AND i.BILLING_MONTH = ? + AND e.OCCURRED_AT >= ? AND e.OCCURRED_AT < ? + AND e.INVOICE_ID IS NULL` + +// CloseMonth invoices every customer for a finished month in one transaction: +// it rates each customer's uninvoiced usage against their plan's tiers, inserts +// the invoices and their lines, and marks the usage invoiced. Either all of it +// is committed or none of it is. +func (s *Service) CloseMonth(ctx context.Context, m Month) (CloseResult, error) { + if s.now().Before(m.End()) { + return CloseResult{}, ErrMonthNotOver + } + result := CloseResult{Month: m} + err := s.inTx(ctx, func(tx *sql.Tx) error { + var existing int + if err := tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM INVOICES WHERE BILLING_MONTH = ?`, + sf.DataTypeDate, m.Start()).Scan(&existing); err != nil { + return fmt.Errorf("check month: %w", err) + } + if existing > 0 { + return ErrMonthClosed + } + inv, lines, err := s.rateMonth(ctx, tx, m) + if err != nil { + return err + } + if err := insertInvoices(ctx, tx, inv, lines); err != nil { + return err + } + result.Invoices = len(inv.ids) + for _, c := range inv.totalCents { + result.TotalCents += c + } + if s.beforeMarkInvoiced != nil { + if err := s.beforeMarkInvoiced(ctx, tx); err != nil { + return err + } + } + res, err := tx.ExecContext(ctx, markUsageInvoiced, sf.DataTypeDate, m.Start(), m.Start(), m.End()) + if err != nil { + return fmt.Errorf("mark usage invoiced: %w", err) + } + result.EventsInvoiced, err = res.RowsAffected() + return err + }) + if err != nil { + return CloseResult{}, err + } + return result, nil +} + +// invoiceColumns and lineColumns hold rows column by column, ready for array binding. +type invoiceColumns struct { + ids, customers, plans []string + months, closedAt []time.Time + calls, totalCents []int64 +} + +type lineColumns struct { + invoiceIDs []string + lineNos, tierNos []int64 + quantities, unitPrices, flatFees, amountsInCents []int64 +} + +func (s *Service) rateMonth(ctx context.Context, tx *sql.Tx, m Month) (invoiceColumns, lineColumns, error) { + var inv invoiceColumns + var lines lineColumns + plans, err := loadPlans(ctx, tx) + if err != nil { + return inv, lines, err + } + rows, err := tx.QueryContext(ctx, usageToInvoice, m.Start(), m.End(), m.End()) + if err != nil { + return inv, lines, fmt.Errorf("read usage: %w", err) + } + defer rows.Close() + closedAt := s.now().UTC() + for rows.Next() { + var customerID, planID string + var calls int64 + if err := rows.Scan(&customerID, &planID, &calls); err != nil { + return inv, lines, err + } + plan, ok := plans[planID] + if !ok { + return inv, lines, fmt.Errorf("customer %s: unknown plan %q", customerID, planID) + } + rated, total, err := plan.Rate(calls) + if err != nil { + return inv, lines, fmt.Errorf("customer %s: %w", customerID, err) + } + id := InvoiceID(m, customerID) + inv.ids = append(inv.ids, id) + inv.customers = append(inv.customers, customerID) + inv.plans = append(inv.plans, planID) + inv.months = append(inv.months, m.Start()) + inv.closedAt = append(inv.closedAt, closedAt) + inv.calls = append(inv.calls, calls) + inv.totalCents = append(inv.totalCents, total) + for i, l := range rated { + lines.invoiceIDs = append(lines.invoiceIDs, id) + lines.lineNos = append(lines.lineNos, int64(i+1)) + lines.tierNos = append(lines.tierNos, int64(l.TierNo)) + lines.quantities = append(lines.quantities, l.Quantity) + lines.unitPrices = append(lines.unitPrices, l.UnitPriceMicros) + lines.flatFees = append(lines.flatFees, l.FlatFeeCents) + lines.amountsInCents = append(lines.amountsInCents, l.AmountCents) + } + } + return inv, lines, rows.Err() +} + +func insertInvoices(ctx context.Context, tx *sql.Tx, inv invoiceColumns, lines lineColumns) error { + if len(inv.ids) == 0 { + return nil + } + binds, err := arrays(inv.ids, inv.customers) + if err != nil { + return err + } + months, err := sf.Array(inv.months, sf.DateType) + if err != nil { + return err + } + rest, err := arrays(inv.plans, inv.calls, inv.totalCents) + if err != nil { + return err + } + closed, err := sf.Array(inv.closedAt, sf.TimestampNTZType) + if err != nil { + return err + } + binds = append(append(append(binds, months), rest...), closed) + if _, err := tx.ExecContext(ctx, `INSERT INTO INVOICES + (INVOICE_ID, CUSTOMER_ID, BILLING_MONTH, PLAN_ID, TOTAL_CALLS, TOTAL_CENTS, CLOSED_AT) + VALUES (?, ?, ?, ?, ?, ?, ?)`, binds...); err != nil { + return fmt.Errorf("insert invoices: %w", err) + } + lineBinds, err := arrays(lines.invoiceIDs, lines.lineNos, lines.tierNos, lines.quantities, + lines.unitPrices, lines.flatFees, lines.amountsInCents) + if err != nil { + return err + } + if _, err := tx.ExecContext(ctx, `INSERT INTO INVOICE_LINES + (INVOICE_ID, LINE_NO, TIER_NO, QUANTITY, UNIT_PRICE_MICROS, FLAT_FEE_CENTS, AMOUNT_CENTS) + VALUES (?, ?, ?, ?, ?, ?, ?)`, lineBinds...); err != nil { + return fmt.Errorf("insert invoice lines: %w", err) + } + return nil +} + +// InvoiceLine is one row of the invoice export. +type InvoiceLine struct { + InvoiceID string + CustomerID string + PlanID string + Month Month + LineNo int64 + TierNo int64 + Quantity int64 + UnitPriceMicros int64 + FlatFeeCents int64 + AmountCents int64 +} + +// EachInvoiceLine streams a closed month's invoice lines, ordered by invoice and +// line number, to fn one row at a time. It returns ErrNoInvoices, without +// calling fn, if the month has no invoices. +func (s *Service) EachInvoiceLine(ctx context.Context, m Month, fn func(InvoiceLine) error) error { + rows, err := s.db.QueryContext(ctx, `SELECT i.INVOICE_ID, i.CUSTOMER_ID, i.PLAN_ID, l.LINE_NO, l.TIER_NO, + l.QUANTITY, l.UNIT_PRICE_MICROS, l.FLAT_FEE_CENTS, l.AMOUNT_CENTS +FROM INVOICES i +JOIN INVOICE_LINES l ON l.INVOICE_ID = i.INVOICE_ID +WHERE i.BILLING_MONTH = ? +ORDER BY i.INVOICE_ID, l.LINE_NO`, sf.DataTypeDate, m.Start()) + if err != nil { + return fmt.Errorf("export invoice lines: %w", err) + } + defer rows.Close() + n := 0 + for rows.Next() { + l := InvoiceLine{Month: m} + if err := rows.Scan(&l.InvoiceID, &l.CustomerID, &l.PlanID, &l.LineNo, &l.TierNo, + &l.Quantity, &l.UnitPriceMicros, &l.FlatFeeCents, &l.AmountCents); err != nil { + return err + } + if err := fn(l); err != nil { + return err + } + n++ + } + if err := rows.Err(); err != nil { + return err + } + if n == 0 { + return ErrNoInvoices + } + return nil +} diff --git a/metering-api/internal/metering/metering_integration_test.go b/metering-api/internal/metering/metering_integration_test.go new file mode 100644 index 0000000..219485c --- /dev/null +++ b/metering-api/internal/metering/metering_integration_test.go @@ -0,0 +1,411 @@ +//go:build integration + +package metering_test + +import ( + "bytes" + "cmp" + "context" + "crypto/sha256" + "database/sql" + "encoding/csv" + "errors" + "fmt" + "math" + "net/http" + "slices" + "strconv" + "testing" + "time" + + "github.com/localstack-samples/localstack-snowflake-samples/metering-api/internal/metering" + "github.com/localstack-samples/localstack-snowflake-samples/metering-api/internal/seed" +) + +// lastMonth is a complete month, so it can be closed. +var lastMonth = metering.MonthOf(time.Now()).Prev() + +// 12 customers: 8 FREE, 3 PRO, 1 ENTERPRISE, about 530 events a month. +func smallDataset(seedValue uint64, months ...metering.Month) seed.Options { + return seed.Options{Seed: seedValue, Customers: 12, Months: months} +} + +// priceList restates the published price list (init/03_price_plans.sf.sql) as +// {last call of the tier, micro-dollars per call, flat fee in cents}. +var priceList = map[string][][3]int64{ + "FREE": {{10_000, 0, 0}, {math.MaxInt64, 2000, 0}}, + "PRO": {{100_000, 0, 4900}, {1_000_000, 800, 0}, {math.MaxInt64, 500, 0}}, + "ENTERPRISE": {{1_000_000, 0, 49900}, {10_000_000, 400, 0}, {math.MaxInt64, 250, 0}}, +} + +type wantLine struct { + tier int + qty, micros, flat, cents int64 +} + +// priceByHand bills calls against a plan's tiers: every tier the usage reaches +// bills the calls above the previous tier's limit, up to its own, plus its flat +// fee; amounts are rounded half up to the cent per line. +func priceByHand(plan string, calls int64) ([]wantLine, int64) { + var lines []wantLine + var total, below int64 + for i, t := range priceList[plan] { + if i > 0 && calls <= below { + break + } + qty := min(calls, t[0]) - below + cents := t[2] + (qty*t[1]+5_000)/10_000 + lines = append(lines, wantLine{tier: i + 1, qty: qty, micros: t[1], flat: t[2], cents: cents}) + total += cents + below = t[0] + } + return lines, total +} + +// callsIn sums each customer's calls and counts the events in a month. +func callsIn(d seed.Dataset, m metering.Month) (map[string]int64, int) { + calls := map[string]int64{} + events := 0 + for _, e := range d.Events { + if !e.OccurredAt.Before(m.Start()) && e.OccurredAt.Before(m.End()) { + calls[e.CustomerID] += e.Quantity + events++ + } + } + return calls, events +} + +func TestIngestIsIdempotent(t *testing.T) { + t.Parallel() + e := newEnv(t) + d := e.seed(smallDataset(1, lastMonth)) + calls, events := callsIn(d, lastMonth) + var wantCalls int64 + for _, c := range calls { + wantCalls += c + } + + // The gateway retries everything, in different batch sizes. + res := e.ingest(d.Events, 250) + if res.Received != events || res.Inserted != 0 || res.Duplicates != events { + t.Errorf("re-ingest = %+v, want %d received, 0 inserted, %d duplicates", res, events, events) + } + if got := e.scalar(`SELECT COUNT(*) FROM USAGE_EVENTS`); got != int64(events) { + t.Errorf("stored events = %d, want %d", got, events) + } + if got := e.scalar(`SELECT SUM(QUANTITY) FROM USAGE_EVENTS`); got != wantCalls { + t.Errorf("stored calls = %d, want %d", got, wantCalls) + } + + // One batch with a new event twice and an existing one: only one row is new. + fresh := metering.Event{EventID: "evt_retry_1", CustomerID: d.Customers[6].ID, Quantity: 7, OccurredAt: lastMonth.Start().Add(time.Hour)} + res = e.ingest([]metering.Event{fresh, fresh, d.Events[0]}, 10) + if res != (metering.IngestResult{Received: 3, Inserted: 1, Duplicates: 2}) { + t.Errorf("mixed batch = %+v, want 3 received, 1 inserted, 2 duplicates", res) + } + if got := e.scalar(`SELECT COUNT(*) FROM USAGE_EVENTS_STAGING`); got != 0 { + t.Errorf("%d rows left in staging", got) + } + + // The usage endpoint agrees with the dataset, per customer and per day. + for _, c := range d.Customers[5:10] { // one FREE, the three PRO and the ENTERPRISE customer + var u metering.Usage + path := fmt.Sprintf("/v1/customers/%s/usage?from=%s&to=%s", c.ID, lastMonth.Start().Format(time.DateOnly), lastMonth.End().Format(time.DateOnly)) + e.call("GET", path, nil, http.StatusOK, &u) + want := calls[c.ID] + if c.ID == fresh.CustomerID { + want += fresh.Quantity + } + var daily int64 + for _, day := range u.Daily { + daily += day.Calls + } + if u.TotalCalls != want || daily != want || u.PlanID != c.PlanID { + t.Errorf("%s usage = %d calls (%d by day) on %s, want %d on %s", c.ID, u.TotalCalls, daily, u.PlanID, want, c.PlanID) + } + } +} + +func TestCloseMonthInvoicesAtTierPrices(t *testing.T) { + t.Parallel() + e := newEnv(t) + d := e.seed(smallDataset(2, lastMonth.Prev(), lastMonth)) + // A PRO customer without usage still pays the monthly fee. + idle := metering.Customer{ID: "cus_idle", Name: "Idle Labs", PlanID: "PRO", CreatedAt: lastMonth.Start().AddDate(-1, 0, 0)} + if err := e.svc.UpsertCustomers(e.ctx, []metering.Customer{idle}); err != nil { + t.Fatal(err) + } + customers := append(slices.Clone(d.Customers), idle) + calls, events := callsIn(d, lastMonth) + + wantTotals := map[string]int64{} + var wantTotal int64 + for _, c := range customers { + _, cents := priceByHand(c.PlanID, calls[c.ID]) + wantTotals[c.ID] = cents + wantTotal += cents + } + + var res metering.CloseResult + e.call("POST", "/v1/invoices/"+lastMonth.String()+"/close", nil, http.StatusCreated, &res) + if res.Invoices != len(customers) || res.TotalCents != wantTotal || res.EventsInvoiced != int64(events) { + t.Errorf("close = %+v, want %d invoices, %d cents, %d events", res, len(customers), wantTotal, events) + } + + rows, err := e.db.QueryContext(e.ctx, `SELECT CUSTOMER_ID, TOTAL_CALLS, TOTAL_CENTS FROM INVOICES WHERE BILLING_MONTH = ?`, lastMonth.Start()) + if err != nil { + t.Fatal(err) + } + defer rows.Close() + seen := 0 + for rows.Next() { + var id string + var gotCalls, gotCents int64 + if err := rows.Scan(&id, &gotCalls, &gotCents); err != nil { + t.Fatal(err) + } + seen++ + if gotCalls != calls[id] || gotCents != wantTotals[id] { + t.Errorf("invoice of %s: %d calls, %d cents; want %d calls, %d cents", id, gotCalls, gotCents, calls[id], wantTotals[id]) + } + } + if err := rows.Err(); err != nil || seen != len(customers) { + t.Errorf("read %d invoices (%v), want %d", seen, err, len(customers)) + } + if got := e.scalar(`SELECT TOTAL_CENTS FROM INVOICES WHERE CUSTOMER_ID = 'cus_idle'`); got != 4900 { + t.Errorf("idle PRO customer billed %d cents, want the 4900 monthly fee", got) + } + + // The month's events are invoiced; the month before is untouched. + if got := e.scalar(`SELECT COUNT(*) FROM USAGE_EVENTS WHERE INVOICE_ID IS NOT NULL`); got != int64(events) { + t.Errorf("%d events invoiced, want %d", got, events) + } + _, earlier := callsIn(d, lastMonth.Prev()) + if got := e.scalar(`SELECT COUNT(*) FROM USAGE_EVENTS WHERE INVOICE_ID IS NULL`); got != int64(earlier) { + t.Errorf("%d events uninvoiced, want the %d of %s", got, earlier, lastMonth.Prev()) + } + + // Invoices are issued once. + var p metering.Problem + e.call("POST", "/v1/invoices/"+lastMonth.String()+"/close", nil, http.StatusConflict, &p) + if got := e.scalar(`SELECT COUNT(*) FROM INVOICES`); got != int64(len(customers)) { + t.Errorf("second close left %d invoices, want %d", got, len(customers)) + } +} + +func TestCloseMonthIsAtomic(t *testing.T) { + t.Parallel() + e := newEnv(t) + d := e.seed(smallDataset(3, lastMonth)) + _, events := callsIn(d, lastMonth) + + var writtenInTx int64 + metering.SetBeforeMarkInvoiced(e.svc, func(ctx context.Context, tx *sql.Tx) error { + if err := tx.QueryRowContext(ctx, `SELECT COUNT(*) FROM INVOICES`).Scan(&writtenInTx); err != nil { + return err + } + return errors.New("injected failure after writing invoices") + }) + var p metering.Problem + e.call("POST", "/v1/invoices/"+lastMonth.String()+"/close", nil, http.StatusInternalServerError, &p) + if writtenInTx != int64(len(d.Customers)) { + t.Fatalf("the transaction had written %d invoices before the failure, want %d", writtenInTx, len(d.Customers)) + } + for query, want := range map[string]int64{ + `SELECT COUNT(*) FROM INVOICES`: 0, + `SELECT COUNT(*) FROM INVOICE_LINES`: 0, + `SELECT COUNT(*) FROM USAGE_EVENTS WHERE INVOICE_ID IS NOT NULL`: 0, + `SELECT COUNT(*) FROM USAGE_EVENTS WHERE INVOICE_ID IS NULL`: int64(events), + } { + if got := e.scalar(query); got != want { + t.Errorf("after the failed close, %s = %d, want %d", query, got, want) + } + } + + // Once the fault is gone, the same close succeeds. + metering.SetBeforeMarkInvoiced(e.svc, nil) + var res metering.CloseResult + e.call("POST", "/v1/invoices/"+lastMonth.String()+"/close", nil, http.StatusCreated, &res) + if res.Invoices != len(d.Customers) || res.EventsInvoiced != int64(events) { + t.Errorf("retried close = %+v, want %d invoices and %d events", res, len(d.Customers), events) + } +} + +func TestRollupRunsAsync(t *testing.T) { + t.Parallel() + e := newEnv(t) + d := e.seed(smallDataset(4, lastMonth.Prev(), lastMonth)) + + plans := map[string]string{} + for _, c := range d.Customers { + plans[c.ID] = c.PlanID + } + type key struct{ plan, day string } + agg := map[key]*metering.RollupRow{} + active := map[key]map[string]bool{} + for _, ev := range d.Events { + if ev.OccurredAt.Before(lastMonth.Start()) || !ev.OccurredAt.Before(lastMonth.End()) { + continue + } + k := key{plans[ev.CustomerID], metering.DateOf(ev.OccurredAt).String()} + if agg[k] == nil { + agg[k] = &metering.RollupRow{PlanID: k.plan, Date: metering.DateOf(ev.OccurredAt)} + active[k] = map[string]bool{} + } + agg[k].Events++ + agg[k].Calls += ev.Quantity + active[k][ev.CustomerID] = true + } + var want []metering.RollupRow + for k, r := range agg { + r.ActiveCustomers = int64(len(active[k])) + want = append(want, *r) + } + slices.SortFunc(want, func(a, b metering.RollupRow) int { + if c := a.Date.Time().Compare(b.Date.Time()); c != 0 { + return c + } + return cmp.Compare(a.PlanID, b.PlanID) + }) + + var started struct { + QueryID string `json:"query_id"` + StatusURL string `json:"status_url"` + } + header := e.call("POST", "/v1/rollups/"+lastMonth.String(), nil, http.StatusAccepted, &started) + if !metering.ValidQueryID(started.QueryID) || header.Get("Location") != "/v1/rollups/"+started.QueryID { + t.Fatalf("started = %+v, Location %q", started, header.Get("Location")) + } + + deadline := time.Now().Add(2 * time.Minute) + var r metering.Rollup + for { + e.call("GET", started.StatusURL, nil, http.StatusOK, &r) + if r.Status != metering.RollupRunning { + break + } + if time.Now().After(deadline) { + t.Fatal("rollup still running after 2 minutes") + } + time.Sleep(250 * time.Millisecond) + } + if r.Status != metering.RollupSucceeded { + t.Fatalf("rollup %s: %s", r.Status, r.Error) + } + if !slices.Equal(r.Rows, want) { + t.Errorf("rollup has %d rows, want %d; first got %+v, first want %+v", len(r.Rows), len(want), first(r.Rows), first(want)) + } +} + +func TestInvoiceExportCSV(t *testing.T) { + t.Parallel() + e := newEnv(t) + d := e.seed(smallDataset(5, lastMonth)) + calls, _ := callsIn(d, lastMonth) + + // The expected file, built from the dataset and the price list. + var want bytes.Buffer + w := csv.NewWriter(&want) + _ = w.Write([]string{"invoice_id", "customer_id", "plan_id", "billing_month", "line_no", "tier_no", + "quantity", "unit_price_micros", "flat_fee_cents", "amount_cents"}) + wantRows := 0 + for _, c := range d.Customers { // ids sort in generation order + lines, _ := priceByHand(c.PlanID, calls[c.ID]) + for i, l := range lines { + _ = w.Write([]string{"inv_" + lastMonth.Start().Format("200601") + "_" + c.ID, c.ID, c.PlanID, lastMonth.String(), + strconv.Itoa(i + 1), strconv.Itoa(l.tier), fmt.Sprint(l.qty), fmt.Sprint(l.micros), fmt.Sprint(l.flat), fmt.Sprint(l.cents)}) + wantRows++ + } + } + w.Flush() + + path := "/v1/invoices/" + lastMonth.String() + "/export.csv" + resp, _ := e.do("GET", path, nil) + if resp.StatusCode != http.StatusNotFound { + t.Errorf("export before close: status %d, want 404", resp.StatusCode) + } + e.call("POST", "/v1/invoices/"+lastMonth.String()+"/close", nil, http.StatusCreated, nil) + + resp, body := e.do("GET", path, nil) + if resp.StatusCode != http.StatusOK || resp.Header.Get("Content-Type") != "text/csv; charset=utf-8" { + t.Fatalf("export: status %d, Content-Type %q", resp.StatusCode, resp.Header.Get("Content-Type")) + } + records, err := csv.NewReader(bytes.NewReader(body)).ReadAll() + if err != nil { + t.Fatal(err) + } + if len(records) != wantRows+1 { + t.Errorf("export has %d rows plus header, want %d", len(records)-1, wantRows) + } + if got, wantSum := sha256.Sum256(body), sha256.Sum256(want.Bytes()); got != wantSum { + t.Errorf("export checksum %x, want %x\ngot:\n%s\nwant:\n%s", got[:8], wantSum[:8], head(body), head(want.Bytes())) + } +} + +func TestErrorPaths(t *testing.T) { + t.Parallel() + e := newEnv(t) + d := e.seed(smallDataset(6, lastMonth)) + events := e.scalar(`SELECT COUNT(*) FROM USAGE_EVENTS`) + + t.Run("unknown customer is a 404 problem", func(t *testing.T) { + resp, body := e.do("GET", "/v1/customers/cus_9999/usage", nil) + if resp.StatusCode != http.StatusNotFound || resp.Header.Get("Content-Type") != "application/problem+json" { + t.Fatalf("status %d, Content-Type %q", resp.StatusCode, resp.Header.Get("Content-Type")) + } + if !bytes.Contains(body, []byte(`"type":"urn:metering:problem:not-found"`)) || !bytes.Contains(body, []byte(`"status":404`)) { + t.Errorf("body %s", body) + } + }) + t.Run("malformed payload is a 400 problem", func(t *testing.T) { + var p metering.Problem + e.call("POST", "/v1/events", `{"events": [{"event_id": "e1",`, http.StatusBadRequest, &p) + if p.Type != "urn:metering:problem:malformed-request" { + t.Errorf("problem %+v", p) + } + }) + t.Run("invalid events are a 422 problem", func(t *testing.T) { + var p metering.Problem + e.call("POST", "/v1/events", map[string]any{"events": []map[string]any{{"event_id": "e1", "customer_id": d.Customers[0].ID, "quantity": 0}}}, + http.StatusUnprocessableEntity, &p) + if p.Type != "urn:metering:problem:validation-failed" || len(p.Errors) != 2 { + t.Errorf("problem %+v", p) + } + }) + t.Run("a batch naming an unknown customer stores nothing", func(t *testing.T) { + batch := []metering.Event{ + {EventID: "evt_ok", CustomerID: d.Customers[0].ID, Quantity: 5, OccurredAt: lastMonth.Start()}, + {EventID: "evt_ghost", CustomerID: "cus_ghost", Quantity: 5, OccurredAt: lastMonth.Start()}, + } + var p metering.Problem + e.call("POST", "/v1/events", map[string]any{"events": batch}, http.StatusUnprocessableEntity, &p) + if p.Type != "urn:metering:problem:unknown-customer" || !slices.Equal(p.UnknownCustomers, []string{"cus_ghost"}) { + t.Errorf("problem %+v", p) + } + if got := e.scalar(`SELECT COUNT(*) FROM USAGE_EVENTS`); got != events { + t.Errorf("%d events stored, want %d", got, events) + } + if got := e.scalar(`SELECT COUNT(*) FROM USAGE_EVENTS_STAGING`); got != 0 { + t.Errorf("%d rows left in staging", got) + } + }) + t.Run("unknown rollup query is a 404 problem", func(t *testing.T) { + var p metering.Problem + e.call("GET", "/v1/rollups/01bd0000-0000-0000-0000-000000000000", nil, http.StatusNotFound, &p) + if p.Type != "urn:metering:problem:not-found" { + t.Errorf("problem %+v", p) + } + }) +} + +func first[T any](s []T) any { + if len(s) == 0 { + return nil + } + return s[0] +} + +func head(b []byte) []byte { + lines := bytes.SplitAfterN(b, []byte("\n"), 6) + return bytes.Join(lines[:min(5, len(lines))], nil) +} diff --git a/metering-api/internal/metering/problem.go b/metering-api/internal/metering/problem.go new file mode 100644 index 0000000..d0a4bfd --- /dev/null +++ b/metering-api/internal/metering/problem.go @@ -0,0 +1,46 @@ +package metering + +import ( + "encoding/json" + "net/http" +) + +// Problem is an RFC 9457 problem details response. +type Problem struct { + Type string `json:"type"` + Title string `json:"title"` + Status int `json:"status"` + Detail string `json:"detail,omitempty"` + Instance string `json:"instance,omitempty"` + Errors []FieldError `json:"errors,omitempty"` + UnknownCustomers []string `json:"unknown_customers,omitempty"` +} + +// FieldError points at one invalid field of a request body. +type FieldError struct { + Field string `json:"field"` + Detail string `json:"detail"` +} + +// Problem types, as URNs so they need no hosted documentation. +const ( + problemMalformed = "urn:metering:problem:malformed-request" + problemInvalid = "urn:metering:problem:validation-failed" + problemUnknownCustomer = "urn:metering:problem:unknown-customer" + problemNotFound = "urn:metering:problem:not-found" + problemConflict = "urn:metering:problem:conflict" + problemInternal = "urn:metering:problem:internal" +) + +func writeProblem(w http.ResponseWriter, r *http.Request, p Problem) { + p.Instance = r.URL.Path + w.Header().Set("Content-Type", "application/problem+json") + w.WriteHeader(p.Status) + _ = json.NewEncoder(w).Encode(p) +} + +func writeJSON(w http.ResponseWriter, status int, v any) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _ = json.NewEncoder(w).Encode(v) +} diff --git a/metering-api/internal/metering/rollups.go b/metering-api/internal/metering/rollups.go new file mode 100644 index 0000000..c9b5ac1 --- /dev/null +++ b/metering-api/internal/metering/rollups.go @@ -0,0 +1,144 @@ +package metering + +import ( + "context" + "database/sql/driver" + "errors" + "fmt" + "regexp" + "time" + + sf "github.com/snowflakedb/gosnowflake/v2" +) + +// The monthly rollup: calls, events and active customers per plan and day. +const rollupQuery = `SELECT c.PLAN_ID, + TO_DATE(e.OCCURRED_AT) AS USAGE_DATE, + COUNT(DISTINCT e.CUSTOMER_ID) AS ACTIVE_CUSTOMERS, + COUNT(*) AS EVENTS, + SUM(e.QUANTITY) AS CALLS +FROM USAGE_EVENTS e +JOIN CUSTOMERS c ON c.CUSTOMER_ID = e.CUSTOMER_ID +WHERE e.OCCURRED_AT >= ? AND e.OCCURRED_AT < ? +GROUP BY c.PLAN_ID, USAGE_DATE +ORDER BY USAGE_DATE, c.PLAN_ID` + +// rollupTimeout bounds how long the driver keeps fetching a rollup's result in +// the background after the request that started it has returned. +const rollupTimeout = 15 * time.Minute + +// RollupRow is one row of the monthly rollup. +type RollupRow struct { + PlanID string `json:"plan_id"` + Date Date `json:"date"` + ActiveCustomers int64 `json:"active_customers"` + Events int64 `json:"events"` + Calls int64 `json:"calls"` +} + +// RollupStatus values. +const ( + RollupRunning = "running" + RollupSucceeded = "succeeded" + RollupFailed = "failed" +) + +// Rollup is the state of a rollup query. Rows is set once it has succeeded. +type Rollup struct { + QueryID string `json:"query_id"` + Status string `json:"status"` + Error string `json:"error,omitempty"` + Rows []RollupRow `json:"rows,omitempty"` +} + +// StartRollup submits the rollup for a month as an asynchronous Snowflake query +// and returns its query id without waiting for it. Any instance of the service +// can later read its status and result by that id (see GetRollup). +func (s *Service) StartRollup(ctx context.Context, m Month) (string, error) { + conn, err := s.db.Conn(ctx) + if err != nil { + return "", err + } + defer conn.Close() + // The query outlives the HTTP request, so it must not be canceled with it. + // The query tag on ctx is kept. + bg, cancel := context.WithTimeout(context.WithoutCancel(ctx), rollupTimeout) + var queryID string + err = conn.Raw(func(dc any) error { + rows, err := dc.(driver.QueryerContext).QueryContext(sf.WithAsyncMode(bg), rollupQuery, []driver.NamedValue{ + {Ordinal: 1, Value: m.Start()}, + {Ordinal: 2, Value: m.End()}, + }) + if err != nil { + return err + } + queryID = rows.(sf.SnowflakeRows).GetQueryID() + // In async mode the driver fetches the result in the background; Close + // waits for that and releases it. Results are read again by id later. + go func() { + defer cancel() + _ = rows.Close() + }() + return nil + }) + if err != nil { + cancel() + return "", fmt.Errorf("submit rollup: %w", err) + } + return queryID, nil +} + +var queryIDPattern = regexp.MustCompile(`^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$`) + +// ValidQueryID reports whether id has the shape of a Snowflake query id. +func ValidQueryID(id string) bool { return queryIDPattern.MatchString(id) } + +// GetRollup reports a rollup query's status and, once it has succeeded, its +// rows (fetched by query id). It returns ErrQueryNotFound for an unknown id. +func (s *Service) GetRollup(ctx context.Context, queryID string) (Rollup, error) { + r := Rollup{QueryID: queryID} + conn, err := s.db.Conn(ctx) + if err != nil { + return r, err + } + err = conn.Raw(func(dc any) error { + _, err := dc.(sf.SnowflakeConnection).GetQueryStatus(ctx, queryID) + return err + }) + _ = conn.Close() + var sfErr *sf.SnowflakeError + switch { + case err == nil: + r.Status = RollupSucceeded + case errors.As(err, &sfErr) && sfErr.Number == sf.ErrQueryIsRunning: + r.Status = RollupRunning + return r, nil + case errors.As(err, &sfErr) && sfErr.Number == sf.ErrQueryReportedError: + r.Status = RollupFailed + r.Error = sfErr.Message + return r, nil + case errors.As(err, &sfErr) && sfErr.Number == sf.ErrQueryStatus: + return r, ErrQueryNotFound + default: + return r, fmt.Errorf("rollup status: %w", err) + } + + rows, err := s.db.QueryContext(sf.WithFetchResultByID(ctx, queryID), "") + if err != nil { + return r, fmt.Errorf("fetch rollup result: %w", err) + } + defer rows.Close() + r.Rows = []RollupRow{} + for rows.Next() { + var ( + row RollupRow + day time.Time + ) + if err := rows.Scan(&row.PlanID, &day, &row.ActiveCustomers, &row.Events, &row.Calls); err != nil { + return r, fmt.Errorf("read rollup result: %w", err) + } + row.Date = DateOf(day) + r.Rows = append(r.Rows, row) + } + return r, rows.Err() +} diff --git a/metering-api/internal/metering/service.go b/metering-api/internal/metering/service.go new file mode 100644 index 0000000..a41404e --- /dev/null +++ b/metering-api/internal/metering/service.go @@ -0,0 +1,141 @@ +// Package metering is the usage-metering and invoicing service: event ingestion, +// usage queries, monthly invoice close, async rollups and CSV export, on +// Snowflake through database/sql and gosnowflake. +package metering + +import ( + "context" + "database/sql" + "errors" + "fmt" + "log/slog" + "time" + + sf "github.com/snowflakedb/gosnowflake/v2" + + "github.com/localstack-samples/localstack-snowflake-samples/metering-api/internal/pricing" +) + +// Service runs the metering use cases against one Snowflake schema. +type Service struct { + db *sql.DB + log *slog.Logger + now func() time.Time + + // beforeMarkInvoiced runs inside the invoice-close transaction after the + // invoices are written and before usage is marked invoiced. Tests use it to + // inject a failure mid-transaction. + beforeMarkInvoiced func(context.Context, *sql.Tx) error +} + +// NewService returns a Service on db, whose session must default to the +// metering schema. +func NewService(db *sql.DB, log *slog.Logger) *Service { + return &Service{db: db, log: log, now: time.Now} +} + +// Ping checks that Snowflake answers. +func (s *Service) Ping(ctx context.Context) error { + return s.db.PingContext(ctx) +} + +// Customer is an account on the API platform. +type Customer struct { + ID string + Name string + PlanID string + CreatedAt time.Time +} + +// Errors the HTTP layer maps to problem responses. +var ( + ErrCustomerNotFound = errors.New("customer not found") + ErrMonthClosed = errors.New("month already closed") + ErrMonthNotOver = errors.New("month has not ended") + ErrNoInvoices = errors.New("no invoices for month") + ErrQueryNotFound = errors.New("query not found") +) + +// UnknownCustomersError rejects an ingest batch that names customers that do +// not exist. +type UnknownCustomersError struct{ IDs []string } + +func (e *UnknownCustomersError) Error() string { + return fmt.Sprintf("%d unknown customer id(s)", len(e.IDs)) +} + +// WithQueryTag labels every Snowflake statement run with ctx, so QUERY_HISTORY +// can be filtered by endpoint (for example QUERY_TAG = 'metering-api:invoices.close'). +func WithQueryTag(ctx context.Context, operation string) context.Context { + return sf.WithQueryTag(ctx, "metering-api:"+operation) +} + +// UpsertCustomers creates or updates customers by id, 100 per statement. +func (s *Service) UpsertCustomers(ctx context.Context, customers []Customer) error { + const chunk = 100 + for start := 0; start < len(customers); start += chunk { + part := customers[start:min(start+chunk, len(customers))] + args := make([]any, 0, 4*len(part)) + values := make([]byte, 0, 16*len(part)) + for i, c := range part { + if i > 0 { + values = append(values, ", "...) + } + values = append(values, "(?, ?, ?, ?)"...) + args = append(args, c.ID, c.Name, c.PlanID, c.CreatedAt.UTC()) + } + query := `MERGE INTO CUSTOMERS t +USING (SELECT $1 AS CUSTOMER_ID, $2 AS NAME, $3 AS PLAN_ID, $4::TIMESTAMP_NTZ AS CREATED_AT + FROM VALUES ` + string(values) + `) s +ON t.CUSTOMER_ID = s.CUSTOMER_ID +WHEN MATCHED THEN UPDATE SET NAME = s.NAME, PLAN_ID = s.PLAN_ID, CREATED_AT = s.CREATED_AT +WHEN NOT MATCHED THEN INSERT (CUSTOMER_ID, NAME, PLAN_ID, CREATED_AT) + VALUES (s.CUSTOMER_ID, s.NAME, s.PLAN_ID, s.CREATED_AT)` + if _, err := s.db.ExecContext(ctx, query, args...); err != nil { + return fmt.Errorf("upsert customers: %w", err) + } + } + return nil +} + +type queryer interface { + QueryContext(ctx context.Context, query string, args ...any) (*sql.Rows, error) +} + +// loadPlans reads PRICE_PLANS into validated pricing plans keyed by plan id. +func loadPlans(ctx context.Context, q queryer) (map[string]pricing.Plan, error) { + rows, err := q.QueryContext(ctx, `SELECT PLAN_ID, UP_TO_CALLS, UNIT_PRICE_MICROS, FLAT_FEE_CENTS +FROM PRICE_PLANS ORDER BY PLAN_ID, TIER_NO`) + if err != nil { + return nil, fmt.Errorf("load price plans: %w", err) + } + defer rows.Close() + plans := map[string]pricing.Plan{} + for rows.Next() { + var ( + id string + upTo sql.NullInt64 + tier pricing.Tier + ) + if err := rows.Scan(&id, &upTo, &tier.UnitPriceMicros, &tier.FlatFeeCents); err != nil { + return nil, err + } + tier.UpTo = pricing.Unlimited + if upTo.Valid { + tier.UpTo = upTo.Int64 + } + p := plans[id] + p.ID = id + p.Tiers = append(p.Tiers, tier) + plans[id] = p + } + if err := rows.Err(); err != nil { + return nil, err + } + for _, p := range plans { + if err := p.Validate(); err != nil { + return nil, err + } + } + return plans, nil +} diff --git a/metering-api/internal/metering/usage.go b/metering-api/internal/metering/usage.go new file mode 100644 index 0000000..5a55a98 --- /dev/null +++ b/metering-api/internal/metering/usage.go @@ -0,0 +1,63 @@ +package metering + +import ( + "context" + "database/sql" + "errors" + "fmt" + "time" +) + +// Usage is a customer's usage over [From, To). +type Usage struct { + CustomerID string `json:"customer_id"` + PlanID string `json:"plan_id"` + From Date `json:"from"` + To Date `json:"to"` + TotalCalls int64 `json:"total_calls"` + Events int64 `json:"events"` + Daily []DayUsage `json:"daily"` +} + +// DayUsage is one day of a Usage. +type DayUsage struct { + Date Date `json:"date"` + Calls int64 `json:"calls"` + Events int64 `json:"events"` +} + +// CustomerUsage aggregates a customer's events per day. It returns +// ErrCustomerNotFound for an unknown customer. +func (s *Service) CustomerUsage(ctx context.Context, customerID string, from, to Date) (Usage, error) { + u := Usage{CustomerID: customerID, From: from, To: to, Daily: []DayUsage{}} + err := s.db.QueryRowContext(ctx, `SELECT PLAN_ID FROM CUSTOMERS WHERE CUSTOMER_ID = ?`, customerID).Scan(&u.PlanID) + if errors.Is(err, sql.ErrNoRows) { + return Usage{}, ErrCustomerNotFound + } + if err != nil { + return Usage{}, fmt.Errorf("look up customer: %w", err) + } + rows, err := s.db.QueryContext(ctx, `SELECT TO_DATE(OCCURRED_AT) AS USAGE_DATE, SUM(QUANTITY), COUNT(*) +FROM USAGE_EVENTS +WHERE CUSTOMER_ID = ? AND OCCURRED_AT >= ? AND OCCURRED_AT < ? +GROUP BY USAGE_DATE +ORDER BY USAGE_DATE`, customerID, from.Time(), to.Time()) + if err != nil { + return Usage{}, fmt.Errorf("aggregate usage: %w", err) + } + defer rows.Close() + for rows.Next() { + var ( + day time.Time + d DayUsage + ) + if err := rows.Scan(&day, &d.Calls, &d.Events); err != nil { + return Usage{}, err + } + d.Date = DateOf(day) + u.TotalCalls += d.Calls + u.Events += d.Events + u.Daily = append(u.Daily, d) + } + return u, rows.Err() +} diff --git a/metering-api/internal/metering/validate.go b/metering-api/internal/metering/validate.go new file mode 100644 index 0000000..e5f4a42 --- /dev/null +++ b/metering-api/internal/metering/validate.go @@ -0,0 +1,101 @@ +package metering + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "regexp" + "time" +) + +// maxClockSkew is how far in the future an event's timestamp may be. +const maxClockSkew = 5 * time.Minute + +// maxUsageRange bounds the usage query window. +const maxUsageRange = 366 * 24 * time.Hour + +var idPattern = regexp.MustCompile(`^[A-Za-z0-9_.:-]+$`) + +type ingestRequest struct { + Events []Event `json:"events"` +} + +// decodeEvents parses an ingest body. Any JSON error is a malformed request. +func decodeEvents(body io.Reader) ([]Event, error) { + dec := json.NewDecoder(body) + dec.DisallowUnknownFields() + var req ingestRequest + if err := dec.Decode(&req); err != nil { + return nil, err + } + if dec.More() { + return nil, errors.New("unexpected data after the JSON object") + } + return req.Events, nil +} + +// ValidateEvents checks an ingest batch and returns one FieldError per problem. +func ValidateEvents(events []Event, now time.Time) []FieldError { + var errs []FieldError + add := func(field, format string, args ...any) { + errs = append(errs, FieldError{Field: field, Detail: fmt.Sprintf(format, args...)}) + } + switch { + case len(events) == 0: + add("events", "must contain at least one event") + case len(events) > MaxBatch: + add("events", "must contain at most %d events, got %d", MaxBatch, len(events)) + } + for i, e := range events { + at := func(f string) string { return fmt.Sprintf("events[%d].%s", i, f) } + checkID(add, at("event_id"), e.EventID, 64) + checkID(add, at("customer_id"), e.CustomerID, 32) + if e.Quantity < 1 { + add(at("quantity"), "must be a positive number of calls") + } + switch { + case e.OccurredAt.IsZero(): + add(at("occurred_at"), "is required (RFC 3339 timestamp)") + case e.OccurredAt.After(now.Add(maxClockSkew)): + add(at("occurred_at"), "is in the future") + } + } + return errs +} + +func checkID(add func(string, string, ...any), field, v string, maxLen int) { + switch { + case v == "": + add(field, "is required") + case len(v) > maxLen: + add(field, "must be at most %d characters", maxLen) + case !idPattern.MatchString(v): + add(field, "may contain only letters, digits and _ . : -") + } +} + +// parseUsageRange reads ?from=YYYY-MM-DD&to=YYYY-MM-DD (to exclusive). Both +// default to the current month. +func parseUsageRange(from, to string, now time.Time) (Date, Date, error) { + month := MonthOf(now) + start, end := DateOf(month.Start()), DateOf(month.End()) + var err error + if from != "" { + if start, err = ParseDate(from); err != nil { + return Date{}, Date{}, fmt.Errorf("from: %w", err) + } + } + if to != "" { + if end, err = ParseDate(to); err != nil { + return Date{}, Date{}, fmt.Errorf("to: %w", err) + } + } + if !start.Time().Before(end.Time()) { + return Date{}, Date{}, errors.New("from must be before to") + } + if end.Time().Sub(start.Time()) > maxUsageRange { + return Date{}, Date{}, errors.New("the range may span at most 366 days") + } + return start, end, nil +} diff --git a/metering-api/internal/metering/validate_test.go b/metering-api/internal/metering/validate_test.go new file mode 100644 index 0000000..bddf4e7 --- /dev/null +++ b/metering-api/internal/metering/validate_test.go @@ -0,0 +1,111 @@ +package metering + +import ( + "strings" + "testing" + "time" +) + +var now = time.Date(2026, 10, 15, 12, 0, 0, 0, time.UTC) + +func validEvent() Event { + return Event{EventID: "evt_1", CustomerID: "cus_0001", Quantity: 10, OccurredAt: now.Add(-time.Hour)} +} + +func TestValidateEvents(t *testing.T) { + tests := []struct { + name string + mutate func(*Event) + field string // expected failing field, "" for valid + }{ + {"valid", func(*Event) {}, ""}, + {"missing event id", func(e *Event) { e.EventID = "" }, "events[0].event_id"}, + {"event id too long", func(e *Event) { e.EventID = strings.Repeat("a", 65) }, "events[0].event_id"}, + {"event id with spaces", func(e *Event) { e.EventID = "evt 1" }, "events[0].event_id"}, + {"missing customer", func(e *Event) { e.CustomerID = "" }, "events[0].customer_id"}, + {"zero quantity", func(e *Event) { e.Quantity = 0 }, "events[0].quantity"}, + {"negative quantity", func(e *Event) { e.Quantity = -3 }, "events[0].quantity"}, + {"missing timestamp", func(e *Event) { e.OccurredAt = time.Time{} }, "events[0].occurred_at"}, + {"timestamp in the future", func(e *Event) { e.OccurredAt = now.Add(6 * time.Minute) }, "events[0].occurred_at"}, + {"small clock skew is fine", func(e *Event) { e.OccurredAt = now.Add(4 * time.Minute) }, ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + e := validEvent() + tt.mutate(&e) + errs := ValidateEvents([]Event{e}, now) + if tt.field == "" { + if len(errs) != 0 { + t.Fatalf("unexpected errors: %+v", errs) + } + return + } + if len(errs) != 1 || errs[0].Field != tt.field { + t.Fatalf("errors = %+v, want one on %s", errs, tt.field) + } + }) + } +} + +func TestValidateEventsBatchSize(t *testing.T) { + if errs := ValidateEvents(nil, now); len(errs) != 1 || errs[0].Field != "events" { + t.Errorf("empty batch: %+v", errs) + } + big := make([]Event, MaxBatch+1) + for i := range big { + big[i] = validEvent() + } + if errs := ValidateEvents(big, now); len(errs) != 1 || errs[0].Field != "events" { + t.Errorf("oversized batch: %+v", errs) + } + if errs := ValidateEvents(big[:MaxBatch], now); len(errs) != 0 { + t.Errorf("full batch: %+v", errs) + } +} + +func TestParseUsageRange(t *testing.T) { + from, to, err := parseUsageRange("", "", now) + if err != nil || from.String() != "2026-10-01" || to.String() != "2026-11-01" { + t.Errorf("default range = %s..%s, %v; want the current month", from, to, err) + } + from, to, err = parseUsageRange("2026-09-01", "2026-09-08", now) + if err != nil || from.String() != "2026-09-01" || to.String() != "2026-09-08" { + t.Errorf("explicit range = %s..%s, %v", from, to, err) + } + for _, bad := range [][2]string{ + {"2026-09-08", "2026-09-08"}, // empty + {"2026-09-08", "2026-09-01"}, // reversed + {"2025-01-01", "2026-01-03"}, // longer than 366 days + {"2026-9-1", "2026-10-01"}, // not YYYY-MM-DD + {"2026-09-01", "tomorrow"}, + } { + if _, _, err := parseUsageRange(bad[0], bad[1], now); err == nil { + t.Errorf("parseUsageRange(%q, %q) accepted", bad[0], bad[1]) + } + } +} + +func TestParseMonth(t *testing.T) { + m, err := ParseMonth("2026-02") + if err != nil { + t.Fatal(err) + } + if got := m.End().Sub(m.Start()); got != 28*24*time.Hour { + t.Errorf("February 2026 spans %s", got) + } + if m.Prev().String() != "2026-01" || MonthOf(now).String() != "2026-10" { + t.Errorf("Prev or MonthOf wrong: %s, %s", m.Prev(), MonthOf(now)) + } + for _, bad := range []string{"", "2026-13", "2026-1", "202602", "2026-02-01"} { + if _, err := ParseMonth(bad); err == nil { + t.Errorf("ParseMonth(%q) accepted", bad) + } + } +} + +func TestInvoiceID(t *testing.T) { + m, _ := ParseMonth("2026-09") + if got := InvoiceID(m, "cus_0042"); got != "inv_202609_cus_0042" { + t.Errorf("InvoiceID = %s", got) + } +} diff --git a/metering-api/internal/pricing/pricing.go b/metering-api/internal/pricing/pricing.go new file mode 100644 index 0000000..21b1976 --- /dev/null +++ b/metering-api/internal/pricing/pricing.go @@ -0,0 +1,107 @@ +// Package pricing rates monthly API usage against graduated price tiers. +// +// A plan is an ordered list of tiers. Calls fill the tiers from the bottom up: +// each tier bills the calls between the previous tier's limit and its own at its +// unit price, plus its flat fee once usage reaches it. Tier 1 is always reached, +// so its flat fee is the plan's monthly fee. +// +// Money is integer: unit prices in micro-dollars per call, amounts in cents. +package pricing + +import ( + "errors" + "fmt" + "math" +) + +// Unlimited is the UpTo of the open-ended top tier. +const Unlimited int64 = math.MaxInt64 + +// microsPerCent converts micro-dollars to cents. +const microsPerCent = 10_000 + +// Tier is one price band. +type Tier struct { + UpTo int64 // last call in this tier, or Unlimited + UnitPriceMicros int64 // price per call in this tier, in micro-dollars + FlatFeeCents int64 // charged once when usage reaches this tier +} + +// Plan is a named list of tiers in ascending order. +type Plan struct { + ID string + Tiers []Tier +} + +// Line is the charge for one tier of one invoice. +type Line struct { + TierNo int + Quantity int64 + UnitPriceMicros int64 + FlatFeeCents int64 + AmountCents int64 +} + +// Validate checks that the tiers are well formed: at least one, strictly +// ascending limits, the last one Unlimited, and no negative prices. +func (p Plan) Validate() error { + if len(p.Tiers) == 0 { + return fmt.Errorf("plan %s: no tiers", p.ID) + } + var prev int64 + for i, t := range p.Tiers { + if t.UnitPriceMicros < 0 || t.FlatFeeCents < 0 { + return fmt.Errorf("plan %s tier %d: negative price", p.ID, i+1) + } + if t.UpTo <= prev { + return fmt.Errorf("plan %s tier %d: limit %d is not above the previous tier's %d", p.ID, i+1, t.UpTo, prev) + } + prev = t.UpTo + } + if prev != Unlimited { + return fmt.Errorf("plan %s: the last tier must be unlimited", p.ID) + } + return nil +} + +// ErrNegativeUsage is returned by Rate for a negative call count. +var ErrNegativeUsage = errors.New("pricing: negative usage") + +// Rate returns the invoice lines for a month with the given number of calls and +// their total in cents. Tiers the usage does not reach produce no line; tier 1 +// always produces one, even for zero calls. +func (p Plan) Rate(calls int64) ([]Line, int64, error) { + if calls < 0 { + return nil, 0, ErrNegativeUsage + } + if err := p.Validate(); err != nil { + return nil, 0, err + } + var ( + lines []Line + total int64 + floor int64 // calls billed by the tiers below + ) + for i, t := range p.Tiers { + if i > 0 && calls <= floor { + break + } + qty := min(calls, t.UpTo) - floor + amount := t.FlatFeeCents + roundHalfUp(qty*t.UnitPriceMicros, microsPerCent) + lines = append(lines, Line{ + TierNo: i + 1, + Quantity: qty, + UnitPriceMicros: t.UnitPriceMicros, + FlatFeeCents: t.FlatFeeCents, + AmountCents: amount, + }) + total += amount + floor = t.UpTo + } + return lines, total, nil +} + +// roundHalfUp divides a non-negative n by d, rounding halves up. +func roundHalfUp(n, d int64) int64 { + return (n + d/2) / d +} diff --git a/metering-api/internal/pricing/pricing_test.go b/metering-api/internal/pricing/pricing_test.go new file mode 100644 index 0000000..c864c5c --- /dev/null +++ b/metering-api/internal/pricing/pricing_test.go @@ -0,0 +1,128 @@ +package pricing + +import ( + "errors" + "slices" + "testing" +) + +// The published price list (init/03_price_plans.sf.sql), restated here. +var ( + free = Plan{ID: "FREE", Tiers: []Tier{ + {UpTo: 10_000}, + {UpTo: Unlimited, UnitPriceMicros: 2000}, // $2.00 per 1,000 + }} + pro = Plan{ID: "PRO", Tiers: []Tier{ + {UpTo: 100_000, FlatFeeCents: 4900}, // $49 incl. 100k + {UpTo: 1_000_000, UnitPriceMicros: 800}, // $0.80 per 1,000 + {UpTo: Unlimited, UnitPriceMicros: 500}, // $0.50 per 1,000 + }} + enterprise = Plan{ID: "ENTERPRISE", Tiers: []Tier{ + {UpTo: 1_000_000, FlatFeeCents: 49900}, // $499 incl. 1M + {UpTo: 10_000_000, UnitPriceMicros: 400}, // $0.40 per 1,000 + {UpTo: Unlimited, UnitPriceMicros: 250}, // $0.25 per 1,000 + }} +) + +func TestRateTotals(t *testing.T) { + tests := []struct { + name string + plan Plan + calls int64 + wantCents int64 + wantTiers []int64 // quantity per billed tier + }{ + {"free, no usage", free, 0, 0, []int64{0}}, + {"free, exactly the allowance", free, 10_000, 0, []int64{10_000}}, + {"free, 2,500 over at $2.00/1k = $5.00", free, 12_500, 500, []int64{10_000, 2_500}}, + {"pro, no usage still pays the $49 fee", pro, 0, 4900, []int64{0}}, + {"pro, exactly the included calls", pro, 100_000, 4900, []int64{100_000}}, + {"pro, 150k at $0.80/1k = $120 + $49", pro, 250_000, 16_900, []int64{100_000, 150_000}}, + {"pro, top of tier 2: 900k at $0.80/1k = $720 + $49", pro, 1_000_000, 76_900, []int64{100_000, 900_000}}, + {"pro, 200k in tier 3 at $0.50/1k adds $100", pro, 1_200_000, 86_900, []int64{100_000, 900_000, 200_000}}, + {"enterprise, 9M at $0.40 + 2M at $0.25 + $499", enterprise, 12_000_000, 459_900, []int64{1_000_000, 9_000_000, 2_000_000}}, + {"pro, 7 calls over = 0.56 cents, rounds up", pro, 100_007, 4901, []int64{100_000, 7}}, + {"pro, 6 calls over = 0.48 cents, rounds down", pro, 100_006, 4900, []int64{100_000, 6}}, + {"free, 1 call over = 0.2 cents, rounds down", free, 10_001, 0, []int64{10_000, 1}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + lines, total, err := tt.plan.Rate(tt.calls) + if err != nil { + t.Fatalf("Rate(%d): %v", tt.calls, err) + } + if total != tt.wantCents { + t.Errorf("total = %d cents, want %d", total, tt.wantCents) + } + var qty []int64 + var sum, calls int64 + for i, l := range lines { + if l.TierNo != i+1 { + t.Errorf("line %d has tier %d", i, l.TierNo) + } + qty = append(qty, l.Quantity) + sum += l.AmountCents + calls += l.Quantity + } + if !slices.Equal(qty, tt.wantTiers) { + t.Errorf("tier quantities = %v, want %v", qty, tt.wantTiers) + } + if sum != total { + t.Errorf("lines sum to %d, total is %d", sum, total) + } + if calls != tt.calls { + t.Errorf("lines bill %d calls, usage was %d", calls, tt.calls) + } + }) + } +} + +func TestRateLineDetails(t *testing.T) { + lines, _, err := pro.Rate(250_000) + if err != nil { + t.Fatal(err) + } + want := []Line{ + {TierNo: 1, Quantity: 100_000, UnitPriceMicros: 0, FlatFeeCents: 4900, AmountCents: 4900}, + {TierNo: 2, Quantity: 150_000, UnitPriceMicros: 800, FlatFeeCents: 0, AmountCents: 12_000}, + } + if !slices.Equal(lines, want) { + t.Errorf("lines = %+v\nwant %+v", lines, want) + } +} + +func TestRateRejectsNegativeUsage(t *testing.T) { + if _, _, err := pro.Rate(-1); !errors.Is(err, ErrNegativeUsage) { + t.Errorf("Rate(-1) error = %v, want ErrNegativeUsage", err) + } +} + +func TestValidate(t *testing.T) { + tests := []struct { + name string + plan Plan + ok bool + }{ + {"published plans are valid", enterprise, true}, + {"single unlimited tier", Plan{ID: "FLAT", Tiers: []Tier{{UpTo: Unlimited, UnitPriceMicros: 100}}}, true}, + {"no tiers", Plan{ID: "EMPTY"}, false}, + {"last tier bounded", Plan{ID: "X", Tiers: []Tier{{UpTo: 10}}}, false}, + {"limits not ascending", Plan{ID: "X", Tiers: []Tier{{UpTo: 10}, {UpTo: 10}, {UpTo: Unlimited}}}, false}, + {"zero first limit", Plan{ID: "X", Tiers: []Tier{{UpTo: 0}, {UpTo: Unlimited}}}, false}, + {"negative unit price", Plan{ID: "X", Tiers: []Tier{{UpTo: Unlimited, UnitPriceMicros: -1}}}, false}, + {"negative flat fee", Plan{ID: "X", Tiers: []Tier{{UpTo: Unlimited, FlatFeeCents: -1}}}, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.plan.Validate() + if (err == nil) != tt.ok { + t.Errorf("Validate() = %v, want ok=%v", err, tt.ok) + } + if !tt.ok { + if _, _, rateErr := tt.plan.Rate(1); rateErr == nil { + t.Error("Rate accepted an invalid plan") + } + } + }) + } +} diff --git a/metering-api/internal/seed/seed.go b/metering-api/internal/seed/seed.go new file mode 100644 index 0000000..1a4b452 --- /dev/null +++ b/metering-api/internal/seed/seed.go @@ -0,0 +1,128 @@ +// Package seed generates a deterministic synthetic dataset: customers on the +// three plans and the usage reports an API gateway would send for them. +// +// Every customer and every (customer, month) has its own random stream derived +// from the seed, so the same customer-month always yields the same events with +// the same ids, whichever months are generated together. Loading a dataset twice +// is therefore a no-op. +package seed + +import ( + "fmt" + "math" + "math/rand/v2" + "time" + + "github.com/localstack-samples/localstack-snowflake-samples/metering-api/internal/metering" +) + +// Options sizes a dataset. +type Options struct { + Seed uint64 + Customers int + Months []metering.Month +} + +// Dataset is the generated data. +type Dataset struct { + Customers []metering.Customer + Events []metering.Event +} + +// profile is how a plan's customers use the API. +type profile struct { + minCalls, maxCalls float64 // monthly calls, drawn log-uniformly + eventsPerDay int // gateway reports per active day + activeShare float64 // probability that a customer is active on a day +} + +var profiles = map[string]profile{ + "FREE": {minCalls: 1_000, maxCalls: 20_000, eventsPerDay: 1, activeShare: 0.7}, + "PRO": {minCalls: 40_000, maxCalls: 1_500_000, eventsPerDay: 2, activeShare: 1}, + "ENTERPRISE": {minCalls: 600_000, maxCalls: 15_000_000, eventsPerDay: 6, activeShare: 1}, +} + +// PlanFor is the plan of the customer at index i: 60% FREE, 30% PRO and 10% +// ENTERPRISE, interleaved so any prefix of the customers has the same mix. +func PlanFor(i int) string { + switch i % 10 { + case 9: + return "ENTERPRISE" + case 6, 7, 8: + return "PRO" + default: + return "FREE" + } +} + +// CustomerID is the id of the customer at index i. +func CustomerID(i int) string { return fmt.Sprintf("cus_%04d", i+1) } + +var ( + adjectives = []string{"Acme", "Blue", "Bright", "Cobalt", "Copper", "Delta", "Ember", "Granite", "Harbor", "Iron", + "Juniper", "Lumen", "Maple", "Nimbus", "North", "Orbit", "Pine", "Quartz", "Silver", "Summit"} + nouns = []string{"Analytics", "Apps", "Cloud", "Commerce", "Data", "Dynamics", "Health", "Labs", "Logistics", + "Media", "Mobility", "Networks", "Payments", "Robotics", "Software", "Systems", "Travel", "Works"} +) + +// signupEpoch is the earliest signup date; customers joined within 18 months of it. +var signupEpoch = time.Date(2024, time.January, 1, 9, 0, 0, 0, time.UTC) + +// Generate builds the dataset. +func Generate(o Options) Dataset { + var d Dataset + for i := range o.Customers { + rng := rand.New(rand.NewPCG(o.Seed, uint64(i))) + c := metering.Customer{ + ID: CustomerID(i), + Name: adjectives[rng.IntN(len(adjectives))] + " " + nouns[rng.IntN(len(nouns))], + PlanID: PlanFor(i), + CreatedAt: signupEpoch.AddDate(0, 0, rng.IntN(540)), + } + d.Customers = append(d.Customers, c) + for _, m := range o.Months { + d.Events = append(d.Events, monthOfEvents(o.Seed, i, c, m)...) + } + } + return d +} + +func monthOfEvents(seed uint64, index int, c metering.Customer, m metering.Month) []metering.Event { + p := profiles[c.PlanID] + monthKey := uint64(m.Start().Year()*100 + int(m.Start().Month())) + rng := rand.New(rand.NewPCG(seed^0x9e3779b97f4a7c15, uint64(index)<<32|monthKey)) + target := math.Exp(math.Log(p.minCalls) + rng.Float64()*(math.Log(p.maxCalls)-math.Log(p.minCalls))) + days := int(m.End().Sub(m.Start()).Hours() / 24) + perEvent := target / (float64(days) * float64(p.eventsPerDay) * p.activeShare) + var events []metering.Event + for day := range days { + if rng.Float64() >= p.activeShare { + continue + } + for slot := range p.eventsPerDay { + // Reports land in their slot of the day, at a random second. + slotLen := 24 * time.Hour / time.Duration(p.eventsPerDay) + at := m.Start().AddDate(0, 0, day).Add(time.Duration(slot)*slotLen + + time.Duration(rng.Int64N(int64(slotLen/time.Second)))*time.Second) + qty := int64(math.Round(perEvent * (0.6 + 0.8*rng.Float64()))) + events = append(events, metering.Event{ + EventID: fmt.Sprintf("evt_%016x", rng.Uint64()), + CustomerID: c.ID, + Quantity: max(qty, 1), + OccurredAt: at, + }) + } + } + return events +} + +// LastCompleteMonths returns the n months before the month of now, oldest first. +func LastCompleteMonths(now time.Time, n int) []metering.Month { + months := make([]metering.Month, n) + m := metering.MonthOf(now) + for i := n - 1; i >= 0; i-- { + m = m.Prev() + months[i] = m + } + return months +} diff --git a/metering-api/internal/seed/seed_test.go b/metering-api/internal/seed/seed_test.go new file mode 100644 index 0000000..dd125cc --- /dev/null +++ b/metering-api/internal/seed/seed_test.go @@ -0,0 +1,87 @@ +package seed + +import ( + "slices" + "testing" + "time" + + "github.com/localstack-samples/localstack-snowflake-samples/metering-api/internal/metering" +) + +func months(t *testing.T, ss ...string) []metering.Month { + t.Helper() + var ms []metering.Month + for _, s := range ss { + m, err := metering.ParseMonth(s) + if err != nil { + t.Fatal(err) + } + ms = append(ms, m) + } + return ms +} + +func TestPlanMix(t *testing.T) { + d := Generate(Options{Seed: 42, Customers: 200, Months: months(t, "2026-09")}) + counts := map[string]int{} + for _, c := range d.Customers { + counts[c.PlanID]++ + } + if counts["FREE"] != 120 || counts["PRO"] != 60 || counts["ENTERPRISE"] != 20 { + t.Errorf("plan mix = %v, want FREE 120, PRO 60, ENTERPRISE 20", counts) + } +} + +func TestEventsAreValidAndInsideTheirMonth(t *testing.T) { + window := months(t, "2026-08", "2026-09") + d := Generate(Options{Seed: 1, Customers: 30, Months: window}) + if errs := metering.ValidateEvents(d.Events[:metering.MaxBatch], time.Now()); len(errs) > 0 { + t.Fatalf("generated invalid events: %+v", errs[0]) + } + seen := map[string]bool{} + for _, e := range d.Events { + if seen[e.EventID] { + t.Fatalf("duplicate event id %s", e.EventID) + } + seen[e.EventID] = true + if e.OccurredAt.Before(window[0].Start()) || !e.OccurredAt.Before(window[1].End()) { + t.Fatalf("event %s at %s is outside %s..%s", e.EventID, e.OccurredAt, window[0], window[1]) + } + if e.Quantity < 1 { + t.Fatalf("event %s has quantity %d", e.EventID, e.Quantity) + } + } +} + +// A customer-month yields the same events whichever window it is generated in, +// so re-running the seed in a later month does not duplicate usage. +func TestCustomerMonthsAreStableAcrossWindows(t *testing.T) { + sept := func(d Dataset) []metering.Event { + return slices.DeleteFunc(slices.Clone(d.Events), func(e metering.Event) bool { + return e.OccurredAt.Month() != time.September + }) + } + a := Generate(Options{Seed: 7, Customers: 20, Months: months(t, "2026-07", "2026-08", "2026-09")}) + b := Generate(Options{Seed: 7, Customers: 20, Months: months(t, "2026-09", "2026-10")}) + if !slices.Equal(sept(a), sept(b)) || len(sept(a)) == 0 { + t.Error("September differs between generation windows") + } + if !slices.Equal(a.Customers, b.Customers) { + t.Error("customers differ between generation windows") + } + c := Generate(Options{Seed: 8, Customers: 20, Months: months(t, "2026-09")}) + if slices.Equal(sept(a), sept(c)) { + t.Error("a different seed produced the same events") + } +} + +func TestLastCompleteMonths(t *testing.T) { + got := LastCompleteMonths(time.Date(2026, 1, 20, 0, 0, 0, 0, time.UTC), 3) + var s []string + for _, m := range got { + s = append(s, m.String()) + } + if want := []string{"2025-10", "2025-11", "2025-12"}; !slices.Equal(s, want) { + t.Errorf("LastCompleteMonths = %v, want %v", s, want) + } +} diff --git a/metering-api/scripts/demo.sh b/metering-api/scripts/demo.sh new file mode 100755 index 0000000..68ed07c --- /dev/null +++ b/metering-api/scripts/demo.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# make demo: run the demo client against the API at HTTP_ADDR. If no API is +# listening there, build one, start it in the background and stop it afterwards. +set -euo pipefail +cd "$(dirname "$0")/.." + +addr="${HTTP_ADDR:-127.0.0.1:8080}" +url="http://${addr}" +healthy() { curl -fsS -o /dev/null "${url}/healthz" 2>/dev/null; } + +if ! healthy; then + mkdir -p bin + go build -o bin/metering-api ./cmd/api + HTTP_ADDR="$addr" LOG_LEVEL=WARN bin/metering-api & + api_pid=$! + trap 'kill "$api_pid" 2>/dev/null; wait "$api_pid" 2>/dev/null || true' EXIT + deadline=$((SECONDS + 60)) + until healthy; do + if (( SECONDS >= deadline )) || ! kill -0 "$api_pid" 2>/dev/null; then + echo "The API did not become healthy at ${url} (is the emulator up? run 'make up')." >&2 + exit 1 + fi + sleep 1 + done +fi + +go run ./cmd/demo -addr "$url" diff --git a/metering-api/scripts/wait-for-emulator.sh b/metering-api/scripts/wait-for-emulator.sh new file mode 100755 index 0000000..eecb632 --- /dev/null +++ b/metering-api/scripts/wait-for-emulator.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# Wait until the Snowflake-Next emulator answers a SQL statement. +# +# Usage: scripts/wait-for-emulator.sh [timeout-seconds] +# Reads SNOWFLAKE_PORT (default 4566). Readiness is checked from the host. +set -euo pipefail + +port="${SNOWFLAKE_PORT:-4566}" +timeout="${1:-120}" +url="https://snowflake.localhost.localstack.cloud:${port}" +deadline=$((SECONDS + timeout)) + +until curl -fsS --compressed -o /dev/null -X POST "${url}/api/v2/statements" \ + -H 'Authorization: Bearer local' -H 'Content-Type: application/json' \ + -H 'Accept: application/json' -d '{"statement":"SELECT 1"}' 2>/dev/null; do + if (( SECONDS >= deadline )); then + echo "Emulator at ${url} did not become ready within ${timeout}s." >&2 + echo "Check 'docker compose logs snowflake' (is LOCALSTACK_AUTH_TOKEN valid? is port ${port} free?)." >&2 + exit 1 + fi + sleep 2 +done +echo "Emulator ready at ${url}" diff --git a/migrate-and-swap/.env.example b/migrate-and-swap/.env.example new file mode 100644 index 0000000..14c274e --- /dev/null +++ b/migrate-and-swap/.env.example @@ -0,0 +1,35 @@ +# Every variable this sample reads, with its default. Copy to .env and +# uncomment what you want to change; the Makefile reads .env. + +# LocalStack auth token. Default: from the environment, else ~/.localstack/default.env. +# Never commit it. +# LOCALSTACK_AUTH_TOKEN= + +# Emulator image and host port. +# SNOWFLAKE_IMAGE=localstack/snowflake-next:latest +# SNOWFLAKE_PORT=4566 + +# EMULATOR or SNOWFLAKE_CLOUD. +# TEST_TARGET=EMULATOR + +# Database and live schema for make seed / demo / release / rollback / status. +# make seed DROPS and recreates the database. The green schema is always +# _GREEN. Tests create their own MAS_TEST_ databases +# and drop them; KEEP_TEST_DB=1 keeps them for debugging. +# INVENTORY_DATABASE=RETAIL +# INVENTORY_SCHEMA=INVENTORY +# KEEP_TEST_DB= + +# pytest-xdist workers for make test. +# PYTEST_WORKERS=4 + +# Key pair used by every connection (generated by make up). +# SNOWFLAKE_PRIVATE_KEY_PATH=.secrets/rsa_key.p8 + +# TEST_TARGET=SNOWFLAKE_CLOUD only. Use a non-production account: the suite +# creates the account-level role INVENTORY_READER and MAS_TEST_ +# databases that it drops afterwards. +# SNOWFLAKE_ACCOUNT= +# SNOWFLAKE_USER= +# SNOWFLAKE_ROLE=ACCOUNTADMIN +# SNOWFLAKE_WAREHOUSE= diff --git a/migrate-and-swap/.python-version b/migrate-and-swap/.python-version new file mode 100644 index 0000000..e4fba21 --- /dev/null +++ b/migrate-and-swap/.python-version @@ -0,0 +1 @@ +3.12 diff --git a/migrate-and-swap/Makefile b/migrate-and-swap/Makefile new file mode 100644 index 0000000..3e6b57c --- /dev/null +++ b/migrate-and-swap/Makefile @@ -0,0 +1,73 @@ +# Migrate and Swap: blue/green schema releases against the Snowflake-Next emulator. +# Run `make help` for the targets. + +-include $(HOME)/.localstack/default.env +export LOCALSTACK_AUTH_TOKEN +SNOWFLAKE_PORT ?= 4566 +export SNOWFLAKE_PORT +COMPOSE := docker compose -p snfs-migrate-and-swap +UV_RUN := uv run --locked +PYTEST_ARGS ?= + +# Optional local overrides (copy .env.example to .env). +-include .env +PYTEST_WORKERS ?= 4 +export TEST_TARGET SNOWFLAKE_IMAGE INVENTORY_DATABASE INVENTORY_SCHEMA KEEP_TEST_DB PYTEST_WORKERS \ + SNOWFLAKE_ACCOUNT SNOWFLAKE_USER SNOWFLAKE_ROLE SNOWFLAKE_WAREHOUSE SNOWFLAKE_PRIVATE_KEY_PATH + +.DEFAULT_GOAL := help +.PHONY: help keys up seed test demo release rollback status down test-cloud install start stop logs + +help: ## List targets + @grep -E '^[a-z-]+:.*## ' $(MAKEFILE_LIST) | awk -F':.*## ' '{printf " %-11s %s\n", $$1, $$2}' + +keys: .secrets/rsa_key.p8 ## Generate the throwaway key pair used for SNOWFLAKE_JWT auth + +# Throwaway, unencrypted dev key pair for local key-pair (SNOWFLAKE_JWT) auth. +.secrets/rsa_key.p8: + @mkdir -p .secrets && chmod 700 .secrets + @openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 -out .secrets/rsa_key.p8 2>/dev/null + @openssl rsa -in .secrets/rsa_key.p8 -pubout -out .secrets/rsa_key.pub 2>/dev/null + @chmod 600 .secrets/rsa_key.p8 + @echo "Generated .secrets/rsa_key.p8 (throwaway, for local use)" + +up: keys ## Pull and start the emulator, install the Python env, wait until SQL answers + $(COMPOSE) pull -q + $(COMPOSE) up -d + uv sync --locked --quiet + ./scripts/wait-for-emulator.sh 180 + +seed: ## (Re)create RETAIL with live INVENTORY at V1.0.0 and the synthetic data set + $(UV_RUN) python scripts/seed.py + +test: ## Run the suite against the running emulator (each worker builds its own databases) + $(UV_RUN) pytest -n $(PYTEST_WORKERS) $(PYTEST_ARGS) + +demo: seed ## Bad release blocked, good release swapped in, rollback, recovery drills + $(UV_RUN) python -m release.demo + +release: ## Release migrations/ to live: clone, migrate green, validate, swap + $(UV_RUN) python -m release release + +rollback: ## Swap the previous version back in + $(UV_RUN) python -m release rollback + +status: ## Show the version of live and green + $(UV_RUN) python -m release status + +down: ## Stop the emulator and remove its volumes + $(COMPOSE) down -v --remove-orphans + +test-cloud: ## Run the same suite against real Snowflake (billable; needs SNOWFLAKE_* env) + TEST_TARGET=SNOWFLAKE_CLOUD $(UV_RUN) pytest -n $(PYTEST_WORKERS) $(PYTEST_ARGS) + +# The targets the repository CI runs for every sample (see the root README) +install: ## Nothing to install up front: make up and make test fetch the dependencies + @echo "Nothing to install, make up and make test fetch the dependencies" + +start: up ## Same as make up + +stop: down ## Same as make down + +logs: ## Tail the emulator log (init hook results included) + $(COMPOSE) logs --tail=200 snowflake diff --git a/migrate-and-swap/README.md b/migrate-and-swap/README.md new file mode 100644 index 0000000..77066d2 --- /dev/null +++ b/migrate-and-swap/README.md @@ -0,0 +1,119 @@ +# Migrate and Swap + +Zero-downtime schema releases for a retail inventory service: migrate a zero-copy clone with schemachange, validate it, swap it in atomically, and roll back with one more swap. Built for data platform engineers who want to rehearse releases against production-shaped data on their laptop or in CI, before anything reaches a live warehouse. + +## Architecture + +```mermaid +flowchart LR + repo["migrations/
V1.0.0 .. V1.2.0, R__inventory_views, A__grants"] + gate["release/validations.sql
(the release gate)"] + tool["release.py
release | rollback | status"] + sc["schemachange 4.4"] + seed["scripts/seed.py
bootstrap V1.0.0 + data"] + subgraph emu["Snowflake-Next emulator"] + direction TB + live["RETAIL.INVENTORY (live)
tables, views, CHANGE_HISTORY"] + green["RETAIL.INVENTORY_GREEN
candidate, then previous version"] + end + readers(["INVENTORY_READER
reports and apps"]) + seed --> live + tool -- "1. CREATE SCHEMA ... CLONE" --> green + tool -- "2. deploy to green" --> sc + repo --> sc --> green + gate --> tool + tool -- "3. checks" --> green + tool -- "4. ALTER SCHEMA ... SWAP WITH" --> live + readers -- "SELECT on views" --> live +``` + +Readers only ever see `RETAIL.INVENTORY`. A release builds the next version beside it in `RETAIL.INVENTORY_GREEN`, from a zero-copy clone of live, so a migration meets production data without touching production. The swap is a single metadata operation, and the old version stays intact in green, which is what makes rollback one more swap. Design notes are in [docs/architecture.md](docs/architecture.md). + +## How it works + +1. **Clone.** `CREATE OR REPLACE SCHEMA INVENTORY_GREEN CLONE INVENTORY` copies metadata, not data, so it takes seconds at any size. +2. **Migrate.** schemachange runs against green. The change history lives inside the schema and is cloned with it, so only the scripts live does not have are applied. A failing script exits 2. +3. **Validate.** Every check in [`release/validations.sql`](release/validations.sql) runs against green: row-count parity, no negative stock, no orphan ledger rows, receipts match purchase orders, no view dropped, every view selectable. One failure exits 1 and live is untouched. +4. **Swap.** `ALTER SCHEMA INVENTORY SWAP WITH INVENTORY_GREEN` exchanges both schemas in one statement. +5. **Roll back.** `rollback` swaps again, but only when green's history is a strict subset of live's, so it can never roll forward. + +`make demo` shows the gate at work: [`demos/V1.3.0__bad_negative_stock.sql`](demos/V1.3.0__bad_negative_stock.sql) looks harmless on staging data but drives nine backordered rows negative on production data. The gate catches it on the clone and the swap never happens. The demo then releases the good migrations, rolls back, and runs two recovery drills: `UNDROP TABLE` for a dropped table, and a clone `BEFORE(STATEMENT => ...)` plus `INSERT OVERWRITE` to undo a bad `UPDATE` in place. + +```mermaid +sequenceDiagram + autonumber + actor Eng as Platform engineer + participant Tool as release.py + participant SC as schemachange + participant Green as INVENTORY_GREEN + participant Live as INVENTORY (live) + Eng->>Tool: python -m release release + Tool->>Live: read CHANGE_HISTORY (live is at 1.0.0) + Tool->>Green: CREATE OR REPLACE SCHEMA ... CLONE INVENTORY + Tool->>SC: deploy, schema = INVENTORY_GREEN + SC->>Green: V1.1.0, V1.2.0, R__ views, A__ grants + Tool->>Green: run every check in validations.sql + alt any check fails + Tool-->>Eng: exit 1, RELEASE BLOCKED (live untouched) + else all checks pass + Tool->>Live: ALTER SCHEMA INVENTORY SWAP WITH INVENTORY_GREEN + Tool-->>Eng: exit 0, RELEASED 1.2.0 + end + Eng->>Tool: python -m release rollback + Tool->>Live: ALTER SCHEMA INVENTORY SWAP WITH INVENTORY_GREEN + Tool-->>Eng: exit 0, live is 1.0.0 again +``` + +## Prerequisites + +- Docker with Compose v2 +- [`uv`](https://docs.astral.sh/uv/) 0.5 or newer (installs Python 3.12 if needed) +- `openssl` and `curl` +- A LocalStack auth token in `LOCALSTACK_AUTH_TOKEN` or `~/.localstack/default.env` + +## Quickstart + +```sh +cd migrate-and-swap +make up # generate a dev key pair, start the emulator, install the Python env +make seed # (re)create RETAIL with live INVENTORY at V1.0.0 and the data set +make demo # blocked release, good release, rollback, two recovery drills +make test # the full suite +make down # stop the emulator and remove its volumes +``` + +To drive releases by hand after `make seed`, use `make release`, `make status` and `make rollback`. If port 4566 is taken, set another one: `SNOWFLAKE_PORT=14566 make up test down`. + +## Tests + +Every test seeds its own `MAS_TEST_` database with the real baseline migration and drops it afterwards, so the suite runs in parallel. Expected values come from the generated data set and the rules written in the migrations. The suite covers: + +- a release of V1.1.0 and V1.2.0 that swaps in, with every backfilled value and view row checked; +- the bad migration blocked by the gate, with live's row counts, history, columns and views unchanged; +- rollback to the exact previous snapshot, and a refused second rollback; +- each gate check failing on the defect it exists for; +- readers keeping `USAGE` and `SELECT` across the swap; +- the `UNDROP` and point-in-time restore runbooks, and their error paths. + +`make test-cloud` runs the same suite against a real Snowflake account (billable). Set `SNOWFLAKE_ACCOUNT`, `SNOWFLAKE_USER`, `SNOWFLAKE_WAREHOUSE` and `SNOWFLAKE_PRIVATE_KEY_PATH` (an unencrypted PKCS#8 key), and optionally `SNOWFLAKE_ROLE` (default `ACCOUNTADMIN`). Use a non-production account. + +## Project structure + +``` +migrate-and-swap/ + migrations/ V1.0.0 .. V1.2.0, R__inventory_views.sql, A__grants.sql + demos/ the bad migration the gate must stop + release/ release tool: clone, migrate, validate, swap, rollback, recovery + validations.sql the release gate + scripts/ seed.py, wait-for-emulator.sh + init/ emulator init hook: warehouse, service user, reader role + tests/ pytest suite + docs/architecture.md design notes +``` + +## Learn more + +- [Cloning considerations](https://docs.snowflake.com/en/user-guide/object-clone) and [`CREATE ... CLONE`](https://docs.snowflake.com/en/sql-reference/sql/create-clone) +- [`ALTER SCHEMA ... SWAP WITH`](https://docs.snowflake.com/en/sql-reference/sql/alter-schema) +- [Understanding and using Time Travel](https://docs.snowflake.com/en/user-guide/data-time-travel) and [`UNDROP TABLE`](https://docs.snowflake.com/en/sql-reference/sql/undrop-table) +- [schemachange](https://github.com/Snowflake-Labs/schemachange) diff --git a/migrate-and-swap/demos/V1.3.0__bad_negative_stock.sql b/migrate-and-swap/demos/V1.3.0__bad_negative_stock.sql new file mode 100644 index 0000000..43b6eb6 --- /dev/null +++ b/migrate-and-swap/demos/V1.3.0__bad_negative_stock.sql @@ -0,0 +1,15 @@ +-- V1.3.0 (deliberately bad, for the demo): "report sellable stock". +-- +-- The intent: ON_HAND should mean units a shopper can buy, so subtract the +-- units already allocated to open online orders and clear the allocation. +-- It passes review and passes on a staging data set where every store has +-- more stock than allocations. On production data it does not: a store that +-- sold into backorder has ALLOCATED > ON_HAND, and this statement drives +-- ON_HAND negative for exactly those rows. +-- +-- The release gate's no_negative_on_hand check catches it on the green clone, +-- and the swap never happens. + +UPDATE {{ database }}.{{ schema }}.STOCK_LEVELS +SET ON_HAND = ON_HAND - ALLOCATED, + ALLOCATED = 0; diff --git a/migrate-and-swap/docker-compose.yml b/migrate-and-swap/docker-compose.yml new file mode 100644 index 0000000..b12255e --- /dev/null +++ b/migrate-and-swap/docker-compose.yml @@ -0,0 +1,18 @@ +# Snowflake-Next emulator for the Migrate and Swap sample. +# Host port is ${SNOWFLAKE_PORT:-4566}, bound to 127.0.0.1 only, so it is reachable from this machine alone. +services: + snowflake: + image: ${SNOWFLAKE_IMAGE:-localstack/snowflake-next:latest} + pull_policy: always + ports: + - "127.0.0.1:${SNOWFLAKE_PORT:-4566}:4566" + environment: + LOCALSTACK_AUTH_TOKEN: ${LOCALSTACK_AUTH_TOKEN:?set LOCALSTACK_AUTH_TOKEN} + volumes: + - ./init:/etc/localstack/init/ready.d:ro + networks: + default: + aliases: + - snowflake.localhost.localstack.cloud + - stages.s3.localhost.localstack.cloud + - stages-enc.s3.localhost.localstack.cloud diff --git a/migrate-and-swap/docs/architecture.md b/migrate-and-swap/docs/architecture.md new file mode 100644 index 0000000..c00792d --- /dev/null +++ b/migrate-and-swap/docs/architecture.md @@ -0,0 +1,95 @@ +# Architecture: Migrate and Swap + +Design notes for the release tool: the data model, the main design decisions, and what a swap does not cover. The README covers how to run it. + +## Data model + +```mermaid +erDiagram + STORES ||--o{ STOCK_LEVELS : "stocks" + PRODUCTS ||--o{ STOCK_LEVELS : "is stocked as" + STOCK_LEVELS ||--o{ STOCK_MOVEMENTS : "ledger of" + STOCK_LEVELS ||--o{ PURCHASE_ORDERS : "replenished by" + PURCHASE_ORDERS ||--o{ STOCK_MOVEMENTS : "received as" + STORES { + varchar STORE_ID PK + varchar STORE_NAME + varchar REGION + date OPENED_ON + } + PRODUCTS { + varchar SKU PK + varchar PRODUCT_NAME + varchar CATEGORY + number UNIT_COST + } + STOCK_LEVELS { + varchar STORE_ID PK + varchar SKU PK + number ON_HAND + number ALLOCATED + varchar LOCATION "V1.0.0 only" + number REORDER_POINT "V1.1.0" + varchar AISLE "V1.2.0" + varchar BAY "V1.2.0" + } + PURCHASE_ORDERS { + varchar PO_ID PK + varchar STORE_ID FK + varchar SKU FK + number ORDERED_QTY + varchar STATUS + } + STOCK_MOVEMENTS { + number MOVEMENT_ID PK + varchar STORE_ID FK + varchar SKU FK + varchar MOVEMENT_TYPE + number QUANTITY + varchar PO_ID FK + } +``` + +`STOCK_LEVELS` is the current snapshot per store and SKU; `STOCK_MOVEMENTS` is the signed ledger. `ALLOCATED` may exceed `ON_HAND` when a store sells into backorder. Keys are informational in Snowflake, so the release gate is what checks referential integrity. The data set ([`release/dataset.py`](../release/dataset.py)) is generated from a fixed seed and plants nine backordered rows and some unnormalised shelf addresses for the migrations to meet. + +## Design decisions + +**Migrate a clone, not live.** schemachange has no rollback and DDL commits implicitly, so a migration that fails halfway, or succeeds but is wrong, would leave live in an untested state. A zero-copy clone gives the migration real data at no risk, a gate before exposure, and an instant rollback target. + +**The change history lives inside the schema.** `.CHANGE_HISTORY` is cloned and swapped with the schema it describes, so each side always knows its own version. `rollback` swaps only when green's history is a strict subset of live's; otherwise it would be a roll-forward and is refused. + +**Swap, then keep the previous version.** `ALTER SCHEMA ... SWAP WITH` renames both schemas in one metadata operation, so there is no moment when `RETAIL.INVENTORY` points at nothing. + +```mermaid +stateDiagram-v2 + direction LR + [*] --> Absent: make seed + Absent --> Candidate: release clones live and migrates green + Candidate --> Candidate: gate fails, exit 1 (live untouched) + Candidate --> Previous: gate passes, swap + Previous --> Candidate: rollback (swap again) + Previous --> Candidate: next release (green replaced by a fresh clone) + Candidate --> Candidate: rollback refused, exit 3 +``` + +The diagram follows what the green schema holds. *Previous* is the only state in which `rollback` swaps. + +**Views are unqualified and owned by the repeatable script.** View bodies name tables without a schema, so the same view reads green's tables in green and live's tables after the swap. Views are dropped before structural changes and rebuilt by `R__inventory_views.sql`, so every release leaves them consistent. + +**Grants are an always-script.** `A__grants.sql` grants `USAGE` on the schema and `SELECT` on all its views to `INVENTORY_READER` on every deploy, so the schema that is swapped in always carries the reader grants. The init hook creates the reader role up front, so it exists before the first deploy. + +**The gate is SQL, in one file.** Each check in [`release/validations.sql`](../release/validations.sql) is a query whose rows are violations. Checks guard invariants every release must keep (row counts, physical stock, referential integrity, view availability). `view_selectable` counts each whole view, so the engine expands and executes it end to end. + +## What a swap does not do + +- **It does not merge writes.** Writes to live between the clone and the swap would be lost, so `row_count_parity` fails the release if any count moved. A rollback is point-in-time; `rollback` prints a note per table whose row counts differ. +- **It does not recover deleted data.** That is time travel's job ([`release/recovery.py`](../release/recovery.py)): `UNDROP TABLE` for a dropped table, and for a bad statement a clone `BEFORE(STATEMENT => ...)` restored with `INSERT OVERWRITE`, which keeps the table's identity, grants and dependent views. Both work inside the retention period. + +## Alternatives considered + +| Alternative | Why not here | +| --- | --- | +| Migrate live in place with down-migrations | DDL autocommits, and down-migrations cannot restore data a destructive step dropped | +| Clone and swap the whole database | Right when a release spans schemas; a schema is the smallest unit that holds this service | +| Rename instead of swap | Two statements, with a window in which `INVENTORY` does not exist | +| A metadata database for the change history | The history would no longer move with the schema on a swap or a rollback | diff --git a/migrate-and-swap/init/01_bootstrap.sf.sql b/migrate-and-swap/init/01_bootstrap.sf.sql new file mode 100644 index 0000000..159e56a --- /dev/null +++ b/migrate-and-swap/init/01_bootstrap.sf.sql @@ -0,0 +1,24 @@ +-- Runs once when the emulator starts, in account SNOWFLAKE as role PUBLIC. +-- Everything inside the inventory database (schema, tables, views, grants) is +-- owned by the schemachange migrations in migrations/. This file only provides +-- what a fresh account needs before the first deploy: a warehouse and the +-- release service user, and the reader role the migrations grant to. +USE ROLE ACCOUNTADMIN; + +CREATE WAREHOUSE IF NOT EXISTS RELEASE_WH + WAREHOUSE_SIZE = XSMALL + AUTO_SUSPEND = 60 + AUTO_RESUME = TRUE + INITIALLY_SUSPENDED = TRUE; + +CREATE USER IF NOT EXISTS SAMPLE_SVC + TYPE = SERVICE + DEFAULT_ROLE = ACCOUNTADMIN + DEFAULT_WAREHOUSE = RELEASE_WH + COMMENT = 'release service user for the Migrate and Swap sample'; +GRANT ROLE ACCOUNTADMIN TO USER SAMPLE_SVC; + +-- The account-level role the migrations grant to. V1.0.0 creates it too (IF +-- NOT EXISTS); creating it here means it exists before the first deploy runs. +CREATE ROLE IF NOT EXISTS INVENTORY_READER; +GRANT ROLE INVENTORY_READER TO ROLE SYSADMIN; diff --git a/migrate-and-swap/migrations/A__grants.sql b/migrate-and-swap/migrations/A__grants.sql new file mode 100644 index 0000000..ac153a2 --- /dev/null +++ b/migrate-and-swap/migrations/A__grants.sql @@ -0,0 +1,10 @@ +-- Re-applied on every deploy (schemachange "always" script, runs last). +-- +-- Grants are why this script exists: CREATE SCHEMA ... CLONE does not copy +-- grants on the schema itself, and ALTER SCHEMA ... SWAP WITH keeps each +-- schema's grants with the schema object. Without this script the swapped-in +-- schema would come up without USAGE for INVENTORY_READER and every report +-- would fail at the moment of the release. + +GRANT USAGE ON SCHEMA {{ database }}.{{ schema }} TO ROLE INVENTORY_READER; +GRANT SELECT ON ALL VIEWS IN SCHEMA {{ database }}.{{ schema }} TO ROLE INVENTORY_READER; diff --git a/migrate-and-swap/migrations/R__inventory_views.sql b/migrate-and-swap/migrations/R__inventory_views.sql new file mode 100644 index 0000000..e588b9e --- /dev/null +++ b/migrate-and-swap/migrations/R__inventory_views.sql @@ -0,0 +1,42 @@ +-- The reporting views, owned by this repeatable script. schemachange re-runs it +-- whenever its content changes, after all versioned scripts of the release. +-- +-- View bodies reference tables WITHOUT a database or schema. Snowflake +-- resolves unqualified names in a view against the view's own schema, so the +-- same view reads green's tables while it lives in green and live's tables +-- after the swap. A fully qualified body (FROM RETAIL.INVENTORY.STOCK_LEVELS) +-- would keep pointing at whatever schema carries that name, which after a +-- swap is the other side. + +CREATE OR REPLACE VIEW {{ database }}.{{ schema }}.V_STOCK_POSITION AS +SELECT + s.STORE_ID, + st.STORE_NAME, + s.SKU, + p.PRODUCT_NAME, + s.ON_HAND, + s.ALLOCATED, + s.ON_HAND - s.ALLOCATED AS AVAILABLE, + s.REORDER_POINT, + s.AISLE, + s.BAY +FROM STOCK_LEVELS s +JOIN STORES st ON st.STORE_ID = s.STORE_ID +JOIN PRODUCTS p ON p.SKU = s.SKU; + +-- Rows at or below their reorder point with nothing already on order. +CREATE OR REPLACE VIEW {{ database }}.{{ schema }}.V_REORDER_CANDIDATES AS +SELECT s.STORE_ID, s.SKU, s.ON_HAND - s.ALLOCATED AS AVAILABLE, s.REORDER_POINT, s.AISLE, s.BAY +FROM STOCK_LEVELS s +WHERE s.ON_HAND - s.ALLOCATED <= s.REORDER_POINT + AND s.REORDER_POINT > 0 + AND NOT EXISTS ( + SELECT 1 FROM PURCHASE_ORDERS po + WHERE po.STORE_ID = s.STORE_ID AND po.SKU = s.SKU AND po.STATUS = 'OPEN' + ); + +CREATE OR REPLACE VIEW {{ database }}.{{ schema }}.V_OPEN_PURCHASE_ORDERS AS +SELECT po.PO_ID, po.STORE_ID, po.SKU, p.PRODUCT_NAME, po.ORDERED_QTY, po.ORDERED_AT, po.EXPECTED_ON +FROM PURCHASE_ORDERS po +JOIN PRODUCTS p ON p.SKU = po.SKU +WHERE po.STATUS = 'OPEN'; diff --git a/migrate-and-swap/migrations/V1.0.0__baseline.sql b/migrate-and-swap/migrations/V1.0.0__baseline.sql new file mode 100644 index 0000000..64f8d47 --- /dev/null +++ b/migrate-and-swap/migrations/V1.0.0__baseline.sql @@ -0,0 +1,99 @@ +-- V1.0.0: the inventory schema as it stood in production when the team adopted +-- schemachange (tables, the reporting views of the time, and the reader role). +-- +-- Every object is addressed as {{ database }}.{{ schema }}.: release.py +-- renders {{ schema }} as the green candidate on a release and as the live +-- schema on the one-off bootstrap (make seed). View bodies use unqualified +-- names on purpose, so a view resolves tables in its own schema after a clone +-- or a swap (see R__inventory_views.sql). +-- +-- Constraints are informational in Snowflake (only NOT NULL is enforced); they +-- document the model for people and BI tools. The release gate +-- (release/validations.sql) is what actually checks referential integrity. + +CREATE ROLE IF NOT EXISTS INVENTORY_READER; +GRANT ROLE INVENTORY_READER TO ROLE SYSADMIN; +GRANT USAGE ON DATABASE {{ database }} TO ROLE INVENTORY_READER; + +CREATE TABLE {{ database }}.{{ schema }}.STORES ( + STORE_ID VARCHAR(8) NOT NULL, + STORE_NAME VARCHAR(100) NOT NULL, + REGION VARCHAR(16) NOT NULL, + OPENED_ON DATE NOT NULL, + CONSTRAINT PK_STORES PRIMARY KEY (STORE_ID) +); + +CREATE TABLE {{ database }}.{{ schema }}.PRODUCTS ( + SKU VARCHAR(16) NOT NULL, + PRODUCT_NAME VARCHAR(200) NOT NULL, + CATEGORY VARCHAR(32) NOT NULL, + UNIT_COST NUMBER(10, 2) NOT NULL, + CONSTRAINT PK_PRODUCTS PRIMARY KEY (SKU) +); + +-- One row per store and SKU. LOCATION is the shelf address printed on the +-- label, '-' (for example 'A07-B12'); NULL while unassigned. +-- ALLOCATED counts units promised to open online orders. It may exceed +-- ON_HAND: the store sells into backorder and fills it from the next delivery. +CREATE TABLE {{ database }}.{{ schema }}.STOCK_LEVELS ( + STORE_ID VARCHAR(8) NOT NULL, + SKU VARCHAR(16) NOT NULL, + ON_HAND NUMBER(10, 0) NOT NULL, + ALLOCATED NUMBER(10, 0) NOT NULL DEFAULT 0, + LOCATION VARCHAR(16), + UPDATED_AT TIMESTAMP_NTZ NOT NULL, + CONSTRAINT PK_STOCK_LEVELS PRIMARY KEY (STORE_ID, SKU), + CONSTRAINT FK_STOCK_LEVELS_STORE FOREIGN KEY (STORE_ID) REFERENCES {{ database }}.{{ schema }}.STORES (STORE_ID), + CONSTRAINT FK_STOCK_LEVELS_PRODUCT FOREIGN KEY (SKU) REFERENCES {{ database }}.{{ schema }}.PRODUCTS (SKU) +); + +CREATE TABLE {{ database }}.{{ schema }}.PURCHASE_ORDERS ( + PO_ID VARCHAR(12) NOT NULL, + STORE_ID VARCHAR(8) NOT NULL, + SKU VARCHAR(16) NOT NULL, + ORDERED_QTY NUMBER(10, 0) NOT NULL, + STATUS VARCHAR(10) NOT NULL, -- OPEN | RECEIVED | CANCELLED + ORDERED_AT TIMESTAMP_NTZ NOT NULL, + EXPECTED_ON DATE NOT NULL, + CONSTRAINT PK_PURCHASE_ORDERS PRIMARY KEY (PO_ID), + CONSTRAINT FK_PURCHASE_ORDERS_STOCK FOREIGN KEY (STORE_ID, SKU) + REFERENCES {{ database }}.{{ schema }}.STOCK_LEVELS (STORE_ID, SKU) +); + +-- The stock ledger. QUANTITY is signed: SALE < 0, RECEIPT > 0 (with the PO it +-- received), ADJUSTMENT either way (counts, damage, shrink). +CREATE TABLE {{ database }}.{{ schema }}.STOCK_MOVEMENTS ( + MOVEMENT_ID NUMBER(12, 0) NOT NULL, + STORE_ID VARCHAR(8) NOT NULL, + SKU VARCHAR(16) NOT NULL, + MOVEMENT_TYPE VARCHAR(12) NOT NULL, -- SALE | RECEIPT | ADJUSTMENT + QUANTITY NUMBER(10, 0) NOT NULL, + PO_ID VARCHAR(12), + MOVED_AT TIMESTAMP_NTZ NOT NULL, + CONSTRAINT PK_STOCK_MOVEMENTS PRIMARY KEY (MOVEMENT_ID), + CONSTRAINT FK_STOCK_MOVEMENTS_STOCK FOREIGN KEY (STORE_ID, SKU) + REFERENCES {{ database }}.{{ schema }}.STOCK_LEVELS (STORE_ID, SKU), + CONSTRAINT FK_STOCK_MOVEMENTS_PO FOREIGN KEY (PO_ID) REFERENCES {{ database }}.{{ schema }}.PURCHASE_ORDERS (PO_ID) +); + +-- The reporting views of the time. R__inventory_views.sql has owned the views +-- since the team adopted schemachange. +CREATE VIEW {{ database }}.{{ schema }}.V_STOCK_POSITION AS +SELECT + s.STORE_ID, + st.STORE_NAME, + s.SKU, + p.PRODUCT_NAME, + s.ON_HAND, + s.ALLOCATED, + s.ON_HAND - s.ALLOCATED AS AVAILABLE, + s.LOCATION +FROM STOCK_LEVELS s +JOIN STORES st ON st.STORE_ID = s.STORE_ID +JOIN PRODUCTS p ON p.SKU = s.SKU; + +CREATE VIEW {{ database }}.{{ schema }}.V_OPEN_PURCHASE_ORDERS AS +SELECT po.PO_ID, po.STORE_ID, po.SKU, p.PRODUCT_NAME, po.ORDERED_QTY, po.ORDERED_AT, po.EXPECTED_ON +FROM PURCHASE_ORDERS po +JOIN PRODUCTS p ON p.SKU = po.SKU +WHERE po.STATUS = 'OPEN'; diff --git a/migrate-and-swap/migrations/V1.1.0__add_reorder_point.sql b/migrate-and-swap/migrations/V1.1.0__add_reorder_point.sql new file mode 100644 index 0000000..b5f59ad --- /dev/null +++ b/migrate-and-swap/migrations/V1.1.0__add_reorder_point.sql @@ -0,0 +1,24 @@ +-- V1.1.0: a reorder point per store and SKU, so replenishment can flag low stock. +-- +-- Rule: seven days of cover at the store's recent sales rate, that is +-- REORDER_POINT = CEIL(units sold in the 28 days up to the latest movement / 4). +-- The window is anchored on the newest movement rather than CURRENT_DATE so a +-- re-run (or a rehearsal on a clone weeks later) computes the same values. +-- Rows with no sales in the window get 0. + +ALTER TABLE {{ database }}.{{ schema }}.STOCK_LEVELS ADD COLUMN REORDER_POINT NUMBER(10, 0); + +UPDATE {{ database }}.{{ schema }}.STOCK_LEVELS SET REORDER_POINT = 0; + +UPDATE {{ database }}.{{ schema }}.STOCK_LEVELS s +SET REORDER_POINT = sales.REORDER_POINT +FROM ( + SELECT m.STORE_ID, m.SKU, CEIL(SUM(-m.QUANTITY) / 4) AS REORDER_POINT + FROM {{ database }}.{{ schema }}.STOCK_MOVEMENTS m + WHERE m.MOVEMENT_TYPE = 'SALE' + AND m.MOVED_AT > (SELECT DATEADD(day, -28, MAX(MOVED_AT)) FROM {{ database }}.{{ schema }}.STOCK_MOVEMENTS) + GROUP BY m.STORE_ID, m.SKU +) sales +WHERE s.STORE_ID = sales.STORE_ID AND s.SKU = sales.SKU; + +ALTER TABLE {{ database }}.{{ schema }}.STOCK_LEVELS ALTER COLUMN REORDER_POINT SET NOT NULL; diff --git a/migrate-and-swap/migrations/V1.2.0__split_location.sql b/migrate-and-swap/migrations/V1.2.0__split_location.sql new file mode 100644 index 0000000..b4afc8e --- /dev/null +++ b/migrate-and-swap/migrations/V1.2.0__split_location.sql @@ -0,0 +1,27 @@ +-- V1.2.0: split the shelf address LOCATION ('A07-B12') into AISLE and BAY, so +-- pick lists can sort by aisle. Older handheld scanners stored some addresses +-- in lower case or with stray spaces ('a07-b12 '): the backfill normalises them. +-- Unassigned rows (LOCATION IS NULL) keep NULL aisle and bay. +-- +-- This is a destructive change: LOCATION is dropped once the backfill is done. +-- On the green clone that is safe to rehearse; done in place on live it could +-- only be undone with time travel. +-- +-- Views are dropped before structural changes and rebuilt by the repeatable +-- script, so every release leaves them consistent: V_STOCK_POSITION reads +-- LOCATION, so it is dropped first and recreated in its new shape by +-- R__inventory_views.sql in the same release (schemachange runs repeatable +-- scripts after all versioned ones). On green, nobody queries the view in +-- between. + +DROP VIEW IF EXISTS {{ database }}.{{ schema }}.V_STOCK_POSITION; + +ALTER TABLE {{ database }}.{{ schema }}.STOCK_LEVELS ADD COLUMN AISLE VARCHAR(4); +ALTER TABLE {{ database }}.{{ schema }}.STOCK_LEVELS ADD COLUMN BAY VARCHAR(4); + +UPDATE {{ database }}.{{ schema }}.STOCK_LEVELS +SET AISLE = SPLIT_PART(UPPER(TRIM(LOCATION)), '-', 1), + BAY = SPLIT_PART(UPPER(TRIM(LOCATION)), '-', 2) +WHERE LOCATION IS NOT NULL; + +ALTER TABLE {{ database }}.{{ schema }}.STOCK_LEVELS DROP COLUMN LOCATION; diff --git a/migrate-and-swap/pyproject.toml b/migrate-and-swap/pyproject.toml new file mode 100644 index 0000000..7d819f6 --- /dev/null +++ b/migrate-and-swap/pyproject.toml @@ -0,0 +1,44 @@ +[project] +name = "migrate-and-swap" +version = "0.1.0" +description = "Zero-downtime schema releases with schemachange: migrate a clone, validate it, swap it in, roll back with one more swap." +readme = "README.md" +license = "Apache-2.0" +requires-python = ">=3.12,<3.13" +dependencies = [ + "schemachange==4.4.0", + # schemachange 4.4.0 requires snowflake-connector-python<5.0. + "snowflake-connector-python>=4.0,<5.0", + "cryptography>=44", + "jinja2>=3.1", +] + +[dependency-groups] +dev = [ + "pytest>=8.3", + "pytest-xdist>=3.6", +] + +[tool.uv] +package = true + +[build-system] +requires = ["hatchling>=1.25"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["release"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +addopts = "-ra --strict-markers" +filterwarnings = [ + # Raised inside the connector's vendored urllib3, not by this sample. + "ignore:X509.get_subject is deprecated:DeprecationWarning", +] + +[tool.ruff] +line-length = 120 + +[tool.ruff.lint] +select = ["E", "F", "W", "I", "B", "UP"] diff --git a/migrate-and-swap/release/__init__.py b/migrate-and-swap/release/__init__.py new file mode 100644 index 0000000..65ee805 --- /dev/null +++ b/migrate-and-swap/release/__init__.py @@ -0,0 +1 @@ +"""Blue/green schema releases for the inventory service: clone, migrate, validate, swap.""" diff --git a/migrate-and-swap/release/__main__.py b/migrate-and-swap/release/__main__.py new file mode 100644 index 0000000..9415faa --- /dev/null +++ b/migrate-and-swap/release/__main__.py @@ -0,0 +1,5 @@ +import sys + +from release.release import main + +sys.exit(main()) diff --git a/migrate-and-swap/release/dataset.py b/migrate-and-swap/release/dataset.py new file mode 100644 index 0000000..4e57ed2 --- /dev/null +++ b/migrate-and-swap/release/dataset.py @@ -0,0 +1,192 @@ +"""A deterministic, synthetic store-by-SKU inventory data set. + +Six stores, 40 SKUs, one stock row per store and SKU, about 120 purchase orders +and a 35-day stock ledger. Everything comes from one seeded ``random.Random``, +so every run (and every test session) loads byte-identical data, and tests can +compute expected values from these rows instead of reading them back. + +Two traits matter for the demo and are planted on purpose: + +* ``BACKORDERED`` stock rows have ``ALLOCATED > ON_HAND`` (the store sold into + backorder). The bad migration ``V1.3.0`` turns exactly these rows negative. +* Some legacy shelf addresses are lower case or carry stray spaces + (``' a07-b12'``), so the ``V1.2.0`` backfill has real normalising to do. +""" + +from __future__ import annotations + +import random +from dataclasses import dataclass +from datetime import date, datetime, timedelta + +from snowflake.connector import SnowflakeConnection + +from release import settings as cfg +from release.deploy import deploy, staged +from release.settings import Settings + +SEED = 20261024 +LEDGER_START = datetime(2026, 9, 1, 8, 0, 0) +LEDGER_DAYS = 35 +BACKORDERED = 9 + +STORES = [ + ("S01", "Portland Pearl District", "WEST", date(2014, 3, 1)), + ("S02", "Seattle Capitol Hill", "WEST", date(2016, 5, 14)), + ("S03", "Denver Union Station", "CENTRAL", date(2017, 9, 2)), + ("S04", "Chicago Wicker Park", "CENTRAL", date(2015, 11, 20)), + ("S05", "Brooklyn Williamsburg", "EAST", date(2018, 4, 7)), + ("S06", "Boston Back Bay", "EAST", date(2019, 8, 30)), +] + +CATALOG = { + "PANTRY": ["Arborio Rice 1kg", "Rolled Oats 750g", "Basmati Rice 2kg", "Red Lentils 500g", + "Penne Rigate 500g", "Chickpeas 400g", "Coconut Milk 400ml", "Tomato Passata 700g"], + "BEVERAGE": ["Cold Brew Coffee 1l", "Sparkling Water 12x", "Green Tea 40ct", "Orange Juice 1.5l", + "Oat Milk 1l", "Kombucha Ginger 330ml", "Espresso Beans 1kg", "Mineral Water 6x"], + "HOUSEHOLD": ["Dish Soap 750ml", "Laundry Pods 32ct", "Paper Towels 6x", "Trash Bags 40ct", + "Sponges 4x", "Glass Cleaner 500ml", "Toilet Paper 12x", "Aluminium Foil 30m"], + "PERSONAL": ["Toothpaste 100ml", "Shampoo 400ml", "Body Wash 500ml", "Sunscreen SPF50", + "Hand Soap 300ml", "Deodorant 50ml", "Cotton Pads 80ct", "Lip Balm 4g"], + "SNACKS": ["Sea Salt Crisps 150g", "Dark Chocolate 100g", "Trail Mix 400g", "Rice Cakes 130g", + "Salted Almonds 200g", "Granola Bars 6x", "Popcorn 90g", "Pretzel Twists 250g"], +} # fmt: skip + + +@dataclass(frozen=True) +class Dataset: + stores: list[tuple] + products: list[tuple] + stock_levels: list[dict] + purchase_orders: list[dict] + stock_movements: list[dict] + + def counts(self) -> dict[str, int]: + return { + "STORES": len(self.stores), + "PRODUCTS": len(self.products), + "STOCK_LEVELS": len(self.stock_levels), + "PURCHASE_ORDERS": len(self.purchase_orders), + "STOCK_MOVEMENTS": len(self.stock_movements), + } + + +def _location(rng: random.Random) -> str | None: + roll = rng.random() + if roll < 0.05: + return None + address = f"A{rng.randint(1, 14):02d}-B{rng.randint(1, 20):02d}" + if roll < 0.10: + return address.lower() + if roll < 0.14: + return f" {address} " + return address + + +def generate() -> Dataset: + rng = random.Random(SEED) + products = [] + for category_no, (category, names) in enumerate(CATALOG.items()): + for i, name in enumerate(names): + sku = f"SKU-{10001 + category_no * 100 + i}" + products.append((sku, name, category, round(rng.uniform(0.8, 24.0), 2))) + + pairs = [(store[0], product[0]) for store in STORES for product in products] + backordered = set(rng.sample(pairs, BACKORDERED)) + stock_levels = [] + for store_id, sku in pairs: + if (store_id, sku) in backordered: + on_hand = rng.randint(0, 3) + allocated = on_hand + rng.randint(1, 6) + else: + on_hand = rng.randint(4, 120) + allocated = rng.randint(0, min(12, on_hand)) + stock_levels.append( + { + "STORE_ID": store_id, + "SKU": sku, + "ON_HAND": on_hand, + "ALLOCATED": allocated, + "LOCATION": _location(rng), + "UPDATED_AT": LEDGER_START + timedelta(days=LEDGER_DAYS, minutes=rng.randint(0, 600)), + } + ) + + purchase_orders = [] + movements = [] + for n in range(1, 121): + store_id, sku = rng.choice(pairs) + ordered_at = LEDGER_START + timedelta(days=rng.randint(0, LEDGER_DAYS - 8), minutes=rng.randint(0, 600)) + status = rng.choices(["RECEIVED", "OPEN", "CANCELLED"], weights=[6, 3, 1])[0] + po = { + "PO_ID": f"PO-{n:06d}", + "STORE_ID": store_id, + "SKU": sku, + "ORDERED_QTY": rng.choice([12, 24, 36, 48, 60]), + "STATUS": status, + "ORDERED_AT": ordered_at, + "EXPECTED_ON": (ordered_at + timedelta(days=rng.randint(4, 7))).date(), + } + purchase_orders.append(po) + if status == "RECEIVED": + received_at = ordered_at + timedelta(days=rng.randint(3, 6)) + movements.append((store_id, sku, "RECEIPT", po["ORDERED_QTY"], po["PO_ID"], received_at)) + + for _ in range(1400): + store_id, sku = rng.choice(pairs) + moved_at = LEDGER_START + timedelta(days=rng.randint(0, LEDGER_DAYS - 1), minutes=rng.randint(0, 780)) + movements.append((store_id, sku, "SALE", -rng.randint(1, 6), None, moved_at)) + for _ in range(60): + store_id, sku = rng.choice(pairs) + moved_at = LEDGER_START + timedelta(days=rng.randint(0, LEDGER_DAYS - 1), minutes=rng.randint(0, 780)) + movements.append((store_id, sku, "ADJUSTMENT", rng.choice([-4, -3, -2, -1, 1, 2]), None, moved_at)) + + movements.sort(key=lambda m: (m[5], m[0], m[1])) + stock_movements = [ + dict(zip(("STORE_ID", "SKU", "MOVEMENT_TYPE", "QUANTITY", "PO_ID", "MOVED_AT"), m, strict=True)) + | {"MOVEMENT_ID": i} + for i, m in enumerate(movements, start=1) + ] + return Dataset(STORES, products, stock_levels, purchase_orders, stock_movements) + + +BASELINE_SCRIPTS = ("migrations/V1.0.0__baseline.sql", "migrations/A__grants.sql") + + +def seed(settings: Settings, data: Dataset | None = None) -> Dataset: + """(Re)create ``settings.database`` as production stood before this change set. + + Drops and recreates the database, bootstraps the live schema with the + baseline migration only (plus the always-run grants), and loads the data + set. The release under test (V1.1.0, V1.2.0, the new views) is left for + ``release.py release``. + """ + data = data or generate() + with cfg.connect(settings) as conn: + cur = conn.cursor() + cur.execute(f"CREATE OR REPLACE DATABASE {settings.database}") + cur.execute(f"CREATE SCHEMA {settings.database}.{settings.schema}") + cur.execute(f"DROP SCHEMA IF EXISTS {settings.database}.PUBLIC") + with staged(*BASELINE_SCRIPTS) as root: + deploy(settings, settings.schema, root_folder=root) + with cfg.connect(settings) as conn: + load(conn, f"{settings.database}.{settings.schema}", data) + return data + + +def _insert(conn: SnowflakeConnection, table: str, columns: list[str], rows: list[tuple]) -> None: + placeholders = ", ".join(["%s"] * len(columns)) + conn.cursor().executemany(f"INSERT INTO {table} ({', '.join(columns)}) VALUES ({placeholders})", rows) + + +def load(conn: SnowflakeConnection, schema: str, data: Dataset) -> None: + """Insert the data set into the (empty) baseline tables of ``schema`` (``DB.SCHEMA``).""" + _insert(conn, f"{schema}.STORES", ["STORE_ID", "STORE_NAME", "REGION", "OPENED_ON"], data.stores) + _insert(conn, f"{schema}.PRODUCTS", ["SKU", "PRODUCT_NAME", "CATEGORY", "UNIT_COST"], data.products) + for table, rows in ( + ("STOCK_LEVELS", data.stock_levels), + ("PURCHASE_ORDERS", data.purchase_orders), + ("STOCK_MOVEMENTS", data.stock_movements), + ): + columns = list(rows[0]) + _insert(conn, f"{schema}.{table}", columns, [tuple(r[c] for c in columns) for r in rows]) diff --git a/migrate-and-swap/release/demo.py b/migrate-and-swap/release/demo.py new file mode 100644 index 0000000..4e6bf9e --- /dev/null +++ b/migrate-and-swap/release/demo.py @@ -0,0 +1,115 @@ +"""``make demo``: a blocked release, a good release, a rollback, two recovery drills. + +Runs against the sample database (``$INVENTORY_DATABASE``, default ``RETAIL``) +right after ``make seed``, so live starts at version 1.0.0. +""" + +from __future__ import annotations + +from snowflake.connector import SnowflakeConnection + +from release import recovery +from release import settings as cfg +from release.deploy import migration_files, staged +from release.release import Exit, Releaser +from release.settings import Settings + +BAD_MIGRATION = "demos/V1.3.0__bad_negative_stock.sql" + + +def heading(text: str) -> None: + print(f"\n== {text}") + + +def indented(line: str) -> None: + print(f" {line}") + + +def expect(code: Exit, wanted: Exit) -> None: + indented(f"exit code {int(code)} ({code.name})") + if code is not wanted: + raise SystemExit(f"demo: expected {wanted.name}, got {code.name}") + + +def snapshot(releaser: Releaser) -> tuple[dict[str, int], frozenset]: + """What "live untouched" means here: every table's row count and the change history.""" + return releaser.row_counts(releaser.settings.schema), releaser.side(releaser.settings.schema).history + + +def columns(conn: SnowflakeConnection, settings: Settings, table: str) -> list[str]: + rows = ( + conn.cursor() + .execute( + f"SELECT COLUMN_NAME FROM {settings.database}.INFORMATION_SCHEMA.COLUMNS " + "WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s ORDER BY ORDINAL_POSITION", + (settings.schema, table), + ) + .fetchall() + ) + return [name for (name,) in rows] + + +def main() -> None: + settings = cfg.load() + live = f"{settings.database}.{settings.schema}" + with cfg.connect(settings) as conn: + releaser = Releaser(settings, conn, out=indented) + + heading(f"0. Production today: {live}") + releaser.status() + indented(f"STOCK_LEVELS columns: {', '.join(columns(conn, settings, 'STOCK_LEVELS'))}") + + heading(f"1. A release with a bad migration stacked on top: {BAD_MIGRATION}") + before = snapshot(releaser) + with staged(BAD_MIGRATION, base=migration_files()) as root: + expect(releaser.release(root, label=f"migrations/ + {BAD_MIGRATION}"), Exit.VALIDATION_FAILED) + after = snapshot(releaser) + indented(f"live row counts unchanged: {before[0] == after[0]}") + indented(f"live change history unchanged: {before[1] == after[1]}") + + heading("2. The good release: migrations/ (V1.1.0, V1.2.0, R__inventory_views, A__grants)") + expect(releaser.release(), Exit.OK) + indented(f"STOCK_LEVELS columns: {', '.join(columns(conn, settings, 'STOCK_LEVELS'))}") + row = ( + conn.cursor() + .execute( + f"SELECT STORE_ID, SKU, AVAILABLE, REORDER_POINT, AISLE, BAY FROM {live}.V_STOCK_POSITION " + "ORDER BY STORE_ID, SKU LIMIT 1" + ) + .fetchone() + ) + indented(f"V_STOCK_POSITION: STORE_ID, SKU, AVAILABLE, REORDER_POINT, AISLE, BAY = {row}") + releaser.status() + + heading("3. Roll back with one more swap") + expect(releaser.rollback(), Exit.OK) + releaser.status() + indented(f"STOCK_LEVELS columns: {', '.join(columns(conn, settings, 'STOCK_LEVELS'))}") + indented("A second rollback would be a roll-forward, so it is refused:") + expect(releaser.rollback(), Exit.PRECONDITION) + + heading(f"4. Recovery drill: DROP TABLE {live}.PURCHASE_ORDERS, then UNDROP") + cur = conn.cursor() + count = cur.execute(f"SELECT COUNT(*) FROM {live}.PURCHASE_ORDERS").fetchone()[0] + cur.execute(f"DROP TABLE {live}.PURCHASE_ORDERS") + indented(f"dropped a table of {count} rows") + recovery.undrop(conn, f"{live}.PURCHASE_ORDERS") + restored = cur.execute(f"SELECT COUNT(*) FROM {live}.PURCHASE_ORDERS").fetchone()[0] + open_pos = cur.execute(f"SELECT COUNT(*) FROM {live}.V_OPEN_PURCHASE_ORDERS").fetchone()[0] + indented(f"UNDROP TABLE: {restored} rows back; V_OPEN_PURCHASE_ORDERS answers again ({open_pos} open)") + + heading("5. Recovery drill: a bad UPDATE, restored with BEFORE(STATEMENT => LAST_QUERY_ID())") + total = f"SELECT SUM(ON_HAND) FROM {live}.STOCK_LEVELS" + good = cur.execute(total).fetchone()[0] + cur.execute(f"UPDATE {live}.STOCK_LEVELS SET ON_HAND = 0 WHERE STORE_ID = 'S03'") # meant: one SKU + bad_statement = recovery.last_query_id(conn) + indented(f"UPDATE ... SET ON_HAND = 0 WHERE STORE_ID = 'S03' (query id {bad_statement})") + indented(f"SUM(ON_HAND): {good} before, {cur.execute(total).fetchone()[0]} after the bad statement") + rows = recovery.restore_before(conn, f"{live}.STOCK_LEVELS", bad_statement) + indented(f"restored {rows} rows from a clone BEFORE(STATEMENT => ...)") + indented(f"SUM(ON_HAND) = {cur.execute(total).fetchone()[0]}") + print() + + +if __name__ == "__main__": + main() diff --git a/migrate-and-swap/release/deploy.py b/migrate-and-swap/release/deploy.py new file mode 100644 index 0000000..05aae0c --- /dev/null +++ b/migrate-and-swap/release/deploy.py @@ -0,0 +1,127 @@ +"""Run schemachange against one schema of the inventory database. + +schemachange forwards every ``SNOWFLAKE_*`` environment variable to the +connector, so the subprocess gets a scrubbed environment that contains only the +connection this sample means to use. A developer's cloud credentials in the +shell can therefore never redirect an emulator deploy. + +The change history table lives *inside* the schema it describes +(``.CHANGE_HISTORY``). A clone carries the history with it, schemachange +on the clone applies only what is new, and a swap moves schema and history +together, so ``status`` and ``rollback`` can always read the version of each +side from the side itself. +""" + +from __future__ import annotations + +import json +import os +import re +import shutil +import subprocess +import sys +import tempfile +from collections.abc import Iterable, Iterator +from contextlib import contextmanager +from pathlib import Path + +from release import settings as cfg +from release.settings import SAMPLE_ROOT, Settings + +SCHEMACHANGE = Path(sys.executable).with_name("schemachange") +MIGRATIONS = SAMPLE_ROOT / "migrations" +CHANGE_HISTORY = "CHANGE_HISTORY" + + +class DeployError(RuntimeError): + """schemachange exited non-zero: a migration failed.""" + + +def change_history_table(settings: Settings, schema: str) -> str: + return f"{settings.database}.{schema}.{CHANGE_HISTORY}" + + +def schemachange_env(settings: Settings, schema: str) -> dict[str, str]: + env = {k: v for k, v in os.environ.items() if not k.startswith(("SNOWFLAKE_", "SNOWSQL_"))} + env.update( + SNOWFLAKE_ACCOUNT=settings.account, + SNOWFLAKE_USER=settings.user, + SNOWFLAKE_ROLE=settings.role, + SNOWFLAKE_WAREHOUSE=settings.warehouse, + SNOWFLAKE_DATABASE=settings.database, + SNOWFLAKE_SCHEMA=schema, + SNOWFLAKE_AUTHENTICATOR="SNOWFLAKE_JWT", + SNOWFLAKE_PRIVATE_KEY_FILE=str(settings.private_key_path), + ) + env[cfg.PLATFORM_DETECTION_ENV] = os.environ[cfg.PLATFORM_DETECTION_ENV] + if settings.is_emulator: + env.update( + SNOWFLAKE_HOST=str(settings.host), + SNOWFLAKE_PORT=str(settings.port), + SNOWFLAKE_PROTOCOL=str(settings.protocol), + SNOWFLAKE_DISABLE_OCSP_CHECKS="true", # see Settings.connect_kwargs + ) + return env + + +def deploy(settings: Settings, schema: str, *, root_folder: Path = MIGRATIONS, log_level: str = "WARNING") -> None: + """Apply every pending script in ``root_folder`` to ``settings.database``.``schema``. + + Scripts address their objects as ``{{ database }}.{{ schema }}.``, so the + same migration files target the live schema on bootstrap and the green + candidate on every release. Raises ``DeployError`` when a script fails. + """ + template_vars = {"database": settings.database, "schema": schema} + result = subprocess.run( + [ + str(SCHEMACHANGE), + "deploy", + "--config-folder", + str(SAMPLE_ROOT), + "--root-folder", + str(root_folder), + "--change-history-table", + change_history_table(settings, schema), + "--vars", + json.dumps(template_vars), + "--log-level", + log_level, + ], + env=schemachange_env(settings, schema), + cwd=SAMPLE_ROOT, + capture_output=True, + text=True, + ) + if result.returncode != 0: + raise DeployError(_error_summary(result.stderr + result.stdout)) + + +_ANSI = re.compile(r"\x1b\[[0-9;]*m") +_LOG_ERROR = re.compile(r"\[error\s*\]\s*(?P.*\S)") + + +def _error_summary(output: str) -> str: + """The Snowflake error schemachange logged, without colours and boilerplate.""" + messages = [m["message"] for m in _LOG_ERROR.finditer(_ANSI.sub("", output))] + errors = [m for m in messages if re.match(r"\d{6} \(", m)] + return (errors or messages or [output.strip().splitlines()[-1] if output.strip() else "no output"])[0] + + +@contextmanager +def staged(*scripts: Path | str, base: Iterable[Path] = ()) -> Iterator[Path]: + """A temporary root folder holding exactly ``base`` plus ``scripts``. + + Used to deploy a subset of the migrations (``make seed`` bootstraps the + baseline only) or to add a script to the set, the way a pull request + would (the demo stacks ``demos/V1.3.0__bad_negative_stock.sql`` on top). + """ + with tempfile.TemporaryDirectory(prefix="migrations-") as tmp: + root = Path(tmp) + for script in [*base, *scripts]: + path = Path(script) + shutil.copy(path if path.is_absolute() else SAMPLE_ROOT / path, root / path.name) + yield root + + +def migration_files() -> list[Path]: + return sorted(MIGRATIONS.glob("*.sql")) diff --git a/migrate-and-swap/release/gate.py b/migrate-and-swap/release/gate.py new file mode 100644 index 0000000..c9134c2 --- /dev/null +++ b/migrate-and-swap/release/gate.py @@ -0,0 +1,110 @@ +"""The release gate: parse ``validations.sql`` and run every check against green.""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field +from pathlib import Path + +from jinja2 import Environment, StrictUndefined +from snowflake.connector import SnowflakeConnection +from snowflake.connector.errors import ProgrammingError + +from release.deploy import CHANGE_HISTORY +from release.settings import Settings + +VALIDATIONS = Path(__file__).with_name("validations.sql") +_CHECK = re.compile(r"^-- check: (?P\S+)\s*$", re.MULTILINE) + + +@dataclass(frozen=True) +class Check: + name: str + description: str + sql: str + + +@dataclass(frozen=True) +class CheckResult: + check: Check + columns: tuple[str, ...] = () + violations: list[tuple] = field(default_factory=list) + error: str | None = None + + @property + def passed(self) -> bool: + return self.error is None and not self.violations + + def summary(self) -> str: + if self.error: + return f"error: {self.error}" + if not self.violations: + return self.check.description.split(". ")[0].rstrip(".") + example = ", ".join(f"{c}={v}" for c, v in zip(self.columns, self.violations[0], strict=True)) + return f"{len(self.violations)} violation(s), e.g. {example}" + + +def parse(text: str) -> list[Check]: + """Split a rendered validations file into checks (anything before the first check is a header).""" + marks = list(_CHECK.finditer(text)) + checks = [] + for mark, nxt in zip(marks, [*marks[1:], None], strict=True): + body = text[mark.end() : nxt.start() if nxt else len(text)].strip().splitlines() + description = " ".join(line[3:].strip() for line in body if line.startswith("-- ")) + sql = "\n".join(line for line in body if not line.startswith("--")).strip().rstrip(";").strip() + if not sql: + raise ValueError(f"check {mark['name']} in {VALIDATIONS.name} has no query") + checks.append(Check(mark["name"], description, sql)) + return checks + + +def render(context: dict, template: Path = VALIDATIONS) -> list[Check]: + env = Environment(undefined=StrictUndefined, keep_trailing_newline=True) + return parse(env.from_string(template.read_text(encoding="utf-8")).render(**context)) + + +def base_tables(conn: SnowflakeConnection, settings: Settings, schema: str) -> list[str]: + """Base tables in ``schema``, without schemachange's change history table.""" + rows = ( + conn.cursor() + .execute( + f"SELECT TABLE_NAME FROM {settings.database}.INFORMATION_SCHEMA.TABLES " + "WHERE TABLE_SCHEMA = %s AND TABLE_TYPE = 'BASE TABLE'", + (schema,), + ) + .fetchall() + ) + return sorted(name for (name,) in rows if name != CHANGE_HISTORY) + + +def views(conn: SnowflakeConnection, settings: Settings, schema: str) -> list[str]: + """Views in ``schema``.""" + sql = f"SELECT TABLE_NAME FROM {settings.database}.INFORMATION_SCHEMA.VIEWS WHERE TABLE_SCHEMA = %s" + rows = conn.cursor().execute(sql, (schema,)).fetchall() + return sorted(name for (name,) in rows) + + +def context(conn: SnowflakeConnection, settings: Settings) -> dict: + return { + "database": settings.database, + "live_schema": settings.schema, + "green_schema": settings.green_schema, + "live": f"{settings.database}.{settings.schema}", + "green": f"{settings.database}.{settings.green_schema}", + "live_tables": base_tables(conn, settings, settings.schema), + "green_views": views(conn, settings, settings.green_schema), + } + + +def run_check(conn: SnowflakeConnection, check: Check) -> CheckResult: + cur = conn.cursor() + try: + rows = cur.execute(check.sql).fetchall() + except ProgrammingError as exc: + message = str(exc.msg or exc).strip().splitlines()[-1] + return CheckResult(check, error=f"{exc.errno or 0:06d} {message}") + return CheckResult(check, tuple(d.name for d in cur.description), list(rows)) + + +def run(conn: SnowflakeConnection, settings: Settings) -> list[CheckResult]: + return [run_check(conn, check) for check in render(context(conn, settings))] diff --git a/migrate-and-swap/release/recovery.py b/migrate-and-swap/release/recovery.py new file mode 100644 index 0000000..bb49373 --- /dev/null +++ b/migrate-and-swap/release/recovery.py @@ -0,0 +1,51 @@ +"""Recovery runbook for the two incidents a release cannot roll back. + +A schema swap restores *structure*. It does not help when someone drops a live +table or runs a bad ``UPDATE`` between releases. Those are time-travel jobs: + +``undrop`` + ``UNDROP TABLE`` brings back the most recent dropped version of a table, + with its data, inside the table's retention period + (``DATA_RETENTION_TIME_IN_DAYS``, 1 by default). + +``restore_before`` + Rebuilds a table as it was immediately before one statement: + + 1. ``CREATE TABLE _BEFORE_ CLONE BEFORE(STATEMENT => '')``: + a zero-copy image of the table before the bad statement, kept until the + restore is verified, so the damage can be diffed first; + 2. ``INSERT OVERWRITE INTO SELECT * FROM ``: restore in place, so + the table keeps its identity, grants and dependent views; + 3. drop the clone. + + The statement id comes from the bad session (``SELECT LAST_QUERY_ID()`` + right after it), from ``QUERY_HISTORY``, or from Snowsight. Time travel + only reaches statements inside the retention period; anything older fails + with ``000707 Time travel data is not available``. +""" + +from __future__ import annotations + +from snowflake.connector import SnowflakeConnection + + +def undrop(conn: SnowflakeConnection, table: str) -> None: + """Restore the most recently dropped version of ``table`` (``DB.SCHEMA.TABLE``).""" + conn.cursor().execute(f"UNDROP TABLE {table}") + + +def last_query_id(conn: SnowflakeConnection) -> str: + """The id of the statement this session ran last (the one to undo).""" + return conn.cursor().execute("SELECT LAST_QUERY_ID()").fetchone()[0] + + +def restore_before(conn: SnowflakeConnection, table: str, query_id: str) -> int: + """Restore ``table`` to its state just before statement ``query_id``; returns the restored row count.""" + before = f"{table}_BEFORE_{query_id.replace('-', '')[:12].upper()}" + cur = conn.cursor() + cur.execute(f"CREATE TABLE {before} CLONE {table} BEFORE(STATEMENT => %s)", (query_id,)) + try: + restored = cur.execute(f"INSERT OVERWRITE INTO {table} SELECT * FROM {before}").fetchone()[0] + finally: + cur.execute(f"DROP TABLE IF EXISTS {before}") + return restored diff --git a/migrate-and-swap/release/release.py b/migrate-and-swap/release/release.py new file mode 100644 index 0000000..e08ce77 --- /dev/null +++ b/migrate-and-swap/release/release.py @@ -0,0 +1,206 @@ +"""Blue/green schema releases: clone live, migrate the clone, validate it, swap it in. + +Usage: ``python -m release {release,rollback,status} [options]`` + +``release`` + 1. ``CREATE OR REPLACE SCHEMA _GREEN CLONE `` (zero-copy, seconds) + 2. ``schemachange deploy`` against green (the change history travels with it) + 3. the gate in ``release/validations.sql`` against green + 4. ``ALTER SCHEMA SWAP WITH _GREEN`` (one atomic rename) + + Any failure in steps 2 or 3 exits non-zero *before* step 4, so live is never + touched. After a swap, green holds the previous version. + +``rollback`` + Swaps once more, if green holds the version live was released from. + +``status`` + The version on each side and how the two sides relate. + +Exit codes: 0 done, 1 validation failed (not swapped), 2 a migration failed +(not swapped), 3 precondition not met (nothing changed). +""" + +from __future__ import annotations + +import argparse +import sys +from collections.abc import Callable +from dataclasses import dataclass +from enum import IntEnum +from pathlib import Path + +from snowflake.connector import SnowflakeConnection +from snowflake.connector.errors import ProgrammingError + +from release import gate +from release import settings as cfg +from release.deploy import CHANGE_HISTORY, MIGRATIONS, DeployError, deploy +from release.settings import Settings + +Printer = Callable[[str], None] + + +class Exit(IntEnum): + OK = 0 + VALIDATION_FAILED = 1 + MIGRATION_FAILED = 2 + PRECONDITION = 3 + + +@dataclass(frozen=True) +class Side: + """One schema and the schemachange history it carries.""" + + schema: str + exists: bool + history: frozenset[tuple] = frozenset() # (VERSION, SCRIPT, CHECKSUM, INSTALLED_ON) rows + + @property + def version(self) -> str | None: + versions = [row[0] for row in self.history if row[0]] + return max(versions, key=_version_key) if versions else None + + +def _version_key(version: str) -> tuple[int, ...]: + return tuple(int(part) for part in version.split(".")) + + +def describe(live: Side, green: Side) -> str: + """How green relates to live, from their change histories.""" + if not live.exists: + return "n/a (no live schema)" + if not green.exists: + return "absent" + if green.history < live.history: + return "previous version (rollback target)" + if green.history > live.history: + extra = len(green.history - live.history) + return f"ahead of live by {extra} script run(s): a blocked, aborted or rolled-back candidate" + if green.history == live.history: + return "identical to live" + return "diverged from live (neither history contains the other)" + + +class Releaser: + def __init__(self, settings: Settings, conn: SnowflakeConnection, out: Printer = print) -> None: + self.settings = settings + self.conn = conn + self.out = out + self.live = f"{settings.database}.{settings.schema}" + self.green = f"{settings.database}.{settings.green_schema}" + + # -- building blocks (public so a test can interleave its own statements) -- + + def side(self, schema: str) -> Side: + table = f"{self.settings.database}.{schema}.{CHANGE_HISTORY}" + try: + rows = self.conn.cursor().execute(f"SELECT VERSION, SCRIPT, CHECKSUM, INSTALLED_ON FROM {table}").fetchall() + except ProgrammingError as exc: + if exc.errno == 2003: # the schema or its history table does not exist + return Side(schema, exists=False) + raise + return Side(schema, exists=True, history=frozenset(rows)) + + def sides(self) -> tuple[Side, Side]: + return self.side(self.settings.schema), self.side(self.settings.green_schema) + + def row_counts(self, schema: str) -> dict[str, int]: + cur = self.conn.cursor() + return { + table: cur.execute(f"SELECT COUNT(*) FROM {self.settings.database}.{schema}.{table}").fetchone()[0] + for table in gate.base_tables(self.conn, self.settings, schema) + } + + def clone(self) -> None: + self.conn.cursor().execute(f"CREATE OR REPLACE SCHEMA {self.green} CLONE {self.live}") + + def migrate(self, root_folder: Path) -> None: + deploy(self.settings, self.settings.green_schema, root_folder=root_folder) + + def validate(self) -> list[gate.CheckResult]: + return gate.run(self.conn, self.settings) + + def swap(self) -> None: + self.conn.cursor().execute(f"ALTER SCHEMA {self.live} SWAP WITH {self.green}") + + # -- commands -- + + def release(self, root_folder: Path = MIGRATIONS, *, label: str | None = None) -> Exit: + live, _ = self.sides() + if not live.exists: + self.out(f"{self.live} has no change history: bootstrap it first (make seed).") + return Exit.PRECONDITION + self.out(f"Releasing {label or root_folder.name + '/'} to {self.live} (live at version {live.version})") + + self.clone() + self.out(f" 1. cloned {self.live} -> {self.green}") + + try: + self.migrate(root_folder) + except DeployError as exc: + self.out(f" 2. migration FAILED on {self.green}: {exc}") + self.out(f"RELEASE ABORTED: {self.live} untouched; inspect {self.green}.") + return Exit.MIGRATION_FAILED + candidate = self.side(self.settings.green_schema) + ran = [row[1] for row in sorted(candidate.history - live.history, key=lambda row: row[3])] + self.out(f" 2. schemachange on {self.green}: version {candidate.version}, ran {', '.join(ran) or 'nothing'}") + + results = self.validate() + failed = [r for r in results if not r.passed] + self.out(f" 3. validations: {len(results) - len(failed)} of {len(results)} passed") + for result in results: + self.out(f" {'PASS' if result.passed else 'FAIL'} {result.check.name:<40} {result.summary()}") + if failed: + self.out(f"RELEASE BLOCKED: {len(failed)} check(s) failed; {self.live} untouched.") + self.out(f"The candidate stays in {self.green} for inspection.") + return Exit.VALIDATION_FAILED + + self.swap() + self.out(f" 4. swapped: {self.live} is now version {candidate.version}; {self.green} holds {live.version}") + self.out(f"RELEASED {candidate.version}. Roll back with: python -m release rollback") + return Exit.OK + + def rollback(self) -> Exit: + live, green = self.sides() + if not (live.exists and green.exists and green.history < live.history): + self.out(f"Nothing to roll back: {self.green} is {describe(live, green)}.") + return Exit.PRECONDITION + self.out(f"Rolling back {self.live} from {live.version} to {green.version}") + live_rows, green_rows = self.row_counts(live.schema), self.row_counts(green.schema) + drift = {t: (n, green_rows.get(t)) for t, n in live_rows.items() if green_rows.get(t) != n} + for table, (now, then) in sorted(drift.items()): + self.out(f" note: {table} has {now} rows live, {then} in the previous version (writes since the release)") + self.swap() + self.out(f"ROLLED BACK: {self.live} is version {green.version} again; {self.green} holds {live.version}") + return Exit.OK + + def status(self) -> Exit: + live, green = self.sides() + self.out(f"live {self.live:<32} version {live.version if live.exists else '(missing)'}") + green_version = green.version if green.exists else "-" + self.out(f"green {self.green:<32} version {green_version}: {describe(live, green)}") + return Exit.OK + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(prog="python -m release", description=__doc__.splitlines()[0]) + sub = parser.add_subparsers(dest="command", required=True) + rel = sub.add_parser("release", help="clone, migrate, validate and swap") + rel.add_argument("--root-folder", type=Path, default=MIGRATIONS, help="schemachange scripts (default: migrations/)") + sub.add_parser("rollback", help="swap back to the previous version") + sub.add_parser("status", help="show the version on each side") + args = parser.parse_args(argv) + + settings = cfg.load() + with cfg.connect(settings) as conn: + releaser = Releaser(settings, conn) + if args.command == "release": + return releaser.release(args.root_folder.resolve()) + if args.command == "rollback": + return releaser.rollback() + return releaser.status() + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/migrate-and-swap/release/settings.py b/migrate-and-swap/release/settings.py new file mode 100644 index 0000000..b233b52 --- /dev/null +++ b/migrate-and-swap/release/settings.py @@ -0,0 +1,148 @@ +"""Connection settings, read from the environment in one place. + +``TEST_TARGET`` picks the target: + +* ``EMULATOR`` (default): the Snowflake-Next emulator on + ``snowflake.localhost.localstack.cloud:${SNOWFLAKE_PORT:-4566}``. Only the + port is configurable, so a stray cloud variable in the shell can never point + an emulator run at a real account. +* ``SNOWFLAKE_CLOUD``: a real account, from the ``SNOWFLAKE_*`` variables. + +Every connection uses key-pair (``SNOWFLAKE_JWT``) auth. Using the same shape +everywhere means the config that passes locally is the config that runs +against a real account. +""" + +from __future__ import annotations + +import os +from dataclasses import dataclass, replace +from pathlib import Path + +import snowflake.connector +from cryptography.hazmat.primitives import serialization +from snowflake.connector import SnowflakeConnection + +SAMPLE_ROOT = Path(__file__).resolve().parents[1] +EMULATOR_HOST = "snowflake.localhost.localstack.cloud" +QUERY_TAG = "migrate-and-swap" + +# The connector's login telemetry probes for cloud platforms (it builds a boto3 +# client to look for AWS credentials, among others), which costs seconds per +# process and tells Snowflake nothing useful about a release. Opt out unless +# the caller decided otherwise. +PLATFORM_DETECTION_ENV = "SNOWFLAKE_DISABLE_PLATFORM_DETECTION" +os.environ.setdefault(PLATFORM_DETECTION_ENV, "true") + + +@dataclass(frozen=True) +class Settings: + target: str + account: str + user: str + private_key_path: Path + role: str + warehouse: str + database: str + schema: str + host: str | None = None + port: int | None = None + protocol: str | None = None + + @property + def is_emulator(self) -> bool: + return self.target == "EMULATOR" + + @property + def green_schema(self) -> str: + """The candidate schema a release migrates; after a swap it holds the previous version.""" + return f"{self.schema}_GREEN" + + def with_database(self, database: str) -> Settings: + return replace(self, database=database.upper()) + + def private_key_der(self) -> bytes: + """The private key as DER bytes. + + The connector's ``private_key_file`` option refuses key files that are + group- or world-writable, which is every file on some filesystems + (a WSL ``/mnt/c`` checkout, for one). Loading the key here works on any + filesystem and keeps the key out of the process environment. + """ + key = serialization.load_pem_private_key(self.private_key_path.read_bytes(), password=None) + return key.private_bytes( + encoding=serialization.Encoding.DER, + format=serialization.PrivateFormat.PKCS8, + encryption_algorithm=serialization.NoEncryption(), + ) + + def connect_kwargs(self) -> dict: + kwargs: dict = { + "account": self.account, + "user": self.user, + "authenticator": "SNOWFLAKE_JWT", + "private_key": self.private_key_der(), + "role": self.role, + "warehouse": self.warehouse, + "session_parameters": {"QUERY_TAG": QUERY_TAG}, + "login_timeout": 60, + } + if self.is_emulator: + # TLS is still verified. The OCSP revocation lookup is skipped for + # the local certificate of a 127.0.0.1 endpoint. + kwargs.update(host=self.host, port=self.port, protocol=self.protocol, disable_ocsp_checks=True) + return kwargs + + +def _env(name: str, default: str) -> str: + """An environment variable, treating an empty value (``VAR=`` in .env) as unset.""" + return os.getenv(name) or default + + +def _required(name: str) -> str: + value = os.getenv(name) + if not value: + raise RuntimeError(f"{name} must be set when TEST_TARGET=SNOWFLAKE_CLOUD") + return value + + +def _key_path() -> Path: + path = Path(_env("SNOWFLAKE_PRIVATE_KEY_PATH", ".secrets/rsa_key.p8")) + return path if path.is_absolute() else SAMPLE_ROOT / path + + +def load() -> Settings: + target = _env("TEST_TARGET", "EMULATOR").upper() + database = _env("INVENTORY_DATABASE", "RETAIL").upper() + schema = _env("INVENTORY_SCHEMA", "INVENTORY").upper() + if target == "EMULATOR": + return Settings( + target=target, + account="snowflake", + user="SAMPLE_SVC", + private_key_path=_key_path(), + role="ACCOUNTADMIN", + warehouse="RELEASE_WH", + database=database, + schema=schema, + host=EMULATOR_HOST, + port=int(_env("SNOWFLAKE_PORT", "4566")), + protocol="https", + ) + if target == "SNOWFLAKE_CLOUD": + return Settings( + target=target, + account=_required("SNOWFLAKE_ACCOUNT"), + user=_required("SNOWFLAKE_USER"), + private_key_path=_key_path(), + role=_env("SNOWFLAKE_ROLE", "ACCOUNTADMIN"), + warehouse=_required("SNOWFLAKE_WAREHOUSE"), + database=database, + schema=schema, + ) + raise RuntimeError(f"TEST_TARGET must be EMULATOR or SNOWFLAKE_CLOUD, got {target!r}") + + +def connect(settings: Settings) -> SnowflakeConnection: + """Open a session as the release identity. Callers use fully qualified names.""" + return snowflake.connector.connect(**settings.connect_kwargs()) diff --git a/migrate-and-swap/release/validations.sql b/migrate-and-swap/release/validations.sql new file mode 100644 index 0000000..2ebc03a --- /dev/null +++ b/migrate-and-swap/release/validations.sql @@ -0,0 +1,67 @@ +-- The release gate. release.py runs every check against the migrated green +-- candidate before the swap; one failing check stops the release with exit +-- code 1 and live is never touched. +-- +-- Format: each check starts with a "-- check: " line, followed by +-- "-- " description lines and one query. A check PASSES when its query returns +-- no rows; every returned row is a violation. A query that fails to run (a +-- dropped table, a broken view) fails its check with the Snowflake error. +-- +-- The file is a Jinja template. Context: +-- live, green fully qualified schemas, e.g. RETAIL.INVENTORY +-- database, live_schema, the same, unqualified, for INFORMATION_SCHEMA +-- green_schema +-- live_tables base tables in live (change history excluded) +-- green_views views in green + +-- check: row_count_parity +-- A schema release must not add or lose rows: every table live has keeps +-- its row count in green. Also catches writes that reached live after the +-- clone was taken (they would be lost by the swap). +{% for t in live_tables %} +SELECT '{{ t }}' AS TABLE_NAME, LIVE_ROWS, GREEN_ROWS +FROM ( + SELECT + (SELECT COUNT(*) FROM {{ live }}.{{ t }}) AS LIVE_ROWS, + (SELECT COUNT(*) FROM {{ green }}.{{ t }}) AS GREEN_ROWS +) +WHERE LIVE_ROWS <> GREEN_ROWS +{% if not loop.last %}UNION ALL{% endif %} +{% endfor %}; + +-- check: no_negative_on_hand +-- Physical stock cannot be negative. Backorders are carried in ALLOCATED. +SELECT STORE_ID, SKU, ON_HAND +FROM {{ green }}.STOCK_LEVELS +WHERE ON_HAND < 0 +ORDER BY STORE_ID, SKU; + +-- check: no_orphan_movements +-- Every ledger movement belongs to a stock row (store and SKU). +SELECT m.MOVEMENT_ID, m.STORE_ID, m.SKU +FROM {{ green }}.STOCK_MOVEMENTS m +LEFT JOIN {{ green }}.STOCK_LEVELS s ON s.STORE_ID = m.STORE_ID AND s.SKU = m.SKU +WHERE s.SKU IS NULL +ORDER BY m.MOVEMENT_ID; + +-- check: receipts_match_purchase_orders +-- Every RECEIPT movement references a purchase order that exists. +SELECT m.MOVEMENT_ID, m.PO_ID +FROM {{ green }}.STOCK_MOVEMENTS m +LEFT JOIN {{ green }}.PURCHASE_ORDERS po ON po.PO_ID = m.PO_ID +WHERE m.MOVEMENT_TYPE = 'RECEIPT' AND po.PO_ID IS NULL +ORDER BY m.MOVEMENT_ID; + +-- check: no_view_dropped +-- Every view that live serves still exists in green: reports and the +-- INVENTORY_READER grants depend on the names. +SELECT TABLE_NAME FROM {{ database }}.INFORMATION_SCHEMA.VIEWS WHERE TABLE_SCHEMA = '{{ live_schema }}' +MINUS +SELECT TABLE_NAME FROM {{ database }}.INFORMATION_SCHEMA.VIEWS WHERE TABLE_SCHEMA = '{{ green_schema }}'; + +{% for v in green_views %} +-- check: view_selectable[{{ v }}] +-- The view compiles and runs against green's tables. COUNT(*) makes the +-- engine expand and execute the whole view; the outer filter never matches. +SELECT VIEW_ROWS FROM (SELECT COUNT(*) AS VIEW_ROWS FROM {{ green }}.{{ v }}) WHERE VIEW_ROWS < 0; +{% endfor %} diff --git a/migrate-and-swap/schemachange-config.yml b/migrate-and-swap/schemachange-config.yml new file mode 100644 index 0000000..0145b75 --- /dev/null +++ b/migrate-and-swap/schemachange-config.yml @@ -0,0 +1,9 @@ +# schemachange 4.4 configuration. Connection settings are not stored here: +# release.deploy passes them as SNOWFLAKE_* variables in a scrubbed environment, +# plus --root-folder, --change-history-table and --vars for the target schema. +config-version: 2 + +schemachange: + root-folder: migrations + create-change-history-table: true + query-tag: migrate-and-swap diff --git a/migrate-and-swap/scripts/seed.py b/migrate-and-swap/scripts/seed.py new file mode 100644 index 0000000..537a896 --- /dev/null +++ b/migrate-and-swap/scripts/seed.py @@ -0,0 +1,23 @@ +"""``make seed``: (re)create the inventory database as production stood before this change set. + +Drops and recreates ``$INVENTORY_DATABASE`` (default ``RETAIL``), bootstraps +the live schema at version 1.0.0 with schemachange, and loads the +deterministic data set from ``release.dataset``. Re-running it gives the same +state, byte for byte. +""" + +from __future__ import annotations + +from release import dataset +from release import settings as cfg + + +def main() -> None: + settings = cfg.load() + data = dataset.seed(settings) + counts = ", ".join(f"{table} {n}" for table, n in data.counts().items()) + print(f"Seeded {settings.database}.{settings.schema} at version 1.0.0: {counts}") + + +if __name__ == "__main__": + main() diff --git a/migrate-and-swap/scripts/wait-for-emulator.sh b/migrate-and-swap/scripts/wait-for-emulator.sh new file mode 100755 index 0000000..501c7e2 --- /dev/null +++ b/migrate-and-swap/scripts/wait-for-emulator.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# Wait until the Snowflake-Next emulator answers a SQL statement. +# +# Copy this file into a sample as scripts/wait-for-emulator.sh (samples stay +# self-contained). Usage: scripts/wait-for-emulator.sh [timeout-seconds] +# Reads SNOWFLAKE_PORT (default 4566). Readiness is checked from the host. +set -euo pipefail + +port="${SNOWFLAKE_PORT:-4566}" +timeout="${1:-120}" +url="https://snowflake.localhost.localstack.cloud:${port}" +deadline=$((SECONDS + timeout)) + +until curl -fsS --compressed -o /dev/null -X POST "${url}/api/v2/statements" \ + -H 'Authorization: Bearer local' -H 'Content-Type: application/json' \ + -H 'Accept: application/json' -d '{"statement":"SELECT 1"}' 2>/dev/null; do + if (( SECONDS >= deadline )); then + echo "Emulator at ${url} did not become ready within ${timeout}s." >&2 + echo "Check 'docker compose logs snowflake' (is LOCALSTACK_AUTH_TOKEN valid? is port ${port} free?)." >&2 + exit 1 + fi + sleep 2 +done +echo "Emulator ready at ${url}" diff --git a/migrate-and-swap/tests/conftest.py b/migrate-and-swap/tests/conftest.py new file mode 100644 index 0000000..cd4c6e0 --- /dev/null +++ b/migrate-and-swap/tests/conftest.py @@ -0,0 +1,117 @@ +"""Shared fixtures: every test gets its own freshly seeded inventory database. + +``env`` creates ``MAS_TEST_``, bootstraps its live ``INVENTORY`` +schema at version 1.0.0 with the real schemachange baseline and loads the +deterministic data set (a few seconds), then drops the database afterwards +(``KEEP_TEST_DB=1`` keeps it for debugging). Tests never share a database, so +they run in parallel (``pytest -n``) and in any order. + +Expected values in the tests come from ``data`` (the generated rows) and the +business rules written in the migrations, never from reading the result of +the code under test. +""" + +from __future__ import annotations + +import os +import uuid +from collections.abc import Iterator +from contextlib import contextmanager +from pathlib import Path + +import pytest +from snowflake.connector import SnowflakeConnection + +from release import dataset +from release import settings as cfg +from release.deploy import migration_files, staged +from release.release import Releaser +from release.settings import Settings + +BASELINE_STOCK_COLUMNS = ["STORE_ID", "SKU", "ON_HAND", "ALLOCATED", "LOCATION", "UPDATED_AT"] +RELEASED_STOCK_COLUMNS = ["STORE_ID", "SKU", "ON_HAND", "ALLOCATED", "UPDATED_AT", "REORDER_POINT", "AISLE", "BAY"] + + +def unique(prefix: str) -> str: + return f"{prefix}_{uuid.uuid4().hex[:8].upper()}" + + +@pytest.fixture(scope="session") +def data() -> dataset.Dataset: + return dataset.generate() + + +@pytest.fixture +def env(data: dataset.Dataset) -> Iterator[Settings]: + """Settings for a database whose live schema is at V1.0.0 with the data set loaded.""" + settings = cfg.load().with_database(unique("MAS_TEST")) + try: + dataset.seed(settings, data) + yield settings + finally: + if not os.getenv("KEEP_TEST_DB"): + with cfg.connect(settings) as conn: + conn.cursor().execute(f"DROP DATABASE IF EXISTS {settings.database}") + + +@pytest.fixture +def conn(env: Settings) -> Iterator[SnowflakeConnection]: + """A session with no current database or schema: every name is fully qualified.""" + connection = cfg.connect(env) + yield connection + connection.close() + + +@pytest.fixture +def releaser(env: Settings, conn: SnowflakeConnection) -> Releaser: + return Releaser(env, conn, out=lambda _line: None) + + +@contextmanager +def release_set(*extra: str | Path, tmp_sql: dict[str, str] | None = None) -> Iterator[Path]: + """A schemachange root folder: migrations/ plus ``extra`` files plus ``tmp_sql`` {name: body} scripts.""" + with staged(*extra, base=migration_files()) as root: + for name, body in (tmp_sql or {}).items(): + (root / name).write_text(body, encoding="utf-8") + yield root + + +@contextmanager +def baseline_plus(scripts: dict[str, str]) -> Iterator[Path]: + """A root folder holding only the bootstrap scripts plus ``scripts``: a release of just those changes.""" + with staged(*dataset.BASELINE_SCRIPTS) as root: + for name, body in scripts.items(): + (root / name).write_text(body, encoding="utf-8") + yield root + + +def query(conn: SnowflakeConnection, sql: str, params: tuple = ()) -> list[tuple]: + return conn.cursor().execute(sql, params).fetchall() + + +def columns(conn: SnowflakeConnection, settings: Settings, schema: str, table: str) -> list[str]: + return [ + name + for (name,) in query( + conn, + f"SELECT COLUMN_NAME FROM {settings.database}.INFORMATION_SCHEMA.COLUMNS " + "WHERE TABLE_SCHEMA = %s AND TABLE_NAME = %s ORDER BY ORDINAL_POSITION", + (schema, table), + ) + ] + + +def live_snapshot(conn: SnowflakeConnection, releaser: Releaser) -> dict: + """Everything "live untouched" covers: row counts, change history, table shapes, view list.""" + settings = releaser.settings + tables = sorted(releaser.row_counts(settings.schema)) + return { + "row_counts": releaser.row_counts(settings.schema), + "history": releaser.side(settings.schema).history, + "columns": {t: columns(conn, settings, settings.schema, t) for t in tables}, + "views": query( + conn, + f"SELECT TABLE_NAME FROM {settings.database}.INFORMATION_SCHEMA.VIEWS WHERE TABLE_SCHEMA = %s ORDER BY 1", + (settings.schema,), + ), + } diff --git a/migrate-and-swap/tests/test_recovery.py b/migrate-and-swap/tests/test_recovery.py new file mode 100644 index 0000000..ccf4d67 --- /dev/null +++ b/migrate-and-swap/tests/test_recovery.py @@ -0,0 +1,95 @@ +"""Recovery drills: UNDROP and time travel to just before a bad statement. + +Every statement a drill travels back to is executed inside the test, after +seeding, so it is always inside the time-travel window. +""" + +from __future__ import annotations + +import pytest +from conftest import query +from snowflake.connector import SnowflakeConnection +from snowflake.connector.errors import ProgrammingError + +from release import recovery +from release.dataset import Dataset +from release.settings import Settings + +STOCK_COLUMNS = "STORE_ID, SKU, ON_HAND, ALLOCATED, LOCATION" + + +def stock_rows(conn: SnowflakeConnection, table: str) -> set[tuple]: + return set(query(conn, f"SELECT {STOCK_COLUMNS} FROM {table}")) + + +def seeded_stock(data: Dataset) -> set[tuple]: + return {(s["STORE_ID"], s["SKU"], s["ON_HAND"], s["ALLOCATED"], s["LOCATION"]) for s in data.stock_levels} + + +def test_undrop_restores_a_dropped_table(conn: SnowflakeConnection, env: Settings, data: Dataset) -> None: + table = f"{env.database}.{env.schema}.PURCHASE_ORDERS" + conn.cursor().execute(f"DROP TABLE {table}") + with pytest.raises(ProgrammingError) as dropped: + conn.cursor().execute(f"SELECT COUNT(*) FROM {table}") + assert dropped.value.errno == 2003 + + recovery.undrop(conn, table) + + restored = query(conn, f"SELECT PO_ID, STORE_ID, SKU, ORDERED_QTY, STATUS FROM {table}") + assert sorted(restored) == sorted( + (p["PO_ID"], p["STORE_ID"], p["SKU"], p["ORDERED_QTY"], p["STATUS"]) for p in data.purchase_orders + ) + open_orders = query(conn, f"SELECT COUNT(*) FROM {env.database}.{env.schema}.V_OPEN_PURCHASE_ORDERS") + assert open_orders == [(sum(p["STATUS"] == "OPEN" for p in data.purchase_orders),)] + + +def test_restore_before_a_bad_update(conn: SnowflakeConnection, env: Settings, data: Dataset) -> None: + table = f"{env.database}.{env.schema}.STOCK_LEVELS" + conn.cursor().execute(f"UPDATE {table} SET ON_HAND = 0 WHERE STORE_ID = 'S03'") # meant: one SKU + bad_statement = recovery.last_query_id(conn) + damaged = stock_rows(conn, table) + assert len(seeded_stock(data) - damaged) == sum( + 1 for s in data.stock_levels if s["STORE_ID"] == "S03" and s["ON_HAND"] != 0 + ) + + restored = recovery.restore_before(conn, table, bad_statement) + + assert restored == len(data.stock_levels) + assert stock_rows(conn, table) == seeded_stock(data) + leftovers = query( + conn, + f"SELECT TABLE_NAME FROM {env.database}.INFORMATION_SCHEMA.TABLES " + "WHERE TABLE_SCHEMA = %s AND TABLE_NAME LIKE 'STOCK_LEVELS_BEFORE_%%'", + (env.schema,), + ) + assert leftovers == [], "the before-image clone is dropped once the restore is done" + + +def test_restore_keeps_later_statements_out(conn: SnowflakeConnection, env: Settings, data: Dataset) -> None: + """BEFORE(STATEMENT => bad) is a point in time: a later write is undone too, which the runbook must say.""" + table = f"{env.database}.{env.schema}.STOCK_LEVELS" + cur = conn.cursor() + cur.execute(f"UPDATE {table} SET ALLOCATED = 0 WHERE STORE_ID = 'S05'") + bad_statement = recovery.last_query_id(conn) + cur.execute(f"UPDATE {table} SET ON_HAND = ON_HAND + 1 WHERE STORE_ID = 'S06'") + recovery.restore_before(conn, table, bad_statement) + assert stock_rows(conn, table) == seeded_stock(data) + + +def test_undrop_of_a_table_that_exists_fails(conn: SnowflakeConnection, env: Settings) -> None: + with pytest.raises(ProgrammingError) as exc: + recovery.undrop(conn, f"{env.database}.{env.schema}.PURCHASE_ORDERS") + assert (exc.value.errno, exc.value.sqlstate) == (2002, "42710") + assert "Object 'PURCHASE_ORDERS' already exists" in exc.value.msg + + +def test_time_travel_before_the_table_existed_fails(conn: SnowflakeConnection, env: Settings) -> None: + schema = f"{env.database}.{env.schema}" + cur = conn.cursor() + cur.execute(f"SELECT COUNT(*) FROM {schema}.STORES") + earlier = recovery.last_query_id(conn) + cur.execute(f"CREATE TABLE {schema}.REPLENISHMENT_RUNS (RUN_ID NUMBER)") + with pytest.raises(ProgrammingError) as exc: + recovery.restore_before(conn, f"{schema}.REPLENISHMENT_RUNS", earlier) + assert (exc.value.errno, exc.value.sqlstate) == (707, "02000") + assert "Time travel data is not available for table REPLENISHMENT_RUNS" in exc.value.msg diff --git a/migrate-and-swap/tests/test_release.py b/migrate-and-swap/tests/test_release.py new file mode 100644 index 0000000..50b6b18 --- /dev/null +++ b/migrate-and-swap/tests/test_release.py @@ -0,0 +1,284 @@ +"""Blue/green releases: the swap, the validation gate, rollback, and grants across the swap.""" + +from __future__ import annotations + +import os +import subprocess +import sys +from datetime import timedelta + +from conftest import ( + BASELINE_STOCK_COLUMNS, + RELEASED_STOCK_COLUMNS, + baseline_plus, + columns, + live_snapshot, + query, + release_set, +) +from snowflake.connector import SnowflakeConnection + +from release.dataset import Dataset +from release.release import Exit, Releaser +from release.settings import SAMPLE_ROOT, Settings + +BAD_MIGRATION = "demos/V1.3.0__bad_negative_stock.sql" + + +# -- expected values, from the data set and the rules stated in the migrations -- + + +def expected_reorder_points(data: Dataset) -> dict[tuple[str, str], int]: + """V1.1.0: CEIL(units sold in the 28 days up to the newest movement / 4), 0 without sales.""" + newest = max(m["MOVED_AT"] for m in data.stock_movements) + sold: dict[tuple[str, str], int] = {} + for m in data.stock_movements: + if m["MOVEMENT_TYPE"] == "SALE" and m["MOVED_AT"] > newest - timedelta(days=28): + key = (m["STORE_ID"], m["SKU"]) + sold[key] = sold.get(key, 0) - m["QUANTITY"] + return {(s["STORE_ID"], s["SKU"]): -(-sold.get((s["STORE_ID"], s["SKU"]), 0) // 4) for s in data.stock_levels} + + +def expected_shelf(location: str | None) -> tuple[str | None, str | None]: + """V1.2.0: 'a07-b12 ' -> ('A07', 'B12'); NULL stays NULL.""" + if location is None: + return None, None + aisle, bay = location.strip().upper().split("-") + return aisle, bay + + +def backordered(data: Dataset) -> set[tuple[str, str]]: + return {(s["STORE_ID"], s["SKU"]) for s in data.stock_levels if s["ALLOCATED"] > s["ON_HAND"]} + + +def failed_checks(results) -> set[str]: + return {r.check.name for r in results if not r.passed} + + +# -- the release -- + + +def test_release_swaps_in_the_new_shape( + releaser: Releaser, conn: SnowflakeConnection, env: Settings, data: Dataset +) -> None: + assert releaser.release() is Exit.OK + + live, green = releaser.sides() + assert live.version == "1.2.0" + assert sorted(r[1] for r in live.history) == [ + "A__grants.sql", + "A__grants.sql", + "R__inventory_views.sql", + "V1.0.0__baseline.sql", + "V1.1.0__add_reorder_point.sql", + "V1.2.0__split_location.sql", + ] + assert green.version == "1.0.0", "the previous version stays in green for rollback" + assert columns(conn, env, env.schema, "STOCK_LEVELS") == RELEASED_STOCK_COLUMNS + assert columns(conn, env, env.green_schema, "STOCK_LEVELS") == BASELINE_STOCK_COLUMNS + + table = f"{env.database}.{env.schema}.STOCK_LEVELS" + rows = query(conn, f"SELECT STORE_ID, SKU, ON_HAND, ALLOCATED, REORDER_POINT, AISLE, BAY FROM {table}") + reorder = expected_reorder_points(data) + expected = { + (s["STORE_ID"], s["SKU"]): ( + s["ON_HAND"], + s["ALLOCATED"], + reorder[s["STORE_ID"], s["SKU"]], + *expected_shelf(s["LOCATION"]), + ) + for s in data.stock_levels + } + assert {(r[0], r[1]): tuple(r[2:]) for r in rows} == expected + assert releaser.row_counts(env.schema) == data.counts() + + +def test_views_serve_the_new_shape_from_live_tables( + releaser: Releaser, conn: SnowflakeConnection, env: Settings, data: Dataset +) -> None: + assert releaser.release() is Exit.OK + live = f"{env.database}.{env.schema}" + + # Unqualified table names in the view bodies resolve in the view's own + # schema, so after the swap the views read live's (the migrated) tables. + position = query(conn, f"SELECT STORE_ID, SKU, AVAILABLE, REORDER_POINT, AISLE, BAY FROM {live}.V_STOCK_POSITION") + reorder = expected_reorder_points(data) + assert {(r[0], r[1]): r[2:] for r in position} == { + (s["STORE_ID"], s["SKU"]): ( + s["ON_HAND"] - s["ALLOCATED"], + reorder[s["STORE_ID"], s["SKU"]], + *expected_shelf(s["LOCATION"]), + ) + for s in data.stock_levels + } + + on_order = {(p["STORE_ID"], p["SKU"]) for p in data.purchase_orders if p["STATUS"] == "OPEN"} + expected_candidates = { + (s["STORE_ID"], s["SKU"]) + for s in data.stock_levels + if 0 < reorder[s["STORE_ID"], s["SKU"]] + and s["ON_HAND"] - s["ALLOCATED"] <= reorder[s["STORE_ID"], s["SKU"]] + and (s["STORE_ID"], s["SKU"]) not in on_order + } + candidates = query(conn, f"SELECT STORE_ID, SKU FROM {live}.V_REORDER_CANDIDATES") + assert expected_candidates, "the data set must exercise the reorder view" + assert set(candidates) == expected_candidates + assert len(candidates) == len(expected_candidates) + + +def test_bad_migration_blocks_the_swap_and_leaves_live_untouched( + releaser: Releaser, conn: SnowflakeConnection, env: Settings, data: Dataset +) -> None: + """The headline: V1.3.0 drives ON_HAND negative on backordered rows; the gate stops the swap.""" + before = live_snapshot(conn, releaser) + with release_set(BAD_MIGRATION) as root: + cli = subprocess.run( + [sys.executable, "-m", "release", "release", "--root-folder", str(root)], + env=os.environ | {"INVENTORY_DATABASE": env.database}, + cwd=SAMPLE_ROOT, + capture_output=True, + text=True, + ) + assert cli.returncode == Exit.VALIDATION_FAILED, cli.stdout + cli.stderr + assert "FAIL no_negative_on_hand" in cli.stdout + assert "RELEASE BLOCKED" in cli.stdout + + assert live_snapshot(conn, releaser) == before, "live must be byte-for-byte untouched" + _, green = releaser.sides() + assert green.version == "1.3.0", "the failed candidate stays in green for inspection" + + results = releaser.validate() + assert failed_checks(results) == {"no_negative_on_hand"} + [negative] = [r for r in results if r.check.name == "no_negative_on_hand"] + assert {(store, sku) for store, sku, _ in negative.violations} == backordered(data) + expected_on_hand = {(s["STORE_ID"], s["SKU"]): s["ON_HAND"] - s["ALLOCATED"] for s in data.stock_levels} + assert all(on_hand == expected_on_hand[store, sku] < 0 for store, sku, on_hand in negative.violations) + + +def test_rollback_restores_the_previous_version(releaser: Releaser, conn: SnowflakeConnection, env: Settings) -> None: + before = live_snapshot(conn, releaser) + assert releaser.release() is Exit.OK + released = releaser.side(env.schema).history + + assert releaser.rollback() is Exit.OK + assert live_snapshot(conn, releaser) == before + assert columns(conn, env, env.schema, "V_STOCK_POSITION")[-1] == "LOCATION" + _, green = releaser.sides() + assert green.version == "1.2.0" + assert green.history == released + + # A second rollback would be a roll-forward: refused, nothing changes. + assert releaser.rollback() is Exit.PRECONDITION + assert live_snapshot(conn, releaser) == before + + +def test_failed_migration_exits_before_the_swap(releaser: Releaser, conn: SnowflakeConnection) -> None: + before = live_snapshot(conn, releaser) + broken = "UPDATE {{ database }}.{{ schema }}.STOCK_LEVELS SET ON_HAND = ON_HAND + ;\n" + with release_set(tmp_sql={"V1.3.0__typo.sql": broken}) as root: + assert releaser.release(root) is Exit.MIGRATION_FAILED + assert live_snapshot(conn, releaser) == before + + +def test_release_requires_a_bootstrapped_live_schema(env: Settings, conn: SnowflakeConnection) -> None: + empty = env.with_database(f"{env.database}_EMPTY") + conn.cursor().execute(f"CREATE DATABASE {empty.database}") + try: + assert Releaser(empty, conn, out=lambda _: None).release() is Exit.PRECONDITION + assert Releaser(empty, conn, out=lambda _: None).rollback() is Exit.PRECONDITION + finally: + conn.cursor().execute(f"DROP DATABASE {empty.database}") + + +# -- the gate catches each class of defect -- + + +def test_writes_to_live_after_the_clone_fail_row_count_parity( + releaser: Releaser, conn: SnowflakeConnection, env: Settings, data: Dataset +) -> None: + """A write that reaches live mid-release would be lost by the swap; the parity check sees it.""" + releaser.clone() + with release_set() as root: + releaser.migrate(root) + conn.cursor().execute( + f"INSERT INTO {env.database}.{env.schema}.STOCK_MOVEMENTS " + "(MOVEMENT_ID, STORE_ID, SKU, MOVEMENT_TYPE, QUANTITY, PO_ID, MOVED_AT) " + "VALUES (999999, 'S01', 'SKU-10001', 'SALE', -1, NULL, '2026-10-06 09:00:00')" + ) + results = releaser.validate() + assert failed_checks(results) == {"row_count_parity"} + [parity] = [r for r in results if r.check.name == "row_count_parity"] + movements = len(data.stock_movements) + assert parity.violations == [("STOCK_MOVEMENTS", movements + 1, movements)] + + +def test_orphan_movements_are_caught(releaser: Releaser, data: Dataset) -> None: + delist = "DELETE FROM {{ database }}.{{ schema }}.STOCK_LEVELS WHERE SKU = 'SKU-10407';\n" + with baseline_plus({"V1.1.0__delist_sku.sql": delist}) as root: + assert releaser.release(root) is Exit.VALIDATION_FAILED + results = releaser.validate() + assert failed_checks(results) == {"row_count_parity", "no_orphan_movements"} + [orphans] = [r for r in results if r.check.name == "no_orphan_movements"] + expected = sorted(m["MOVEMENT_ID"] for m in data.stock_movements if m["SKU"] == "SKU-10407") + assert expected, "the data set must have movements for the delisted SKU" + assert [movement_id for movement_id, _, _ in orphans.violations] == expected + + +def test_dropped_view_is_caught(releaser: Releaser) -> None: + drop = "DROP VIEW {{ database }}.{{ schema }}.V_OPEN_PURCHASE_ORDERS;\n" + with baseline_plus({"V1.1.0__drop_view.sql": drop}) as root: + assert releaser.release(root) is Exit.VALIDATION_FAILED + results = releaser.validate() + assert failed_checks(results) == {"no_view_dropped"} + [dropped] = [r for r in results if r.check.name == "no_view_dropped"] + assert dropped.violations == [("V_OPEN_PURCHASE_ORDERS",)] + + +def test_view_broken_by_a_dropped_table_is_caught(releaser: Releaser) -> None: + """A migration replaces PURCHASE_ORDERS but forgets the view that reads it.""" + replace = ( + "CREATE TABLE {{ database }}.{{ schema }}.PURCHASE_ORDER_LINES AS\n" + " SELECT * FROM {{ database }}.{{ schema }}.PURCHASE_ORDERS;\n" + "DROP TABLE {{ database }}.{{ schema }}.PURCHASE_ORDERS;\n" + ) + with baseline_plus({"V1.1.0__replace_purchase_orders.sql": replace}) as root: + assert releaser.release(root) is Exit.VALIDATION_FAILED + results = {r.check.name: r for r in releaser.validate()} + assert {name for name, r in results.items() if not r.passed} == { + "row_count_parity", + "receipts_match_purchase_orders", + "view_selectable[V_OPEN_PURCHASE_ORDERS]", + } + assert results["view_selectable[V_OPEN_PURCHASE_ORDERS]"].error.startswith("002037") + assert results["view_selectable[V_STOCK_POSITION]"].passed + + +# -- grants across CLONE and SWAP -- + + +def _grantees(conn: SnowflakeConnection, on: str, privilege: str) -> set[str]: + cur = conn.cursor() + cur.execute(f"SHOW GRANTS ON {on}") + names = [d.name for d in cur.description] + return {row[names.index("grantee_name")] for row in cur.fetchall() if row[names.index("privilege")] == privilege} + + +def test_clone_does_not_copy_grants_on_the_schema_itself(conn: SnowflakeConnection, env: Settings) -> None: + """Why A__grants.sql exists: the swapped-in schema would otherwise lack USAGE for readers.""" + live = f"{env.database}.{env.schema}" + clone = f"{env.database}.GRANT_PROBE" + assert "INVENTORY_READER" in _grantees(conn, f"SCHEMA {live}", "USAGE") + conn.cursor().execute(f"CREATE SCHEMA {clone} CLONE {live}") + assert "INVENTORY_READER" not in _grantees(conn, f"SCHEMA {clone}", "USAGE") + + +def test_readers_keep_access_across_the_swap(releaser: Releaser, conn: SnowflakeConnection, env: Settings) -> None: + assert releaser.release() is Exit.OK + live = f"{env.database}.{env.schema}" + assert "INVENTORY_READER" in _grantees(conn, f"SCHEMA {live}", "USAGE") + for view in ("V_STOCK_POSITION", "V_REORDER_CANDIDATES", "V_OPEN_PURCHASE_ORDERS"): + assert "INVENTORY_READER" in _grantees(conn, f"VIEW {live}.{view}", "SELECT"), view + cur = conn.cursor() + cur.execute(f"GRANT USAGE ON WAREHOUSE {env.warehouse} TO ROLE INVENTORY_READER") # environment, not schema + cur.execute("USE ROLE INVENTORY_READER") + assert cur.execute(f"SELECT COUNT(*) FROM {live}.V_REORDER_CANDIDATES").fetchone()[0] > 0 diff --git a/migrate-and-swap/uv.lock b/migrate-and-swap/uv.lock new file mode 100644 index 0000000..8a2ecd1 --- /dev/null +++ b/migrate-and-swap/uv.lock @@ -0,0 +1,547 @@ +version = 1 +revision = 3 +requires-python = "==3.12.*" + +[[package]] +name = "asn1crypto" +version = "1.5.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/de/cf/d547feed25b5244fcb9392e288ff9fdc3280b10260362fc45d37a798a6ee/asn1crypto-1.5.1.tar.gz", hash = "sha256:13ae38502be632115abf8a24cbe5f4da52e3b5231990aff31123c805306ccb9c", size = 121080, upload-time = "2022-03-15T14:46:52.889Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c9/7f/09065fd9e27da0eda08b4d6897f1c13535066174cc023af248fc2a8d5e5a/asn1crypto-1.5.1-py2.py3-none-any.whl", hash = "sha256:db4e40728b728508912cbb3d44f19ce188f218e9eba635821bb4b68564f8fd67", size = 105045, upload-time = "2022-03-15T14:46:51.055Z" }, +] + +[[package]] +name = "boto3" +version = "1.43.118" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, + { name = "jmespath" }, + { name = "s3transfer" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/96/8b/5694c8433b5202f34a82def4edee274f7df1eba2919c66d9d389d2202ec8/boto3-1.43.118.tar.gz", hash = "sha256:c0620f799e0f0c222ba42c337b65cdbcd3da38a3121187f634d8926cbbb28a9c", size = 112649, upload-time = "2026-10-22T19:22:09.629Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bb/a3/65bb3d64928f3c2da17640b92044084150317f02adf22442cb0c52af2a8b/boto3-1.43.118-py3-none-any.whl", hash = "sha256:ab8f3d2c622ef039338b4f228d97ea55f1abd573d84bb9b9a5ee8cb6780c0ccd", size = 140045, upload-time = "2026-10-22T19:22:08.353Z" }, +] + +[[package]] +name = "botocore" +version = "1.43.118" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jmespath" }, + { name = "python-dateutil" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d4/b8/3e55154ddaf1a6d1d4ddda85c0e5983205a3b7657edacd8ebba8c6fd774c/botocore-1.43.118.tar.gz", hash = "sha256:7f3f9d88cc9e847587690f82a42e3c3c292c3026c92c1e60d68a9310018e003f", size = 16420126, upload-time = "2026-10-22T19:22:04.81Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9c/ce/61b39a8e7a69af5ea43c1c3e1b0065a4866b42ccddf107e16686a3143614/botocore-1.43.118-py3-none-any.whl", hash = "sha256:a45d7ae570674161e4a3542d8c68633bf7503f92d1de781ef7361b79488eda6f", size = 16122016, upload-time = "2026-10-22T19:22:00.414Z" }, +] + +[[package]] +name = "certifi" +version = "2026.10.23" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fd/11/b524cbd9bbdaca58608e4cf4053fd463417549e1959fa8d3aa18319f9555/certifi-2026.10.23.tar.gz", hash = "sha256:a9895e6e2758b371e0545e321a3a8fd1fcfd9829ada8824a936dfb16aaaf36cd", size = 136225, upload-time = "2026-10-23T11:36:16.478Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/d1/a3652fe2c19134d0d15922177821d863cbd7e842d53a21bbe7e8e3a1b5ac/certifi-2026.10.23-py3-none-any.whl", hash = "sha256:e9b54de76e530bb6cf0ccd036787103acb58fd42ba930ea8bf948ddbaa2f61d1", size = 135082, upload-time = "2026-10-23T11:36:14.763Z" }, +] + +[[package]] +name = "cffi" +version = "2.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pycparser", marker = "implementation_name != 'PyPy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9e/ef/008a1939e372c06329a3fce4279c02f328488f3526744906eeec3da7ad5f/cffi-2.1.1.tar.gz", hash = "sha256:dd31f52ea1086513bb9df30f8fcee9b8918323ae067a3d5b78bc826a000712be", size = 530807, upload-time = "2026-08-03T21:21:18.939Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/10/69/43965eccfdead3b9220015fd1320e117be8c6ed01a62ffab76eeb752f5d5/cffi-2.1.1-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:c8c69575568085ba0b1b10c0249d779a214aea6f6522e949a0fc9fb0fcb449d0", size = 184821, upload-time = "2026-08-03T21:19:44.887Z" }, + { url = "https://files.pythonhosted.org/packages/54/7d/16e5a096677b5e313ca80cd5e5170efa3ea44624a82bb111925522da64b1/cffi-2.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f81b3b8f3d4e343550fa4baa0e479bba9f2d29ce9c2e9b51d1ce1718d7442fcf", size = 184719, upload-time = "2026-08-03T21:19:46.129Z" }, + { url = "https://files.pythonhosted.org/packages/56/e6/8941622732edec876dd17d0453dce07317ae96db34f2ec1436c9d3785986/cffi-2.1.1-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:811bd1e21d32de12efca32393a0ab3f5133b54fce9bd44b8bd77ab07da14bf6a", size = 214799, upload-time = "2026-08-03T21:19:47.218Z" }, + { url = "https://files.pythonhosted.org/packages/44/de/f98430906df1545ffde0d543dd124a7a439bc2cd32b36b9c53f805df7333/cffi-2.1.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:68e62fe11f30d5ca8289242866f0a5291402d8529ca2178ab8afc5c9694ae890", size = 222389, upload-time = "2026-08-03T21:19:48.331Z" }, + { url = "https://files.pythonhosted.org/packages/6a/5b/717f1526b9957b34456313c31645c5b82b8fb5c3fe9e4752999be7128bfc/cffi-2.1.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:4a7c934f7360e8cd64fe9efadcbd10c7c6364f531e432b9a4bf5ccbc9e0e8b50", size = 210249, upload-time = "2026-08-03T21:19:49.543Z" }, + { url = "https://files.pythonhosted.org/packages/64/b3/f8aa4f3e34986c7e4ec45072d1b1b9dd295b6b18007b45518d79726dd725/cffi-2.1.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:3143d81e29e1e20a9ce10901ec369012947876596f75a222235965f2b7ae832e", size = 208775, upload-time = "2026-08-03T21:19:50.918Z" }, + { url = "https://files.pythonhosted.org/packages/b1/db/dceb9dd5b231e1da801793f8acc9f3c52a7e1afe40bb1aae37e02b0faad5/cffi-2.1.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c1453022f490d2459a11819d83ad1d586e9ff65a12ac3e705ffebd46d3685dcf", size = 221822, upload-time = "2026-08-03T21:19:52.054Z" }, + { url = "https://files.pythonhosted.org/packages/a0/d2/6cd24ae3be000a634109c247d1475d62e5616d0dc78c82770942ec384248/cffi-2.1.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:208f941bb9d18e768138677f0a6d2ce01f590df56043dda1df1535ac57c88517", size = 225232, upload-time = "2026-08-03T21:19:53.109Z" }, + { url = "https://files.pythonhosted.org/packages/cb/52/3fa190537004dd7f0ab860a6dc7c0175b8667f68d1e618a46f5498d30250/cffi-2.1.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:210019b6c7cf07f081b4c54635c8cf744377001350e29cc0f81c4377b4797735", size = 223597, upload-time = "2026-08-03T21:19:54.515Z" }, + { url = "https://files.pythonhosted.org/packages/80/fb/0bb75b7039588c074b37ae99f40d9bfddf990ecb2fbc346ebccd2e56b9be/cffi-2.1.1-cp312-cp312-win32.whl", hash = "sha256:046bfc24911b37851ee1b51aab8bffe713d89c68c6a057b09484ce9fd5f69b4e", size = 175292, upload-time = "2026-08-03T21:19:55.566Z" }, + { url = "https://files.pythonhosted.org/packages/d9/79/615cc094e2fb508cade7de88d3b4f6c4ec2bab695c97bce9153dc65aadf5/cffi-2.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:f53e442b08449d42821fa4a4fba000095af9f62742a500f978a9f557ec44339a", size = 185919, upload-time = "2026-08-03T21:19:56.89Z" }, + { url = "https://files.pythonhosted.org/packages/70/c6/d0ea84713fe46b243a436a18fcd47d639732747e21635c8a27191b06dc30/cffi-2.1.1-cp312-cp312-win_arm64.whl", hash = "sha256:7bde5e4cc5c10140859842b9d383af292b22639a4dffb725314baf45968cef80", size = 180093, upload-time = "2026-08-03T21:19:58.155Z" }, +] + +[[package]] +name = "charset-normalizer" +version = "3.5.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/33/1c/f41d4e74c28ab327ff3acd36053f7ea506c55872d7a90b0fa71aa3ab0c89/charset_normalizer-3.5.2.tar.gz", hash = "sha256:39de2a259fc954455c57274dc94c79d5842774e1247a016aff30bc0efed0f4ef", size = 172659, upload-time = "2026-09-30T04:39:23.398Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e7/c8/693809898870237d82785a03f3b2b58fe4c9f14669f84a7d4e623c92a59e/charset_normalizer-3.5.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:ed2a239c0ea213acc1908150a3037257083c7c083128f1a4cec2ec4b97dca491", size = 367780, upload-time = "2026-09-30T04:35:30.888Z" }, + { url = "https://files.pythonhosted.org/packages/c9/87/2fea8c13dc24b3ca9c6f803a5b2dfdeae73eb4f9e12c7885ed908ff0433c/charset_normalizer-3.5.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b91363207bd9dc966a691e959bb47f64b30f7ac4b072be9968b366982f7db77c", size = 246730, upload-time = "2026-09-30T04:35:32.286Z" }, + { url = "https://files.pythonhosted.org/packages/a8/9e/09efac30b937722f46d3110ba30b875b24b2e3a266ed746cc4e376a94d80/charset_normalizer-3.5.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:38a873987f3be698494da8b2e3085e29da02da7b633dce73e79c699a113d7bf0", size = 237707, upload-time = "2026-09-30T04:35:33.709Z" }, + { url = "https://files.pythonhosted.org/packages/9e/18/70d76670b13686237863a379928d60bd10e021f17d243ab3d7014c4a5f4e/charset_normalizer-3.5.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:355ad8011081dec5412240c087a9a0c9d4d5039f3ed11a3f13e18c2b29b56c51", size = 273050, upload-time = "2026-09-30T04:35:35.138Z" }, + { url = "https://files.pythonhosted.org/packages/54/e2/77a8b09d5adc013ed07b95b01b8b8fa5441c4e810e83ee7e4aae2fa4d91a/charset_normalizer-3.5.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ee21e28f0430bd6dc9086c6e525d5e818a44a5ad19720c8a0ef766792f3eb5e5", size = 270345, upload-time = "2026-09-30T04:35:36.502Z" }, + { url = "https://files.pythonhosted.org/packages/7f/c5/38806a25ab5e65fc178f39affeda20858efafede2fce1ffc2556cfc9fe73/charset_normalizer-3.5.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3d31298449090ab8d47b7b1b2a555ff73cac7ed438a08b7ac160980c7ebed649", size = 257601, upload-time = "2026-09-30T04:35:37.919Z" }, + { url = "https://files.pythonhosted.org/packages/ae/8d/213565184708fdb263ae55e2c04ee1ff748129dd65d48ed0e3502da9c85a/charset_normalizer-3.5.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5cde776b7cc66e4f6c99612cea4aa7269aa65863f7a15841b2c264f103822f4e", size = 252222, upload-time = "2026-09-30T04:35:39.544Z" }, + { url = "https://files.pythonhosted.org/packages/7e/24/76d2cefc25472531e4c5c7dfff68865eb1c39b78482f0fdc15b46f047830/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ae4f5fea5b8b8ccff88238cc8569303e5ee95efae67fa62922a311397a71f346", size = 248482, upload-time = "2026-09-30T04:35:41.088Z" }, + { url = "https://files.pythonhosted.org/packages/7d/dc/65a801b66ab4c197e22c433ab25e7ac24324ac6f45a2269aca42cce309bf/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:f7d486c83842422badd511868fd8a9a20e9407ace71564b6af47ce7e60a336c1", size = 241206, upload-time = "2026-09-30T04:35:42.59Z" }, + { url = "https://files.pythonhosted.org/packages/a7/95/ca9b5eabde673002c6f1e7ada1b223916fe18f6d661da7aabd4d643718f1/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:11a4d68a6ecda3292cb1e50239e111543ba5d709bb62a6b4ea1afcfa729d8875", size = 273190, upload-time = "2026-09-30T04:35:44.347Z" }, + { url = "https://files.pythonhosted.org/packages/2d/8b/803b4d2a3f6e1740f63f1e87b04d14b42f3d4fdfe6ed7d4db2d34102b14f/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:d6734d2ef8a50fbf8445c139477da401f50d62a0606bf00e20ec6d87773fefb1", size = 253527, upload-time = "2026-09-30T04:35:45.915Z" }, + { url = "https://files.pythonhosted.org/packages/a9/55/93c0e5dbd085ae0471346026abbe7e0db9ea2d6fea74e51f0b5a46f233a7/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:a815775b6c38d4e0ff7bcffbeba67feded90202bb6a226b8dd35f1c855217413", size = 271285, upload-time = "2026-09-30T04:35:47.49Z" }, + { url = "https://files.pythonhosted.org/packages/95/69/0dbd0e0b9b16cfa816cdfcb3e2e3854a1f680dc07fb1245ea125e7448060/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:23851fb4e1b85ed3f6c2a27b777cdfe2e19fb5b38429a8faf38c7542b7665869", size = 260010, upload-time = "2026-09-30T04:35:48.996Z" }, + { url = "https://files.pythonhosted.org/packages/58/9d/e7b88e7b1bf403590c3b573277b5e1e488c68c7a6fbacca310a2c324e90c/charset_normalizer-3.5.2-cp312-cp312-win32.whl", hash = "sha256:db19d07e2e0129e974a0e65d0064fc222a446cd5122c2fd4184d2af9fc734a9e", size = 184125, upload-time = "2026-09-30T04:35:50.777Z" }, + { url = "https://files.pythonhosted.org/packages/eb/e6/e6e083884cbcfd49c64865af05027fe7011be7b2d9179524f099a1b611f3/charset_normalizer-3.5.2-cp312-cp312-win_amd64.whl", hash = "sha256:780fbe7cab297b81dad9fb8dc5eb003c0468ffb0d9e5f65068c53a34661a96bc", size = 207486, upload-time = "2026-09-30T04:35:52.194Z" }, + { url = "https://files.pythonhosted.org/packages/c4/e3/017aea0911ada7405a825c7d937eb3a13009664e2f5b38e8c4bbf2abf894/charset_normalizer-3.5.2-cp312-cp312-win_arm64.whl", hash = "sha256:e2af3aad578aa6bd1384bcf4750fc285e5a9de53f40b7d41e5a0bf748edeb2b3", size = 196734, upload-time = "2026-09-30T04:35:53.636Z" }, + { url = "https://files.pythonhosted.org/packages/8c/ab/176fbfd5b64939c55d652366aa5b9ef1d767af207a3aa6ebeb0d226c484d/charset_normalizer-3.5.2-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:4275811936e2f06feff5e598fb42a1b7ae852da8e39605211892b56b81a34efd", size = 331815, upload-time = "2026-09-30T04:38:26.216Z" }, + { url = "https://files.pythonhosted.org/packages/7e/84/371eac6b30bdbcbf2d632a1a01809103459216fcaae61b8b8d922c1bfb8a/charset_normalizer-3.5.2-cp37-abi3-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:1c50fe28bbc2ced33386f298650d91218076c05420e6cbd790b913adc41659e7", size = 253276, upload-time = "2026-09-30T04:38:28.032Z" }, + { url = "https://files.pythonhosted.org/packages/43/6f/c4fbae58febff71709c51bc7e18fdfa55341dc382704740f9f0cbf03817b/charset_normalizer-3.5.2-cp37-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d19fbd981a488e22cd04883659ca6b08f50b5974f9fd7c95655ef6a043e5893f", size = 241239, upload-time = "2026-09-30T04:38:29.732Z" }, + { url = "https://files.pythonhosted.org/packages/61/71/458c3f42164a07d0c5210798e9e704b39e540a6793b05aba67f3a35243a9/charset_normalizer-3.5.2-cp37-abi3-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0fed1d06615f022ee3b13caf5e8b180cfea32bb2c5aded8a9d44277afc040f93", size = 231121, upload-time = "2026-09-30T04:38:31.462Z" }, + { url = "https://files.pythonhosted.org/packages/09/54/ab9e89367076f6331bb6c65c4bf14a5361fa5191cb6561bf534f18504e1b/charset_normalizer-3.5.2-cp37-abi3-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:838dcc90063569a0448120554591a1d6c4a4ffe11babf048908793154ab86ade", size = 260350, upload-time = "2026-09-30T04:38:33.239Z" }, + { url = "https://files.pythonhosted.org/packages/7c/c1/061431ecc688d9d76602502cb57cc01e691e682c18f1beb45f9673b5bbd2/charset_normalizer-3.5.2-cp37-abi3-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2ce45c6627b22c47e390bc91a41c3d13032192e699fa0bea96e9671b373d69b0", size = 255430, upload-time = "2026-09-30T04:38:34.865Z" }, + { url = "https://files.pythonhosted.org/packages/8d/1f/20c8949f0676f7ab811abdeb7f4d7f1cbc6e61ff20bef08b44edeb092bc8/charset_normalizer-3.5.2-cp37-abi3-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0774bf9bf620249fee3e0b8b9fd3065de213be30f3aa94ce2494b3b638949e26", size = 250612, upload-time = "2026-09-30T04:38:36.649Z" }, + { url = "https://files.pythonhosted.org/packages/2b/9e/46f2fa4c431fc98c4ae76a8cb5bdca54e0341e3cfc3fcfd8e82740250818/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1db38f4c5496827c1a501846d64d14c3b80c7e6714e406cd7dc36a9899fa1011", size = 242083, upload-time = "2026-09-30T04:38:38.26Z" }, + { url = "https://files.pythonhosted.org/packages/bd/39/559be29a0c0f086e0bba6922babd38916cc5e0b58ced4de13ee01ea05508/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:304d8e4d493af723536393eee0c689eb7813f4a474c8b479dee63f1fdd98f621", size = 232738, upload-time = "2026-09-30T04:38:39.81Z" }, + { url = "https://files.pythonhosted.org/packages/ff/6c/387b0e4f756a282831c1d9fc6aeb6c51ca4507ca202767c8de15ce9b12e2/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_ppc64le.whl", hash = "sha256:9b7f416ff0978e2f2249330527f0ad6fa02f4932e6199692d3b52da2048c19e4", size = 260703, upload-time = "2026-09-30T04:38:41.346Z" }, + { url = "https://files.pythonhosted.org/packages/96/92/1fdf015f09ef449f50d3ac4b67c90887c9c318b727daa95cc4f866e6521d/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:01077390b03f7988f11d700a2194e69b119741a86b1a638b1db88891e3eced8e", size = 247622, upload-time = "2026-09-30T04:38:42.937Z" }, + { url = "https://files.pythonhosted.org/packages/dc/3c/8e7b8a5671ad5d433669fb2a76f1a0164df2d9b1718b0206bc2a16d840cc/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_s390x.whl", hash = "sha256:7e841fb9010836c992c9f12fcbd43a831de93a5f726fc1ccd8ca1d0268c5014c", size = 257500, upload-time = "2026-09-30T04:38:44.604Z" }, + { url = "https://files.pythonhosted.org/packages/b4/f0/45b579df5cabc1d5d53ea1cc35e8437d3ca768c0acccc7041517cb6fbb32/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:9cae88599c7219005d879f98e5ed53341e9a122af585e1091200358a3003d2a0", size = 255100, upload-time = "2026-09-30T04:38:46.289Z" }, + { url = "https://files.pythonhosted.org/packages/31/68/fdec18a343f5fb3f310588dd478b09ac4799e0b187dbade3a8cd776f03ef/charset_normalizer-3.5.2-cp37-abi3-win32.whl", hash = "sha256:01b0c0d2262a9e28e8484a278c7e1b5d650e3ac8cf2683d2967e25899f208bdf", size = 174499, upload-time = "2026-09-30T04:38:47.999Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8a/b618149cc5207943a0242068d7a27897f56a62947b5a039085f2a22029f8/charset_normalizer-3.5.2-cp37-abi3-win_amd64.whl", hash = "sha256:9f56f72050826f63dcee7a7f55b0a77168cb3bfc553fd405e7f8f9ece75a4036", size = 200092, upload-time = "2026-09-30T04:38:49.707Z" }, + { url = "https://files.pythonhosted.org/packages/03/cf/4c66866fa9e2b1c78e3c911516d1de497a677b7ac60f1eceda74ce777ca3/charset_normalizer-3.5.2-cp37-abi3-win_arm64.whl", hash = "sha256:40ab6bffa02ae10a0581e6c198be7d2d8ca5c2a0c64e4ed3465d766df457573e", size = 294363, upload-time = "2026-09-30T04:38:51.312Z" }, + { url = "https://files.pythonhosted.org/packages/fc/ad/d07d7862a62ffa6d79d68074d14823243dd235a77c45262acbf6adeb28bf/charset_normalizer-3.5.2-py3-none-any.whl", hash = "sha256:b6b751274acb69d77b3323d6b7dbaa3c7fdfc1eb829b7eb61d262f32e1af9685", size = 68872, upload-time = "2026-09-30T04:39:21.828Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "cryptography" +version = "50.0.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9d/af/182eb91b0df3fe75c4d9f26fe70684569566745f6ba7e5c9c73a862c5252/cryptography-50.0.2.tar.gz", hash = "sha256:7b46165bb56eb4704e2eaaf86f3c940d19154535d9b0ca7d6d590b04060e00d5", size = 880623, upload-time = "2026-09-30T15:30:04.884Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/56/d194340cc4a57535e82e1bee9e89667ac4b7c13b5d3f59686deae3094dd5/cryptography-50.0.2-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:fa8f5efb344d6908a1ce62f4a24e2e5780f825d6f53f5f50ec5ffacac72936cb", size = 3914904, upload-time = "2026-09-30T14:43:44.339Z" }, + { url = "https://files.pythonhosted.org/packages/d9/69/c9bd862c3bf43d6399c433caf002df16e2dffd4be49bdf515cda38038711/cryptography-50.0.2-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:79def8d059362e7831389ed3be0ecdf58a89386e1271e35dd9f5af84e81bffd0", size = 4731146, upload-time = "2026-09-30T14:43:47.113Z" }, + { url = "https://files.pythonhosted.org/packages/21/69/64cef1f702bf6657e0cc186ed1a2891d50d29fb41586b254e1c07adea261/cryptography-50.0.2-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:630ebfea3bf689d075f82316324ff7433dc447fe6bc1bfc76524b74b4a9567d2", size = 4719841, upload-time = "2026-09-30T14:43:49.01Z" }, + { url = "https://files.pythonhosted.org/packages/38/6b/61a3f8d8c5e1e49a6cddccafc4015cc1c0021360ab0acb4080e7a423644a/cryptography-50.0.2-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:f9f6143a8c75945eb960d9eb98905a441394abfa24afaae239d514ffb2586480", size = 4738340, upload-time = "2026-09-30T14:43:50.932Z" }, + { url = "https://files.pythonhosted.org/packages/7b/2e/7212ca32fd43dc91f2f41db20160b268098874b4c9a0e7be94d6835f5b2e/cryptography-50.0.2-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:a582ab2ae1d34f67112cadc86702774c9ea4374df6bca6afe672817203c99134", size = 5367029, upload-time = "2026-09-30T14:43:52.911Z" }, + { url = "https://files.pythonhosted.org/packages/1a/f1/b474e930c4d910328780e3940da76f5aa5cbc48ce1fc14e44d239d9ea9db/cryptography-50.0.2-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:4061c0079120205fb760c58acab6443e217307dcf05e3702cf970e0689972856", size = 4753050, upload-time = "2026-09-30T14:43:55.272Z" }, + { url = "https://files.pythonhosted.org/packages/7c/52/9af10e80ac16b0fcc2123f9cbd5e7afbd0fd5075bb7a607c592258a39cda/cryptography-50.0.2-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:ac9ed99d81760c62fe89d5f0815cdfa1ba9a35141cf30f1c2d044f04b4803d2e", size = 4376724, upload-time = "2026-09-30T14:43:57.24Z" }, + { url = "https://files.pythonhosted.org/packages/71/37/6202e488cc1eb625ea110c292c6bda92823176e023f427d8d5660ce8d632/cryptography-50.0.2-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:87e9ce85beb6b328ba370cc6e6aea483c92617b4c95b1d33a49297eb662bfb04", size = 4737859, upload-time = "2026-09-30T14:43:59.541Z" }, + { url = "https://files.pythonhosted.org/packages/8f/30/e86d7d518489b0ae2497091a35287abcb1a2ce4037837a34afbe9b1d6964/cryptography-50.0.2-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:f265528741e048bce55c3463ed721fb0aa45a5888d8add8cfeccb3035451bbdc", size = 5324103, upload-time = "2026-09-30T14:44:01.901Z" }, + { url = "https://files.pythonhosted.org/packages/d3/69/2c833a049475e0a3444e94c7d0aca0aa51d166374a449b09e92ac98138de/cryptography-50.0.2-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:9dab55f57c74c3cad24c323bacbbd04be4705ba6eb0d92e920b1fc4837ed5079", size = 4752576, upload-time = "2026-09-30T14:44:04.545Z" }, + { url = "https://files.pythonhosted.org/packages/6c/5d/906970b83bbfc1f5bbfb677a143c181f2801f23b6a7204a3b47c42c97e65/cryptography-50.0.2-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:25784ce8b9621c90c643efb9e1e2162ab3b0224cae446ad5e70e7fcb1ce18b51", size = 4870819, upload-time = "2026-09-30T14:44:06.884Z" }, + { url = "https://files.pythonhosted.org/packages/68/e3/f2298d3bb55e0c4a91841ec4d01b3f020ba8c5fbf15ccdcc6dcf03f97025/cryptography-50.0.2-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:85d0d9a31b9098e98534226d5686b47264b95e62ce459dc2e62fdfc809f9fe93", size = 5030152, upload-time = "2026-09-30T14:44:09.443Z" }, + { url = "https://files.pythonhosted.org/packages/9a/4f/adfc442765721292fff86d314ce385d3249d22db42295c0dd057727b60f3/cryptography-50.0.2-cp311-abi3-win_amd64.whl", hash = "sha256:7afa5a6602a9f29af1f3a2965f831bae7c9d5d597b7cbb716d41ab3b7d89879c", size = 3824692, upload-time = "2026-09-30T14:44:11.671Z" }, + { url = "https://files.pythonhosted.org/packages/2d/49/93f6a6e7a87c9aa68d44d3e1cdb5fe8f60c90d5d2f46acae9a56892816b8/cryptography-50.0.2-cp315-abi3.abi3t-macosx_11_0_arm64.whl", hash = "sha256:edc3342adf8f697fc5f59c887a304356f147b397809440ed64e2fa6af2f50f37", size = 4133708, upload-time = "2026-09-30T14:44:41.807Z" }, + { url = "https://files.pythonhosted.org/packages/8c/75/32ac2a56243d778805c16ca6a32b8f74fb757df7e28d7ecb560afafb59cf/cryptography-50.0.2-cp315-abi3.abi3t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d370b8d1dfcdf7130178137f6fbee6140774a1acc6cacefc4b42643ec11d0a3a", size = 4956267, upload-time = "2026-09-30T14:44:43.693Z" }, + { url = "https://files.pythonhosted.org/packages/aa/a4/2c8d734e43d97f0842ee9f1b7b4bfb3d0cf5e19edebf43c2afe6675c2320/cryptography-50.0.2-cp315-abi3.abi3t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f2f9bd7f90c64fe89253f0a2c05e3c4856072660429ce8831b4235bf29403a67", size = 4966465, upload-time = "2026-09-30T14:44:45.769Z" }, + { url = "https://files.pythonhosted.org/packages/c2/58/ee288c829a6f41f6235ae9dd33d82fd19b45442b65b4c8a3da36963d9f7a/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_28_aarch64.whl", hash = "sha256:e275096ea1e60cc595cda2836fd4a6c725d1125108b868be17f53684d164e2cc", size = 4959356, upload-time = "2026-09-30T14:44:48.211Z" }, + { url = "https://files.pythonhosted.org/packages/92/20/9ded6d51ddd9897f6b6e81fb9ebea7951d7cc5d6c890b0ed8abf77a51a80/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_28_ppc64le.whl", hash = "sha256:b13478603dcd0a2479ff8e87e2c19a7d525734686fe3c49542472293a204212d", size = 5548822, upload-time = "2026-09-30T14:44:50.86Z" }, + { url = "https://files.pythonhosted.org/packages/02/a8/8df951850d6b31d2a00218f19e2b3f999523437ed7a819df7fa427942fca/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_28_x86_64.whl", hash = "sha256:58a0c478eeca76fe5e07993c5a0703def34a6dc6a0cda4f5564639b33112ffe7", size = 5001199, upload-time = "2026-09-30T14:44:53.379Z" }, + { url = "https://files.pythonhosted.org/packages/8b/f9/36b3022218ce75b7cdf068fb95f809f9bd0d820e4955ef43b90c255cc7ac/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_31_armv7l.whl", hash = "sha256:d38cdff612d06fa6a32840d5e1b1f7a27cee4a349aa9085d94a67789d6bfd408", size = 4629333, upload-time = "2026-09-30T14:44:55.635Z" }, + { url = "https://files.pythonhosted.org/packages/8c/72/20f99a219f6af47cdd1cbd978c243b92d71496e168a746138af44ded4f29/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_34_aarch64.whl", hash = "sha256:fdd28f912fccfec1846a94e2e1e8f9b0012f557f0c46fe4f3eb0d7a87afcf90b", size = 4958822, upload-time = "2026-09-30T14:44:59.639Z" }, + { url = "https://files.pythonhosted.org/packages/f2/20/196f112617fb08eb4d608a2a6c422373d46f9cc2857f38fc0667033c0899/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_34_ppc64le.whl", hash = "sha256:cbc8738fd8526d80f35cb3a40d41f41a2e7030bb3b18b09a6778ef63d291c2fd", size = 5506351, upload-time = "2026-09-30T14:45:02.267Z" }, + { url = "https://files.pythonhosted.org/packages/24/95/83378121ef3eaaaf71d4b781577ff794acb39b9e1b87a3f156898c8497ed/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_34_x86_64.whl", hash = "sha256:e105ab60406787da31fccc883fc0f733af1efd78f0136a4599692c4083a73d0c", size = 5000859, upload-time = "2026-09-30T14:45:05.009Z" }, + { url = "https://files.pythonhosted.org/packages/22/f7/70fd7ae4d1dbfa7ba29b02e1b9068771519a86027756510b700ce81086a8/cryptography-50.0.2-cp315-abi3.abi3t-musllinux_1_2_aarch64.whl", hash = "sha256:6f8700550aa1474a91e5dc07049c46f98b423b5b1ddd0483e0b51362eeeaf5be", size = 5092151, upload-time = "2026-09-30T15:29:15.932Z" }, + { url = "https://files.pythonhosted.org/packages/d4/be/688367b74de86984bd58d8efacfc7c9e68b89a6a22ced0fb4f38db50254a/cryptography-50.0.2-cp315-abi3.abi3t-musllinux_1_2_x86_64.whl", hash = "sha256:c71be1cbfa5cd9a41ee452acf1eccd82b2c05950358b106ec8ceb83411d1a020", size = 5286120, upload-time = "2026-09-30T15:29:18.309Z" }, + { url = "https://files.pythonhosted.org/packages/39/d1/55f8a3f2ef5d1529e16835ef10cf0fe3d559ce237b46dddc440c0bba3649/cryptography-50.0.2-cp315-abi3.abi3t-win_amd64.whl", hash = "sha256:c423ab384a46c4dff7217b2ea5ba2e11cffdeab6441acd04cf65a369caf0366c", size = 4111557, upload-time = "2026-09-30T15:29:20.155Z" }, + { url = "https://files.pythonhosted.org/packages/23/ad/ac987755d00e1e64273760228d2635ae38dae2be83e3c6e0d3289d91dec3/cryptography-50.0.2-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:0ec5f09541743261e66e291b4a0cbf0fb2997aeaab6d9e9c740b9dba1b58d1c2", size = 3943588, upload-time = "2026-09-30T15:29:22.265Z" }, + { url = "https://files.pythonhosted.org/packages/d5/8d/6d585339bedf85d45044c85d8412dac53f2bb6f918e8b7777efba1787844/cryptography-50.0.2-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c5e67125c7dca78d199ec4e116aa93dbb83494808ecbb8211a2cb09b1bf41dbd", size = 4756166, upload-time = "2026-09-30T15:29:24.58Z" }, + { url = "https://files.pythonhosted.org/packages/bf/f1/1c1f6874e8550cfddd4b688ceb38cefb6ed15ceed224d56f133f3d88c214/cryptography-50.0.2-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ee247f5c245c9a2fe7c8e2214e295918838e44e00a45a6718451e4004219e767", size = 4749145, upload-time = "2026-09-30T15:29:26.807Z" }, + { url = "https://files.pythonhosted.org/packages/c1/63/61b15dc1a8de03fe0adbe3fd7608b3ad5c73bf50993bbcb1faaa930afe33/cryptography-50.0.2-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:dfe9763530994147d9af1def057a5b9658b00e8f8fe8743d144d1e0911c2e454", size = 4763638, upload-time = "2026-09-30T15:29:28.588Z" }, + { url = "https://files.pythonhosted.org/packages/fc/35/b345bdfa40c9126df1a9d33236aa98418367931b8725f84fc3ae2b98dc59/cryptography-50.0.2-cp39-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:58ddb5a8e3179d12f19e4ea34d2d32e9d63a4baa142c875c1eb59f41b7243acd", size = 5382217, upload-time = "2026-09-30T15:29:30.589Z" }, + { url = "https://files.pythonhosted.org/packages/4f/87/ef344a9e616871f2519c22d6afcda79ddd5d35e9592d95eb6e677608d055/cryptography-50.0.2-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:f21e8a22c8605750c7af886bab299a363721264061b4ac0a30efb73cfd58efc5", size = 4781387, upload-time = "2026-09-30T15:29:32.605Z" }, + { url = "https://files.pythonhosted.org/packages/90/5b/f2fdb13cd0b96f6f932c8627bb292a45f11c64d21620a8e120aee9a3b848/cryptography-50.0.2-cp39-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:9c8402a82ea0dc4ceeab793db05f0fafa8ca139ca34fcde5df0f596103c74107", size = 4403790, upload-time = "2026-09-30T15:29:34.374Z" }, + { url = "https://files.pythonhosted.org/packages/bc/ce/7e4f662b1e3c393513569e402cfc85ac7da0bd3d5435e122a3140219eb2d/cryptography-50.0.2-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:0ddc924c04591c2811ca024d62ecad4f7f6f08af8939c211438f48a16bd23602", size = 4764319, upload-time = "2026-09-30T15:29:36.149Z" }, + { url = "https://files.pythonhosted.org/packages/3c/3f/86ff33ce34cc0de6847fb96e035a1a760d81652e38643f617c02ad32ef7a/cryptography-50.0.2-cp39-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:a6557e5f38e065ca9fbdaf7cfc7435ecb1d113aa81a022d1b51921ee7432e227", size = 5338560, upload-time = "2026-09-30T15:29:39.053Z" }, + { url = "https://files.pythonhosted.org/packages/40/cf/6b5c8e2fd9202d98988ab7cb5cc5c991704c4ad55f492ff408e4969f83f1/cryptography-50.0.2-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:1981f1db4630889b9ef7803fadef12b056f428cb6b85c27ba57b774793b6093c", size = 4780973, upload-time = "2026-09-30T15:29:41.251Z" }, + { url = "https://files.pythonhosted.org/packages/10/bf/8d6ebc7dded797bd0f0160d52188021211f011a2b164ef0ae1dac4587465/cryptography-50.0.2-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:7a8701d6b584d76e909e3d305b7d126b41439876a5aaf76cddc67fc230eafa2e", size = 4897738, upload-time = "2026-09-30T15:29:43.106Z" }, + { url = "https://files.pythonhosted.org/packages/d4/aa/f3f6e0de7e6253b8baa8b2d8fb9d50924fa75cee3d4624bd4bc1208ee923/cryptography-50.0.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:ce47f66801c20ec6c6632453bb5960fe38939e9306970b48b3a5a26de7745d94", size = 5058280, upload-time = "2026-09-30T15:29:44.827Z" }, + { url = "https://files.pythonhosted.org/packages/f6/b6/a1faf3a27ae9405fb34b1713cc73b2d8a26b04d5c561578fa2e6ef3e5bb9/cryptography-50.0.2-cp39-abi3-win_amd64.whl", hash = "sha256:4e81d95e5bafc2d6e34e4bed780e53e4d5b9a2f928573428aa4d35fbec1eb0de", size = 3854095, upload-time = "2026-09-30T15:29:46.782Z" }, +] + +[[package]] +name = "execnet" +version = "2.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bf/89/780e11f9588d9e7128a3f87788354c7946a9cbb1401ad38a48c4db9a4f07/execnet-2.1.2.tar.gz", hash = "sha256:63d83bfdd9a23e35b9c6a3261412324f964c2ec8dcd8d3c6916ee9373e0befcd", size = 166622, upload-time = "2025-11-12T09:56:37.75Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ab/84/02fc1827e8cdded4aa65baef11296a9bbe595c474f0d6d758af082d849fd/execnet-2.1.2-py3-none-any.whl", hash = "sha256:67fba928dd5a544b783f6056f449e5e3931a5c378b128bc18501f7ea79e296ec", size = 40708, upload-time = "2025-11-12T09:56:36.333Z" }, +] + +[[package]] +name = "filelock" +version = "3.32.7" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0f/59/e19834834cb01a32febfbb0f8a23a9088088f5d45991824ff2bc3b5e8acb/filelock-3.32.7.tar.gz", hash = "sha256:37b8a3d9811b0f9aef7e5ec5c71bb320de52df51e6ca9bcd6f5ad81187660da7", size = 225154, upload-time = "2026-09-16T00:24:20.907Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/15/df/31098c5aeb4d966b553641472bd55fcf5fdfac953549894b8a765ba44e91/filelock-3.32.7-py3-none-any.whl", hash = "sha256:65ff0d0190ea42038b32bda4b77834fb05be2cad4c5b9b01aa4dfb3614536e52", size = 100157, upload-time = "2026-09-16T00:24:19.543Z" }, +] + +[[package]] +name = "idna" +version = "3.20" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f5/08/8eea9d4b8302028f3abb2c0813953f7aec26d33b7a8960ed760e65ff29fa/idna-3.20.tar.gz", hash = "sha256:a7db850025b95ded1eae8a46181a1a6c56c92c96f0e2b005d9ff8dc0210cab44", size = 216463, upload-time = "2026-09-17T14:11:04.752Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/58/a2/bb081bab032533a855d44de1d56f8e8426114ff1ba5d1f07a438a0a654f8/idna-3.20-py3-none-any.whl", hash = "sha256:ab7ae7122974553370f0bdb919e1a960b2cd1bc1ef0276416d896db81c14582c", size = 69583, upload-time = "2026-09-17T14:11:03.168Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/e1/2069291243c926a2ff1cd706c7f3eeb9b62144bf60f77c9fb9ff2fb26bd3/iniconfig-2.3.1.tar.gz", hash = "sha256:67f4b9c50da0dedf52af349e7749a80a9057a5031199791b906c3bb3ae878960", size = 21209, upload-time = "2026-10-06T22:48:38.076Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/56/43/4ca9e49d27a1fcf6bece6f6aec0ea46bb9112489b93d4b688fb415457bdb/iniconfig-2.3.1-py3-none-any.whl", hash = "sha256:9121e2c1fdb355232495be3194c8dfe87ccc2d5dee45947b78e68f499790d7a7", size = 7552, upload-time = "2026-10-06T22:48:36.959Z" }, +] + +[[package]] +name = "jinja2" +version = "3.1.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d", size = 245115, upload-time = "2025-03-05T20:05:02.478Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, +] + +[[package]] +name = "jmespath" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d3/59/322338183ecda247fb5d1763a6cbe46eff7222eaeebafd9fa65d4bf5cb11/jmespath-1.1.0.tar.gz", hash = "sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d", size = 27377, upload-time = "2026-01-22T16:35:26.279Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/14/2f/967ba146e6d58cf6a652da73885f52fc68001525b4197effc174321d70b4/jmespath-1.1.0-py3-none-any.whl", hash = "sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64", size = 20419, upload-time = "2026-01-22T16:35:24.919Z" }, +] + +[[package]] +name = "markupsafe" +version = "3.0.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/38/9b/e422a865e1d5d57d0e509b4e0bf1c1a70a7f6382c29a5aa428df994c8bc8/markupsafe-3.0.4.tar.gz", hash = "sha256:2e9ad7dd851bf45fab9f75cbff4cb493fee9979e8d8c7c9c3ee119022518edd6", size = 153777, upload-time = "2026-10-02T23:07:22.29Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/81/09/4c59d56b8461ae8eb0d8ba34bb25b7e618547044679d58a82ef9b2479fc1/markupsafe-3.0.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:61631e08084be9e21a8967ec3139c7616ed7c5e9368e05c86d1b39562c8a57b6", size = 11658, upload-time = "2026-10-02T23:04:51.876Z" }, + { url = "https://files.pythonhosted.org/packages/a2/f0/d6613774d86fbf6d145751d43c59875e47a6f9f17daee0aef173bd36d90e/markupsafe-3.0.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0930db9bdc62d22944e10b066448bb65dc9abe9112880c7cab8da54db4284d5f", size = 12050, upload-time = "2026-10-02T23:04:52.931Z" }, + { url = "https://files.pythonhosted.org/packages/0d/f2/8f18e0b806eb13c1f8d07d917a720831ead54253a6dec011fbc78098a6f8/markupsafe-3.0.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6a45c3d514f2436064db00d7fc8778d888f0236ebfed649b53d13a59e69ad51b", size = 24363, upload-time = "2026-10-02T23:04:53.895Z" }, + { url = "https://files.pythonhosted.org/packages/60/ce/fa07dbe8a5675558fa36dea033e19995bc783de2dec5f540ccb9030b06aa/markupsafe-3.0.4-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:1e1451fab512d1bcc3dc26988ec1edb0b82c2db909132872cd9356070a6b63df", size = 28525, upload-time = "2026-10-02T23:04:54.905Z" }, + { url = "https://files.pythonhosted.org/packages/85/40/be87c01f3868ec217f8a2015089d71c22c8c5a75324822e5ed1cdd87210d/markupsafe-3.0.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:bd3ce56ae2cbae3ba82b683bc425cd7e48d2ed8b10f3e818186b6f5646d9271c", size = 24733, upload-time = "2026-10-02T23:04:56.229Z" }, + { url = "https://files.pythonhosted.org/packages/4f/a7/aeedb5140afa41fc74c225e9184ab96723a6e873b6ee1c9fede7283456d8/markupsafe-3.0.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8e124f974786f831d6043728e38296969d3579db8896fe004682f5758e613581", size = 22985, upload-time = "2026-10-02T23:04:57.521Z" }, + { url = "https://files.pythonhosted.org/packages/c3/fc/e91352bb08c6a59da3ef0909d457bf95a5f5908fbf151b30a06d9dbcfbb4/markupsafe-3.0.4-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c02e8f18bdedba082cef725942ac823b9b60656db07f7e265cb31618dfd00d77", size = 22001, upload-time = "2026-10-02T23:04:58.597Z" }, + { url = "https://files.pythonhosted.org/packages/5d/f8/bffee5e7d2a3deb59748a797650a48af7e672025cf641a79344a771ad106/markupsafe-3.0.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:9f098115c247e11d138ab83a28fa0323c77015007ea2df73ba5fd714dfefd67c", size = 23793, upload-time = "2026-10-02T23:04:59.686Z" }, + { url = "https://files.pythonhosted.org/packages/ed/59/b853d6628ecb4d658e1d637224846d5e9bb4adf4f8df97f3be9f29dce2ec/markupsafe-3.0.4-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:d5f93ebbeb8032d47e349328ec8662d973d9b05a70b3c35df1f91fe419b84749", size = 22640, upload-time = "2026-10-02T23:05:00.768Z" }, + { url = "https://files.pythonhosted.org/packages/09/b2/1506df394f0f075797c418d0301498f49e43be194e3ffcb49e6fe6ccf022/markupsafe-3.0.4-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:64511c54db4e4987aef4c41923235927428729e8174c5dba488429be70a998ed", size = 24074, upload-time = "2026-10-02T23:05:01.813Z" }, + { url = "https://files.pythonhosted.org/packages/c7/81/5ed69cda630ac69ef60d06c09ba5a7f84ff66a2e28cf986fd5614ab3c6e6/markupsafe-3.0.4-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:e1a622f13970d81f95d0c72f9dc090dce9085fccfa4c9f2174377ee32bd15786", size = 21563, upload-time = "2026-10-02T23:05:03.239Z" }, + { url = "https://files.pythonhosted.org/packages/0c/fe/fb1e79be0fea60aa32602ebefc9c35a82bb42b4df157285ab7dfec12341a/markupsafe-3.0.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c9a7f43c0b202b334cc9184af09bb8f21d3a209e038efaf106936fb69e6b026e", size = 23048, upload-time = "2026-10-02T23:05:04.479Z" }, + { url = "https://files.pythonhosted.org/packages/c8/52/7632a53360671a9b750cdbabaf9cdd89f18b42248b8e4cb42c0b0296e459/markupsafe-3.0.4-cp312-cp312-win32.whl", hash = "sha256:f0ec3b750b59375eab5b0fb2b9254810c00a3375be6d789899f1055a1d556237", size = 14108, upload-time = "2026-10-02T23:05:05.513Z" }, + { url = "https://files.pythonhosted.org/packages/3f/bf/62495e180b7000aaf30000fff849e933f74264638057176cf46852500adc/markupsafe-3.0.4-cp312-cp312-win_amd64.whl", hash = "sha256:11935df9bf455ed0c04eb87bcd720f02b1fe5e02128a9430f23aed6f93336fc7", size = 14303, upload-time = "2026-10-02T23:05:06.538Z" }, + { url = "https://files.pythonhosted.org/packages/c5/8e/4c24208776a65878d656996945aacfbfe010d3720d1a98fc0eb8491fc03b/markupsafe-3.0.4-cp312-cp312-win_arm64.whl", hash = "sha256:a4bbd2d87dd233b9fc5812160c3d0ffbe42edc22a26ce0469f58479ede633fe9", size = 14174, upload-time = "2026-10-02T23:05:07.617Z" }, +] + +[[package]] +name = "migrate-and-swap" +version = "0.1.0" +source = { editable = "." } +dependencies = [ + { name = "cryptography" }, + { name = "jinja2" }, + { name = "schemachange" }, + { name = "snowflake-connector-python" }, +] + +[package.dev-dependencies] +dev = [ + { name = "pytest" }, + { name = "pytest-xdist" }, +] + +[package.metadata] +requires-dist = [ + { name = "cryptography", specifier = ">=44" }, + { name = "jinja2", specifier = ">=3.1" }, + { name = "schemachange", specifier = "==4.4.0" }, + { name = "snowflake-connector-python", specifier = ">=4.0,<5.0" }, +] + +[package.metadata.requires-dev] +dev = [ + { name = "pytest", specifier = ">=8.3" }, + { name = "pytest-xdist", specifier = ">=3.6" }, +] + +[[package]] +name = "packaging" +version = "26.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/fa/3944b40b07da9ce895c0e6303a5ab7d53da063554f534556b134a54d6093/packaging-26.3.tar.gz", hash = "sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79", size = 313412, upload-time = "2026-08-04T18:15:28.737Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", hash = "sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", size = 129956, upload-time = "2026-08-04T18:15:27.159Z" }, +] + +[[package]] +name = "platformdirs" +version = "4.13.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/58/e6/d0abbb256b44e6a05748c6942511ff67db8dbb068de0c35b24bf1b395b09/platformdirs-4.13.5.tar.gz", hash = "sha256:05831adf87d392760f4b5c3a0a95579ab5a10b18b9b9bd43680dcb47544788a0", size = 62362, upload-time = "2026-10-19T01:02:11.778Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6c/8a/1d8cb3842909dc193a73835adedf412e32e18f8ffd59f43983949622fe6f/platformdirs-4.13.5-py3-none-any.whl", hash = "sha256:53d3ec4280b815c1e0b02edad7f9ee8fda1ba9c0865dd9c2e6e63849ce82fc1e", size = 32779, upload-time = "2026-10-19T01:02:10.17Z" }, +] + +[[package]] +name = "pluggy" +version = "1.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bf/db/7fc19e6f2dc92a966727031389fc2e08b558f0f25eb7403c1119ad4713cd/pluggy-1.7.0.tar.gz", hash = "sha256:d1eaa46ebb595891b860ab086b4d09c8588af65ebd4361b8e8f4bb8920b90ba8", size = 123304, upload-time = "2026-10-15T09:50:58.343Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/40/9e/2b38731e0fc536806f16490e1a12d7f0dc2a1235aa8cc07bcc75416a7daa/pluggy-1.7.0-py3-none-any.whl", hash = "sha256:7dd7b0d8832ba3cb632c306926ded123429211b83641b35dc5c41ad2d34f9bec", size = 27082, upload-time = "2026-10-15T09:50:56.808Z" }, +] + +[[package]] +name = "pycparser" +version = "3.12" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cb/27/eeb9f307bf07060f3d9f9aa0fb8c15b01bdfded0a941e4b2719c7f94cacd/pycparser-3.12.tar.gz", hash = "sha256:f03387877eab82ce01bbdfd0ba7252ee55a90b1c708d5129e2ce915475cf1db6", size = 158785, upload-time = "2026-10-22T13:40:03.56Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b6/ae/315bbeee126a769cbde43cf4908acb4b40408d8bf8bbf334dd7368d8dadf/pycparser-3.12-py3-none-any.whl", hash = "sha256:a076f3f489b39b43ed13563a1dafe42447445bec2e294154ec5c57fcdda4c85e", size = 62600, upload-time = "2026-10-22T13:40:02.021Z" }, +] + +[[package]] +name = "pygments" +version = "2.21.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/49/2e/ced460408999b33da6b31b0021b0f37d329e202d4169aeb164493778f25b/pygments-2.21.0.tar.gz", hash = "sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c", size = 5005329, upload-time = "2026-08-17T08:02:48.824Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/46/17f022dd3e953bf20a04a028a21ec746d942f8d2af30fa0f124fa0e6a684/pygments-2.21.0-py3-none-any.whl", hash = "sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9", size = 1250147, upload-time = "2026-08-17T08:02:44.912Z" }, +] + +[[package]] +name = "pyjwt" +version = "2.15.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/43/ea/5194e52748b0da83d71e082d75496eaec6e58f419f5e184786ded517e6a9/pyjwt-2.15.1.tar.gz", hash = "sha256:4f259e80cdfb6b3fc18a7de51fd1ef9ec79652f25019bae68975ca2468a34df8", size = 121252, upload-time = "2026-09-28T18:40:42.598Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/50/ca/44de4e75f8aadc457f0634be3b542815078ded46dca30efb960edeecad6e/pyjwt-2.15.1-py3-none-any.whl", hash = "sha256:42d59d631f7768a1028a64c7ff581a9bf7519804daf91fc5b6c56e30eec5e193", size = 33860, upload-time = "2026-09-28T18:40:41.429Z" }, +] + +[[package]] +name = "pyopenssl" +version = "26.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cryptography" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3f/e8/7325d258199b159eb2c03fe32107533e2832e70e63f4fb88a6aa00023201/pyopenssl-26.4.0.tar.gz", hash = "sha256:28dfcce0162b9211413e26dfbfdf1d24317fbeba18fc93c12400a1856b2a0bc7", size = 182046, upload-time = "2026-08-01T19:50:50.512Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/51/ad/2cf6d3fa2fae5c79e1ed9960c0d42badd0f94d81dd12b50604cdc839e648/pyopenssl-26.4.0-py3-none-any.whl", hash = "sha256:f0eb0cb2d581d3ad2b9c489468485e7f2ab6727d08401bcf9d824c3caddf3c1c", size = 56026, upload-time = "2026-08-01T19:50:48.94Z" }, +] + +[[package]] +name = "pytest" +version = "9.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, +] + +[[package]] +name = "pytest-xdist" +version = "3.8.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "execnet" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/78/b4/439b179d1ff526791eb921115fca8e44e596a13efeda518b9d845a619450/pytest_xdist-3.8.0.tar.gz", hash = "sha256:7e578125ec9bc6050861aa93f2d59f1d8d085595d6551c2c90b6f4fad8d3a9f1", size = 88069, upload-time = "2025-07-01T13:30:59.346Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ca/31/d4e37e9e550c2b92a9cbc2e4d0b7420a27224968580b5a447f420847c975/pytest_xdist-3.8.0-py3-none-any.whl", hash = "sha256:202ca578cfeb7370784a8c33d6d05bc6e13b4f25b5053c30a152269fd10f0b88", size = 46396, upload-time = "2025-07-01T13:30:56.632Z" }, +] + +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, +] + +[[package]] +name = "pytz" +version = "2026.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/14/21/d83d6ef28c4c912c4bb4d1dcf591f7b8c6bde87b9c66f9f454677314e16d/pytz-2026.5.tar.gz", hash = "sha256:fa23724b9c486543b9ff54a327ee7569ac83ade54bb9afd0fc18676620401c86", size = 318572, upload-time = "2026-10-04T02:37:58.719Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4f/ef/c66110d46fb800dda0bf33164182dfadabe26a90e4476844d502a23dca8e/pytz-2026.5-py2.py3-none-any.whl", hash = "sha256:e658af3757f9e26a9d25dd2aff38335acd92bc9104f890a894b2c1ba28311b03", size = 506342, upload-time = "2026-10-04T02:37:56.814Z" }, +] + +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, + { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, + { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, +] + +[[package]] +name = "requests" +version = "2.34.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ac/c3/e2a2b89f2d3e2179abd6d00ebd70bff6273f37fb3e0cc209f48b39d00cbf/requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed", size = 142856, upload-time = "2026-05-14T19:25:27.735Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, +] + +[[package]] +name = "s3transfer" +version = "0.19.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/49/c5/0a5ce655109cb96b8dfad36908a4e108d15f5dc2eff8fbc3146c4312e704/s3transfer-0.19.3.tar.gz", hash = "sha256:8326acc3d056d7f7b8a37110c7d2aea41f67e83f6ae79cd3b5fbcca34c726406", size = 166087, upload-time = "2026-10-19T19:19:49.751Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e2/a4/88f9e10498759121060b8050a1554753dfc04df39bbbcc7aad9ecc8d9440/s3transfer-0.19.3-py3-none-any.whl", hash = "sha256:24eb7535804306355e31c952a90022bf5b2f9d760f9a11062c7493a217c08553", size = 90252, upload-time = "2026-10-19T19:19:48.309Z" }, +] + +[[package]] +name = "schemachange" +version = "4.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "jinja2" }, + { name = "pyyaml" }, + { name = "snowflake-connector-python" }, + { name = "structlog" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1b/81/3df283afe06d4931e126e985ea2f7a8c83be889912797c3f11d4e9c1a188/schemachange-4.4.0.tar.gz", hash = "sha256:2c8b2b13d93d20b7749ea2dc0e5c05691a3c4f660198f08bddba426c59ebb034", size = 104004, upload-time = "2026-10-23T11:08:12.461Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/76/d6/d4d7331b1c648e3615a678985cc65d33b6d4601fe34d0b81ac02cdac9e14/schemachange-4.4.0-py3-none-any.whl", hash = "sha256:648882cee8f1bece25e9735c66eed9e33a4941fd3ec132f358bcf1e693bfe441", size = 92711, upload-time = "2026-10-23T11:08:11.189Z" }, +] + +[[package]] +name = "six" +version = "1.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, +] + +[[package]] +name = "snowflake-connector-python" +version = "4.8.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "asn1crypto" }, + { name = "boto3" }, + { name = "botocore" }, + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "cryptography" }, + { name = "filelock" }, + { name = "idna" }, + { name = "packaging" }, + { name = "platformdirs" }, + { name = "pyjwt" }, + { name = "pyopenssl" }, + { name = "pytz" }, + { name = "requests" }, + { name = "sortedcontainers" }, + { name = "tomlkit" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c2/9e/5e54a9ab5a9e02caa777750e0054b14cb5007be75505815084f97a067ab1/snowflake_connector_python-4.8.0.tar.gz", hash = "sha256:1bf40aa8684eb0769fedeb53a702b274472571812835d5a03fb9dcf9254333b9", size = 992670, upload-time = "2026-09-30T16:31:09.818Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/51/50/1c4471bccde09971136884be0ea099a3206e320ed24b65b5c916126e316a/snowflake_connector_python-4.8.0-cp312-cp312-macosx_15_0_arm64.whl", hash = "sha256:2fde1216c4c1e4015ad7e39b7b39b6a44a29551f4d7a3dfc64b0905e66dc24ac", size = 1220915, upload-time = "2026-09-30T16:31:17.723Z" }, + { url = "https://files.pythonhosted.org/packages/6f/8a/21dff856ce456874a93816b9e07417f5c253ca0e9b5f65111bcc8164ffdc/snowflake_connector_python-4.8.0-cp312-cp312-macosx_15_0_x86_64.whl", hash = "sha256:c40a058f87dfbca1e65122932afb41c42cb9c3b7cb12826204527b41dd0f39ef", size = 1227694, upload-time = "2026-09-30T16:31:19.093Z" }, + { url = "https://files.pythonhosted.org/packages/9f/1a/e225f7c901550c4eb06430f6ac7b2c0847a885492e49ce97f2fc56ddc558/snowflake_connector_python-4.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f2a3cbcd3e6c4a449165f78bb331e1570ce6026cc21a2673313b4912d80745c0", size = 2953261, upload-time = "2026-09-30T16:30:57.611Z" }, + { url = "https://files.pythonhosted.org/packages/67/60/d77cfe97234be726c1635de98119373b9ebc6ca052f5cf9ce16b2bab77dc/snowflake_connector_python-4.8.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1c80df2fc5e16f3a49e6d16b1cdca0358308c1db2eae22d82aa7c038c80cc913", size = 2986741, upload-time = "2026-09-30T16:30:59.131Z" }, + { url = "https://files.pythonhosted.org/packages/8d/66/34f0d29c37c4bd25ca325ab4e8a11df75866abc4180aa0a0a044a0139bec/snowflake_connector_python-4.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:b9bddddc713aaa20dc5ff5635896f66e50c0fed88b6a905b088e727d046e361d", size = 5452920, upload-time = "2026-09-30T16:31:34.679Z" }, +] + +[[package]] +name = "sortedcontainers" +version = "2.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e8/c4/ba2f8066cceb6f23394729afe52f3bf7adec04bf9ed2c820b39e19299111/sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88", size = 30594, upload-time = "2021-05-16T22:03:42.897Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0", size = 29575, upload-time = "2021-05-16T22:03:41.177Z" }, +] + +[[package]] +name = "structlog" +version = "25.5.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ef/52/9ba0f43b686e7f3ddfeaa78ac3af750292662284b3661e91ad5494f21dbc/structlog-25.5.0.tar.gz", hash = "sha256:098522a3bebed9153d4570c6d0288abf80a031dfdb2048d59a49e9dc2190fc98", size = 1460830, upload-time = "2025-10-27T08:28:23.028Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a8/45/a132b9074aa18e799b891b91ad72133c98d8042c70f6240e4c5f9dabee2f/structlog-25.5.0-py3-none-any.whl", hash = "sha256:a8453e9b9e636ec59bd9e79bbd4a72f025981b3ba0f5837aebf48f02f37a7f9f", size = 72510, upload-time = "2025-10-27T08:28:21.535Z" }, +] + +[[package]] +name = "tomlkit" +version = "0.15.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/08/72/fc01563df95c189a9f65850096bdc205bc2fe37da6c740cd64afaf18d07e/tomlkit-0.15.2.tar.gz", hash = "sha256:83e32755ba1d20d9340bd069ee97e9f6232ba2346b63ffcba425de01f7e327a6", size = 190522, upload-time = "2026-10-16T04:07:08.166Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/da/46/7d84138a60a8bd796825b2d38ebe92d0d16501ca57d501b7c5d677401bbc/tomlkit-0.15.2-py3-none-any.whl", hash = "sha256:05562412163a3037b94497385fb7f9204c637bc605af7a48199e41cf2272f437", size = 52225, upload-time = "2026-10-16T04:07:09.216Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, +] + +[[package]] +name = "urllib3" +version = "2.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e3/05/b17359e1cefb4f909b5e40b1b90a496d987258916dbbf88e842c729f510e/urllib3-2.8.0.tar.gz", hash = "sha256:63bf2ead4c879426ebf22ef2a781eeb4aa3b4ae798a0435506f8687fd5bb9b63", size = 458972, upload-time = "2026-09-15T19:29:36.253Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/92/9d/c4e665119135114480843e7ab388fa94d8480650450e6f8e26b70d323a4c/urllib3-2.8.0-py3-none-any.whl", hash = "sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3", size = 135717, upload-time = "2026-09-15T19:29:34.577Z" }, +] diff --git a/order-to-cash-cdc/.env.example b/order-to-cash-cdc/.env.example new file mode 100644 index 0000000..b4aece7 --- /dev/null +++ b/order-to-cash-cdc/.env.example @@ -0,0 +1,30 @@ +# Every variable this sample reads, with its default. Export the ones you +# change in your shell: the Makefile passes them to docker compose and to the +# Python code (docker compose alone would also read a .env file here). + +# LocalStack auth token for the emulator (falls back to ~/.localstack/default.env). +LOCALSTACK_AUTH_TOKEN= + +# Emulator image (defaults to the newest) and host ports, all bound to 127.0.0.1. +SNOWFLAKE_IMAGE=localstack/snowflake-next:latest +SNOWFLAKE_PORT=4566 +KAFKA_PORT=19092 +CONNECT_PORT=18083 + +# EMULATOR (default) or SNOWFLAKE_CLOUD. +TEST_TARGET=EMULATOR + +# Connection. Emulator defaults shown; on SNOWFLAKE_CLOUD, SNOWFLAKE_ACCOUNT and +# SNOWFLAKE_ROLE are required and SNOWFLAKE_HOST defaults to +# .snowflakecomputing.com. +SNOWFLAKE_ACCOUNT=snowflake +SNOWFLAKE_HOST= +SNOWFLAKE_USER=SAMPLE_SVC +SNOWFLAKE_PRIVATE_KEY_PATH=.secrets/rsa_key.p8 +SNOWFLAKE_ROLE=ACCOUNTADMIN +SNOWFLAKE_WAREHOUSE=O2C_WH + +# Database created by init/ (used by make seed and the registered sink), and +# the role the Kafka connector uses. +O2C_DATABASE=O2C +O2C_CONNECTOR_ROLE=O2C_INGEST diff --git a/order-to-cash-cdc/Makefile b/order-to-cash-cdc/Makefile new file mode 100644 index 0000000..9082738 --- /dev/null +++ b/order-to-cash-cdc/Makefile @@ -0,0 +1,72 @@ +# Order-to-Cash CDC: Kafka -> Snowflake Kafka connector -> stream -> task -> core and mart. +# +# make up generate the dev key, start emulator + Kafka + Connect, register the sink +# make seed push the seeded change events through the O2C deployment +# make demo forced failure, rollback, then exactly-once apply (readable output) +# make test full suite against the running stack (each run uses its own database) +# make down stop and remove everything + +-include $(HOME)/.localstack/default.env +export LOCALSTACK_AUTH_TOKEN +SNOWFLAKE_PORT ?= 4566 +KAFKA_PORT ?= 19092 +CONNECT_PORT ?= 18083 +export SNOWFLAKE_PORT KAFKA_PORT CONNECT_PORT + +COMPOSE := docker compose -p snfs-order-to-cash-cdc +UV_RUN := uv run --frozen +SECRETS := .secrets +SNOWFLAKE_PRIVATE_KEY_PATH ?= $(SECRETS)/rsa_key.p8 +export SNOWFLAKE_PRIVATE_KEY_PATH +PYTEST_ARGS ?= + +.PHONY: up seed test demo down test-cloud logs install start stop + +up: $(SNOWFLAKE_PRIVATE_KEY_PATH) $(SECRETS)/connector.properties + uv sync --frozen + $(COMPOSE) up -d --build + scripts/wait-for-emulator.sh 180 + $(UV_RUN) python -m pipeline.kafka_connect setup + +seed: + $(UV_RUN) python -m pipeline.seed + +test: + $(UV_RUN) pytest $(PYTEST_ARGS) + +demo: + $(UV_RUN) python -m pipeline.demo + +down: + $(COMPOSE) down -v --remove-orphans + +# Real Snowflake: Kafka and Connect still run locally (make up), the sink and +# the tests point at SNOWFLAKE_ACCOUNT. Consumes credits. +test-cloud: + TEST_TARGET=SNOWFLAKE_CLOUD $(UV_RUN) pytest $(PYTEST_ARGS) + +logs: + $(COMPOSE) logs --tail=200 connect snowflake + +# Throwaway unencrypted PKCS#8 key pair for key-pair (SNOWFLAKE_JWT) auth. +$(SECRETS)/rsa_key.p8: + mkdir -p $(SECRETS) + openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 -out $@ + openssl rsa -in $@ -pubout -out $(SECRETS)/rsa_key.pub + chmod 600 $@ + +# The connector takes the key as one base64 line without PEM armor. Connect +# reads it through FileConfigProvider; the file must be readable by the +# container's user. +$(SECRETS)/connector.properties: $(SNOWFLAKE_PRIVATE_KEY_PATH) + mkdir -p $(SECRETS) + printf 'snowflake.private.key=%s\n' "$$(grep -v -- '-----' $< | tr -d '\n')" > $@ + chmod 644 $@ + +# The targets the repository CI runs for every sample (see the root README) +install: + @echo "Nothing to install, make up and make test fetch the dependencies" + +start: up + +stop: down diff --git a/order-to-cash-cdc/README.md b/order-to-cash-cdc/README.md new file mode 100644 index 0000000..53d11f2 --- /dev/null +++ b/order-to-cash-cdc/README.md @@ -0,0 +1,114 @@ +# Order-to-Cash CDC + +A near-real-time order pipeline from Kafka to marts, built from Snowflake streams, tasks, SQL Scripting procedures and dynamic tables. It is for data engineers who own a CDC pipeline and want to test its failure paths (a run that dies after its MERGE, replayed and late events) deterministically on a laptop. + +## Architecture + +```mermaid +flowchart LR + producer["producer/produce.py
seeded Debezium-style events"] -- "topic orders.cdc
3 partitions, keyed by PK" --> kafka[("Kafka 3.7.1
KRaft")] + kafka --> connect["Kafka Connect
Snowflake Kafka connector 2.1.2
SNOWPIPE_STREAMING"] + subgraph sf["Snowflake"] + landing["RAW.ORDERS_CDC"] --> stream["RAW.ORDERS_CDC_STREAM"] + stream -- "WHEN SYSTEM$STREAM_HAS_DATA" --> task["Task OPS.APPLY_ORDERS_CDC"] + task -- "CALL" --> proc["Procedure OPS.APPLY_ORDERS_CDC
one transaction"] + proc -- "MERGE, newest LSN wins" --> orders["CORE.ORDERS"] + proc -- "SCD2 MERGE" --> dim["CORE.DIM_CUSTOMER_HISTORY"] + proc -- "invalid events" --> dlq["OPS.DEAD_LETTER"] + orders --> mart["MART.DAILY_BOOKINGS
dynamic table"] + dim --> mart + end + connect -- "key-pair auth, HTTPS" --> landing +``` + +A seeded producer publishes Debezium-style change events for TPC-H `orders` and `customer` to Kafka. The Snowflake Kafka connector lands them with Snowpipe Streaming, and a stream-driven task calls one procedure that applies each batch to a current-state table, an SCD2 history and a dead-letter table. A dynamic table serves daily bookings by market segment. + +## How it works + +1. **Change events.** `producer/produce.py` emits a customer snapshot, then inserts, updates and deletes, one envelope per commit with a strictly increasing `source.lsn`. Delivery adds duplicates, late events and three invalid records. +2. **Landing.** The connector writes each record into `RAW.ORDERS_CDC` as `RECORD_METADATA` and `RECORD_CONTENT` VARIANT columns. Its private key is read through Kafka Connect's `FileConfigProvider`, never stored in the connector config. +3. **Apply.** In one transaction, `OPS.APPLY_ORDERS_CDC()` parses the stream, dead-letters invalid events, de-duplicates with `QUALIFY ROW_NUMBER()`, MERGEs `CORE.ORDERS` (soft delete) and SCD2-MERGEs `CORE.DIM_CUSTOMER_HISTORY`. Every write is idempotent, so replays and retries never double-apply. +4. **Failure.** Setting `OPS.PIPELINE_CONTROL.FAIL_NEXT_RUN` makes the procedure raise after its MERGEs. The handler rolls back and re-raises, so the task run is `FAILED` in `TASK_HISTORY()` and the next run applies the same events exactly once. +5. **Serve.** `MART.DAILY_BOOKINGS` aggregates live orders by order date and each customer's current segment. + +```mermaid +sequenceDiagram + autonumber + participant T as pytest / make demo + participant K as Kafka + connector + participant L as RAW.ORDERS_CDC + stream + participant S as Task + procedure + participant C as CORE tables + T->>K: produce batch 2 + K->>L: Snowpipe Streaming insert + T->>S: set FAIL_NEXT_RUN, EXECUTE TASK + S->>C: MERGE, then RAISE -20001 and ROLLBACK + T->>C: assert core tables unchanged + T->>S: clear the flag, EXECUTE TASK + S->>L: read the same events + S->>C: MERGE and SCD2 MERGE, COMMIT + T->>C: assert each event applied exactly once +``` + +## Prerequisites + +- Docker with Compose v2 (about 2 GB of free memory) +- [`uv`](https://docs.astral.sh/uv/) (installs Python 3.12 if needed) +- `openssl`, `make`, `curl` +- A LocalStack auth token, exported as `LOCALSTACK_AUTH_TOKEN` or stored in `~/.localstack/default.env` + +## Quickstart + +```sh +make up # dev key pair, Snowflake + Kafka + Connect, sink connector registered +make seed # push the seeded events through the O2C database +make demo # forced failure, rollback, exactly-once retry (database O2C_DEMO) +make test # run the test suite +make down # stop and remove everything +``` + +Host ports default to `4566` (Snowflake), `19092` (Kafka) and `18083` (Kafka Connect REST); override them with `SNOWFLAKE_PORT`, `KAFKA_PORT` and `CONNECT_PORT`. After `make seed`, connect any Snowflake client to host `snowflake.localhost.localstack.cloud` and try: + +```sql +SELECT * FROM O2C.CORE.DIM_CUSTOMER_HISTORY WHERE C_CUSTKEY = 1 ORDER BY VALID_FROM; +SELECT * FROM TABLE(O2C.INFORMATION_SCHEMA.TASK_HISTORY(TASK_NAME => 'APPLY_ORDERS_CDC')) +WHERE DATABASE_NAME = 'O2C'; +``` + +## Tests + +Each scenario runs in its own database, topic and sink connector, and every expected value is computed independently from the producer's events in `tests/model.py`. The suite covers: + +- every Kafka record lands exactly once through the Snowflake Kafka connector; +- `CORE.ORDERS` holds the newest version of every order, with soft deletes; +- the SCD2 history has one version per change, and a customer delete closes the open version; +- duplicate, late and replayed events never create rows or regress state; +- invalid events land in `OPS.DEAD_LETTER` with the right reason; +- `MART.DAILY_BOOKINGS` matches independently computed totals; +- a MERGE without de-duplication fails with `100090 Duplicate row detected during DML action`; +- a forced failure rolls back completely and the retry applies each event exactly once. + +To run the same suite against a Snowflake account, start Kafka and Connect with `make up`, set `SNOWFLAKE_ACCOUNT`, `SNOWFLAKE_USER`, `SNOWFLAKE_ROLE` (with `CREATE DATABASE`), `SNOWFLAKE_WAREHOUSE` and `SNOWFLAKE_PRIVATE_KEY_PATH` (a key registered for that user), then run `make test-cloud`. One-time account setup is in [docs/architecture.md](docs/architecture.md#running-on-a-snowflake-account). + +## Project structure + +``` +order-to-cash-cdc/ + Makefile up, seed, demo, test, down, test-cloud, logs + docker-compose.yml Snowflake, Kafka, Kafka Connect + connect/ Connect image, worker config, sink connector config + init/ landing zone, core tables, procedure, task, dynamic table + producer/produce.py seeded change generator and Kafka publisher + pipeline/ settings, deploy, Connect client, harness, seed and demo + tests/ end-to-end, transaction and model tests + scripts/ readiness check + docs/architecture.md design notes +``` + +## Learn more + +- [Snowflake Kafka connector with Snowpipe Streaming](https://docs.snowflake.com/en/user-guide/data-load-snowpipe-streaming-kafka) +- [Streams](https://docs.snowflake.com/en/user-guide/streams-intro) and [tasks](https://docs.snowflake.com/en/user-guide/tasks-intro) +- [Transactions in Snowflake Scripting](https://docs.snowflake.com/en/developer-guide/snowflake-scripting/transactions) +- [`MERGE`](https://docs.snowflake.com/en/sql-reference/sql/merge) and [dynamic tables](https://docs.snowflake.com/en/user-guide/dynamic-tables-intro) +- [Debezium PostgreSQL connector event format](https://debezium.io/documentation/reference/stable/connectors/postgresql.html#postgresql-events) diff --git a/order-to-cash-cdc/connect/Dockerfile b/order-to-cash-cdc/connect/Dockerfile new file mode 100644 index 0000000..394aa50 --- /dev/null +++ b/order-to-cash-cdc/connect/Dockerfile @@ -0,0 +1,35 @@ +# Kafka Connect worker with the Snowflake Kafka connector. +# +# The worker runs the Kafka 3.7.1 distribution (copied from the official +# apache/kafka image) on a pinned Java 11 JRE, with the connector version and +# its checksum pinned below. +ARG KAFKA_IMAGE=apache/kafka:3.7.1 +ARG JRE_IMAGE=eclipse-temurin:11.0.24_8-jre-jammy + +FROM ${KAFKA_IMAGE} AS kafka + +FROM ${JRE_IMAGE} + +ARG CONNECTOR_VERSION=2.1.2 +# SHA-256 of snowflake-kafka-connector-2.1.2.jar from Maven Central +# (its published SHA-1 is 206abda3a8ca617789926deb55a9ba223404ff2a). +ARG CONNECTOR_SHA256=9a160a9cf5e3fd62d40cfcb4d97323b0ba61a5be2e2714422d8bbe7cf68cacfb + +COPY --from=kafka /opt/kafka /opt/kafka + +RUN set -eux; \ + mkdir -p /opt/connect/plugins/snowflake-kafka-connector; \ + jar="snowflake-kafka-connector-${CONNECTOR_VERSION}.jar"; \ + curl -fsSL -o "/opt/connect/plugins/snowflake-kafka-connector/${jar}" \ + "https://repo1.maven.org/maven2/com/snowflake/snowflake-kafka-connector/${CONNECTOR_VERSION}/${jar}"; \ + echo "${CONNECTOR_SHA256} /opt/connect/plugins/snowflake-kafka-connector/${jar}" | sha256sum -c -; \ + useradd --system --uid 1001 --home-dir /opt/connect connect; \ + chown -R connect /opt/connect + +COPY --chown=connect worker.properties /opt/connect/worker.properties + +USER connect +ENV KAFKA_HEAP_OPTS="-Xms256m -Xmx512m" \ + LOG_DIR=/tmp/connect-logs +EXPOSE 8083 +ENTRYPOINT ["/opt/kafka/bin/connect-distributed.sh", "/opt/connect/worker.properties"] diff --git a/order-to-cash-cdc/connect/snowflake-sink.json b/order-to-cash-cdc/connect/snowflake-sink.json new file mode 100644 index 0000000..cdc8cb6 --- /dev/null +++ b/order-to-cash-cdc/connect/snowflake-sink.json @@ -0,0 +1,26 @@ +{ + "name": "o2c-orders-cdc-sink", + "config": { + "connector.class": "com.snowflake.kafka.connector.SnowflakeSinkConnector", + "tasks.max": "1", + "topics": "orders.cdc", + "snowflake.topic2table.map": "orders.cdc:ORDERS_CDC", + "snowflake.url.name": "https://snowflake.localhost.localstack.cloud:4566", + "snowflake.user.name": "SAMPLE_SVC", + "snowflake.role.name": "O2C_INGEST", + "snowflake.private.key": "${file:/opt/connect/secrets/connector.properties:snowflake.private.key}", + "snowflake.database.name": "O2C", + "snowflake.schema.name": "RAW", + "snowflake.ingestion.method": "SNOWPIPE_STREAMING", + "snowflake.enable.schematization": "false", + "enable.streaming.channel.offset.migration": "false", + "buffer.count.records": "500", + "buffer.flush.time": "2", + "buffer.size.bytes": "5000000", + "errors.tolerance": "none", + "errors.log.enable": "true", + "key.converter": "org.apache.kafka.connect.storage.StringConverter", + "value.converter": "org.apache.kafka.connect.json.JsonConverter", + "value.converter.schemas.enable": "false" + } +} diff --git a/order-to-cash-cdc/connect/worker.properties b/order-to-cash-cdc/connect/worker.properties new file mode 100644 index 0000000..6ffc798 --- /dev/null +++ b/order-to-cash-cdc/connect/worker.properties @@ -0,0 +1,25 @@ +# Kafka Connect distributed worker (single node). +bootstrap.servers=kafka:29092 +group.id=o2c-connect +listeners=http://0.0.0.0:8083 +rest.advertised.host.name=connect +plugin.path=/opt/connect/plugins + +# Internal topics. Replication factor 1 because the sample runs one broker. +config.storage.topic=_connect-configs +offset.storage.topic=_connect-offsets +status.storage.topic=_connect-status +config.storage.replication.factor=1 +offset.storage.replication.factor=1 +status.storage.replication.factor=1 +offset.flush.interval.ms=1000 + +# Worker-level converters; the Snowflake sink overrides them per connector. +key.converter=org.apache.kafka.connect.storage.StringConverter +value.converter=org.apache.kafka.connect.json.JsonConverter +value.converter.schemas.enable=false + +# Secrets stay out of the connector config (and out of GET /connectors): +# the sink references ${file:/opt/connect/secrets/connector.properties:...}. +config.providers=file +config.providers.file.class=org.apache.kafka.common.config.provider.FileConfigProvider diff --git a/order-to-cash-cdc/docker-compose.yml b/order-to-cash-cdc/docker-compose.yml new file mode 100644 index 0000000..6e56fec --- /dev/null +++ b/order-to-cash-cdc/docker-compose.yml @@ -0,0 +1,64 @@ +name: snfs-order-to-cash-cdc + +services: + snowflake: + image: ${SNOWFLAKE_IMAGE:-localstack/snowflake-next:latest} + pull_policy: always + ports: + - "127.0.0.1:${SNOWFLAKE_PORT:-4566}:4566" + environment: + LOCALSTACK_AUTH_TOKEN: ${LOCALSTACK_AUTH_TOKEN:?set LOCALSTACK_AUTH_TOKEN} + volumes: + - ./init:/etc/localstack/init/ready.d:ro + networks: + default: + aliases: + # Kafka Connect reaches the emulator by its public hostname on 4566, + # so the connector sees the same TLS certificate a host client does. + - snowflake.localhost.localstack.cloud + - stages.s3.localhost.localstack.cloud + - stages-enc.s3.localhost.localstack.cloud + + kafka: + # Single-node KRaft broker (combined broker + controller, no ZooKeeper). + image: apache/kafka:3.7.1 + ports: + - "127.0.0.1:${KAFKA_PORT:-19092}:19092" + environment: + KAFKA_NODE_ID: 1 + KAFKA_PROCESS_ROLES: broker,controller + KAFKA_CONTROLLER_QUORUM_VOTERS: 1@localhost:9093 + KAFKA_LISTENERS: INTERNAL://:29092,EXTERNAL://:19092,CONTROLLER://:9093 + # INTERNAL is for Kafka Connect on the compose network, EXTERNAL for the + # producer and tests on the host. + KAFKA_ADVERTISED_LISTENERS: INTERNAL://kafka:29092,EXTERNAL://localhost:${KAFKA_PORT:-19092} + KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: INTERNAL:PLAINTEXT,EXTERNAL:PLAINTEXT,CONTROLLER:PLAINTEXT + KAFKA_INTER_BROKER_LISTENER_NAME: INTERNAL + KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER + KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1 + KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1 + KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0 + KAFKA_AUTO_CREATE_TOPICS_ENABLE: "false" + KAFKA_HEAP_OPTS: "-Xms256m -Xmx256m" + healthcheck: + test: ["CMD", "/opt/kafka/bin/kafka-topics.sh", "--bootstrap-server", "localhost:29092", "--list"] + interval: 5s + timeout: 20s + retries: 24 + + connect: + build: + context: ./connect + image: snfs-order-to-cash-cdc-connect:2.1.2 + depends_on: + kafka: + condition: service_healthy + snowflake: + condition: service_started + ports: + - "127.0.0.1:${CONNECT_PORT:-18083}:8083" + volumes: + # connector.properties holds the private key for the sink's + # FileConfigProvider reference (generated by `make up`, gitignored). + - ./.secrets:/opt/connect/secrets:ro diff --git a/order-to-cash-cdc/docs/architecture.md b/order-to-cash-cdc/docs/architecture.md new file mode 100644 index 0000000..3bfa856 --- /dev/null +++ b/order-to-cash-cdc/docs/architecture.md @@ -0,0 +1,91 @@ +# Architecture notes + +Design notes for the Order-to-Cash CDC sample. Start with the [README](../README.md) for the overview. + +## Event contract + +Each Kafka record value is a Debezium change envelope (PostgreSQL connector shape, `decimal.handling.mode=string`, dates as days since the epoch). The key is the primary key as JSON, for example `{"o_orderkey": 17}`. + +```json +{ + "before": null, + "after": {"o_orderkey": 17, "o_custkey": 4, "o_orderstatus": "O", "o_totalprice": "182315.40", + "o_orderdate": 20729, "o_orderpriority": "2-HIGH", "o_clerk": "Clerk#000000412"}, + "source": {"connector": "postgresql", "db": "tpch", "schema": "public", "table": "orders", + "lsn": 24001250, "ts_ms": 1790845200000}, + "op": "c", + "ts_ms": 1790845200250 +} +``` + +- `op`: `r` (snapshot read), `c`, `u`, `d`. A delete carries the row in `before`. +- `source.lsn`: strictly increasing per source database, so it orders all changes to a key regardless of Kafka partition or arrival order. It is the pipeline's version number. +- `source.ts_ms`: commit time in the source; it becomes `SOURCE_TS` and the SCD2 validity boundaries. + +## Data model + +```mermaid +erDiagram + RAW_ORDERS_CDC ||--o{ OPS_APPLY_BATCH : "parsed each run" + OPS_APPLY_BATCH ||--o{ CORE_ORDERS : "MERGE newest LSN" + OPS_APPLY_BATCH ||--o{ CORE_DIM_CUSTOMER_HISTORY : "SCD2 MERGE" + OPS_APPLY_BATCH ||--o{ OPS_DEAD_LETTER : "invalid events" + CORE_ORDERS }o--|| MART_DAILY_BOOKINGS : "dynamic table" + CORE_DIM_CUSTOMER_HISTORY }o--|| MART_DAILY_BOOKINGS : "current segment" +``` + +| Object | Kind | Notes | +| --- | --- | --- | +| `RAW.ORDERS_CDC` | table | Connector-managed landing zone, append only | +| `RAW.ORDERS_CDC_STREAM` | standard stream | Unprocessed rows of the landing table | +| `OPS.APPLY_BATCH` | transient table | Parsed events of the latest run, kept for debugging | +| `CORE.ORDERS` | table | Current state per order, soft delete | +| `CORE.DIM_CUSTOMER_HISTORY` | table | SCD2, validity `[VALID_FROM, VALID_TO)` | +| `OPS.DEAD_LETTER` | table | Rejected events with the reason, keyed by topic, partition and offset | +| `OPS.PIPELINE_CONTROL` | table | `FAIL_NEXT_RUN` switch | +| `OPS.APPLY_ORDERS_CDC` | procedure and task | The whole apply in one transaction; the task is created suspended and triggered with `EXECUTE TASK` | +| `MART.DAILY_BOOKINGS` | dynamic table | Live orders by order date and current segment | + +## The apply procedure + +`OPS.APPLY_ORDERS_CDC()` runs inside `BEGIN TRANSACTION ... COMMIT`: + +1. Rewrite `OPS.APPLY_BATCH` from `RAW.ORDERS_CDC_STREAM`, setting `REJECT_REASON` for an unknown table or op, or a missing LSN, timestamp or key. +2. `MERGE INTO OPS.DEAD_LETTER` for rejected rows, inserting only records not yet dead-lettered. +3. `MERGE INTO CORE.ORDERS` from the newest event per key (`QUALIFY ROW_NUMBER()`), updating only when the event's LSN is newer than the stored one. A delete sets `IS_DELETED`. +4. SCD2 `MERGE INTO CORE.DIM_CUSTOMER_HISTORY`: customer events newer than the newest applied version become versions with `VALID_TO = LEAD(SOURCE_TS)`, and the open version is closed. A delete closes the open version and opens none. +5. If `OPS.PIPELINE_CONTROL.FAIL_NEXT_RUN` is `TRUE`, raise a user exception (-20001); otherwise `COMMIT`. + +The exception handler runs `ROLLBACK; RAISE;`: nothing of the run persists, and re-raising makes the task run `FAILED` in `TASK_HISTORY()`, where alerting can see it. + +Every write is idempotent, so replays and retries never double-apply: + +| Situation | What prevents a wrong result | +| --- | --- | +| Duplicate delivery in one batch | `QUALIFY` keeps one row per key (orders) or per key and LSN (customers) | +| Duplicate, replayed or late delivery | LSN-guarded MERGEs: an event no newer than the applied state changes nothing | +| Delete followed by a late create | The soft-deleted row keeps the delete's LSN, so the older create loses | +| Failed run | One transaction, rolled back; the next run applies the same events | +| Invalid record seen again | Dead-letter MERGE keyed by topic, partition and offset | + +```mermaid +stateDiagram-v2 + [*] --> Pending: events land in RAW.ORDERS_CDC + Pending --> Running: EXECUTE TASK, or schedule with STREAM_HAS_DATA + Running --> Applied: COMMIT + Running --> RolledBack: error or FAIL_NEXT_RUN, then ROLLBACK and RAISE + RolledBack --> Pending: TASK_HISTORY shows FAILED + Applied --> [*] +``` + +## Test isolation + +Every scenario gets its own database (`O2C_T_`), deployed from the part of each `init/*.sf.sql` file below its `-- @deploy:` marker, plus its own topic and sink connector from `connect/snowflake-sink.json`. All three are removed when the scenario ends, so tests run against exactly the DDL and connector configuration that `make up` deploys and never share data. `make demo` uses the same mechanism with the fixed database `O2C_DEMO`. Every wait polls a condition with a timeout; nothing waits on the task's schedule. + +## Running on a Snowflake account + +`TEST_TARGET=SNOWFLAKE_CLOUD` (set by `make test-cloud`) points the tests and the sink's `snowflake.url.name` at `SNOWFLAKE_ACCOUNT`; Kafka and Connect still run locally. Before the first run: + +1. Run the part of `init/01_landing.sf.sql` above the `@deploy` marker (warehouse, `O2C_INGEST` role, `SAMPLE_SVC` user), or point `SNOWFLAKE_WAREHOUSE`, `SNOWFLAKE_USER` and `O2C_CONNECTOR_ROLE` at existing objects. +2. Register the public key: `ALTER USER SAMPLE_SVC SET RSA_PUBLIC_KEY = '...'` with the body of `.secrets/rsa_key.pub`, or set `SNOWFLAKE_PRIVATE_KEY_PATH` to an already registered key and delete `.secrets/connector.properties` so `make up` regenerates it. +3. `make test-cloud`. Each scenario creates and drops its own database, so the role needs `CREATE DATABASE`. diff --git a/order-to-cash-cdc/init/01_landing.sf.sql b/order-to-cash-cdc/init/01_landing.sf.sql new file mode 100644 index 0000000..0dcf4c1 --- /dev/null +++ b/order-to-cash-cdc/init/01_landing.sf.sql @@ -0,0 +1,45 @@ +-- Order-to-Cash CDC: account objects and the Kafka landing zone. +-- +-- Take ACCOUNTADMIN to create the warehouse, the ingest role and user, and +-- the database the pipeline uses. +USE ROLE ACCOUNTADMIN; + +CREATE WAREHOUSE IF NOT EXISTS O2C_WH + WAREHOUSE_SIZE = XSMALL + AUTO_SUSPEND = 60 + INITIALLY_SUSPENDED = TRUE; + +-- Least-privilege role for the Kafka connector's service user. On real +-- Snowflake, register the public key with: +-- ALTER USER SAMPLE_SVC SET RSA_PUBLIC_KEY = ''; +CREATE ROLE IF NOT EXISTS O2C_INGEST; +CREATE USER IF NOT EXISTS SAMPLE_SVC + DEFAULT_ROLE = O2C_INGEST + DEFAULT_WAREHOUSE = O2C_WH + COMMENT = 'Kafka connector service user (key-pair auth)'; +GRANT ROLE O2C_INGEST TO USER SAMPLE_SVC; + +CREATE DATABASE IF NOT EXISTS O2C; +USE DATABASE O2C; +GRANT USAGE ON DATABASE O2C TO ROLE O2C_INGEST; + +-- @deploy: everything below this line is database-relative. The test suite +-- and `make demo` run it inside a fresh database per run. + +CREATE SCHEMA IF NOT EXISTS RAW; +GRANT USAGE ON SCHEMA RAW TO ROLE O2C_INGEST; + +-- Landing table written by the Snowflake Kafka connector (Snowpipe Streaming, +-- non-schematized). One row per Kafka record: RECORD_METADATA carries topic, +-- partition, offset and key; RECORD_CONTENT is the Debezium change envelope. +CREATE TABLE IF NOT EXISTS RAW.ORDERS_CDC ( + RECORD_METADATA VARIANT, + RECORD_CONTENT VARIANT +) +COMMENT = 'Debezium-style change events for TPC-H orders and customer, landed by the Kafka connector'; +GRANT SELECT, INSERT ON TABLE RAW.ORDERS_CDC TO ROLE O2C_INGEST; + +-- Standard (delta) stream of unprocessed change events. OPS.APPLY_ORDERS_CDC +-- consumes it inside one transaction, so the offset advances only when the +-- merge commits. +CREATE STREAM IF NOT EXISTS RAW.ORDERS_CDC_STREAM ON TABLE RAW.ORDERS_CDC; diff --git a/order-to-cash-cdc/init/02_core.sf.sql b/order-to-cash-cdc/init/02_core.sf.sql new file mode 100644 index 0000000..437013b --- /dev/null +++ b/order-to-cash-cdc/init/02_core.sf.sql @@ -0,0 +1,94 @@ +-- Order-to-Cash CDC: current-state and history tables, plus operational tables. +USE ROLE ACCOUNTADMIN; +USE DATABASE O2C; + +-- @deploy: everything below this line is database-relative. + +CREATE SCHEMA IF NOT EXISTS CORE; +CREATE SCHEMA IF NOT EXISTS MART; +CREATE SCHEMA IF NOT EXISTS OPS; + +-- Current state of every order the source has ever emitted. Deletes are soft: +-- the row stays with IS_DELETED = TRUE so late or replayed events for that key +-- cannot resurrect it (their SOURCE_LSN is lower). +CREATE TABLE IF NOT EXISTS CORE.ORDERS ( + O_ORDERKEY NUMBER(38, 0) NOT NULL, + O_CUSTKEY NUMBER(38, 0) NOT NULL, + O_ORDERSTATUS VARCHAR(1) NOT NULL, + O_TOTALPRICE NUMBER(12, 2) NOT NULL, + O_ORDERDATE DATE NOT NULL, + O_ORDERPRIORITY VARCHAR(15) NOT NULL, + O_CLERK VARCHAR(15) NOT NULL, + IS_DELETED BOOLEAN NOT NULL, + SOURCE_LSN NUMBER(38, 0) NOT NULL, + SOURCE_TS TIMESTAMP_NTZ(3) NOT NULL, + APPLIED_AT TIMESTAMP_NTZ(3) NOT NULL, + CONSTRAINT PK_ORDERS PRIMARY KEY (O_ORDERKEY) +) +COMMENT = 'Current state per order key (soft delete), newest source LSN wins'; + +-- Type 2 history of customers. Validity is half-open: [VALID_FROM, VALID_TO). +-- A delete closes the open version and opens none. +CREATE TABLE IF NOT EXISTS CORE.DIM_CUSTOMER_HISTORY ( + C_CUSTKEY NUMBER(38, 0) NOT NULL, + C_NAME VARCHAR(25) NOT NULL, + C_NATIONKEY NUMBER(38, 0) NOT NULL, + C_ACCTBAL NUMBER(12, 2) NOT NULL, + C_MKTSEGMENT VARCHAR(10) NOT NULL, + VALID_FROM TIMESTAMP_NTZ(3) NOT NULL, + VALID_TO TIMESTAMP_NTZ(3), + IS_CURRENT BOOLEAN NOT NULL, + SOURCE_LSN NUMBER(38, 0) NOT NULL, + CONSTRAINT PK_DIM_CUSTOMER_HISTORY PRIMARY KEY (C_CUSTKEY, SOURCE_LSN) +) +COMMENT = 'SCD2 history per customer key, one row per applied version'; + +-- Working set of the current task run, parsed from the stream. Rewritten at +-- the start of every run (inside the run's transaction), kept afterwards for +-- debugging. +CREATE TRANSIENT TABLE IF NOT EXISTS OPS.APPLY_BATCH ( + KAFKA_TOPIC VARCHAR, + KAFKA_PARTITION NUMBER(38, 0), + KAFKA_OFFSET NUMBER(38, 0), + SOURCE_TABLE VARCHAR, + OP VARCHAR, + ENTITY_KEY NUMBER(38, 0), + SOURCE_LSN NUMBER(38, 0), + SOURCE_TS TIMESTAMP_NTZ(3), + IMAGE VARIANT, + REJECT_REASON VARCHAR, + RECORD_CONTENT VARIANT, + RECORD_METADATA VARIANT +) +COMMENT = 'Parsed change events of the latest OPS.APPLY_ORDERS_CDC run'; + +-- Events that fail validation. Keyed by Kafka coordinates so re-reading the +-- same record never dead-letters it twice. +CREATE TABLE IF NOT EXISTS OPS.DEAD_LETTER ( + KAFKA_TOPIC VARCHAR NOT NULL, + KAFKA_PARTITION NUMBER(38, 0) NOT NULL, + KAFKA_OFFSET NUMBER(38, 0) NOT NULL, + REASON VARCHAR NOT NULL, + RECORD_CONTENT VARIANT, + RECORD_METADATA VARIANT, + DEAD_LETTERED_AT TIMESTAMP_NTZ(3) NOT NULL, + CONSTRAINT PK_DEAD_LETTER PRIMARY KEY (KAFKA_TOPIC, KAFKA_PARTITION, KAFKA_OFFSET) +) +COMMENT = 'Rejected change events with the reason, for replay after a fix'; + +-- Operator switches for the pipeline. FAIL_NEXT_RUN makes the apply +-- procedure raise after its merges, which exercises the rollback path. +CREATE TABLE IF NOT EXISTS OPS.PIPELINE_CONTROL ( + PIPELINE_NAME VARCHAR NOT NULL, + FAIL_NEXT_RUN BOOLEAN NOT NULL, + UPDATED_AT TIMESTAMP_NTZ(3) NOT NULL, + CONSTRAINT PK_PIPELINE_CONTROL PRIMARY KEY (PIPELINE_NAME) +) +COMMENT = 'Operator switches for the order-to-cash pipeline'; + +MERGE INTO OPS.PIPELINE_CONTROL t +USING (SELECT 'orders_cdc' AS PIPELINE_NAME) s + ON t.PIPELINE_NAME = s.PIPELINE_NAME +WHEN NOT MATCHED THEN + INSERT (PIPELINE_NAME, FAIL_NEXT_RUN, UPDATED_AT) + VALUES (s.PIPELINE_NAME, FALSE, CURRENT_TIMESTAMP()::TIMESTAMP_NTZ); diff --git a/order-to-cash-cdc/init/03_pipeline.sf.sql b/order-to-cash-cdc/init/03_pipeline.sf.sql new file mode 100644 index 0000000..fc2269d --- /dev/null +++ b/order-to-cash-cdc/init/03_pipeline.sf.sql @@ -0,0 +1,243 @@ +-- Order-to-Cash CDC: the apply procedure, the stream-driven task and the mart. +USE ROLE ACCOUNTADMIN; +USE DATABASE O2C; + +-- @deploy: everything below this line is database-relative. + +-- Applies every unprocessed change event in one transaction: +-- 1. parse the stream into OPS.APPLY_BATCH (this read consumes the stream), +-- 2. dead-letter invalid events, +-- 3. MERGE orders into CORE.ORDERS (newest LSN wins, soft delete), +-- 4. SCD2 MERGE customers into CORE.DIM_CUSTOMER_HISTORY, +-- 5. COMMIT, which is also when the stream offset advances. +-- Any error rolls the whole run back, so the stream keeps its offset and the +-- next run sees the same events again. Every MERGE is guarded by SOURCE_LSN, +-- which makes re-applying an already applied event a no-op. +CREATE OR REPLACE PROCEDURE OPS.APPLY_ORDERS_CDC() +RETURNS VARCHAR +LANGUAGE SQL +EXECUTE AS OWNER +AS +$$ +DECLARE + injected_failure EXCEPTION (-20001, 'Injected failure: OPS.PIPELINE_CONTROL.FAIL_NEXT_RUN is TRUE for orders_cdc'); + -- The control flag is read through a declared cursor. + control_cur CURSOR FOR + SELECT FAIL_NEXT_RUN FROM OPS.PIPELINE_CONTROL WHERE PIPELINE_NAME = 'orders_cdc'; + fail_next_run BOOLEAN DEFAULT FALSE; + events_read INTEGER DEFAULT 0; + dead_lettered INTEGER DEFAULT 0; + orders_applied INTEGER DEFAULT 0; + customer_rows INTEGER DEFAULT 0; +BEGIN + BEGIN TRANSACTION; + + DELETE FROM OPS.APPLY_BATCH; + + INSERT INTO OPS.APPLY_BATCH ( + KAFKA_TOPIC, KAFKA_PARTITION, KAFKA_OFFSET, SOURCE_TABLE, OP, ENTITY_KEY, + SOURCE_LSN, SOURCE_TS, IMAGE, REJECT_REASON, RECORD_CONTENT, RECORD_METADATA + ) + SELECT + kafka_topic, kafka_partition, kafka_offset, source_table, op, entity_key, + source_lsn, source_ts, image, + CASE + WHEN source_table IS NULL OR source_table NOT IN ('orders', 'customer') THEN 'UNKNOWN_SOURCE_TABLE' + WHEN op IS NULL OR op NOT IN ('c', 'r', 'u', 'd') THEN 'UNKNOWN_OP' + WHEN source_lsn IS NULL THEN 'MISSING_LSN' + WHEN source_ts IS NULL THEN 'MISSING_SOURCE_TS' + WHEN entity_key IS NULL THEN 'MISSING_KEY' + END, + record_content, record_metadata + FROM ( + SELECT + e.*, + CASE e.source_table + WHEN 'orders' THEN e.image:o_orderkey::NUMBER(38, 0) + WHEN 'customer' THEN e.image:c_custkey::NUMBER(38, 0) + END AS entity_key + FROM ( + SELECT + s.RECORD_METADATA:topic::VARCHAR AS kafka_topic, + s.RECORD_METADATA:partition::NUMBER(38, 0) AS kafka_partition, + s.RECORD_METADATA:offset::NUMBER(38, 0) AS kafka_offset, + s.RECORD_CONTENT:source:table::VARCHAR AS source_table, + s.RECORD_CONTENT:op::VARCHAR AS op, + s.RECORD_CONTENT:source:lsn::NUMBER(38, 0) AS source_lsn, + TO_TIMESTAMP_NTZ(s.RECORD_CONTENT:source:ts_ms::NUMBER(38, 0), 3) AS source_ts, + -- Debezium puts the row in "after" for c/r/u and in "before" for d. + IFF(s.RECORD_CONTENT:op::VARCHAR = 'd', s.RECORD_CONTENT:before, s.RECORD_CONTENT:after) AS image, + s.RECORD_CONTENT AS record_content, + s.RECORD_METADATA AS record_metadata + FROM RAW.ORDERS_CDC_STREAM s + WHERE s.METADATA$ACTION = 'INSERT' + ) e + ) p; + events_read := SQLROWCOUNT; + + MERGE INTO OPS.DEAD_LETTER t + USING ( + SELECT * FROM OPS.APPLY_BATCH WHERE REJECT_REASON IS NOT NULL + ) s + ON t.KAFKA_TOPIC = s.KAFKA_TOPIC + AND t.KAFKA_PARTITION = s.KAFKA_PARTITION + AND t.KAFKA_OFFSET = s.KAFKA_OFFSET + WHEN NOT MATCHED THEN + INSERT (KAFKA_TOPIC, KAFKA_PARTITION, KAFKA_OFFSET, REASON, RECORD_CONTENT, RECORD_METADATA, DEAD_LETTERED_AT) + VALUES (s.KAFKA_TOPIC, s.KAFKA_PARTITION, s.KAFKA_OFFSET, s.REJECT_REASON, s.RECORD_CONTENT, s.RECORD_METADATA, + CURRENT_TIMESTAMP()::TIMESTAMP_NTZ); + dead_lettered := SQLROWCOUNT; + + -- Current state: keep the newest event per order key in this batch (this + -- also collapses redelivered duplicates), then apply it only if it is newer + -- than what the table already holds (late events lose). + MERGE INTO CORE.ORDERS t + USING ( + SELECT + ENTITY_KEY AS o_orderkey, + IMAGE:o_custkey::NUMBER(38, 0) AS o_custkey, + IMAGE:o_orderstatus::VARCHAR AS o_orderstatus, + IMAGE:o_totalprice::NUMBER(12, 2) AS o_totalprice, + -- Debezium encodes DATE columns as days since the epoch. + DATEADD(DAY, IMAGE:o_orderdate::INTEGER, '1970-01-01'::DATE) AS o_orderdate, + IMAGE:o_orderpriority::VARCHAR AS o_orderpriority, + IMAGE:o_clerk::VARCHAR AS o_clerk, + OP = 'd' AS is_deleted, + SOURCE_LSN AS source_lsn, + SOURCE_TS AS source_ts + FROM OPS.APPLY_BATCH + WHERE SOURCE_TABLE = 'orders' AND REJECT_REASON IS NULL + QUALIFY ROW_NUMBER() OVER ( + PARTITION BY ENTITY_KEY ORDER BY SOURCE_LSN DESC, KAFKA_PARTITION, KAFKA_OFFSET + ) = 1 + ) s + ON t.O_ORDERKEY = s.o_orderkey + WHEN MATCHED AND s.source_lsn > t.SOURCE_LSN THEN UPDATE SET + O_CUSTKEY = s.o_custkey, + O_ORDERSTATUS = s.o_orderstatus, + O_TOTALPRICE = s.o_totalprice, + O_ORDERDATE = s.o_orderdate, + O_ORDERPRIORITY = s.o_orderpriority, + O_CLERK = s.o_clerk, + IS_DELETED = s.is_deleted, + SOURCE_LSN = s.source_lsn, + SOURCE_TS = s.source_ts, + APPLIED_AT = CURRENT_TIMESTAMP()::TIMESTAMP_NTZ + -- A delete for a key never seen before still inserts a tombstone (from the + -- "before" image), so a late create cannot bring the order back. + WHEN NOT MATCHED THEN INSERT ( + O_ORDERKEY, O_CUSTKEY, O_ORDERSTATUS, O_TOTALPRICE, O_ORDERDATE, O_ORDERPRIORITY, + O_CLERK, IS_DELETED, SOURCE_LSN, SOURCE_TS, APPLIED_AT + ) VALUES ( + s.o_orderkey, s.o_custkey, s.o_orderstatus, s.o_totalprice, s.o_orderdate, s.o_orderpriority, + s.o_clerk, s.is_deleted, s.source_lsn, s.source_ts, CURRENT_TIMESTAMP()::TIMESTAMP_NTZ + ); + orders_applied := SQLROWCOUNT; + + -- SCD2: every customer event newer than the newest applied version becomes + -- a version (deletes only close). The source rows are: + -- * one row per new version with a NULL merge key (never matches: INSERT), + -- * one row per customer with its key (matches the open version: close it + -- at the timestamp of the first change in this batch). + MERGE INTO CORE.DIM_CUSTOMER_HISTORY t + USING ( + WITH applied AS ( + SELECT C_CUSTKEY, MAX(SOURCE_LSN) AS max_lsn + FROM CORE.DIM_CUSTOMER_HISTORY + GROUP BY C_CUSTKEY + ), + changes AS ( + SELECT b.* + FROM OPS.APPLY_BATCH b + LEFT JOIN applied a ON a.C_CUSTKEY = b.ENTITY_KEY + WHERE b.SOURCE_TABLE = 'customer' + AND b.REJECT_REASON IS NULL + AND (a.max_lsn IS NULL OR b.SOURCE_LSN > a.max_lsn) + QUALIFY ROW_NUMBER() OVER ( + PARTITION BY b.ENTITY_KEY, b.SOURCE_LSN ORDER BY b.KAFKA_PARTITION, b.KAFKA_OFFSET + ) = 1 + ), + versions AS ( + SELECT + c.*, + LEAD(c.SOURCE_TS) OVER (PARTITION BY c.ENTITY_KEY ORDER BY c.SOURCE_LSN) AS next_ts, + ROW_NUMBER() OVER (PARTITION BY c.ENTITY_KEY ORDER BY c.SOURCE_LSN) AS change_seq + FROM changes c + ) + SELECT + NULL::NUMBER(38, 0) AS merge_key, + ENTITY_KEY AS c_custkey, + IMAGE:c_name::VARCHAR AS c_name, + IMAGE:c_nationkey::NUMBER(38, 0) AS c_nationkey, + IMAGE:c_acctbal::NUMBER(12, 2) AS c_acctbal, + IMAGE:c_mktsegment::VARCHAR AS c_mktsegment, + SOURCE_TS AS valid_from, + next_ts AS valid_to, + SOURCE_LSN AS source_lsn + FROM versions + WHERE OP <> 'd' + UNION ALL + SELECT + ENTITY_KEY, ENTITY_KEY, NULL, NULL, NULL, NULL, SOURCE_TS, NULL, SOURCE_LSN + FROM versions + WHERE change_seq = 1 + ) s + ON t.C_CUSTKEY = s.merge_key AND t.IS_CURRENT + WHEN MATCHED THEN UPDATE SET + VALID_TO = s.valid_from, + IS_CURRENT = FALSE + WHEN NOT MATCHED AND s.merge_key IS NULL THEN INSERT ( + C_CUSTKEY, C_NAME, C_NATIONKEY, C_ACCTBAL, C_MKTSEGMENT, VALID_FROM, VALID_TO, IS_CURRENT, SOURCE_LSN + ) VALUES ( + s.c_custkey, s.c_name, s.c_nationkey, s.c_acctbal, s.c_mktsegment, s.valid_from, s.valid_to, + s.valid_to IS NULL, s.source_lsn + ); + customer_rows := SQLROWCOUNT; + + -- Failure injection point: after every merge, before COMMIT. + OPEN control_cur; + FETCH control_cur INTO fail_next_run; + CLOSE control_cur; + IF (fail_next_run) THEN + RAISE injected_failure; + END IF; + + COMMIT; + RETURN 'events_read=' || events_read || ' dead_lettered=' || dead_lettered || + ' orders_applied=' || orders_applied || ' customer_rows=' || customer_rows; +EXCEPTION + WHEN OTHER THEN + ROLLBACK; + RAISE; +END; +$$; + +-- Runs the procedure whenever the stream has data. Created suspended, as +-- Snowflake does; tests and the demo trigger runs with EXECUTE TASK. To run it +-- on its schedule: ALTER TASK OPS.APPLY_ORDERS_CDC RESUME; +CREATE OR REPLACE TASK OPS.APPLY_ORDERS_CDC + WAREHOUSE = O2C_WH + SCHEDULE = '1 MINUTE' + COMMENT = 'Apply new change events from RAW.ORDERS_CDC_STREAM' + WHEN SYSTEM$STREAM_HAS_DATA('RAW.ORDERS_CDC_STREAM') +AS + CALL OPS.APPLY_ORDERS_CDC(); + +-- Daily bookings by market segment, from live (not deleted) orders and each +-- customer's current segment. Orders whose customer has no open version +-- (deleted or not yet seen) fall into UNKNOWN. +CREATE OR REPLACE DYNAMIC TABLE MART.DAILY_BOOKINGS + TARGET_LAG = '1 minute' + WAREHOUSE = O2C_WH + COMMENT = 'Order count and gross bookings per order date and market segment' +AS + SELECT + o.O_ORDERDATE AS BOOKING_DATE, + COALESCE(c.C_MKTSEGMENT, 'UNKNOWN') AS MARKET_SEGMENT, + COUNT(*) AS ORDER_COUNT, + SUM(o.O_TOTALPRICE) AS GROSS_BOOKINGS + FROM CORE.ORDERS o + LEFT JOIN CORE.DIM_CUSTOMER_HISTORY c + ON c.C_CUSTKEY = o.O_CUSTKEY AND c.IS_CURRENT + WHERE NOT o.IS_DELETED + GROUP BY o.O_ORDERDATE, COALESCE(c.C_MKTSEGMENT, 'UNKNOWN'); diff --git a/order-to-cash-cdc/pipeline/__init__.py b/order-to-cash-cdc/pipeline/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/order-to-cash-cdc/pipeline/demo.py b/order-to-cash-cdc/pipeline/demo.py new file mode 100644 index 0000000..5cc4425 --- /dev/null +++ b/order-to-cash-cdc/pipeline/demo.py @@ -0,0 +1,109 @@ +"""`make demo`: a failed task run leaves no trace, the retry applies each event once. + +Runs in its own database (O2C_DEMO, recreated each time and kept afterwards +for exploring) with its own topic and sink connector, so the numbers are the +same on every run. +""" + +from __future__ import annotations + +import time + +from pipeline import settings as settings_module +from pipeline import snowflake_db +from pipeline.harness import IsolatedPipeline, Pipeline +from producer.produce import ChangeEvent, initial_scenario, next_batch +from tests import model + +WIDTH = 78 + + +def step(n: int, total: int, text: str) -> None: + print(f"\n[{n}/{total}] {text}") + + +def detail(text: str) -> None: + print(f" {text}") + + +def describe(events: list[ChangeEvent]) -> str: + kinds = {k: sum(1 for e in events if e.kind == k) for k in ("change", "duplicate", "late", "poison")} + return ", ".join(f"{n} {k}" for k, n in kinds.items() if n) + + +def state(p: Pipeline) -> tuple[list, list, list]: + return p.orders(), p.customer_history(), p.dead_letters() + + +def summary(p: Pipeline) -> str: + orders, history, dead = state(p) + return (f"core.orders {len(orders)} rows ({sum(1 for o in orders if o[7])} soft-deleted) | " + f"dim_customer_history {len(history)} versions | dead_letter {len(dead)}") + + +def yes(flag: bool) -> str: + return "yes" if flag else "NO" + + +def main() -> None: + settings = settings_module.load() + total = 6 + print("=" * WIDTH) + print("Order-to-Cash CDC: forced failure, rollback, then exactly-once apply") + print("=" * WIDTH) + with snowflake_db.connect(settings) as conn, \ + IsolatedPipeline(settings, conn, database="O2C_DEMO", keep_database=True) as p: + step(1, total, f"Deployed the pipeline into database {p.database}; topic {p.topic} -> RAW.ORDERS_CDC") + + scenario = initial_scenario(42) + started = time.monotonic() + produced = p.produce(scenario.events) + p.wait_for_landing(produced) + step(2, total, f"Batch 1: {produced} Debezium events ({describe(scenario.events)})") + detail(f"landed in RAW.ORDERS_CDC via the Kafka connector in {time.monotonic() - started:.1f}s") + run = p.run_task() + detail(f"EXECUTE TASK OPS.APPLY_ORDERS_CDC -> {run.state}") + detail(summary(p)) + + batch2 = next_batch(scenario) + p.produce(batch2) + p.wait_for_landing(len(scenario.events)) + before = state(p) + step(3, total, f"Batch 2: {len(batch2)} more events ({describe(batch2)}) landed; not applied yet") + + p.set_fail_next_run(True) + step(4, total, "UPDATE OPS.PIPELINE_CONTROL SET FAIL_NEXT_RUN = TRUE; EXECUTE TASK") + failed = p.run_task() + detail(f"TASK_HISTORY: {failed.state} ({failed.error_code}) {failed.error_message}") + after = state(p) + detail(f"core.orders unchanged: {yes(after[0] == before[0])}") + detail(f"dim_customer_history unchanged: {yes(after[1] == before[1])}") + detail(f"dead_letter unchanged: {yes(after[2] == before[2])}") + detail(f"SYSTEM$STREAM_HAS_DATA: {p.stream_has_data()} (batch 2 still pending)") + + p.set_fail_next_run(False) + step(5, total, "UPDATE OPS.PIPELINE_CONTROL SET FAIL_NEXT_RUN = FALSE; EXECUTE TASK") + retried = p.run_task() + detail(f"TASK_HISTORY: {retried.state}") + detail(summary(p)) + events = scenario.events + detail(f"core.orders equals newest event per key: {yes(p.orders() == model.expected_orders(events))}") + detail(f"one SCD2 version per customer change: " + f"{yes(p.customer_history() == model.expected_customer_history(events))}") + + p.refresh_mart() + bookings = p.daily_bookings() + step(6, total, "ALTER DYNAMIC TABLE MART.DAILY_BOOKINGS REFRESH") + detail(f"matches totals computed from the {len(events)} events: " + f"{yes(bookings == model.expected_daily_bookings(events))}") + detail(f"{'booking_date':<12} {'segment':<11} {'orders':>6} {'gross_bookings':>15}") + for booking_date, segment, count, gross in bookings[:8]: + detail(f"{booking_date!s:<12} {segment:<11} {count:>6} {gross:>15,.2f}") + if len(bookings) > 8: + detail(f"... {len(bookings) - 8} more rows") + + print("\nExplore it: SELECT * FROM O2C_DEMO.CORE.DIM_CUSTOMER_HISTORY ORDER BY C_CUSTKEY, VALID_FROM;") + + +if __name__ == "__main__": + main() diff --git a/order-to-cash-cdc/pipeline/harness.py b/order-to-cash-cdc/pipeline/harness.py new file mode 100644 index 0000000..7238dd2 --- /dev/null +++ b/order-to-cash-cdc/pipeline/harness.py @@ -0,0 +1,166 @@ +"""Drive the pipeline deterministically: produce, wait for landing, run the task. + +`IsolatedPipeline` gives a test (or the demo) its own database, topic and sink +connector, built from the same init SQL and connector config as the O2C +deployment, and removes them afterwards. Nothing here sleeps on a schedule: +every wait polls a concrete condition with a bounded timeout. +""" + +from __future__ import annotations + +import time +import uuid +from dataclasses import dataclass +from datetime import datetime + +from snowflake.connector import SnowflakeConnection + +from pipeline import kafka_connect, snowflake_db +from pipeline.settings import Settings +from producer.produce import ChangeEvent, publish + +TASK = "OPS.APPLY_ORDERS_CDC" +TERMINAL_STATES = ("SUCCEEDED", "FAILED", "CANCELLED", "SKIPPED") + + +@dataclass(frozen=True) +class TaskRun: + state: str + error_code: str | None + error_message: str | None + query_id: str | None + scheduled_time: datetime | None + + +class Pipeline: + """Operations on one deployed pipeline (a database plus its sink topic).""" + + def __init__(self, settings: Settings, conn: SnowflakeConnection, database: str, topic: str) -> None: + self.settings = settings + self.conn = conn + self.database = database + self.topic = topic + + # -- helpers -------------------------------------------------------------- + def q(self, sql: str, params: tuple | None = None) -> list[tuple]: + return snowflake_db.query(self.conn, sql, params) + + def scalar(self, sql: str, params: tuple | None = None): + return snowflake_db.scalar(self.conn, sql, params) + + def fq(self, name: str) -> str: + return f"{self.database}.{name}" + + # -- ingest --------------------------------------------------------------- + def produce(self, events: list[ChangeEvent]) -> int: + return publish(events, self.topic, self.settings.kafka_bootstrap) + + def landed(self) -> int: + return self.scalar(f"SELECT COUNT(*) FROM {self.fq('RAW.ORDERS_CDC')}") + + def wait_for_landing(self, expected: int, timeout: float = 180, interval: float = 1.0) -> int: + """Poll the landing table until it holds `expected` rows (never more).""" + deadline = time.monotonic() + timeout + count = self.landed() + while count < expected and time.monotonic() < deadline: + time.sleep(interval) + count = self.landed() + if count != expected: + raise TimeoutError( + f"{self.fq('RAW.ORDERS_CDC')} has {count} rows, expected {expected} within {timeout}s " + f"(topic {self.topic}; check 'docker compose logs connect')" + ) + return count + + # -- task ----------------------------------------------------------------- + def _task_runs(self) -> list[TaskRun]: + rows = self.q( + f"""SELECT STATE, ERROR_CODE, ERROR_MESSAGE, QUERY_ID, SCHEDULED_TIME + FROM TABLE({self.database}.INFORMATION_SCHEMA.TASK_HISTORY(TASK_NAME => 'APPLY_ORDERS_CDC')) + ORDER BY SCHEDULED_TIME DESC""" + ) + return [TaskRun(*row) for row in rows if row[0] in TERMINAL_STATES] + + def run_task(self, timeout: float = 120, interval: float = 0.5) -> TaskRun: + """EXECUTE TASK, then poll TASK_HISTORY() until that run finishes.""" + before = len(self._task_runs()) + self.q(f"EXECUTE TASK {self.fq(TASK)}") + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + runs = self._task_runs() + if len(runs) > before: + return runs[0] + time.sleep(interval) + raise TimeoutError(f"no finished run of {self.fq(TASK)} in TASK_HISTORY() within {timeout}s") + + def set_fail_next_run(self, fail: bool) -> None: + self.q( + f"""UPDATE {self.fq('OPS.PIPELINE_CONTROL')} + SET FAIL_NEXT_RUN = %s, UPDATED_AT = CURRENT_TIMESTAMP()::TIMESTAMP_NTZ + WHERE PIPELINE_NAME = 'orders_cdc'""", + (fail,), + ) + + def stream_has_data(self) -> bool: + return bool(self.scalar(f"SELECT SYSTEM$STREAM_HAS_DATA('{self.fq('RAW.ORDERS_CDC_STREAM')}')")) + + def refresh_mart(self) -> None: + self.q(f"ALTER DYNAMIC TABLE {self.fq('MART.DAILY_BOOKINGS')} REFRESH") + + # -- reads ------------------------------------------------------------------ + def orders(self) -> list[tuple]: + return self.q( + f"""SELECT O_ORDERKEY, O_CUSTKEY, O_ORDERSTATUS, O_TOTALPRICE, O_ORDERDATE, O_ORDERPRIORITY, + O_CLERK, IS_DELETED, SOURCE_LSN + FROM {self.fq('CORE.ORDERS')} ORDER BY O_ORDERKEY""" + ) + + def customer_history(self) -> list[tuple]: + return self.q( + f"""SELECT C_CUSTKEY, C_NAME, C_NATIONKEY, C_ACCTBAL, C_MKTSEGMENT, VALID_FROM, VALID_TO, + IS_CURRENT, SOURCE_LSN + FROM {self.fq('CORE.DIM_CUSTOMER_HISTORY')} ORDER BY C_CUSTKEY, SOURCE_LSN""" + ) + + def daily_bookings(self) -> list[tuple]: + return self.q( + f"""SELECT BOOKING_DATE, MARKET_SEGMENT, ORDER_COUNT, GROSS_BOOKINGS + FROM {self.fq('MART.DAILY_BOOKINGS')} ORDER BY BOOKING_DATE, MARKET_SEGMENT""" + ) + + def dead_letters(self) -> list[tuple]: + return self.q( + f"""SELECT REASON, RECORD_CONTENT:source:lsn::NUMBER + FROM {self.fq('OPS.DEAD_LETTER')} ORDER BY 2, 1""" + ) + + +class IsolatedPipeline: + """Context manager: a fresh database, topic and sink connector per run.""" + + def __init__(self, settings: Settings, conn: SnowflakeConnection, prefix: str = "O2C_T", + database: str | None = None, keep_database: bool = False) -> None: + run_id = uuid.uuid4().hex[:8] + self.settings = settings + self.conn = conn + self.database = (database or f"{prefix}_{run_id}").upper() + # Topic and connector are unique per run even when the database name is + # fixed (the demo), so a crashed run can never feed a later one. + self.topic = f"orders.cdc.{self.database.lower()}.{run_id}" + self.connector = f"o2c-sink-{self.database.lower().replace('_', '-')}-{run_id}" + self.keep_database = keep_database + self.topics = kafka_connect.Topics(settings) + self.connect = kafka_connect.ConnectClient(settings) + + def __enter__(self) -> Pipeline: + snowflake_db.deploy(self.conn, self.settings, self.database) + self.topics.create(self.topic) + self.connect.put(self.connector, kafka_connect.sink_config(self.settings, self.topic, self.database)) + self.connect.wait_running(self.connector) + return Pipeline(self.settings, self.conn, self.database, self.topic) + + def __exit__(self, *exc) -> None: + self.connect.delete(self.connector) + self.topics.delete(self.topic) + if not self.keep_database: + snowflake_db.drop(self.conn, self.database) diff --git a/order-to-cash-cdc/pipeline/kafka_connect.py b/order-to-cash-cdc/pipeline/kafka_connect.py new file mode 100644 index 0000000..73d3aed --- /dev/null +++ b/order-to-cash-cdc/pipeline/kafka_connect.py @@ -0,0 +1,152 @@ +"""Kafka topics and the Snowflake sink connector (Kafka Connect REST API). + +CLI (used by `make up`): + python -m pipeline.kafka_connect setup # create orders.cdc, register the sink + python -m pipeline.kafka_connect status +""" + +from __future__ import annotations + +import copy +import json +import sys +import time + +import requests +from confluent_kafka.admin import AdminClient, NewTopic + +from pipeline.settings import SAMPLE_ROOT, Settings, load + +SINK_TEMPLATE = SAMPLE_ROOT / "connect" / "snowflake-sink.json" +TOPIC_PARTITIONS = 3 + + +def sink_config(settings: Settings, topic: str, database: str, table: str = "ORDERS_CDC") -> dict: + """The committed connector config, pointed at `topic` -> `database`.RAW.`table`.""" + config = copy.deepcopy(json.loads(SINK_TEMPLATE.read_text(encoding="utf-8"))["config"]) + config.update({ + "topics": topic, + "snowflake.topic2table.map": f"{topic}:{table}", + "snowflake.database.name": database, + "snowflake.url.name": settings.connector_url, + "snowflake.user.name": settings.user, + "snowflake.role.name": settings.connector_role, + }) + return config + + +def default_connector_name() -> str: + return json.loads(SINK_TEMPLATE.read_text(encoding="utf-8"))["name"] + + +class Topics: + def __init__(self, settings: Settings) -> None: + self.admin = AdminClient({"bootstrap.servers": settings.kafka_bootstrap}) + + def create(self, name: str, partitions: int = TOPIC_PARTITIONS) -> None: + futures = self.admin.create_topics([NewTopic(name, num_partitions=partitions, replication_factor=1)]) + for future in futures.values(): + try: + future.result(timeout=30) + except Exception as exc: # noqa: BLE001 - KafkaException carries the error code + if "TOPIC_ALREADY_EXISTS" not in str(exc): + raise + + def delete(self, name: str) -> None: + for future in self.admin.delete_topics([name], operation_timeout=30).values(): + try: + future.result(timeout=30) + except Exception as exc: # noqa: BLE001 + if "UNKNOWN_TOPIC_OR_PART" not in str(exc): + raise + + +class ConnectClient: + def __init__(self, settings: Settings) -> None: + self.url = settings.connect_url + + def wait_ready(self, timeout: float = 180) -> None: + deadline = time.monotonic() + timeout + last = "" + while time.monotonic() < deadline: + try: + plugins = requests.get(f"{self.url}/connector-plugins", timeout=5).json() + if any(p["class"].endswith("SnowflakeSinkConnector") for p in plugins): + return + last = f"plugins: {[p['class'] for p in plugins]}" + except (requests.RequestException, ValueError) as exc: + last = str(exc) + time.sleep(2) + raise TimeoutError(f"Kafka Connect at {self.url} not ready after {timeout}s ({last}); " + "check 'docker compose logs connect'") + + def put(self, name: str, config: dict) -> None: + resp = requests.put(f"{self.url}/connectors/{name}/config", json=config, timeout=30) + if resp.status_code not in (200, 201): + raise RuntimeError(f"registering connector {name} failed: {resp.status_code} {resp.text}") + + def status(self, name: str) -> dict | None: + resp = requests.get(f"{self.url}/connectors/{name}/status", timeout=10) + return resp.json() if resp.status_code == 200 else None + + def wait_running(self, name: str, timeout: float = 120) -> None: + deadline = time.monotonic() + timeout + last: dict | None = None + while time.monotonic() < deadline: + last = self.status(name) + if last: + states = [last["connector"]["state"]] + [t["state"] for t in last.get("tasks", [])] + if "FAILED" in states: + trace = next((t.get("trace", "") for t in last.get("tasks", []) if t["state"] == "FAILED"), "") + raise RuntimeError(f"connector {name} FAILED: {trace[:2000]}") + if last.get("tasks") and all(s == "RUNNING" for s in states): + return + time.sleep(1) + raise TimeoutError(f"connector {name} not RUNNING after {timeout}s: {last}") + + def delete(self, name: str) -> None: + resp = requests.delete(f"{self.url}/connectors/{name}", timeout=30) + if resp.status_code not in (204, 404): + raise RuntimeError(f"deleting connector {name} failed: {resp.status_code} {resp.text}") + + +def setup(settings: Settings) -> None: + """Create the orders.cdc topic and register the sink for O2C.RAW.ORDERS_CDC.""" + from snowflake.connector.errors import ProgrammingError + + from pipeline import snowflake_db + + # Init hooks run after the emulator starts answering; wait for the landing table. + with snowflake_db.connect(settings) as conn: + deadline = time.monotonic() + 120 + while True: + try: + snowflake_db.query(conn, f"SELECT COUNT(*) FROM {settings.database}.RAW.ORDERS_CDC") + break + except ProgrammingError: + if time.monotonic() > deadline: + raise + time.sleep(2) + connect = ConnectClient(settings) + connect.wait_ready() + config = sink_config(settings, "orders.cdc", settings.database) + Topics(settings).create(config["topics"]) + name = default_connector_name() + connect.put(name, config) + connect.wait_running(name) + print(f"Connector {name} RUNNING: {config['topics']} -> {settings.database}.RAW.ORDERS_CDC") + + +def main() -> None: + settings = load() + command = sys.argv[1] if len(sys.argv) > 1 else "setup" + if command == "setup": + setup(settings) + elif command == "status": + print(json.dumps(ConnectClient(settings).status(default_connector_name()), indent=2)) + else: + sys.exit(f"unknown command {command!r} (setup | status)") + + +if __name__ == "__main__": + main() diff --git a/order-to-cash-cdc/pipeline/seed.py b/order-to-cash-cdc/pipeline/seed.py new file mode 100644 index 0000000..ca16d31 --- /dev/null +++ b/order-to-cash-cdc/pipeline/seed.py @@ -0,0 +1,39 @@ +"""`make seed`: push the seeded scenario through the O2C deployment. + +Produces batch 1 to `orders.cdc` (the sink registered by `make up`), waits for +it to land in O2C.RAW.ORDERS_CDC, runs the task once and refreshes the mart. +Safe to repeat: replayed events are recognised by their source LSN, so core +and mart tables end up identical (only the landing table grows). +""" + +from __future__ import annotations + +from pipeline import settings as settings_module +from pipeline import snowflake_db +from pipeline.harness import Pipeline +from producer.produce import initial_scenario + + +def main() -> None: + settings = settings_module.load() + with snowflake_db.connect(settings) as conn: + pipeline = Pipeline(settings, conn, settings.database, "orders.cdc") + scenario = initial_scenario(42) + before = pipeline.landed() + produced = pipeline.produce(scenario.events) + pipeline.wait_for_landing(before + produced) + run = pipeline.run_task() + if run.state != "SUCCEEDED": + raise SystemExit(f"task run {run.state}: {run.error_code} {run.error_message}") + pipeline.refresh_mart() + orders = pipeline.orders() + print(f"Seeded {settings.database}: {produced} events produced to orders.cdc, " + f"{before + produced} rows in RAW.ORDERS_CDC") + print(f" CORE.ORDERS {len(orders)} rows ({sum(1 for o in orders if o[7])} soft-deleted), " + f"CORE.DIM_CUSTOMER_HISTORY {len(pipeline.customer_history())} versions, " + f"MART.DAILY_BOOKINGS {len(pipeline.daily_bookings())} rows, " + f"OPS.DEAD_LETTER {len(pipeline.dead_letters())} rows") + + +if __name__ == "__main__": + main() diff --git a/order-to-cash-cdc/pipeline/settings.py b/order-to-cash-cdc/pipeline/settings.py new file mode 100644 index 0000000..1eb5312 --- /dev/null +++ b/order-to-cash-cdc/pipeline/settings.py @@ -0,0 +1,96 @@ +"""Connection settings for the emulator and for real Snowflake, read in one place. + +TEST_TARGET picks the defaults: EMULATOR (the local Snowflake-Next container) +or SNOWFLAKE_CLOUD (the SNOWFLAKE_* variables). Nothing else in the sample +branches on the target. +""" + +from __future__ import annotations + +import os +from dataclasses import dataclass +from pathlib import Path + +SAMPLE_ROOT = Path(__file__).resolve().parent.parent +EMULATOR_HOST = "snowflake.localhost.localstack.cloud" + + +def _env(name: str, default: str | None = None) -> str | None: + value = os.getenv(name) + return value if value not in (None, "") else default + + +def _required(name: str) -> str: + value = _env(name) + if value is None: + raise RuntimeError(f"{name} must be set when TEST_TARGET=SNOWFLAKE_CLOUD") + return value + + +@dataclass(frozen=True) +class Settings: + target: str + account: str + host: str + port: int + protocol: str + user: str + private_key_path: Path + role: str + warehouse: str + database: str + # What the Kafka Connect container uses to reach Snowflake. On the + # emulator this is the compose network alias on port 4566, not the host + # port mapping. + connector_url: str + connector_role: str + kafka_bootstrap: str + connect_url: str + + @property + def is_emulator(self) -> bool: + return self.target == "EMULATOR" + + +def load() -> Settings: + target = (_env("TEST_TARGET", "EMULATOR") or "EMULATOR").upper() + if target not in ("EMULATOR", "SNOWFLAKE_CLOUD"): + raise RuntimeError(f"TEST_TARGET must be EMULATOR or SNOWFLAKE_CLOUD, got {target!r}") + + key_path = Path(_env("SNOWFLAKE_PRIVATE_KEY_PATH", str(SAMPLE_ROOT / ".secrets" / "rsa_key.p8"))) + if not key_path.is_absolute(): + key_path = SAMPLE_ROOT / key_path + common = { + "user": _env("SNOWFLAKE_USER", "SAMPLE_SVC"), + "private_key_path": key_path, + "warehouse": _env("SNOWFLAKE_WAREHOUSE", "O2C_WH"), + "database": _env("O2C_DATABASE", "O2C"), + "connector_role": _env("O2C_CONNECTOR_ROLE", "O2C_INGEST"), + "kafka_bootstrap": f"localhost:{_env('KAFKA_PORT', '19092')}", + "connect_url": f"http://127.0.0.1:{_env('CONNECT_PORT', '18083')}", + } + + if target == "EMULATOR": + return Settings( + target=target, + account=_env("SNOWFLAKE_ACCOUNT", "snowflake"), + host=EMULATOR_HOST, + port=int(_env("SNOWFLAKE_PORT", "4566")), + protocol="https", + role=_env("SNOWFLAKE_ROLE", "ACCOUNTADMIN"), + connector_url=f"https://{EMULATOR_HOST}:4566", + **common, + ) + + account = _required("SNOWFLAKE_ACCOUNT") + host = _env("SNOWFLAKE_HOST", f"{account}.snowflakecomputing.com") + return Settings( + target=target, + account=account, + host=host, + port=443, + protocol="https", + role=_required("SNOWFLAKE_ROLE"), + connector_url=f"https://{host}:443", + **common, + ) diff --git a/order-to-cash-cdc/pipeline/snowflake_db.py b/order-to-cash-cdc/pipeline/snowflake_db.py new file mode 100644 index 0000000..6a99029 --- /dev/null +++ b/order-to-cash-cdc/pipeline/snowflake_db.py @@ -0,0 +1,97 @@ +"""Snowflake connections and pipeline deployment.""" + +from __future__ import annotations + +import re +from collections.abc import Sequence +from typing import Any + +import snowflake.connector +from cryptography.hazmat.primitives import serialization +from snowflake.connector import SnowflakeConnection + +from pipeline.settings import SAMPLE_ROOT, Settings + +QUERY_TAG = "order-to-cash-cdc" +INIT_DIR = SAMPLE_ROOT / "init" +# Init files put account-level setup (role, warehouse, the O2C database) above +# this marker and database-relative objects below it. +DEPLOY_MARKER = "-- @deploy:" + + +def private_key_der(settings: Settings) -> bytes: + """The unencrypted PKCS#8 key as DER bytes. + + Loaded here rather than passed as private_key_file: connector 5.x refuses + key files that are group- or world-writable, which is every file on some + mounted filesystems (WSL /mnt/c, Docker Desktop bind mounts). + """ + key = serialization.load_pem_private_key(settings.private_key_path.read_bytes(), password=None) + return key.private_bytes( + encoding=serialization.Encoding.DER, + format=serialization.PrivateFormat.PKCS8, + encryption_algorithm=serialization.NoEncryption(), + ) + + +def connect(settings: Settings, database: str | None = None) -> SnowflakeConnection: + """Open a key-pair (SNOWFLAKE_JWT) connection with an explicit role and warehouse.""" + return snowflake.connector.connect( + account=settings.account, + host=settings.host, + port=settings.port, + protocol=settings.protocol, + user=settings.user, + authenticator="SNOWFLAKE_JWT", + private_key=private_key_der(settings), + role=settings.role, + warehouse=settings.warehouse, + database=database, + schema="OPS" if database else None, + session_parameters={"QUERY_TAG": QUERY_TAG}, + client_session_keep_alive=False, + # Always log out on close; no async queries outlive a connection here. + enable_server_session_keep_alive_auto_detection=False, + ) + + +def query(conn: SnowflakeConnection, sql: str, params: Sequence[Any] | None = None) -> list[tuple]: + with conn.cursor() as cur: + cur.execute(sql, params) + return cur.fetchall() if cur.description else [] + + +def scalar(conn: SnowflakeConnection, sql: str, params: Sequence[Any] | None = None) -> Any: + rows = query(conn, sql, params) + return rows[0][0] if rows else None + + +def deployable_sql(settings: Settings) -> str: + """Database-relative DDL from init/*.sf.sql, in file order. + + The warehouse name is the one in the init files unless SNOWFLAKE_WAREHOUSE + names another (real accounts rarely have O2C_WH). + """ + parts = [] + for path in sorted(INIT_DIR.glob("*.sf.sql")): + text = path.read_text(encoding="utf-8") + if DEPLOY_MARKER not in text: + raise RuntimeError(f"{path.name} has no '{DEPLOY_MARKER}' marker") + parts.append(text.split(DEPLOY_MARKER, 1)[1].split("\n", 1)[1]) + sql = "\n".join(parts) + return re.sub(r"\bWAREHOUSE = O2C_WH\b", f"WAREHOUSE = {settings.warehouse}", sql) + + +def deploy(conn: SnowflakeConnection, settings: Settings, database: str) -> None: + """Create `database` from scratch with every pipeline object in it.""" + with conn.cursor() as cur: + cur.execute(f"CREATE OR REPLACE DATABASE {database}") + cur.execute(f"USE DATABASE {database}") + cur.execute(f"GRANT USAGE ON DATABASE {database} TO ROLE {settings.connector_role}") + for cur in conn.execute_string(deployable_sql(settings), remove_comments=True): + cur.close() + query(conn, f"USE SCHEMA {database}.OPS") + + +def drop(conn: SnowflakeConnection, database: str) -> None: + query(conn, f"DROP DATABASE IF EXISTS {database}") diff --git a/order-to-cash-cdc/producer/__init__.py b/order-to-cash-cdc/producer/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/order-to-cash-cdc/producer/produce.py b/order-to-cash-cdc/producer/produce.py new file mode 100644 index 0000000..d0117ee --- /dev/null +++ b/order-to-cash-cdc/producer/produce.py @@ -0,0 +1,356 @@ +"""Seeded Debezium-style change generator for TPC-H shaped `orders` and `customer`. + +A small simulated source database applies inserts, updates and deletes and +writes one change event per commit, with a monotonically increasing log +sequence number (LSN), like Debezium's PostgreSQL connector. Delivery then +adds what real CDC pipelines see on the wire: + +* duplicates: an event delivered twice (at-least-once redelivery), +* late events: an older change for a key delivered after a newer one, +* poison events: records the pipeline must reject (truncate op, missing key, + a table the pipeline does not own). + +The same seed always yields the same events, so tests can compute expected +results from the events alone. + +CLI: python -m producer.produce --topic orders.cdc --seed 42 +""" + +from __future__ import annotations + +import argparse +import copy +import json +import random +from dataclasses import dataclass, field, replace +from datetime import UTC, date, datetime, timedelta +from decimal import Decimal + +SEGMENTS = ("AUTOMOBILE", "BUILDING", "FURNITURE", "HOUSEHOLD", "MACHINERY") +PRIORITIES = ("1-URGENT", "2-HIGH", "3-MEDIUM", "4-NOT SPECIFIED", "5-LOW") +EPOCH = date(1970, 1, 1) +FIRST_ORDER_DATE = date(2026, 10, 1) +ORDER_DATE_SPAN_DAYS = 5 +DEBEZIUM_VERSION = "2.7.3.Final" + + +@dataclass(frozen=True) +class ChangeEvent: + """One record on the `orders.cdc` topic. + + `kind` is the generator's ground truth: "change" (first delivery), + "duplicate", "late" or "poison". `reject_reason` is set for poison + events only. + """ + + table: str + op: str + key: int | None + lsn: int + ts_ms: int + before: dict | None + after: dict | None + kind: str = "change" + reject_reason: str | None = None + envelope_override: dict | None = field(default=None, compare=False) + + @property + def image(self) -> dict | None: + return self.before if self.op == "d" else self.after + + def envelope(self) -> dict: + if self.envelope_override is not None: + return self.envelope_override + return { + "before": self.before, + "after": self.after, + "source": { + "version": DEBEZIUM_VERSION, + "connector": "postgresql", + "name": "tpch", + "ts_ms": self.ts_ms, + "db": "tpch", + "schema": "public", + "table": self.table, + "lsn": self.lsn, + }, + "op": self.op, + "ts_ms": self.ts_ms + 250, + } + + def kafka_key(self) -> str: + key_column = "o_orderkey" if self.table == "orders" else "c_custkey" + return json.dumps({key_column: self.key}) + + def kafka_value(self) -> str: + return json.dumps(self.envelope(), separators=(",", ":")) + + +def money(value: Decimal) -> str: + # Debezium decimal.handling.mode=string + return str(value.quantize(Decimal("0.01"))) + + +class SourceDatabase: + """The upstream OLTP database: current rows plus a change log.""" + + def __init__(self, seed: int, start_lsn: int = 24_000_000, start: datetime | None = None) -> None: + self.rng = random.Random(seed) + self.lsn = start_lsn + self.now = start or datetime(2026, 10, 1, 8, 0, 0, tzinfo=UTC) + self.customers: dict[int, dict] = {} + self.orders: dict[int, dict] = {} + self.next_custkey = 1 + self.next_orderkey = 1 + + # -- change log --------------------------------------------------------- + def _commit(self, table: str, op: str, key: int, before: dict | None, after: dict | None) -> ChangeEvent: + self.lsn += self.rng.randint(8, 96) + self.now += timedelta(milliseconds=self.rng.randint(150, 4_000)) + return ChangeEvent( + table=table, + op=op, + key=key, + lsn=self.lsn, + ts_ms=int(self.now.timestamp() * 1000), + before=copy.deepcopy(before), + after=copy.deepcopy(after), + ) + + # -- customer ----------------------------------------------------------- + def insert_customer(self, op: str = "c") -> ChangeEvent: + key = self.next_custkey + self.next_custkey += 1 + row = { + "c_custkey": key, + "c_name": f"Customer#{key:09d}", + "c_nationkey": self.rng.randint(0, 24), + "c_acctbal": money(Decimal(self.rng.randint(-99_999, 999_999)) / 100), + "c_mktsegment": self.rng.choice(SEGMENTS), + } + self.customers[key] = row + return self._commit("customer", op, key, None, row) + + def update_customer(self, key: int | None = None) -> ChangeEvent: + key = key if key is not None else self.rng.choice(sorted(self.customers)) + before = self.customers[key] + after = dict(before) + if self.rng.random() < 0.6: + after["c_mktsegment"] = self.rng.choice([s for s in SEGMENTS if s != before["c_mktsegment"]]) + after["c_acctbal"] = money(Decimal(before["c_acctbal"]) + Decimal(self.rng.randint(-50_000, 50_000)) / 100) + self.customers[key] = after + return self._commit("customer", "u", key, before, after) + + def delete_customer(self, key: int | None = None) -> ChangeEvent: + key = key if key is not None else self.rng.choice(sorted(self.customers)) + before = self.customers.pop(key) + return self._commit("customer", "d", key, before, None) + + # -- orders ------------------------------------------------------------- + def insert_order(self) -> ChangeEvent: + key = self.next_orderkey + self.next_orderkey += 1 + order_date = FIRST_ORDER_DATE + timedelta(days=self.rng.randrange(ORDER_DATE_SPAN_DAYS)) + row = { + "o_orderkey": key, + "o_custkey": self.rng.choice(sorted(self.customers)), + "o_orderstatus": "O", + "o_totalprice": money(Decimal(self.rng.randint(90_000, 45_000_000)) / 100), + "o_orderdate": (order_date - EPOCH).days, # Debezium io.debezium.time.Date + "o_orderpriority": self.rng.choice(PRIORITIES), + "o_clerk": f"Clerk#{self.rng.randint(1, 1000):09d}", + } + self.orders[key] = row + return self._commit("orders", "c", key, None, row) + + def update_order(self, key: int | None = None) -> ChangeEvent: + key = key if key is not None else self.rng.choice(sorted(self.orders)) + before = self.orders[key] + after = dict(before) + after["o_orderstatus"] = {"O": "P", "P": "F", "F": "F"}[before["o_orderstatus"]] + if self.rng.random() < 0.5: + after["o_totalprice"] = money(Decimal(before["o_totalprice"]) * Decimal("0.95")) + self.orders[key] = after + return self._commit("orders", "u", key, before, after) + + def delete_order(self, key: int | None = None) -> ChangeEvent: + key = key if key is not None else self.rng.choice(sorted(self.orders)) + before = self.orders.pop(key) + return self._commit("orders", "d", key, before, None) + + # -- workload ----------------------------------------------------------- + def snapshot(self, customers: int) -> list[ChangeEvent]: + """Initial snapshot read (op "r"), as Debezium emits on first start.""" + return [self.insert_customer(op="r") for _ in range(customers)] + + def workload(self, changes: int) -> list[ChangeEvent]: + events: list[ChangeEvent] = [] + for _ in range(changes): + roll = self.rng.random() + if roll < 0.40 or not self.orders: + events.append(self.insert_order()) + elif roll < 0.68: + events.append(self.update_order()) + elif roll < 0.74 and len(self.orders) > 3: + events.append(self.delete_order()) + elif roll < 0.80: + events.append(self.insert_customer()) + elif roll < 0.97: + events.append(self.update_customer()) + elif len(self.customers) > 3: + events.append(self.delete_customer()) + else: + events.append(self.update_customer()) + return events + + +def deliver(events: list[ChangeEvent], rng: random.Random, duplicates: int, late: int) -> list[ChangeEvent]: + """Delivery order on the topic: commit order plus duplicates and late events.""" + delivered = list(events) + # Late: an update delivered after the next change for the same key. + candidates = [] + for i, ev in enumerate(events): + if ev.op != "u": + continue + nxt = next((j for j in range(i + 1, len(events)) if (events[j].table, events[j].key) == (ev.table, ev.key)), None) + if nxt is not None: + candidates.append((i, nxt)) + moved: set[int] = set() + for i, j in rng.sample(candidates, min(late, len(candidates))): + if i in moved or j in moved: + continue + moved.update((i, j)) # j stays put so i really arrives after it + delivered.remove(events[i]) + delivered.insert(delivered.index(events[j]) + 1, replace(events[i], kind="late")) + # Duplicates: redeliver a few events a little later (same LSN, same payload). + for ev in rng.sample([e for k, e in enumerate(events) if k not in moved], duplicates): + pos = min(len(delivered), delivered.index(ev) + rng.randint(1, 10)) + delivered.insert(pos, replace(ev, kind="duplicate")) + return delivered + + +def poison_events(source: SourceDatabase) -> list[ChangeEvent]: + """Records the pipeline must dead-letter instead of applying.""" + source.now += timedelta(seconds=1) + ts_ms = int(source.now.timestamp() * 1000) + base = {"version": DEBEZIUM_VERSION, "connector": "postgresql", "name": "tpch", "db": "tpch", "schema": "public"} + truncate = { + "before": None, + "after": None, + "source": {**base, "ts_ms": ts_ms, "table": "orders", "lsn": source.lsn + 1}, + "op": "t", + "ts_ms": ts_ms, + } + missing_key = { + "before": None, + "after": {"o_custkey": 1, "o_orderstatus": "O", "o_totalprice": "10.00"}, + "source": {**base, "ts_ms": ts_ms, "table": "orders", "lsn": source.lsn + 2}, + "op": "c", + "ts_ms": ts_ms, + } + foreign_table = { + "before": None, + "after": {"l_orderkey": 1, "l_linenumber": 1}, + "source": {**base, "ts_ms": ts_ms, "table": "lineitem", "lsn": source.lsn + 3}, + "op": "c", + "ts_ms": ts_ms, + } + source.lsn += 3 + return [ + ChangeEvent("orders", "t", None, source.lsn - 2, ts_ms, None, None, "poison", "UNKNOWN_OP", truncate), + ChangeEvent("orders", "c", None, source.lsn - 1, ts_ms, None, None, "poison", "MISSING_KEY", missing_key), + ChangeEvent("lineitem", "c", None, source.lsn, ts_ms, None, None, "poison", "UNKNOWN_SOURCE_TABLE", + foreign_table), + ] + + +@dataclass +class Scenario: + """A source database and the batches of events it has delivered so far.""" + + seed: int + source: SourceDatabase + batches: list[list[ChangeEvent]] = field(default_factory=list) + + @property + def events(self) -> list[ChangeEvent]: + return [ev for batch in self.batches for ev in batch] + + +def initial_scenario(seed: int = 42) -> Scenario: + """Batch 1: snapshot, a mixed workload, guaranteed edge cases, poison.""" + source = SourceDatabase(seed) + rng = random.Random(seed + 1) + commits = source.snapshot(customers=12) + commits += source.workload(changes=70) + # Guaranteed coverage regardless of the random mix: a customer updated + # twice, a customer deleted while one of their orders is still live (its + # bookings move to segment UNKNOWN), an order soft-deleted. + twice = min(source.customers) + commits += [source.update_customer(twice), source.update_customer(twice)] + with_orders = sorted({o["o_custkey"] for o in source.orders.values()} - {twice}) + commits.append(source.delete_customer(with_orders[-1])) + commits.append(source.delete_order(min(source.orders))) + batch = deliver(commits, rng, duplicates=6, late=4) + poison_events(source) + return Scenario(seed=seed, source=source, batches=[batch]) + + +def next_batch(scenario: Scenario, changes: int = 30) -> list[ChangeEvent]: + """A follow-up batch from the same source (higher LSNs), with duplicates. + + It also redelivers an already superseded order event and customer event + from earlier batches (a connector or source restart replaying old + offsets), which the pipeline must recognise as stale by LSN. + """ + rng = random.Random(scenario.seed + len(scenario.batches) + 1) + earlier = [e for e in scenario.events if e.kind == "change"] + batch = deliver(scenario.source.workload(changes), rng, duplicates=3, late=2) + for table in ("orders", "customer"): + stale = next(e for e in earlier if e.table == table and any( + later.key == e.key and later.lsn > e.lsn for later in earlier if later.table == table)) + batch.insert(rng.randrange(len(batch) + 1), replace(stale, kind="duplicate")) + scenario.batches.append(batch) + return batch + + +def publish(events: list[ChangeEvent], topic: str, bootstrap: str) -> int: + """Produce events in delivery order, keyed by primary key, and wait for acks.""" + from confluent_kafka import Producer + + errors: list[str] = [] + + def on_delivery(err, _msg) -> None: + if err is not None: + errors.append(str(err)) + + producer = Producer({ + "bootstrap.servers": bootstrap, + "acks": "all", + "enable.idempotence": True, + "linger.ms": 5, + "client.id": "o2c-cdc-producer", + }) + for ev in events: + producer.produce(topic, key=ev.kafka_key(), value=ev.kafka_value(), on_delivery=on_delivery) + remaining = producer.flush(60) + if remaining or errors: + raise RuntimeError(f"{remaining} events not delivered to {topic}; errors: {errors[:3]}") + return len(events) + + +def main() -> None: + import os + + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("--topic", default="orders.cdc") + parser.add_argument("--seed", type=int, default=42) + parser.add_argument("--bootstrap", default=f"localhost:{os.getenv('KAFKA_PORT', '19092')}") + args = parser.parse_args() + scenario = initial_scenario(args.seed) + count = publish(scenario.events, args.topic, args.bootstrap) + kinds = {k: sum(1 for e in scenario.events if e.kind == k) for k in ("change", "duplicate", "late", "poison")} + print(f"Produced {count} events to {args.topic} (seed {args.seed}): {kinds}") + + +if __name__ == "__main__": + main() diff --git a/order-to-cash-cdc/pyproject.toml b/order-to-cash-cdc/pyproject.toml new file mode 100644 index 0000000..39a88f2 --- /dev/null +++ b/order-to-cash-cdc/pyproject.toml @@ -0,0 +1,26 @@ +[project] +name = "order-to-cash-cdc" +version = "0.1.0" +description = "Kafka change events into current-state and SCD2 tables through a stream-driven Snowflake task" +readme = "README.md" +license = "Apache-2.0" +requires-python = "==3.12.*" +dependencies = [ + "snowflake-connector-python>=5.1,<6", + "confluent-kafka>=2.6,<3", + "requests>=2.32,<3", + "cryptography>=43", +] + +[dependency-groups] +dev = [ + "pytest>=8.3,<9", +] + +[tool.uv] +package = false + +[tool.pytest.ini_options] +testpaths = ["tests"] +pythonpath = ["."] +addopts = "-ra" diff --git a/order-to-cash-cdc/scripts/wait-for-emulator.sh b/order-to-cash-cdc/scripts/wait-for-emulator.sh new file mode 100755 index 0000000..501c7e2 --- /dev/null +++ b/order-to-cash-cdc/scripts/wait-for-emulator.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# Wait until the Snowflake-Next emulator answers a SQL statement. +# +# Copy this file into a sample as scripts/wait-for-emulator.sh (samples stay +# self-contained). Usage: scripts/wait-for-emulator.sh [timeout-seconds] +# Reads SNOWFLAKE_PORT (default 4566). Readiness is checked from the host. +set -euo pipefail + +port="${SNOWFLAKE_PORT:-4566}" +timeout="${1:-120}" +url="https://snowflake.localhost.localstack.cloud:${port}" +deadline=$((SECONDS + timeout)) + +until curl -fsS --compressed -o /dev/null -X POST "${url}/api/v2/statements" \ + -H 'Authorization: Bearer local' -H 'Content-Type: application/json' \ + -H 'Accept: application/json' -d '{"statement":"SELECT 1"}' 2>/dev/null; do + if (( SECONDS >= deadline )); then + echo "Emulator at ${url} did not become ready within ${timeout}s." >&2 + echo "Check 'docker compose logs snowflake' (is LOCALSTACK_AUTH_TOKEN valid? is port ${port} free?)." >&2 + exit 1 + fi + sleep 2 +done +echo "Emulator ready at ${url}" diff --git a/order-to-cash-cdc/tests/__init__.py b/order-to-cash-cdc/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/order-to-cash-cdc/tests/conftest.py b/order-to-cash-cdc/tests/conftest.py new file mode 100644 index 0000000..b32aba6 --- /dev/null +++ b/order-to-cash-cdc/tests/conftest.py @@ -0,0 +1,65 @@ +"""Fixtures: one connection per session, one isolated pipeline per scenario. + +Each pipeline gets its own database (fresh DDL from init/*.sf.sql), Kafka topic +and Snowflake sink connector, and all three are removed afterwards, so runs +never see each other's data and the O2C database `make seed` fills is never +touched. +""" + +from __future__ import annotations + +from collections.abc import Iterator +from dataclasses import dataclass + +import pytest + +from pipeline import settings as settings_module +from pipeline import snowflake_db +from pipeline.harness import IsolatedPipeline, Pipeline, TaskRun +from producer.produce import Scenario, initial_scenario + +SEED = 42 + + +@pytest.fixture(scope="session") +def settings() -> settings_module.Settings: + return settings_module.load() + + +@pytest.fixture(scope="session") +def conn(settings): + connection = snowflake_db.connect(settings) + yield connection + connection.close() + + +@pytest.fixture +def isolated_pipeline(settings, conn) -> Iterator[Pipeline]: + with IsolatedPipeline(settings, conn) as pipeline: + yield pipeline + + +@dataclass +class Loaded: + """A pipeline after batch 1 has been produced, landed and applied once.""" + + pipeline: Pipeline + scenario: Scenario + first_run: TaskRun + + +def load_initial_batch(pipeline: Pipeline) -> Loaded: + scenario = initial_scenario(SEED) + produced = pipeline.produce(scenario.events) + pipeline.wait_for_landing(produced) + run = pipeline.run_task() + assert run.state == "SUCCEEDED", f"initial apply failed: {run}" + pipeline.refresh_mart() + return Loaded(pipeline, scenario, run) + + +@pytest.fixture(scope="module") +def loaded(settings, conn) -> Iterator[Loaded]: + """Shared, read-only state for tests that only inspect results.""" + with IsolatedPipeline(settings, conn) as pipeline: + yield load_initial_batch(pipeline) diff --git a/order-to-cash-cdc/tests/model.py b/order-to-cash-cdc/tests/model.py new file mode 100644 index 0000000..bd2d64f --- /dev/null +++ b/order-to-cash-cdc/tests/model.py @@ -0,0 +1,105 @@ +"""Expected pipeline results, computed in plain Python from the emitted events. + +This is the test oracle. It applies the business rules directly to the +producer's events (newest LSN wins per key, deletes are soft, one SCD2 version +per customer change, poison events are dead-lettered) without touching any +SQL from the pipeline, so a bug in the procedure cannot hide in both places. +""" + +from __future__ import annotations + +from collections import defaultdict +from datetime import UTC, date, datetime, timedelta +from decimal import Decimal + +from producer.produce import EPOCH, ChangeEvent + + +def _ts(ts_ms: int) -> datetime: + return datetime.fromtimestamp(ts_ms / 1000, UTC).replace(tzinfo=None) + + +def _valid(events: list[ChangeEvent], table: str) -> list[ChangeEvent]: + return [e for e in events if e.kind != "poison" and e.table == table] + + +def _latest_by_key(events: list[ChangeEvent]) -> dict[int, ChangeEvent]: + latest: dict[int, ChangeEvent] = {} + for ev in events: + if ev.key not in latest or ev.lsn > latest[ev.key].lsn: + latest[ev.key] = ev + return latest + + +def expected_orders(events: list[ChangeEvent]) -> list[tuple]: + """Rows of CORE.ORDERS, in the column order of Pipeline.orders().""" + rows = [] + for key, ev in sorted(_latest_by_key(_valid(events, "orders")).items()): + img = ev.image + rows.append(( + key, + img["o_custkey"], + img["o_orderstatus"], + Decimal(img["o_totalprice"]), + EPOCH + timedelta(days=img["o_orderdate"]), + img["o_orderpriority"], + img["o_clerk"], + ev.op == "d", + ev.lsn, + )) + return rows + + +def expected_customer_history(events: list[ChangeEvent]) -> list[tuple]: + """Rows of CORE.DIM_CUSTOMER_HISTORY, in the column order of Pipeline.customer_history().""" + changes: dict[int, dict[int, ChangeEvent]] = defaultdict(dict) + for ev in _valid(events, "customer"): + changes[ev.key][ev.lsn] = ev # a redelivered duplicate has the same LSN + rows = [] + for key in sorted(changes): + ordered = [changes[key][lsn] for lsn in sorted(changes[key])] + for i, ev in enumerate(ordered): + if ev.op == "d": + continue + valid_to = _ts(ordered[i + 1].ts_ms) if i + 1 < len(ordered) else None + img = ev.image + rows.append(( + key, + img["c_name"], + img["c_nationkey"], + Decimal(img["c_acctbal"]), + img["c_mktsegment"], + _ts(ev.ts_ms), + valid_to, + valid_to is None, + ev.lsn, + )) + return rows + + +def deleted_customers(events: list[ChangeEvent]) -> dict[int, datetime]: + """Customer keys whose newest change is a delete, with the delete time.""" + latest = _latest_by_key(_valid(events, "customer")) + return {k: _ts(e.ts_ms) for k, e in latest.items() if e.op == "d"} + + +def expected_daily_bookings(events: list[ChangeEvent]) -> list[tuple]: + """Rows of MART.DAILY_BOOKINGS: live orders by order date and current segment.""" + segment = {} + for key, ev in _latest_by_key(_valid(events, "customer")).items(): + if ev.op != "d": + segment[key] = ev.image["c_mktsegment"] + totals: dict[tuple[date, str], list] = defaultdict(lambda: [0, Decimal("0.00")]) + for _key, ev in _latest_by_key(_valid(events, "orders")).items(): + if ev.op == "d": + continue + img = ev.image + bucket = totals[(EPOCH + timedelta(days=img["o_orderdate"]), segment.get(img["o_custkey"], "UNKNOWN"))] + bucket[0] += 1 + bucket[1] += Decimal(img["o_totalprice"]) + return [(d, s, n, total) for (d, s), (n, total) in sorted(totals.items())] + + +def expected_dead_letters(events: list[ChangeEvent]) -> list[tuple]: + """(reason, source LSN) per poison delivery, in the order of Pipeline.dead_letters().""" + return sorted(((e.reject_reason, e.lsn) for e in events if e.kind == "poison"), key=lambda r: (r[1], r[0])) diff --git a/order-to-cash-cdc/tests/test_model.py b/order-to-cash-cdc/tests/test_model.py new file mode 100644 index 0000000..0cdf13d --- /dev/null +++ b/order-to-cash-cdc/tests/test_model.py @@ -0,0 +1,38 @@ +"""Checks on the test oracle itself (no emulator needed). + +tests/model.py derives expected tables from the delivered events. Here it is +compared with the simulated source database's final rows, a second +independent view, so a mistake in the oracle cannot silently pass the +pipeline tests. +""" + +from __future__ import annotations + +from decimal import Decimal + +from producer.produce import initial_scenario, next_batch +from tests import model + + +def test_expected_state_matches_the_source_database(): + scenario = initial_scenario(42) + next_batch(scenario) + events = scenario.events + live_orders = {row[0]: row for row in model.expected_orders(events) if not row[7]} + assert sorted(live_orders) == sorted(scenario.source.orders) + for key, src in scenario.source.orders.items(): + assert live_orders[key][2] == src["o_orderstatus"] + assert live_orders[key][3] == Decimal(src["o_totalprice"]) + current = {row[0]: row for row in model.expected_customer_history(events) if row[7]} + assert sorted(current) == sorted(scenario.source.customers) + for key, src in scenario.source.customers.items(): + assert current[key][4] == src["c_mktsegment"] + assert current[key][3] == Decimal(src["c_acctbal"]) + + +def test_scenario_is_deterministic_and_covers_every_event_kind(): + first, second = initial_scenario(42).events, initial_scenario(42).events + assert [e.kafka_value() for e in first] == [e.kafka_value() for e in second] + kinds = {e.kind for e in first} + assert kinds == {"change", "duplicate", "late", "poison"} + assert {e.op for e in first if e.kind != "poison"} == {"r", "c", "u", "d"} diff --git a/order-to-cash-cdc/tests/test_pipeline.py b/order-to-cash-cdc/tests/test_pipeline.py new file mode 100644 index 0000000..a51e12a --- /dev/null +++ b/order-to-cash-cdc/tests/test_pipeline.py @@ -0,0 +1,176 @@ +"""End-to-end tests: Kafka -> connector -> stream -> task -> core tables -> mart. + +Every expected value comes from tests/model.py, which recomputes the business +rules from the producer's events. No test waits on the task's schedule: runs +are triggered with EXECUTE TASK and observed through TASK_HISTORY(). +""" + +from __future__ import annotations + +from collections import Counter + +import pytest +from snowflake.connector.errors import ProgrammingError + +from producer.produce import next_batch +from tests import model +from tests.conftest import load_initial_batch + + +# --- ingest --------------------------------------------------------------------- + + +def test_connector_lands_every_produced_event_once(loaded): + p, events = loaded.pipeline, loaded.scenario.events + rows = p.q( + f"""SELECT RECORD_METADATA:topic::VARCHAR, RECORD_METADATA:partition::INT, + RECORD_METADATA:offset::INT, RECORD_CONTENT:source:lsn::NUMBER + FROM {p.fq('RAW.ORDERS_CDC')}""" + ) + assert len(rows) == len(events) + assert {r[0] for r in rows} == {p.topic} + assert len({(r[1], r[2]) for r in rows}) == len(rows), "a Kafka offset landed twice" + assert {r[1] for r in rows} <= {0, 1, 2} + assert Counter(r[3] for r in rows) == Counter(e.lsn for e in events) + + +# --- current state and history -------------------------------------------------------- + + +def test_core_orders_hold_newest_version_of_every_order(loaded): + assert loaded.pipeline.orders() == model.expected_orders(loaded.scenario.events) + + +def test_order_delete_is_a_soft_delete(loaded): + p, events = loaded.pipeline, loaded.scenario.events + expected_deleted = {row[0]: row for row in model.expected_orders(events) if row[7]} + assert expected_deleted, "scenario must contain deleted orders" + actual_deleted = {row[0]: row for row in p.orders() if row[7]} + # The row survives with the attributes of the delete's "before" image. + assert actual_deleted == expected_deleted + + +def test_customer_delete_closes_the_open_scd2_version(loaded): + p, events = loaded.pipeline, loaded.scenario.events + deleted = model.deleted_customers(events) + assert deleted, "scenario must contain deleted customers" + history = p.customer_history() + for key, deleted_at in deleted.items(): + versions = [row for row in history if row[0] == key] + assert versions, f"customer {key} has no history" + assert not any(row[7] for row in versions), f"customer {key} still has a current version" + assert versions[-1][6] == deleted_at, f"customer {key}: last VALID_TO is not the delete time" + + +def test_customer_history_has_one_version_per_change(loaded): + p, events = loaded.pipeline, loaded.scenario.events + history = p.customer_history() + assert history == model.expected_customer_history(events) + # Contiguous validity: each closed version ends where the next one starts. + for prev, nxt in zip(history, history[1:], strict=False): + if prev[0] == nxt[0]: + assert prev[6] == nxt[5] + + +def test_duplicate_and_late_events_do_not_create_rows_or_regress_state(loaded): + p, events = loaded.pipeline, loaded.scenario.events + assert any(e.kind == "duplicate" for e in events) and any(e.kind == "late" for e in events) + assert p.q(f"SELECT O_ORDERKEY FROM {p.fq('CORE.ORDERS')} GROUP BY 1 HAVING COUNT(*) > 1") == [] + assert p.q( + f"SELECT C_CUSTKEY, SOURCE_LSN FROM {p.fq('CORE.DIM_CUSTOMER_HISTORY')} GROUP BY 1, 2 HAVING COUNT(*) > 1" + ) == [] + applied_lsn = {row[0]: row[8] for row in p.orders()} + for late in (e for e in events if e.kind == "late" and e.table == "orders"): + assert applied_lsn[late.key] > late.lsn, f"late event for order {late.key} overwrote a newer one" + + +def test_invalid_events_are_dead_lettered_not_applied(loaded): + p, events = loaded.pipeline, loaded.scenario.events + assert p.dead_letters() == model.expected_dead_letters(events) + assert p.scalar(f"SELECT COUNT(*) FROM {p.fq('CORE.ORDERS')} WHERE O_ORDERKEY IS NULL") == 0 + + +# --- mart ----------------------------------------------------------------------- + + +def test_daily_bookings_match_totals_computed_from_events(loaded): + expected = model.expected_daily_bookings(loaded.scenario.events) + assert len({row[0] for row in expected}) > 1, "scenario must span several booking dates" + assert "UNKNOWN" in {row[1] for row in expected}, "scenario must book orders of a deleted customer" + assert loaded.pipeline.daily_bookings() == expected + + +# --- error paths ---------------------------------------------------------------------- + + +def test_merge_without_dedup_fails_with_duplicate_row_error(loaded): + """Why the procedure ranks with QUALIFY ROW_NUMBER(): CDC batches carry several + events per key, and a MERGE that matches one target row with several source + rows is rejected (ERROR_ON_NONDETERMINISTIC_MERGE).""" + p = loaded.pipeline + before = p.orders() + with pytest.raises(ProgrammingError) as excinfo: + p.q( + f"""MERGE INTO {p.fq('CORE.ORDERS')} t + USING (SELECT * FROM {p.fq('OPS.APPLY_BATCH')} + WHERE SOURCE_TABLE = 'orders' AND REJECT_REASON IS NULL) s + ON t.O_ORDERKEY = s.ENTITY_KEY + WHEN MATCHED THEN UPDATE SET O_ORDERSTATUS = s.IMAGE:o_orderstatus::VARCHAR""" + ) + assert excinfo.value.errno == 100090 + assert "Duplicate row detected during DML action" in str(excinfo.value) + assert p.orders() == before + + +def test_replaying_the_same_events_changes_nothing(isolated_pipeline): + p = isolated_pipeline + loaded = load_initial_batch(p) + events = loaded.scenario.events + orders, history, bookings = p.orders(), p.customer_history(), p.daily_bookings() + + # The source (or a connector reset) redelivers every event of batch 1. + p.produce(events) + p.wait_for_landing(2 * len(events)) + run = p.run_task() + p.refresh_mart() + + assert run.state == "SUCCEEDED", run + assert p.orders() == orders == model.expected_orders(events) + assert p.customer_history() == history == model.expected_customer_history(events) + assert p.daily_bookings() == bookings + # Each poison *delivery* is a distinct Kafka record, so it is dead-lettered once per delivery. + assert p.dead_letters() == sorted(model.expected_dead_letters(events) * 2, key=lambda r: (r[1], r[0])) + + +def test_forced_failure_rolls_back_then_next_run_applies_each_event_exactly_once(isolated_pipeline): + """Headline: a run that fails after its MERGEs leaves no trace, and the retry + applies batch 2 exactly once.""" + p = isolated_pipeline + loaded = load_initial_batch(p) + scenario = loaded.scenario + batch2 = next_batch(scenario) + p.produce(batch2) + p.wait_for_landing(len(scenario.events)) + orders, history, dead = p.orders(), p.customer_history(), p.dead_letters() + assert orders != model.expected_orders(scenario.events), "batch 2 must change core.orders" + + p.set_fail_next_run(True) + failed = p.run_task() + + assert failed.state == "FAILED" + assert str(failed.error_code) == "-20001" + assert "FAIL_NEXT_RUN is TRUE" in failed.error_message + assert p.orders() == orders, "core.orders changed although the run failed" + assert p.customer_history() == history, "dim_customer_history changed although the run failed" + assert p.dead_letters() == dead + assert p.stream_has_data() is True, "stream offset advanced although the run failed" + + p.set_fail_next_run(False) + retried = p.run_task() + p.refresh_mart() + + assert retried.state == "SUCCEEDED", retried + assert p.orders() == model.expected_orders(scenario.events) + assert p.customer_history() == model.expected_customer_history(scenario.events) + assert p.daily_bookings() == model.expected_daily_bookings(scenario.events) + assert p.dead_letters() == dead diff --git a/order-to-cash-cdc/tests/test_transactions.py b/order-to-cash-cdc/tests/test_transactions.py new file mode 100644 index 0000000..e7b4310 --- /dev/null +++ b/order-to-cash-cdc/tests/test_transactions.py @@ -0,0 +1,47 @@ +"""Snowflake Scripting transaction semantics the apply procedure relies on. + +OPS.APPLY_ORDERS_CDC wraps its whole run in one explicit transaction and rolls +it back in the exception handler, so a failure after the first MERGE never +leaves a half-applied batch. +""" + +from __future__ import annotations + +import uuid + +import pytest +from snowflake.connector.errors import ProgrammingError + +from pipeline import snowflake_db + + +@pytest.fixture +def scratch_schema(conn): + database = f"O2C_S_{uuid.uuid4().hex[:8]}".upper() + snowflake_db.query(conn, f"CREATE DATABASE {database}") + snowflake_db.query(conn, f"CREATE TABLE {database}.PUBLIC.T (N INTEGER)") + yield f"{database}.PUBLIC" + snowflake_db.drop(conn, database) + + +def test_rollback_in_handler_discards_the_failed_transaction(conn, scratch_schema): + snowflake_db.query(conn, f""" + CREATE PROCEDURE {scratch_schema}.P() RETURNS VARCHAR LANGUAGE SQL AS + $$ + DECLARE + boom EXCEPTION (-20002, 'boom'); + BEGIN + BEGIN TRANSACTION; + INSERT INTO {scratch_schema}.T VALUES (1); + RAISE boom; + EXCEPTION + WHEN OTHER THEN + ROLLBACK; + RAISE; + END; + $$""") + with pytest.raises(ProgrammingError) as excinfo: + snowflake_db.query(conn, f"CALL {scratch_schema}.P()") + assert excinfo.value.errno == -20002 + assert snowflake_db.query(conn, f"SELECT N FROM {scratch_schema}.T") == [] + diff --git a/order-to-cash-cdc/uv.lock b/order-to-cash-cdc/uv.lock new file mode 100644 index 0000000..bed55df --- /dev/null +++ b/order-to-cash-cdc/uv.lock @@ -0,0 +1,323 @@ +version = 1 +revision = 3 +requires-python = "==3.12.*" + +[[package]] +name = "certifi" +version = "2026.10.23" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fd/11/b524cbd9bbdaca58608e4cf4053fd463417549e1959fa8d3aa18319f9555/certifi-2026.10.23.tar.gz", hash = "sha256:a9895e6e2758b371e0545e321a3a8fd1fcfd9829ada8824a936dfb16aaaf36cd", size = 136225, upload-time = "2026-10-23T11:36:16.478Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/d1/a3652fe2c19134d0d15922177821d863cbd7e842d53a21bbe7e8e3a1b5ac/certifi-2026.10.23-py3-none-any.whl", hash = "sha256:e9b54de76e530bb6cf0ccd036787103acb58fd42ba930ea8bf948ddbaa2f61d1", size = 135082, upload-time = "2026-10-23T11:36:14.763Z" }, +] + +[[package]] +name = "cffi" +version = "2.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pycparser", marker = "implementation_name != 'PyPy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9e/ef/008a1939e372c06329a3fce4279c02f328488f3526744906eeec3da7ad5f/cffi-2.1.1.tar.gz", hash = "sha256:dd31f52ea1086513bb9df30f8fcee9b8918323ae067a3d5b78bc826a000712be", size = 530807, upload-time = "2026-08-03T21:21:18.939Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/10/69/43965eccfdead3b9220015fd1320e117be8c6ed01a62ffab76eeb752f5d5/cffi-2.1.1-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:c8c69575568085ba0b1b10c0249d779a214aea6f6522e949a0fc9fb0fcb449d0", size = 184821, upload-time = "2026-08-03T21:19:44.887Z" }, + { url = "https://files.pythonhosted.org/packages/54/7d/16e5a096677b5e313ca80cd5e5170efa3ea44624a82bb111925522da64b1/cffi-2.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f81b3b8f3d4e343550fa4baa0e479bba9f2d29ce9c2e9b51d1ce1718d7442fcf", size = 184719, upload-time = "2026-08-03T21:19:46.129Z" }, + { url = "https://files.pythonhosted.org/packages/56/e6/8941622732edec876dd17d0453dce07317ae96db34f2ec1436c9d3785986/cffi-2.1.1-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:811bd1e21d32de12efca32393a0ab3f5133b54fce9bd44b8bd77ab07da14bf6a", size = 214799, upload-time = "2026-08-03T21:19:47.218Z" }, + { url = "https://files.pythonhosted.org/packages/44/de/f98430906df1545ffde0d543dd124a7a439bc2cd32b36b9c53f805df7333/cffi-2.1.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:68e62fe11f30d5ca8289242866f0a5291402d8529ca2178ab8afc5c9694ae890", size = 222389, upload-time = "2026-08-03T21:19:48.331Z" }, + { url = "https://files.pythonhosted.org/packages/6a/5b/717f1526b9957b34456313c31645c5b82b8fb5c3fe9e4752999be7128bfc/cffi-2.1.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:4a7c934f7360e8cd64fe9efadcbd10c7c6364f531e432b9a4bf5ccbc9e0e8b50", size = 210249, upload-time = "2026-08-03T21:19:49.543Z" }, + { url = "https://files.pythonhosted.org/packages/64/b3/f8aa4f3e34986c7e4ec45072d1b1b9dd295b6b18007b45518d79726dd725/cffi-2.1.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:3143d81e29e1e20a9ce10901ec369012947876596f75a222235965f2b7ae832e", size = 208775, upload-time = "2026-08-03T21:19:50.918Z" }, + { url = "https://files.pythonhosted.org/packages/b1/db/dceb9dd5b231e1da801793f8acc9f3c52a7e1afe40bb1aae37e02b0faad5/cffi-2.1.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c1453022f490d2459a11819d83ad1d586e9ff65a12ac3e705ffebd46d3685dcf", size = 221822, upload-time = "2026-08-03T21:19:52.054Z" }, + { url = "https://files.pythonhosted.org/packages/a0/d2/6cd24ae3be000a634109c247d1475d62e5616d0dc78c82770942ec384248/cffi-2.1.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:208f941bb9d18e768138677f0a6d2ce01f590df56043dda1df1535ac57c88517", size = 225232, upload-time = "2026-08-03T21:19:53.109Z" }, + { url = "https://files.pythonhosted.org/packages/cb/52/3fa190537004dd7f0ab860a6dc7c0175b8667f68d1e618a46f5498d30250/cffi-2.1.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:210019b6c7cf07f081b4c54635c8cf744377001350e29cc0f81c4377b4797735", size = 223597, upload-time = "2026-08-03T21:19:54.515Z" }, + { url = "https://files.pythonhosted.org/packages/80/fb/0bb75b7039588c074b37ae99f40d9bfddf990ecb2fbc346ebccd2e56b9be/cffi-2.1.1-cp312-cp312-win32.whl", hash = "sha256:046bfc24911b37851ee1b51aab8bffe713d89c68c6a057b09484ce9fd5f69b4e", size = 175292, upload-time = "2026-08-03T21:19:55.566Z" }, + { url = "https://files.pythonhosted.org/packages/d9/79/615cc094e2fb508cade7de88d3b4f6c4ec2bab695c97bce9153dc65aadf5/cffi-2.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:f53e442b08449d42821fa4a4fba000095af9f62742a500f978a9f557ec44339a", size = 185919, upload-time = "2026-08-03T21:19:56.89Z" }, + { url = "https://files.pythonhosted.org/packages/70/c6/d0ea84713fe46b243a436a18fcd47d639732747e21635c8a27191b06dc30/cffi-2.1.1-cp312-cp312-win_arm64.whl", hash = "sha256:7bde5e4cc5c10140859842b9d383af292b22639a4dffb725314baf45968cef80", size = 180093, upload-time = "2026-08-03T21:19:58.155Z" }, +] + +[[package]] +name = "charset-normalizer" +version = "3.5.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/33/1c/f41d4e74c28ab327ff3acd36053f7ea506c55872d7a90b0fa71aa3ab0c89/charset_normalizer-3.5.2.tar.gz", hash = "sha256:39de2a259fc954455c57274dc94c79d5842774e1247a016aff30bc0efed0f4ef", size = 172659, upload-time = "2026-09-30T04:39:23.398Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e7/c8/693809898870237d82785a03f3b2b58fe4c9f14669f84a7d4e623c92a59e/charset_normalizer-3.5.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:ed2a239c0ea213acc1908150a3037257083c7c083128f1a4cec2ec4b97dca491", size = 367780, upload-time = "2026-09-30T04:35:30.888Z" }, + { url = "https://files.pythonhosted.org/packages/c9/87/2fea8c13dc24b3ca9c6f803a5b2dfdeae73eb4f9e12c7885ed908ff0433c/charset_normalizer-3.5.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b91363207bd9dc966a691e959bb47f64b30f7ac4b072be9968b366982f7db77c", size = 246730, upload-time = "2026-09-30T04:35:32.286Z" }, + { url = "https://files.pythonhosted.org/packages/a8/9e/09efac30b937722f46d3110ba30b875b24b2e3a266ed746cc4e376a94d80/charset_normalizer-3.5.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:38a873987f3be698494da8b2e3085e29da02da7b633dce73e79c699a113d7bf0", size = 237707, upload-time = "2026-09-30T04:35:33.709Z" }, + { url = "https://files.pythonhosted.org/packages/9e/18/70d76670b13686237863a379928d60bd10e021f17d243ab3d7014c4a5f4e/charset_normalizer-3.5.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:355ad8011081dec5412240c087a9a0c9d4d5039f3ed11a3f13e18c2b29b56c51", size = 273050, upload-time = "2026-09-30T04:35:35.138Z" }, + { url = "https://files.pythonhosted.org/packages/54/e2/77a8b09d5adc013ed07b95b01b8b8fa5441c4e810e83ee7e4aae2fa4d91a/charset_normalizer-3.5.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ee21e28f0430bd6dc9086c6e525d5e818a44a5ad19720c8a0ef766792f3eb5e5", size = 270345, upload-time = "2026-09-30T04:35:36.502Z" }, + { url = "https://files.pythonhosted.org/packages/7f/c5/38806a25ab5e65fc178f39affeda20858efafede2fce1ffc2556cfc9fe73/charset_normalizer-3.5.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3d31298449090ab8d47b7b1b2a555ff73cac7ed438a08b7ac160980c7ebed649", size = 257601, upload-time = "2026-09-30T04:35:37.919Z" }, + { url = "https://files.pythonhosted.org/packages/ae/8d/213565184708fdb263ae55e2c04ee1ff748129dd65d48ed0e3502da9c85a/charset_normalizer-3.5.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5cde776b7cc66e4f6c99612cea4aa7269aa65863f7a15841b2c264f103822f4e", size = 252222, upload-time = "2026-09-30T04:35:39.544Z" }, + { url = "https://files.pythonhosted.org/packages/7e/24/76d2cefc25472531e4c5c7dfff68865eb1c39b78482f0fdc15b46f047830/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ae4f5fea5b8b8ccff88238cc8569303e5ee95efae67fa62922a311397a71f346", size = 248482, upload-time = "2026-09-30T04:35:41.088Z" }, + { url = "https://files.pythonhosted.org/packages/7d/dc/65a801b66ab4c197e22c433ab25e7ac24324ac6f45a2269aca42cce309bf/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:f7d486c83842422badd511868fd8a9a20e9407ace71564b6af47ce7e60a336c1", size = 241206, upload-time = "2026-09-30T04:35:42.59Z" }, + { url = "https://files.pythonhosted.org/packages/a7/95/ca9b5eabde673002c6f1e7ada1b223916fe18f6d661da7aabd4d643718f1/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:11a4d68a6ecda3292cb1e50239e111543ba5d709bb62a6b4ea1afcfa729d8875", size = 273190, upload-time = "2026-09-30T04:35:44.347Z" }, + { url = "https://files.pythonhosted.org/packages/2d/8b/803b4d2a3f6e1740f63f1e87b04d14b42f3d4fdfe6ed7d4db2d34102b14f/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:d6734d2ef8a50fbf8445c139477da401f50d62a0606bf00e20ec6d87773fefb1", size = 253527, upload-time = "2026-09-30T04:35:45.915Z" }, + { url = "https://files.pythonhosted.org/packages/a9/55/93c0e5dbd085ae0471346026abbe7e0db9ea2d6fea74e51f0b5a46f233a7/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:a815775b6c38d4e0ff7bcffbeba67feded90202bb6a226b8dd35f1c855217413", size = 271285, upload-time = "2026-09-30T04:35:47.49Z" }, + { url = "https://files.pythonhosted.org/packages/95/69/0dbd0e0b9b16cfa816cdfcb3e2e3854a1f680dc07fb1245ea125e7448060/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:23851fb4e1b85ed3f6c2a27b777cdfe2e19fb5b38429a8faf38c7542b7665869", size = 260010, upload-time = "2026-09-30T04:35:48.996Z" }, + { url = "https://files.pythonhosted.org/packages/58/9d/e7b88e7b1bf403590c3b573277b5e1e488c68c7a6fbacca310a2c324e90c/charset_normalizer-3.5.2-cp312-cp312-win32.whl", hash = "sha256:db19d07e2e0129e974a0e65d0064fc222a446cd5122c2fd4184d2af9fc734a9e", size = 184125, upload-time = "2026-09-30T04:35:50.777Z" }, + { url = "https://files.pythonhosted.org/packages/eb/e6/e6e083884cbcfd49c64865af05027fe7011be7b2d9179524f099a1b611f3/charset_normalizer-3.5.2-cp312-cp312-win_amd64.whl", hash = "sha256:780fbe7cab297b81dad9fb8dc5eb003c0468ffb0d9e5f65068c53a34661a96bc", size = 207486, upload-time = "2026-09-30T04:35:52.194Z" }, + { url = "https://files.pythonhosted.org/packages/c4/e3/017aea0911ada7405a825c7d937eb3a13009664e2f5b38e8c4bbf2abf894/charset_normalizer-3.5.2-cp312-cp312-win_arm64.whl", hash = "sha256:e2af3aad578aa6bd1384bcf4750fc285e5a9de53f40b7d41e5a0bf748edeb2b3", size = 196734, upload-time = "2026-09-30T04:35:53.636Z" }, + { url = "https://files.pythonhosted.org/packages/8c/ab/176fbfd5b64939c55d652366aa5b9ef1d767af207a3aa6ebeb0d226c484d/charset_normalizer-3.5.2-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:4275811936e2f06feff5e598fb42a1b7ae852da8e39605211892b56b81a34efd", size = 331815, upload-time = "2026-09-30T04:38:26.216Z" }, + { url = "https://files.pythonhosted.org/packages/7e/84/371eac6b30bdbcbf2d632a1a01809103459216fcaae61b8b8d922c1bfb8a/charset_normalizer-3.5.2-cp37-abi3-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:1c50fe28bbc2ced33386f298650d91218076c05420e6cbd790b913adc41659e7", size = 253276, upload-time = "2026-09-30T04:38:28.032Z" }, + { url = "https://files.pythonhosted.org/packages/43/6f/c4fbae58febff71709c51bc7e18fdfa55341dc382704740f9f0cbf03817b/charset_normalizer-3.5.2-cp37-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d19fbd981a488e22cd04883659ca6b08f50b5974f9fd7c95655ef6a043e5893f", size = 241239, upload-time = "2026-09-30T04:38:29.732Z" }, + { url = "https://files.pythonhosted.org/packages/61/71/458c3f42164a07d0c5210798e9e704b39e540a6793b05aba67f3a35243a9/charset_normalizer-3.5.2-cp37-abi3-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0fed1d06615f022ee3b13caf5e8b180cfea32bb2c5aded8a9d44277afc040f93", size = 231121, upload-time = "2026-09-30T04:38:31.462Z" }, + { url = "https://files.pythonhosted.org/packages/09/54/ab9e89367076f6331bb6c65c4bf14a5361fa5191cb6561bf534f18504e1b/charset_normalizer-3.5.2-cp37-abi3-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:838dcc90063569a0448120554591a1d6c4a4ffe11babf048908793154ab86ade", size = 260350, upload-time = "2026-09-30T04:38:33.239Z" }, + { url = "https://files.pythonhosted.org/packages/7c/c1/061431ecc688d9d76602502cb57cc01e691e682c18f1beb45f9673b5bbd2/charset_normalizer-3.5.2-cp37-abi3-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2ce45c6627b22c47e390bc91a41c3d13032192e699fa0bea96e9671b373d69b0", size = 255430, upload-time = "2026-09-30T04:38:34.865Z" }, + { url = "https://files.pythonhosted.org/packages/8d/1f/20c8949f0676f7ab811abdeb7f4d7f1cbc6e61ff20bef08b44edeb092bc8/charset_normalizer-3.5.2-cp37-abi3-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0774bf9bf620249fee3e0b8b9fd3065de213be30f3aa94ce2494b3b638949e26", size = 250612, upload-time = "2026-09-30T04:38:36.649Z" }, + { url = "https://files.pythonhosted.org/packages/2b/9e/46f2fa4c431fc98c4ae76a8cb5bdca54e0341e3cfc3fcfd8e82740250818/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1db38f4c5496827c1a501846d64d14c3b80c7e6714e406cd7dc36a9899fa1011", size = 242083, upload-time = "2026-09-30T04:38:38.26Z" }, + { url = "https://files.pythonhosted.org/packages/bd/39/559be29a0c0f086e0bba6922babd38916cc5e0b58ced4de13ee01ea05508/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:304d8e4d493af723536393eee0c689eb7813f4a474c8b479dee63f1fdd98f621", size = 232738, upload-time = "2026-09-30T04:38:39.81Z" }, + { url = "https://files.pythonhosted.org/packages/ff/6c/387b0e4f756a282831c1d9fc6aeb6c51ca4507ca202767c8de15ce9b12e2/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_ppc64le.whl", hash = "sha256:9b7f416ff0978e2f2249330527f0ad6fa02f4932e6199692d3b52da2048c19e4", size = 260703, upload-time = "2026-09-30T04:38:41.346Z" }, + { url = "https://files.pythonhosted.org/packages/96/92/1fdf015f09ef449f50d3ac4b67c90887c9c318b727daa95cc4f866e6521d/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:01077390b03f7988f11d700a2194e69b119741a86b1a638b1db88891e3eced8e", size = 247622, upload-time = "2026-09-30T04:38:42.937Z" }, + { url = "https://files.pythonhosted.org/packages/dc/3c/8e7b8a5671ad5d433669fb2a76f1a0164df2d9b1718b0206bc2a16d840cc/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_s390x.whl", hash = "sha256:7e841fb9010836c992c9f12fcbd43a831de93a5f726fc1ccd8ca1d0268c5014c", size = 257500, upload-time = "2026-09-30T04:38:44.604Z" }, + { url = "https://files.pythonhosted.org/packages/b4/f0/45b579df5cabc1d5d53ea1cc35e8437d3ca768c0acccc7041517cb6fbb32/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:9cae88599c7219005d879f98e5ed53341e9a122af585e1091200358a3003d2a0", size = 255100, upload-time = "2026-09-30T04:38:46.289Z" }, + { url = "https://files.pythonhosted.org/packages/31/68/fdec18a343f5fb3f310588dd478b09ac4799e0b187dbade3a8cd776f03ef/charset_normalizer-3.5.2-cp37-abi3-win32.whl", hash = "sha256:01b0c0d2262a9e28e8484a278c7e1b5d650e3ac8cf2683d2967e25899f208bdf", size = 174499, upload-time = "2026-09-30T04:38:47.999Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8a/b618149cc5207943a0242068d7a27897f56a62947b5a039085f2a22029f8/charset_normalizer-3.5.2-cp37-abi3-win_amd64.whl", hash = "sha256:9f56f72050826f63dcee7a7f55b0a77168cb3bfc553fd405e7f8f9ece75a4036", size = 200092, upload-time = "2026-09-30T04:38:49.707Z" }, + { url = "https://files.pythonhosted.org/packages/03/cf/4c66866fa9e2b1c78e3c911516d1de497a677b7ac60f1eceda74ce777ca3/charset_normalizer-3.5.2-cp37-abi3-win_arm64.whl", hash = "sha256:40ab6bffa02ae10a0581e6c198be7d2d8ca5c2a0c64e4ed3465d766df457573e", size = 294363, upload-time = "2026-09-30T04:38:51.312Z" }, + { url = "https://files.pythonhosted.org/packages/fc/ad/d07d7862a62ffa6d79d68074d14823243dd235a77c45262acbf6adeb28bf/charset_normalizer-3.5.2-py3-none-any.whl", hash = "sha256:b6b751274acb69d77b3323d6b7dbaa3c7fdfc1eb829b7eb61d262f32e1af9685", size = 68872, upload-time = "2026-09-30T04:39:21.828Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "confluent-kafka" +version = "2.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/b4/28/ef5544a6c1120b5e5da5098ec93238a8f753b01a701351e3fc83ba72e1d2/confluent_kafka-2.16.0.tar.gz", hash = "sha256:8268b8763a0c0503a99a55a9cac0132ed010932135d4222f67e2c804d1597508", size = 453625, upload-time = "2026-10-07T09:13:50.46Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ee/13/c411fb55d0c59e1ed1bf87ce4fde0185ef0e539fccf85a83afcb7e7bf5d0/confluent_kafka-2.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:52bbb9e5352d1db6a4fc9132d831b6ae34c7a2cb2c38a4ce6b464ae3268b6f6a", size = 4533086, upload-time = "2026-10-07T09:12:49.702Z" }, + { url = "https://files.pythonhosted.org/packages/96/b4/71c76cc556c95f5d0b86e5add0150cd9014051263afbf5bbae90df61aec3/confluent_kafka-2.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d727998de5fdc305be99e5d32ffe1e66abaad4fba8588634f81519052aa0df31", size = 4502124, upload-time = "2026-10-07T09:12:51.115Z" }, + { url = "https://files.pythonhosted.org/packages/49/6b/8d1c4dac153fbfd5c00a86c1301a0c2f3a37618ce7b68bf224690e015cfc/confluent_kafka-2.16.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:0eabaccf63c08791db84d00e0ed800b9429a4765c0fa9cf462c3c64bc354a4b3", size = 5150989, upload-time = "2026-10-07T09:12:52.674Z" }, + { url = "https://files.pythonhosted.org/packages/19/d2/c8779c9f985883a6ac1308ac815a40066b02372750d226cff037cd90b878/confluent_kafka-2.16.0-cp312-cp312-manylinux_2_28_s390x.whl", hash = "sha256:25226a4c3f8529cb86e057feab497edfedab9cee1f2f902e31fe0fc7e526be29", size = 6103877, upload-time = "2026-10-07T09:12:54.24Z" }, + { url = "https://files.pythonhosted.org/packages/f2/02/972fb6e1c987fc5edd09bd3d9510797a69369aa4e1a73ac0880b0b7f684f/confluent_kafka-2.16.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:5b3adb61cfbde5eab27e0a46bdda6913ed70fb5bb716e7f78b8bf664e10781da", size = 4962560, upload-time = "2026-10-07T09:12:55.601Z" }, + { url = "https://files.pythonhosted.org/packages/1e/3a/f0f0fd0b9460133e9e89afa1d9d91cbffbbf07e12d19c71d8ec9347e284b/confluent_kafka-2.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:abb386d796aa6cfd0276787b1e8570af82ee293cb77a8cbbb9b0f88d20f99eeb", size = 4710241, upload-time = "2026-10-07T09:12:57.437Z" }, +] + +[[package]] +name = "cryptography" +version = "50.0.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9d/af/182eb91b0df3fe75c4d9f26fe70684569566745f6ba7e5c9c73a862c5252/cryptography-50.0.2.tar.gz", hash = "sha256:7b46165bb56eb4704e2eaaf86f3c940d19154535d9b0ca7d6d590b04060e00d5", size = 880623, upload-time = "2026-09-30T15:30:04.884Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/56/d194340cc4a57535e82e1bee9e89667ac4b7c13b5d3f59686deae3094dd5/cryptography-50.0.2-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:fa8f5efb344d6908a1ce62f4a24e2e5780f825d6f53f5f50ec5ffacac72936cb", size = 3914904, upload-time = "2026-09-30T14:43:44.339Z" }, + { url = "https://files.pythonhosted.org/packages/d9/69/c9bd862c3bf43d6399c433caf002df16e2dffd4be49bdf515cda38038711/cryptography-50.0.2-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:79def8d059362e7831389ed3be0ecdf58a89386e1271e35dd9f5af84e81bffd0", size = 4731146, upload-time = "2026-09-30T14:43:47.113Z" }, + { url = "https://files.pythonhosted.org/packages/21/69/64cef1f702bf6657e0cc186ed1a2891d50d29fb41586b254e1c07adea261/cryptography-50.0.2-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:630ebfea3bf689d075f82316324ff7433dc447fe6bc1bfc76524b74b4a9567d2", size = 4719841, upload-time = "2026-09-30T14:43:49.01Z" }, + { url = "https://files.pythonhosted.org/packages/38/6b/61a3f8d8c5e1e49a6cddccafc4015cc1c0021360ab0acb4080e7a423644a/cryptography-50.0.2-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:f9f6143a8c75945eb960d9eb98905a441394abfa24afaae239d514ffb2586480", size = 4738340, upload-time = "2026-09-30T14:43:50.932Z" }, + { url = "https://files.pythonhosted.org/packages/7b/2e/7212ca32fd43dc91f2f41db20160b268098874b4c9a0e7be94d6835f5b2e/cryptography-50.0.2-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:a582ab2ae1d34f67112cadc86702774c9ea4374df6bca6afe672817203c99134", size = 5367029, upload-time = "2026-09-30T14:43:52.911Z" }, + { url = "https://files.pythonhosted.org/packages/1a/f1/b474e930c4d910328780e3940da76f5aa5cbc48ce1fc14e44d239d9ea9db/cryptography-50.0.2-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:4061c0079120205fb760c58acab6443e217307dcf05e3702cf970e0689972856", size = 4753050, upload-time = "2026-09-30T14:43:55.272Z" }, + { url = "https://files.pythonhosted.org/packages/7c/52/9af10e80ac16b0fcc2123f9cbd5e7afbd0fd5075bb7a607c592258a39cda/cryptography-50.0.2-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:ac9ed99d81760c62fe89d5f0815cdfa1ba9a35141cf30f1c2d044f04b4803d2e", size = 4376724, upload-time = "2026-09-30T14:43:57.24Z" }, + { url = "https://files.pythonhosted.org/packages/71/37/6202e488cc1eb625ea110c292c6bda92823176e023f427d8d5660ce8d632/cryptography-50.0.2-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:87e9ce85beb6b328ba370cc6e6aea483c92617b4c95b1d33a49297eb662bfb04", size = 4737859, upload-time = "2026-09-30T14:43:59.541Z" }, + { url = "https://files.pythonhosted.org/packages/8f/30/e86d7d518489b0ae2497091a35287abcb1a2ce4037837a34afbe9b1d6964/cryptography-50.0.2-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:f265528741e048bce55c3463ed721fb0aa45a5888d8add8cfeccb3035451bbdc", size = 5324103, upload-time = "2026-09-30T14:44:01.901Z" }, + { url = "https://files.pythonhosted.org/packages/d3/69/2c833a049475e0a3444e94c7d0aca0aa51d166374a449b09e92ac98138de/cryptography-50.0.2-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:9dab55f57c74c3cad24c323bacbbd04be4705ba6eb0d92e920b1fc4837ed5079", size = 4752576, upload-time = "2026-09-30T14:44:04.545Z" }, + { url = "https://files.pythonhosted.org/packages/6c/5d/906970b83bbfc1f5bbfb677a143c181f2801f23b6a7204a3b47c42c97e65/cryptography-50.0.2-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:25784ce8b9621c90c643efb9e1e2162ab3b0224cae446ad5e70e7fcb1ce18b51", size = 4870819, upload-time = "2026-09-30T14:44:06.884Z" }, + { url = "https://files.pythonhosted.org/packages/68/e3/f2298d3bb55e0c4a91841ec4d01b3f020ba8c5fbf15ccdcc6dcf03f97025/cryptography-50.0.2-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:85d0d9a31b9098e98534226d5686b47264b95e62ce459dc2e62fdfc809f9fe93", size = 5030152, upload-time = "2026-09-30T14:44:09.443Z" }, + { url = "https://files.pythonhosted.org/packages/9a/4f/adfc442765721292fff86d314ce385d3249d22db42295c0dd057727b60f3/cryptography-50.0.2-cp311-abi3-win_amd64.whl", hash = "sha256:7afa5a6602a9f29af1f3a2965f831bae7c9d5d597b7cbb716d41ab3b7d89879c", size = 3824692, upload-time = "2026-09-30T14:44:11.671Z" }, + { url = "https://files.pythonhosted.org/packages/2d/49/93f6a6e7a87c9aa68d44d3e1cdb5fe8f60c90d5d2f46acae9a56892816b8/cryptography-50.0.2-cp315-abi3.abi3t-macosx_11_0_arm64.whl", hash = "sha256:edc3342adf8f697fc5f59c887a304356f147b397809440ed64e2fa6af2f50f37", size = 4133708, upload-time = "2026-09-30T14:44:41.807Z" }, + { url = "https://files.pythonhosted.org/packages/8c/75/32ac2a56243d778805c16ca6a32b8f74fb757df7e28d7ecb560afafb59cf/cryptography-50.0.2-cp315-abi3.abi3t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d370b8d1dfcdf7130178137f6fbee6140774a1acc6cacefc4b42643ec11d0a3a", size = 4956267, upload-time = "2026-09-30T14:44:43.693Z" }, + { url = "https://files.pythonhosted.org/packages/aa/a4/2c8d734e43d97f0842ee9f1b7b4bfb3d0cf5e19edebf43c2afe6675c2320/cryptography-50.0.2-cp315-abi3.abi3t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f2f9bd7f90c64fe89253f0a2c05e3c4856072660429ce8831b4235bf29403a67", size = 4966465, upload-time = "2026-09-30T14:44:45.769Z" }, + { url = "https://files.pythonhosted.org/packages/c2/58/ee288c829a6f41f6235ae9dd33d82fd19b45442b65b4c8a3da36963d9f7a/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_28_aarch64.whl", hash = "sha256:e275096ea1e60cc595cda2836fd4a6c725d1125108b868be17f53684d164e2cc", size = 4959356, upload-time = "2026-09-30T14:44:48.211Z" }, + { url = "https://files.pythonhosted.org/packages/92/20/9ded6d51ddd9897f6b6e81fb9ebea7951d7cc5d6c890b0ed8abf77a51a80/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_28_ppc64le.whl", hash = "sha256:b13478603dcd0a2479ff8e87e2c19a7d525734686fe3c49542472293a204212d", size = 5548822, upload-time = "2026-09-30T14:44:50.86Z" }, + { url = "https://files.pythonhosted.org/packages/02/a8/8df951850d6b31d2a00218f19e2b3f999523437ed7a819df7fa427942fca/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_28_x86_64.whl", hash = "sha256:58a0c478eeca76fe5e07993c5a0703def34a6dc6a0cda4f5564639b33112ffe7", size = 5001199, upload-time = "2026-09-30T14:44:53.379Z" }, + { url = "https://files.pythonhosted.org/packages/8b/f9/36b3022218ce75b7cdf068fb95f809f9bd0d820e4955ef43b90c255cc7ac/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_31_armv7l.whl", hash = "sha256:d38cdff612d06fa6a32840d5e1b1f7a27cee4a349aa9085d94a67789d6bfd408", size = 4629333, upload-time = "2026-09-30T14:44:55.635Z" }, + { url = "https://files.pythonhosted.org/packages/8c/72/20f99a219f6af47cdd1cbd978c243b92d71496e168a746138af44ded4f29/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_34_aarch64.whl", hash = "sha256:fdd28f912fccfec1846a94e2e1e8f9b0012f557f0c46fe4f3eb0d7a87afcf90b", size = 4958822, upload-time = "2026-09-30T14:44:59.639Z" }, + { url = "https://files.pythonhosted.org/packages/f2/20/196f112617fb08eb4d608a2a6c422373d46f9cc2857f38fc0667033c0899/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_34_ppc64le.whl", hash = "sha256:cbc8738fd8526d80f35cb3a40d41f41a2e7030bb3b18b09a6778ef63d291c2fd", size = 5506351, upload-time = "2026-09-30T14:45:02.267Z" }, + { url = "https://files.pythonhosted.org/packages/24/95/83378121ef3eaaaf71d4b781577ff794acb39b9e1b87a3f156898c8497ed/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_34_x86_64.whl", hash = "sha256:e105ab60406787da31fccc883fc0f733af1efd78f0136a4599692c4083a73d0c", size = 5000859, upload-time = "2026-09-30T14:45:05.009Z" }, + { url = "https://files.pythonhosted.org/packages/22/f7/70fd7ae4d1dbfa7ba29b02e1b9068771519a86027756510b700ce81086a8/cryptography-50.0.2-cp315-abi3.abi3t-musllinux_1_2_aarch64.whl", hash = "sha256:6f8700550aa1474a91e5dc07049c46f98b423b5b1ddd0483e0b51362eeeaf5be", size = 5092151, upload-time = "2026-09-30T15:29:15.932Z" }, + { url = "https://files.pythonhosted.org/packages/d4/be/688367b74de86984bd58d8efacfc7c9e68b89a6a22ced0fb4f38db50254a/cryptography-50.0.2-cp315-abi3.abi3t-musllinux_1_2_x86_64.whl", hash = "sha256:c71be1cbfa5cd9a41ee452acf1eccd82b2c05950358b106ec8ceb83411d1a020", size = 5286120, upload-time = "2026-09-30T15:29:18.309Z" }, + { url = "https://files.pythonhosted.org/packages/39/d1/55f8a3f2ef5d1529e16835ef10cf0fe3d559ce237b46dddc440c0bba3649/cryptography-50.0.2-cp315-abi3.abi3t-win_amd64.whl", hash = "sha256:c423ab384a46c4dff7217b2ea5ba2e11cffdeab6441acd04cf65a369caf0366c", size = 4111557, upload-time = "2026-09-30T15:29:20.155Z" }, + { url = "https://files.pythonhosted.org/packages/23/ad/ac987755d00e1e64273760228d2635ae38dae2be83e3c6e0d3289d91dec3/cryptography-50.0.2-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:0ec5f09541743261e66e291b4a0cbf0fb2997aeaab6d9e9c740b9dba1b58d1c2", size = 3943588, upload-time = "2026-09-30T15:29:22.265Z" }, + { url = "https://files.pythonhosted.org/packages/d5/8d/6d585339bedf85d45044c85d8412dac53f2bb6f918e8b7777efba1787844/cryptography-50.0.2-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c5e67125c7dca78d199ec4e116aa93dbb83494808ecbb8211a2cb09b1bf41dbd", size = 4756166, upload-time = "2026-09-30T15:29:24.58Z" }, + { url = "https://files.pythonhosted.org/packages/bf/f1/1c1f6874e8550cfddd4b688ceb38cefb6ed15ceed224d56f133f3d88c214/cryptography-50.0.2-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ee247f5c245c9a2fe7c8e2214e295918838e44e00a45a6718451e4004219e767", size = 4749145, upload-time = "2026-09-30T15:29:26.807Z" }, + { url = "https://files.pythonhosted.org/packages/c1/63/61b15dc1a8de03fe0adbe3fd7608b3ad5c73bf50993bbcb1faaa930afe33/cryptography-50.0.2-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:dfe9763530994147d9af1def057a5b9658b00e8f8fe8743d144d1e0911c2e454", size = 4763638, upload-time = "2026-09-30T15:29:28.588Z" }, + { url = "https://files.pythonhosted.org/packages/fc/35/b345bdfa40c9126df1a9d33236aa98418367931b8725f84fc3ae2b98dc59/cryptography-50.0.2-cp39-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:58ddb5a8e3179d12f19e4ea34d2d32e9d63a4baa142c875c1eb59f41b7243acd", size = 5382217, upload-time = "2026-09-30T15:29:30.589Z" }, + { url = "https://files.pythonhosted.org/packages/4f/87/ef344a9e616871f2519c22d6afcda79ddd5d35e9592d95eb6e677608d055/cryptography-50.0.2-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:f21e8a22c8605750c7af886bab299a363721264061b4ac0a30efb73cfd58efc5", size = 4781387, upload-time = "2026-09-30T15:29:32.605Z" }, + { url = "https://files.pythonhosted.org/packages/90/5b/f2fdb13cd0b96f6f932c8627bb292a45f11c64d21620a8e120aee9a3b848/cryptography-50.0.2-cp39-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:9c8402a82ea0dc4ceeab793db05f0fafa8ca139ca34fcde5df0f596103c74107", size = 4403790, upload-time = "2026-09-30T15:29:34.374Z" }, + { url = "https://files.pythonhosted.org/packages/bc/ce/7e4f662b1e3c393513569e402cfc85ac7da0bd3d5435e122a3140219eb2d/cryptography-50.0.2-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:0ddc924c04591c2811ca024d62ecad4f7f6f08af8939c211438f48a16bd23602", size = 4764319, upload-time = "2026-09-30T15:29:36.149Z" }, + { url = "https://files.pythonhosted.org/packages/3c/3f/86ff33ce34cc0de6847fb96e035a1a760d81652e38643f617c02ad32ef7a/cryptography-50.0.2-cp39-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:a6557e5f38e065ca9fbdaf7cfc7435ecb1d113aa81a022d1b51921ee7432e227", size = 5338560, upload-time = "2026-09-30T15:29:39.053Z" }, + { url = "https://files.pythonhosted.org/packages/40/cf/6b5c8e2fd9202d98988ab7cb5cc5c991704c4ad55f492ff408e4969f83f1/cryptography-50.0.2-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:1981f1db4630889b9ef7803fadef12b056f428cb6b85c27ba57b774793b6093c", size = 4780973, upload-time = "2026-09-30T15:29:41.251Z" }, + { url = "https://files.pythonhosted.org/packages/10/bf/8d6ebc7dded797bd0f0160d52188021211f011a2b164ef0ae1dac4587465/cryptography-50.0.2-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:7a8701d6b584d76e909e3d305b7d126b41439876a5aaf76cddc67fc230eafa2e", size = 4897738, upload-time = "2026-09-30T15:29:43.106Z" }, + { url = "https://files.pythonhosted.org/packages/d4/aa/f3f6e0de7e6253b8baa8b2d8fb9d50924fa75cee3d4624bd4bc1208ee923/cryptography-50.0.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:ce47f66801c20ec6c6632453bb5960fe38939e9306970b48b3a5a26de7745d94", size = 5058280, upload-time = "2026-09-30T15:29:44.827Z" }, + { url = "https://files.pythonhosted.org/packages/f6/b6/a1faf3a27ae9405fb34b1713cc73b2d8a26b04d5c561578fa2e6ef3e5bb9/cryptography-50.0.2-cp39-abi3-win_amd64.whl", hash = "sha256:4e81d95e5bafc2d6e34e4bed780e53e4d5b9a2f928573428aa4d35fbec1eb0de", size = 3854095, upload-time = "2026-09-30T15:29:46.782Z" }, +] + +[[package]] +name = "idna" +version = "3.20" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f5/08/8eea9d4b8302028f3abb2c0813953f7aec26d33b7a8960ed760e65ff29fa/idna-3.20.tar.gz", hash = "sha256:a7db850025b95ded1eae8a46181a1a6c56c92c96f0e2b005d9ff8dc0210cab44", size = 216463, upload-time = "2026-09-17T14:11:04.752Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/58/a2/bb081bab032533a855d44de1d56f8e8426114ff1ba5d1f07a438a0a654f8/idna-3.20-py3-none-any.whl", hash = "sha256:ab7ae7122974553370f0bdb919e1a960b2cd1bc1ef0276416d896db81c14582c", size = 69583, upload-time = "2026-09-17T14:11:03.168Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/e1/2069291243c926a2ff1cd706c7f3eeb9b62144bf60f77c9fb9ff2fb26bd3/iniconfig-2.3.1.tar.gz", hash = "sha256:67f4b9c50da0dedf52af349e7749a80a9057a5031199791b906c3bb3ae878960", size = 21209, upload-time = "2026-10-06T22:48:38.076Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/56/43/4ca9e49d27a1fcf6bece6f6aec0ea46bb9112489b93d4b688fb415457bdb/iniconfig-2.3.1-py3-none-any.whl", hash = "sha256:9121e2c1fdb355232495be3194c8dfe87ccc2d5dee45947b78e68f499790d7a7", size = 7552, upload-time = "2026-10-06T22:48:36.959Z" }, +] + +[[package]] +name = "order-to-cash-cdc" +version = "0.1.0" +source = { virtual = "." } +dependencies = [ + { name = "confluent-kafka" }, + { name = "cryptography" }, + { name = "requests" }, + { name = "snowflake-connector-python" }, +] + +[package.dev-dependencies] +dev = [ + { name = "pytest" }, +] + +[package.metadata] +requires-dist = [ + { name = "confluent-kafka", specifier = ">=2.6,<3" }, + { name = "cryptography", specifier = ">=43" }, + { name = "requests", specifier = ">=2.32,<3" }, + { name = "snowflake-connector-python", specifier = ">=5.1,<6" }, +] + +[package.metadata.requires-dev] +dev = [{ name = "pytest", specifier = ">=8.3,<9" }] + +[[package]] +name = "packaging" +version = "26.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/fa/3944b40b07da9ce895c0e6303a5ab7d53da063554f534556b134a54d6093/packaging-26.3.tar.gz", hash = "sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79", size = 313412, upload-time = "2026-08-04T18:15:28.737Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", hash = "sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", size = 129956, upload-time = "2026-08-04T18:15:27.159Z" }, +] + +[[package]] +name = "pluggy" +version = "1.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bf/db/7fc19e6f2dc92a966727031389fc2e08b558f0f25eb7403c1119ad4713cd/pluggy-1.7.0.tar.gz", hash = "sha256:d1eaa46ebb595891b860ab086b4d09c8588af65ebd4361b8e8f4bb8920b90ba8", size = 123304, upload-time = "2026-10-15T09:50:58.343Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/40/9e/2b38731e0fc536806f16490e1a12d7f0dc2a1235aa8cc07bcc75416a7daa/pluggy-1.7.0-py3-none-any.whl", hash = "sha256:7dd7b0d8832ba3cb632c306926ded123429211b83641b35dc5c41ad2d34f9bec", size = 27082, upload-time = "2026-10-15T09:50:56.808Z" }, +] + +[[package]] +name = "protobuf" +version = "6.33.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/66/70/e908e9c5e52ef7c3a6c7902c9dfbb34c7e29c25d2f81ade3856445fd5c94/protobuf-6.33.6.tar.gz", hash = "sha256:a6768d25248312c297558af96a9f9c929e8c4cee0659cb07e780731095f38135", size = 444531, upload-time = "2026-03-18T19:05:00.988Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fc/9f/2f509339e89cfa6f6a4c4ff50438db9ca488dec341f7e454adad60150b00/protobuf-6.33.6-cp310-abi3-win32.whl", hash = "sha256:7d29d9b65f8afef196f8334e80d6bc1d5d4adedb449971fefd3723824e6e77d3", size = 425739, upload-time = "2026-03-18T19:04:48.373Z" }, + { url = "https://files.pythonhosted.org/packages/76/5d/683efcd4798e0030c1bab27374fd13a89f7c2515fb1f3123efdfaa5eab57/protobuf-6.33.6-cp310-abi3-win_amd64.whl", hash = "sha256:0cd27b587afca21b7cfa59a74dcbd48a50f0a6400cfb59391340ad729d91d326", size = 437089, upload-time = "2026-03-18T19:04:50.381Z" }, + { url = "https://files.pythonhosted.org/packages/5c/01/a3c3ed5cd186f39e7880f8303cc51385a198a81469d53d0fdecf1f64d929/protobuf-6.33.6-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:9720e6961b251bde64edfdab7d500725a2af5280f3f4c87e57c0208376aa8c3a", size = 427737, upload-time = "2026-03-18T19:04:51.866Z" }, + { url = "https://files.pythonhosted.org/packages/ee/90/b3c01fdec7d2f627b3a6884243ba328c1217ed2d978def5c12dc50d328a3/protobuf-6.33.6-cp39-abi3-manylinux2014_aarch64.whl", hash = "sha256:e2afbae9b8e1825e3529f88d514754e094278bb95eadc0e199751cdd9a2e82a2", size = 324610, upload-time = "2026-03-18T19:04:53.096Z" }, + { url = "https://files.pythonhosted.org/packages/9b/ca/25afc144934014700c52e05103c2421997482d561f3101ff352e1292fb81/protobuf-6.33.6-cp39-abi3-manylinux2014_s390x.whl", hash = "sha256:c96c37eec15086b79762ed265d59ab204dabc53056e3443e702d2681f4b39ce3", size = 339381, upload-time = "2026-03-18T19:04:54.616Z" }, + { url = "https://files.pythonhosted.org/packages/16/92/d1e32e3e0d894fe00b15ce28ad4944ab692713f2e7f0a99787405e43533a/protobuf-6.33.6-cp39-abi3-manylinux2014_x86_64.whl", hash = "sha256:e9db7e292e0ab79dd108d7f1a94fe31601ce1ee3f7b79e0692043423020b0593", size = 323436, upload-time = "2026-03-18T19:04:55.768Z" }, + { url = "https://files.pythonhosted.org/packages/c4/72/02445137af02769918a93807b2b7890047c32bfb9f90371cbc12688819eb/protobuf-6.33.6-py3-none-any.whl", hash = "sha256:77179e006c476e69bf8e8ce866640091ec42e1beb80b213c3900006ecfba6901", size = 170656, upload-time = "2026-03-18T19:04:59.826Z" }, +] + +[[package]] +name = "pycparser" +version = "3.12" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cb/27/eeb9f307bf07060f3d9f9aa0fb8c15b01bdfded0a941e4b2719c7f94cacd/pycparser-3.12.tar.gz", hash = "sha256:f03387877eab82ce01bbdfd0ba7252ee55a90b1c708d5129e2ce915475cf1db6", size = 158785, upload-time = "2026-10-22T13:40:03.56Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b6/ae/315bbeee126a769cbde43cf4908acb4b40408d8bf8bbf334dd7368d8dadf/pycparser-3.12-py3-none-any.whl", hash = "sha256:a076f3f489b39b43ed13563a1dafe42447445bec2e294154ec5c57fcdda4c85e", size = 62600, upload-time = "2026-10-22T13:40:02.021Z" }, +] + +[[package]] +name = "pygments" +version = "2.21.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/49/2e/ced460408999b33da6b31b0021b0f37d329e202d4169aeb164493778f25b/pygments-2.21.0.tar.gz", hash = "sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c", size = 5005329, upload-time = "2026-08-17T08:02:48.824Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/46/17f022dd3e953bf20a04a028a21ec746d942f8d2af30fa0f124fa0e6a684/pygments-2.21.0-py3-none-any.whl", hash = "sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9", size = 1250147, upload-time = "2026-08-17T08:02:44.912Z" }, +] + +[[package]] +name = "pytest" +version = "8.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a3/5c/00a0e072241553e1a7496d638deababa67c5058571567b92a7eaa258397c/pytest-8.4.2.tar.gz", hash = "sha256:86c0d0b93306b961d58d62a4db4879f27fe25513d4b969df351abdddb3c30e01", size = 1519618, upload-time = "2025-09-04T14:34:22.711Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a8/a4/20da314d277121d6534b3a980b29035dcd51e6744bd79075a6ce8fa4eb8d/pytest-8.4.2-py3-none-any.whl", hash = "sha256:872f880de3fc3a5bdc88a11b39c9710c3497a547cfa9320bc3c5e62fbf272e79", size = 365750, upload-time = "2025-09-04T14:34:20.226Z" }, +] + +[[package]] +name = "pytz" +version = "2026.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/14/21/d83d6ef28c4c912c4bb4d1dcf591f7b8c6bde87b9c66f9f454677314e16d/pytz-2026.5.tar.gz", hash = "sha256:fa23724b9c486543b9ff54a327ee7569ac83ade54bb9afd0fc18676620401c86", size = 318572, upload-time = "2026-10-04T02:37:58.719Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4f/ef/c66110d46fb800dda0bf33164182dfadabe26a90e4476844d502a23dca8e/pytz-2026.5-py2.py3-none-any.whl", hash = "sha256:e658af3757f9e26a9d25dd2aff38335acd92bc9104f890a894b2c1ba28311b03", size = 506342, upload-time = "2026-10-04T02:37:56.814Z" }, +] + +[[package]] +name = "requests" +version = "2.34.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ac/c3/e2a2b89f2d3e2179abd6d00ebd70bff6273f37fb3e0cc209f48b39d00cbf/requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed", size = 142856, upload-time = "2026-05-14T19:25:27.735Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, +] + +[[package]] +name = "snowflake-connector-python" +version = "5.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cryptography" }, + { name = "protobuf" }, + { name = "pytz" }, + { name = "tomlkit" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7a/e4/05727a3af8c240ff6880c0becf729968a34ef691f1bb9f6e0d983bb685e6/snowflake_connector_python-5.1.0.tar.gz", hash = "sha256:2d9eec7fbaa329b4013b9d67e01e2120fe2878854f6de421cc9d07c620fa270d", size = 1950546, upload-time = "2026-10-21T21:33:05.19Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/e7/0519ab883f7073f356b99ad84475011759578949abfaf9ac0325cbd899d7/snowflake_connector_python-5.1.0-cp312-cp312-macosx_10_14_x86_64.whl", hash = "sha256:86fb60cace989dd6d27769a2bfffc0b5e894095776140006e6c57fd4b5065070", size = 11229627, upload-time = "2026-10-21T21:32:09.051Z" }, + { url = "https://files.pythonhosted.org/packages/14/d3/c7bd1b127d2b2fe57d01e1ba29df24ca4a400f2a9fb4da2400185af597e8/snowflake_connector_python-5.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:525eca4af24aed27fed48c8500c2a2c6490b613b3813260a867d4430fc08dcbe", size = 10581308, upload-time = "2026-10-21T21:31:57.59Z" }, + { url = "https://files.pythonhosted.org/packages/29/22/0dc1479ff11662e6e2e65aeab6953c65efeaceb444cb2d95c339f00615c6/snowflake_connector_python-5.1.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:7eb533a075dae3b76b9567ecdcb6c66d57b97156cc4366f349843e49f295746a", size = 14241926, upload-time = "2026-10-21T21:32:33.682Z" }, + { url = "https://files.pythonhosted.org/packages/f4/ac/fb606e7be6c58d01175cf6f7e05b84abeaddccccbc5cde93e846aa2ea7eb/snowflake_connector_python-5.1.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:e899d9d69bc7fb40ab531c3f34049d064dae57fcc4805d0c286aa35570e683de", size = 15249669, upload-time = "2026-10-21T21:32:20.298Z" }, + { url = "https://files.pythonhosted.org/packages/82/75/4d20ce4e4f0704d35ebfbc17fcc62c3a9f5bca2a5345509174590686fdb2/snowflake_connector_python-5.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:ebbdfa1a53548bb308460d75f132cc85f45dc618c16ba450b6e1486d4d08050b", size = 11567901, upload-time = "2026-10-21T21:32:48.965Z" }, + { url = "https://files.pythonhosted.org/packages/e7/37/c609d9599e6831854cca14e00122f586e14a09cacb5d223744563cdc80f2/snowflake_connector_python-5.1.0-cp312-cp312-win_arm64.whl", hash = "sha256:64c15ed531ff03d42c383250ffd87b7a81ff9ba78cb6b4afaaefae36e8109851", size = 10880368, upload-time = "2026-10-21T21:33:02.974Z" }, +] + +[[package]] +name = "tomlkit" +version = "0.15.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/08/72/fc01563df95c189a9f65850096bdc205bc2fe37da6c740cd64afaf18d07e/tomlkit-0.15.2.tar.gz", hash = "sha256:83e32755ba1d20d9340bd069ee97e9f6232ba2346b63ffcba425de01f7e327a6", size = 190522, upload-time = "2026-10-16T04:07:08.166Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/da/46/7d84138a60a8bd796825b2d38ebe92d0d16501ca57d501b7c5d677401bbc/tomlkit-0.15.2-py3-none-any.whl", hash = "sha256:05562412163a3037b94497385fb7f9204c637bc605af7a48199e41cf2272f437", size = 52225, upload-time = "2026-10-16T04:07:09.216Z" }, +] + +[[package]] +name = "urllib3" +version = "2.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e3/05/b17359e1cefb4f909b5e40b1b90a496d987258916dbbf88e842c729f510e/urllib3-2.8.0.tar.gz", hash = "sha256:63bf2ead4c879426ebf22ef2a781eeb4aa3b4ae798a0435506f8687fd5bb9b63", size = 458972, upload-time = "2026-09-15T19:29:36.253Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/92/9d/c4e665119135114480843e7ab388fa94d8480650450e6f8e26b70d323a4c/urllib3-2.8.0-py3-none-any.whl", hash = "sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3", size = 135717, upload-time = "2026-09-15T19:29:34.577Z" }, +] diff --git a/phi-guardrails/.env.example b/phi-guardrails/.env.example new file mode 100644 index 0000000..fd5b369 --- /dev/null +++ b/phi-guardrails/.env.example @@ -0,0 +1,29 @@ +# Every variable this sample reads, with its default. Copy to .env and +# uncomment what you want to change; the Makefile reads .env. + +# LocalStack auth token. Default: from the environment, else ~/.localstack/default.env. +# Never commit it. +# LOCALSTACK_AUTH_TOKEN= + +# Emulator image and host port. +# SNOWFLAKE_IMAGE=localstack/snowflake-next:latest +# SNOWFLAKE_PORT=4566 + +# EMULATOR or SNOWFLAKE_CLOUD. +# TEST_TARGET=EMULATOR + +# Database for make migrate / seed / demo / gate. Tests always deploy their own +# PHI_TEST_ database and drop it; KEEP_TEST_DB=1 keeps it for debugging. +# PHI_DATABASE=PHI +# KEEP_TEST_DB= + +# Key pair used by every connection (generated by make up). +# SNOWFLAKE_PRIVATE_KEY_PATH=.secrets/rsa_key.p8 + +# TEST_TARGET=SNOWFLAKE_CLOUD only. Use a non-production account: the suite +# creates account-level roles (CLINICIAN, ..., FACILITY_101..103) and PERSONA_* +# users, plus a PHI_TEST_ database that it drops afterwards. +# SNOWFLAKE_ACCOUNT= +# SNOWFLAKE_USER= +# SNOWFLAKE_ROLE=ACCOUNTADMIN +# SNOWFLAKE_WAREHOUSE= diff --git a/phi-guardrails/.python-version b/phi-guardrails/.python-version new file mode 100644 index 0000000..e4fba21 --- /dev/null +++ b/phi-guardrails/.python-version @@ -0,0 +1 @@ +3.12 diff --git a/phi-guardrails/Makefile b/phi-guardrails/Makefile new file mode 100644 index 0000000..bd36c6c --- /dev/null +++ b/phi-guardrails/Makefile @@ -0,0 +1,59 @@ +# PHI Guardrails: governance as code against the Snowflake-Next emulator. +# Targets: up, migrate, seed, test, demo, gate, down, test-cloud (see README.md). + +-include $(HOME)/.localstack/default.env +export LOCALSTACK_AUTH_TOKEN +SNOWFLAKE_PORT ?= 4566 +export SNOWFLAKE_PORT +COMPOSE := docker compose -p snfs-phi-guardrails +UV_RUN := uv run --locked + +# Optional local overrides (copy .env.example to .env). +-include .env +export TEST_TARGET SNOWFLAKE_IMAGE PHI_DATABASE KEEP_TEST_DB \ + SNOWFLAKE_ACCOUNT SNOWFLAKE_USER SNOWFLAKE_ROLE SNOWFLAKE_WAREHOUSE SNOWFLAKE_PRIVATE_KEY_PATH + +.PHONY: up migrate seed test demo gate down test-cloud install start stop logs + +up: .secrets/rsa_key.p8 + $(COMPOSE) pull -q + $(COMPOSE) up -d + ./scripts/wait-for-emulator.sh 180 + +# Throwaway, unencrypted key pair for local runs; never reuse it elsewhere. +.secrets/rsa_key.p8: + @mkdir -p .secrets + openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 -out .secrets/rsa_key.p8 + openssl rsa -in .secrets/rsa_key.p8 -pubout -out .secrets/rsa_key.pub + +migrate: ## apply pending schemachange migrations to $(PHI_DATABASE) (default PHI) + $(UV_RUN) python -m phi_guardrails.deploy + +seed: migrate ## download (once), verify and load the Synthea extract; idempotent + $(UV_RUN) python scripts/load_synthea.py + +test: ## the full suite; deploys and drops its own database + $(UV_RUN) pytest + +demo: seed ## the access matrix, a new PHI column, and a caught regression + $(UV_RUN) python -m phi_guardrails.demo + +gate: migrate ## the policy-coverage gate on $(PHI_DATABASE); exits 1 on an unprotected column + $(UV_RUN) python -m phi_guardrails.coverage + +down: + $(COMPOSE) down -v --remove-orphans + +test-cloud: ## same suite against a real account (consumes credits) + TEST_TARGET=SNOWFLAKE_CLOUD $(UV_RUN) pytest + +# The targets the repository CI runs for every sample (see the root README) +install: ## Nothing to install up front: make up and make test fetch the dependencies + @echo "Nothing to install, make up and make test fetch the dependencies" + +start: up ## Same as make up + +stop: down ## Same as make down + +logs: ## Tail the emulator log (init hook results included) + $(COMPOSE) logs --tail=200 snowflake diff --git a/phi-guardrails/README.md b/phi-guardrails/README.md new file mode 100644 index 0000000..77b43f2 --- /dev/null +++ b/phi-guardrails/README.md @@ -0,0 +1,125 @@ +# PHI Guardrails + +Governance as code for a healthcare dataset: every role-by-column access rule is a test generated from a reviewed contract file, and a policy-coverage gate fails the build when a PHI column carries a tag but no masking policy. It is for platform and governance engineers who want RBAC, masking and row access policies tested on every pull request, locally, before they reach production. + +## Architecture + +```mermaid +flowchart LR + contract["access-contract.yml
(reviewed by Compliance)"] + migrations["migrations/
V1.0.0 .. V1.3.0, R__analyst_views"] + sc["schemachange"] + synthea["Synthea CSV extract
(pinned URL + SHA-256)"] + loader["Loader"] + subgraph sf["Snowflake (LocalStack emulator)"] + direction TB + roles["Roles and PERSONA_* users
FACILITY_101..103"] + governance["GOVERNANCE
PHI_TYPE tag, masking policies,
FACILITY_ROW_ACCESS"] + clinical["CLINICAL
FACILITIES, PATIENTS, ENCOUNTERS"] + analytics["ANALYTICS
secure views"] + end + subgraph suite["pytest"] + matrix["Access matrix
one test per contract cell"] + gate["Coverage gate
tagged column without policy = fail"] + end + migrations --> sc --> sf + synthea --> loader --> clinical + governance -. "masks and filters" .-> clinical + analytics -- "selects from" --> clinical + contract --> matrix + synthea -- "expected values" --> matrix + matrix -- "one session per persona user" --> sf + gate -- "POLICY_REFERENCES,
TAG_REFERENCES_ALL_COLUMNS" --> sf +``` + +Everything that grants or hides data is deployed by schemachange from `migrations/`; the init hook only creates a warehouse and the deploy user. Each test session deploys the same migrations into its own `PHI_TEST_` database, loads the data and drops the database afterwards, so the suite tests the migrations rather than a hand-built fixture. The data is Synthea's fully synthetic 1K-patient sample, downloaded at run time and verified by SHA-256. Design notes: [docs/architecture.md](docs/architecture.md). + +## How it works + +1. `V1.0.0` creates five functional roles (`CLINICIAN`, `CLINICIAN_LEAD` which inherits it, `ANALYST`, `AUDITOR`, `DATA_ENGINEER`), one key-pair user per persona, the schemas and the tables. +2. `V1.1.0` creates the `PHI_TYPE` tag with `ALLOWED_VALUES` and attaches one masking policy per data type to it, so classifying a column is all it takes to mask it. +3. `V1.2.0` adds `FACILITY_ROW_ACCESS`: a row is visible when the session holds the row's `FACILITY_` role, or is an auditor or analyst. +4. `V1.3.0` adds `PATIENTS.EMAIL` with nothing but a tag, and `R__analyst_views.sql` publishes the secure views analysts read. +5. The access matrix connects as each persona's own user, reads every object, and compares each row with values computed from the Synthea source rows and the contract, never from the policies. +6. The coverage gate lists tagged columns and their effective masking policies, and exits 1 if any has none. + +`make demo` runs the matrix, shows `EMAIL` masked by its tag alone, then applies [`demos/broken-current-role-policy.sql`](demos/broken-current-role-policy.sql): a one-line "simplification" that uses `CURRENT_ROLE() = 'CLINICIAN'` instead of `IS_ROLE_IN_SESSION('CLINICIAN')`. The matrix fails exactly the 10 text-PHI cells of `CLINICIAN_LEAD`, and the demo restores the policy. + +```mermaid +sequenceDiagram + autonumber + actor Lead as PERSONA_CLINICIAN_LEAD + participant SF as Snowflake + participant RAP as FACILITY_ROW_ACCESS + participant Tag as PHI_TYPE tag + participant Mask as PHI_MASK_STRING + Lead->>SF: SELECT * FROM CLINICAL.PATIENTS (role CLINICIAN_LEAD) + SF->>SF: privilege check, SELECT inherited from CLINICIAN + SF->>RAP: per row, IS_ROLE_IN_SESSION('FACILITY_' || PRIMARY_FACILITY_ID) + RAP-->>SF: TRUE for 101 and 102, FALSE for 103 + SF->>Tag: which policy masks SSN (VARCHAR)? + Tag-->>SF: PHI_MASK_STRING + SF->>Mask: is CLINICIAN in session? + Mask-->>SF: yes, inherited, return the stored value + SF-->>Lead: rows of facilities 101 and 102, PHI in clear +``` + +## Prerequisites + +- Docker with Compose v2 +- [`uv`](https://docs.astral.sh/uv/) 0.5 or newer (it installs Python 3.12 if needed) +- `openssl`, for the throwaway local key pair +- A LocalStack auth token in `LOCALSTACK_AUTH_TOKEN` or `~/.localstack/default.env` +- Internet access on first run, for the emulator image and the 9 MB Synthea archive (cached in `.cache/`) + +## Quickstart + +```sh +cd phi-guardrails +make up # generate a key pair, start the emulator, wait until it answers +make seed # deploy the migrations into PHI and load the Synthea extract +make demo # access matrix, the tag-only EMAIL column, the broken-policy drill +make test # full suite in its own PHI_TEST_ database +make down # stop the emulator and remove its volumes +``` + +Set `SNOWFLAKE_PORT` (for example `SNOWFLAKE_PORT=14566 make up test down`) if host port 4566 is taken. `make gate` runs the coverage gate against `PHI`. + +## Tests + +`make test` deploys, loads, runs everything and drops its database. The suite covers: + +- **Access matrix**: one test per cell of [`access-contract.yml`](access-contract.yml) (174 cells): each persona sees exactly the rows (`all`, `facility`, `none`, `denied`) and the `clear` or `masked` values the contract states. +- **Regression detection**: with the broken policy applied, exactly the 10 clear text-PHI cells of `CLINICIAN_LEAD` fail. +- **Tag-only masking**: `EMAIL`'s only masking policy is the one its tag carries. +- **Coverage gate**: passes on the deployed database (14 tagged columns), and fails, naming exactly those columns, on a scratch table with tagged `NUMBER` and `TIMESTAMP_TZ` columns. +- **Classification and error paths**: tags match the contract, every clinical column is classified, an unknown PHI type is rejected (`391826`), the tag accepts no second `VARCHAR` policy (`391837`), and an attached policy cannot be replaced (`003531`). +- **Deployment**: schemachange records every migration, and a second deploy applies nothing. + +To run the same suite against a real account, set `SNOWFLAKE_ACCOUNT`, `SNOWFLAKE_USER`, `SNOWFLAKE_WAREHOUSE` and `SNOWFLAKE_PRIVATE_KEY_PATH` (an unencrypted PKCS#8 key with its `.pub` next to it; optionally `SNOWFLAKE_ROLE`, default `ACCOUNTADMIN`) and run `make test-cloud`. Use a non-production account: the suite creates account-level roles and `PERSONA_*` users, and consumes credits. + +## Project structure + +``` +phi-guardrails/ + access-contract.yml who may see what: the reviewed source of truth + migrations/ schemachange: V1.0.0 .. V1.3.0, R__analyst_views.sql + demos/ the broken policy the matrix must catch + src/phi_guardrails/ + contract.py parses access-contract.yml into test cases + matrix.py expected values and per-persona queries + coverage.py the policy-coverage gate (also a CLI) + deploy.py, load.py schemachange runner and Synthea loader + demo.py, drill.py make demo + tests/ access matrix, coverage gate, deployment + init/ warehouse and deploy user + docs/architecture.md data model and design decisions +``` + +## Learn more + +- [Tag-based masking policies](https://docs.snowflake.com/en/user-guide/tag-based-masking-policies) +- [Understanding row access policies](https://docs.snowflake.com/en/user-guide/security-row-intro) +- [`IS_ROLE_IN_SESSION`](https://docs.snowflake.com/en/sql-reference/functions/is_role_in_session) and [access control overview](https://docs.snowflake.com/en/user-guide/security-access-control-overview) +- [`POLICY_REFERENCES`](https://docs.snowflake.com/en/sql-reference/functions/policy_references) and [`TAG_REFERENCES_ALL_COLUMNS`](https://docs.snowflake.com/en/sql-reference/functions/tag_references_all_columns) +- [schemachange](https://github.com/Snowflake-Labs/schemachange) and [Synthea](https://synthetichealth.github.io/synthea/) diff --git a/phi-guardrails/access-contract.yml b/phi-guardrails/access-contract.yml new file mode 100644 index 0000000..6758808 --- /dev/null +++ b/phi-guardrails/access-contract.yml @@ -0,0 +1,111 @@ +# PHI access contract +# +# The single source of truth for who may see what. Compliance reviews this file; +# tests/test_access_matrix.py turns every cell into one test case and runs it +# against the deployed database as the persona's own user and role. A change to +# a policy that breaks a cell fails CI; a change to this file is a change to the +# access rules and goes through review. +# +# Outcomes +# rows all every row of the object +# facility only rows of the persona's facilities (facility_column) +# none the query succeeds and returns no rows +# denied the query fails with an authorization error (002003 or 003001) +# columns clear the stored value +# masked '***MASKED***' for text; 1 January of the same year for dates and timestamps +# A column lists one cell per persona that can see rows of the object. +# +# phi_type is the PHI_TYPE tag value the column must carry (null: not PHI). Views +# list no phi_type: their columns inherit protection from the base table. + +version: 1 + +personas: + CLINICIAN: {user: PERSONA_CLINICIAN, facilities: [101]} + CLINICIAN_LEAD: {user: PERSONA_CLINICIAN_LEAD, facilities: [101, 102]} + ANALYST: {user: PERSONA_ANALYST} + AUDITOR: {user: PERSONA_AUDITOR} + DATA_ENGINEER: {user: PERSONA_DATA_ENGINEER} + +objects: + CLINICAL.FACILITIES: + key: [FACILITY_ID] + rows: {CLINICIAN: all, CLINICIAN_LEAD: all, ANALYST: all, AUDITOR: all, DATA_ENGINEER: all} + columns: + FACILITY_ID: {phi_type: null, CLINICIAN: clear, CLINICIAN_LEAD: clear, ANALYST: clear, AUDITOR: clear, DATA_ENGINEER: clear} + NAME: {phi_type: null, CLINICIAN: clear, CLINICIAN_LEAD: clear, ANALYST: clear, AUDITOR: clear, DATA_ENGINEER: clear} + ADDRESS: {phi_type: null, CLINICIAN: clear, CLINICIAN_LEAD: clear, ANALYST: clear, AUDITOR: clear, DATA_ENGINEER: clear} + CITY: {phi_type: null, CLINICIAN: clear, CLINICIAN_LEAD: clear, ANALYST: clear, AUDITOR: clear, DATA_ENGINEER: clear} + STATE: {phi_type: null, CLINICIAN: clear, CLINICIAN_LEAD: clear, ANALYST: clear, AUDITOR: clear, DATA_ENGINEER: clear} + ZIP: {phi_type: null, CLINICIAN: clear, CLINICIAN_LEAD: clear, ANALYST: clear, AUDITOR: clear, DATA_ENGINEER: clear} + PHONE: {phi_type: null, CLINICIAN: clear, CLINICIAN_LEAD: clear, ANALYST: clear, AUDITOR: clear, DATA_ENGINEER: clear} + SOURCE_ORGANIZATION_ID: {phi_type: null, CLINICIAN: clear, CLINICIAN_LEAD: clear, ANALYST: clear, AUDITOR: clear, DATA_ENGINEER: clear} + + CLINICAL.PATIENTS: + key: [PATIENT_ID] + facility_column: PRIMARY_FACILITY_ID + rows: {CLINICIAN: facility, CLINICIAN_LEAD: facility, ANALYST: denied, AUDITOR: all, DATA_ENGINEER: none} + columns: + PATIENT_ID: {phi_type: null, CLINICIAN: clear, CLINICIAN_LEAD: clear, AUDITOR: clear} + PRIMARY_FACILITY_ID: {phi_type: null, CLINICIAN: clear, CLINICIAN_LEAD: clear, AUDITOR: clear} + FIRST_NAME: {phi_type: name, CLINICIAN: clear, CLINICIAN_LEAD: clear, AUDITOR: clear} + LAST_NAME: {phi_type: name, CLINICIAN: clear, CLINICIAN_LEAD: clear, AUDITOR: clear} + BIRTH_DATE: {phi_type: birth_date, CLINICIAN: clear, CLINICIAN_LEAD: clear, AUDITOR: masked} + DEATH_DATE: {phi_type: death_date, CLINICIAN: clear, CLINICIAN_LEAD: clear, AUDITOR: masked} + SSN: {phi_type: ssn, CLINICIAN: clear, CLINICIAN_LEAD: clear, AUDITOR: masked} + DRIVERS_LICENSE: {phi_type: license_number, CLINICIAN: clear, CLINICIAN_LEAD: clear, AUDITOR: masked} + PASSPORT: {phi_type: passport_number, CLINICIAN: clear, CLINICIAN_LEAD: clear, AUDITOR: masked} + GENDER: {phi_type: null, CLINICIAN: clear, CLINICIAN_LEAD: clear, AUDITOR: clear} + RACE: {phi_type: null, CLINICIAN: clear, CLINICIAN_LEAD: clear, AUDITOR: clear} + ETHNICITY: {phi_type: null, CLINICIAN: clear, CLINICIAN_LEAD: clear, AUDITOR: clear} + MARITAL_STATUS: {phi_type: null, CLINICIAN: clear, CLINICIAN_LEAD: clear, AUDITOR: clear} + ADDRESS: {phi_type: address, CLINICIAN: clear, CLINICIAN_LEAD: clear, AUDITOR: masked} + CITY: {phi_type: address, CLINICIAN: clear, CLINICIAN_LEAD: clear, AUDITOR: masked} + COUNTY: {phi_type: address, CLINICIAN: clear, CLINICIAN_LEAD: clear, AUDITOR: masked} + STATE: {phi_type: null, CLINICIAN: clear, CLINICIAN_LEAD: clear, AUDITOR: clear} + ZIP: {phi_type: zip, CLINICIAN: clear, CLINICIAN_LEAD: clear, AUDITOR: masked} + EMAIL: {phi_type: email, CLINICIAN: clear, CLINICIAN_LEAD: clear, AUDITOR: masked} # V1.3.0 + + CLINICAL.ENCOUNTERS: + key: [ENCOUNTER_ID] + facility_column: FACILITY_ID + rows: {CLINICIAN: facility, CLINICIAN_LEAD: facility, ANALYST: denied, AUDITOR: all, DATA_ENGINEER: none} + columns: + ENCOUNTER_ID: {phi_type: null, CLINICIAN: clear, CLINICIAN_LEAD: clear, AUDITOR: clear} + PATIENT_ID: {phi_type: null, CLINICIAN: clear, CLINICIAN_LEAD: clear, AUDITOR: clear} + FACILITY_ID: {phi_type: null, CLINICIAN: clear, CLINICIAN_LEAD: clear, AUDITOR: clear} + START_TS: {phi_type: service_date, CLINICIAN: clear, CLINICIAN_LEAD: clear, AUDITOR: masked} + STOP_TS: {phi_type: service_date, CLINICIAN: clear, CLINICIAN_LEAD: clear, AUDITOR: masked} + ENCOUNTER_CLASS: {phi_type: null, CLINICIAN: clear, CLINICIAN_LEAD: clear, AUDITOR: clear} + CODE: {phi_type: null, CLINICIAN: clear, CLINICIAN_LEAD: clear, AUDITOR: clear} + DESCRIPTION: {phi_type: null, CLINICIAN: clear, CLINICIAN_LEAD: clear, AUDITOR: clear} + REASON_CODE: {phi_type: null, CLINICIAN: clear, CLINICIAN_LEAD: clear, AUDITOR: clear} + REASON_DESCRIPTION: {phi_type: null, CLINICIAN: clear, CLINICIAN_LEAD: clear, AUDITOR: clear} + TOTAL_CLAIM_COST: {phi_type: null, CLINICIAN: clear, CLINICIAN_LEAD: clear, AUDITOR: clear} + PAYER_COVERAGE: {phi_type: null, CLINICIAN: clear, CLINICIAN_LEAD: clear, AUDITOR: clear} + + ANALYTICS.ENCOUNTER_SUMMARY: + key: [FACILITY_ID, ENCOUNTER_YEAR, ENCOUNTER_CLASS] + rows: {CLINICIAN: denied, CLINICIAN_LEAD: denied, ANALYST: all, AUDITOR: denied, DATA_ENGINEER: denied} + columns: + FACILITY_ID: {ANALYST: clear} + FACILITY_NAME: {ANALYST: clear} + ENCOUNTER_YEAR: {ANALYST: clear} + ENCOUNTER_CLASS: {ANALYST: clear} + ENCOUNTERS: {ANALYST: clear} + PATIENTS: {ANALYST: clear} + TOTAL_CLAIM_COST: {ANALYST: clear} + + ANALYTICS.PATIENT_COHORT: + key: [PATIENT_ID] + rows: {CLINICIAN: denied, CLINICIAN_LEAD: denied, ANALYST: all, AUDITOR: denied, DATA_ENGINEER: denied} + columns: + PATIENT_ID: {ANALYST: clear} + PRIMARY_FACILITY_ID: {ANALYST: clear} + BIRTH_DATE: {ANALYST: masked} + GENDER: {ANALYST: clear} + RACE: {ANALYST: clear} + ETHNICITY: {ANALYST: clear} + STATE: {ANALYST: clear} + ZIP: {ANALYST: masked} + EMAIL: {ANALYST: masked} # V1.3.0: masked with no new policy code diff --git a/phi-guardrails/demos/broken-current-role-policy.sql b/phi-guardrails/demos/broken-current-role-policy.sql new file mode 100644 index 0000000..9a04730 --- /dev/null +++ b/phi-guardrails/demos/broken-current-role-policy.sql @@ -0,0 +1,22 @@ +-- A plausible "simplification" of the string masking policy, as it might +-- arrive in a pull request: compare the session's role by name instead of +-- calling IS_ROLE_IN_SESSION. +-- +-- It reads the same and still works for CLINICIAN and AUDITOR. It breaks +-- CLINICIAN_LEAD: CURRENT_ROLE() is the session's primary role only, so a role +-- that inherits CLINICIAN no longer counts as a clinician, and every text PHI +-- column turns to '***MASKED***' for clinical leads. Dates and timestamps are +-- unaffected (their policies are untouched), which makes the bug easy to miss +-- in a manual spot check. +-- +-- `make demo` applies this to the sample database, shows the access matrix +-- catching it, and restores the original body. Rendered like a migration: +-- {{ database }} is the target database. + +ALTER MASKING POLICY {{ database }}.GOVERNANCE.PHI_MASK_STRING SET BODY -> + CASE + WHEN CURRENT_ROLE() = 'CLINICIAN' THEN val + WHEN CURRENT_ROLE() = 'AUDITOR' + AND SYSTEM$GET_TAG_ON_CURRENT_COLUMN('{{ database }}.GOVERNANCE.PHI_TYPE') = 'name' THEN val + ELSE '***MASKED***' + END; diff --git a/phi-guardrails/docker-compose.yml b/phi-guardrails/docker-compose.yml new file mode 100644 index 0000000..d1bb2e0 --- /dev/null +++ b/phi-guardrails/docker-compose.yml @@ -0,0 +1,18 @@ +# Snowflake-Next emulator for the PHI Guardrails sample. +# Host port is ${SNOWFLAKE_PORT:-4566}, bound to 127.0.0.1 so it is reachable from this machine only. +services: + snowflake: + image: ${SNOWFLAKE_IMAGE:-localstack/snowflake-next:latest} + pull_policy: always + ports: + - "127.0.0.1:${SNOWFLAKE_PORT:-4566}:4566" + environment: + LOCALSTACK_AUTH_TOKEN: ${LOCALSTACK_AUTH_TOKEN:?set LOCALSTACK_AUTH_TOKEN} + volumes: + - ./init:/etc/localstack/init/ready.d:ro + networks: + default: + aliases: + - snowflake.localhost.localstack.cloud + - stages.s3.localhost.localstack.cloud + - stages-enc.s3.localhost.localstack.cloud diff --git a/phi-guardrails/docs/architecture.md b/phi-guardrails/docs/architecture.md new file mode 100644 index 0000000..ce055db --- /dev/null +++ b/phi-guardrails/docs/architecture.md @@ -0,0 +1,107 @@ +# PHI Guardrails: design notes + +The README covers what the sample does and how to run it. This page records the data model and the design decisions behind the roles, policies and tests. + +## Data model + +```mermaid +erDiagram + FACILITIES ||--o{ PATIENTS : "primary facility" + FACILITIES ||--o{ ENCOUNTERS : "hosts" + PATIENTS ||--o{ ENCOUNTERS : "has" + FACILITIES { + number FACILITY_ID PK + varchar NAME + varchar ADDRESS + varchar CITY + varchar STATE + varchar ZIP + varchar PHONE + varchar SOURCE_ORGANIZATION_ID + } + PATIENTS { + varchar PATIENT_ID PK + number PRIMARY_FACILITY_ID FK + varchar FIRST_NAME "PHI name" + varchar LAST_NAME "PHI name" + date BIRTH_DATE "PHI birth_date" + date DEATH_DATE "PHI death_date" + varchar SSN "PHI ssn" + varchar DRIVERS_LICENSE "PHI license_number" + varchar PASSPORT "PHI passport_number" + varchar GENDER + varchar RACE + varchar ETHNICITY + varchar MARITAL_STATUS + varchar ADDRESS "PHI address" + varchar CITY "PHI address" + varchar COUNTY "PHI address" + varchar STATE + varchar ZIP "PHI zip" + varchar EMAIL "PHI email, V1.3.0" + } + ENCOUNTERS { + varchar ENCOUNTER_ID PK + varchar PATIENT_ID FK + number FACILITY_ID FK + timestamp_ntz START_TS "PHI service_date" + timestamp_ntz STOP_TS "PHI service_date" + varchar ENCOUNTER_CLASS + varchar CODE + varchar DESCRIPTION + varchar REASON_CODE + varchar REASON_DESCRIPTION + number TOTAL_CLAIM_COST + number PAYER_COVERAGE + } +``` + +The rows come from the Synthea April 2020 sample: three Massachusetts hospitals (facility ids 101, 102 and 103), their encounters since 2010, and the 156 patients seen there. A patient's primary facility is where most of their encounters took place. `PATIENT_ID` is a random surrogate key, not a medical record number, so it is not tagged. + +Classification follows the HIPAA Safe Harbor identifier list: names, geography smaller than a state, all elements of dates except the year, SSN, licence and passport numbers, and email. Clinical codes and descriptions stay unmasked: on their own they identify nobody, and they are what analysts need. + +## Layers of protection + +A query passes three independent checks, so a mistake in one layer does not expose PHI. + +```mermaid +flowchart TB + q["SELECT as a persona"] --> grants{"Privileges
USAGE + SELECT"} + grants -- "missing" --> denied["authorization error"] + grants -- "granted" --> rap{"FACILITY_ROW_ACCESS
per row"} + rap -- "FALSE" --> hidden["row not returned"] + rap -- "TRUE" --> mask{"Masking policy of the
column's PHI_TYPE tag"} + mask -- "clinician" --> clear["stored value"] + mask -- "auditor, name column" --> clear + mask -- "anyone else" --> masked["***MASKED*** or year only"] +``` + +| Role | `SELECT` on `PATIENTS`, `ENCOUNTERS` | `ANALYTICS` views | Rows let through | Masking | +| --- | --- | --- | --- | --- | +| `CLINICIAN` | yes | no | facility 101 | clear | +| `CLINICIAN_LEAD` | inherited | no | facilities 101 (inherited) and 102 | clear (inherited) | +| `AUDITOR` | yes | no | all | names clear, rest masked | +| `DATA_ENGINEER` | yes, plus `INSERT`, `UPDATE`, `DELETE`, `TRUNCATE` | no | none | masked | +| `ANALYST` | no | `SELECT` on both views | all, through the views only | masked | + +Every role reads the facility directory (`CLINICAL.FACILITIES`, not PHI). + +## Key design decisions + +- **Facility roles carry row access.** Each facility has a `FACILITY_` role, granted to the functional roles, and the row policy checks `IS_ROLE_IN_SESSION('FACILITY_' || facility_id)`. Who may see which facility is an ordinary grant, visible in `SHOW GRANTS` and reviewed like any other privilege. Persona sessions use their primary role only, so each test exercises exactly one role hierarchy. +- **`IS_ROLE_IN_SESSION`, not `CURRENT_ROLE()`.** It honours inheritance, so `CLINICIAN_LEAD` is treated as a clinician. The demo's broken policy shows what happens otherwise. +- **One tag, one policy per data type.** `GOVERNANCE.PHI_TYPE` uses `ALLOWED_VALUES` for a closed vocabulary and carries `PHI_MASK_STRING` (`VARCHAR`), `PHI_MASK_DATE` and `PHI_MASK_TIMESTAMP` (`TIMESTAMP_NTZ`). Classifying a new column of those types is the whole change (`V1.3.0`); a tagged column of any other type is caught by the coverage gate. +- **Masked dates keep the year.** Safe Harbor allows it, and it keeps year-level analytics such as `ENCOUNTER_SUMMARY` working. Auditors see names in clear so they can identify whose record was accessed. +- **The analyst passes the row policy but not the grants.** Secure views evaluate row policies as the querying role, so the policy lets analysts through and the missing `SELECT` grants keep them out of the raw tables. Masking is the third layer if a grant is ever added by mistake. +- **The data engineer writes but sees nothing.** Load jobs need `INSERT` and `TRUNCATE`, not readable PHI; the row policy hides every patient row from `DATA_ENGINEER`. +- **Policies are created once and altered in place.** Migrations use `CREATE ... IF NOT EXISTS` and change a policy with `ALTER MASKING POLICY ... SET BODY`, because an attached policy cannot be replaced (`003531`). Each column is classified in its own statement, so a diff shows exactly which classification changed. +- **Expectations never come from the policies.** The oracle computes visible rows from the persona's facilities in the contract, clear values from the Synthea rows, and masked values from the rule in the contract header. The two views are recomputed in Python, including the 11-patient suppression threshold. A policy bug cannot leak into the expectation. +- **The gate reads `INFORMATION_SCHEMA`.** `TAG_REFERENCES_ALL_COLUMNS` and `POLICY_REFERENCES` reflect a deploy immediately, whereas `ACCOUNT_USAGE` views have ingestion latency. +- **Every test session deploys from scratch.** The suite runs the real migrations into a fresh `PHI_TEST_` database, so it tests the migrations themselves. schemachange runs with a scrubbed environment, so `SNOWFLAKE_*` variables in a developer's shell cannot redirect a deploy. + +## Adapting it for production + +- Split the deploy identity: `USERADMIN` / `SECURITYADMIN` for roles, users and grants, and a governance role that owns `GOVERNANCE` and holds `APPLY MASKING POLICY`, `APPLY ROW ACCESS POLICY` and `APPLY TAG`. The sample deploys as `ACCOUNTADMIN` to stay short. +- Persona users are test fixtures: grant the functional roles to people or groups through SCIM, and keep `PERSONA_*` users in the non-production account that runs the matrix. +- Run `make gate` after every deploy, and the matrix on every pull request that touches `migrations/` or `access-contract.yml`. +- Treat changes to `access-contract.yml` as access-rule changes and require a compliance reviewer on that path (for example with `CODEOWNERS`). diff --git a/phi-guardrails/init/01_bootstrap.sf.sql b/phi-guardrails/init/01_bootstrap.sf.sql new file mode 100644 index 0000000..27447f2 --- /dev/null +++ b/phi-guardrails/init/01_bootstrap.sf.sql @@ -0,0 +1,18 @@ +-- Runs once when the emulator starts, in account SNOWFLAKE as role PUBLIC. +-- Everything else (roles, schema, tags, policies) is owned by the schemachange +-- migrations in migrations/, so this file only provides what a fresh account +-- needs before the first deploy: a warehouse and the deploy service user. +USE ROLE ACCOUNTADMIN; + +CREATE WAREHOUSE IF NOT EXISTS PHI_WH + WAREHOUSE_SIZE = XSMALL + AUTO_SUSPEND = 60 + AUTO_RESUME = TRUE + INITIALLY_SUSPENDED = TRUE; + +CREATE USER IF NOT EXISTS SAMPLE_SVC + TYPE = SERVICE + DEFAULT_ROLE = ACCOUNTADMIN + DEFAULT_WAREHOUSE = PHI_WH + COMMENT = 'schemachange deploy user for the PHI Guardrails sample'; +GRANT ROLE ACCOUNTADMIN TO USER SAMPLE_SVC; diff --git a/phi-guardrails/migrations/R__analyst_views.sql b/phi-guardrails/migrations/R__analyst_views.sql new file mode 100644 index 0000000..a39617e --- /dev/null +++ b/phi-guardrails/migrations/R__analyst_views.sql @@ -0,0 +1,51 @@ +-- Repeatable: the analyst-facing views. schemachange re-runs this file +-- whenever it changes, after all versioned migrations. +-- +-- Both views are SECURE, so their definitions and the base tables stay hidden +-- from analysts. Policies on the base columns still apply to whoever queries +-- the view: there is no masking logic here, and none is needed. + +-- Encounter volume and cost per facility, year and class. Cells covering fewer +-- than 11 patients are suppressed (the CMS cell-size suppression threshold), +-- so small groups cannot be used to re-identify a patient. YEAR() works on the +-- masked START_TS because masking keeps the year. +CREATE OR REPLACE SECURE VIEW {{ database }}.ANALYTICS.ENCOUNTER_SUMMARY + COPY GRANTS + COMMENT = 'Encounters per facility, year and class; cells under 11 patients suppressed' +AS +SELECT + e.FACILITY_ID, + f.NAME AS FACILITY_NAME, + YEAR(e.START_TS) AS ENCOUNTER_YEAR, + e.ENCOUNTER_CLASS, + COUNT(*) AS ENCOUNTERS, + COUNT(DISTINCT e.PATIENT_ID) AS PATIENTS, + SUM(e.TOTAL_CLAIM_COST) AS TOTAL_CLAIM_COST +FROM {{ database }}.CLINICAL.ENCOUNTERS AS e +JOIN {{ database }}.CLINICAL.FACILITIES AS f + ON f.FACILITY_ID = e.FACILITY_ID +GROUP BY e.FACILITY_ID, f.NAME, YEAR(e.START_TS), e.ENCOUNTER_CLASS +HAVING COUNT(DISTINCT e.PATIENT_ID) >= 11; + +-- Patient-level cohort for building study populations. PHI columns pass +-- through as-is and arrive masked for analysts, because their tags travel with +-- them; EMAIL (added in V1.3.0) needed no change beyond listing it here. +CREATE OR REPLACE SECURE VIEW {{ database }}.ANALYTICS.PATIENT_COHORT + COPY GRANTS + COMMENT = 'Patient demographics for cohort building; PHI masked by tag' +AS +SELECT + PATIENT_ID, + PRIMARY_FACILITY_ID, + BIRTH_DATE, + GENDER, + RACE, + ETHNICITY, + STATE, + ZIP, + EMAIL +FROM {{ database }}.CLINICAL.PATIENTS; + +GRANT USAGE ON SCHEMA {{ database }}.ANALYTICS TO ROLE ANALYST; +GRANT SELECT ON VIEW {{ database }}.ANALYTICS.ENCOUNTER_SUMMARY TO ROLE ANALYST; +GRANT SELECT ON VIEW {{ database }}.ANALYTICS.PATIENT_COHORT TO ROLE ANALYST; diff --git a/phi-guardrails/migrations/V1.0.0__roles_and_schema.sql b/phi-guardrails/migrations/V1.0.0__roles_and_schema.sql new file mode 100644 index 0000000..70f3b55 --- /dev/null +++ b/phi-guardrails/migrations/V1.0.0__roles_and_schema.sql @@ -0,0 +1,122 @@ +-- V1.0.0: clinical role hierarchy, one test user per persona, schemas and tables. +-- +-- Roles and users are account-level, so every statement here is idempotent: +-- the same migrations deploy into several databases (one per test run) in one +-- account. Grants are explicit per object: PHI never relies on future grants. + +-- Functional roles --------------------------------------------------------- +CREATE ROLE IF NOT EXISTS CLINICIAN COMMENT = 'Treats patients: clear PHI, rows of own facilities only'; +CREATE ROLE IF NOT EXISTS CLINICIAN_LEAD COMMENT = 'Leads a care team: everything CLINICIAN has, plus more facilities'; +CREATE ROLE IF NOT EXISTS ANALYST COMMENT = 'Population analytics: ANALYTICS views only, PHI masked'; +CREATE ROLE IF NOT EXISTS AUDITOR COMMENT = 'Privacy audit: all rows, identities clear, other PHI masked'; +CREATE ROLE IF NOT EXISTS DATA_ENGINEER COMMENT = 'Operates pipelines: may write, reads no patient rows'; + +GRANT ROLE CLINICIAN TO ROLE CLINICIAN_LEAD; + +-- One test user per persona. Key-pair only (no password); sessions use the +-- persona role alone (no secondary roles) so each test sees exactly one +-- role hierarchy. +CREATE USER IF NOT EXISTS PERSONA_CLINICIAN TYPE = PERSON DEFAULT_ROLE = CLINICIAN DEFAULT_WAREHOUSE = {{ warehouse }} DEFAULT_SECONDARY_ROLES = (); +CREATE USER IF NOT EXISTS PERSONA_CLINICIAN_LEAD TYPE = PERSON DEFAULT_ROLE = CLINICIAN_LEAD DEFAULT_WAREHOUSE = {{ warehouse }} DEFAULT_SECONDARY_ROLES = (); +CREATE USER IF NOT EXISTS PERSONA_ANALYST TYPE = PERSON DEFAULT_ROLE = ANALYST DEFAULT_WAREHOUSE = {{ warehouse }} DEFAULT_SECONDARY_ROLES = (); +CREATE USER IF NOT EXISTS PERSONA_AUDITOR TYPE = PERSON DEFAULT_ROLE = AUDITOR DEFAULT_WAREHOUSE = {{ warehouse }} DEFAULT_SECONDARY_ROLES = (); +CREATE USER IF NOT EXISTS PERSONA_DATA_ENGINEER TYPE = PERSON DEFAULT_ROLE = DATA_ENGINEER DEFAULT_WAREHOUSE = {{ warehouse }} DEFAULT_SECONDARY_ROLES = (); + +ALTER USER PERSONA_CLINICIAN SET RSA_PUBLIC_KEY = '{{ persona_public_key }}'; +ALTER USER PERSONA_CLINICIAN_LEAD SET RSA_PUBLIC_KEY = '{{ persona_public_key }}'; +ALTER USER PERSONA_ANALYST SET RSA_PUBLIC_KEY = '{{ persona_public_key }}'; +ALTER USER PERSONA_AUDITOR SET RSA_PUBLIC_KEY = '{{ persona_public_key }}'; +ALTER USER PERSONA_DATA_ENGINEER SET RSA_PUBLIC_KEY = '{{ persona_public_key }}'; + +GRANT ROLE CLINICIAN TO USER PERSONA_CLINICIAN; +GRANT ROLE CLINICIAN_LEAD TO USER PERSONA_CLINICIAN_LEAD; +GRANT ROLE ANALYST TO USER PERSONA_ANALYST; +GRANT ROLE AUDITOR TO USER PERSONA_AUDITOR; +GRANT ROLE DATA_ENGINEER TO USER PERSONA_DATA_ENGINEER; + +GRANT USAGE ON WAREHOUSE {{ warehouse }} TO ROLE CLINICIAN; +GRANT USAGE ON WAREHOUSE {{ warehouse }} TO ROLE ANALYST; +GRANT USAGE ON WAREHOUSE {{ warehouse }} TO ROLE AUDITOR; +GRANT USAGE ON WAREHOUSE {{ warehouse }} TO ROLE DATA_ENGINEER; + +-- Schemas ------------------------------------------------------------------ +-- CLINICAL holds the raw tables, GOVERNANCE the tag and policies (owned by the +-- deploy role, never granted to readers), ANALYTICS the views analysts read. +CREATE SCHEMA IF NOT EXISTS {{ database }}.CLINICAL COMMENT = 'Raw clinical tables (PHI)'; +CREATE SCHEMA IF NOT EXISTS {{ database }}.GOVERNANCE COMMENT = 'PHI tag, masking and row access policies'; +CREATE SCHEMA IF NOT EXISTS {{ database }}.ANALYTICS COMMENT = 'Secure views for analysts'; + +CREATE TABLE IF NOT EXISTS {{ database }}.CLINICAL.FACILITIES ( + FACILITY_ID NUMBER(6, 0) NOT NULL PRIMARY KEY, + NAME VARCHAR(200) NOT NULL, + ADDRESS VARCHAR(200), + CITY VARCHAR(100), + STATE VARCHAR(2), + ZIP VARCHAR(10), + PHONE VARCHAR(20), + SOURCE_ORGANIZATION_ID VARCHAR(36) NOT NULL +) COMMENT = 'Hospitals (Synthea organizations); directory data, not PHI'; + +CREATE TABLE IF NOT EXISTS {{ database }}.CLINICAL.PATIENTS ( + PATIENT_ID VARCHAR(36) NOT NULL PRIMARY KEY COMMENT 'Random surrogate key', + PRIMARY_FACILITY_ID NUMBER(6, 0) NOT NULL REFERENCES {{ database }}.CLINICAL.FACILITIES (FACILITY_ID), + FIRST_NAME VARCHAR(100) NOT NULL, + LAST_NAME VARCHAR(100) NOT NULL, + BIRTH_DATE DATE NOT NULL, + DEATH_DATE DATE, + SSN VARCHAR(11) NOT NULL, + DRIVERS_LICENSE VARCHAR(20), + PASSPORT VARCHAR(20), + GENDER VARCHAR(1), + RACE VARCHAR(30), + ETHNICITY VARCHAR(30), + MARITAL_STATUS VARCHAR(1), + ADDRESS VARCHAR(200), + CITY VARCHAR(100), + COUNTY VARCHAR(100), + STATE VARCHAR(50), + ZIP VARCHAR(10) +) COMMENT = 'Synthea patients seen at a sample facility since 2010'; + +CREATE TABLE IF NOT EXISTS {{ database }}.CLINICAL.ENCOUNTERS ( + ENCOUNTER_ID VARCHAR(36) NOT NULL PRIMARY KEY, + PATIENT_ID VARCHAR(36) NOT NULL REFERENCES {{ database }}.CLINICAL.PATIENTS (PATIENT_ID), + FACILITY_ID NUMBER(6, 0) NOT NULL REFERENCES {{ database }}.CLINICAL.FACILITIES (FACILITY_ID), + START_TS TIMESTAMP_NTZ NOT NULL COMMENT 'UTC', + STOP_TS TIMESTAMP_NTZ COMMENT 'UTC', + ENCOUNTER_CLASS VARCHAR(20) NOT NULL, + CODE VARCHAR(20) NOT NULL COMMENT 'SNOMED CT', + DESCRIPTION VARCHAR(200), + REASON_CODE VARCHAR(20), + REASON_DESCRIPTION VARCHAR(200), + TOTAL_CLAIM_COST NUMBER(12, 2), + PAYER_COVERAGE NUMBER(12, 2) +) COMMENT = 'Synthea encounters at sample facilities since 2010'; + +-- Grants ------------------------------------------------------------------- +GRANT USAGE ON DATABASE {{ database }} TO ROLE CLINICIAN; +GRANT USAGE ON DATABASE {{ database }} TO ROLE ANALYST; +GRANT USAGE ON DATABASE {{ database }} TO ROLE AUDITOR; +GRANT USAGE ON DATABASE {{ database }} TO ROLE DATA_ENGINEER; + +GRANT USAGE ON SCHEMA {{ database }}.CLINICAL TO ROLE CLINICIAN; +GRANT USAGE ON SCHEMA {{ database }}.CLINICAL TO ROLE AUDITOR; +GRANT USAGE ON SCHEMA {{ database }}.CLINICAL TO ROLE DATA_ENGINEER; +-- ANALYST reads the facility directory, but no patient-level table. +GRANT USAGE ON SCHEMA {{ database }}.CLINICAL TO ROLE ANALYST; + +GRANT SELECT ON TABLE {{ database }}.CLINICAL.FACILITIES TO ROLE CLINICIAN; +GRANT SELECT ON TABLE {{ database }}.CLINICAL.FACILITIES TO ROLE ANALYST; +GRANT SELECT ON TABLE {{ database }}.CLINICAL.FACILITIES TO ROLE AUDITOR; +GRANT SELECT ON TABLE {{ database }}.CLINICAL.FACILITIES TO ROLE DATA_ENGINEER; + +GRANT SELECT ON TABLE {{ database }}.CLINICAL.PATIENTS TO ROLE CLINICIAN; +GRANT SELECT ON TABLE {{ database }}.CLINICAL.PATIENTS TO ROLE AUDITOR; +GRANT SELECT ON TABLE {{ database }}.CLINICAL.ENCOUNTERS TO ROLE CLINICIAN; +GRANT SELECT ON TABLE {{ database }}.CLINICAL.ENCOUNTERS TO ROLE AUDITOR; + +-- Data engineers run the load jobs: they write, and the row access policy in +-- V1.2.0 keeps every patient row invisible to them. +GRANT SELECT, INSERT, UPDATE, DELETE, TRUNCATE ON TABLE {{ database }}.CLINICAL.PATIENTS TO ROLE DATA_ENGINEER; +GRANT SELECT, INSERT, UPDATE, DELETE, TRUNCATE ON TABLE {{ database }}.CLINICAL.ENCOUNTERS TO ROLE DATA_ENGINEER; +GRANT SELECT, INSERT, UPDATE, DELETE, TRUNCATE ON TABLE {{ database }}.CLINICAL.FACILITIES TO ROLE DATA_ENGINEER; diff --git a/phi-guardrails/migrations/V1.1.0__tags_and_masking.sql b/phi-guardrails/migrations/V1.1.0__tags_and_masking.sql new file mode 100644 index 0000000..3593427 --- /dev/null +++ b/phi-guardrails/migrations/V1.1.0__tags_and_masking.sql @@ -0,0 +1,71 @@ +-- V1.1.0: the PHI_TYPE tag and tag-based masking. +-- +-- Columns are classified with one tag. The tag carries one masking policy per +-- data type, so any VARCHAR, DATE or TIMESTAMP_NTZ column that is tagged later +-- is masked with no new policy code (V1.3.0 relies on this). A tagged column +-- of any other data type is NOT masked: tests/test_policy_coverage.py is the +-- gate that catches it. +-- +-- Who sees what: +-- CLINICIAN (and CLINICIAN_LEAD, which inherits it) every value in clear +-- AUDITOR names in clear (to identify whose record was accessed), the rest masked +-- everyone else masked +-- Masked strings read '***MASKED***'; masked dates and timestamps keep only the +-- year (HIPAA Safe Harbor allows the year), so year-level analytics still work. +-- +-- IS_ROLE_IN_SESSION, not CURRENT_ROLE: it honours the role hierarchy, so a +-- role that inherits CLINICIAN is treated as a clinician. + +CREATE TAG IF NOT EXISTS {{ database }}.GOVERNANCE.PHI_TYPE + ALLOWED_VALUES 'name', 'address', 'zip', 'birth_date', 'death_date', 'service_date', + 'phone', 'email', 'ssn', 'mrn', 'license_number', 'passport_number' + COMMENT = 'HIPAA identifier class of a column; drives masking'; + +CREATE MASKING POLICY IF NOT EXISTS {{ database }}.GOVERNANCE.PHI_MASK_STRING + AS (val VARCHAR) RETURNS VARCHAR -> + CASE + WHEN IS_ROLE_IN_SESSION('CLINICIAN') THEN val + WHEN IS_ROLE_IN_SESSION('AUDITOR') + AND SYSTEM$GET_TAG_ON_CURRENT_COLUMN('{{ database }}.GOVERNANCE.PHI_TYPE') = 'name' THEN val + ELSE '***MASKED***' + END + COMMENT = 'PHI strings: clear for clinicians, names clear for auditors'; + +CREATE MASKING POLICY IF NOT EXISTS {{ database }}.GOVERNANCE.PHI_MASK_DATE + AS (val DATE) RETURNS DATE -> + CASE + WHEN IS_ROLE_IN_SESSION('CLINICIAN') THEN val + ELSE DATE_TRUNC('YEAR', val) + END + COMMENT = 'PHI dates: clear for clinicians, year only otherwise'; + +CREATE MASKING POLICY IF NOT EXISTS {{ database }}.GOVERNANCE.PHI_MASK_TIMESTAMP + AS (val TIMESTAMP_NTZ) RETURNS TIMESTAMP_NTZ -> + CASE + WHEN IS_ROLE_IN_SESSION('CLINICIAN') THEN val + ELSE DATE_TRUNC('YEAR', val) + END + COMMENT = 'PHI timestamps: clear for clinicians, year only otherwise'; + +ALTER TAG {{ database }}.GOVERNANCE.PHI_TYPE SET + MASKING POLICY {{ database }}.GOVERNANCE.PHI_MASK_STRING, + MASKING POLICY {{ database }}.GOVERNANCE.PHI_MASK_DATE, + MASKING POLICY {{ database }}.GOVERNANCE.PHI_MASK_TIMESTAMP; + +-- Classify the existing columns ------------------------------------------- +-- One column per statement, so each classification is its own reviewable line +-- and a later diff shows exactly which column changed. +ALTER TABLE {{ database }}.CLINICAL.PATIENTS MODIFY COLUMN FIRST_NAME SET TAG {{ database }}.GOVERNANCE.PHI_TYPE = 'name'; +ALTER TABLE {{ database }}.CLINICAL.PATIENTS MODIFY COLUMN LAST_NAME SET TAG {{ database }}.GOVERNANCE.PHI_TYPE = 'name'; +ALTER TABLE {{ database }}.CLINICAL.PATIENTS MODIFY COLUMN BIRTH_DATE SET TAG {{ database }}.GOVERNANCE.PHI_TYPE = 'birth_date'; +ALTER TABLE {{ database }}.CLINICAL.PATIENTS MODIFY COLUMN DEATH_DATE SET TAG {{ database }}.GOVERNANCE.PHI_TYPE = 'death_date'; +ALTER TABLE {{ database }}.CLINICAL.PATIENTS MODIFY COLUMN SSN SET TAG {{ database }}.GOVERNANCE.PHI_TYPE = 'ssn'; +ALTER TABLE {{ database }}.CLINICAL.PATIENTS MODIFY COLUMN DRIVERS_LICENSE SET TAG {{ database }}.GOVERNANCE.PHI_TYPE = 'license_number'; +ALTER TABLE {{ database }}.CLINICAL.PATIENTS MODIFY COLUMN PASSPORT SET TAG {{ database }}.GOVERNANCE.PHI_TYPE = 'passport_number'; +ALTER TABLE {{ database }}.CLINICAL.PATIENTS MODIFY COLUMN ADDRESS SET TAG {{ database }}.GOVERNANCE.PHI_TYPE = 'address'; +ALTER TABLE {{ database }}.CLINICAL.PATIENTS MODIFY COLUMN CITY SET TAG {{ database }}.GOVERNANCE.PHI_TYPE = 'address'; +ALTER TABLE {{ database }}.CLINICAL.PATIENTS MODIFY COLUMN COUNTY SET TAG {{ database }}.GOVERNANCE.PHI_TYPE = 'address'; +ALTER TABLE {{ database }}.CLINICAL.PATIENTS MODIFY COLUMN ZIP SET TAG {{ database }}.GOVERNANCE.PHI_TYPE = 'zip'; + +ALTER TABLE {{ database }}.CLINICAL.ENCOUNTERS MODIFY COLUMN START_TS SET TAG {{ database }}.GOVERNANCE.PHI_TYPE = 'service_date'; +ALTER TABLE {{ database }}.CLINICAL.ENCOUNTERS MODIFY COLUMN STOP_TS SET TAG {{ database }}.GOVERNANCE.PHI_TYPE = 'service_date'; diff --git a/phi-guardrails/migrations/V1.2.0__facility_row_access.sql b/phi-guardrails/migrations/V1.2.0__facility_row_access.sql new file mode 100644 index 0000000..5b1f405 --- /dev/null +++ b/phi-guardrails/migrations/V1.2.0__facility_row_access.sql @@ -0,0 +1,34 @@ +-- V1.2.0: facility-scoped row access. +-- +-- Each facility has an access role FACILITY_. A row is visible when the +-- session holds the role of the row's facility, or a role that reads across +-- facilities: +-- AUDITOR audits every facility +-- ANALYST aggregates across facilities, but only through ANALYTICS views +-- (it has no SELECT grant on the patient-level tables) +-- DATA_ENGINEER holds no facility role, so it sees no patient rows at all. +-- +-- The role-to-facility mapping lives in the role hierarchy, so who may see +-- which facility is an ordinary grant: visible in SHOW GRANTS, reviewed like +-- any other privilege, and evaluated without a lookup table. + +CREATE ROLE IF NOT EXISTS FACILITY_101 COMMENT = 'Row access: HALLMARK HEALTH SYSTEM'; +CREATE ROLE IF NOT EXISTS FACILITY_102 COMMENT = 'Row access: CAMBRIDGE HEALTH ALLIANCE'; +CREATE ROLE IF NOT EXISTS FACILITY_103 COMMENT = 'Row access: NEWTON-WELLESLEY HOSPITAL'; + +-- Facility roles are granted to the functional roles, so row access follows +-- the role hierarchy and a session needs only its primary role. +GRANT ROLE FACILITY_101 TO ROLE CLINICIAN; -- clinicians work at facility 101 +GRANT ROLE FACILITY_102 TO ROLE CLINICIAN_LEAD; -- the lead also covers 102 (and 101 through CLINICIAN) + +CREATE ROW ACCESS POLICY IF NOT EXISTS {{ database }}.GOVERNANCE.FACILITY_ROW_ACCESS + AS (facility_id NUMBER) RETURNS BOOLEAN -> + IS_ROLE_IN_SESSION('AUDITOR') + OR IS_ROLE_IN_SESSION('ANALYST') + OR IS_ROLE_IN_SESSION('FACILITY_' || facility_id) + COMMENT = 'Rows of facilities whose FACILITY_ role is in session'; + +ALTER TABLE {{ database }}.CLINICAL.PATIENTS + ADD ROW ACCESS POLICY {{ database }}.GOVERNANCE.FACILITY_ROW_ACCESS ON (PRIMARY_FACILITY_ID); +ALTER TABLE {{ database }}.CLINICAL.ENCOUNTERS + ADD ROW ACCESS POLICY {{ database }}.GOVERNANCE.FACILITY_ROW_ACCESS ON (FACILITY_ID); diff --git a/phi-guardrails/migrations/V1.3.0__add_patient_email.sql b/phi-guardrails/migrations/V1.3.0__add_patient_email.sql new file mode 100644 index 0000000..42b866e --- /dev/null +++ b/phi-guardrails/migrations/V1.3.0__add_patient_email.sql @@ -0,0 +1,13 @@ +-- V1.3.0: a new PHI column. +-- +-- The column is classified at creation, and that is the whole change: the +-- PHI_TYPE tag already carries a VARCHAR masking policy, so EMAIL is masked for +-- every non-clinical role with no new policy code. access-contract.yml gains the +-- matching rows, and the coverage gate confirms the column is protected. + +ALTER TABLE {{ database }}.CLINICAL.PATIENTS + ADD COLUMN EMAIL VARCHAR(254) + WITH TAG ({{ database }}.GOVERNANCE.PHI_TYPE = 'email'); + +-- Describe the column for catalog users. +ALTER TABLE {{ database }}.CLINICAL.PATIENTS ALTER COLUMN EMAIL COMMENT 'Patient contact email'; diff --git a/phi-guardrails/pyproject.toml b/phi-guardrails/pyproject.toml new file mode 100644 index 0000000..ba86290 --- /dev/null +++ b/phi-guardrails/pyproject.toml @@ -0,0 +1,43 @@ +[project] +name = "phi-guardrails" +version = "0.1.0" +description = "Governance as code for a healthcare dataset: a role-by-column access matrix generated from a contract file, plus a policy-coverage gate." +readme = "README.md" +license = "Apache-2.0" +requires-python = ">=3.12,<3.13" +dependencies = [ + "schemachange==4.4.0", + "snowflake-connector-python>=4.0,<5.0", + "cryptography>=44", + "jinja2>=3.1", + "pyyaml>=6.0.2", +] + +[dependency-groups] +dev = [ + "pytest>=8.3", +] + +[tool.uv] +package = true + +[build-system] +requires = ["hatchling>=1.25"] +build-backend = "hatchling.build" + +[tool.hatch.build.targets.wheel] +packages = ["src/phi_guardrails"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +addopts = "-ra --strict-markers" +filterwarnings = [ + # Raised inside the connector's vendored urllib3, not by this sample. + "ignore:X509.get_subject is deprecated:DeprecationWarning", +] + +[tool.ruff] +line-length = 120 + +[tool.ruff.lint] +select = ["E", "F", "W", "I", "B", "UP"] diff --git a/phi-guardrails/schemachange-config.yml b/phi-guardrails/schemachange-config.yml new file mode 100644 index 0000000..0e7951d --- /dev/null +++ b/phi-guardrails/schemachange-config.yml @@ -0,0 +1,9 @@ +# schemachange 4.4 configuration. Connection settings are not stored here: +# phi_guardrails.deploy passes them as SNOWFLAKE_* variables in a scrubbed +# environment, plus --change-history-table and --vars for the target database. +config-version: 2 + +schemachange: + root-folder: migrations + create-change-history-table: true + query-tag: phi-guardrails diff --git a/phi-guardrails/scripts/load_synthea.py b/phi-guardrails/scripts/load_synthea.py new file mode 100644 index 0000000..d9cab33 --- /dev/null +++ b/phi-guardrails/scripts/load_synthea.py @@ -0,0 +1,36 @@ +"""Download (once), verify and load the pinned Synthea extract. + +Usage: uv run python scripts/load_synthea.py [--database NAME] + +Dataset: Synthea 1K sample CSV export, April 2020 build + URL https://synthetichealth.github.io/synthea-sample-data/downloads/synthea_sample_data_csv_apr2020.zip + SHA-256 4194b18c11eaedcf0d5d5dd448d8ac9661f14381e2ef9f109215dc42266cd38a + Cached .cache/synthea_sample_data_csv_apr2020.zip +Synthea output is fully synthetic. The generator is Apache-2.0; the project +states the data "can be used without concern for legal or privacy restrictions". +The pinned values live in phi_guardrails/synthea.py, which the test oracle shares. +""" + +from __future__ import annotations + +import argparse + +from phi_guardrails import settings as cfg +from phi_guardrails.load import load +from phi_guardrails.synthea import DATASET_SHA256, DATASET_URL + + +def main() -> None: + parser = argparse.ArgumentParser(description="Load the pinned Synthea extract.") + parser.add_argument("--database", help="target database (default: $PHI_DATABASE or PHI)") + args = parser.parse_args() + settings = cfg.load() + if args.database: + settings = settings.with_database(args.database.upper()) + print(f"Dataset {DATASET_URL}\n sha256 {DATASET_SHA256}") + for table, count in load(settings).items(): + print(f" {settings.database}.{table}: {count} rows") + + +if __name__ == "__main__": + main() diff --git a/phi-guardrails/scripts/wait-for-emulator.sh b/phi-guardrails/scripts/wait-for-emulator.sh new file mode 100755 index 0000000..501c7e2 --- /dev/null +++ b/phi-guardrails/scripts/wait-for-emulator.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# Wait until the Snowflake-Next emulator answers a SQL statement. +# +# Copy this file into a sample as scripts/wait-for-emulator.sh (samples stay +# self-contained). Usage: scripts/wait-for-emulator.sh [timeout-seconds] +# Reads SNOWFLAKE_PORT (default 4566). Readiness is checked from the host. +set -euo pipefail + +port="${SNOWFLAKE_PORT:-4566}" +timeout="${1:-120}" +url="https://snowflake.localhost.localstack.cloud:${port}" +deadline=$((SECONDS + timeout)) + +until curl -fsS --compressed -o /dev/null -X POST "${url}/api/v2/statements" \ + -H 'Authorization: Bearer local' -H 'Content-Type: application/json' \ + -H 'Accept: application/json' -d '{"statement":"SELECT 1"}' 2>/dev/null; do + if (( SECONDS >= deadline )); then + echo "Emulator at ${url} did not become ready within ${timeout}s." >&2 + echo "Check 'docker compose logs snowflake' (is LOCALSTACK_AUTH_TOKEN valid? is port ${port} free?)." >&2 + exit 1 + fi + sleep 2 +done +echo "Emulator ready at ${url}" diff --git a/phi-guardrails/src/phi_guardrails/__init__.py b/phi-guardrails/src/phi_guardrails/__init__.py new file mode 100644 index 0000000..3068d9a --- /dev/null +++ b/phi-guardrails/src/phi_guardrails/__init__.py @@ -0,0 +1 @@ +"""PHI Guardrails: governance as code for a synthetic (Synthea) healthcare dataset.""" diff --git a/phi-guardrails/src/phi_guardrails/contract.py b/phi-guardrails/src/phi_guardrails/contract.py new file mode 100644 index 0000000..fe4318e --- /dev/null +++ b/phi-guardrails/src/phi_guardrails/contract.py @@ -0,0 +1,132 @@ +"""Parse and validate ``access-contract.yml`` into test cases. + +Each cell of the contract becomes one :class:`Case`: + +* a rows case per (persona, object): ``all``, ``facility``, ``none`` or ``denied``; +* a column case per (persona, object, column): ``clear`` or ``masked``, for + personas that can see rows of the object. + +Validation is strict, so the matrix cannot silently shrink: a persona missing +from a ``rows`` rule, or a column missing a cell for a persona that can see the +rows, is an error rather than an untested cell. +""" + +from __future__ import annotations + +from dataclasses import dataclass +from pathlib import Path + +import yaml + +from phi_guardrails.settings import SAMPLE_ROOT + +CONTRACT_PATH = SAMPLE_ROOT / "access-contract.yml" +ROW_OUTCOMES = frozenset({"all", "facility", "none", "denied"}) +COLUMN_OUTCOMES = frozenset({"clear", "masked"}) +VISIBLE = frozenset({"all", "facility"}) + + +class ContractError(ValueError): + pass + + +@dataclass(frozen=True) +class Persona: + role: str + user: str + facilities: tuple[int, ...] + + +@dataclass(frozen=True) +class Column: + name: str + access: dict[str, str] + phi_type: str | None + declares_phi_type: bool + + +@dataclass(frozen=True) +class AccessObject: + name: str + key: tuple[str, ...] + rows: dict[str, str] + columns: dict[str, Column] + facility_column: str | None + + @property + def is_table(self) -> bool: + """Tables classify their columns (phi_type); views inherit classification.""" + return all(c.declares_phi_type for c in self.columns.values()) + + +@dataclass(frozen=True) +class Case: + persona: str + object: str + column: str | None + expect: str + + @property + def id(self) -> str: + return f"{self.persona}-{self.object}-{self.column or 'rows'}-{self.expect}" + + +@dataclass(frozen=True) +class Contract: + personas: dict[str, Persona] + objects: dict[str, AccessObject] + + def cases(self) -> list[Case]: + out: list[Case] = [] + for obj in self.objects.values(): + for role in self.personas: + out.append(Case(role, obj.name, None, obj.rows[role])) + if obj.rows[role] in VISIBLE: + out.extend(Case(role, obj.name, col.name, col.access[role]) for col in obj.columns.values()) + return out + + +def _persona(role: str, spec: dict) -> Persona: + return Persona(role=role, user=spec["user"], facilities=tuple(spec.get("facilities", ()))) + + +def _object(name: str, spec: dict, personas: dict[str, Persona]) -> AccessObject: + rows = spec["rows"] + if set(rows) != set(personas): + raise ContractError(f"{name}: rows must list exactly {sorted(personas)}, got {sorted(rows)}") + if bad := {r: o for r, o in rows.items() if o not in ROW_OUTCOMES}: + raise ContractError(f"{name}: unknown row outcome(s) {bad}") + facility_column = spec.get("facility_column") + for role, outcome in rows.items(): + if outcome == "facility" and not (facility_column and personas[role].facilities): + raise ContractError(f"{name}: {role} is facility-scoped but facilities/facility_column is missing") + visible = {r for r, o in rows.items() if o in VISIBLE} + columns = {} + for col_name, cell in spec["columns"].items(): + access = {k: v for k, v in cell.items() if k != "phi_type"} + if set(access) != visible: + raise ContractError( + f"{name}.{col_name}: needs one cell for each persona that sees rows {sorted(visible)}, " + f"got {sorted(access)}" + ) + if bad := {r: o for r, o in access.items() if o not in COLUMN_OUTCOMES}: + raise ContractError(f"{name}.{col_name}: unknown column outcome(s) {bad}") + columns[col_name] = Column( + name=col_name, + access=access, + phi_type=cell.get("phi_type"), + declares_phi_type="phi_type" in cell, + ) + key = tuple(spec["key"]) + if missing := set(key) - set(columns): + raise ContractError(f"{name}: key column(s) {sorted(missing)} are not listed in columns") + return AccessObject(name, key, rows, columns, facility_column) + + +def load_contract(path: Path = CONTRACT_PATH) -> Contract: + raw = yaml.safe_load(path.read_text(encoding="utf-8")) + if raw.get("version") != 1: + raise ContractError(f"{path}: expected contract version 1, got {raw.get('version')!r}") + personas = {role: _persona(role, spec) for role, spec in raw["personas"].items()} + objects = {name: _object(name, spec, personas) for name, spec in raw["objects"].items()} + return Contract(personas, objects) diff --git a/phi-guardrails/src/phi_guardrails/coverage.py b/phi-guardrails/src/phi_guardrails/coverage.py new file mode 100644 index 0000000..9017e12 --- /dev/null +++ b/phi-guardrails/src/phi_guardrails/coverage.py @@ -0,0 +1,119 @@ +"""Policy-coverage gate: every PHI_TYPE-tagged column must be masked. + +Usage: ``python -m phi_guardrails.coverage [--database NAME]`` (exit 1 on an unprotected column) + +Tag-based masking only protects a tagged column when the tag carries a policy +for that column's data type. A ZIP stored as NUMBER, or a timestamp stored as +TIMESTAMP_TZ, keeps its tag and silently stays in clear. The gate lists every +tagged column in the database, looks up its effective masking policy, and +fails on any column that has none. + +Only INFORMATION_SCHEMA table functions are used (TAG_REFERENCES_ALL_COLUMNS, +POLICY_REFERENCES): unlike the ACCOUNT_USAGE views they have no ingestion +latency, so the gate can run straight after a deploy. +""" + +from __future__ import annotations + +import argparse +import sys +from dataclasses import dataclass + +from snowflake.connector import SnowflakeConnection + +from phi_guardrails import settings as cfg + +TAG_SCHEMA = "GOVERNANCE" +TAG_NAME = "PHI_TYPE" + + +@dataclass(frozen=True) +class TaggedColumn: + table: str # SCHEMA.TABLE + column: str + data_type: str + phi_type: str + masking_policy: str | None # SCHEMA.POLICY, or None when unprotected + + @property + def protected(self) -> bool: + return self.masking_policy is not None + + +def _tables(cur, database: str) -> list[tuple[str, str]]: + cur.execute( + f"SELECT TABLE_SCHEMA, TABLE_NAME FROM {database}.INFORMATION_SCHEMA.TABLES " + "WHERE TABLE_TYPE = 'BASE TABLE' AND TABLE_SCHEMA <> 'INFORMATION_SCHEMA' " + "ORDER BY 1, 2" + ) + return cur.fetchall() + + +def scan(conn: SnowflakeConnection, database: str) -> list[TaggedColumn]: + """Every column of every table in ``database`` that carries the PHI_TYPE tag.""" + cur = conn.cursor() + cur.execute(f"SELECT TABLE_SCHEMA, TABLE_NAME, COLUMN_NAME, DATA_TYPE FROM {database}.INFORMATION_SCHEMA.COLUMNS") + data_types = {(s, t, c): d for s, t, c, d in cur.fetchall()} + found: list[TaggedColumn] = [] + for schema, table in _tables(cur, database): + fq = f"{database}.{schema}.{table}" + cur.execute( + "SELECT COLUMN_NAME, TAG_VALUE " + f"FROM TABLE({database}.INFORMATION_SCHEMA.TAG_REFERENCES_ALL_COLUMNS(%s, 'table')) " + "WHERE TAG_DATABASE = %s AND TAG_SCHEMA = %s AND TAG_NAME = %s", + (fq, database, TAG_SCHEMA, TAG_NAME), + ) + tagged = dict(cur.fetchall()) + if not tagged: + continue + cur.execute( + "SELECT REF_COLUMN_NAME, POLICY_SCHEMA || '.' || POLICY_NAME " + f"FROM TABLE({database}.INFORMATION_SCHEMA.POLICY_REFERENCES(" + "REF_ENTITY_NAME => %s, REF_ENTITY_DOMAIN => 'table')) " + "WHERE POLICY_KIND = 'MASKING_POLICY' AND POLICY_STATUS = 'ACTIVE'", + (fq,), + ) + policies = dict(cur.fetchall()) + found.extend( + TaggedColumn( + table=f"{schema}.{table}", + column=column, + data_type=data_types.get((schema, table, column), "?"), + phi_type=phi_type, + masking_policy=policies.get(column), + ) + for column, phi_type in sorted(tagged.items()) + ) + return found + + +def uncovered(columns: list[TaggedColumn]) -> list[TaggedColumn]: + return [c for c in columns if not c.protected] + + +def report(columns: list[TaggedColumn]) -> str: + lines = [f"{'column':<40} {'data type':<14} {'phi_type':<16} masking policy"] + for c in columns: + policy = c.masking_policy or "NONE <-- unprotected" + lines.append(f"{c.table + '.' + c.column:<40} {c.data_type:<14} {c.phi_type:<16} {policy}") + gaps = uncovered(columns) + lines.append( + f"{len(columns)} tagged column(s), {len(gaps)} without a masking policy" + ("" if gaps else ": gate passed") + ) + return "\n".join(lines) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Fail if a PHI_TYPE-tagged column has no masking policy.") + parser.add_argument("--database", help="database to scan (default: $PHI_DATABASE or PHI)") + args = parser.parse_args() + settings = cfg.load() + database = (args.database or settings.database).upper() + with cfg.connect(settings) as conn: + columns = scan(conn, database) + print(report(columns)) + sys.exit(1 if uncovered(columns) else 0) + + +if __name__ == "__main__": + main() diff --git a/phi-guardrails/src/phi_guardrails/demo.py b/phi-guardrails/src/phi_guardrails/demo.py new file mode 100644 index 0000000..f9b365d --- /dev/null +++ b/phi-guardrails/src/phi_guardrails/demo.py @@ -0,0 +1,116 @@ +"""``make demo``: the access matrix, a new PHI column, and a caught regression. + +Runs against the sample database (``$PHI_DATABASE``, default ``PHI``) after +``make migrate seed``. The regression drill changes a masking policy and +restores its original body before exiting, even on Ctrl-C. +""" + +from __future__ import annotations + +from collections import Counter + +from phi_guardrails import settings as cfg +from phi_guardrails.contract import Case, Contract, load_contract +from phi_guardrails.drill import BROKEN_POLICY_SQL, STRING_POLICY, temporarily_applied +from phi_guardrails.matrix import Matrix +from phi_guardrails.settings import SAMPLE_ROOT, Settings +from phi_guardrails.synthea import load_dataset + +Results = list[tuple[Case, list[str]]] + + +def heading(text: str) -> None: + print(f"\n== {text}") + + +def print_grid(contract: Contract, results: Results) -> Results: + """One line per persona, one column per object: passed/total cells.""" + passed: Counter = Counter() + total: Counter = Counter() + failed = [] + for case, failures in results: + total[case.persona, case.object] += 1 + if failures: + failed.append((case, failures)) + else: + passed[case.persona, case.object] += 1 + objects = list(contract.objects) + labels = [o.split(".")[1] for o in objects] + width = max(len(label) for label in labels) + 2 + print(f" {'persona':<16}" + "".join(f"{label:<{width}}" for label in labels).rstrip()) + for persona in contract.personas: + cells = [] + for obj in objects: + p, t = passed[persona, obj], total[persona, obj] + marker = "ok" if p == t else "FAIL" + cells.append(f"{f'{marker} {p}/{t}':<{width}}") + print(f" {persona:<16}" + "".join(cells).rstrip()) + return failed + + +def show_email(m: Matrix, settings: Settings) -> None: + db = settings.database + with cfg.connect(settings) as conn: + policy, tag = ( + conn.cursor() + .execute( + f"SELECT POLICY_SCHEMA || '.' || POLICY_NAME, TAG_SCHEMA || '.' || TAG_NAME " + f"FROM TABLE({db}.INFORMATION_SCHEMA.POLICY_REFERENCES(REF_ENTITY_NAME => '{db}.CLINICAL.PATIENTS', " + "REF_ENTITY_DOMAIN => 'table')) WHERE POLICY_KIND = 'MASKING_POLICY' AND REF_COLUMN_NAME = 'EMAIL'" + ) + .fetchone() + ) + print(f" EMAIL is masked by {policy}, inherited from tag {tag}: no policy was written for it.") + patient = min(p["PATIENT_ID"] for p in m.oracle.data.patients if p["PRIMARY_FACILITY_ID"] == 101) + for persona, obj in ( + ("CLINICIAN", "CLINICAL.PATIENTS"), + ("AUDITOR", "CLINICAL.PATIENTS"), + ("ANALYST", "ANALYTICS.PATIENT_COHORT"), + ): + row = m.observer.observe(persona, m.contract.objects[obj]).rows.get((patient,), {}) + print(f" {persona:<10} reading {obj:<26} sees EMAIL = {row.get('EMAIL')}") + + +def drill(m: Matrix, settings: Settings) -> bool: + """Apply the broken policy, show what the matrix catches, restore; True if all is well.""" + print(f" {BROKEN_POLICY_SQL.relative_to(SAMPLE_ROOT)} rewrites {STRING_POLICY} with") + print(" CURRENT_ROLE() = 'CLINICIAN' instead of IS_ROLE_IN_SESSION('CLINICIAN').") + with cfg.connect(settings) as conn, temporarily_applied(conn, settings.database): + m.observer.invalidate() + failed = [(c, f) for c, f in m.run() if f] + for case, failures in failed: + print(f" FAIL {case.persona} {case.object}.{case.column}: {failures[0]}") + personas = sorted({c.persona for c, _ in failed}) + print( + f" {len(failed)} cell(s) failed, for {', '.join(personas) or 'nobody'}: the matrix caught the regression." + ) + m.observer.invalidate() + still_failing = [c.id for c, f in m.run([c for c, _ in failed]) if f] + print(f" Original policy body restored; the {len(failed)} cell(s) pass again: {not still_failing}") + return bool(failed) and not still_failing + + +def main() -> None: + settings = cfg.load() + contract = load_contract() + m = Matrix(settings, contract, load_dataset()) + try: + heading( + f"1. Access matrix: access-contract.yml -> {len(contract.cases())} cells, " + "each queried as the persona's own user" + ) + failed = print_grid(contract, m.run()) + for case, failures in failed: + print(f" FAIL {case.id}: {failures[0]}") + heading("2. A new PHI column with no new policy code (V1.3.0 adds CLINICAL.PATIENTS.EMAIL)") + show_email(m, settings) + heading("3. Regression drill: a broken masking policy") + drill_ok = drill(m, settings) + finally: + m.close() + if failed or not drill_ok: + raise SystemExit("The access matrix did not match access-contract.yml; see the FAIL lines above.") + + +if __name__ == "__main__": + main() diff --git a/phi-guardrails/src/phi_guardrails/deploy.py b/phi-guardrails/src/phi_guardrails/deploy.py new file mode 100644 index 0000000..74f9106 --- /dev/null +++ b/phi-guardrails/src/phi_guardrails/deploy.py @@ -0,0 +1,92 @@ +"""Run the schemachange migrations against one database. + +Usage: ``python -m phi_guardrails.deploy [--database NAME]`` + +schemachange forwards every ``SNOWFLAKE_*`` environment variable to the +connector, so the subprocess gets a scrubbed environment that contains only the +connection this sample means to use. A developer's cloud credentials in the +shell can therefore never redirect an emulator deploy. +""" + +from __future__ import annotations + +import argparse +import json +import os +import subprocess +import sys +from pathlib import Path + +from phi_guardrails import settings as cfg +from phi_guardrails.settings import SAMPLE_ROOT, Settings + +SCHEMACHANGE = Path(sys.executable).with_name("schemachange") + + +def change_history_table(database: str) -> str: + return f"{database}.SCHEMACHANGE.CHANGE_HISTORY" + + +def schemachange_env(settings: Settings) -> dict[str, str]: + env = {k: v for k, v in os.environ.items() if not k.startswith(("SNOWFLAKE_", "SNOWSQL_"))} + env.update( + SNOWFLAKE_ACCOUNT=settings.account, + SNOWFLAKE_USER=settings.user, + SNOWFLAKE_ROLE=settings.role, + SNOWFLAKE_WAREHOUSE=settings.warehouse, + SNOWFLAKE_DATABASE=settings.database, + SNOWFLAKE_AUTHENTICATOR="SNOWFLAKE_JWT", + SNOWFLAKE_PRIVATE_KEY_FILE=str(settings.private_key_path), + ) + env[cfg.PLATFORM_DETECTION_ENV] = os.environ[cfg.PLATFORM_DETECTION_ENV] + if settings.is_emulator: + env.update( + SNOWFLAKE_HOST=str(settings.host), + SNOWFLAKE_PORT=str(settings.port), + SNOWFLAKE_PROTOCOL=str(settings.protocol), + SNOWFLAKE_DISABLE_OCSP_CHECKS="true", # see Settings.connect_kwargs + ) + return env + + +def deploy(settings: Settings, *, log_level: str = "WARNING") -> None: + """Create the database if needed, then apply every pending migration in ``migrations/``.""" + with cfg.connect(settings) as conn: + conn.cursor().execute(f"CREATE DATABASE IF NOT EXISTS IDENTIFIER('{settings.database}')") + template_vars = { + "database": settings.database, + "warehouse": settings.warehouse, + "persona_public_key": cfg.public_key_body(settings), + } + subprocess.run( + [ + str(SCHEMACHANGE), + "deploy", + "--config-folder", + str(SAMPLE_ROOT), + "--change-history-table", + change_history_table(settings.database), + "--vars", + json.dumps(template_vars), + "--log-level", + log_level, + ], + check=True, + env=schemachange_env(settings), + cwd=SAMPLE_ROOT, + ) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--database", help="target database (default: $PHI_DATABASE or PHI)") + parser.add_argument("--log-level", default="INFO") + args = parser.parse_args() + settings = cfg.load() + if args.database: + settings = settings.with_database(args.database.upper()) + deploy(settings, log_level=args.log_level) + + +if __name__ == "__main__": + main() diff --git a/phi-guardrails/src/phi_guardrails/drill.py b/phi-guardrails/src/phi_guardrails/drill.py new file mode 100644 index 0000000..920b040 --- /dev/null +++ b/phi-guardrails/src/phi_guardrails/drill.py @@ -0,0 +1,47 @@ +"""Apply a policy change temporarily, then put the original policy back. + +Used by the regression test and ``make demo`` to show that the access matrix +catches a broken masking policy, without leaving the database broken. +""" + +from __future__ import annotations + +from collections.abc import Iterator +from contextlib import contextmanager +from pathlib import Path + +import jinja2 +from snowflake.connector import SnowflakeConnection + +from phi_guardrails.settings import SAMPLE_ROOT + +BROKEN_POLICY_SQL = SAMPLE_ROOT / "demos" / "broken-current-role-policy.sql" +STRING_POLICY = "GOVERNANCE.PHI_MASK_STRING" + + +def render(path: Path, database: str) -> str: + """Render a SQL file the way schemachange renders migrations.""" + return jinja2.Template(path.read_text(encoding="utf-8")).render(database=database) + + +def policy_body(conn: SnowflakeConnection, policy: str) -> str: + cur = conn.cursor().execute(f"DESCRIBE MASKING POLICY {policy}") + columns = [d[0].lower() for d in cur.description] + return cur.fetchone()[columns.index("body")] + + +@contextmanager +def temporarily_applied( + conn: SnowflakeConnection, + database: str, + sql_file: Path = BROKEN_POLICY_SQL, + policy: str = STRING_POLICY, +) -> Iterator[None]: + """Run ``sql_file`` (which alters ``policy``), and restore ``policy``'s body on exit.""" + qualified = f"{database}.{policy}" + original = policy_body(conn, qualified) + conn.execute_string(render(sql_file, database)) + try: + yield + finally: + conn.cursor().execute(f"ALTER MASKING POLICY {qualified} SET BODY -> {original}") diff --git a/phi-guardrails/src/phi_guardrails/load.py b/phi-guardrails/src/phi_guardrails/load.py new file mode 100644 index 0000000..c860f36 --- /dev/null +++ b/phi-guardrails/src/phi_guardrails/load.py @@ -0,0 +1,41 @@ +"""Load the pinned Synthea extract into a deployed database (idempotent). + +The load runs as the deploy identity in one transaction: tables are truncated +and refilled, so re-running it leaves exactly one copy of the data. TRUNCATE +(not DELETE) empties every row: the deploy role is not a facility role, so a +DELETE would only see the rows the facility row access policy shows it. +""" + +from __future__ import annotations + +from phi_guardrails import settings as cfg +from phi_guardrails.settings import Settings +from phi_guardrails.synthea import Dataset, load_dataset + +LOAD_ORDER = ("CLINICAL.FACILITIES", "CLINICAL.PATIENTS", "CLINICAL.ENCOUNTERS") + + +def load(settings: Settings, dataset: Dataset | None = None) -> dict[str, int]: + dataset = dataset or load_dataset() + counts: dict[str, int] = {} + with cfg.connect(settings) as conn: + cur = conn.cursor() + cur.execute(f"USE DATABASE IDENTIFIER('{settings.database}')") + cur.execute("BEGIN") + try: + for table in reversed(LOAD_ORDER): + cur.execute(f"TRUNCATE TABLE {table}") + for table in LOAD_ORDER: + rows = dataset.table(table) + columns = list(rows[0]) + placeholders = ", ".join(["%s"] * len(columns)) + cur.executemany( + f"INSERT INTO {table} ({', '.join(columns)}) VALUES ({placeholders})", + [tuple(row[c] for c in columns) for row in rows], + ) + counts[table] = len(rows) + cur.execute("COMMIT") + except Exception: + cur.execute("ROLLBACK") + raise + return counts diff --git a/phi-guardrails/src/phi_guardrails/matrix.py b/phi-guardrails/src/phi_guardrails/matrix.py new file mode 100644 index 0000000..777be89 --- /dev/null +++ b/phi-guardrails/src/phi_guardrails/matrix.py @@ -0,0 +1,227 @@ +"""Evaluate access-contract cases against a deployed database. + +The :class:`Oracle` derives every expected value from the loaded Synthea rows +and the contract alone: visible rows from the persona's facilities, clear +values from the source data, masked values from the masking rule stated in the +contract header. It never reads a policy, so a policy bug cannot leak into the +expectation. + +The :class:`Observer` queries each object once per persona, connected as that +persona's user with its role, and caches the result for all column cases. +""" + +from __future__ import annotations + +from collections import defaultdict +from dataclasses import dataclass, field +from datetime import date, datetime +from decimal import Decimal + +from snowflake.connector import SnowflakeConnection +from snowflake.connector.errors import ProgrammingError + +from phi_guardrails import settings as cfg +from phi_guardrails.contract import AccessObject, Case, Contract +from phi_guardrails.settings import Settings +from phi_guardrails.synthea import Dataset + +MASK = "***MASKED***" +# Authorization errors that satisfy a "denied" cell: 002003 (object does not +# exist or not authorized) and 003001 (insufficient privileges). The contract +# pins that access is refused, not which of the two messages is returned. +NOT_AUTHORIZED = frozenset({2003, 3001}) +SUPPRESSION_THRESHOLD = 11 # ENCOUNTER_SUMMARY hides cells with fewer patients +Row = dict[str, object] +Key = tuple + + +def masked(value: object, sample: object) -> object: + """The contract's masking rule; ``sample`` is any non-null value of the column.""" + if isinstance(sample, datetime): + return None if value is None else datetime(value.year, 1, 1) # type: ignore[union-attr] + if isinstance(sample, date): + return None if value is None else date(value.year, 1, 1) # type: ignore[union-attr] + return MASK + + +# Python equivalents of the views in migrations/R__analyst_views.sql, computed +# over the source rows (not over the database). +def _encounter_summary(data: Dataset) -> list[Row]: + names = {f["FACILITY_ID"]: f["NAME"] for f in data.facilities} + groups: dict[Key, list[Row]] = defaultdict(list) + for enc in data.encounters: + groups[(enc["FACILITY_ID"], enc["START_TS"].year, enc["ENCOUNTER_CLASS"])].append(enc) # type: ignore[union-attr] + out = [] + for (facility, year, klass), encs in groups.items(): + patients = {e["PATIENT_ID"] for e in encs} + if len(patients) >= SUPPRESSION_THRESHOLD: + out.append( + { + "FACILITY_ID": facility, + "FACILITY_NAME": names[facility], + "ENCOUNTER_YEAR": year, + "ENCOUNTER_CLASS": klass, + "ENCOUNTERS": len(encs), + "PATIENTS": len(patients), + "TOTAL_CLAIM_COST": sum((e["TOTAL_CLAIM_COST"] for e in encs), Decimal("0")), + } + ) + return out + + +def _patient_cohort(data: Dataset) -> list[Row]: + cols = ("PATIENT_ID", "PRIMARY_FACILITY_ID", "BIRTH_DATE", "GENDER", "RACE", "ETHNICITY", "STATE", "ZIP", "EMAIL") + return [{c: p[c] for c in cols} for p in data.patients] + + +VIEW_SOURCES = { + "ANALYTICS.ENCOUNTER_SUMMARY": _encounter_summary, + "ANALYTICS.PATIENT_COHORT": _patient_cohort, +} + + +class Oracle: + def __init__(self, contract: Contract, data: Dataset) -> None: + self.contract = contract + self.data = data + + def source_rows(self, obj: AccessObject) -> list[Row]: + view = VIEW_SOURCES.get(obj.name) + return view(self.data) if view else self.data.table(obj.name) + + def visible_rows(self, persona: str, obj: AccessObject) -> dict[Key, Row]: + rule = obj.rows[persona] + rows = self.source_rows(obj) + if rule == "facility": + allowed = set(self.contract.personas[persona].facilities) + rows = [r for r in rows if r[obj.facility_column] in allowed] # type: ignore[index] + elif rule in ("none", "denied"): + rows = [] + return {tuple(r[k] for k in obj.key): r for r in rows} + + def expected_value(self, obj: AccessObject, column: str, outcome: str, row: Row) -> object: + if outcome == "clear": + return row[column] + sample = next((r[column] for r in self.source_rows(obj) if r[column] is not None), "") + return masked(row[column], sample) + + +@dataclass +class Observation: + columns: list[str] = field(default_factory=list) + rows: dict[Key, Row] = field(default_factory=dict) + error: ProgrammingError | None = None + + +class Observer: + """Reads each object once per persona, as the persona's user and role.""" + + def __init__(self, settings: Settings, contract: Contract) -> None: + self.settings = settings + self.contract = contract + self._conns: dict[str, SnowflakeConnection] = {} + self._cache: dict[tuple[str, str], Observation] = {} + + def connection(self, persona: str) -> SnowflakeConnection: + if persona not in self._conns: + p = self.contract.personas[persona] + self._conns[persona] = cfg.connect(self.settings, user=p.user, role=p.role) + return self._conns[persona] + + def observe(self, persona: str, obj: AccessObject) -> Observation: + cache_key = (persona, obj.name) + if cache_key not in self._cache: + self._cache[cache_key] = self._query(persona, obj) + return self._cache[cache_key] + + def _query(self, persona: str, obj: AccessObject) -> Observation: + cur = self.connection(persona).cursor() + try: + cur.execute(f"SELECT * FROM {self.settings.database}.{obj.name}") + except ProgrammingError as exc: + return Observation(error=exc) + columns = [d[0] for d in cur.description] + rows = {} + for values in cur.fetchall(): + row = dict(zip(columns, values, strict=True)) + rows[tuple(row[k] for k in obj.key)] = row + return Observation(columns=columns, rows=rows) + + def invalidate(self) -> None: + """Forget cached results, e.g. after a policy change.""" + self._cache.clear() + + def close(self) -> None: + for conn in self._conns.values(): + conn.close() + self._conns.clear() + + +def _describe_error(exc: ProgrammingError) -> str: + return f"{exc.errno:06d} {str(exc.msg).splitlines()[-1].strip()}" + + +def _sample(keys: set, limit: int = 2) -> str: + return ", ".join(repr(k[0] if len(k) == 1 else k) for k in sorted(keys, key=repr)[:limit]) + + +class Matrix: + def __init__(self, settings: Settings, contract: Contract, data: Dataset) -> None: + self.contract = contract + self.oracle = Oracle(contract, data) + self.observer = Observer(settings, contract) + + def check(self, case: Case) -> list[str]: + """Return why ``case`` fails; an empty list means it passes.""" + obj = self.contract.objects[case.object] + obs = self.observer.observe(case.persona, obj) + if case.column is None: + return self._check_rows(case, obj, obs) + return self._check_column(case, obj, obs) + + def _check_rows(self, case: Case, obj: AccessObject, obs: Observation) -> list[str]: + if case.expect == "denied": + if obs.error is None: + return [f"expected an authorization error, but the query returned {len(obs.rows)} row(s)"] + if obs.error.errno not in NOT_AUTHORIZED: + return [f"expected an authorization error, got {_describe_error(obs.error)}"] + return [] + if obs.error is not None: + return [f"query failed: {_describe_error(obs.error)}"] + expected = set(self.oracle.visible_rows(case.persona, obj)) + actual = set(obs.rows) + problems = [] + if missing := expected - actual: + problems.append(f"{len(missing)} expected row(s) not visible, e.g. {_sample(missing)}") + if extra := actual - expected: + problems.append(f"{len(extra)} row(s) visible that should not be, e.g. {_sample(extra)}") + return problems + + def _check_column(self, case: Case, obj: AccessObject, obs: Observation) -> list[str]: + if obs.error is not None: + return [f"query failed: {_describe_error(obs.error)}"] + if case.column not in obs.columns: + return [f"column {case.column} is not in the result ({', '.join(obs.columns)})"] + expected_rows = self.oracle.visible_rows(case.persona, obj) + keys = expected_rows.keys() & obs.rows.keys() + if not keys: + return ["no visible rows to compare (see the rows case)"] + wrong = [] + for key in sorted(keys, key=repr): + want = self.oracle.expected_value(obj, case.column, case.expect, expected_rows[key]) + got = obs.rows[key][case.column] + if got != want: + wrong.append((key, want, got)) + if not wrong: + return [] + key, want, got = wrong[0] + return [ + f"{len(wrong)} of {len(keys)} row(s) not {case.expect}, " + f"e.g. {_sample({key}, 1)}: expected {want!r}, got {got!r}" + ] + + def run(self, cases: list[Case] | None = None) -> list[tuple[Case, list[str]]]: + return [(case, self.check(case)) for case in (cases or self.contract.cases())] + + def close(self) -> None: + self.observer.close() diff --git a/phi-guardrails/src/phi_guardrails/settings.py b/phi-guardrails/src/phi_guardrails/settings.py new file mode 100644 index 0000000..017a4dc --- /dev/null +++ b/phi-guardrails/src/phi_guardrails/settings.py @@ -0,0 +1,129 @@ +"""Connection settings, read from the environment in one place. + +``TEST_TARGET`` picks the target: + +* ``EMULATOR`` (default): the Snowflake-Next emulator on + ``snowflake.localhost.localstack.cloud:${SNOWFLAKE_PORT:-4566}``. Only the + port is configurable, so a stray cloud variable in the shell can never point + an emulator run at a real account. +* ``SNOWFLAKE_CLOUD``: a real account, from the ``SNOWFLAKE_*`` variables. + +Every connection uses key-pair (``SNOWFLAKE_JWT``) auth on both targets, so +the config that passes locally is the config that runs against a real account. +""" + +from __future__ import annotations + +import os +from dataclasses import dataclass, replace +from pathlib import Path + +import snowflake.connector +from snowflake.connector import SnowflakeConnection + +SAMPLE_ROOT = Path(__file__).resolve().parents[2] +EMULATOR_HOST = "snowflake.localhost.localstack.cloud" +QUERY_TAG = "phi-guardrails" + +# The connector's login telemetry probes for cloud platforms (it builds a boto3 +# client to look for AWS credentials, among others), which costs seconds per +# process and tells Snowflake nothing useful about a test run. Opt out unless +# the caller decided otherwise. +PLATFORM_DETECTION_ENV = "SNOWFLAKE_DISABLE_PLATFORM_DETECTION" +os.environ.setdefault(PLATFORM_DETECTION_ENV, "true") + + +@dataclass(frozen=True) +class Settings: + target: str + account: str + user: str + private_key_path: Path + role: str + warehouse: str + database: str + host: str | None = None + port: int | None = None + protocol: str | None = None + + @property + def is_emulator(self) -> bool: + return self.target == "EMULATOR" + + def with_database(self, database: str) -> Settings: + return replace(self, database=database) + + def connect_kwargs(self, *, user: str | None = None, role: str | None = None) -> dict: + kwargs: dict = { + "account": self.account, + "user": user or self.user, + "authenticator": "SNOWFLAKE_JWT", + "private_key_file": str(self.private_key_path), + "role": role or self.role, + "warehouse": self.warehouse, + "session_parameters": {"QUERY_TAG": QUERY_TAG}, + "login_timeout": 60, + } + if self.is_emulator: + # TLS is still verified; the OCSP revocation lookup is turned off + # because the endpoint is local (127.0.0.1). + kwargs.update(host=self.host, port=self.port, protocol=self.protocol, disable_ocsp_checks=True) + return kwargs + + +def _env(name: str, default: str) -> str: + """An environment variable, treating an empty value (``VAR=`` in .env) as unset.""" + return os.getenv(name) or default + + +def _required(name: str) -> str: + value = os.getenv(name) + if not value: + raise RuntimeError(f"{name} must be set when TEST_TARGET=SNOWFLAKE_CLOUD") + return value + + +def _key_path() -> Path: + path = Path(_env("SNOWFLAKE_PRIVATE_KEY_PATH", ".secrets/rsa_key.p8")) + return path if path.is_absolute() else SAMPLE_ROOT / path + + +def load() -> Settings: + target = _env("TEST_TARGET", "EMULATOR").upper() + database = _env("PHI_DATABASE", "PHI").upper() + if target == "EMULATOR": + return Settings( + target=target, + account="snowflake", + user="SAMPLE_SVC", + private_key_path=_key_path(), + role="ACCOUNTADMIN", + warehouse="PHI_WH", + database=database, + host=EMULATOR_HOST, + port=int(_env("SNOWFLAKE_PORT", "4566")), + protocol="https", + ) + if target == "SNOWFLAKE_CLOUD": + return Settings( + target=target, + account=_required("SNOWFLAKE_ACCOUNT"), + user=_required("SNOWFLAKE_USER"), + private_key_path=_key_path(), + role=_env("SNOWFLAKE_ROLE", "ACCOUNTADMIN"), + warehouse=_required("SNOWFLAKE_WAREHOUSE"), + database=database, + ) + raise RuntimeError(f"TEST_TARGET must be EMULATOR or SNOWFLAKE_CLOUD, got {target!r}") + + +def connect(settings: Settings, *, user: str | None = None, role: str | None = None) -> SnowflakeConnection: + """Open a session as ``user`` with primary role ``role`` (defaults: the deploy identity).""" + return snowflake.connector.connect(**settings.connect_kwargs(user=user, role=role)) + + +def public_key_body(settings: Settings) -> str: + """The deploy key's public half, in the form ``ALTER USER ... SET RSA_PUBLIC_KEY`` expects.""" + pub = settings.private_key_path.with_suffix(".pub") + lines = pub.read_text(encoding="ascii").splitlines() + return "".join(line for line in lines if not line.startswith("-----")) diff --git a/phi-guardrails/src/phi_guardrails/synthea.py b/phi-guardrails/src/phi_guardrails/synthea.py new file mode 100644 index 0000000..e3736f8 --- /dev/null +++ b/phi-guardrails/src/phi_guardrails/synthea.py @@ -0,0 +1,183 @@ +"""The pinned Synthea extract this sample loads, and the rows derived from it. + +Source: the Synthea "1K sample" CSV export, April 2020 build, published by the +Synthea project (MITRE) at synthetichealth.github.io. Synthea data is fully +synthetic: no real patient is described. The generator is Apache-2.0 licensed +and the project states its output "can be used without concern for legal or +privacy restrictions". The archive is downloaded at run time and verified by +SHA-256, never committed. + +The extract keeps three hospitals, their encounters since 2010, and the +patients seen there. This module is the single place that turns the CSVs into +table rows, so the loader and the test oracle can never disagree about the +input data. The oracle's expected values come from these rows and the access +contract, never from querying the policies under test. +""" + +from __future__ import annotations + +import csv +import hashlib +import io +import re +import unicodedata +import urllib.request +import zipfile +from collections import Counter, defaultdict +from dataclasses import dataclass +from datetime import date, datetime +from decimal import Decimal +from functools import cache +from pathlib import Path + +from phi_guardrails.settings import SAMPLE_ROOT + +DATASET_URL = "https://synthetichealth.github.io/synthea-sample-data/downloads/" "synthea_sample_data_csv_apr2020.zip" +DATASET_SHA256 = "4194b18c11eaedcf0d5d5dd448d8ac9661f14381e2ef9f109215dc42266cd38a" +CACHE_PATH = SAMPLE_ROOT / ".cache" / "synthea_sample_data_csv_apr2020.zip" + +# Facility id -> Synthea organization id. The ids are the sample's own stable +# keys; facility roles are named after them (FACILITY_101, ...). +FACILITY_ORGANIZATIONS = { + 101: "d692e283-0833-3201-8e55-4f868a9c0736", # HALLMARK HEALTH SYSTEM, Melrose + 102: "e002090d-4e92-300e-b41e-7d1f21dee4c6", # CAMBRIDGE HEALTH ALLIANCE, Cambridge + 103: "3d10019f-c88e-3de5-9916-6107b9c0263d", # NEWTON-WELLESLEY HOSPITAL, Newton +} +ENCOUNTERS_SINCE = "2010-01-01" + + +@dataclass(frozen=True) +class Dataset: + facilities: list[dict] + patients: list[dict] + encounters: list[dict] + + def table(self, name: str) -> list[dict]: + return { + "CLINICAL.FACILITIES": self.facilities, + "CLINICAL.PATIENTS": self.patients, + "CLINICAL.ENCOUNTERS": self.encounters, + }[name] + + +def download(path: Path = CACHE_PATH) -> Path: + """Fetch the archive once into ``.cache/`` and verify its checksum on every use.""" + if not path.exists(): + path.parent.mkdir(parents=True, exist_ok=True) + partial = path.with_suffix(".part") + with urllib.request.urlopen(DATASET_URL, timeout=120) as response: # noqa: S310 (pinned https URL) + partial.write_bytes(response.read()) + partial.replace(path) + digest = hashlib.sha256(path.read_bytes()).hexdigest() + if digest != DATASET_SHA256: + raise RuntimeError(f"{path} has SHA-256 {digest}, expected {DATASET_SHA256}. Delete it and retry.") + return path + + +def _read(archive: zipfile.ZipFile, name: str) -> list[dict]: + with archive.open(f"csv/{name}") as raw: + return list(csv.DictReader(io.TextIOWrapper(raw, encoding="utf-8"))) + + +def _text(value: str) -> str | None: + return value.strip() or None + + +def _date(value: str) -> date | None: + return date.fromisoformat(value) if value else None + + +def _timestamp(value: str) -> datetime: + # Synthea writes UTC instants ("2010-01-23T17:45:28Z"); the table stores TIMESTAMP_NTZ in UTC. + return datetime.fromisoformat(value.removesuffix("Z")) + + +def _name_part(value: str) -> str: + # Synthea appends digits to names ("Milo271"); emails use the plain ASCII letters. + ascii_only = unicodedata.normalize("NFKD", value).encode("ascii", "ignore").decode() + return re.sub(r"[^a-z]", "", ascii_only.lower()) + + +def patient_email(first: str, last: str) -> str: + """Synthetic contact address backfilled into the EMAIL column added by V1.3.0.""" + return f"{_name_part(first)}.{_name_part(last)}@example.org" + + +@cache +def load_dataset() -> Dataset: + with zipfile.ZipFile(download()) as archive: + organizations = {row["Id"]: row for row in _read(archive, "organizations.csv")} + facility_of = {org: fid for fid, org in FACILITY_ORGANIZATIONS.items()} + raw_encounters = [ + row + for row in _read(archive, "encounters.csv") + if row["ORGANIZATION"] in facility_of and row["START"] >= ENCOUNTERS_SINCE + ] + raw_patients = {row["Id"]: row for row in _read(archive, "patients.csv")} + + facilities = [ + { + "FACILITY_ID": fid, + "NAME": org["NAME"], + "ADDRESS": org["ADDRESS"], + "CITY": org["CITY"], + "STATE": org["STATE"], + "ZIP": org["ZIP"], + "PHONE": org["PHONE"], + "SOURCE_ORGANIZATION_ID": org_id, + } + for fid, org_id in sorted(FACILITY_ORGANIZATIONS.items()) + for org in [organizations[org_id]] + ] + + encounters = [ + { + "ENCOUNTER_ID": row["Id"], + "PATIENT_ID": row["PATIENT"], + "FACILITY_ID": facility_of[row["ORGANIZATION"]], + "START_TS": _timestamp(row["START"]), + "STOP_TS": _timestamp(row["STOP"]), + "ENCOUNTER_CLASS": row["ENCOUNTERCLASS"], + "CODE": row["CODE"], + "DESCRIPTION": row["DESCRIPTION"], + "REASON_CODE": _text(row["REASONCODE"]), + "REASON_DESCRIPTION": _text(row["REASONDESCRIPTION"]), + "TOTAL_CLAIM_COST": Decimal(row["TOTAL_CLAIM_COST"]).quantize(Decimal("0.01")), + "PAYER_COVERAGE": Decimal(row["PAYER_COVERAGE"]).quantize(Decimal("0.01")), + } + for row in sorted(raw_encounters, key=lambda r: r["Id"]) + ] + + # A patient's primary facility is where most of their encounters happened + # (ties go to the lowest facility id). + visits: dict[str, Counter] = defaultdict(Counter) + for enc in encounters: + visits[enc["PATIENT_ID"]][enc["FACILITY_ID"]] += 1 + patients = [] + for patient_id in sorted(visits): + row = raw_patients[patient_id] + counts = visits[patient_id] + patients.append( + { + "PATIENT_ID": patient_id, + "PRIMARY_FACILITY_ID": min(counts, key=lambda fid: (-counts[fid], fid)), + "FIRST_NAME": row["FIRST"], + "LAST_NAME": row["LAST"], + "BIRTH_DATE": _date(row["BIRTHDATE"]), + "DEATH_DATE": _date(row["DEATHDATE"]), + "SSN": row["SSN"], + "DRIVERS_LICENSE": _text(row["DRIVERS"]), + "PASSPORT": _text(row["PASSPORT"]), + "GENDER": row["GENDER"], + "RACE": row["RACE"], + "ETHNICITY": row["ETHNICITY"], + "MARITAL_STATUS": _text(row["MARITAL"]), + "ADDRESS": row["ADDRESS"], + "CITY": row["CITY"], + "COUNTY": row["COUNTY"], + "STATE": row["STATE"], + "ZIP": _text(row["ZIP"]), + "EMAIL": patient_email(row["FIRST"], row["LAST"]), + } + ) + return Dataset(facilities=facilities, patients=patients, encounters=encounters) diff --git a/phi-guardrails/tests/conftest.py b/phi-guardrails/tests/conftest.py new file mode 100644 index 0000000..04aa667 --- /dev/null +++ b/phi-guardrails/tests/conftest.py @@ -0,0 +1,78 @@ +"""Shared fixtures: one freshly deployed database per test session. + +The session deploys the real migrations with schemachange into +``PHI_TEST_``, loads the pinned Synthea extract, and drops the database +afterwards (set ``KEEP_TEST_DB=1`` to keep it for debugging). Roles and users +are account-level and created idempotently by V1.0.0, so sessions do not +interfere with each other or with ``make migrate``. +""" + +from __future__ import annotations + +import os +import uuid +from collections.abc import Iterator + +import pytest +from snowflake.connector import SnowflakeConnection + +from phi_guardrails import settings as cfg +from phi_guardrails.contract import Contract, load_contract +from phi_guardrails.deploy import deploy +from phi_guardrails.load import load +from phi_guardrails.matrix import Matrix +from phi_guardrails.settings import Settings +from phi_guardrails.synthea import Dataset, load_dataset + + +def unique(prefix: str) -> str: + return f"{prefix}_{uuid.uuid4().hex[:8].upper()}" + + +@pytest.fixture(scope="session") +def contract() -> Contract: + return load_contract() + + +@pytest.fixture(scope="session") +def dataset() -> Dataset: + return load_dataset() + + +@pytest.fixture(scope="session") +def deployed() -> Iterator[Settings]: + """Settings pointing at a database with every migration applied and data loaded.""" + settings = cfg.load().with_database(unique("PHI_TEST")) + try: + deploy(settings) + load(settings) + yield settings + finally: + if not os.getenv("KEEP_TEST_DB"): + with cfg.connect(settings) as conn: + conn.cursor().execute(f"DROP DATABASE IF EXISTS {settings.database}") + + +@pytest.fixture(scope="session") +def admin(deployed: Settings) -> Iterator[SnowflakeConnection]: + """The deploy identity, with the test database as the current database.""" + conn = cfg.connect(deployed) + conn.cursor().execute(f"USE DATABASE {deployed.database}") + yield conn + conn.close() + + +@pytest.fixture(scope="session") +def matrix(deployed: Settings, contract: Contract, dataset: Dataset) -> Iterator[Matrix]: + m = Matrix(deployed, contract, dataset) + yield m + m.close() + + +@pytest.fixture +def scratch_schema(admin: SnowflakeConnection, deployed: Settings) -> Iterator[str]: + """A throwaway schema in the test database, as ``DB.SCHEMA``.""" + name = f"{deployed.database}.{unique('SCRATCH')}" + admin.cursor().execute(f"CREATE SCHEMA {name}") + yield name + admin.cursor().execute(f"DROP SCHEMA IF EXISTS {name} CASCADE") diff --git a/phi-guardrails/tests/test_access_matrix.py b/phi-guardrails/tests/test_access_matrix.py new file mode 100644 index 0000000..9104765 --- /dev/null +++ b/phi-guardrails/tests/test_access_matrix.py @@ -0,0 +1,80 @@ +"""The access matrix: one test per cell of access-contract.yml, plus the +behaviours around it (regression detection and tag-only masking).""" + +from __future__ import annotations + +import pytest +from snowflake.connector import SnowflakeConnection + +from phi_guardrails.contract import VISIBLE, Case, Contract, load_contract +from phi_guardrails.drill import temporarily_applied +from phi_guardrails.matrix import Matrix +from phi_guardrails.settings import Settings +from phi_guardrails.synthea import Dataset + + +@pytest.mark.parametrize("case", load_contract().cases(), ids=lambda c: c.id) +def test_contract_cell(matrix: Matrix, case: Case) -> None: + failures = matrix.check(case) + assert not failures, f"{case.id}: " + "; ".join(failures) + + +def test_matrix_covers_every_cell(contract: Contract) -> None: + """The generated cases are exactly the contract's cells: 5 objects x 5 personas + rows cells, plus one column cell per persona that can see an object's rows.""" + expected = sum( + len(contract.personas) + sum(1 for o in obj.rows.values() if o in VISIBLE) * len(obj.columns) + for obj in contract.objects.values() + ) + assert len(contract.cases()) == expected == 174 + + +def test_broken_current_role_policy_is_caught( + matrix: Matrix, admin: SnowflakeConnection, deployed: Settings, contract: Contract, dataset: Dataset +) -> None: + """Headline regression: demos/broken-current-role-policy.sql replaces + IS_ROLE_IN_SESSION('CLINICIAN') with CURRENT_ROLE() = 'CLINICIAN' in the + string masking policy. The matrix must fail exactly the clinical lead's + clear text-PHI cells, and nothing else.""" + text_phi = { + (name, col.name) + for name, obj in contract.objects.items() + if obj.is_table + for col in obj.columns.values() + if col.phi_type and isinstance(next(r[col.name] for r in dataset.table(name) if r[col.name]), str) + } + expected = { + ("CLINICIAN_LEAD", obj, col) + for obj, col in text_phi + if contract.objects[obj].columns[col].access.get("CLINICIAN_LEAD") == "clear" + } + # The text PHI columns are all on PATIENTS: two names, SSN, licence and + # passport numbers, three address parts, ZIP and EMAIL. + assert {obj for _, obj, _ in expected} == {"CLINICAL.PATIENTS"} and len(expected) == 10 + + try: + with temporarily_applied(admin, deployed.database): + matrix.observer.invalidate() + failing = {(case.persona, case.object, case.column) for case, failures in matrix.run() if failures} + finally: + matrix.observer.invalidate() + assert failing == expected + + +def test_patient_email_is_masked_by_its_tag_alone(admin: SnowflakeConnection, deployed: Settings) -> None: + """V1.3.0 adds EMAIL with a tag and no policy code: its only masking policy + is the one the PHI_TYPE tag carries, and GOVERNANCE still has the three + policies V1.1.0 created.""" + db = deployed.database + refs = ( + admin.cursor() + .execute( + f"SELECT POLICY_NAME, TAG_NAME FROM TABLE({db}.INFORMATION_SCHEMA.POLICY_REFERENCES(" + f"REF_ENTITY_NAME => '{db}.CLINICAL.PATIENTS', REF_ENTITY_DOMAIN => 'table')) " + "WHERE POLICY_KIND = 'MASKING_POLICY' AND REF_COLUMN_NAME = 'EMAIL'" + ) + .fetchall() + ) + assert refs == [("PHI_MASK_STRING", "PHI_TYPE")] + policies = {row[1] for row in admin.cursor().execute(f"SHOW MASKING POLICIES IN SCHEMA {db}.GOVERNANCE").fetchall()} + assert policies == {"PHI_MASK_STRING", "PHI_MASK_DATE", "PHI_MASK_TIMESTAMP"} diff --git a/phi-guardrails/tests/test_deploy.py b/phi-guardrails/tests/test_deploy.py new file mode 100644 index 0000000..e76f657 --- /dev/null +++ b/phi-guardrails/tests/test_deploy.py @@ -0,0 +1,32 @@ +"""schemachange deploys: change history and idempotency.""" + +from __future__ import annotations + +from snowflake.connector import SnowflakeConnection + +from phi_guardrails.deploy import change_history_table, deploy +from phi_guardrails.settings import SAMPLE_ROOT, Settings + +MIGRATIONS = sorted(p.name for p in (SAMPLE_ROOT / "migrations").glob("*.sql")) + + +def _history(admin: SnowflakeConnection, database: str) -> list[tuple]: + return ( + admin.cursor() + .execute(f"SELECT VERSION, SCRIPT, STATUS FROM {change_history_table(database)} ORDER BY INSTALLED_ON, SCRIPT") + .fetchall() + ) + + +def test_change_history_lists_every_migration(admin: SnowflakeConnection, deployed: Settings) -> None: + history = _history(admin, deployed.database) + assert sorted(script for _, script, _ in history) == MIGRATIONS + assert {status for _, _, status in history} == {"Success"} + versions = [version for version, script, _ in history if script.startswith("V")] + assert versions == ["1.0.0", "1.1.0", "1.2.0", "1.3.0"] + + +def test_redeploy_applies_nothing(admin: SnowflakeConnection, deployed: Settings) -> None: + before = _history(admin, deployed.database) + deploy(deployed) + assert _history(admin, deployed.database) == before diff --git a/phi-guardrails/tests/test_load.py b/phi-guardrails/tests/test_load.py new file mode 100644 index 0000000..3ddf8ef --- /dev/null +++ b/phi-guardrails/tests/test_load.py @@ -0,0 +1,21 @@ +"""Loading the Synthea extract is idempotent.""" + +from __future__ import annotations + +from phi_guardrails import settings as cfg +from phi_guardrails.load import LOAD_ORDER, load +from phi_guardrails.settings import Settings +from phi_guardrails.synthea import Dataset + + +def test_reloading_leaves_exactly_one_copy(deployed: Settings, dataset: Dataset) -> None: + load(deployed, dataset) + load(deployed, dataset) + + # AUDITOR passes the facility row access policy, so it counts every row. + with cfg.connect(deployed, user="PERSONA_AUDITOR", role="AUDITOR") as conn: + cur = conn.cursor() + cur.execute(f"USE DATABASE {deployed.database}") + counts = {table: cur.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0] for table in LOAD_ORDER} + + assert counts == {table: len(dataset.table(table)) for table in LOAD_ORDER} diff --git a/phi-guardrails/tests/test_policy_coverage.py b/phi-guardrails/tests/test_policy_coverage.py new file mode 100644 index 0000000..537465e --- /dev/null +++ b/phi-guardrails/tests/test_policy_coverage.py @@ -0,0 +1,137 @@ +"""The policy-coverage gate and the classification rules around it.""" + +from __future__ import annotations + +import subprocess +import sys + +import pytest +from conftest import unique +from snowflake.connector import SnowflakeConnection +from snowflake.connector.errors import ProgrammingError + +from phi_guardrails import coverage +from phi_guardrails.contract import Contract +from phi_guardrails.settings import SAMPLE_ROOT, Settings + + +def _contract_phi_columns(contract: Contract) -> dict[tuple[str, str], str]: + return { + (name, col.name): col.phi_type + for name, obj in contract.objects.items() + if obj.is_table + for col in obj.columns.values() + if col.phi_type + } + + +def test_every_tagged_column_has_a_masking_policy(admin: SnowflakeConnection, deployed: Settings) -> None: + columns = coverage.scan(admin, deployed.database) + assert coverage.uncovered(columns) == [] + assert len(columns) == 14 # 12 on PATIENTS (EMAIL included) + START_TS, STOP_TS + + +def test_tags_match_the_contract(admin: SnowflakeConnection, deployed: Settings, contract: Contract) -> None: + """Every column the contract classifies as PHI carries exactly that tag + value, and no other column is tagged.""" + tagged = {(c.table, c.column): c.phi_type for c in coverage.scan(admin, deployed.database)} + assert tagged == _contract_phi_columns(contract) + + +def test_every_table_column_is_classified_in_the_contract( + admin: SnowflakeConnection, deployed: Settings, contract: Contract +) -> None: + """A new column cannot slip in unclassified: each column of a CLINICAL + table needs a contract entry (with phi_type, or phi_type: null).""" + rows = ( + admin.cursor() + .execute( + "SELECT TABLE_SCHEMA || '.' || TABLE_NAME, COLUMN_NAME " + f"FROM {deployed.database}.INFORMATION_SCHEMA.COLUMNS " + "WHERE TABLE_SCHEMA = 'CLINICAL'" + ) + .fetchall() + ) + in_database = set(rows) + in_contract = {(name, col) for name, obj in contract.objects.items() if obj.is_table for col in obj.columns} + assert in_database == in_contract + + +def test_gate_fails_on_a_tagged_but_unprotected_column( + admin: SnowflakeConnection, deployed: Settings, scratch_schema: str +) -> None: + """Tag-based masking covers only the data types the tag has policies for. + An MRN stored as NUMBER and a timestamp stored as TIMESTAMP_TZ keep their + tags but stay in clear; the gate must name exactly those two columns.""" + db = deployed.database + schema = scratch_schema.split(".")[1] + tag = f"{db}.GOVERNANCE.PHI_TYPE" + cur = admin.cursor() + cur.execute( + f"CREATE TABLE {scratch_schema}.LAB_RESULTS (" + f"MRN NUMBER(10, 0) WITH TAG ({tag} = 'mrn'), " + f"COLLECTED_AT TIMESTAMP_TZ WITH TAG ({tag} = 'service_date'), " + f"PATIENT_NAME VARCHAR(100) WITH TAG ({tag} = 'name'), " + "RESULT VARCHAR(100))" + ) + cur.execute( + f"INSERT INTO {scratch_schema}.LAB_RESULTS " + "SELECT 4711, '2020-03-04 05:06:07 +01:00'::TIMESTAMP_TZ, 'Milo Feil', 'negative'" + ) + + # The leak the gate exists for: the deploy role is not a clinician, so the + # VARCHAR name is masked for it, but the NUMBER and TIMESTAMP_TZ values are not. + mrn, name = cur.execute(f"SELECT MRN, PATIENT_NAME FROM {scratch_schema}.LAB_RESULTS").fetchone() + assert (mrn, name) == (4711, "***MASKED***") + + gaps = {(c.table, c.column, c.data_type) for c in coverage.uncovered(coverage.scan(admin, db))} + assert gaps == { + (f"{schema}.LAB_RESULTS", "MRN", "NUMBER"), + (f"{schema}.LAB_RESULTS", "COLLECTED_AT", "TIMESTAMP_TZ"), + } + + # The CLI is what CI runs: it must exit non-zero and name the columns. + result = subprocess.run( + [sys.executable, "-m", "phi_guardrails.coverage", "--database", db], + capture_output=True, + text=True, + cwd=SAMPLE_ROOT, + check=False, + ) + assert result.returncode == 1, result.stdout + result.stderr + assert f"{schema}.LAB_RESULTS.MRN" in result.stdout + assert "2 without a masking policy" in result.stdout + + +def test_tag_rejects_an_unknown_phi_type(admin: SnowflakeConnection, deployed: Settings, scratch_schema: str) -> None: + admin.cursor().execute(f"CREATE TABLE {scratch_schema}.NOTES (BODY VARCHAR)") + with pytest.raises(ProgrammingError) as excinfo: + admin.cursor().execute( + f"ALTER TABLE {scratch_schema}.NOTES MODIFY COLUMN BODY " + f"SET TAG {deployed.database}.GOVERNANCE.PHI_TYPE = 'diagnosis'" + ) + assert excinfo.value.errno == 391826, excinfo.value # value not in ALLOWED_VALUES + + +def test_tag_carries_one_masking_policy_per_data_type( + admin: SnowflakeConnection, deployed: Settings, scratch_schema: str +) -> None: + """A second VARCHAR policy cannot be attached to the tag, so there is never + doubt about which policy masks a text column.""" + policy = f"{scratch_schema}.{unique('SHADOW_MASK')}" + admin.cursor().execute(f"CREATE MASKING POLICY {policy} AS (val VARCHAR) RETURNS VARCHAR -> val") + with pytest.raises(ProgrammingError) as excinfo: + admin.cursor().execute(f"ALTER TAG {deployed.database}.GOVERNANCE.PHI_TYPE SET MASKING POLICY {policy}") + assert excinfo.value.errno == 391837, excinfo.value + + +def test_attached_policy_cannot_be_replaced(admin: SnowflakeConnection, scratch_schema: str) -> None: + """Why the migrations use CREATE ... IF NOT EXISTS and change policies with + ALTER ... SET BODY: CREATE OR REPLACE on an attached policy fails.""" + cur = admin.cursor() + cur.execute(f"CREATE TABLE {scratch_schema}.T (V VARCHAR)") + cur.execute(f"CREATE MASKING POLICY {scratch_schema}.P AS (val VARCHAR) RETURNS VARCHAR -> '***'") + cur.execute(f"ALTER TABLE {scratch_schema}.T MODIFY COLUMN V SET MASKING POLICY {scratch_schema}.P") + with pytest.raises(ProgrammingError) as excinfo: + cur.execute(f"CREATE OR REPLACE MASKING POLICY {scratch_schema}.P AS (val VARCHAR) RETURNS VARCHAR -> val") + assert excinfo.value.errno == 3531, excinfo.value diff --git a/phi-guardrails/uv.lock b/phi-guardrails/uv.lock new file mode 100644 index 0000000..3e6f3e2 --- /dev/null +++ b/phi-guardrails/uv.lock @@ -0,0 +1,523 @@ +version = 1 +revision = 3 +requires-python = "==3.12.*" + +[[package]] +name = "asn1crypto" +version = "1.5.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/de/cf/d547feed25b5244fcb9392e288ff9fdc3280b10260362fc45d37a798a6ee/asn1crypto-1.5.1.tar.gz", hash = "sha256:13ae38502be632115abf8a24cbe5f4da52e3b5231990aff31123c805306ccb9c", size = 121080, upload-time = "2022-03-15T14:46:52.889Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c9/7f/09065fd9e27da0eda08b4d6897f1c13535066174cc023af248fc2a8d5e5a/asn1crypto-1.5.1-py2.py3-none-any.whl", hash = "sha256:db4e40728b728508912cbb3d44f19ce188f218e9eba635821bb4b68564f8fd67", size = 105045, upload-time = "2022-03-15T14:46:51.055Z" }, +] + +[[package]] +name = "boto3" +version = "1.43.118" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, + { name = "jmespath" }, + { name = "s3transfer" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/96/8b/5694c8433b5202f34a82def4edee274f7df1eba2919c66d9d389d2202ec8/boto3-1.43.118.tar.gz", hash = "sha256:c0620f799e0f0c222ba42c337b65cdbcd3da38a3121187f634d8926cbbb28a9c", size = 112649, upload-time = "2026-10-22T19:22:09.629Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bb/a3/65bb3d64928f3c2da17640b92044084150317f02adf22442cb0c52af2a8b/boto3-1.43.118-py3-none-any.whl", hash = "sha256:ab8f3d2c622ef039338b4f228d97ea55f1abd573d84bb9b9a5ee8cb6780c0ccd", size = 140045, upload-time = "2026-10-22T19:22:08.353Z" }, +] + +[[package]] +name = "botocore" +version = "1.43.118" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jmespath" }, + { name = "python-dateutil" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d4/b8/3e55154ddaf1a6d1d4ddda85c0e5983205a3b7657edacd8ebba8c6fd774c/botocore-1.43.118.tar.gz", hash = "sha256:7f3f9d88cc9e847587690f82a42e3c3c292c3026c92c1e60d68a9310018e003f", size = 16420126, upload-time = "2026-10-22T19:22:04.81Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9c/ce/61b39a8e7a69af5ea43c1c3e1b0065a4866b42ccddf107e16686a3143614/botocore-1.43.118-py3-none-any.whl", hash = "sha256:a45d7ae570674161e4a3542d8c68633bf7503f92d1de781ef7361b79488eda6f", size = 16122016, upload-time = "2026-10-22T19:22:00.414Z" }, +] + +[[package]] +name = "certifi" +version = "2026.10.23" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fd/11/b524cbd9bbdaca58608e4cf4053fd463417549e1959fa8d3aa18319f9555/certifi-2026.10.23.tar.gz", hash = "sha256:a9895e6e2758b371e0545e321a3a8fd1fcfd9829ada8824a936dfb16aaaf36cd", size = 136225, upload-time = "2026-10-23T11:36:16.478Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/d1/a3652fe2c19134d0d15922177821d863cbd7e842d53a21bbe7e8e3a1b5ac/certifi-2026.10.23-py3-none-any.whl", hash = "sha256:e9b54de76e530bb6cf0ccd036787103acb58fd42ba930ea8bf948ddbaa2f61d1", size = 135082, upload-time = "2026-10-23T11:36:14.763Z" }, +] + +[[package]] +name = "cffi" +version = "2.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pycparser", marker = "implementation_name != 'PyPy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9e/ef/008a1939e372c06329a3fce4279c02f328488f3526744906eeec3da7ad5f/cffi-2.1.1.tar.gz", hash = "sha256:dd31f52ea1086513bb9df30f8fcee9b8918323ae067a3d5b78bc826a000712be", size = 530807, upload-time = "2026-08-03T21:21:18.939Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/10/69/43965eccfdead3b9220015fd1320e117be8c6ed01a62ffab76eeb752f5d5/cffi-2.1.1-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:c8c69575568085ba0b1b10c0249d779a214aea6f6522e949a0fc9fb0fcb449d0", size = 184821, upload-time = "2026-08-03T21:19:44.887Z" }, + { url = "https://files.pythonhosted.org/packages/54/7d/16e5a096677b5e313ca80cd5e5170efa3ea44624a82bb111925522da64b1/cffi-2.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f81b3b8f3d4e343550fa4baa0e479bba9f2d29ce9c2e9b51d1ce1718d7442fcf", size = 184719, upload-time = "2026-08-03T21:19:46.129Z" }, + { url = "https://files.pythonhosted.org/packages/56/e6/8941622732edec876dd17d0453dce07317ae96db34f2ec1436c9d3785986/cffi-2.1.1-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:811bd1e21d32de12efca32393a0ab3f5133b54fce9bd44b8bd77ab07da14bf6a", size = 214799, upload-time = "2026-08-03T21:19:47.218Z" }, + { url = "https://files.pythonhosted.org/packages/44/de/f98430906df1545ffde0d543dd124a7a439bc2cd32b36b9c53f805df7333/cffi-2.1.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:68e62fe11f30d5ca8289242866f0a5291402d8529ca2178ab8afc5c9694ae890", size = 222389, upload-time = "2026-08-03T21:19:48.331Z" }, + { url = "https://files.pythonhosted.org/packages/6a/5b/717f1526b9957b34456313c31645c5b82b8fb5c3fe9e4752999be7128bfc/cffi-2.1.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:4a7c934f7360e8cd64fe9efadcbd10c7c6364f531e432b9a4bf5ccbc9e0e8b50", size = 210249, upload-time = "2026-08-03T21:19:49.543Z" }, + { url = "https://files.pythonhosted.org/packages/64/b3/f8aa4f3e34986c7e4ec45072d1b1b9dd295b6b18007b45518d79726dd725/cffi-2.1.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:3143d81e29e1e20a9ce10901ec369012947876596f75a222235965f2b7ae832e", size = 208775, upload-time = "2026-08-03T21:19:50.918Z" }, + { url = "https://files.pythonhosted.org/packages/b1/db/dceb9dd5b231e1da801793f8acc9f3c52a7e1afe40bb1aae37e02b0faad5/cffi-2.1.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c1453022f490d2459a11819d83ad1d586e9ff65a12ac3e705ffebd46d3685dcf", size = 221822, upload-time = "2026-08-03T21:19:52.054Z" }, + { url = "https://files.pythonhosted.org/packages/a0/d2/6cd24ae3be000a634109c247d1475d62e5616d0dc78c82770942ec384248/cffi-2.1.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:208f941bb9d18e768138677f0a6d2ce01f590df56043dda1df1535ac57c88517", size = 225232, upload-time = "2026-08-03T21:19:53.109Z" }, + { url = "https://files.pythonhosted.org/packages/cb/52/3fa190537004dd7f0ab860a6dc7c0175b8667f68d1e618a46f5498d30250/cffi-2.1.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:210019b6c7cf07f081b4c54635c8cf744377001350e29cc0f81c4377b4797735", size = 223597, upload-time = "2026-08-03T21:19:54.515Z" }, + { url = "https://files.pythonhosted.org/packages/80/fb/0bb75b7039588c074b37ae99f40d9bfddf990ecb2fbc346ebccd2e56b9be/cffi-2.1.1-cp312-cp312-win32.whl", hash = "sha256:046bfc24911b37851ee1b51aab8bffe713d89c68c6a057b09484ce9fd5f69b4e", size = 175292, upload-time = "2026-08-03T21:19:55.566Z" }, + { url = "https://files.pythonhosted.org/packages/d9/79/615cc094e2fb508cade7de88d3b4f6c4ec2bab695c97bce9153dc65aadf5/cffi-2.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:f53e442b08449d42821fa4a4fba000095af9f62742a500f978a9f557ec44339a", size = 185919, upload-time = "2026-08-03T21:19:56.89Z" }, + { url = "https://files.pythonhosted.org/packages/70/c6/d0ea84713fe46b243a436a18fcd47d639732747e21635c8a27191b06dc30/cffi-2.1.1-cp312-cp312-win_arm64.whl", hash = "sha256:7bde5e4cc5c10140859842b9d383af292b22639a4dffb725314baf45968cef80", size = 180093, upload-time = "2026-08-03T21:19:58.155Z" }, +] + +[[package]] +name = "charset-normalizer" +version = "3.5.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/33/1c/f41d4e74c28ab327ff3acd36053f7ea506c55872d7a90b0fa71aa3ab0c89/charset_normalizer-3.5.2.tar.gz", hash = "sha256:39de2a259fc954455c57274dc94c79d5842774e1247a016aff30bc0efed0f4ef", size = 172659, upload-time = "2026-09-30T04:39:23.398Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e7/c8/693809898870237d82785a03f3b2b58fe4c9f14669f84a7d4e623c92a59e/charset_normalizer-3.5.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:ed2a239c0ea213acc1908150a3037257083c7c083128f1a4cec2ec4b97dca491", size = 367780, upload-time = "2026-09-30T04:35:30.888Z" }, + { url = "https://files.pythonhosted.org/packages/c9/87/2fea8c13dc24b3ca9c6f803a5b2dfdeae73eb4f9e12c7885ed908ff0433c/charset_normalizer-3.5.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b91363207bd9dc966a691e959bb47f64b30f7ac4b072be9968b366982f7db77c", size = 246730, upload-time = "2026-09-30T04:35:32.286Z" }, + { url = "https://files.pythonhosted.org/packages/a8/9e/09efac30b937722f46d3110ba30b875b24b2e3a266ed746cc4e376a94d80/charset_normalizer-3.5.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:38a873987f3be698494da8b2e3085e29da02da7b633dce73e79c699a113d7bf0", size = 237707, upload-time = "2026-09-30T04:35:33.709Z" }, + { url = "https://files.pythonhosted.org/packages/9e/18/70d76670b13686237863a379928d60bd10e021f17d243ab3d7014c4a5f4e/charset_normalizer-3.5.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:355ad8011081dec5412240c087a9a0c9d4d5039f3ed11a3f13e18c2b29b56c51", size = 273050, upload-time = "2026-09-30T04:35:35.138Z" }, + { url = "https://files.pythonhosted.org/packages/54/e2/77a8b09d5adc013ed07b95b01b8b8fa5441c4e810e83ee7e4aae2fa4d91a/charset_normalizer-3.5.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ee21e28f0430bd6dc9086c6e525d5e818a44a5ad19720c8a0ef766792f3eb5e5", size = 270345, upload-time = "2026-09-30T04:35:36.502Z" }, + { url = "https://files.pythonhosted.org/packages/7f/c5/38806a25ab5e65fc178f39affeda20858efafede2fce1ffc2556cfc9fe73/charset_normalizer-3.5.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3d31298449090ab8d47b7b1b2a555ff73cac7ed438a08b7ac160980c7ebed649", size = 257601, upload-time = "2026-09-30T04:35:37.919Z" }, + { url = "https://files.pythonhosted.org/packages/ae/8d/213565184708fdb263ae55e2c04ee1ff748129dd65d48ed0e3502da9c85a/charset_normalizer-3.5.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5cde776b7cc66e4f6c99612cea4aa7269aa65863f7a15841b2c264f103822f4e", size = 252222, upload-time = "2026-09-30T04:35:39.544Z" }, + { url = "https://files.pythonhosted.org/packages/7e/24/76d2cefc25472531e4c5c7dfff68865eb1c39b78482f0fdc15b46f047830/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ae4f5fea5b8b8ccff88238cc8569303e5ee95efae67fa62922a311397a71f346", size = 248482, upload-time = "2026-09-30T04:35:41.088Z" }, + { url = "https://files.pythonhosted.org/packages/7d/dc/65a801b66ab4c197e22c433ab25e7ac24324ac6f45a2269aca42cce309bf/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:f7d486c83842422badd511868fd8a9a20e9407ace71564b6af47ce7e60a336c1", size = 241206, upload-time = "2026-09-30T04:35:42.59Z" }, + { url = "https://files.pythonhosted.org/packages/a7/95/ca9b5eabde673002c6f1e7ada1b223916fe18f6d661da7aabd4d643718f1/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:11a4d68a6ecda3292cb1e50239e111543ba5d709bb62a6b4ea1afcfa729d8875", size = 273190, upload-time = "2026-09-30T04:35:44.347Z" }, + { url = "https://files.pythonhosted.org/packages/2d/8b/803b4d2a3f6e1740f63f1e87b04d14b42f3d4fdfe6ed7d4db2d34102b14f/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:d6734d2ef8a50fbf8445c139477da401f50d62a0606bf00e20ec6d87773fefb1", size = 253527, upload-time = "2026-09-30T04:35:45.915Z" }, + { url = "https://files.pythonhosted.org/packages/a9/55/93c0e5dbd085ae0471346026abbe7e0db9ea2d6fea74e51f0b5a46f233a7/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:a815775b6c38d4e0ff7bcffbeba67feded90202bb6a226b8dd35f1c855217413", size = 271285, upload-time = "2026-09-30T04:35:47.49Z" }, + { url = "https://files.pythonhosted.org/packages/95/69/0dbd0e0b9b16cfa816cdfcb3e2e3854a1f680dc07fb1245ea125e7448060/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:23851fb4e1b85ed3f6c2a27b777cdfe2e19fb5b38429a8faf38c7542b7665869", size = 260010, upload-time = "2026-09-30T04:35:48.996Z" }, + { url = "https://files.pythonhosted.org/packages/58/9d/e7b88e7b1bf403590c3b573277b5e1e488c68c7a6fbacca310a2c324e90c/charset_normalizer-3.5.2-cp312-cp312-win32.whl", hash = "sha256:db19d07e2e0129e974a0e65d0064fc222a446cd5122c2fd4184d2af9fc734a9e", size = 184125, upload-time = "2026-09-30T04:35:50.777Z" }, + { url = "https://files.pythonhosted.org/packages/eb/e6/e6e083884cbcfd49c64865af05027fe7011be7b2d9179524f099a1b611f3/charset_normalizer-3.5.2-cp312-cp312-win_amd64.whl", hash = "sha256:780fbe7cab297b81dad9fb8dc5eb003c0468ffb0d9e5f65068c53a34661a96bc", size = 207486, upload-time = "2026-09-30T04:35:52.194Z" }, + { url = "https://files.pythonhosted.org/packages/c4/e3/017aea0911ada7405a825c7d937eb3a13009664e2f5b38e8c4bbf2abf894/charset_normalizer-3.5.2-cp312-cp312-win_arm64.whl", hash = "sha256:e2af3aad578aa6bd1384bcf4750fc285e5a9de53f40b7d41e5a0bf748edeb2b3", size = 196734, upload-time = "2026-09-30T04:35:53.636Z" }, + { url = "https://files.pythonhosted.org/packages/8c/ab/176fbfd5b64939c55d652366aa5b9ef1d767af207a3aa6ebeb0d226c484d/charset_normalizer-3.5.2-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:4275811936e2f06feff5e598fb42a1b7ae852da8e39605211892b56b81a34efd", size = 331815, upload-time = "2026-09-30T04:38:26.216Z" }, + { url = "https://files.pythonhosted.org/packages/7e/84/371eac6b30bdbcbf2d632a1a01809103459216fcaae61b8b8d922c1bfb8a/charset_normalizer-3.5.2-cp37-abi3-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:1c50fe28bbc2ced33386f298650d91218076c05420e6cbd790b913adc41659e7", size = 253276, upload-time = "2026-09-30T04:38:28.032Z" }, + { url = "https://files.pythonhosted.org/packages/43/6f/c4fbae58febff71709c51bc7e18fdfa55341dc382704740f9f0cbf03817b/charset_normalizer-3.5.2-cp37-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d19fbd981a488e22cd04883659ca6b08f50b5974f9fd7c95655ef6a043e5893f", size = 241239, upload-time = "2026-09-30T04:38:29.732Z" }, + { url = "https://files.pythonhosted.org/packages/61/71/458c3f42164a07d0c5210798e9e704b39e540a6793b05aba67f3a35243a9/charset_normalizer-3.5.2-cp37-abi3-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0fed1d06615f022ee3b13caf5e8b180cfea32bb2c5aded8a9d44277afc040f93", size = 231121, upload-time = "2026-09-30T04:38:31.462Z" }, + { url = "https://files.pythonhosted.org/packages/09/54/ab9e89367076f6331bb6c65c4bf14a5361fa5191cb6561bf534f18504e1b/charset_normalizer-3.5.2-cp37-abi3-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:838dcc90063569a0448120554591a1d6c4a4ffe11babf048908793154ab86ade", size = 260350, upload-time = "2026-09-30T04:38:33.239Z" }, + { url = "https://files.pythonhosted.org/packages/7c/c1/061431ecc688d9d76602502cb57cc01e691e682c18f1beb45f9673b5bbd2/charset_normalizer-3.5.2-cp37-abi3-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2ce45c6627b22c47e390bc91a41c3d13032192e699fa0bea96e9671b373d69b0", size = 255430, upload-time = "2026-09-30T04:38:34.865Z" }, + { url = "https://files.pythonhosted.org/packages/8d/1f/20c8949f0676f7ab811abdeb7f4d7f1cbc6e61ff20bef08b44edeb092bc8/charset_normalizer-3.5.2-cp37-abi3-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0774bf9bf620249fee3e0b8b9fd3065de213be30f3aa94ce2494b3b638949e26", size = 250612, upload-time = "2026-09-30T04:38:36.649Z" }, + { url = "https://files.pythonhosted.org/packages/2b/9e/46f2fa4c431fc98c4ae76a8cb5bdca54e0341e3cfc3fcfd8e82740250818/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1db38f4c5496827c1a501846d64d14c3b80c7e6714e406cd7dc36a9899fa1011", size = 242083, upload-time = "2026-09-30T04:38:38.26Z" }, + { url = "https://files.pythonhosted.org/packages/bd/39/559be29a0c0f086e0bba6922babd38916cc5e0b58ced4de13ee01ea05508/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:304d8e4d493af723536393eee0c689eb7813f4a474c8b479dee63f1fdd98f621", size = 232738, upload-time = "2026-09-30T04:38:39.81Z" }, + { url = "https://files.pythonhosted.org/packages/ff/6c/387b0e4f756a282831c1d9fc6aeb6c51ca4507ca202767c8de15ce9b12e2/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_ppc64le.whl", hash = "sha256:9b7f416ff0978e2f2249330527f0ad6fa02f4932e6199692d3b52da2048c19e4", size = 260703, upload-time = "2026-09-30T04:38:41.346Z" }, + { url = "https://files.pythonhosted.org/packages/96/92/1fdf015f09ef449f50d3ac4b67c90887c9c318b727daa95cc4f866e6521d/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:01077390b03f7988f11d700a2194e69b119741a86b1a638b1db88891e3eced8e", size = 247622, upload-time = "2026-09-30T04:38:42.937Z" }, + { url = "https://files.pythonhosted.org/packages/dc/3c/8e7b8a5671ad5d433669fb2a76f1a0164df2d9b1718b0206bc2a16d840cc/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_s390x.whl", hash = "sha256:7e841fb9010836c992c9f12fcbd43a831de93a5f726fc1ccd8ca1d0268c5014c", size = 257500, upload-time = "2026-09-30T04:38:44.604Z" }, + { url = "https://files.pythonhosted.org/packages/b4/f0/45b579df5cabc1d5d53ea1cc35e8437d3ca768c0acccc7041517cb6fbb32/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:9cae88599c7219005d879f98e5ed53341e9a122af585e1091200358a3003d2a0", size = 255100, upload-time = "2026-09-30T04:38:46.289Z" }, + { url = "https://files.pythonhosted.org/packages/31/68/fdec18a343f5fb3f310588dd478b09ac4799e0b187dbade3a8cd776f03ef/charset_normalizer-3.5.2-cp37-abi3-win32.whl", hash = "sha256:01b0c0d2262a9e28e8484a278c7e1b5d650e3ac8cf2683d2967e25899f208bdf", size = 174499, upload-time = "2026-09-30T04:38:47.999Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8a/b618149cc5207943a0242068d7a27897f56a62947b5a039085f2a22029f8/charset_normalizer-3.5.2-cp37-abi3-win_amd64.whl", hash = "sha256:9f56f72050826f63dcee7a7f55b0a77168cb3bfc553fd405e7f8f9ece75a4036", size = 200092, upload-time = "2026-09-30T04:38:49.707Z" }, + { url = "https://files.pythonhosted.org/packages/03/cf/4c66866fa9e2b1c78e3c911516d1de497a677b7ac60f1eceda74ce777ca3/charset_normalizer-3.5.2-cp37-abi3-win_arm64.whl", hash = "sha256:40ab6bffa02ae10a0581e6c198be7d2d8ca5c2a0c64e4ed3465d766df457573e", size = 294363, upload-time = "2026-09-30T04:38:51.312Z" }, + { url = "https://files.pythonhosted.org/packages/fc/ad/d07d7862a62ffa6d79d68074d14823243dd235a77c45262acbf6adeb28bf/charset_normalizer-3.5.2-py3-none-any.whl", hash = "sha256:b6b751274acb69d77b3323d6b7dbaa3c7fdfc1eb829b7eb61d262f32e1af9685", size = 68872, upload-time = "2026-09-30T04:39:21.828Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "cryptography" +version = "50.0.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9d/af/182eb91b0df3fe75c4d9f26fe70684569566745f6ba7e5c9c73a862c5252/cryptography-50.0.2.tar.gz", hash = "sha256:7b46165bb56eb4704e2eaaf86f3c940d19154535d9b0ca7d6d590b04060e00d5", size = 880623, upload-time = "2026-09-30T15:30:04.884Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/56/d194340cc4a57535e82e1bee9e89667ac4b7c13b5d3f59686deae3094dd5/cryptography-50.0.2-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:fa8f5efb344d6908a1ce62f4a24e2e5780f825d6f53f5f50ec5ffacac72936cb", size = 3914904, upload-time = "2026-09-30T14:43:44.339Z" }, + { url = "https://files.pythonhosted.org/packages/d9/69/c9bd862c3bf43d6399c433caf002df16e2dffd4be49bdf515cda38038711/cryptography-50.0.2-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:79def8d059362e7831389ed3be0ecdf58a89386e1271e35dd9f5af84e81bffd0", size = 4731146, upload-time = "2026-09-30T14:43:47.113Z" }, + { url = "https://files.pythonhosted.org/packages/21/69/64cef1f702bf6657e0cc186ed1a2891d50d29fb41586b254e1c07adea261/cryptography-50.0.2-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:630ebfea3bf689d075f82316324ff7433dc447fe6bc1bfc76524b74b4a9567d2", size = 4719841, upload-time = "2026-09-30T14:43:49.01Z" }, + { url = "https://files.pythonhosted.org/packages/38/6b/61a3f8d8c5e1e49a6cddccafc4015cc1c0021360ab0acb4080e7a423644a/cryptography-50.0.2-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:f9f6143a8c75945eb960d9eb98905a441394abfa24afaae239d514ffb2586480", size = 4738340, upload-time = "2026-09-30T14:43:50.932Z" }, + { url = "https://files.pythonhosted.org/packages/7b/2e/7212ca32fd43dc91f2f41db20160b268098874b4c9a0e7be94d6835f5b2e/cryptography-50.0.2-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:a582ab2ae1d34f67112cadc86702774c9ea4374df6bca6afe672817203c99134", size = 5367029, upload-time = "2026-09-30T14:43:52.911Z" }, + { url = "https://files.pythonhosted.org/packages/1a/f1/b474e930c4d910328780e3940da76f5aa5cbc48ce1fc14e44d239d9ea9db/cryptography-50.0.2-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:4061c0079120205fb760c58acab6443e217307dcf05e3702cf970e0689972856", size = 4753050, upload-time = "2026-09-30T14:43:55.272Z" }, + { url = "https://files.pythonhosted.org/packages/7c/52/9af10e80ac16b0fcc2123f9cbd5e7afbd0fd5075bb7a607c592258a39cda/cryptography-50.0.2-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:ac9ed99d81760c62fe89d5f0815cdfa1ba9a35141cf30f1c2d044f04b4803d2e", size = 4376724, upload-time = "2026-09-30T14:43:57.24Z" }, + { url = "https://files.pythonhosted.org/packages/71/37/6202e488cc1eb625ea110c292c6bda92823176e023f427d8d5660ce8d632/cryptography-50.0.2-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:87e9ce85beb6b328ba370cc6e6aea483c92617b4c95b1d33a49297eb662bfb04", size = 4737859, upload-time = "2026-09-30T14:43:59.541Z" }, + { url = "https://files.pythonhosted.org/packages/8f/30/e86d7d518489b0ae2497091a35287abcb1a2ce4037837a34afbe9b1d6964/cryptography-50.0.2-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:f265528741e048bce55c3463ed721fb0aa45a5888d8add8cfeccb3035451bbdc", size = 5324103, upload-time = "2026-09-30T14:44:01.901Z" }, + { url = "https://files.pythonhosted.org/packages/d3/69/2c833a049475e0a3444e94c7d0aca0aa51d166374a449b09e92ac98138de/cryptography-50.0.2-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:9dab55f57c74c3cad24c323bacbbd04be4705ba6eb0d92e920b1fc4837ed5079", size = 4752576, upload-time = "2026-09-30T14:44:04.545Z" }, + { url = "https://files.pythonhosted.org/packages/6c/5d/906970b83bbfc1f5bbfb677a143c181f2801f23b6a7204a3b47c42c97e65/cryptography-50.0.2-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:25784ce8b9621c90c643efb9e1e2162ab3b0224cae446ad5e70e7fcb1ce18b51", size = 4870819, upload-time = "2026-09-30T14:44:06.884Z" }, + { url = "https://files.pythonhosted.org/packages/68/e3/f2298d3bb55e0c4a91841ec4d01b3f020ba8c5fbf15ccdcc6dcf03f97025/cryptography-50.0.2-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:85d0d9a31b9098e98534226d5686b47264b95e62ce459dc2e62fdfc809f9fe93", size = 5030152, upload-time = "2026-09-30T14:44:09.443Z" }, + { url = "https://files.pythonhosted.org/packages/9a/4f/adfc442765721292fff86d314ce385d3249d22db42295c0dd057727b60f3/cryptography-50.0.2-cp311-abi3-win_amd64.whl", hash = "sha256:7afa5a6602a9f29af1f3a2965f831bae7c9d5d597b7cbb716d41ab3b7d89879c", size = 3824692, upload-time = "2026-09-30T14:44:11.671Z" }, + { url = "https://files.pythonhosted.org/packages/2d/49/93f6a6e7a87c9aa68d44d3e1cdb5fe8f60c90d5d2f46acae9a56892816b8/cryptography-50.0.2-cp315-abi3.abi3t-macosx_11_0_arm64.whl", hash = "sha256:edc3342adf8f697fc5f59c887a304356f147b397809440ed64e2fa6af2f50f37", size = 4133708, upload-time = "2026-09-30T14:44:41.807Z" }, + { url = "https://files.pythonhosted.org/packages/8c/75/32ac2a56243d778805c16ca6a32b8f74fb757df7e28d7ecb560afafb59cf/cryptography-50.0.2-cp315-abi3.abi3t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d370b8d1dfcdf7130178137f6fbee6140774a1acc6cacefc4b42643ec11d0a3a", size = 4956267, upload-time = "2026-09-30T14:44:43.693Z" }, + { url = "https://files.pythonhosted.org/packages/aa/a4/2c8d734e43d97f0842ee9f1b7b4bfb3d0cf5e19edebf43c2afe6675c2320/cryptography-50.0.2-cp315-abi3.abi3t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f2f9bd7f90c64fe89253f0a2c05e3c4856072660429ce8831b4235bf29403a67", size = 4966465, upload-time = "2026-09-30T14:44:45.769Z" }, + { url = "https://files.pythonhosted.org/packages/c2/58/ee288c829a6f41f6235ae9dd33d82fd19b45442b65b4c8a3da36963d9f7a/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_28_aarch64.whl", hash = "sha256:e275096ea1e60cc595cda2836fd4a6c725d1125108b868be17f53684d164e2cc", size = 4959356, upload-time = "2026-09-30T14:44:48.211Z" }, + { url = "https://files.pythonhosted.org/packages/92/20/9ded6d51ddd9897f6b6e81fb9ebea7951d7cc5d6c890b0ed8abf77a51a80/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_28_ppc64le.whl", hash = "sha256:b13478603dcd0a2479ff8e87e2c19a7d525734686fe3c49542472293a204212d", size = 5548822, upload-time = "2026-09-30T14:44:50.86Z" }, + { url = "https://files.pythonhosted.org/packages/02/a8/8df951850d6b31d2a00218f19e2b3f999523437ed7a819df7fa427942fca/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_28_x86_64.whl", hash = "sha256:58a0c478eeca76fe5e07993c5a0703def34a6dc6a0cda4f5564639b33112ffe7", size = 5001199, upload-time = "2026-09-30T14:44:53.379Z" }, + { url = "https://files.pythonhosted.org/packages/8b/f9/36b3022218ce75b7cdf068fb95f809f9bd0d820e4955ef43b90c255cc7ac/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_31_armv7l.whl", hash = "sha256:d38cdff612d06fa6a32840d5e1b1f7a27cee4a349aa9085d94a67789d6bfd408", size = 4629333, upload-time = "2026-09-30T14:44:55.635Z" }, + { url = "https://files.pythonhosted.org/packages/8c/72/20f99a219f6af47cdd1cbd978c243b92d71496e168a746138af44ded4f29/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_34_aarch64.whl", hash = "sha256:fdd28f912fccfec1846a94e2e1e8f9b0012f557f0c46fe4f3eb0d7a87afcf90b", size = 4958822, upload-time = "2026-09-30T14:44:59.639Z" }, + { url = "https://files.pythonhosted.org/packages/f2/20/196f112617fb08eb4d608a2a6c422373d46f9cc2857f38fc0667033c0899/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_34_ppc64le.whl", hash = "sha256:cbc8738fd8526d80f35cb3a40d41f41a2e7030bb3b18b09a6778ef63d291c2fd", size = 5506351, upload-time = "2026-09-30T14:45:02.267Z" }, + { url = "https://files.pythonhosted.org/packages/24/95/83378121ef3eaaaf71d4b781577ff794acb39b9e1b87a3f156898c8497ed/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_34_x86_64.whl", hash = "sha256:e105ab60406787da31fccc883fc0f733af1efd78f0136a4599692c4083a73d0c", size = 5000859, upload-time = "2026-09-30T14:45:05.009Z" }, + { url = "https://files.pythonhosted.org/packages/22/f7/70fd7ae4d1dbfa7ba29b02e1b9068771519a86027756510b700ce81086a8/cryptography-50.0.2-cp315-abi3.abi3t-musllinux_1_2_aarch64.whl", hash = "sha256:6f8700550aa1474a91e5dc07049c46f98b423b5b1ddd0483e0b51362eeeaf5be", size = 5092151, upload-time = "2026-09-30T15:29:15.932Z" }, + { url = "https://files.pythonhosted.org/packages/d4/be/688367b74de86984bd58d8efacfc7c9e68b89a6a22ced0fb4f38db50254a/cryptography-50.0.2-cp315-abi3.abi3t-musllinux_1_2_x86_64.whl", hash = "sha256:c71be1cbfa5cd9a41ee452acf1eccd82b2c05950358b106ec8ceb83411d1a020", size = 5286120, upload-time = "2026-09-30T15:29:18.309Z" }, + { url = "https://files.pythonhosted.org/packages/39/d1/55f8a3f2ef5d1529e16835ef10cf0fe3d559ce237b46dddc440c0bba3649/cryptography-50.0.2-cp315-abi3.abi3t-win_amd64.whl", hash = "sha256:c423ab384a46c4dff7217b2ea5ba2e11cffdeab6441acd04cf65a369caf0366c", size = 4111557, upload-time = "2026-09-30T15:29:20.155Z" }, + { url = "https://files.pythonhosted.org/packages/23/ad/ac987755d00e1e64273760228d2635ae38dae2be83e3c6e0d3289d91dec3/cryptography-50.0.2-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:0ec5f09541743261e66e291b4a0cbf0fb2997aeaab6d9e9c740b9dba1b58d1c2", size = 3943588, upload-time = "2026-09-30T15:29:22.265Z" }, + { url = "https://files.pythonhosted.org/packages/d5/8d/6d585339bedf85d45044c85d8412dac53f2bb6f918e8b7777efba1787844/cryptography-50.0.2-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c5e67125c7dca78d199ec4e116aa93dbb83494808ecbb8211a2cb09b1bf41dbd", size = 4756166, upload-time = "2026-09-30T15:29:24.58Z" }, + { url = "https://files.pythonhosted.org/packages/bf/f1/1c1f6874e8550cfddd4b688ceb38cefb6ed15ceed224d56f133f3d88c214/cryptography-50.0.2-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ee247f5c245c9a2fe7c8e2214e295918838e44e00a45a6718451e4004219e767", size = 4749145, upload-time = "2026-09-30T15:29:26.807Z" }, + { url = "https://files.pythonhosted.org/packages/c1/63/61b15dc1a8de03fe0adbe3fd7608b3ad5c73bf50993bbcb1faaa930afe33/cryptography-50.0.2-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:dfe9763530994147d9af1def057a5b9658b00e8f8fe8743d144d1e0911c2e454", size = 4763638, upload-time = "2026-09-30T15:29:28.588Z" }, + { url = "https://files.pythonhosted.org/packages/fc/35/b345bdfa40c9126df1a9d33236aa98418367931b8725f84fc3ae2b98dc59/cryptography-50.0.2-cp39-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:58ddb5a8e3179d12f19e4ea34d2d32e9d63a4baa142c875c1eb59f41b7243acd", size = 5382217, upload-time = "2026-09-30T15:29:30.589Z" }, + { url = "https://files.pythonhosted.org/packages/4f/87/ef344a9e616871f2519c22d6afcda79ddd5d35e9592d95eb6e677608d055/cryptography-50.0.2-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:f21e8a22c8605750c7af886bab299a363721264061b4ac0a30efb73cfd58efc5", size = 4781387, upload-time = "2026-09-30T15:29:32.605Z" }, + { url = "https://files.pythonhosted.org/packages/90/5b/f2fdb13cd0b96f6f932c8627bb292a45f11c64d21620a8e120aee9a3b848/cryptography-50.0.2-cp39-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:9c8402a82ea0dc4ceeab793db05f0fafa8ca139ca34fcde5df0f596103c74107", size = 4403790, upload-time = "2026-09-30T15:29:34.374Z" }, + { url = "https://files.pythonhosted.org/packages/bc/ce/7e4f662b1e3c393513569e402cfc85ac7da0bd3d5435e122a3140219eb2d/cryptography-50.0.2-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:0ddc924c04591c2811ca024d62ecad4f7f6f08af8939c211438f48a16bd23602", size = 4764319, upload-time = "2026-09-30T15:29:36.149Z" }, + { url = "https://files.pythonhosted.org/packages/3c/3f/86ff33ce34cc0de6847fb96e035a1a760d81652e38643f617c02ad32ef7a/cryptography-50.0.2-cp39-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:a6557e5f38e065ca9fbdaf7cfc7435ecb1d113aa81a022d1b51921ee7432e227", size = 5338560, upload-time = "2026-09-30T15:29:39.053Z" }, + { url = "https://files.pythonhosted.org/packages/40/cf/6b5c8e2fd9202d98988ab7cb5cc5c991704c4ad55f492ff408e4969f83f1/cryptography-50.0.2-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:1981f1db4630889b9ef7803fadef12b056f428cb6b85c27ba57b774793b6093c", size = 4780973, upload-time = "2026-09-30T15:29:41.251Z" }, + { url = "https://files.pythonhosted.org/packages/10/bf/8d6ebc7dded797bd0f0160d52188021211f011a2b164ef0ae1dac4587465/cryptography-50.0.2-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:7a8701d6b584d76e909e3d305b7d126b41439876a5aaf76cddc67fc230eafa2e", size = 4897738, upload-time = "2026-09-30T15:29:43.106Z" }, + { url = "https://files.pythonhosted.org/packages/d4/aa/f3f6e0de7e6253b8baa8b2d8fb9d50924fa75cee3d4624bd4bc1208ee923/cryptography-50.0.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:ce47f66801c20ec6c6632453bb5960fe38939e9306970b48b3a5a26de7745d94", size = 5058280, upload-time = "2026-09-30T15:29:44.827Z" }, + { url = "https://files.pythonhosted.org/packages/f6/b6/a1faf3a27ae9405fb34b1713cc73b2d8a26b04d5c561578fa2e6ef3e5bb9/cryptography-50.0.2-cp39-abi3-win_amd64.whl", hash = "sha256:4e81d95e5bafc2d6e34e4bed780e53e4d5b9a2f928573428aa4d35fbec1eb0de", size = 3854095, upload-time = "2026-09-30T15:29:46.782Z" }, +] + +[[package]] +name = "filelock" +version = "3.32.7" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0f/59/e19834834cb01a32febfbb0f8a23a9088088f5d45991824ff2bc3b5e8acb/filelock-3.32.7.tar.gz", hash = "sha256:37b8a3d9811b0f9aef7e5ec5c71bb320de52df51e6ca9bcd6f5ad81187660da7", size = 225154, upload-time = "2026-09-16T00:24:20.907Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/15/df/31098c5aeb4d966b553641472bd55fcf5fdfac953549894b8a765ba44e91/filelock-3.32.7-py3-none-any.whl", hash = "sha256:65ff0d0190ea42038b32bda4b77834fb05be2cad4c5b9b01aa4dfb3614536e52", size = 100157, upload-time = "2026-09-16T00:24:19.543Z" }, +] + +[[package]] +name = "idna" +version = "3.20" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f5/08/8eea9d4b8302028f3abb2c0813953f7aec26d33b7a8960ed760e65ff29fa/idna-3.20.tar.gz", hash = "sha256:a7db850025b95ded1eae8a46181a1a6c56c92c96f0e2b005d9ff8dc0210cab44", size = 216463, upload-time = "2026-09-17T14:11:04.752Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/58/a2/bb081bab032533a855d44de1d56f8e8426114ff1ba5d1f07a438a0a654f8/idna-3.20-py3-none-any.whl", hash = "sha256:ab7ae7122974553370f0bdb919e1a960b2cd1bc1ef0276416d896db81c14582c", size = 69583, upload-time = "2026-09-17T14:11:03.168Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/e1/2069291243c926a2ff1cd706c7f3eeb9b62144bf60f77c9fb9ff2fb26bd3/iniconfig-2.3.1.tar.gz", hash = "sha256:67f4b9c50da0dedf52af349e7749a80a9057a5031199791b906c3bb3ae878960", size = 21209, upload-time = "2026-10-06T22:48:38.076Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/56/43/4ca9e49d27a1fcf6bece6f6aec0ea46bb9112489b93d4b688fb415457bdb/iniconfig-2.3.1-py3-none-any.whl", hash = "sha256:9121e2c1fdb355232495be3194c8dfe87ccc2d5dee45947b78e68f499790d7a7", size = 7552, upload-time = "2026-10-06T22:48:36.959Z" }, +] + +[[package]] +name = "jinja2" +version = "3.1.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d", size = 245115, upload-time = "2025-03-05T20:05:02.478Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, +] + +[[package]] +name = "jmespath" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d3/59/322338183ecda247fb5d1763a6cbe46eff7222eaeebafd9fa65d4bf5cb11/jmespath-1.1.0.tar.gz", hash = "sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d", size = 27377, upload-time = "2026-01-22T16:35:26.279Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/14/2f/967ba146e6d58cf6a652da73885f52fc68001525b4197effc174321d70b4/jmespath-1.1.0-py3-none-any.whl", hash = "sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64", size = 20419, upload-time = "2026-01-22T16:35:24.919Z" }, +] + +[[package]] +name = "markupsafe" +version = "3.0.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/38/9b/e422a865e1d5d57d0e509b4e0bf1c1a70a7f6382c29a5aa428df994c8bc8/markupsafe-3.0.4.tar.gz", hash = "sha256:2e9ad7dd851bf45fab9f75cbff4cb493fee9979e8d8c7c9c3ee119022518edd6", size = 153777, upload-time = "2026-10-02T23:07:22.29Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/81/09/4c59d56b8461ae8eb0d8ba34bb25b7e618547044679d58a82ef9b2479fc1/markupsafe-3.0.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:61631e08084be9e21a8967ec3139c7616ed7c5e9368e05c86d1b39562c8a57b6", size = 11658, upload-time = "2026-10-02T23:04:51.876Z" }, + { url = "https://files.pythonhosted.org/packages/a2/f0/d6613774d86fbf6d145751d43c59875e47a6f9f17daee0aef173bd36d90e/markupsafe-3.0.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0930db9bdc62d22944e10b066448bb65dc9abe9112880c7cab8da54db4284d5f", size = 12050, upload-time = "2026-10-02T23:04:52.931Z" }, + { url = "https://files.pythonhosted.org/packages/0d/f2/8f18e0b806eb13c1f8d07d917a720831ead54253a6dec011fbc78098a6f8/markupsafe-3.0.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6a45c3d514f2436064db00d7fc8778d888f0236ebfed649b53d13a59e69ad51b", size = 24363, upload-time = "2026-10-02T23:04:53.895Z" }, + { url = "https://files.pythonhosted.org/packages/60/ce/fa07dbe8a5675558fa36dea033e19995bc783de2dec5f540ccb9030b06aa/markupsafe-3.0.4-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:1e1451fab512d1bcc3dc26988ec1edb0b82c2db909132872cd9356070a6b63df", size = 28525, upload-time = "2026-10-02T23:04:54.905Z" }, + { url = "https://files.pythonhosted.org/packages/85/40/be87c01f3868ec217f8a2015089d71c22c8c5a75324822e5ed1cdd87210d/markupsafe-3.0.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:bd3ce56ae2cbae3ba82b683bc425cd7e48d2ed8b10f3e818186b6f5646d9271c", size = 24733, upload-time = "2026-10-02T23:04:56.229Z" }, + { url = "https://files.pythonhosted.org/packages/4f/a7/aeedb5140afa41fc74c225e9184ab96723a6e873b6ee1c9fede7283456d8/markupsafe-3.0.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8e124f974786f831d6043728e38296969d3579db8896fe004682f5758e613581", size = 22985, upload-time = "2026-10-02T23:04:57.521Z" }, + { url = "https://files.pythonhosted.org/packages/c3/fc/e91352bb08c6a59da3ef0909d457bf95a5f5908fbf151b30a06d9dbcfbb4/markupsafe-3.0.4-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c02e8f18bdedba082cef725942ac823b9b60656db07f7e265cb31618dfd00d77", size = 22001, upload-time = "2026-10-02T23:04:58.597Z" }, + { url = "https://files.pythonhosted.org/packages/5d/f8/bffee5e7d2a3deb59748a797650a48af7e672025cf641a79344a771ad106/markupsafe-3.0.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:9f098115c247e11d138ab83a28fa0323c77015007ea2df73ba5fd714dfefd67c", size = 23793, upload-time = "2026-10-02T23:04:59.686Z" }, + { url = "https://files.pythonhosted.org/packages/ed/59/b853d6628ecb4d658e1d637224846d5e9bb4adf4f8df97f3be9f29dce2ec/markupsafe-3.0.4-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:d5f93ebbeb8032d47e349328ec8662d973d9b05a70b3c35df1f91fe419b84749", size = 22640, upload-time = "2026-10-02T23:05:00.768Z" }, + { url = "https://files.pythonhosted.org/packages/09/b2/1506df394f0f075797c418d0301498f49e43be194e3ffcb49e6fe6ccf022/markupsafe-3.0.4-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:64511c54db4e4987aef4c41923235927428729e8174c5dba488429be70a998ed", size = 24074, upload-time = "2026-10-02T23:05:01.813Z" }, + { url = "https://files.pythonhosted.org/packages/c7/81/5ed69cda630ac69ef60d06c09ba5a7f84ff66a2e28cf986fd5614ab3c6e6/markupsafe-3.0.4-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:e1a622f13970d81f95d0c72f9dc090dce9085fccfa4c9f2174377ee32bd15786", size = 21563, upload-time = "2026-10-02T23:05:03.239Z" }, + { url = "https://files.pythonhosted.org/packages/0c/fe/fb1e79be0fea60aa32602ebefc9c35a82bb42b4df157285ab7dfec12341a/markupsafe-3.0.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c9a7f43c0b202b334cc9184af09bb8f21d3a209e038efaf106936fb69e6b026e", size = 23048, upload-time = "2026-10-02T23:05:04.479Z" }, + { url = "https://files.pythonhosted.org/packages/c8/52/7632a53360671a9b750cdbabaf9cdd89f18b42248b8e4cb42c0b0296e459/markupsafe-3.0.4-cp312-cp312-win32.whl", hash = "sha256:f0ec3b750b59375eab5b0fb2b9254810c00a3375be6d789899f1055a1d556237", size = 14108, upload-time = "2026-10-02T23:05:05.513Z" }, + { url = "https://files.pythonhosted.org/packages/3f/bf/62495e180b7000aaf30000fff849e933f74264638057176cf46852500adc/markupsafe-3.0.4-cp312-cp312-win_amd64.whl", hash = "sha256:11935df9bf455ed0c04eb87bcd720f02b1fe5e02128a9430f23aed6f93336fc7", size = 14303, upload-time = "2026-10-02T23:05:06.538Z" }, + { url = "https://files.pythonhosted.org/packages/c5/8e/4c24208776a65878d656996945aacfbfe010d3720d1a98fc0eb8491fc03b/markupsafe-3.0.4-cp312-cp312-win_arm64.whl", hash = "sha256:a4bbd2d87dd233b9fc5812160c3d0ffbe42edc22a26ce0469f58479ede633fe9", size = 14174, upload-time = "2026-10-02T23:05:07.617Z" }, +] + +[[package]] +name = "packaging" +version = "26.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/fa/3944b40b07da9ce895c0e6303a5ab7d53da063554f534556b134a54d6093/packaging-26.3.tar.gz", hash = "sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79", size = 313412, upload-time = "2026-08-04T18:15:28.737Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", hash = "sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", size = 129956, upload-time = "2026-08-04T18:15:27.159Z" }, +] + +[[package]] +name = "phi-guardrails" +version = "0.1.0" +source = { editable = "." } +dependencies = [ + { name = "cryptography" }, + { name = "jinja2" }, + { name = "pyyaml" }, + { name = "schemachange" }, + { name = "snowflake-connector-python" }, +] + +[package.dev-dependencies] +dev = [ + { name = "pytest" }, +] + +[package.metadata] +requires-dist = [ + { name = "cryptography", specifier = ">=44" }, + { name = "jinja2", specifier = ">=3.1" }, + { name = "pyyaml", specifier = ">=6.0.2" }, + { name = "schemachange", specifier = "==4.4.0" }, + { name = "snowflake-connector-python", specifier = ">=4.0,<5.0" }, +] + +[package.metadata.requires-dev] +dev = [{ name = "pytest", specifier = ">=8.3" }] + +[[package]] +name = "platformdirs" +version = "4.13.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/58/e6/d0abbb256b44e6a05748c6942511ff67db8dbb068de0c35b24bf1b395b09/platformdirs-4.13.5.tar.gz", hash = "sha256:05831adf87d392760f4b5c3a0a95579ab5a10b18b9b9bd43680dcb47544788a0", size = 62362, upload-time = "2026-10-19T01:02:11.778Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6c/8a/1d8cb3842909dc193a73835adedf412e32e18f8ffd59f43983949622fe6f/platformdirs-4.13.5-py3-none-any.whl", hash = "sha256:53d3ec4280b815c1e0b02edad7f9ee8fda1ba9c0865dd9c2e6e63849ce82fc1e", size = 32779, upload-time = "2026-10-19T01:02:10.17Z" }, +] + +[[package]] +name = "pluggy" +version = "1.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bf/db/7fc19e6f2dc92a966727031389fc2e08b558f0f25eb7403c1119ad4713cd/pluggy-1.7.0.tar.gz", hash = "sha256:d1eaa46ebb595891b860ab086b4d09c8588af65ebd4361b8e8f4bb8920b90ba8", size = 123304, upload-time = "2026-10-15T09:50:58.343Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/40/9e/2b38731e0fc536806f16490e1a12d7f0dc2a1235aa8cc07bcc75416a7daa/pluggy-1.7.0-py3-none-any.whl", hash = "sha256:7dd7b0d8832ba3cb632c306926ded123429211b83641b35dc5c41ad2d34f9bec", size = 27082, upload-time = "2026-10-15T09:50:56.808Z" }, +] + +[[package]] +name = "pycparser" +version = "3.12" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cb/27/eeb9f307bf07060f3d9f9aa0fb8c15b01bdfded0a941e4b2719c7f94cacd/pycparser-3.12.tar.gz", hash = "sha256:f03387877eab82ce01bbdfd0ba7252ee55a90b1c708d5129e2ce915475cf1db6", size = 158785, upload-time = "2026-10-22T13:40:03.56Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b6/ae/315bbeee126a769cbde43cf4908acb4b40408d8bf8bbf334dd7368d8dadf/pycparser-3.12-py3-none-any.whl", hash = "sha256:a076f3f489b39b43ed13563a1dafe42447445bec2e294154ec5c57fcdda4c85e", size = 62600, upload-time = "2026-10-22T13:40:02.021Z" }, +] + +[[package]] +name = "pygments" +version = "2.21.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/49/2e/ced460408999b33da6b31b0021b0f37d329e202d4169aeb164493778f25b/pygments-2.21.0.tar.gz", hash = "sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c", size = 5005329, upload-time = "2026-08-17T08:02:48.824Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/46/17f022dd3e953bf20a04a028a21ec746d942f8d2af30fa0f124fa0e6a684/pygments-2.21.0-py3-none-any.whl", hash = "sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9", size = 1250147, upload-time = "2026-08-17T08:02:44.912Z" }, +] + +[[package]] +name = "pyjwt" +version = "2.15.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/43/ea/5194e52748b0da83d71e082d75496eaec6e58f419f5e184786ded517e6a9/pyjwt-2.15.1.tar.gz", hash = "sha256:4f259e80cdfb6b3fc18a7de51fd1ef9ec79652f25019bae68975ca2468a34df8", size = 121252, upload-time = "2026-09-28T18:40:42.598Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/50/ca/44de4e75f8aadc457f0634be3b542815078ded46dca30efb960edeecad6e/pyjwt-2.15.1-py3-none-any.whl", hash = "sha256:42d59d631f7768a1028a64c7ff581a9bf7519804daf91fc5b6c56e30eec5e193", size = 33860, upload-time = "2026-09-28T18:40:41.429Z" }, +] + +[[package]] +name = "pyopenssl" +version = "26.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cryptography" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3f/e8/7325d258199b159eb2c03fe32107533e2832e70e63f4fb88a6aa00023201/pyopenssl-26.4.0.tar.gz", hash = "sha256:28dfcce0162b9211413e26dfbfdf1d24317fbeba18fc93c12400a1856b2a0bc7", size = 182046, upload-time = "2026-08-01T19:50:50.512Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/51/ad/2cf6d3fa2fae5c79e1ed9960c0d42badd0f94d81dd12b50604cdc839e648/pyopenssl-26.4.0-py3-none-any.whl", hash = "sha256:f0eb0cb2d581d3ad2b9c489468485e7f2ab6727d08401bcf9d824c3caddf3c1c", size = 56026, upload-time = "2026-08-01T19:50:48.94Z" }, +] + +[[package]] +name = "pytest" +version = "9.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, +] + +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, +] + +[[package]] +name = "pytz" +version = "2026.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/14/21/d83d6ef28c4c912c4bb4d1dcf591f7b8c6bde87b9c66f9f454677314e16d/pytz-2026.5.tar.gz", hash = "sha256:fa23724b9c486543b9ff54a327ee7569ac83ade54bb9afd0fc18676620401c86", size = 318572, upload-time = "2026-10-04T02:37:58.719Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4f/ef/c66110d46fb800dda0bf33164182dfadabe26a90e4476844d502a23dca8e/pytz-2026.5-py2.py3-none-any.whl", hash = "sha256:e658af3757f9e26a9d25dd2aff38335acd92bc9104f890a894b2c1ba28311b03", size = 506342, upload-time = "2026-10-04T02:37:56.814Z" }, +] + +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, + { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, + { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, +] + +[[package]] +name = "requests" +version = "2.34.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ac/c3/e2a2b89f2d3e2179abd6d00ebd70bff6273f37fb3e0cc209f48b39d00cbf/requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed", size = 142856, upload-time = "2026-05-14T19:25:27.735Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, +] + +[[package]] +name = "s3transfer" +version = "0.19.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/49/c5/0a5ce655109cb96b8dfad36908a4e108d15f5dc2eff8fbc3146c4312e704/s3transfer-0.19.3.tar.gz", hash = "sha256:8326acc3d056d7f7b8a37110c7d2aea41f67e83f6ae79cd3b5fbcca34c726406", size = 166087, upload-time = "2026-10-19T19:19:49.751Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e2/a4/88f9e10498759121060b8050a1554753dfc04df39bbbcc7aad9ecc8d9440/s3transfer-0.19.3-py3-none-any.whl", hash = "sha256:24eb7535804306355e31c952a90022bf5b2f9d760f9a11062c7493a217c08553", size = 90252, upload-time = "2026-10-19T19:19:48.309Z" }, +] + +[[package]] +name = "schemachange" +version = "4.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "jinja2" }, + { name = "pyyaml" }, + { name = "snowflake-connector-python" }, + { name = "structlog" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1b/81/3df283afe06d4931e126e985ea2f7a8c83be889912797c3f11d4e9c1a188/schemachange-4.4.0.tar.gz", hash = "sha256:2c8b2b13d93d20b7749ea2dc0e5c05691a3c4f660198f08bddba426c59ebb034", size = 104004, upload-time = "2026-10-23T11:08:12.461Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/76/d6/d4d7331b1c648e3615a678985cc65d33b6d4601fe34d0b81ac02cdac9e14/schemachange-4.4.0-py3-none-any.whl", hash = "sha256:648882cee8f1bece25e9735c66eed9e33a4941fd3ec132f358bcf1e693bfe441", size = 92711, upload-time = "2026-10-23T11:08:11.189Z" }, +] + +[[package]] +name = "six" +version = "1.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, +] + +[[package]] +name = "snowflake-connector-python" +version = "4.8.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "asn1crypto" }, + { name = "boto3" }, + { name = "botocore" }, + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "cryptography" }, + { name = "filelock" }, + { name = "idna" }, + { name = "packaging" }, + { name = "platformdirs" }, + { name = "pyjwt" }, + { name = "pyopenssl" }, + { name = "pytz" }, + { name = "requests" }, + { name = "sortedcontainers" }, + { name = "tomlkit" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c2/9e/5e54a9ab5a9e02caa777750e0054b14cb5007be75505815084f97a067ab1/snowflake_connector_python-4.8.0.tar.gz", hash = "sha256:1bf40aa8684eb0769fedeb53a702b274472571812835d5a03fb9dcf9254333b9", size = 992670, upload-time = "2026-09-30T16:31:09.818Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/51/50/1c4471bccde09971136884be0ea099a3206e320ed24b65b5c916126e316a/snowflake_connector_python-4.8.0-cp312-cp312-macosx_15_0_arm64.whl", hash = "sha256:2fde1216c4c1e4015ad7e39b7b39b6a44a29551f4d7a3dfc64b0905e66dc24ac", size = 1220915, upload-time = "2026-09-30T16:31:17.723Z" }, + { url = "https://files.pythonhosted.org/packages/6f/8a/21dff856ce456874a93816b9e07417f5c253ca0e9b5f65111bcc8164ffdc/snowflake_connector_python-4.8.0-cp312-cp312-macosx_15_0_x86_64.whl", hash = "sha256:c40a058f87dfbca1e65122932afb41c42cb9c3b7cb12826204527b41dd0f39ef", size = 1227694, upload-time = "2026-09-30T16:31:19.093Z" }, + { url = "https://files.pythonhosted.org/packages/9f/1a/e225f7c901550c4eb06430f6ac7b2c0847a885492e49ce97f2fc56ddc558/snowflake_connector_python-4.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f2a3cbcd3e6c4a449165f78bb331e1570ce6026cc21a2673313b4912d80745c0", size = 2953261, upload-time = "2026-09-30T16:30:57.611Z" }, + { url = "https://files.pythonhosted.org/packages/67/60/d77cfe97234be726c1635de98119373b9ebc6ca052f5cf9ce16b2bab77dc/snowflake_connector_python-4.8.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1c80df2fc5e16f3a49e6d16b1cdca0358308c1db2eae22d82aa7c038c80cc913", size = 2986741, upload-time = "2026-09-30T16:30:59.131Z" }, + { url = "https://files.pythonhosted.org/packages/8d/66/34f0d29c37c4bd25ca325ab4e8a11df75866abc4180aa0a0a044a0139bec/snowflake_connector_python-4.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:b9bddddc713aaa20dc5ff5635896f66e50c0fed88b6a905b088e727d046e361d", size = 5452920, upload-time = "2026-09-30T16:31:34.679Z" }, +] + +[[package]] +name = "sortedcontainers" +version = "2.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e8/c4/ba2f8066cceb6f23394729afe52f3bf7adec04bf9ed2c820b39e19299111/sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88", size = 30594, upload-time = "2021-05-16T22:03:42.897Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0", size = 29575, upload-time = "2021-05-16T22:03:41.177Z" }, +] + +[[package]] +name = "structlog" +version = "25.5.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ef/52/9ba0f43b686e7f3ddfeaa78ac3af750292662284b3661e91ad5494f21dbc/structlog-25.5.0.tar.gz", hash = "sha256:098522a3bebed9153d4570c6d0288abf80a031dfdb2048d59a49e9dc2190fc98", size = 1460830, upload-time = "2025-10-27T08:28:23.028Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a8/45/a132b9074aa18e799b891b91ad72133c98d8042c70f6240e4c5f9dabee2f/structlog-25.5.0-py3-none-any.whl", hash = "sha256:a8453e9b9e636ec59bd9e79bbd4a72f025981b3ba0f5837aebf48f02f37a7f9f", size = 72510, upload-time = "2025-10-27T08:28:21.535Z" }, +] + +[[package]] +name = "tomlkit" +version = "0.15.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/08/72/fc01563df95c189a9f65850096bdc205bc2fe37da6c740cd64afaf18d07e/tomlkit-0.15.2.tar.gz", hash = "sha256:83e32755ba1d20d9340bd069ee97e9f6232ba2346b63ffcba425de01f7e327a6", size = 190522, upload-time = "2026-10-16T04:07:08.166Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/da/46/7d84138a60a8bd796825b2d38ebe92d0d16501ca57d501b7c5d677401bbc/tomlkit-0.15.2-py3-none-any.whl", hash = "sha256:05562412163a3037b94497385fb7f9204c637bc605af7a48199e41cf2272f437", size = 52225, upload-time = "2026-10-16T04:07:09.216Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, +] + +[[package]] +name = "urllib3" +version = "2.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e3/05/b17359e1cefb4f909b5e40b1b90a496d987258916dbbf88e842c729f510e/urllib3-2.8.0.tar.gz", hash = "sha256:63bf2ead4c879426ebf22ef2a781eeb4aa3b4ae798a0435506f8687fd5bb9b63", size = 458972, upload-time = "2026-09-15T19:29:36.253Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/92/9d/c4e665119135114480843e7ab388fa94d8480650450e6f8e26b70d323a4c/urllib3-2.8.0-py3-none-any.whl", hash = "sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3", size = 135717, upload-time = "2026-09-15T19:29:34.577Z" }, +] diff --git a/slim-ci-lab/.env.example b/slim-ci-lab/.env.example new file mode 100644 index 0000000..f93befb --- /dev/null +++ b/slim-ci-lab/.env.example @@ -0,0 +1,27 @@ +# Every variable Slim CI Lab reads, with its default. Nothing here is required +# for the emulator path except the LocalStack token. The Makefile does not load +# this file: export what you change (or `set -a; . ./.env; set +a`). + +# --- Emulator -------------------------------------------------------------- +# LocalStack auth token. The Makefile also reads ~/.localstack/default.env. +# LOCALSTACK_AUTH_TOKEN= +SNOWFLAKE_IMAGE=localstack/snowflake-next:latest +SNOWFLAKE_PORT=4566 + +# --- Shared ---------------------------------------------------------------- +# EMULATOR (default) or SNOWFLAKE_CLOUD: picks the connection used by the +# loader, the demo and the tests. +TEST_TARGET=EMULATOR +# Key pair used for SNOWFLAKE_JWT auth. `make up` generates a throwaway one. +SNOWFLAKE_PRIVATE_KEY_PATH=.secrets/rsa_key.p8 +# Database for raw data and dbt models. The test suite creates and drops its own. +DBT_DATABASE=SLIM_CI +# Pull request number for the `pr` target: models build into schema pr_. +DBT_PR_NUMBER=local + +# --- Real Snowflake (TEST_TARGET=SNOWFLAKE_CLOUD and the dbt `cloud` target) -- +# Read only on the cloud path; the emulator path ignores them. +SNOWFLAKE_ACCOUNT= +SNOWFLAKE_USER= +SNOWFLAKE_ROLE=TRANSFORMER +SNOWFLAKE_WAREHOUSE=TRANSFORMING diff --git a/slim-ci-lab/.github-example/workflows/dbt-ci.yml b/slim-ci-lab/.github-example/workflows/dbt-ci.yml new file mode 100644 index 0000000..46d9c93 --- /dev/null +++ b/slim-ci-lab/.github-example/workflows/dbt-ci.yml @@ -0,0 +1,137 @@ +# Slim CI for this dbt project, against the Snowflake-Next emulator. +# +# Copy to .github/workflows/dbt-ci.yml in a repository whose root is this +# sample (dbt/, scripts/, Makefile, ...). The pull request job needs one +# secret, LOCALSTACK_AUTH_TOKEN, and no Snowflake credentials: the emulator +# runs as a container on the runner, so a PR build costs 0 credits. +# +# The manually triggered `parity` job runs the same script against a real +# Snowflake account (billable). It needs the +# SNOWFLAKE_ACCOUNT and SNOWFLAKE_USER variables and the SNOWFLAKE_PRIVATE_KEY +# secret (PKCS#8 PEM for a key-pair user). +name: dbt-ci + +on: + pull_request: + workflow_dispatch: + inputs: + base_ref: + description: Branch whose models form production state + default: main + required: true + +permissions: + contents: read + +concurrency: + group: dbt-ci-${{ github.event.pull_request.number || github.run_id }} + cancel-in-progress: true + +env: + UV_FROZEN: "1" + +jobs: + slim-ci: + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + timeout-minutes: 20 + env: + LOCALSTACK_AUTH_TOKEN: ${{ secrets.LOCALSTACK_AUTH_TOKEN }} + DBT_PR_NUMBER: ${{ github.event.pull_request.number }} + steps: + - name: Check out the pull request + uses: actions/checkout@v4 + + - name: Check out the base branch (production state) + uses: actions/checkout@v4 + with: + ref: ${{ github.base_ref }} + path: .base + + - uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + + - name: Cache the pinned TLC download + uses: actions/cache@v4 + with: + path: .cache/tlc + key: tlc-yellow-2024-01-${{ hashFiles('src/slim_ci_lab/tlc.py') }} + + - name: Start the emulator + run: make up + + - name: Slim CI (prod state from base, PR build with state:modified+ --defer) + env: + PROD_PROJECT_DIR: .base/dbt + PR_PROJECT_DIR: dbt + run: scripts/ci_local.sh + + - name: Keep dbt artifacts + if: always() + uses: actions/upload-artifact@v4 + with: + name: dbt-artifacts-pr-${{ github.event.pull_request.number }} + path: | + prod-artifacts/manifest.json + dbt/target/run_results.json + dbt/target/manifest.json + if-no-files-found: ignore + + - name: Emulator logs on failure + if: failure() + run: docker compose -p snfs-slim-ci-lab logs --tail 300 + + - name: Stop the emulator + if: always() + run: make down + + parity: + if: github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + timeout-minutes: 30 + environment: snowflake-parity + env: + TEST_TARGET: SNOWFLAKE_CLOUD + SNOWFLAKE_ACCOUNT: ${{ vars.SNOWFLAKE_ACCOUNT }} + SNOWFLAKE_USER: ${{ vars.SNOWFLAKE_USER }} + SNOWFLAKE_ROLE: ${{ vars.SNOWFLAKE_ROLE || 'TRANSFORMER' }} + SNOWFLAKE_WAREHOUSE: ${{ vars.SNOWFLAKE_WAREHOUSE || 'TRANSFORMING' }} + SNOWFLAKE_PRIVATE_KEY_PATH: ${{ github.workspace }}/.secrets/rsa_key.p8 + DBT_DATABASE: SLIM_CI_PARITY + DBT_PR_NUMBER: parity_${{ github.run_id }} + PROD_TARGET: cloud + PR_TARGET: cloud + steps: + - uses: actions/checkout@v4 + + - name: Check out the base branch (production state) + uses: actions/checkout@v4 + with: + ref: ${{ inputs.base_ref }} + path: .base + + - uses: astral-sh/setup-uv@v6 + with: + enable-cache: true + + - name: Write the key-pair private key + env: + SNOWFLAKE_PRIVATE_KEY: ${{ secrets.SNOWFLAKE_PRIVATE_KEY }} + run: | + install -d -m 700 .secrets + printf '%s\n' "$SNOWFLAKE_PRIVATE_KEY" > .secrets/rsa_key.p8 + chmod 600 .secrets/rsa_key.p8 + + - name: Install + run: uv sync + + - name: Slim CI on real Snowflake (billable) + env: + PROD_PROJECT_DIR: .base/dbt + PR_PROJECT_DIR: dbt + run: scripts/ci_local.sh + + - name: Drop the parity PR schema + if: always() + run: uv run dbt run-operation drop_pr_schema --target cloud --project-dir dbt --profiles-dir dbt diff --git a/slim-ci-lab/.gitignore b/slim-ci-lab/.gitignore new file mode 100644 index 0000000..8862464 --- /dev/null +++ b/slim-ci-lab/.gitignore @@ -0,0 +1,17 @@ +# Throwaway key pair generated by `make up` +.secrets/ +.env + +# Downloaded TLC files and the downsampled extract +.cache/ + +# dbt build output and the saved production state +dbt/target/ +dbt/logs/ +dbt/dbt_packages/ +prod-artifacts/ + +# Python +.venv/ +__pycache__/ +.pytest_cache/ diff --git a/slim-ci-lab/Makefile b/slim-ci-lab/Makefile new file mode 100644 index 0000000..fd0d046 --- /dev/null +++ b/slim-ci-lab/Makefile @@ -0,0 +1,67 @@ +# Slim CI Lab: dbt Slim CI against the Snowflake-Next emulator. +# Run `make help` for the targets. + +-include $(HOME)/.localstack/default.env +export LOCALSTACK_AUTH_TOKEN +SNOWFLAKE_PORT ?= 4566 +export SNOWFLAKE_PORT +COMPOSE := docker compose -p snfs-slim-ci-lab + +SNOWFLAKE_PRIVATE_KEY_PATH ?= $(CURDIR)/.secrets/rsa_key.p8 +export SNOWFLAKE_PRIVATE_KEY_PATH +export DBT_PROFILES_DIR := $(CURDIR)/dbt +export DBT_PROJECT_DIR := $(CURDIR)/dbt + +UV_RUN := uv run --frozen +PYTEST_ARGS ?= + +.DEFAULT_GOAL := help +.PHONY: help keys up seed test demo demo-bad-pr ci-local down test-cloud install start stop logs + +help: ## List targets + @grep -hE '^[a-z-]+:.*## ' $(firstword $(MAKEFILE_LIST)) | awk -F':.*## ' '{printf " %-12s %s\n", $$1, $$2}' + +keys: .secrets/rsa_key.p8 ## Generate the throwaway key pair used for SNOWFLAKE_JWT auth + +.secrets/rsa_key.p8: + @mkdir -p .secrets && chmod 700 .secrets + @openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 -out .secrets/rsa_key.p8 2>/dev/null + @openssl rsa -in .secrets/rsa_key.p8 -pubout -out .secrets/rsa_key.pub 2>/dev/null + @chmod 600 .secrets/rsa_key.p8 + @echo "Generated .secrets/rsa_key.p8 (throwaway, for local runs)" + +up: keys ## Start the emulator, install the Python env, wait until SQL answers + $(COMPOSE) up -d + uv sync --frozen --quiet + ./scripts/wait-for-emulator.sh 180 + +seed: ## Download, verify and downsample the TLC month, then PUT + COPY it into RAW.TRIPS + $(UV_RUN) slim-ci-load + +test: ## Run the test suite against the running emulator (builds its own database) + $(UV_RUN) pytest $(PYTEST_ARGS) + +demo: demo-bad-pr ## Headline demo: the bad-PR second-run MERGE failure + +demo-bad-pr: ## Apply demos/bad-unique-key.patch and show the second run fail with 100090 + $(UV_RUN) slim-ci-demo-bad-pr + +ci-local: ## Reproduce the GitHub Actions Slim CI pipeline locally + ./scripts/ci_local.sh + +down: ## Stop the emulator and remove its volumes + $(COMPOSE) down -v --remove-orphans + +test-cloud: ## Run the same suite against real Snowflake (billable; needs SNOWFLAKE_* env) + TEST_TARGET=SNOWFLAKE_CLOUD $(UV_RUN) pytest $(PYTEST_ARGS) + +# The targets the repository CI runs for every sample (see the root README) +install: ## Nothing to install up front: make up and make test fetch the dependencies + @echo "Nothing to install, make up and make test fetch the dependencies" + +start: up ## Same as make up + +stop: down ## Same as make down + +logs: ## Tail the emulator log (init hook results included) + $(COMPOSE) logs --tail=200 snowflake diff --git a/slim-ci-lab/README.md b/slim-ci-lab/README.md new file mode 100644 index 0000000..9ea5d6b --- /dev/null +++ b/slim-ci-lab/README.md @@ -0,0 +1,133 @@ +# Slim CI Lab + +A dbt project with a complete Slim CI pull request pipeline (`state:modified+`, `--defer`, one schema per PR) that runs against the LocalStack Snowflake-Next emulator: no Snowflake credentials, 0 credits, and feedback in well under two minutes. It is for analytics engineers running dbt Core and GitHub Actions who want every pull request tested on a warehouse, including the "passes once, breaks on the second run" incremental `MERGE` bug. + +## Architecture + +```mermaid +flowchart LR + tlc["NYC TLC
yellow_tripdata_2024-01.parquet"] -- "download + SHA-256 check" --> cache[".cache/tlc
1-in-300 extract"] + cache -- "PUT" --> stage["@RAW.TLC_LANDING
internal stage"] + stage -- "COPY INTO" --> raw[("RAW.TRIPS")] + seed["taxi_zone_lookup.csv"] -- "dbt seed" --> zones[("REFERENCE.TAXI_ZONE_LOOKUP")] + raw --> stg["stg_trips (view)"] + zones --> stgz["stg_taxi_zones (view)"] + stg --> int["int_trips_enriched
(incremental MERGE)"] + stgz --> int + int --> fct["fct_daily_zone_revenue
(table)"] + stgz -- "dbt snapshot" --> dim["dim_zone
(SCD2 snapshot)"] + subgraph emu["Snowflake-Next emulator (docker compose)"] + stage + raw + zones + stg + stgz + int + fct + dim + end +``` + +The warehouse runs in one `localstack/snowflake-next` container. The loader and dbt run on the host or the CI runner and connect over HTTPS on `snowflake.localhost.localstack.cloud:4566` with key-pair (`SNOWFLAKE_JWT`) auth. Production state lives in the `STAGING`, `INTERMEDIATE`, `MARTS`, `SNAPSHOTS` and `REFERENCE` schemas, and each pull request builds into its own `PR_` schema. Design notes are in [docs/architecture.md](docs/architecture.md). + +## How it works + +1. **Load raw data.** [`loader.py`](src/slim_ci_lab/loader.py) downloads one pinned month of NYC TLC trips, verifies its SHA-256, writes a deterministic 1-in-300 extract (9,883 rows), then runs `PUT` and `COPY INTO RAW.TRIPS ... MATCH_BY_COLUMN_NAME` with lineage columns. `COPY` load metadata makes the step idempotent. +2. **Build production state.** `dbt build --target prod` builds the seed, models and snapshot, runs the data and unit tests, and saves `manifest.json` to `prod-artifacts/`. +3. **Prepare the PR schema.** `drop_pr_schema` gives every push a clean `PR_`, then `dbt clone` zero-copy clones modified incremental models from production, so the PR build runs the same `MERGE` production would. +4. **Build only what changed.** `dbt build --select state:modified+ --defer --state prod-artifacts` builds the modified models and their children; unmodified upstream models resolve to production. +5. **Summarize.** [`ci_summary.py`](src/slim_ci_lab/ci_summary.py) prints the nodes built and deferred, the wall-clock time and the credits used, and writes the same to the GitHub step summary. + +```mermaid +sequenceDiagram + autonumber + actor Dev as Analytics engineer + participant GH as GitHub Actions runner + participant SF as Snowflake-Next emulator + Dev->>GH: push to pull request #101 + GH->>SF: make up, wait for SELECT 1 + GH->>SF: PUT trips extract, COPY INTO RAW.TRIPS + GH->>SF: dbt build --target prod (base branch) + Note over GH: save manifest.json to prod-artifacts/ + GH->>SF: drop_pr_schema, dbt clone modified incrementals into PR_101 + GH->>SF: dbt build --select state:modified+ --defer + alt good change + SF-->>GH: modified mart and its tests pass + GH-->>Dev: check passes, "Snowflake credits used: 0" + else non-unique unique_key + SF-->>GH: 100090 Duplicate row detected during DML action + GH-->>Dev: check fails before review + end + GH->>SF: make down +``` + +## Prerequisites + +- Docker with Compose v2 +- [`uv`](https://docs.astral.sh/uv/) 0.10 or newer (installs Python 3.12 and the locked dependencies) +- `openssl`, `curl`, `patch` and GNU `make` +- A LocalStack auth token in `LOCALSTACK_AUTH_TOKEN` or `~/.localstack/default.env` +- Network access to `d37ci6vzurychx.cloudfront.net` on the first run (about 48 MB, cached under `.cache/`) + +## Quickstart + +```sh +make up # key pair, emulator container, Python env, wait for SQL +make seed # download, verify, downsample, PUT + COPY INTO RAW.TRIPS +make ci-local # the full Slim CI pipeline, with demos/example-pr.patch as the PR +make demo # a bad unique_key passes run 1 and fails the run 2 MERGE +make test # the test suite (creates and drops its own database) +make down # stop and remove the emulator +``` + +`make ci-local` ends with a summary like this: + +```text +==> 5/5 Summary +PR schema: pr_local +Selected: fct_daily_zone_revenue +Nodes run: 1 model, 4 test, 1 unit_test +Results: 5 pass, 1 success +Deferred to prod: 19 of 25 nodes not rebuilt +Wall-clock: 0m 35s +Snowflake credits used: 0 (Snowflake-Next emulator) +``` + +Run the bad PR through the whole pipeline with `CI_PR_PATCH=demos/bad-unique-key.patch make ci-local`: it fails with `100090` on the first CI run. To use another host port, set `SNOWFLAKE_PORT` (for example `SNOWFLAKE_PORT=14566 make up test down`). + +## Tests + +`make test` runs pytest against the emulator in a throwaway `SLIM_CI_TEST_` database. Expected values come from business rules or from pandas over the source file, never from the models. The suite covers: + +- **Loading:** `COPY INTO` row counts and timestamps match the Parquet file, and a re-run loads 0 files. +- **Models:** revenue for one zone and day equals an independent pandas computation; a second incremental run merges without duplicates; the `dim_zone` snapshot records a renamed zone and a hard delete. +- **Slim CI:** the PR build selects only the modified mart and defers the rest; without `--defer` it fails with `002003`; a non-unique `unique_key` fails with `100090` on the second run and on the first CI run. +- **dbt unit tests:** two unit tests in [`dbt/models/schema.yml`](dbt/models/schema.yml) run inside every `dbt build`. + +`make test-cloud` runs the same suite against a real Snowflake account (billable). Set `SNOWFLAKE_ACCOUNT`, `SNOWFLAKE_USER` and `SNOWFLAKE_PRIVATE_KEY_PATH`, and optionally `SNOWFLAKE_ROLE` (default `TRANSFORMER`) and `SNOWFLAKE_WAREHOUSE` (default `TRANSFORMING`); [`init/01_platform.sf.sql`](init/01_platform.sf.sql) creates the objects the role needs. + +A copyable GitHub Actions workflow is in [`.github-example/workflows/dbt-ci.yml`](.github-example/workflows/dbt-ci.yml): the `slim-ci` job runs on every pull request with only the `LOCALSTACK_AUTH_TOKEN` secret, and the manual `parity` job runs the same pipeline on a real account. + +## Project structure + +```text +slim-ci-lab/ + Makefile up, seed, ci-local, demo, test, down, test-cloud + docker-compose.yml the emulator, bound to 127.0.0.1 + init/01_platform.sf.sql warehouse, role, service user, database + src/slim_ci_lab/ settings, TLC download, loader, demo, CI summary + scripts/ci_local.sh the Slim CI pipeline (make ci-local and GitHub Actions) + dbt/ models, macros, snapshot, seed, profiles + demos/ example-pr.patch (good PR), bad-unique-key.patch (bad PR) + tests/ pytest suite + docs/architecture.md design notes and data model + .github-example/ copyable GitHub Actions workflow +``` + +## Learn more + +- [dbt: node selection and state](https://docs.getdbt.com/reference/node-selection/syntax), [defer](https://docs.getdbt.com/reference/node-selection/defer), [dbt clone](https://docs.getdbt.com/reference/commands/clone) +- [dbt: incremental models](https://docs.getdbt.com/docs/build/incremental-models), [snapshots](https://docs.getdbt.com/docs/build/snapshots), [unit tests](https://docs.getdbt.com/docs/build/unit-tests) +- [Snowflake: COPY INTO \](https://docs.snowflake.com/en/sql-reference/sql/copy-into-table), [CREATE FILE FORMAT](https://docs.snowflake.com/en/sql-reference/sql/create-file-format) +- [Snowflake: MERGE](https://docs.snowflake.com/en/sql-reference/sql/merge), [CREATE ... CLONE](https://docs.snowflake.com/en/sql-reference/sql/create-clone) +- [NYC TLC trip record data](https://www.nyc.gov/site/tlc/about/tlc-trip-record-data.page) diff --git a/slim-ci-lab/dbt/dbt_project.yml b/slim-ci-lab/dbt/dbt_project.yml new file mode 100644 index 0000000..7d74255 --- /dev/null +++ b/slim-ci-lab/dbt/dbt_project.yml @@ -0,0 +1,47 @@ +name: slim_ci_lab +version: "1.0.0" +config-version: 2 +profile: slim_ci_lab +require-dbt-version: [">=1.10.0", "<1.11.0"] + +model-paths: ["models"] +seed-paths: ["seeds"] +snapshot-paths: ["snapshots"] +macro-paths: ["macros"] +test-paths: ["tests"] +clean-targets: ["target", "logs"] + +flags: + send_anonymous_usage_stats: false + +vars: + # The month of TLC data this project reports on. Trips outside it (the TLC + # files carry a few stray timestamps) are excluded from the marts. + reporting_start: "2024-01-01" + reporting_end: "2024-02-01" + # Incremental runs re-merge this many days before the latest loaded pickup, + # so late or corrected trips update in place instead of being skipped. + incremental_lookback_days: 3 + +models: + slim_ci_lab: + +persist_docs: + relation: true + staging: + +schema: staging + +materialized: view + intermediate: + +schema: intermediate + marts: + +schema: marts + +materialized: table + +seeds: + slim_ci_lab: + +schema: reference + taxi_zone_lookup: + +column_types: + LocationID: integer + Borough: varchar(32) + Zone: varchar(64) + service_zone: varchar(32) diff --git a/slim-ci-lab/dbt/macros/drop_pr_schema.sql b/slim-ci-lab/dbt/macros/drop_pr_schema.sql new file mode 100644 index 0000000..27599f8 --- /dev/null +++ b/slim-ci-lab/dbt/macros/drop_pr_schema.sql @@ -0,0 +1,10 @@ +{# + dbt run-operation drop_pr_schema --target pr + Drops this pull request's schema. CI runs it before the PR build (fresh + state on every push) and, on a real account, when the PR closes. +#} +{% macro drop_pr_schema() %} + {%- set relation = api.Relation.create(database=target.database, schema=pr_schema_name()) -%} + {% do log("Dropping schema " ~ relation, info=true) %} + {% do adapter.drop_schema(relation) %} +{% endmacro %} diff --git a/slim-ci-lab/dbt/macros/generate_schema_name.sql b/slim-ci-lab/dbt/macros/generate_schema_name.sql new file mode 100644 index 0000000..3b723ae --- /dev/null +++ b/slim-ci-lab/dbt/macros/generate_schema_name.sql @@ -0,0 +1,30 @@ +{# + Where each node is built: + + * pr (and cloud with DBT_PR_NUMBER set): everything goes into one schema, + pr_. A pull request owns exactly one schema, so two open PRs never + collide and cleanup is a single DROP SCHEMA. + * prod (and cloud without a PR number): the layer's custom schema is used + as is: staging, intermediate, marts, snapshots, reference. + * dev and anything else: _, a personal sandbox. +#} +{% macro generate_schema_name(custom_schema_name, node) -%} + {%- set pr_number = env_var('DBT_PR_NUMBER', '') | trim -%} + {%- if target.name == 'pr' or (target.name == 'cloud' and pr_number) -%} + {{ pr_schema_name() }} + {%- elif target.name in ['prod', 'cloud'] and custom_schema_name -%} + {{ custom_schema_name | trim }} + {%- elif custom_schema_name -%} + {{ target.schema }}_{{ custom_schema_name | trim }} + {%- else -%} + {{ target.schema }} + {%- endif -%} +{%- endmacro %} + +{% macro pr_schema_name() -%} + {%- set pr_number = env_var('DBT_PR_NUMBER', '') | trim -%} + {%- if not pr_number -%} + {{ exceptions.raise_compiler_error("DBT_PR_NUMBER must be set to build a pull request schema") }} + {%- endif -%} + pr_{{ pr_number | lower | replace('-', '_') }} +{%- endmacro %} diff --git a/slim-ci-lab/dbt/macros/surrogate_key.sql b/slim-ci-lab/dbt/macros/surrogate_key.sql new file mode 100644 index 0000000..663749a --- /dev/null +++ b/slim-ci-lab/dbt/macros/surrogate_key.sql @@ -0,0 +1,12 @@ +{# + Deterministic surrogate key: MD5 over the columns' text, with NULLs made + explicit so (NULL, 'a') and ('', 'a') hash differently. Same idea as + dbt_utils.generate_surrogate_key, without the package dependency. +#} +{% macro surrogate_key(columns) -%} + md5( + {%- for column in columns %} + coalesce(cast({{ column }} as varchar), '_null_'){% if not loop.last %} || '|' ||{% endif %} + {%- endfor %} + ) +{%- endmacro %} diff --git a/slim-ci-lab/dbt/models/intermediate/int_trips_enriched.sql b/slim-ci-lab/dbt/models/intermediate/int_trips_enriched.sql new file mode 100644 index 0000000..503f0b3 --- /dev/null +++ b/slim-ci-lab/dbt/models/intermediate/int_trips_enriched.sql @@ -0,0 +1,82 @@ +{{ + config( + materialized='incremental', + incremental_strategy='merge', + unique_key='trip_id', + on_schema_change='append_new_columns' + ) +}} + +-- Billable trips in the reporting month, keyed and enriched with zone names. +-- Incremental: each run re-merges a lookback window, so a corrected trip +-- updates its row (matched on trip_id) instead of being appended twice. + +with trips as ( + + select * + from {{ ref('stg_trips') }} + where pickup_at >= '{{ var("reporting_start") }}' + and pickup_at < '{{ var("reporting_end") }}' + -- Negative totals are voids and refunds of an earlier trip, not revenue. + and total_amount >= 0 + {% if is_incremental() %} + and pickup_at >= ( + select dateadd(day, -{{ var('incremental_lookback_days') }}, max(pickup_at)) + from {{ this }} + ) + {% endif %} + +), + +keyed as ( + + select + *, + -- The TLC feed has no trip id, and the same vendor, timestamps and + -- zones can appear on more than one record. Number the repeats so the + -- surrogate key stays unique. + row_number() over ( + partition by vendor_id, pickup_at, dropoff_at, pickup_location_id, dropoff_location_id + order by _source_file, _source_row + ) as natural_key_seq + from trips + +), + +zones as ( + + select * from {{ ref('stg_taxi_zones') }} + +) + +select + {{ surrogate_key([ + 'keyed.vendor_id', 'keyed.pickup_at', 'keyed.dropoff_at', + 'keyed.pickup_location_id', 'keyed.dropoff_location_id', 'keyed.natural_key_seq' + ]) }} as trip_id, + keyed.natural_key_seq, + keyed.vendor_id, + keyed.pickup_at, + cast(keyed.pickup_at as date) as pickup_date, + keyed.dropoff_at, + datediff('second', keyed.pickup_at, keyed.dropoff_at) as trip_duration_seconds, + keyed.pickup_location_id, + pickup_zone.borough as pickup_borough, + pickup_zone.zone_name as pickup_zone, + keyed.dropoff_location_id, + dropoff_zone.borough as dropoff_borough, + dropoff_zone.zone_name as dropoff_zone, + keyed.passenger_count, + keyed.trip_distance_miles, + keyed.payment_type_id, + keyed.fare_amount, + keyed.tip_amount, + keyed.tolls_amount, + keyed.total_amount, + keyed._source_file, + keyed._loaded_at +from keyed +left join zones as pickup_zone + on keyed.pickup_location_id = pickup_zone.location_id +left join zones as dropoff_zone + on keyed.dropoff_location_id = dropoff_zone.location_id diff --git a/slim-ci-lab/dbt/models/marts/fct_daily_zone_revenue.sql b/slim-ci-lab/dbt/models/marts/fct_daily_zone_revenue.sql new file mode 100644 index 0000000..5c1ba0a --- /dev/null +++ b/slim-ci-lab/dbt/models/marts/fct_daily_zone_revenue.sql @@ -0,0 +1,15 @@ +-- Revenue per pickup zone per day. Revenue is the passenger-paid total +-- (fare, surcharges, tolls and tips) of billable trips. +select + {{ surrogate_key(['pickup_date', 'pickup_location_id']) }} as daily_zone_id, + pickup_date, + pickup_location_id, + pickup_borough, + pickup_zone, + count(*) as trip_count, + sum(total_amount) as revenue, + sum(fare_amount) as fare_revenue, + sum(tip_amount) as tip_revenue, + round(avg(trip_distance_miles), 2) as avg_trip_distance_miles +from {{ ref('int_trips_enriched') }} +group by pickup_date, pickup_location_id, pickup_borough, pickup_zone diff --git a/slim-ci-lab/dbt/models/schema.yml b/slim-ci-lab/dbt/models/schema.yml new file mode 100644 index 0000000..11d8705 --- /dev/null +++ b/slim-ci-lab/dbt/models/schema.yml @@ -0,0 +1,87 @@ +models: + - name: stg_trips + description: One row per raw TLC trip record, renamed and typed (money as NUMBER(10,2)). + columns: + - name: pickup_at + data_tests: [not_null] + - name: total_amount + description: Passenger-paid total. Negative for voids and refunds. + + - name: stg_taxi_zones + description: TLC taxi zones from the taxi_zone_lookup seed. + columns: + - name: location_id + data_tests: [unique, not_null] + + - name: int_trips_enriched + description: > + Billable trips (total_amount >= 0) in the reporting month, one row per + trip, keyed by a surrogate trip_id and enriched with pickup and dropoff + zone names. Incremental MERGE on trip_id with a lookback window. + columns: + - name: trip_id + description: MD5 of vendor, pickup and dropoff timestamps and zones, plus natural_key_seq. + data_tests: [unique, not_null] + - name: natural_key_seq + description: 1 for the first record with a given natural key, 2 for a repeat, and so on. + - name: pickup_date + data_tests: [not_null] + - name: pickup_location_id + data_tests: + - not_null + - relationships: + arguments: + to: ref('stg_taxi_zones') + field: location_id + + - name: fct_daily_zone_revenue + description: Trips and revenue per pickup zone per day. + columns: + - name: daily_zone_id + description: Surrogate key of (pickup_date, pickup_location_id), the grain of this table. + data_tests: [unique, not_null] + - name: revenue + description: Sum of total_amount of billable trips. + data_tests: [not_null] + - name: trip_count + data_tests: [not_null] + +unit_tests: + - name: int_trips_enriched_keys_repeats_and_drops_refunds + description: > + Two records with the same natural key both survive with distinct + natural_key_seq values; refunds and trips outside the reporting month + are dropped; zone names come from the zone lookup. + model: int_trips_enriched + overrides: + macros: + is_incremental: false + given: + - input: ref('stg_trips') + rows: + - {vendor_id: 2, pickup_at: "2024-01-15 08:00:00", dropoff_at: "2024-01-15 08:40:00", pickup_location_id: 132, dropoff_location_id: 236, total_amount: 70.25, _source_file: "yellow/2024-01/a.parquet", _source_row: 1} + - {vendor_id: 2, pickup_at: "2024-01-15 08:00:00", dropoff_at: "2024-01-15 08:40:00", pickup_location_id: 132, dropoff_location_id: 236, total_amount: 70.25, _source_file: "yellow/2024-01/a.parquet", _source_row: 2} + - {vendor_id: 1, pickup_at: "2024-01-15 09:00:00", dropoff_at: "2024-01-15 09:10:00", pickup_location_id: 236, dropoff_location_id: 132, total_amount: -12.50, _source_file: "yellow/2024-01/a.parquet", _source_row: 3} + - {vendor_id: 1, pickup_at: "2023-12-31 23:50:00", dropoff_at: "2024-01-01 00:05:00", pickup_location_id: 236, dropoff_location_id: 132, total_amount: 18.00, _source_file: "yellow/2024-01/a.parquet", _source_row: 4} + - input: ref('stg_taxi_zones') + rows: + - {location_id: 132, borough: Queens, zone_name: JFK Airport, service_zone: Airports} + - {location_id: 236, borough: Manhattan, zone_name: Upper East Side North, service_zone: Yellow Zone} + expect: + rows: + - {natural_key_seq: 1, pickup_date: "2024-01-15", pickup_zone: JFK Airport, dropoff_zone: Upper East Side North, trip_duration_seconds: 2400, total_amount: 70.25} + - {natural_key_seq: 2, pickup_date: "2024-01-15", pickup_zone: JFK Airport, dropoff_zone: Upper East Side North, trip_duration_seconds: 2400, total_amount: 70.25} + + - name: fct_daily_zone_revenue_sums_total_amount + description: Revenue is the sum of total_amount per pickup zone and day; fare and tips are reported separately. + model: fct_daily_zone_revenue + given: + - input: ref('int_trips_enriched') + rows: + - {pickup_date: "2024-01-15", pickup_location_id: 132, pickup_borough: Queens, pickup_zone: JFK Airport, fare_amount: 70.00, tip_amount: 15.00, total_amount: 95.25, trip_distance_miles: 17.1} + - {pickup_date: "2024-01-15", pickup_location_id: 132, pickup_borough: Queens, pickup_zone: JFK Airport, fare_amount: 52.80, tip_amount: 0.00, total_amount: 64.55, trip_distance_miles: 12.9} + - {pickup_date: "2024-01-16", pickup_location_id: 132, pickup_borough: Queens, pickup_zone: JFK Airport, fare_amount: 19.80, tip_amount: 4.00, total_amount: 30.30, trip_distance_miles: 3.0} + expect: + rows: + - {pickup_date: "2024-01-15", pickup_location_id: 132, trip_count: 2, revenue: 159.80, fare_revenue: 122.80, tip_revenue: 15.00, avg_trip_distance_miles: 15.00} + - {pickup_date: "2024-01-16", pickup_location_id: 132, trip_count: 1, revenue: 30.30, fare_revenue: 19.80, tip_revenue: 4.00, avg_trip_distance_miles: 3.00} diff --git a/slim-ci-lab/dbt/models/sources.yml b/slim-ci-lab/dbt/models/sources.yml new file mode 100644 index 0000000..8ebf35f --- /dev/null +++ b/slim-ci-lab/dbt/models/sources.yml @@ -0,0 +1,17 @@ +sources: + - name: tlc + description: NYC Taxi and Limousine Commission trip records, landed by the loader (PUT + COPY INTO). + database: "{{ target.database }}" + schema: raw + loaded_at_field: _loaded_at + tables: + - name: trips + description: > + Yellow taxi trips for the pinned month, downsampled 1 in 300. Column + names follow the TLC data dictionary; the loader adds _source_file, + _source_row and _loaded_at from COPY INTO metadata. + columns: + - name: tpep_pickup_datetime + data_tests: [not_null] + - name: _source_file + data_tests: [not_null] diff --git a/slim-ci-lab/dbt/models/staging/stg_taxi_zones.sql b/slim-ci-lab/dbt/models/staging/stg_taxi_zones.sql new file mode 100644 index 0000000..dddd181 --- /dev/null +++ b/slim-ci-lab/dbt/models/staging/stg_taxi_zones.sql @@ -0,0 +1,7 @@ +-- TLC taxi zones (seed), with consistent column names. +select + locationid as location_id, + borough, + zone as zone_name, + service_zone +from {{ ref('taxi_zone_lookup') }} diff --git a/slim-ci-lab/dbt/models/staging/stg_trips.sql b/slim-ci-lab/dbt/models/staging/stg_trips.sql new file mode 100644 index 0000000..6584ad3 --- /dev/null +++ b/slim-ci-lab/dbt/models/staging/stg_trips.sql @@ -0,0 +1,38 @@ +-- One row per raw trip record: renamed, typed, no business rules yet. +with source as ( + + select * from {{ source('tlc', 'trips') }} + +), + +renamed as ( + + select + vendorid as vendor_id, + tpep_pickup_datetime as pickup_at, + tpep_dropoff_datetime as dropoff_at, + passenger_count, + trip_distance as trip_distance_miles, + ratecodeid as rate_code_id, + store_and_fwd_flag = 'Y' as is_store_and_forward, + pulocationid as pickup_location_id, + dolocationid as dropoff_location_id, + payment_type as payment_type_id, + -- Money as fixed-point so sums are exact. + cast(fare_amount as number(10, 2)) as fare_amount, + cast(extra as number(10, 2)) as extra_amount, + cast(mta_tax as number(10, 2)) as mta_tax_amount, + cast(tip_amount as number(10, 2)) as tip_amount, + cast(tolls_amount as number(10, 2)) as tolls_amount, + cast(improvement_surcharge as number(10, 2)) as improvement_surcharge_amount, + cast(congestion_surcharge as number(10, 2)) as congestion_surcharge_amount, + cast(airport_fee as number(10, 2)) as airport_fee_amount, + cast(total_amount as number(10, 2)) as total_amount, + _source_file, + _source_row, + _loaded_at + from source + +) + +select * from renamed diff --git a/slim-ci-lab/dbt/profiles.yml b/slim-ci-lab/dbt/profiles.yml new file mode 100644 index 0000000..dae4646 --- /dev/null +++ b/slim-ci-lab/dbt/profiles.yml @@ -0,0 +1,48 @@ +# Connection profiles. dev, prod and pr point at the Snowflake-Next emulator; +# cloud points at a real account. All four use key-pair (SNOWFLAKE_JWT) auth, +# so there is no password anywhere. Pick one with --target or DBT_TARGET. +slim_ci_lab: + target: "{{ env_var('DBT_TARGET', 'dev') }}" + outputs: + dev: &emulator + type: snowflake + account: snowflake + host: snowflake.localhost.localstack.cloud + port: "{{ env_var('SNOWFLAKE_PORT', '4566') | as_number }}" + protocol: https + user: SAMPLE_SVC + authenticator: snowflake_jwt + private_key_path: "{{ env_var('SNOWFLAKE_PRIVATE_KEY_PATH', '.secrets/rsa_key.p8') }}" + role: TRANSFORMER + warehouse: TRANSFORMING + database: "{{ env_var('DBT_DATABASE', 'SLIM_CI') }}" + # Personal sandbox: dev__staging, dev__marts, ... + schema: "dev_{{ env_var('USER', 'local') | lower }}" + threads: 4 + query_tag: slim-ci-lab + reuse_connections: true + + # Production state: custom schemas used verbatim (staging, marts, ...). + prod: + <<: *emulator + schema: analytics + + # One schema per pull request (see macros/generate_schema_name.sql). + pr: + <<: *emulator + schema: "pr_{{ env_var('DBT_PR_NUMBER') }}" + + # Real Snowflake. Behaves like prod, or like pr when DBT_PR_NUMBER is set. + cloud: + type: snowflake + account: "{{ env_var('SNOWFLAKE_ACCOUNT') }}" + user: "{{ env_var('SNOWFLAKE_USER') }}" + authenticator: snowflake_jwt + private_key_path: "{{ env_var('SNOWFLAKE_PRIVATE_KEY_PATH') }}" + role: "{{ env_var('SNOWFLAKE_ROLE', 'TRANSFORMER') }}" + warehouse: "{{ env_var('SNOWFLAKE_WAREHOUSE', 'TRANSFORMING') }}" + database: "{{ env_var('DBT_DATABASE', 'SLIM_CI') }}" + schema: analytics + threads: 4 + query_tag: slim-ci-lab + reuse_connections: true diff --git a/slim-ci-lab/dbt/seeds/taxi_zone_lookup.csv b/slim-ci-lab/dbt/seeds/taxi_zone_lookup.csv new file mode 100644 index 0000000..5abd72f --- /dev/null +++ b/slim-ci-lab/dbt/seeds/taxi_zone_lookup.csv @@ -0,0 +1,266 @@ +"LocationID","Borough","Zone","service_zone" +1,"EWR","Newark Airport","EWR" +2,"Queens","Jamaica Bay","Boro Zone" +3,"Bronx","Allerton/Pelham Gardens","Boro Zone" +4,"Manhattan","Alphabet City","Yellow Zone" +5,"Staten Island","Arden Heights","Boro Zone" +6,"Staten Island","Arrochar/Fort Wadsworth","Boro Zone" +7,"Queens","Astoria","Boro Zone" +8,"Queens","Astoria Park","Boro Zone" +9,"Queens","Auburndale","Boro Zone" +10,"Queens","Baisley Park","Boro Zone" +11,"Brooklyn","Bath Beach","Boro Zone" +12,"Manhattan","Battery Park","Yellow Zone" +13,"Manhattan","Battery Park City","Yellow Zone" +14,"Brooklyn","Bay Ridge","Boro Zone" +15,"Queens","Bay Terrace/Fort Totten","Boro Zone" +16,"Queens","Bayside","Boro Zone" +17,"Brooklyn","Bedford","Boro Zone" +18,"Bronx","Bedford Park","Boro Zone" +19,"Queens","Bellerose","Boro Zone" +20,"Bronx","Belmont","Boro Zone" +21,"Brooklyn","Bensonhurst East","Boro Zone" +22,"Brooklyn","Bensonhurst West","Boro Zone" +23,"Staten Island","Bloomfield/Emerson Hill","Boro Zone" +24,"Manhattan","Bloomingdale","Yellow Zone" +25,"Brooklyn","Boerum Hill","Boro Zone" +26,"Brooklyn","Borough Park","Boro Zone" +27,"Queens","Breezy Point/Fort Tilden/Riis Beach","Boro Zone" +28,"Queens","Briarwood/Jamaica Hills","Boro Zone" +29,"Brooklyn","Brighton Beach","Boro Zone" +30,"Queens","Broad Channel","Boro Zone" +31,"Bronx","Bronx Park","Boro Zone" +32,"Bronx","Bronxdale","Boro Zone" +33,"Brooklyn","Brooklyn Heights","Boro Zone" +34,"Brooklyn","Brooklyn Navy Yard","Boro Zone" +35,"Brooklyn","Brownsville","Boro Zone" +36,"Brooklyn","Bushwick North","Boro Zone" +37,"Brooklyn","Bushwick South","Boro Zone" +38,"Queens","Cambria Heights","Boro Zone" +39,"Brooklyn","Canarsie","Boro Zone" +40,"Brooklyn","Carroll Gardens","Boro Zone" +41,"Manhattan","Central Harlem","Boro Zone" +42,"Manhattan","Central Harlem North","Boro Zone" +43,"Manhattan","Central Park","Yellow Zone" +44,"Staten Island","Charleston/Tottenville","Boro Zone" +45,"Manhattan","Chinatown","Yellow Zone" +46,"Bronx","City Island","Boro Zone" +47,"Bronx","Claremont/Bathgate","Boro Zone" +48,"Manhattan","Clinton East","Yellow Zone" +49,"Brooklyn","Clinton Hill","Boro Zone" +50,"Manhattan","Clinton West","Yellow Zone" +51,"Bronx","Co-Op City","Boro Zone" +52,"Brooklyn","Cobble Hill","Boro Zone" +53,"Queens","College Point","Boro Zone" +54,"Brooklyn","Columbia Street","Boro Zone" +55,"Brooklyn","Coney Island","Boro Zone" +56,"Queens","Corona","Boro Zone" +57,"Queens","Corona","Boro Zone" +58,"Bronx","Country Club","Boro Zone" +59,"Bronx","Crotona Park","Boro Zone" +60,"Bronx","Crotona Park East","Boro Zone" +61,"Brooklyn","Crown Heights North","Boro Zone" +62,"Brooklyn","Crown Heights South","Boro Zone" +63,"Brooklyn","Cypress Hills","Boro Zone" +64,"Queens","Douglaston","Boro Zone" +65,"Brooklyn","Downtown Brooklyn/MetroTech","Boro Zone" +66,"Brooklyn","DUMBO/Vinegar Hill","Boro Zone" +67,"Brooklyn","Dyker Heights","Boro Zone" +68,"Manhattan","East Chelsea","Yellow Zone" +69,"Bronx","East Concourse/Concourse Village","Boro Zone" +70,"Queens","East Elmhurst","Boro Zone" +71,"Brooklyn","East Flatbush/Farragut","Boro Zone" +72,"Brooklyn","East Flatbush/Remsen Village","Boro Zone" +73,"Queens","East Flushing","Boro Zone" +74,"Manhattan","East Harlem North","Boro Zone" +75,"Manhattan","East Harlem South","Boro Zone" +76,"Brooklyn","East New York","Boro Zone" +77,"Brooklyn","East New York/Pennsylvania Avenue","Boro Zone" +78,"Bronx","East Tremont","Boro Zone" +79,"Manhattan","East Village","Yellow Zone" +80,"Brooklyn","East Williamsburg","Boro Zone" +81,"Bronx","Eastchester","Boro Zone" +82,"Queens","Elmhurst","Boro Zone" +83,"Queens","Elmhurst/Maspeth","Boro Zone" +84,"Staten Island","Eltingville/Annadale/Prince's Bay","Boro Zone" +85,"Brooklyn","Erasmus","Boro Zone" +86,"Queens","Far Rockaway","Boro Zone" +87,"Manhattan","Financial District North","Yellow Zone" +88,"Manhattan","Financial District South","Yellow Zone" +89,"Brooklyn","Flatbush/Ditmas Park","Boro Zone" +90,"Manhattan","Flatiron","Yellow Zone" +91,"Brooklyn","Flatlands","Boro Zone" +92,"Queens","Flushing","Boro Zone" +93,"Queens","Flushing Meadows-Corona Park","Boro Zone" +94,"Bronx","Fordham South","Boro Zone" +95,"Queens","Forest Hills","Boro Zone" +96,"Queens","Forest Park/Highland Park","Boro Zone" +97,"Brooklyn","Fort Greene","Boro Zone" +98,"Queens","Fresh Meadows","Boro Zone" +99,"Staten Island","Freshkills Park","Boro Zone" +100,"Manhattan","Garment District","Yellow Zone" +101,"Queens","Glen Oaks","Boro Zone" +102,"Queens","Glendale","Boro Zone" +103,"Manhattan","Governor's Island/Ellis Island/Liberty Island","Yellow Zone" +104,"Manhattan","Governor's Island/Ellis Island/Liberty Island","Yellow Zone" +105,"Manhattan","Governor's Island/Ellis Island/Liberty Island","Yellow Zone" +106,"Brooklyn","Gowanus","Boro Zone" +107,"Manhattan","Gramercy","Yellow Zone" +108,"Brooklyn","Gravesend","Boro Zone" +109,"Staten Island","Great Kills","Boro Zone" +110,"Staten Island","Great Kills Park","Boro Zone" +111,"Brooklyn","Green-Wood Cemetery","Boro Zone" +112,"Brooklyn","Greenpoint","Boro Zone" +113,"Manhattan","Greenwich Village North","Yellow Zone" +114,"Manhattan","Greenwich Village South","Yellow Zone" +115,"Staten Island","Grymes Hill/Clifton","Boro Zone" +116,"Manhattan","Hamilton Heights","Boro Zone" +117,"Queens","Hammels/Arverne","Boro Zone" +118,"Staten Island","Heartland Village/Todt Hill","Boro Zone" +119,"Bronx","Highbridge","Boro Zone" +120,"Manhattan","Highbridge Park","Boro Zone" +121,"Queens","Hillcrest/Pomonok","Boro Zone" +122,"Queens","Hollis","Boro Zone" +123,"Brooklyn","Homecrest","Boro Zone" +124,"Queens","Howard Beach","Boro Zone" +125,"Manhattan","Hudson Sq","Yellow Zone" +126,"Bronx","Hunts Point","Boro Zone" +127,"Manhattan","Inwood","Boro Zone" +128,"Manhattan","Inwood Hill Park","Boro Zone" +129,"Queens","Jackson Heights","Boro Zone" +130,"Queens","Jamaica","Boro Zone" +131,"Queens","Jamaica Estates","Boro Zone" +132,"Queens","JFK Airport","Airports" +133,"Brooklyn","Kensington","Boro Zone" +134,"Queens","Kew Gardens","Boro Zone" +135,"Queens","Kew Gardens Hills","Boro Zone" +136,"Bronx","Kingsbridge Heights","Boro Zone" +137,"Manhattan","Kips Bay","Yellow Zone" +138,"Queens","LaGuardia Airport","Airports" +139,"Queens","Laurelton","Boro Zone" +140,"Manhattan","Lenox Hill East","Yellow Zone" +141,"Manhattan","Lenox Hill West","Yellow Zone" +142,"Manhattan","Lincoln Square East","Yellow Zone" +143,"Manhattan","Lincoln Square West","Yellow Zone" +144,"Manhattan","Little Italy/NoLiTa","Yellow Zone" +145,"Queens","Long Island City/Hunters Point","Boro Zone" +146,"Queens","Long Island City/Queens Plaza","Boro Zone" +147,"Bronx","Longwood","Boro Zone" +148,"Manhattan","Lower East Side","Yellow Zone" +149,"Brooklyn","Madison","Boro Zone" +150,"Brooklyn","Manhattan Beach","Boro Zone" +151,"Manhattan","Manhattan Valley","Yellow Zone" +152,"Manhattan","Manhattanville","Boro Zone" +153,"Manhattan","Marble Hill","Boro Zone" +154,"Brooklyn","Marine Park/Floyd Bennett Field","Boro Zone" +155,"Brooklyn","Marine Park/Mill Basin","Boro Zone" +156,"Staten Island","Mariners Harbor","Boro Zone" +157,"Queens","Maspeth","Boro Zone" +158,"Manhattan","Meatpacking/West Village West","Yellow Zone" +159,"Bronx","Melrose South","Boro Zone" +160,"Queens","Middle Village","Boro Zone" +161,"Manhattan","Midtown Center","Yellow Zone" +162,"Manhattan","Midtown East","Yellow Zone" +163,"Manhattan","Midtown North","Yellow Zone" +164,"Manhattan","Midtown South","Yellow Zone" +165,"Brooklyn","Midwood","Boro Zone" +166,"Manhattan","Morningside Heights","Boro Zone" +167,"Bronx","Morrisania/Melrose","Boro Zone" +168,"Bronx","Mott Haven/Port Morris","Boro Zone" +169,"Bronx","Mount Hope","Boro Zone" +170,"Manhattan","Murray Hill","Yellow Zone" +171,"Queens","Murray Hill-Queens","Boro Zone" +172,"Staten Island","New Dorp/Midland Beach","Boro Zone" +173,"Queens","North Corona","Boro Zone" +174,"Bronx","Norwood","Boro Zone" +175,"Queens","Oakland Gardens","Boro Zone" +176,"Staten Island","Oakwood","Boro Zone" +177,"Brooklyn","Ocean Hill","Boro Zone" +178,"Brooklyn","Ocean Parkway South","Boro Zone" +179,"Queens","Old Astoria","Boro Zone" +180,"Queens","Ozone Park","Boro Zone" +181,"Brooklyn","Park Slope","Boro Zone" +182,"Bronx","Parkchester","Boro Zone" +183,"Bronx","Pelham Bay","Boro Zone" +184,"Bronx","Pelham Bay Park","Boro Zone" +185,"Bronx","Pelham Parkway","Boro Zone" +186,"Manhattan","Penn Station/Madison Sq West","Yellow Zone" +187,"Staten Island","Port Richmond","Boro Zone" +188,"Brooklyn","Prospect-Lefferts Gardens","Boro Zone" +189,"Brooklyn","Prospect Heights","Boro Zone" +190,"Brooklyn","Prospect Park","Boro Zone" +191,"Queens","Queens Village","Boro Zone" +192,"Queens","Queensboro Hill","Boro Zone" +193,"Queens","Queensbridge/Ravenswood","Boro Zone" +194,"Manhattan","Randalls Island","Yellow Zone" +195,"Brooklyn","Red Hook","Boro Zone" +196,"Queens","Rego Park","Boro Zone" +197,"Queens","Richmond Hill","Boro Zone" +198,"Queens","Ridgewood","Boro Zone" +199,"Bronx","Rikers Island","Boro Zone" +200,"Bronx","Riverdale/North Riverdale/Fieldston","Boro Zone" +201,"Queens","Rockaway Park","Boro Zone" +202,"Manhattan","Roosevelt Island","Boro Zone" +203,"Queens","Rosedale","Boro Zone" +204,"Staten Island","Rossville/Woodrow","Boro Zone" +205,"Queens","Saint Albans","Boro Zone" +206,"Staten Island","Saint George/New Brighton","Boro Zone" +207,"Queens","Saint Michaels Cemetery/Woodside","Boro Zone" +208,"Bronx","Schuylerville/Edgewater Park","Boro Zone" +209,"Manhattan","Seaport","Yellow Zone" +210,"Brooklyn","Sheepshead Bay","Boro Zone" +211,"Manhattan","SoHo","Yellow Zone" +212,"Bronx","Soundview/Bruckner","Boro Zone" +213,"Bronx","Soundview/Castle Hill","Boro Zone" +214,"Staten Island","South Beach/Dongan Hills","Boro Zone" +215,"Queens","South Jamaica","Boro Zone" +216,"Queens","South Ozone Park","Boro Zone" +217,"Brooklyn","South Williamsburg","Boro Zone" +218,"Queens","Springfield Gardens North","Boro Zone" +219,"Queens","Springfield Gardens South","Boro Zone" +220,"Bronx","Spuyten Duyvil/Kingsbridge","Boro Zone" +221,"Staten Island","Stapleton","Boro Zone" +222,"Brooklyn","Starrett City","Boro Zone" +223,"Queens","Steinway","Boro Zone" +224,"Manhattan","Stuy Town/Peter Cooper Village","Yellow Zone" +225,"Brooklyn","Stuyvesant Heights","Boro Zone" +226,"Queens","Sunnyside","Boro Zone" +227,"Brooklyn","Sunset Park East","Boro Zone" +228,"Brooklyn","Sunset Park West","Boro Zone" +229,"Manhattan","Sutton Place/Turtle Bay North","Yellow Zone" +230,"Manhattan","Times Sq/Theatre District","Yellow Zone" +231,"Manhattan","TriBeCa/Civic Center","Yellow Zone" +232,"Manhattan","Two Bridges/Seward Park","Yellow Zone" +233,"Manhattan","UN/Turtle Bay South","Yellow Zone" +234,"Manhattan","Union Sq","Yellow Zone" +235,"Bronx","University Heights/Morris Heights","Boro Zone" +236,"Manhattan","Upper East Side North","Yellow Zone" +237,"Manhattan","Upper East Side South","Yellow Zone" +238,"Manhattan","Upper West Side North","Yellow Zone" +239,"Manhattan","Upper West Side South","Yellow Zone" +240,"Bronx","Van Cortlandt Park","Boro Zone" +241,"Bronx","Van Cortlandt Village","Boro Zone" +242,"Bronx","Van Nest/Morris Park","Boro Zone" +243,"Manhattan","Washington Heights North","Boro Zone" +244,"Manhattan","Washington Heights South","Boro Zone" +245,"Staten Island","West Brighton","Boro Zone" +246,"Manhattan","West Chelsea/Hudson Yards","Yellow Zone" +247,"Bronx","West Concourse","Boro Zone" +248,"Bronx","West Farms/Bronx River","Boro Zone" +249,"Manhattan","West Village","Yellow Zone" +250,"Bronx","Westchester Village/Unionport","Boro Zone" +251,"Staten Island","Westerleigh","Boro Zone" +252,"Queens","Whitestone","Boro Zone" +253,"Queens","Willets Point","Boro Zone" +254,"Bronx","Williamsbridge/Olinville","Boro Zone" +255,"Brooklyn","Williamsburg (North Side)","Boro Zone" +256,"Brooklyn","Williamsburg (South Side)","Boro Zone" +257,"Brooklyn","Windsor Terrace","Boro Zone" +258,"Queens","Woodhaven","Boro Zone" +259,"Bronx","Woodlawn/Wakefield","Boro Zone" +260,"Queens","Woodside","Boro Zone" +261,"Manhattan","World Trade Center","Yellow Zone" +262,"Manhattan","Yorkville East","Yellow Zone" +263,"Manhattan","Yorkville West","Yellow Zone" +264,"Unknown","N/A","N/A" +265,"N/A","Outside of NYC","N/A" diff --git a/slim-ci-lab/dbt/seeds/taxi_zone_lookup.yml b/slim-ci-lab/dbt/seeds/taxi_zone_lookup.yml new file mode 100644 index 0000000..9ec5801 --- /dev/null +++ b/slim-ci-lab/dbt/seeds/taxi_zone_lookup.yml @@ -0,0 +1,10 @@ +seeds: + - name: taxi_zone_lookup + description: > + NYC TLC taxi zone lookup, committed as published at + https://d37ci6vzurychx.cloudfront.net/misc/taxi_zone_lookup.csv + (SHA-256 1a99e105092230f8620f301edcca7f80d3080642ff404d28ed957d3fa222c8ed), + with line endings normalised to LF. + columns: + - name: LocationID + data_tests: [unique, not_null] diff --git a/slim-ci-lab/dbt/snapshots/dim_zone.yml b/slim-ci-lab/dbt/snapshots/dim_zone.yml new file mode 100644 index 0000000..9ebddf0 --- /dev/null +++ b/slim-ci-lab/dbt/snapshots/dim_zone.yml @@ -0,0 +1,16 @@ +snapshots: + - name: dim_zone + description: > + Type 2 history of TLC taxi zones. A changed borough, zone name or service + zone closes the current row and opens a new one; a zone removed from the + lookup gets a new row with dbt_is_deleted = 'True'. + relation: ref('stg_taxi_zones') + config: + schema: snapshots + unique_key: location_id + strategy: check + check_cols: [borough, zone_name, service_zone] + hard_deletes: new_record + columns: + - name: location_id + data_tests: [not_null] diff --git a/slim-ci-lab/demos/bad-unique-key.patch b/slim-ci-lab/demos/bad-unique-key.patch new file mode 100644 index 0000000..c4daabd --- /dev/null +++ b/slim-ci-lab/demos/bad-unique-key.patch @@ -0,0 +1,20 @@ +Subject: [PATCH] int_trips_enriched: merge on the reporting grain + +The daily mart only needs one row per pickup zone and day, so key the +incremental merge on (pickup_location_id, pickup_date) instead of trip_id. + +(Demo patch: this key is not unique. The first run builds the table with +CREATE TABLE AS and every test passes; the second run is a MERGE, and +Snowflake rejects it with 100090 "Duplicate row detected during DML action".) +--- +--- a/models/intermediate/int_trips_enriched.sql ++++ b/models/intermediate/int_trips_enriched.sql +@@ -2,7 +2,7 @@ + config( + materialized='incremental', + incremental_strategy='merge', +- unique_key='trip_id', ++ unique_key=['pickup_location_id', 'pickup_date'], + on_schema_change='append_new_columns' + ) + }} diff --git a/slim-ci-lab/demos/example-pr.patch b/slim-ci-lab/demos/example-pr.patch new file mode 100644 index 0000000..9a1486e --- /dev/null +++ b/slim-ci-lab/demos/example-pr.patch @@ -0,0 +1,19 @@ +Subject: [PATCH] fct_daily_zone_revenue: add tip_rate + +Tips as a share of fares, for the driver-earnings dashboard. + +(Example pull request used by `make ci-local`: it modifies one mart, so +Slim CI builds only that model and its tests, deferring every upstream +reference to production.) +--- +--- a/models/marts/fct_daily_zone_revenue.sql ++++ b/models/marts/fct_daily_zone_revenue.sql +@@ -10,6 +10,8 @@ + sum(total_amount) as revenue, + sum(fare_amount) as fare_revenue, + sum(tip_amount) as tip_revenue, ++ -- Tips as a share of fares, for the driver-earnings dashboard. ++ round(sum(tip_amount) / nullif(sum(fare_amount), 0), 4) as tip_rate, + round(avg(trip_distance_miles), 2) as avg_trip_distance_miles + from {{ ref('int_trips_enriched') }} + group by pickup_date, pickup_location_id, pickup_borough, pickup_zone diff --git a/slim-ci-lab/docker-compose.yml b/slim-ci-lab/docker-compose.yml new file mode 100644 index 0000000..99ce0ec --- /dev/null +++ b/slim-ci-lab/docker-compose.yml @@ -0,0 +1,18 @@ +# Snowflake-Next emulator for Slim CI Lab. dbt and the loader run on the host +# (or the CI runner) and connect through the published port. +services: + snowflake: + image: ${SNOWFLAKE_IMAGE:-localstack/snowflake-next:latest} + pull_policy: always + ports: + - "127.0.0.1:${SNOWFLAKE_PORT:-4566}:4566" + environment: + LOCALSTACK_AUTH_TOKEN: ${LOCALSTACK_AUTH_TOKEN:?set LOCALSTACK_AUTH_TOKEN} + volumes: + - ./init:/etc/localstack/init/ready.d:ro + networks: + default: + aliases: + - snowflake.localhost.localstack.cloud + - stages.s3.localhost.localstack.cloud + - stages-enc.s3.localhost.localstack.cloud diff --git a/slim-ci-lab/docs/architecture.md b/slim-ci-lab/docs/architecture.md new file mode 100644 index 0000000..45c3753 --- /dev/null +++ b/slim-ci-lab/docs/architecture.md @@ -0,0 +1,92 @@ +# Slim CI Lab: design notes + +How data gets in, how the dbt project is layered, where each target builds, and why the pipeline catches a bad `unique_key` on the first CI run. + +## Components + +| Component | Runs where | Role | +| --- | --- | --- | +| `localstack/snowflake-next` | Docker, `127.0.0.1:${SNOWFLAKE_PORT}` | The warehouse: stages, tables, MERGE, CLONE, snapshots | +| `init/01_platform.sf.sql` | Inside the emulator at startup | What an admin provides: warehouse `TRANSFORMING`, role `TRANSFORMER`, user `SAMPLE_SVC`, database `SLIM_CI` | +| `slim_ci_lab.loader` | Host or CI runner | Download, verify, downsample, `PUT`, `COPY INTO` | +| dbt Core + dbt-snowflake | Host or CI runner | Models, tests, unit tests, snapshot, state comparison | +| `scripts/ci_local.sh` | Host or CI runner | The Slim CI pipeline; GitHub Actions runs the same script | + +Compose binds the emulator to loopback only. Every client uses key-pair (`SNOWFLAKE_JWT`) auth with a throwaway key from `make up`, the same configuration a real account uses for service users. + +## Data flow + +```mermaid +flowchart TB + subgraph raw["RAW (loader)"] + trips[("TRIPS
TLC columns + _source_file, _source_row, _loaded_at")] + end + subgraph reference["REFERENCE (dbt seed)"] + lookup[("TAXI_ZONE_LOOKUP")] + end + subgraph staging["STAGING (views)"] + stg_trips["stg_trips
rename, money as NUMBER(10,2)"] + stg_zones["stg_taxi_zones"] + end + subgraph intermediate["INTERMEDIATE"] + enriched["int_trips_enriched
incremental MERGE, unique_key trip_id"] + end + subgraph marts["MARTS"] + fct["fct_daily_zone_revenue
grain: pickup_date x pickup zone"] + end + subgraph snapshots["SNAPSHOTS"] + dim["dim_zone
check strategy, hard_deletes new_record"] + end + trips --> stg_trips --> enriched --> fct + lookup --> stg_zones --> enriched + stg_zones --> dim +``` + +- **Pinned input.** `tlc.py` names the exact URL and SHA-256 of `yellow_tripdata_2024-01.parquet`. A mismatch deletes the cached copy and fails. +- **Deterministic downsample.** The extract sorts by every column and keeps every 300th row (9,883 rows), so tests can compute expected values from the file. +- **Load.** `PUT ... OVERWRITE = FALSE`, then `COPY INTO RAW.TRIPS ... MATCH_BY_COLUMN_NAME = CASE_INSENSITIVE ON_ERROR = ABORT_STATEMENT` with `INCLUDE_METADATA` for lineage. The Parquet file format sets `USE_LOGICAL_TYPE = TRUE`, Snowflake's recommended setting for Parquet logical types such as timestamps. +- **Idempotency.** `COPY` load metadata means a second run reports `Copy executed with 0 files processed.`, so CI can retry the step. +- **Surrogate key.** TLC records have no id. `trip_id` is the MD5 of vendor, pickup and dropoff timestamps and zones plus `natural_key_seq`, a `row_number()` over those columns, so repeated records get distinct, stable keys. +- **Lookback window.** Incremental runs re-select trips from `max(pickup_at) - 3 days` (`var('incremental_lookback_days')`) and MERGE them on `trip_id`, so late or corrected trips update in place. + +## Targets and schemas + +`dbt/macros/generate_schema_name.sql` decides where each node lands: + +| Target | Connection | `staging` model lands in | Purpose | +| --- | --- | --- | --- | +| `dev` | emulator | `DEV__STAGING` | Personal sandbox | +| `prod` | emulator | `STAGING` | Production state that PRs defer to | +| `pr` | emulator | `PR_` | Every node of one PR in one schema | +| `cloud` | real account | `STAGING`, or `PR_` when `DBT_PR_NUMBER` is set | The same workflow on Snowflake | + +One schema per PR keeps isolation simple: two open PRs never write the same relation, and `drop_pr_schema` removes everything a PR created. Only the `cloud` target reads `SNOWFLAKE_ACCOUNT` and related variables, so a stray variable in a shell never redirects a local run to a billable account (`settings.py` follows the same rule). + +## Why clone, then defer + +```mermaid +sequenceDiagram + participant CI as ci_local.sh + participant PR as PR_102 schema + participant Prod as INTERMEDIATE (prod) + CI->>Prod: int_trips_enriched holds 9,762 billable trips + CI->>PR: dbt clone int_trips_enriched (zero-copy) + CI->>PR: dbt build --select state:modified+ --defer + PR->>PR: is_incremental() is true, so MERGE the lookback window + alt unique_key = trip_id + PR-->>CI: MERGE updates matched rows, tests pass + else unique_key = (pickup_location_id, pickup_date) + PR-->>CI: 100090 Duplicate row detected during DML action + end +``` + +Without the clone, a modified incremental model starts empty in the PR schema: dbt runs `CREATE TABLE AS`, every test passes, and the first `MERGE` happens in production. With a non-unique `unique_key`, each target row matches many source rows, and Snowflake rejects the nondeterministic MERGE (`ERROR_ON_NONDETERMINISTIC_MERGE` defaults to true). Cloning first makes the PR build run that MERGE. + +`--defer` points `ref()` for unselected nodes at production. Without it, the PR build of `fct_daily_zone_revenue` references a missing `PR_.INT_TRIPS_ENRICHED` and fails with `002003`; with it, the PR builds one table and reads everything else from production. + +## Test isolation and security + +- The pytest session creates `SLIM_CI_TEST_`, builds production state in a temporary copy of the project, and drops the database at the end. No test or demo edits `dbt/`. +- The emulator listens on `127.0.0.1` only. `.secrets/` is gitignored and created with mode 700/600. +- The `parity` workflow job writes the real private key from a secret and runs in the `snowflake-parity` GitHub environment, so it can require approval. +- Statements carry `QUERY_TAG = 'slim-ci-lab'`, so CI load is easy to find in query history. diff --git a/slim-ci-lab/init/01_platform.sf.sql b/slim-ci-lab/init/01_platform.sf.sql new file mode 100644 index 0000000..922c9d5 --- /dev/null +++ b/slim-ci-lab/init/01_platform.sf.sql @@ -0,0 +1,27 @@ +-- Platform objects a data team gets from its Snowflake admins: a warehouse, +-- a TRANSFORMER role for dbt and the loader, a service user, and the +-- analytics database. Runs once at emulator startup (as role PUBLIC). +-- The loader creates RAW objects idempotently itself, so the same pipeline +-- also runs on a real account where this script is applied by an admin. +USE ROLE ACCOUNTADMIN; + +CREATE WAREHOUSE IF NOT EXISTS TRANSFORMING + WAREHOUSE_SIZE = XSMALL + AUTO_SUSPEND = 60 + AUTO_RESUME = TRUE + INITIALLY_SUSPENDED = TRUE; + +CREATE ROLE IF NOT EXISTS TRANSFORMER; +GRANT ROLE TRANSFORMER TO ROLE SYSADMIN; +GRANT USAGE ON WAREHOUSE TRANSFORMING TO ROLE TRANSFORMER; +-- The test suite creates one throwaway database per run. +GRANT CREATE DATABASE ON ACCOUNT TO ROLE TRANSFORMER; + +CREATE USER IF NOT EXISTS SAMPLE_SVC + TYPE = SERVICE + DEFAULT_ROLE = TRANSFORMER + DEFAULT_WAREHOUSE = TRANSFORMING; +GRANT ROLE TRANSFORMER TO USER SAMPLE_SVC; + +CREATE DATABASE IF NOT EXISTS SLIM_CI; +GRANT OWNERSHIP ON DATABASE SLIM_CI TO ROLE TRANSFORMER COPY CURRENT GRANTS; diff --git a/slim-ci-lab/pyproject.toml b/slim-ci-lab/pyproject.toml new file mode 100644 index 0000000..1953944 --- /dev/null +++ b/slim-ci-lab/pyproject.toml @@ -0,0 +1,36 @@ +[project] +name = "slim-ci-lab" +version = "0.1.0" +description = "dbt Slim CI against the Snowflake-Next emulator: per-PR schemas, state:modified+ with defer, zero credits." +readme = "README.md" +license = "Apache-2.0" +requires-python = ">=3.12,<3.13" +dependencies = [ + # dbt-snowflake 1.10.x pins snowflake-connector-python <5, so the + # connector comes from here (4.x) rather than being pinned directly. + "dbt-core>=1.10,<1.11", + "dbt-snowflake>=1.10,<1.11", + "pandas>=2.2", + "pyarrow>=17", + "requests>=2.32", +] + +[dependency-groups] +dev = ["pytest>=8.3"] + +[project.scripts] +slim-ci-load = "slim_ci_lab.loader:main" +slim-ci-demo-bad-pr = "slim_ci_lab.bad_pr_demo:main" +slim-ci-summary = "slim_ci_lab.ci_summary:main" + +[build-system] +requires = ["uv_build>=0.10,<0.11"] +build-backend = "uv_build" + +[tool.pytest.ini_options] +testpaths = ["tests"] +addopts = "-ra --strict-markers" +filterwarnings = [ + # Raised inside the connector's vendored urllib3, not by this project. + "ignore:X509.get_subject is deprecated:DeprecationWarning", +] diff --git a/slim-ci-lab/scripts/ci_local.sh b/slim-ci-lab/scripts/ci_local.sh new file mode 100755 index 0000000..055c15d --- /dev/null +++ b/slim-ci-lab/scripts/ci_local.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash +# Slim CI for dbt, the same steps GitHub Actions runs (.github-example/workflows/dbt-ci.yml): +# +# 1. load raw data PUT + COPY INTO (idempotent: re-runs load 0 files) +# 2. production state dbt build --target prod, save manifest.json +# 3. PR schema drop pr_, zero-copy clone modified incremental models from prod +# 4. PR build dbt build --select state:modified+ --defer --state +# 5. summary nodes built, wall-clock, credits +# +# Environment (all optional): +# PROD_PROJECT_DIR dbt project for production state (default: dbt/, i.e. main) +# PR_PROJECT_DIR dbt project of the pull request. Default: a temporary copy +# of dbt/ with CI_PR_PATCH applied. +# CI_PR_PATCH patch that simulates the PR locally (default: demos/example-pr.patch) +# PROD_STATE build (default) or reuse: reuse an existing $STATE_DIR/manifest.json +# STATE_DIR where the production manifest is kept (default: prod-artifacts) +# PROD_TARGET / PR_TARGET dbt targets (default prod / pr; both "cloud" for real Snowflake) +# DBT_PR_NUMBER pull request number; models build into schema pr_ (default: local) +set -euo pipefail + +root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$root" +start=$SECONDS + +export DBT_PROFILES_DIR="${DBT_PROFILES_DIR:-$root/dbt}" +export SNOWFLAKE_PRIVATE_KEY_PATH="${SNOWFLAKE_PRIVATE_KEY_PATH:-$root/.secrets/rsa_key.p8}" +export DBT_PR_NUMBER="${DBT_PR_NUMBER:-local}" +PROD_PROJECT_DIR="${PROD_PROJECT_DIR:-$root/dbt}" +PR_PROJECT_DIR="${PR_PROJECT_DIR:-}" +CI_PR_PATCH="$(realpath "${CI_PR_PATCH:-$root/demos/example-pr.patch}")" +PROD_STATE="${PROD_STATE:-build}" +STATE_DIR="$(mkdir -p "${STATE_DIR:-$root/prod-artifacts}" && cd "${STATE_DIR:-$root/prod-artifacts}" && pwd)" +PROD_TARGET="${PROD_TARGET:-prod}" +PR_TARGET="${PR_TARGET:-pr}" +RUN="${RUN:-uv run --frozen}" + +step() { printf '\n==> %s\n' "$*"; } +dbt() { $RUN dbt --no-use-colors "$@"; } + +if [[ -z "$PR_PROJECT_DIR" ]]; then + scratch="$(mktemp -d)" + trap 'rm -rf "$scratch"' EXIT + PR_PROJECT_DIR="$scratch/dbt" + cp -R "$root/dbt" "$PR_PROJECT_DIR" + rm -rf "$PR_PROJECT_DIR/target" "$PR_PROJECT_DIR/logs" + patch --batch --forward --silent -p1 -d "$PR_PROJECT_DIR" -i "$CI_PR_PATCH" + echo "Simulating a pull request: $(sed -n 's/^Subject: //p' "$CI_PR_PATCH")" +fi + +step "1/5 Load raw data (PUT + COPY INTO)" +$RUN slim-ci-load + +if [[ "$PROD_STATE" == "reuse" && -f "$STATE_DIR/manifest.json" ]]; then + step "2/5 Production state: reusing $STATE_DIR/manifest.json" +else + step "2/5 Build production state (target: $PROD_TARGET)" + (unset DBT_PR_NUMBER; dbt build --target "$PROD_TARGET" --project-dir "$PROD_PROJECT_DIR") + cp "$PROD_PROJECT_DIR/target/manifest.json" "$STATE_DIR/manifest.json" +fi + +step "3/5 Fresh PR schema; clone modified incremental models from production" +dbt run-operation drop_pr_schema --target "$PR_TARGET" --project-dir "$PR_PROJECT_DIR" +# A modified incremental model would otherwise be built from scratch in the +# PR schema (CREATE TABLE AS) and never exercise its MERGE. Cloning the +# production table first makes the PR build run the same MERGE production's +# next run will, so a bad unique_key fails here instead of after merge. +dbt clone --target "$PR_TARGET" --project-dir "$PR_PROJECT_DIR" --state "$STATE_DIR" \ + --select "state:modified+,config.materialized:incremental,state:old" + +step "4/5 PR build: dbt build --select state:modified+ --defer" +status=0 +dbt build --target "$PR_TARGET" --project-dir "$PR_PROJECT_DIR" \ + --select state:modified+ --defer --state "$STATE_DIR" || status=$? + +step "5/5 Summary" +$RUN slim-ci-summary --run-results "$PR_PROJECT_DIR/target/run_results.json" \ + --manifest "$PR_PROJECT_DIR/target/manifest.json" --seconds $((SECONDS - start)) +exit "$status" diff --git a/slim-ci-lab/scripts/wait-for-emulator.sh b/slim-ci-lab/scripts/wait-for-emulator.sh new file mode 100755 index 0000000..501c7e2 --- /dev/null +++ b/slim-ci-lab/scripts/wait-for-emulator.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# Wait until the Snowflake-Next emulator answers a SQL statement. +# +# Copy this file into a sample as scripts/wait-for-emulator.sh (samples stay +# self-contained). Usage: scripts/wait-for-emulator.sh [timeout-seconds] +# Reads SNOWFLAKE_PORT (default 4566). Readiness is checked from the host. +set -euo pipefail + +port="${SNOWFLAKE_PORT:-4566}" +timeout="${1:-120}" +url="https://snowflake.localhost.localstack.cloud:${port}" +deadline=$((SECONDS + timeout)) + +until curl -fsS --compressed -o /dev/null -X POST "${url}/api/v2/statements" \ + -H 'Authorization: Bearer local' -H 'Content-Type: application/json' \ + -H 'Accept: application/json' -d '{"statement":"SELECT 1"}' 2>/dev/null; do + if (( SECONDS >= deadline )); then + echo "Emulator at ${url} did not become ready within ${timeout}s." >&2 + echo "Check 'docker compose logs snowflake' (is LOCALSTACK_AUTH_TOKEN valid? is port ${port} free?)." >&2 + exit 1 + fi + sleep 2 +done +echo "Emulator ready at ${url}" diff --git a/slim-ci-lab/src/slim_ci_lab/__init__.py b/slim-ci-lab/src/slim_ci_lab/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/slim-ci-lab/src/slim_ci_lab/bad_pr_demo.py b/slim-ci-lab/src/slim_ci_lab/bad_pr_demo.py new file mode 100644 index 0000000..164e2d0 --- /dev/null +++ b/slim-ci-lab/src/slim_ci_lab/bad_pr_demo.py @@ -0,0 +1,92 @@ +"""Demo: a pull request that passes CI once and breaks production on run two. + +demos/bad-unique-key.patch changes the incremental model's unique_key to a +non-unique pair. The first run creates the table (CREATE TABLE AS) and every +test passes. The second run is a MERGE, which Snowflake rejects with error +100090 "Duplicate row detected during DML action" because several source +rows match one target row. Running the model twice in CI catches it before +review; this demo exits 0 only when exactly that failure is observed. +""" + +from __future__ import annotations + +import argparse +import subprocess +import sys +import tempfile +from dataclasses import dataclass +from pathlib import Path + +from slim_ci_lab import loader, settings +from slim_ci_lab.dbt_runner import apply_patch, copy_project, dbt_env, run_dbt, targets + +PATCH = settings.SAMPLE_ROOT / "demos" / "bad-unique-key.patch" +PR_NUMBER = "bad_unique_key" +MODEL = "int_trips_enriched" +EXPECTED_CODE = "100090" +EXPECTED_MESSAGE = "Duplicate row detected during DML action" + + +@dataclass(frozen=True) +class DemoResult: + first: subprocess.CompletedProcess[str] + second: subprocess.CompletedProcess[str] + + @property + def second_output(self) -> str: + return self.second.stdout + self.second.stderr + + @property + def expected_failure_observed(self) -> bool: + return ( + self.first.returncode == 0 + and self.second.returncode != 0 + and EXPECTED_CODE in self.second_output + and EXPECTED_MESSAGE in self.second_output + ) + + +def run(database: str, patch: Path = PATCH, echo: bool = False) -> DemoResult: + """Apply `patch` to a temporary copy of the project and run the model twice.""" + with tempfile.TemporaryDirectory(prefix="slim-ci-bad-pr-") as tmp: + project = copy_project(Path(tmp) / "dbt") + apply_patch(project, patch) + env = dbt_env(DBT_DATABASE=database, DBT_PR_NUMBER=PR_NUMBER) + target = ["--target", targets()[1]] + run_dbt(["run-operation", "drop_pr_schema", *target], project, env) + _banner(echo, "Run 1: dbt build --select +int_trips_enriched (fresh PR schema)") + first = run_dbt(["build", *target, "--select", f"+{MODEL}"], project, env, echo=echo) + _banner(echo, "Run 2: dbt run --select int_trips_enriched (incremental MERGE)") + second = run_dbt(["run", *target, "--select", MODEL], project, env, echo=echo) + return DemoResult(first=first, second=second) + + +def _banner(echo: bool, text: str) -> None: + if echo: + print(f"\n=== {text} ===", flush=True) + + +def main(argv: list[str] | None = None) -> None: + parser = argparse.ArgumentParser(description="Show the second-run MERGE failure of a non-unique unique_key.") + parser.add_argument("--database", default=settings.load().database) + args = parser.parse_args(argv) + + print(f"Applying {PATCH.relative_to(settings.SAMPLE_ROOT)} to a temporary copy of the dbt project") + result = loader.load(args.database) + print(f"Raw data ready: {result.total_rows} trips in {args.database}.RAW.TRIPS") + demo = run(args.database, echo=True) + + print("\n=== Result ===") + print(f"Run 1 exit code: {demo.first.returncode} (expected 0: CREATE TABLE AS, all tests pass)") + print(f"Run 2 exit code: {demo.second.returncode} (expected non-zero: MERGE with a non-unique key)") + if demo.expected_failure_observed: + print(f'Caught on run 2: {EXPECTED_CODE} "{EXPECTED_MESSAGE}".') + print("A single run would have shipped this bug. Fix: unique_key='trip_id'.") + print("make ci-local catches it on the first run: CI_PR_PATCH=demos/bad-unique-key.patch make ci-local") + return + print(f'Did NOT observe the expected {EXPECTED_CODE} failure on run 2.', file=sys.stderr) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/slim-ci-lab/src/slim_ci_lab/ci_summary.py b/slim-ci-lab/src/slim_ci_lab/ci_summary.py new file mode 100644 index 0000000..651848e --- /dev/null +++ b/slim-ci-lab/src/slim_ci_lab/ci_summary.py @@ -0,0 +1,60 @@ +"""Print the Slim CI summary from dbt's run_results.json and manifest.json. + +Also appends it to $GITHUB_STEP_SUMMARY when running in GitHub Actions. +""" + +from __future__ import annotations + +import argparse +import json +import os +from collections import Counter +from pathlib import Path + +from slim_ci_lab import settings + +BUILDABLE = {"model", "seed", "snapshot", "test", "unit_test"} + + +def summarize(run_results: Path, manifest: Path, seconds: int) -> list[str]: + results = json.loads(run_results.read_text())["results"] + nodes = json.loads(manifest.read_text()) + all_ids = [uid for uid, node in nodes["nodes"].items() if node["resource_type"] in BUILDABLE] + all_ids += list(nodes.get("unit_tests", {})) + built = {r["unique_id"] for r in results} + statuses = Counter(r["status"] for r in results) + kinds = Counter(uid.split(".", 1)[0] for uid in built) + models = sorted(uid.rsplit(".", 1)[-1] for uid in built if uid.startswith(("model.", "snapshot.", "seed."))) + pr_schema = next((nodes["nodes"][uid]["schema"] for uid in built if uid.startswith("model.")), "-") + + if settings.is_emulator(): + credits = "Snowflake credits used: 0 (Snowflake-Next emulator)" + else: + credits = "Snowflake credits used: billable, see SNOWFLAKE.ACCOUNT_USAGE.WAREHOUSE_METERING_HISTORY" + return [ + f"PR schema: {pr_schema}", + f"Selected: {', '.join(models) or 'nothing (no modified models)'}", + "Nodes run: " + ", ".join(f"{n} {k}" for k, n in sorted(kinds.items())) if kinds else "Nodes run: 0", + "Results: " + ", ".join(f"{n} {s}" for s, n in sorted(statuses.items())) if statuses else "Results: -", + f"Deferred to prod: {len(set(all_ids) - built)} of {len(all_ids)} nodes not rebuilt", + f"Wall-clock: {seconds // 60}m {seconds % 60:02d}s", + credits, + ] + + +def main(argv: list[str] | None = None) -> None: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--run-results", type=Path, required=True) + parser.add_argument("--manifest", type=Path, required=True) + parser.add_argument("--seconds", type=int, required=True) + args = parser.parse_args(argv) + lines = summarize(args.run_results, args.manifest, args.seconds) + print("\n".join(lines)) + step_summary = os.getenv("GITHUB_STEP_SUMMARY") + if step_summary: + with Path(step_summary).open("a") as fh: + fh.write("### Slim CI\n\n```\n" + "\n".join(lines) + "\n```\n") + + +if __name__ == "__main__": + main() diff --git a/slim-ci-lab/src/slim_ci_lab/dbt_runner.py b/slim-ci-lab/src/slim_ci_lab/dbt_runner.py new file mode 100644 index 0000000..048ae57 --- /dev/null +++ b/slim-ci-lab/src/slim_ci_lab/dbt_runner.py @@ -0,0 +1,68 @@ +"""Run dbt as a subprocess against a (possibly patched) copy of the project. + +Patched copies stand in for a pull request branch: the demo and the tests +apply a patch from demos/ to a temporary copy, never to dbt/ itself. +""" + +from __future__ import annotations + +import os +import shutil +import subprocess +import sys +from collections.abc import Mapping, Sequence +from pathlib import Path + +from slim_ci_lab.settings import DBT_PROJECT_DIR, SAMPLE_ROOT, is_emulator + +DBT = Path(sys.executable).with_name("dbt") + + +def targets() -> tuple[str, str]: + """(production target, pull request target) for the configured TEST_TARGET.""" + return ("prod", "pr") if is_emulator() else ("cloud", "cloud") + + +def copy_project(dest: Path, source: Path = DBT_PROJECT_DIR) -> Path: + """Copy the dbt project to `dest` (without build output) and return it.""" + shutil.copytree(source, dest, ignore=shutil.ignore_patterns("target", "logs", "dbt_packages")) + return dest + + +def apply_patch(project_dir: Path, patch_file: Path) -> None: + subprocess.run( + ["patch", "--batch", "--forward", "--silent", "-p1", "-d", str(project_dir), "-i", str(patch_file.resolve())], + check=True, + ) + + +def dbt_env(**overrides: str) -> dict[str, str]: + env = dict(os.environ) + env.setdefault("DBT_PROFILES_DIR", str(DBT_PROJECT_DIR)) + key = env.get("SNOWFLAKE_PRIVATE_KEY_PATH") or ".secrets/rsa_key.p8" + key_path = Path(key).expanduser() + env["SNOWFLAKE_PRIVATE_KEY_PATH"] = str(key_path if key_path.is_absolute() else SAMPLE_ROOT / key_path) + env.update(overrides) + return env + + +def run_dbt( + args: Sequence[str], + project_dir: Path = DBT_PROJECT_DIR, + env: Mapping[str, str] | None = None, + echo: bool = False, +) -> subprocess.CompletedProcess[str]: + """Run `dbt ` in `project_dir`. Output is captured (and echoed if asked).""" + cmd = [str(DBT), "--no-use-colors", *args, "--project-dir", str(project_dir)] + proc = subprocess.run( + cmd, + cwd=SAMPLE_ROOT, + env=dict(env) if env is not None else dbt_env(), + capture_output=True, + text=True, + check=False, + ) + if echo: + sys.stdout.write(proc.stdout) + sys.stderr.write(proc.stderr) + return proc diff --git a/slim-ci-lab/src/slim_ci_lab/loader.py b/slim-ci-lab/src/slim_ci_lab/loader.py new file mode 100644 index 0000000..81caafe --- /dev/null +++ b/slim-ci-lab/src/slim_ci_lab/loader.py @@ -0,0 +1,124 @@ +"""Load the downsampled TLC extract into .RAW.TRIPS. + +PUT uploads the Parquet file to an internal stage; COPY INTO matches Parquet +columns to table columns by name and records lineage (file name, row number, +scan time) through INCLUDE_METADATA. COPY keeps per-file load metadata, so +re-running the loader loads 0 new files: the step is idempotent and safe to +retry in CI. +""" + +from __future__ import annotations + +import argparse +from dataclasses import dataclass +from pathlib import Path + +from slim_ci_lab import settings, tlc + +STAGE_PATH = "yellow/2024-01/" + +RAW_DDL = ( + "CREATE DATABASE IF NOT EXISTS {db}", + "CREATE SCHEMA IF NOT EXISTS {db}.RAW", + # USE_LOGICAL_TYPE reads Parquet logical types (TIMESTAMP, DECIMAL) as + # typed values; it is Snowflake's recommended setting for Parquet loads. + """CREATE FILE FORMAT IF NOT EXISTS {db}.RAW.TLC_PARQUET + TYPE = PARQUET + USE_LOGICAL_TYPE = TRUE""", + """CREATE STAGE IF NOT EXISTS {db}.RAW.TLC_LANDING + FILE_FORMAT = (FORMAT_NAME = '{db}.RAW.TLC_PARQUET') + COMMENT = 'NYC TLC trip files, landed as published'""", + # Column names follow the TLC data dictionary so MATCH_BY_COLUMN_NAME can + # map them; renaming and typing happens in dbt staging. + """CREATE TABLE IF NOT EXISTS {db}.RAW.TRIPS ( + VENDORID INTEGER, + TPEP_PICKUP_DATETIME TIMESTAMP_NTZ, + TPEP_DROPOFF_DATETIME TIMESTAMP_NTZ, + PASSENGER_COUNT INTEGER, + TRIP_DISTANCE FLOAT, + RATECODEID INTEGER, + STORE_AND_FWD_FLAG VARCHAR(1), + PULOCATIONID INTEGER, + DOLOCATIONID INTEGER, + PAYMENT_TYPE INTEGER, + FARE_AMOUNT FLOAT, + EXTRA FLOAT, + MTA_TAX FLOAT, + TIP_AMOUNT FLOAT, + TOLLS_AMOUNT FLOAT, + IMPROVEMENT_SURCHARGE FLOAT, + TOTAL_AMOUNT FLOAT, + CONGESTION_SURCHARGE FLOAT, + AIRPORT_FEE FLOAT, + _SOURCE_FILE VARCHAR, + _SOURCE_ROW INTEGER, + _LOADED_AT TIMESTAMP_LTZ + ) COMMENT = 'Yellow taxi trips as delivered by NYC TLC (downsampled)'""", +) + +COPY_SQL = """COPY INTO {db}.RAW.TRIPS + FROM @{db}.RAW.TLC_LANDING/{path} + MATCH_BY_COLUMN_NAME = CASE_INSENSITIVE + INCLUDE_METADATA = ( + _SOURCE_FILE = METADATA$FILENAME, + _SOURCE_ROW = METADATA$FILE_ROW_NUMBER, + _LOADED_AT = METADATA$START_SCAN_TIME + ) + ON_ERROR = ABORT_STATEMENT""" + + +@dataclass(frozen=True) +class LoadResult: + files_loaded: int + rows_loaded: int + total_rows: int + + +def ensure_raw_objects(cursor, database: str) -> None: + for statement in RAW_DDL: + cursor.execute(statement.format(db=database)) + + +def put_file(cursor, database: str, path: Path, stage_path: str = STAGE_PATH) -> None: + # OVERWRITE = FALSE keeps the staged file (and its load metadata) stable. + cursor.execute( + f"PUT 'file://{path.resolve().as_posix()}' @{database}.RAW.TLC_LANDING/{stage_path} " + "AUTO_COMPRESS = FALSE OVERWRITE = FALSE" + ) + + +def copy_into_trips(cursor, database: str, stage_path: str = STAGE_PATH) -> tuple[int, int]: + """Run COPY INTO and return (files loaded, rows loaded).""" + rows = cursor.execute(COPY_SQL.format(db=database, path=stage_path)).fetchall() + columns = [col[0].lower() for col in cursor.description] + if "rows_loaded" not in columns: # one row: "Copy executed with 0 files processed." + return 0, 0 + status, loaded = columns.index("status"), columns.index("rows_loaded") + done = [row for row in rows if row[status] == "LOADED"] + return len(done), sum(int(row[loaded]) for row in done) + + +def load(database: str, path: Path | None = None) -> LoadResult: + path = path or tlc.sample_file() + with settings.connect() as conn: + cursor = conn.cursor() + ensure_raw_objects(cursor, database) + put_file(cursor, database, path) + files, rows = copy_into_trips(cursor, database) + (total,) = cursor.execute(f"SELECT COUNT(*) FROM {database}.RAW.TRIPS").fetchone() + return LoadResult(files_loaded=files, rows_loaded=rows, total_rows=int(total)) + + +def main(argv: list[str] | None = None) -> None: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--database", default=settings.load().database) + args = parser.parse_args(argv) + result = load(args.database) + print( + f"{args.database}.RAW.TRIPS: {result.files_loaded} file(s) and {result.rows_loaded} row(s) " + f"loaded this run; {result.total_rows} row(s) in total" + ) + + +if __name__ == "__main__": + main() diff --git a/slim-ci-lab/src/slim_ci_lab/settings.py b/slim-ci-lab/src/slim_ci_lab/settings.py new file mode 100644 index 0000000..2236199 --- /dev/null +++ b/slim-ci-lab/src/slim_ci_lab/settings.py @@ -0,0 +1,122 @@ +"""Connection settings, read from the environment in one place. + +TEST_TARGET picks the connection: + +* EMULATOR (default): the Snowflake-Next emulator on + snowflake.localhost.localstack.cloud:${SNOWFLAKE_PORT}. Cloud SNOWFLAKE_* + variables are deliberately ignored, so a stray real-account variable in + your shell can never send emulator runs to a billable account. +* SNOWFLAKE_CLOUD: a real account from SNOWFLAKE_ACCOUNT, SNOWFLAKE_USER, + SNOWFLAKE_PRIVATE_KEY_PATH, SNOWFLAKE_ROLE and SNOWFLAKE_WAREHOUSE. + +Both use key-pair (SNOWFLAKE_JWT) auth, the configuration a real account's +service users need; `make up` generates a throwaway key pair for local runs. +""" + +from __future__ import annotations + +import os +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +from cryptography.hazmat.primitives import serialization + +SAMPLE_ROOT = Path(__file__).resolve().parents[2] +DBT_PROJECT_DIR = SAMPLE_ROOT / "dbt" +CACHE_DIR = SAMPLE_ROOT / ".cache" + +EMULATOR_HOST = "snowflake.localhost.localstack.cloud" +EMULATOR_ACCOUNT = "snowflake" +DEFAULT_USER = "SAMPLE_SVC" +DEFAULT_ROLE = "TRANSFORMER" +DEFAULT_WAREHOUSE = "TRANSFORMING" +DEFAULT_DATABASE = "SLIM_CI" +QUERY_TAG = "slim-ci-lab" + + +def test_target() -> str: + value = os.getenv("TEST_TARGET", "EMULATOR").strip().upper() + if value not in {"EMULATOR", "SNOWFLAKE_CLOUD"}: + raise ValueError(f"TEST_TARGET must be EMULATOR or SNOWFLAKE_CLOUD, got {value!r}") + return value + + +def is_emulator() -> bool: + return test_target() == "EMULATOR" + + +def _required(name: str) -> str: + value = os.getenv(name, "").strip() + if not value: + raise RuntimeError(f"{name} must be set when TEST_TARGET=SNOWFLAKE_CLOUD") + return value + + +def _key_path() -> Path: + raw = os.getenv("SNOWFLAKE_PRIVATE_KEY_PATH", "").strip() + if not raw: + if not is_emulator(): + _required("SNOWFLAKE_PRIVATE_KEY_PATH") + raw = ".secrets/rsa_key.p8" + path = Path(raw).expanduser() + return path if path.is_absolute() else SAMPLE_ROOT / path + + +@dataclass(frozen=True) +class Settings: + account: str + user: str + role: str + warehouse: str + database: str + private_key_path: Path + host: str | None = None + port: int | None = None + + def connect_kwargs(self, **overrides: Any) -> dict[str, Any]: + """Keyword arguments for snowflake.connector.connect().""" + key = serialization.load_pem_private_key(self.private_key_path.read_bytes(), password=None) + kwargs: dict[str, Any] = { + "account": self.account, + "user": self.user, + "role": self.role, + "warehouse": self.warehouse, + "authenticator": "SNOWFLAKE_JWT", + "private_key": key, + "session_parameters": {"QUERY_TAG": QUERY_TAG}, + } + if self.host: + kwargs.update(host=self.host, port=self.port, protocol="https") + kwargs.update(overrides) + return kwargs + + +def load(database: str | None = None) -> Settings: + database = database or os.getenv("DBT_DATABASE", DEFAULT_DATABASE) + if is_emulator(): + return Settings( + account=EMULATOR_ACCOUNT, + user=DEFAULT_USER, + role=DEFAULT_ROLE, + warehouse=DEFAULT_WAREHOUSE, + database=database, + private_key_path=_key_path(), + host=EMULATOR_HOST, + port=int(os.getenv("SNOWFLAKE_PORT", "4566")), + ) + return Settings( + account=_required("SNOWFLAKE_ACCOUNT"), + user=_required("SNOWFLAKE_USER"), + role=os.getenv("SNOWFLAKE_ROLE", DEFAULT_ROLE), + warehouse=os.getenv("SNOWFLAKE_WAREHOUSE", DEFAULT_WAREHOUSE), + database=database, + private_key_path=_key_path(), + ) + + +def connect(database: str | None = None, **overrides: Any): + """Open a connector session for the configured target.""" + import snowflake.connector + + return snowflake.connector.connect(**load(database).connect_kwargs(**overrides)) diff --git a/slim-ci-lab/src/slim_ci_lab/tlc.py b/slim-ci-lab/src/slim_ci_lab/tlc.py new file mode 100644 index 0000000..c4d497b --- /dev/null +++ b/slim-ci-lab/src/slim_ci_lab/tlc.py @@ -0,0 +1,89 @@ +"""Pinned NYC TLC trip data: download, verify, downsample. + +The full January 2024 yellow-taxi file has about 3 million rows. CI only needs +enough rows to exercise every model, so the loader keeps a deterministic +1-in-N extract: sort by every column (a total order, so ties are identical +rows), then take every Nth row. The same input always yields the same extract, +on any machine, so test expectations computed from it are stable. +""" + +from __future__ import annotations + +import hashlib +from dataclasses import dataclass +from pathlib import Path + +import pyarrow as pa +import pyarrow.compute as pc +import pyarrow.parquet as pq +import requests + +from slim_ci_lab.settings import CACHE_DIR + + +@dataclass(frozen=True) +class TlcFile: + url: str + sha256: str + + @property + def name(self) -> str: + return self.url.rsplit("/", 1)[-1] + + +YELLOW_2024_01 = TlcFile( + url="https://d37ci6vzurychx.cloudfront.net/trip-data/yellow_tripdata_2024-01.parquet", + sha256="c4d59da7bbc8abaeeeb1727947ee93d9891a71acb42854bd80db1571b2030510", +) +SAMPLE_EVERY = 300 + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as fh: + for chunk in iter(lambda: fh.read(1 << 20), b""): + digest.update(chunk) + return digest.hexdigest() + + +def download(spec: TlcFile = YELLOW_2024_01, cache_dir: Path = CACHE_DIR / "tlc") -> Path: + """Return the cached file, downloading it first if needed. Verifies SHA-256.""" + cache_dir.mkdir(parents=True, exist_ok=True) + target = cache_dir / spec.name + if not target.exists(): + partial = target.with_suffix(".part") + print(f"Downloading {spec.url}") + with requests.get(spec.url, stream=True, timeout=120) as resp: + resp.raise_for_status() + with partial.open("wb") as fh: + for chunk in resp.iter_content(chunk_size=1 << 20): + fh.write(chunk) + partial.rename(target) + actual = _sha256(target) + if actual != spec.sha256: + target.unlink() + raise RuntimeError( + f"SHA-256 mismatch for {spec.name}: expected {spec.sha256}, got {actual}. " + "The cached copy was removed; re-run to download it again." + ) + return target + + +def downsample(source: Path, every: int = SAMPLE_EVERY) -> Path: + """Write (once) and return the deterministic 1-in-`every` extract of `source`.""" + target = source.with_name(f"{source.stem}.sample-1in{every}.parquet") + if target.exists(): + return target + table = pq.read_table(source) + order = pc.sort_indices(table, sort_keys=[(name, "ascending") for name in table.column_names]) + picked = pc.take(order, pa.array(range(0, len(order), every))) + sample = table.take(picked) + partial = target.with_suffix(".part") + pq.write_table(sample, partial) + partial.rename(target) + return target + + +def sample_file() -> Path: + """The downsampled extract the pipeline loads (downloading the month if needed).""" + return downsample(download()) diff --git a/slim-ci-lab/tests/conftest.py b/slim-ci-lab/tests/conftest.py new file mode 100644 index 0000000..2886b81 --- /dev/null +++ b/slim-ci-lab/tests/conftest.py @@ -0,0 +1,71 @@ +"""Shared fixtures: one throwaway database per test session. + +The session loads the downsampled TLC extract into .RAW.TRIPS and builds +production state once (dbt build --target prod in a temporary copy of the +project). Tests then build pull request schemas against that state. The +database is dropped at the end, so runs never touch SLIM_CI or each other. +""" + +from __future__ import annotations + +import shutil +import uuid +from dataclasses import dataclass +from pathlib import Path + +import pytest + +from slim_ci_lab import loader, settings, tlc +from slim_ci_lab.dbt_runner import copy_project, dbt_env, run_dbt, targets + +EMULATOR = settings.is_emulator() + + +@dataclass(frozen=True) +class ProdState: + database: str + project_dir: Path + state_dir: Path + env: dict[str, str] + + +@pytest.fixture(scope="session") +def database(): + name = f"SLIM_CI_TEST_{uuid.uuid4().hex[:8].upper()}" + with settings.connect() as conn: + conn.cursor().execute(f"CREATE DATABASE {name}") + yield name + with settings.connect() as conn: + conn.cursor().execute(f"DROP DATABASE IF EXISTS {name}") + + +@pytest.fixture(scope="session") +def conn(database): + with settings.connect() as connection: + connection.cursor().execute(f"USE DATABASE {database}") + yield connection + + +@pytest.fixture(scope="session") +def sample_file() -> Path: + return tlc.sample_file() + + +@pytest.fixture(scope="session") +def first_load(database, sample_file) -> loader.LoadResult: + return loader.load(database, sample_file) + + +@pytest.fixture(scope="session") +def prod_state(database, first_load, tmp_path_factory) -> ProdState: + """Production state: a full `dbt build` plus its saved manifest.json.""" + work = tmp_path_factory.mktemp("prod") + project = copy_project(work / "dbt") + env = dbt_env(DBT_DATABASE=database) + env.pop("DBT_PR_NUMBER", None) + result = run_dbt(["build", "--target", targets()[0]], project, env) + assert result.returncode == 0, result.stdout[-4000:] + state_dir = work / "state" + state_dir.mkdir() + shutil.copy(project / "target" / "manifest.json", state_dir / "manifest.json") + return ProdState(database=database, project_dir=project, state_dir=state_dir, env=env) diff --git a/slim-ci-lab/tests/test_load.py b/slim-ci-lab/tests/test_load.py new file mode 100644 index 0000000..34ad7f1 --- /dev/null +++ b/slim-ci-lab/tests/test_load.py @@ -0,0 +1,39 @@ +"""Raw load: PUT + COPY INTO .RAW.TRIPS.""" + +from __future__ import annotations + +import pyarrow.compute as pc +import pyarrow.parquet as pq + +from slim_ci_lab import loader + + +def test_raw_row_count_matches_downsampled_file(conn, database, sample_file, first_load): + expected_rows = pq.ParquetFile(sample_file).metadata.num_rows + + (count, files) = conn.cursor().execute( + f"SELECT COUNT(*), COUNT(DISTINCT _SOURCE_FILE) FROM {database}.RAW.TRIPS" + ).fetchone() + + assert first_load.files_loaded == 1 + assert first_load.rows_loaded == expected_rows + assert (count, files) == (expected_rows, 1) + + +def test_raw_timestamps_match_source_file(conn, database, sample_file, first_load): + source = pq.read_table(sample_file, columns=["tpep_pickup_datetime"]).column(0) + expected = (pc.min(source).as_py(), pc.max(source).as_py()) + + actual = conn.cursor().execute( + f"SELECT MIN(TPEP_PICKUP_DATETIME), MAX(TPEP_PICKUP_DATETIME) FROM {database}.RAW.TRIPS" + ).fetchone() + + assert actual == expected + + +def test_copy_rerun_loads_no_new_files(conn, database, sample_file, first_load): + again = loader.load(database, sample_file) + + assert (again.files_loaded, again.rows_loaded) == (0, 0) + assert again.total_rows == first_load.total_rows + diff --git a/slim-ci-lab/tests/test_models.py b/slim-ci-lab/tests/test_models.py new file mode 100644 index 0000000..cf86224 --- /dev/null +++ b/slim-ci-lab/tests/test_models.py @@ -0,0 +1,95 @@ +"""dbt models and snapshot, checked against values computed independently.""" + +from __future__ import annotations + +import csv +import datetime as dt +from decimal import ROUND_HALF_UP, Decimal + +import pandas as pd + +from slim_ci_lab.dbt_runner import copy_project, dbt_env, run_dbt, targets + +# Business rules, restated here rather than imported from the project: +# revenue is the passenger-paid total of billable trips (total >= 0) +# picked up in January 2024, attributed to the pickup zone and day. +MONTH_START, MONTH_END = pd.Timestamp("2024-01-01"), pd.Timestamp("2024-02-01") +JFK_AIRPORT, KNOWN_DAY = 132, dt.date(2024, 1, 15) +CENT = Decimal("0.01") + + +def _billable_trips(sample_file) -> pd.DataFrame: + df = pd.read_parquet(sample_file, columns=["tpep_pickup_datetime", "PULocationID", "total_amount"]) + in_month = (df.tpep_pickup_datetime >= MONTH_START) & (df.tpep_pickup_datetime < MONTH_END) + return df[in_month & (df.total_amount >= 0)] + + +def _money(value: float) -> Decimal: + return Decimal(str(float(value))).quantize(CENT, rounding=ROUND_HALF_UP) + + +def test_daily_zone_revenue_matches_pandas(conn, prod_state, sample_file): + trips = _billable_trips(sample_file) + known = trips[(trips.PULocationID == JFK_AIRPORT) & (trips.tpep_pickup_datetime.dt.date == KNOWN_DAY)] + assert len(known) > 10, "fixture precondition: the extract has enough JFK trips on the known day" + expected = (len(known), sum((_money(v) for v in known.total_amount), Decimal("0.00"))) + + actual = conn.cursor().execute( + f"""SELECT trip_count, revenue, pickup_zone + FROM {prod_state.database}.MARTS.FCT_DAILY_ZONE_REVENUE + WHERE pickup_location_id = %s AND pickup_date = %s""", + (JFK_AIRPORT, KNOWN_DAY), + ).fetchall() + + assert actual == [(expected[0], expected[1], "JFK Airport")] + + +def test_incremental_second_run_merges_without_duplicates(conn, prod_state, sample_file): + expected_rows = len(_billable_trips(sample_file)) + table = f"{prod_state.database}.INTERMEDIATE.INT_TRIPS_ENRICHED" + + second = run_dbt(["run", "--target", targets()[0], "--select", "int_trips_enriched"], prod_state.project_dir, prod_state.env) + + assert second.returncode == 0, second.stdout[-3000:] + rows, keys = conn.cursor().execute(f"SELECT COUNT(*), COUNT(DISTINCT trip_id) FROM {table}").fetchone() + assert (rows, keys) == (expected_rows, expected_rows) + + +def test_snapshot_records_zone_change_and_hard_delete(conn, database, first_load, tmp_path): + project = copy_project(tmp_path / "dbt") + env = dbt_env(DBT_DATABASE=database, DBT_PR_NUMBER="snapshot_test") + build = [ + "build", "--target", targets()[1], "--indirect-selection", "cautious", + "--select", "taxi_zone_lookup", "stg_taxi_zones", "dim_zone", + ] + first = run_dbt(build, project, env) + assert first.returncode == 0, first.stdout[-3000:] + + seed = project / "seeds" / "taxi_zone_lookup.csv" + with seed.open(newline="") as fh: + rows = list(csv.reader(fh)) + header, zones = rows[0], rows[1:] + assert zones[0][:3] == ["1", "EWR", "Newark Airport"] and zones[-1][0] == "265" + zones[0][2] = "Newark Liberty International Airport" # zone renamed upstream + zones = zones[:-1] # zone 265 removed upstream + with seed.open("w", newline="") as fh: + csv.writer(fh).writerows([header, *zones]) + + second = run_dbt(build, project, env) + assert second.returncode == 0, second.stdout[-3000:] + + history = conn.cursor().execute( + f"""SELECT location_id, zone_name, dbt_valid_to IS NULL AS is_current, dbt_is_deleted + FROM {database}.PR_SNAPSHOT_TEST.DIM_ZONE + WHERE location_id IN (1, 265) + ORDER BY location_id, dbt_valid_from, dbt_valid_to NULLS LAST""" + ).fetchall() + (total,) = conn.cursor().execute(f"SELECT COUNT(*) FROM {database}.PR_SNAPSHOT_TEST.DIM_ZONE").fetchone() + + assert history == [ + (1, "Newark Airport", False, "False"), + (1, "Newark Liberty International Airport", True, "False"), + (265, "Outside of NYC", False, "False"), + (265, "Outside of NYC", True, "True"), + ] + assert total == 265 + 2 diff --git a/slim-ci-lab/tests/test_slim_ci.py b/slim-ci-lab/tests/test_slim_ci.py new file mode 100644 index 0000000..ba4f9ea --- /dev/null +++ b/slim-ci-lab/tests/test_slim_ci.py @@ -0,0 +1,89 @@ +"""The pull request pipeline: state:modified+ with defer, per-PR schemas, the bad-PR catch.""" + +from __future__ import annotations + +import subprocess + +from slim_ci_lab import bad_pr_demo, settings +from slim_ci_lab.dbt_runner import apply_patch, copy_project, run_dbt, targets + +EMULATOR = settings.is_emulator() +DEMOS = settings.SAMPLE_ROOT / "demos" +CI_SCRIPT = settings.SAMPLE_ROOT / "scripts" / "ci_local.sh" + + +def run_ci(prod_state, pr_number: str, patch: str) -> subprocess.CompletedProcess[str]: + """Run scripts/ci_local.sh for one simulated pull request, reusing the session's prod state.""" + prod_target, pr_target = targets() + env = { + **prod_state.env, + "DBT_PR_NUMBER": pr_number, + "CI_PR_PATCH": str(DEMOS / patch), + "PROD_STATE": "reuse", + "STATE_DIR": str(prod_state.state_dir), + "PROD_TARGET": prod_target, + "PR_TARGET": pr_target, + } + return subprocess.run([str(CI_SCRIPT)], env=env, capture_output=True, text=True, check=False) + + +def tables_in(conn, database: str, schema: str) -> set[str]: + rows = conn.cursor().execute( + f"SELECT table_name FROM {database}.INFORMATION_SCHEMA.TABLES WHERE table_schema = %s", + (schema,), + ).fetchall() + return {name for (name,) in rows} + + +def test_pr_build_selects_only_the_modified_mart_and_defers_upstream(conn, prod_state): + db = prod_state.database + + result = run_ci(prod_state, "101", "example-pr.patch") + + assert result.returncode == 0, result.stdout[-4000:] + if EMULATOR: + assert "Snowflake credits used: 0" in result.stdout + # Only the modified model lands in the PR schema; its upstreams were read from prod. + assert tables_in(conn, db, "PR_101") == {"FCT_DAILY_ZONE_REVENUE"} + columns = conn.cursor().execute(f"SELECT * FROM {db}.PR_101.FCT_DAILY_ZONE_REVENUE LIMIT 0") + assert "TIP_RATE" in {col[0] for col in columns.description} + pr_total, prod_total = conn.cursor().execute( + f"""SELECT (SELECT SUM(revenue) FROM {db}.PR_101.FCT_DAILY_ZONE_REVENUE), + (SELECT SUM(revenue) FROM {db}.MARTS.FCT_DAILY_ZONE_REVENUE)""" + ).fetchone() + assert pr_total == prod_total + + +def test_pr_build_without_defer_fails_with_002003(prod_state, tmp_path): + project = copy_project(tmp_path / "dbt") + apply_patch(project, DEMOS / "example-pr.patch") + env = {**prod_state.env, "DBT_PR_NUMBER": "no_defer"} + pr_target = ["--target", targets()[1]] + run_dbt(["run-operation", "drop_pr_schema", *pr_target], project, env) + + result = run_dbt(["run", *pr_target, "--select", "state:modified", "--state", str(prod_state.state_dir)], project, env) + + # Without --defer, ref('int_trips_enriched') points into the empty PR schema. + assert result.returncode != 0 + assert "002003" in result.stdout + assert "does not exist or not authorized" in result.stdout + + +def test_bad_unique_key_passes_once_then_fails_with_100090(database, first_load): + demo = bad_pr_demo.run(database) + + assert demo.first.returncode == 0, demo.first.stdout[-3000:] + assert demo.second.returncode != 0 + assert "100090" in demo.second_output + assert "Duplicate row detected during DML action" in demo.second_output + assert demo.expected_failure_observed + + +def test_slim_ci_rejects_bad_unique_key_pr_on_first_ci_run(prod_state): + # The pipeline clones the modified incremental model from prod, so its + # very first PR build already runs the MERGE that production would. + result = run_ci(prod_state, "102", "bad-unique-key.patch") + + assert result.returncode != 0 + assert "100090" in result.stdout + assert "Duplicate row detected during DML action" in result.stdout diff --git a/slim-ci-lab/uv.lock b/slim-ci-lab/uv.lock new file mode 100644 index 0000000..44ce260 --- /dev/null +++ b/slim-ci-lab/uv.lock @@ -0,0 +1,1209 @@ +version = 1 +revision = 3 +requires-python = "==3.12.*" +resolution-markers = [ + "sys_platform == 'win32'", + "sys_platform == 'emscripten'", + "sys_platform != 'emscripten' and sys_platform != 'win32'", +] + +[[package]] +name = "agate" +version = "1.9.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "babel" }, + { name = "isodate" }, + { name = "leather" }, + { name = "parsedatetime" }, + { name = "python-slugify" }, + { name = "pytimeparse" }, + { name = "tzdata", marker = "sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/29/77/6f5df1c68bf056f5fdefc60ccc616303c6211e71cd6033c830c12735f605/agate-1.9.1.tar.gz", hash = "sha256:bc60880c2ee59636a2a80cd8603d63f995be64526abf3cbba12f00767bcd5b3d", size = 202303, upload-time = "2023-12-21T20:05:24.316Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/53/89b197cb472a3175d73384761a3413fd58e6b65a794c1102d148b8de87bd/agate-1.9.1-py2.py3-none-any.whl", hash = "sha256:1cf329510b3dde07c4ad1740b7587c9c679abc3dcd92bb1107eabc10c2e03c50", size = 95085, upload-time = "2023-12-21T20:05:21.954Z" }, +] + +[[package]] +name = "annotated-types" +version = "0.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5f/56/a8120250d128bed162cd73c76d45f6ef9991f3e068f62a8ee060afa3104a/annotated_types-0.8.0.tar.gz", hash = "sha256:13b2beaad985e05e2d6407ee4c4f35590b11f8d693a258a561055cac8f64cab7", size = 15893, upload-time = "2026-07-23T20:16:13.995Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/99/91/8acff4f5e50511b911bbccb72b8628a49c68ce14148cd9f6431094859a90/annotated_types-0.8.0-py3-none-any.whl", hash = "sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0", size = 13427, upload-time = "2026-07-23T20:16:12.938Z" }, +] + +[[package]] +name = "asn1crypto" +version = "1.5.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/de/cf/d547feed25b5244fcb9392e288ff9fdc3280b10260362fc45d37a798a6ee/asn1crypto-1.5.1.tar.gz", hash = "sha256:13ae38502be632115abf8a24cbe5f4da52e3b5231990aff31123c805306ccb9c", size = 121080, upload-time = "2022-03-15T14:46:52.889Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/c9/7f/09065fd9e27da0eda08b4d6897f1c13535066174cc023af248fc2a8d5e5a/asn1crypto-1.5.1-py2.py3-none-any.whl", hash = "sha256:db4e40728b728508912cbb3d44f19ce188f218e9eba635821bb4b68564f8fd67", size = 105045, upload-time = "2022-03-15T14:46:51.055Z" }, +] + +[[package]] +name = "attrs" +version = "26.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9a/8e/82a0fe20a541c03148528be8cac2408564a6c9a0cc7e9171802bc1d26985/attrs-26.1.0.tar.gz", hash = "sha256:d03ceb89cb322a8fd706d4fb91940737b6642aa36998fe130a9bc96c985eff32", size = 952055, upload-time = "2026-03-19T14:22:25.026Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/64/b4/17d4b0b2a2dc85a6df63d1157e028ed19f90d4cd97c36717afef2bc2f395/attrs-26.1.0-py3-none-any.whl", hash = "sha256:c647aa4a12dfbad9333ca4e71fe62ddc36f4e63b2d260a37a8b83d2f043ac309", size = 67548, upload-time = "2026-03-19T14:22:23.645Z" }, +] + +[[package]] +name = "babel" +version = "2.18.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/b2/51899539b6ceeeb420d40ed3cd4b7a40519404f9baf3d4ac99dc413a834b/babel-2.18.0.tar.gz", hash = "sha256:b80b99a14bd085fcacfa15c9165f651fbb3406e66cc603abf11c5750937c992d", size = 9959554, upload-time = "2026-02-01T12:30:56.078Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/77/f5/21d2de20e8b8b0408f0681956ca2c69f1320a3848ac50e6e7f39c6159675/babel-2.18.0-py3-none-any.whl", hash = "sha256:e2b422b277c2b9a9630c1d7903c2a00d0830c409c59ac8cae9081c92f1aeba35", size = 10196845, upload-time = "2026-02-01T12:30:53.445Z" }, +] + +[[package]] +name = "boto3" +version = "1.43.118" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, + { name = "jmespath" }, + { name = "s3transfer" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/96/8b/5694c8433b5202f34a82def4edee274f7df1eba2919c66d9d389d2202ec8/boto3-1.43.118.tar.gz", hash = "sha256:c0620f799e0f0c222ba42c337b65cdbcd3da38a3121187f634d8926cbbb28a9c", size = 112649, upload-time = "2026-10-22T19:22:09.629Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/bb/a3/65bb3d64928f3c2da17640b92044084150317f02adf22442cb0c52af2a8b/boto3-1.43.118-py3-none-any.whl", hash = "sha256:ab8f3d2c622ef039338b4f228d97ea55f1abd573d84bb9b9a5ee8cb6780c0ccd", size = 140045, upload-time = "2026-10-22T19:22:08.353Z" }, +] + +[[package]] +name = "botocore" +version = "1.43.118" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jmespath" }, + { name = "python-dateutil" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d4/b8/3e55154ddaf1a6d1d4ddda85c0e5983205a3b7657edacd8ebba8c6fd774c/botocore-1.43.118.tar.gz", hash = "sha256:7f3f9d88cc9e847587690f82a42e3c3c292c3026c92c1e60d68a9310018e003f", size = 16420126, upload-time = "2026-10-22T19:22:04.81Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9c/ce/61b39a8e7a69af5ea43c1c3e1b0065a4866b42ccddf107e16686a3143614/botocore-1.43.118-py3-none-any.whl", hash = "sha256:a45d7ae570674161e4a3542d8c68633bf7503f92d1de781ef7361b79488eda6f", size = 16122016, upload-time = "2026-10-22T19:22:00.414Z" }, +] + +[[package]] +name = "certifi" +version = "2025.1.31" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/1c/ab/c9f1e32b7b1bf505bf26f0ef697775960db7932abeb7b516de930ba2705f/certifi-2025.1.31.tar.gz", hash = "sha256:3d5da6925056f6f18f119200434a4780a94263f10d1c21d032a6f6b2baa20651", size = 167577, upload-time = "2025-01-31T02:16:47.166Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/38/fc/bce832fd4fd99766c04d1ee0eead6b0ec6486fb100ae5e74c1d91292b982/certifi-2025.1.31-py3-none-any.whl", hash = "sha256:ca78db4565a652026a4db2bcdf68f2fb589ea80d0be70e03929ed730746b84fe", size = 166393, upload-time = "2025-01-31T02:16:45.015Z" }, +] + +[[package]] +name = "cffi" +version = "2.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pycparser", marker = "implementation_name != 'PyPy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9e/ef/008a1939e372c06329a3fce4279c02f328488f3526744906eeec3da7ad5f/cffi-2.1.1.tar.gz", hash = "sha256:dd31f52ea1086513bb9df30f8fcee9b8918323ae067a3d5b78bc826a000712be", size = 530807, upload-time = "2026-08-03T21:21:18.939Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/10/69/43965eccfdead3b9220015fd1320e117be8c6ed01a62ffab76eeb752f5d5/cffi-2.1.1-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:c8c69575568085ba0b1b10c0249d779a214aea6f6522e949a0fc9fb0fcb449d0", size = 184821, upload-time = "2026-08-03T21:19:44.887Z" }, + { url = "https://files.pythonhosted.org/packages/54/7d/16e5a096677b5e313ca80cd5e5170efa3ea44624a82bb111925522da64b1/cffi-2.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f81b3b8f3d4e343550fa4baa0e479bba9f2d29ce9c2e9b51d1ce1718d7442fcf", size = 184719, upload-time = "2026-08-03T21:19:46.129Z" }, + { url = "https://files.pythonhosted.org/packages/56/e6/8941622732edec876dd17d0453dce07317ae96db34f2ec1436c9d3785986/cffi-2.1.1-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:811bd1e21d32de12efca32393a0ab3f5133b54fce9bd44b8bd77ab07da14bf6a", size = 214799, upload-time = "2026-08-03T21:19:47.218Z" }, + { url = "https://files.pythonhosted.org/packages/44/de/f98430906df1545ffde0d543dd124a7a439bc2cd32b36b9c53f805df7333/cffi-2.1.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:68e62fe11f30d5ca8289242866f0a5291402d8529ca2178ab8afc5c9694ae890", size = 222389, upload-time = "2026-08-03T21:19:48.331Z" }, + { url = "https://files.pythonhosted.org/packages/6a/5b/717f1526b9957b34456313c31645c5b82b8fb5c3fe9e4752999be7128bfc/cffi-2.1.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:4a7c934f7360e8cd64fe9efadcbd10c7c6364f531e432b9a4bf5ccbc9e0e8b50", size = 210249, upload-time = "2026-08-03T21:19:49.543Z" }, + { url = "https://files.pythonhosted.org/packages/64/b3/f8aa4f3e34986c7e4ec45072d1b1b9dd295b6b18007b45518d79726dd725/cffi-2.1.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:3143d81e29e1e20a9ce10901ec369012947876596f75a222235965f2b7ae832e", size = 208775, upload-time = "2026-08-03T21:19:50.918Z" }, + { url = "https://files.pythonhosted.org/packages/b1/db/dceb9dd5b231e1da801793f8acc9f3c52a7e1afe40bb1aae37e02b0faad5/cffi-2.1.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c1453022f490d2459a11819d83ad1d586e9ff65a12ac3e705ffebd46d3685dcf", size = 221822, upload-time = "2026-08-03T21:19:52.054Z" }, + { url = "https://files.pythonhosted.org/packages/a0/d2/6cd24ae3be000a634109c247d1475d62e5616d0dc78c82770942ec384248/cffi-2.1.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:208f941bb9d18e768138677f0a6d2ce01f590df56043dda1df1535ac57c88517", size = 225232, upload-time = "2026-08-03T21:19:53.109Z" }, + { url = "https://files.pythonhosted.org/packages/cb/52/3fa190537004dd7f0ab860a6dc7c0175b8667f68d1e618a46f5498d30250/cffi-2.1.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:210019b6c7cf07f081b4c54635c8cf744377001350e29cc0f81c4377b4797735", size = 223597, upload-time = "2026-08-03T21:19:54.515Z" }, + { url = "https://files.pythonhosted.org/packages/80/fb/0bb75b7039588c074b37ae99f40d9bfddf990ecb2fbc346ebccd2e56b9be/cffi-2.1.1-cp312-cp312-win32.whl", hash = "sha256:046bfc24911b37851ee1b51aab8bffe713d89c68c6a057b09484ce9fd5f69b4e", size = 175292, upload-time = "2026-08-03T21:19:55.566Z" }, + { url = "https://files.pythonhosted.org/packages/d9/79/615cc094e2fb508cade7de88d3b4f6c4ec2bab695c97bce9153dc65aadf5/cffi-2.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:f53e442b08449d42821fa4a4fba000095af9f62742a500f978a9f557ec44339a", size = 185919, upload-time = "2026-08-03T21:19:56.89Z" }, + { url = "https://files.pythonhosted.org/packages/70/c6/d0ea84713fe46b243a436a18fcd47d639732747e21635c8a27191b06dc30/cffi-2.1.1-cp312-cp312-win_arm64.whl", hash = "sha256:7bde5e4cc5c10140859842b9d383af292b22639a4dffb725314baf45968cef80", size = 180093, upload-time = "2026-08-03T21:19:58.155Z" }, +] + +[[package]] +name = "charset-normalizer" +version = "3.5.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/33/1c/f41d4e74c28ab327ff3acd36053f7ea506c55872d7a90b0fa71aa3ab0c89/charset_normalizer-3.5.2.tar.gz", hash = "sha256:39de2a259fc954455c57274dc94c79d5842774e1247a016aff30bc0efed0f4ef", size = 172659, upload-time = "2026-09-30T04:39:23.398Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e7/c8/693809898870237d82785a03f3b2b58fe4c9f14669f84a7d4e623c92a59e/charset_normalizer-3.5.2-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:ed2a239c0ea213acc1908150a3037257083c7c083128f1a4cec2ec4b97dca491", size = 367780, upload-time = "2026-09-30T04:35:30.888Z" }, + { url = "https://files.pythonhosted.org/packages/c9/87/2fea8c13dc24b3ca9c6f803a5b2dfdeae73eb4f9e12c7885ed908ff0433c/charset_normalizer-3.5.2-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b91363207bd9dc966a691e959bb47f64b30f7ac4b072be9968b366982f7db77c", size = 246730, upload-time = "2026-09-30T04:35:32.286Z" }, + { url = "https://files.pythonhosted.org/packages/a8/9e/09efac30b937722f46d3110ba30b875b24b2e3a266ed746cc4e376a94d80/charset_normalizer-3.5.2-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:38a873987f3be698494da8b2e3085e29da02da7b633dce73e79c699a113d7bf0", size = 237707, upload-time = "2026-09-30T04:35:33.709Z" }, + { url = "https://files.pythonhosted.org/packages/9e/18/70d76670b13686237863a379928d60bd10e021f17d243ab3d7014c4a5f4e/charset_normalizer-3.5.2-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:355ad8011081dec5412240c087a9a0c9d4d5039f3ed11a3f13e18c2b29b56c51", size = 273050, upload-time = "2026-09-30T04:35:35.138Z" }, + { url = "https://files.pythonhosted.org/packages/54/e2/77a8b09d5adc013ed07b95b01b8b8fa5441c4e810e83ee7e4aae2fa4d91a/charset_normalizer-3.5.2-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ee21e28f0430bd6dc9086c6e525d5e818a44a5ad19720c8a0ef766792f3eb5e5", size = 270345, upload-time = "2026-09-30T04:35:36.502Z" }, + { url = "https://files.pythonhosted.org/packages/7f/c5/38806a25ab5e65fc178f39affeda20858efafede2fce1ffc2556cfc9fe73/charset_normalizer-3.5.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:3d31298449090ab8d47b7b1b2a555ff73cac7ed438a08b7ac160980c7ebed649", size = 257601, upload-time = "2026-09-30T04:35:37.919Z" }, + { url = "https://files.pythonhosted.org/packages/ae/8d/213565184708fdb263ae55e2c04ee1ff748129dd65d48ed0e3502da9c85a/charset_normalizer-3.5.2-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5cde776b7cc66e4f6c99612cea4aa7269aa65863f7a15841b2c264f103822f4e", size = 252222, upload-time = "2026-09-30T04:35:39.544Z" }, + { url = "https://files.pythonhosted.org/packages/7e/24/76d2cefc25472531e4c5c7dfff68865eb1c39b78482f0fdc15b46f047830/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ae4f5fea5b8b8ccff88238cc8569303e5ee95efae67fa62922a311397a71f346", size = 248482, upload-time = "2026-09-30T04:35:41.088Z" }, + { url = "https://files.pythonhosted.org/packages/7d/dc/65a801b66ab4c197e22c433ab25e7ac24324ac6f45a2269aca42cce309bf/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:f7d486c83842422badd511868fd8a9a20e9407ace71564b6af47ce7e60a336c1", size = 241206, upload-time = "2026-09-30T04:35:42.59Z" }, + { url = "https://files.pythonhosted.org/packages/a7/95/ca9b5eabde673002c6f1e7ada1b223916fe18f6d661da7aabd4d643718f1/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:11a4d68a6ecda3292cb1e50239e111543ba5d709bb62a6b4ea1afcfa729d8875", size = 273190, upload-time = "2026-09-30T04:35:44.347Z" }, + { url = "https://files.pythonhosted.org/packages/2d/8b/803b4d2a3f6e1740f63f1e87b04d14b42f3d4fdfe6ed7d4db2d34102b14f/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:d6734d2ef8a50fbf8445c139477da401f50d62a0606bf00e20ec6d87773fefb1", size = 253527, upload-time = "2026-09-30T04:35:45.915Z" }, + { url = "https://files.pythonhosted.org/packages/a9/55/93c0e5dbd085ae0471346026abbe7e0db9ea2d6fea74e51f0b5a46f233a7/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:a815775b6c38d4e0ff7bcffbeba67feded90202bb6a226b8dd35f1c855217413", size = 271285, upload-time = "2026-09-30T04:35:47.49Z" }, + { url = "https://files.pythonhosted.org/packages/95/69/0dbd0e0b9b16cfa816cdfcb3e2e3854a1f680dc07fb1245ea125e7448060/charset_normalizer-3.5.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:23851fb4e1b85ed3f6c2a27b777cdfe2e19fb5b38429a8faf38c7542b7665869", size = 260010, upload-time = "2026-09-30T04:35:48.996Z" }, + { url = "https://files.pythonhosted.org/packages/58/9d/e7b88e7b1bf403590c3b573277b5e1e488c68c7a6fbacca310a2c324e90c/charset_normalizer-3.5.2-cp312-cp312-win32.whl", hash = "sha256:db19d07e2e0129e974a0e65d0064fc222a446cd5122c2fd4184d2af9fc734a9e", size = 184125, upload-time = "2026-09-30T04:35:50.777Z" }, + { url = "https://files.pythonhosted.org/packages/eb/e6/e6e083884cbcfd49c64865af05027fe7011be7b2d9179524f099a1b611f3/charset_normalizer-3.5.2-cp312-cp312-win_amd64.whl", hash = "sha256:780fbe7cab297b81dad9fb8dc5eb003c0468ffb0d9e5f65068c53a34661a96bc", size = 207486, upload-time = "2026-09-30T04:35:52.194Z" }, + { url = "https://files.pythonhosted.org/packages/c4/e3/017aea0911ada7405a825c7d937eb3a13009664e2f5b38e8c4bbf2abf894/charset_normalizer-3.5.2-cp312-cp312-win_arm64.whl", hash = "sha256:e2af3aad578aa6bd1384bcf4750fc285e5a9de53f40b7d41e5a0bf748edeb2b3", size = 196734, upload-time = "2026-09-30T04:35:53.636Z" }, + { url = "https://files.pythonhosted.org/packages/8c/ab/176fbfd5b64939c55d652366aa5b9ef1d767af207a3aa6ebeb0d226c484d/charset_normalizer-3.5.2-cp37-abi3-macosx_10_9_universal2.whl", hash = "sha256:4275811936e2f06feff5e598fb42a1b7ae852da8e39605211892b56b81a34efd", size = 331815, upload-time = "2026-09-30T04:38:26.216Z" }, + { url = "https://files.pythonhosted.org/packages/7e/84/371eac6b30bdbcbf2d632a1a01809103459216fcaae61b8b8d922c1bfb8a/charset_normalizer-3.5.2-cp37-abi3-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:1c50fe28bbc2ced33386f298650d91218076c05420e6cbd790b913adc41659e7", size = 253276, upload-time = "2026-09-30T04:38:28.032Z" }, + { url = "https://files.pythonhosted.org/packages/43/6f/c4fbae58febff71709c51bc7e18fdfa55341dc382704740f9f0cbf03817b/charset_normalizer-3.5.2-cp37-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d19fbd981a488e22cd04883659ca6b08f50b5974f9fd7c95655ef6a043e5893f", size = 241239, upload-time = "2026-09-30T04:38:29.732Z" }, + { url = "https://files.pythonhosted.org/packages/61/71/458c3f42164a07d0c5210798e9e704b39e540a6793b05aba67f3a35243a9/charset_normalizer-3.5.2-cp37-abi3-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:0fed1d06615f022ee3b13caf5e8b180cfea32bb2c5aded8a9d44277afc040f93", size = 231121, upload-time = "2026-09-30T04:38:31.462Z" }, + { url = "https://files.pythonhosted.org/packages/09/54/ab9e89367076f6331bb6c65c4bf14a5361fa5191cb6561bf534f18504e1b/charset_normalizer-3.5.2-cp37-abi3-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:838dcc90063569a0448120554591a1d6c4a4ffe11babf048908793154ab86ade", size = 260350, upload-time = "2026-09-30T04:38:33.239Z" }, + { url = "https://files.pythonhosted.org/packages/7c/c1/061431ecc688d9d76602502cb57cc01e691e682c18f1beb45f9673b5bbd2/charset_normalizer-3.5.2-cp37-abi3-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:2ce45c6627b22c47e390bc91a41c3d13032192e699fa0bea96e9671b373d69b0", size = 255430, upload-time = "2026-09-30T04:38:34.865Z" }, + { url = "https://files.pythonhosted.org/packages/8d/1f/20c8949f0676f7ab811abdeb7f4d7f1cbc6e61ff20bef08b44edeb092bc8/charset_normalizer-3.5.2-cp37-abi3-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:0774bf9bf620249fee3e0b8b9fd3065de213be30f3aa94ce2494b3b638949e26", size = 250612, upload-time = "2026-09-30T04:38:36.649Z" }, + { url = "https://files.pythonhosted.org/packages/2b/9e/46f2fa4c431fc98c4ae76a8cb5bdca54e0341e3cfc3fcfd8e82740250818/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:1db38f4c5496827c1a501846d64d14c3b80c7e6714e406cd7dc36a9899fa1011", size = 242083, upload-time = "2026-09-30T04:38:38.26Z" }, + { url = "https://files.pythonhosted.org/packages/bd/39/559be29a0c0f086e0bba6922babd38916cc5e0b58ced4de13ee01ea05508/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:304d8e4d493af723536393eee0c689eb7813f4a474c8b479dee63f1fdd98f621", size = 232738, upload-time = "2026-09-30T04:38:39.81Z" }, + { url = "https://files.pythonhosted.org/packages/ff/6c/387b0e4f756a282831c1d9fc6aeb6c51ca4507ca202767c8de15ce9b12e2/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_ppc64le.whl", hash = "sha256:9b7f416ff0978e2f2249330527f0ad6fa02f4932e6199692d3b52da2048c19e4", size = 260703, upload-time = "2026-09-30T04:38:41.346Z" }, + { url = "https://files.pythonhosted.org/packages/96/92/1fdf015f09ef449f50d3ac4b67c90887c9c318b727daa95cc4f866e6521d/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_riscv64.whl", hash = "sha256:01077390b03f7988f11d700a2194e69b119741a86b1a638b1db88891e3eced8e", size = 247622, upload-time = "2026-09-30T04:38:42.937Z" }, + { url = "https://files.pythonhosted.org/packages/dc/3c/8e7b8a5671ad5d433669fb2a76f1a0164df2d9b1718b0206bc2a16d840cc/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_s390x.whl", hash = "sha256:7e841fb9010836c992c9f12fcbd43a831de93a5f726fc1ccd8ca1d0268c5014c", size = 257500, upload-time = "2026-09-30T04:38:44.604Z" }, + { url = "https://files.pythonhosted.org/packages/b4/f0/45b579df5cabc1d5d53ea1cc35e8437d3ca768c0acccc7041517cb6fbb32/charset_normalizer-3.5.2-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:9cae88599c7219005d879f98e5ed53341e9a122af585e1091200358a3003d2a0", size = 255100, upload-time = "2026-09-30T04:38:46.289Z" }, + { url = "https://files.pythonhosted.org/packages/31/68/fdec18a343f5fb3f310588dd478b09ac4799e0b187dbade3a8cd776f03ef/charset_normalizer-3.5.2-cp37-abi3-win32.whl", hash = "sha256:01b0c0d2262a9e28e8484a278c7e1b5d650e3ac8cf2683d2967e25899f208bdf", size = 174499, upload-time = "2026-09-30T04:38:47.999Z" }, + { url = "https://files.pythonhosted.org/packages/9d/8a/b618149cc5207943a0242068d7a27897f56a62947b5a039085f2a22029f8/charset_normalizer-3.5.2-cp37-abi3-win_amd64.whl", hash = "sha256:9f56f72050826f63dcee7a7f55b0a77168cb3bfc553fd405e7f8f9ece75a4036", size = 200092, upload-time = "2026-09-30T04:38:49.707Z" }, + { url = "https://files.pythonhosted.org/packages/03/cf/4c66866fa9e2b1c78e3c911516d1de497a677b7ac60f1eceda74ce777ca3/charset_normalizer-3.5.2-cp37-abi3-win_arm64.whl", hash = "sha256:40ab6bffa02ae10a0581e6c198be7d2d8ca5c2a0c64e4ed3465d766df457573e", size = 294363, upload-time = "2026-09-30T04:38:51.312Z" }, + { url = "https://files.pythonhosted.org/packages/fc/ad/d07d7862a62ffa6d79d68074d14823243dd235a77c45262acbf6adeb28bf/charset_normalizer-3.5.2-py3-none-any.whl", hash = "sha256:b6b751274acb69d77b3323d6b7dbaa3c7fdfc1eb829b7eb61d262f32e1af9685", size = 68872, upload-time = "2026-09-30T04:39:21.828Z" }, +] + +[[package]] +name = "click" +version = "8.5.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c7/0e/7fa0ef50764b67090eca4114772a2abf8b6148198475e54c660b97caeee6/click-8.5.0.tar.gz", hash = "sha256:ba0d2089de75ea0310e2dde03160e6ca10009947fb95a182f9b54021bb272e34", size = 382235, upload-time = "2026-08-26T13:33:14.56Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/58/50/6c0d534c5f134586a8e1ba4e330569e32f057e33372ae556463212fb4cd3/click-8.5.0-py3-none-any.whl", hash = "sha256:255bc9599cf7748b4b1a446ccc735421bd08a2ae529a8b88597d3de5664ee360", size = 125251, upload-time = "2026-08-26T13:33:12.928Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "cryptography" +version = "50.0.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9d/af/182eb91b0df3fe75c4d9f26fe70684569566745f6ba7e5c9c73a862c5252/cryptography-50.0.2.tar.gz", hash = "sha256:7b46165bb56eb4704e2eaaf86f3c940d19154535d9b0ca7d6d590b04060e00d5", size = 880623, upload-time = "2026-09-30T15:30:04.884Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/56/d194340cc4a57535e82e1bee9e89667ac4b7c13b5d3f59686deae3094dd5/cryptography-50.0.2-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:fa8f5efb344d6908a1ce62f4a24e2e5780f825d6f53f5f50ec5ffacac72936cb", size = 3914904, upload-time = "2026-09-30T14:43:44.339Z" }, + { url = "https://files.pythonhosted.org/packages/d9/69/c9bd862c3bf43d6399c433caf002df16e2dffd4be49bdf515cda38038711/cryptography-50.0.2-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:79def8d059362e7831389ed3be0ecdf58a89386e1271e35dd9f5af84e81bffd0", size = 4731146, upload-time = "2026-09-30T14:43:47.113Z" }, + { url = "https://files.pythonhosted.org/packages/21/69/64cef1f702bf6657e0cc186ed1a2891d50d29fb41586b254e1c07adea261/cryptography-50.0.2-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:630ebfea3bf689d075f82316324ff7433dc447fe6bc1bfc76524b74b4a9567d2", size = 4719841, upload-time = "2026-09-30T14:43:49.01Z" }, + { url = "https://files.pythonhosted.org/packages/38/6b/61a3f8d8c5e1e49a6cddccafc4015cc1c0021360ab0acb4080e7a423644a/cryptography-50.0.2-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:f9f6143a8c75945eb960d9eb98905a441394abfa24afaae239d514ffb2586480", size = 4738340, upload-time = "2026-09-30T14:43:50.932Z" }, + { url = "https://files.pythonhosted.org/packages/7b/2e/7212ca32fd43dc91f2f41db20160b268098874b4c9a0e7be94d6835f5b2e/cryptography-50.0.2-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:a582ab2ae1d34f67112cadc86702774c9ea4374df6bca6afe672817203c99134", size = 5367029, upload-time = "2026-09-30T14:43:52.911Z" }, + { url = "https://files.pythonhosted.org/packages/1a/f1/b474e930c4d910328780e3940da76f5aa5cbc48ce1fc14e44d239d9ea9db/cryptography-50.0.2-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:4061c0079120205fb760c58acab6443e217307dcf05e3702cf970e0689972856", size = 4753050, upload-time = "2026-09-30T14:43:55.272Z" }, + { url = "https://files.pythonhosted.org/packages/7c/52/9af10e80ac16b0fcc2123f9cbd5e7afbd0fd5075bb7a607c592258a39cda/cryptography-50.0.2-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:ac9ed99d81760c62fe89d5f0815cdfa1ba9a35141cf30f1c2d044f04b4803d2e", size = 4376724, upload-time = "2026-09-30T14:43:57.24Z" }, + { url = "https://files.pythonhosted.org/packages/71/37/6202e488cc1eb625ea110c292c6bda92823176e023f427d8d5660ce8d632/cryptography-50.0.2-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:87e9ce85beb6b328ba370cc6e6aea483c92617b4c95b1d33a49297eb662bfb04", size = 4737859, upload-time = "2026-09-30T14:43:59.541Z" }, + { url = "https://files.pythonhosted.org/packages/8f/30/e86d7d518489b0ae2497091a35287abcb1a2ce4037837a34afbe9b1d6964/cryptography-50.0.2-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:f265528741e048bce55c3463ed721fb0aa45a5888d8add8cfeccb3035451bbdc", size = 5324103, upload-time = "2026-09-30T14:44:01.901Z" }, + { url = "https://files.pythonhosted.org/packages/d3/69/2c833a049475e0a3444e94c7d0aca0aa51d166374a449b09e92ac98138de/cryptography-50.0.2-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:9dab55f57c74c3cad24c323bacbbd04be4705ba6eb0d92e920b1fc4837ed5079", size = 4752576, upload-time = "2026-09-30T14:44:04.545Z" }, + { url = "https://files.pythonhosted.org/packages/6c/5d/906970b83bbfc1f5bbfb677a143c181f2801f23b6a7204a3b47c42c97e65/cryptography-50.0.2-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:25784ce8b9621c90c643efb9e1e2162ab3b0224cae446ad5e70e7fcb1ce18b51", size = 4870819, upload-time = "2026-09-30T14:44:06.884Z" }, + { url = "https://files.pythonhosted.org/packages/68/e3/f2298d3bb55e0c4a91841ec4d01b3f020ba8c5fbf15ccdcc6dcf03f97025/cryptography-50.0.2-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:85d0d9a31b9098e98534226d5686b47264b95e62ce459dc2e62fdfc809f9fe93", size = 5030152, upload-time = "2026-09-30T14:44:09.443Z" }, + { url = "https://files.pythonhosted.org/packages/9a/4f/adfc442765721292fff86d314ce385d3249d22db42295c0dd057727b60f3/cryptography-50.0.2-cp311-abi3-win_amd64.whl", hash = "sha256:7afa5a6602a9f29af1f3a2965f831bae7c9d5d597b7cbb716d41ab3b7d89879c", size = 3824692, upload-time = "2026-09-30T14:44:11.671Z" }, + { url = "https://files.pythonhosted.org/packages/2d/49/93f6a6e7a87c9aa68d44d3e1cdb5fe8f60c90d5d2f46acae9a56892816b8/cryptography-50.0.2-cp315-abi3.abi3t-macosx_11_0_arm64.whl", hash = "sha256:edc3342adf8f697fc5f59c887a304356f147b397809440ed64e2fa6af2f50f37", size = 4133708, upload-time = "2026-09-30T14:44:41.807Z" }, + { url = "https://files.pythonhosted.org/packages/8c/75/32ac2a56243d778805c16ca6a32b8f74fb757df7e28d7ecb560afafb59cf/cryptography-50.0.2-cp315-abi3.abi3t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d370b8d1dfcdf7130178137f6fbee6140774a1acc6cacefc4b42643ec11d0a3a", size = 4956267, upload-time = "2026-09-30T14:44:43.693Z" }, + { url = "https://files.pythonhosted.org/packages/aa/a4/2c8d734e43d97f0842ee9f1b7b4bfb3d0cf5e19edebf43c2afe6675c2320/cryptography-50.0.2-cp315-abi3.abi3t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f2f9bd7f90c64fe89253f0a2c05e3c4856072660429ce8831b4235bf29403a67", size = 4966465, upload-time = "2026-09-30T14:44:45.769Z" }, + { url = "https://files.pythonhosted.org/packages/c2/58/ee288c829a6f41f6235ae9dd33d82fd19b45442b65b4c8a3da36963d9f7a/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_28_aarch64.whl", hash = "sha256:e275096ea1e60cc595cda2836fd4a6c725d1125108b868be17f53684d164e2cc", size = 4959356, upload-time = "2026-09-30T14:44:48.211Z" }, + { url = "https://files.pythonhosted.org/packages/92/20/9ded6d51ddd9897f6b6e81fb9ebea7951d7cc5d6c890b0ed8abf77a51a80/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_28_ppc64le.whl", hash = "sha256:b13478603dcd0a2479ff8e87e2c19a7d525734686fe3c49542472293a204212d", size = 5548822, upload-time = "2026-09-30T14:44:50.86Z" }, + { url = "https://files.pythonhosted.org/packages/02/a8/8df951850d6b31d2a00218f19e2b3f999523437ed7a819df7fa427942fca/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_28_x86_64.whl", hash = "sha256:58a0c478eeca76fe5e07993c5a0703def34a6dc6a0cda4f5564639b33112ffe7", size = 5001199, upload-time = "2026-09-30T14:44:53.379Z" }, + { url = "https://files.pythonhosted.org/packages/8b/f9/36b3022218ce75b7cdf068fb95f809f9bd0d820e4955ef43b90c255cc7ac/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_31_armv7l.whl", hash = "sha256:d38cdff612d06fa6a32840d5e1b1f7a27cee4a349aa9085d94a67789d6bfd408", size = 4629333, upload-time = "2026-09-30T14:44:55.635Z" }, + { url = "https://files.pythonhosted.org/packages/8c/72/20f99a219f6af47cdd1cbd978c243b92d71496e168a746138af44ded4f29/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_34_aarch64.whl", hash = "sha256:fdd28f912fccfec1846a94e2e1e8f9b0012f557f0c46fe4f3eb0d7a87afcf90b", size = 4958822, upload-time = "2026-09-30T14:44:59.639Z" }, + { url = "https://files.pythonhosted.org/packages/f2/20/196f112617fb08eb4d608a2a6c422373d46f9cc2857f38fc0667033c0899/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_34_ppc64le.whl", hash = "sha256:cbc8738fd8526d80f35cb3a40d41f41a2e7030bb3b18b09a6778ef63d291c2fd", size = 5506351, upload-time = "2026-09-30T14:45:02.267Z" }, + { url = "https://files.pythonhosted.org/packages/24/95/83378121ef3eaaaf71d4b781577ff794acb39b9e1b87a3f156898c8497ed/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_34_x86_64.whl", hash = "sha256:e105ab60406787da31fccc883fc0f733af1efd78f0136a4599692c4083a73d0c", size = 5000859, upload-time = "2026-09-30T14:45:05.009Z" }, + { url = "https://files.pythonhosted.org/packages/22/f7/70fd7ae4d1dbfa7ba29b02e1b9068771519a86027756510b700ce81086a8/cryptography-50.0.2-cp315-abi3.abi3t-musllinux_1_2_aarch64.whl", hash = "sha256:6f8700550aa1474a91e5dc07049c46f98b423b5b1ddd0483e0b51362eeeaf5be", size = 5092151, upload-time = "2026-09-30T15:29:15.932Z" }, + { url = "https://files.pythonhosted.org/packages/d4/be/688367b74de86984bd58d8efacfc7c9e68b89a6a22ced0fb4f38db50254a/cryptography-50.0.2-cp315-abi3.abi3t-musllinux_1_2_x86_64.whl", hash = "sha256:c71be1cbfa5cd9a41ee452acf1eccd82b2c05950358b106ec8ceb83411d1a020", size = 5286120, upload-time = "2026-09-30T15:29:18.309Z" }, + { url = "https://files.pythonhosted.org/packages/39/d1/55f8a3f2ef5d1529e16835ef10cf0fe3d559ce237b46dddc440c0bba3649/cryptography-50.0.2-cp315-abi3.abi3t-win_amd64.whl", hash = "sha256:c423ab384a46c4dff7217b2ea5ba2e11cffdeab6441acd04cf65a369caf0366c", size = 4111557, upload-time = "2026-09-30T15:29:20.155Z" }, + { url = "https://files.pythonhosted.org/packages/23/ad/ac987755d00e1e64273760228d2635ae38dae2be83e3c6e0d3289d91dec3/cryptography-50.0.2-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:0ec5f09541743261e66e291b4a0cbf0fb2997aeaab6d9e9c740b9dba1b58d1c2", size = 3943588, upload-time = "2026-09-30T15:29:22.265Z" }, + { url = "https://files.pythonhosted.org/packages/d5/8d/6d585339bedf85d45044c85d8412dac53f2bb6f918e8b7777efba1787844/cryptography-50.0.2-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c5e67125c7dca78d199ec4e116aa93dbb83494808ecbb8211a2cb09b1bf41dbd", size = 4756166, upload-time = "2026-09-30T15:29:24.58Z" }, + { url = "https://files.pythonhosted.org/packages/bf/f1/1c1f6874e8550cfddd4b688ceb38cefb6ed15ceed224d56f133f3d88c214/cryptography-50.0.2-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ee247f5c245c9a2fe7c8e2214e295918838e44e00a45a6718451e4004219e767", size = 4749145, upload-time = "2026-09-30T15:29:26.807Z" }, + { url = "https://files.pythonhosted.org/packages/c1/63/61b15dc1a8de03fe0adbe3fd7608b3ad5c73bf50993bbcb1faaa930afe33/cryptography-50.0.2-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:dfe9763530994147d9af1def057a5b9658b00e8f8fe8743d144d1e0911c2e454", size = 4763638, upload-time = "2026-09-30T15:29:28.588Z" }, + { url = "https://files.pythonhosted.org/packages/fc/35/b345bdfa40c9126df1a9d33236aa98418367931b8725f84fc3ae2b98dc59/cryptography-50.0.2-cp39-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:58ddb5a8e3179d12f19e4ea34d2d32e9d63a4baa142c875c1eb59f41b7243acd", size = 5382217, upload-time = "2026-09-30T15:29:30.589Z" }, + { url = "https://files.pythonhosted.org/packages/4f/87/ef344a9e616871f2519c22d6afcda79ddd5d35e9592d95eb6e677608d055/cryptography-50.0.2-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:f21e8a22c8605750c7af886bab299a363721264061b4ac0a30efb73cfd58efc5", size = 4781387, upload-time = "2026-09-30T15:29:32.605Z" }, + { url = "https://files.pythonhosted.org/packages/90/5b/f2fdb13cd0b96f6f932c8627bb292a45f11c64d21620a8e120aee9a3b848/cryptography-50.0.2-cp39-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:9c8402a82ea0dc4ceeab793db05f0fafa8ca139ca34fcde5df0f596103c74107", size = 4403790, upload-time = "2026-09-30T15:29:34.374Z" }, + { url = "https://files.pythonhosted.org/packages/bc/ce/7e4f662b1e3c393513569e402cfc85ac7da0bd3d5435e122a3140219eb2d/cryptography-50.0.2-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:0ddc924c04591c2811ca024d62ecad4f7f6f08af8939c211438f48a16bd23602", size = 4764319, upload-time = "2026-09-30T15:29:36.149Z" }, + { url = "https://files.pythonhosted.org/packages/3c/3f/86ff33ce34cc0de6847fb96e035a1a760d81652e38643f617c02ad32ef7a/cryptography-50.0.2-cp39-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:a6557e5f38e065ca9fbdaf7cfc7435ecb1d113aa81a022d1b51921ee7432e227", size = 5338560, upload-time = "2026-09-30T15:29:39.053Z" }, + { url = "https://files.pythonhosted.org/packages/40/cf/6b5c8e2fd9202d98988ab7cb5cc5c991704c4ad55f492ff408e4969f83f1/cryptography-50.0.2-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:1981f1db4630889b9ef7803fadef12b056f428cb6b85c27ba57b774793b6093c", size = 4780973, upload-time = "2026-09-30T15:29:41.251Z" }, + { url = "https://files.pythonhosted.org/packages/10/bf/8d6ebc7dded797bd0f0160d52188021211f011a2b164ef0ae1dac4587465/cryptography-50.0.2-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:7a8701d6b584d76e909e3d305b7d126b41439876a5aaf76cddc67fc230eafa2e", size = 4897738, upload-time = "2026-09-30T15:29:43.106Z" }, + { url = "https://files.pythonhosted.org/packages/d4/aa/f3f6e0de7e6253b8baa8b2d8fb9d50924fa75cee3d4624bd4bc1208ee923/cryptography-50.0.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:ce47f66801c20ec6c6632453bb5960fe38939e9306970b48b3a5a26de7745d94", size = 5058280, upload-time = "2026-09-30T15:29:44.827Z" }, + { url = "https://files.pythonhosted.org/packages/f6/b6/a1faf3a27ae9405fb34b1713cc73b2d8a26b04d5c561578fa2e6ef3e5bb9/cryptography-50.0.2-cp39-abi3-win_amd64.whl", hash = "sha256:4e81d95e5bafc2d6e34e4bed780e53e4d5b9a2f928573428aa4d35fbec1eb0de", size = 3854095, upload-time = "2026-09-30T15:29:46.782Z" }, +] + +[[package]] +name = "daff" +version = "1.4.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e1/d0/c0a1374db3afad0f9dfe6c795e5df102af03d49ad5e6e8502fb09eb88110/daff-1.4.2.tar.gz", hash = "sha256:47f0391eda7e2b5011f7ccac006b9178accb465bcb94a2c9f284257fff5d2686", size = 148251, upload-time = "2025-05-04T19:24:11.521Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/29/fe/d54a874e8d7b88bc03c459f63a993305db50039b734fab751a0466dabfc1/daff-1.4.2-py3-none-any.whl", hash = "sha256:88981a21d065e4378b5c4bd40b975dbfdea9b7ff540071f3bb5e20cc8b3590b5", size = 144922, upload-time = "2025-05-04T19:24:09.999Z" }, +] + +[[package]] +name = "dbt-adapters" +version = "1.24.5" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "agate" }, + { name = "dbt-common" }, + { name = "dbt-protos" }, + { name = "mashumaro", extra = ["msgpack"] }, + { name = "protobuf" }, + { name = "pytz" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b3/d4/bea32dd730def5b875a76c05d281d397ed720b9184d936cc4cedac605948/dbt_adapters-1.24.5.tar.gz", hash = "sha256:87de096a0d5b219905f1079b002f13e3447ffb63558f4f3c5b7278670d198acb", size = 146532, upload-time = "2026-07-15T07:18:49.036Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9c/c0/07f0e315458128c6ef246ba9698dc0f5e9333394ab8844508adac468c617/dbt_adapters-1.24.5-py3-none-any.whl", hash = "sha256:32345118b921160722a13f12c3e369b9d59a0fb8fb9b3297e29d8cd53f973411", size = 177259, upload-time = "2026-07-15T07:18:47.314Z" }, +] + +[[package]] +name = "dbt-common" +version = "1.39.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "agate" }, + { name = "colorama" }, + { name = "dbt-protos" }, + { name = "deepdiff" }, + { name = "isodate" }, + { name = "jinja2" }, + { name = "jsonschema" }, + { name = "mashumaro", extra = ["msgpack"] }, + { name = "opentelemetry-api" }, + { name = "pathspec" }, + { name = "protobuf" }, + { name = "python-dateutil" }, + { name = "requests" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/01/60/83f4a90ffa271b74b209d584e3be3b54b32f45aca36b0e0861e84e3f2d8c/dbt_common-1.39.0.tar.gz", hash = "sha256:868256ab2b59328efdd455644172ad17303f3f6a78a015f06667e865e068d620", size = 88069, upload-time = "2026-08-11T09:15:16.136Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3a/bc/56a9e62e67634631c7a99b246d92d1be22df746d265b36185fd7517b9163/dbt_common-1.39.0-py3-none-any.whl", hash = "sha256:30d63490b9ef2aad6f469ec841fb758430fcb1d997b79ff3df36fcb7b79a241f", size = 89092, upload-time = "2026-08-11T09:15:14.595Z" }, +] + +[[package]] +name = "dbt-core" +version = "1.10.23" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "agate" }, + { name = "click" }, + { name = "daff" }, + { name = "dbt-adapters" }, + { name = "dbt-common" }, + { name = "dbt-extractor" }, + { name = "dbt-protos" }, + { name = "dbt-semantic-interfaces" }, + { name = "jinja2" }, + { name = "jsonschema" }, + { name = "mashumaro", extra = ["msgpack"] }, + { name = "networkx" }, + { name = "packaging" }, + { name = "pathspec" }, + { name = "protobuf" }, + { name = "pydantic" }, + { name = "pytz" }, + { name = "pyyaml" }, + { name = "requests" }, + { name = "snowplow-tracker" }, + { name = "sqlparse" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0a/8d/27bba30052dd7c248be0b40389223d5b3d9dc6b06761d1297101197475a9/dbt_core-1.10.23.tar.gz", hash = "sha256:0781b9b887b20a66d439bd1ec33eae57db663c729825a19c11b7529766b0a5dd", size = 901971, upload-time = "2026-08-12T16:28:58.259Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/45/1d/995c8d233f930ab562a8bd31920509079967d1068299b9b7d895ef76e5f0/dbt_core-1.10.23-py3-none-any.whl", hash = "sha256:bdce6698a2ee93be7375a8be481818849b2438c5f8dddb0b6b24e4cb58a94e36", size = 988329, upload-time = "2026-08-12T16:28:57.087Z" }, +] + +[[package]] +name = "dbt-extractor" +version = "0.6.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f1/06/1f7b5d277af4bd7c3ab5065f79407c46a73950f0879fac69e51067c87649/dbt_extractor-0.6.0.tar.gz", hash = "sha256:d6cf08ec793b8bc2bd6e260ef818230ae68a4f71436fa489f08d7db1a52e2ffe", size = 270461, upload-time = "2025-04-07T16:46:30.532Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9f/dd/ec8f9e48e7dd5a52a69cca7907681d1779cf1cc8b02f2aa2acb6a2bf8bb4/dbt_extractor-0.6.0-cp39-abi3-macosx_10_12_x86_64.macosx_11_0_arm64.macosx_10_12_universal2.whl", hash = "sha256:4b6b1e70dde78cb904ca7a8958c2c803e77779b6ce108f4ea7ac479f5700db89", size = 790206, upload-time = "2025-04-07T16:46:05.352Z" }, + { url = "https://files.pythonhosted.org/packages/03/5f/233f326336aa21fbd9e7268f239a8464af145abd398a360d894c3286699d/dbt_extractor-0.6.0-cp39-abi3-macosx_10_12_x86_64.whl", hash = "sha256:dcf14ed245de8df269815ff4c4f555fa72d2621f4fff37c023b8c99d0e421b4f", size = 404381, upload-time = "2025-04-07T16:46:07.471Z" }, + { url = "https://files.pythonhosted.org/packages/c9/2a/e14c13b9a437780c5712525ce537915b531bba45481fc7102deb4492ff83/dbt_extractor-0.6.0-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:af451633390ac19669d3bde6c79822e657d32f5d903b3388bb00d56333fd52d5", size = 435109, upload-time = "2025-04-07T16:46:09.443Z" }, + { url = "https://files.pythonhosted.org/packages/58/2e/1ef1cd2b36973bea0a6823a7b7cd1b3db29b61ddebb015ceaea88b9e9347/dbt_extractor-0.6.0-cp39-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:05bcfab7ebd70296ceb31742e8333ba66a2c939de44e61a7088bebafa939aaf6", size = 434550, upload-time = "2025-04-07T16:46:10.916Z" }, + { url = "https://files.pythonhosted.org/packages/40/5a/468a2855181aaee5402efbf9ef757d074cd306eec22bbcd267cdd0edbe94/dbt_extractor-0.6.0-cp39-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:71b3f8897138cc6698d313b9a3d0450fd021937ff5463269ee18ed415541781b", size = 470137, upload-time = "2025-04-07T16:46:12.36Z" }, + { url = "https://files.pythonhosted.org/packages/b2/18/611dceb2fa7ea668471f290f34fec55fa3283e3ee9d0475d964e6ffaff97/dbt_extractor-0.6.0-cp39-abi3-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:868af715a6328d7317ce6e4db238f850f660fef13fb36b7ab4cf9163ed5f54ff", size = 524331, upload-time = "2025-04-07T16:46:14.177Z" }, + { url = "https://files.pythonhosted.org/packages/9e/ad/9dd410d4d95e336ae6b10c53c939bf1ff8e9991e1adb5ea4aefc4a87c445/dbt_extractor-0.6.0-cp39-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c1fd2b083a75e80b13e9874dc9699bfdfddf3baa9b6a8dea48de06d51a082733", size = 517959, upload-time = "2025-04-07T16:46:15.68Z" }, + { url = "https://files.pythonhosted.org/packages/a4/4f/6994cdfb51c5652fad0c8f9cf5b3ec1816cb10e99ed145eb27e6a9bcc16b/dbt_extractor-0.6.0-cp39-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:311f0d3a4994751c541a4fa303d205727ba90e90c85286c03d3d9284e2bf0bd4", size = 494850, upload-time = "2025-04-07T16:46:17.265Z" }, + { url = "https://files.pythonhosted.org/packages/df/5e/fad01e18d68ffd09c0f39cdedeed8fcaaea74a8b46d1a944472b5f95b72b/dbt_extractor-0.6.0-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:aecfa43f7e6f139e76d47e4e1d7b189655ae19a8cf697686230bacb89a94ae74", size = 442739, upload-time = "2025-04-07T16:46:19.002Z" }, + { url = "https://files.pythonhosted.org/packages/9d/82/49068ee2b9f38aa34d0f3196bb7b71d11af86630d5ed5cb6626108c97cd6/dbt_extractor-0.6.0-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:a5cb810edc60c0486f78cc29739ebda70c81b10a1686861e78addc9f91fcd7de", size = 618014, upload-time = "2025-04-07T16:46:21.571Z" }, + { url = "https://files.pythonhosted.org/packages/18/c6/cdaf1ac8959d571b5cb3587b8afef9e5fe60b99fe59aca94560808501d8b/dbt_extractor-0.6.0-cp39-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:080fd1edf123926ed97929c65a75874d0fea687ccd5d3ebbc9e81b339f099604", size = 697290, upload-time = "2025-04-07T16:46:23.089Z" }, + { url = "https://files.pythonhosted.org/packages/94/6d/46bdb9a809c66784fcc19b853311568cfd3041c075f0a578cb7116686841/dbt_extractor-0.6.0-cp39-abi3-musllinux_1_2_i686.whl", hash = "sha256:1b9ed7b15df983a735f87773f6765db8458680c02fcebbf89df4e238503c0e08", size = 644443, upload-time = "2025-04-07T16:46:24.463Z" }, + { url = "https://files.pythonhosted.org/packages/3b/02/b111856273e414ac80ef58d2103c9b7c6a5b29b1ec248999d3d5873ada00/dbt_extractor-0.6.0-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:caeaba8d8c813f8e32d586c12615c0c7d6b99bee4f1be845312e80ef731de164", size = 613017, upload-time = "2025-04-07T16:46:25.913Z" }, + { url = "https://files.pythonhosted.org/packages/c4/de/d1492ab6beaf0a18aee17c7a9562592ac2981e962b4058262f5eb6dabfc5/dbt_extractor-0.6.0-cp39-abi3-win32.whl", hash = "sha256:369dcc3499f160256756585783f1308868076d5a65d0a051348d22da8b90e67d", size = 252721, upload-time = "2025-04-07T16:46:27.295Z" }, + { url = "https://files.pythonhosted.org/packages/60/36/f5b1c4159fa911607f3a49fcbc535e4783870fd887bc0a1b3ad42587cb73/dbt_extractor-0.6.0-cp39-abi3-win_amd64.whl", hash = "sha256:a79a570fdcb672505ac2bdc12360a2a7aec622ef604d8c607225854ff862518c", size = 277146, upload-time = "2025-04-07T16:46:28.991Z" }, +] + +[[package]] +name = "dbt-protos" +version = "1.0.615" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "protobuf" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/4c/c9/9d176ee7d60737e16d990681989f7574c9074be3530d50e5d15470b1f0db/dbt_protos-1.0.615.tar.gz", hash = "sha256:05ee5544ffbc4d105728bdcf78b4d75c9881d66eda36c26a786a1d2f53d49694", size = 224939, upload-time = "2026-10-23T15:57:07.438Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/36/3c/8744fb10e0a943c5805ddc9372d9a3ad706439d6e8be09b6a95e2ff754db/dbt_protos-1.0.615-py3-none-any.whl", hash = "sha256:86a2237841b2b4b1f282d0dec6919ae6540544318cfd2d63239c961d49d0f195", size = 303036, upload-time = "2026-10-23T15:57:05.982Z" }, +] + +[[package]] +name = "dbt-semantic-interfaces" +version = "0.9.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "importlib-metadata" }, + { name = "jinja2" }, + { name = "jsonschema" }, + { name = "more-itertools" }, + { name = "pydantic" }, + { name = "python-dateutil" }, + { name = "pyyaml" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b0/91/c702d8fb143541fda10f5eb7a7a89f34bda38ee043ecb3e3653363d0c5a0/dbt_semantic_interfaces-0.9.0.tar.gz", hash = "sha256:5c921257dce8bb51c9ffb5479f2bdd959e16ebfb98ee833de6daa70788c47271", size = 93865, upload-time = "2025-07-09T20:06:30.454Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/82/41708b2b69d5fead88dea5ca0d863d6291da83ca6f1bd19246842d397e2b/dbt_semantic_interfaces-0.9.0-py3-none-any.whl", hash = "sha256:1b54c06ba89190a47a7f0563360930a0cce869e55b484ca09d261ade0e319155", size = 147008, upload-time = "2025-07-09T20:06:32.466Z" }, +] + +[[package]] +name = "dbt-snowflake" +version = "1.10.8" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "agate" }, + { name = "certifi" }, + { name = "dbt-adapters" }, + { name = "dbt-common" }, + { name = "dbt-core" }, + { name = "snowflake-connector-python", extra = ["secure-local-storage"] }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a5/d8/420cc25a4204804058ce32ae1d90e9bbf9a9c16f0ae1060fd9bba3078c7d/dbt_snowflake-1.10.8.tar.gz", hash = "sha256:f49e4197d9dd8710ff666d098c2224eed581aa8c29fb66f84916fc201512062a", size = 121154, upload-time = "2026-06-18T07:08:05.288Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6e/73/178283cc4274dcd6171216a73458cda3a8bfb4d3fafdd406eed38c3add7f/dbt_snowflake-1.10.8-py3-none-any.whl", hash = "sha256:dca21bcb49610ab67b5ce8d475e630d0187182247a43bd33de8b6a384665c02b", size = 71482, upload-time = "2026-06-18T07:08:03.232Z" }, +] + +[[package]] +name = "deepdiff" +version = "8.6.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "orderly-set" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/89/50/767448e792d41bfb6094ee317a355c1cb221dca24b2e178e2203bbea2a77/deepdiff-8.6.2.tar.gz", hash = "sha256:186dcbd181e4d76cef11ab05f802d0056c5d6083c5a6748c1473e9d7481e183e", size = 634860, upload-time = "2026-03-18T17:16:33.785Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2b/5f/c52bd1255db763d0cdcb7084d2e90c42119cb229302c56bdf1d0aa78abd2/deepdiff-8.6.2-py3-none-any.whl", hash = "sha256:4d22034a866c3928303a9332c279362f714192d9305bac17c498720d095fd1b4", size = 91979, upload-time = "2026-03-18T17:16:32.171Z" }, +] + +[[package]] +name = "filelock" +version = "3.32.7" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0f/59/e19834834cb01a32febfbb0f8a23a9088088f5d45991824ff2bc3b5e8acb/filelock-3.32.7.tar.gz", hash = "sha256:37b8a3d9811b0f9aef7e5ec5c71bb320de52df51e6ca9bcd6f5ad81187660da7", size = 225154, upload-time = "2026-09-16T00:24:20.907Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/15/df/31098c5aeb4d966b553641472bd55fcf5fdfac953549894b8a765ba44e91/filelock-3.32.7-py3-none-any.whl", hash = "sha256:65ff0d0190ea42038b32bda4b77834fb05be2cad4c5b9b01aa4dfb3614536e52", size = 100157, upload-time = "2026-09-16T00:24:19.543Z" }, +] + +[[package]] +name = "idna" +version = "3.20" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f5/08/8eea9d4b8302028f3abb2c0813953f7aec26d33b7a8960ed760e65ff29fa/idna-3.20.tar.gz", hash = "sha256:a7db850025b95ded1eae8a46181a1a6c56c92c96f0e2b005d9ff8dc0210cab44", size = 216463, upload-time = "2026-09-17T14:11:04.752Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/58/a2/bb081bab032533a855d44de1d56f8e8426114ff1ba5d1f07a438a0a654f8/idna-3.20-py3-none-any.whl", hash = "sha256:ab7ae7122974553370f0bdb919e1a960b2cd1bc1ef0276416d896db81c14582c", size = 69583, upload-time = "2026-09-17T14:11:03.168Z" }, +] + +[[package]] +name = "importlib-metadata" +version = "8.9.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "zipp" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e7/72/c600ae4f68c28fc19f9c31b9403053e5dbb8cace2e6842c7b7c3e4d42fe9/importlib_metadata-8.9.0.tar.gz", hash = "sha256:58850626cef4bd2df100378b0f2aea9724a7b92f10770d547725b047078f99ee", size = 56140, upload-time = "2026-03-20T16:56:26.362Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7d/f9/97f2ca8bb3ec6e4b1d64f983ebe98b9a192faddff67fac3d6303a537e670/importlib_metadata-8.9.0-py3-none-any.whl", hash = "sha256:e0f761b6ea91ced3b0844c14c9d955224d538105921f8e6754c00f6ca79fba7f", size = 27220, upload-time = "2026-03-20T16:56:25.07Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/e1/2069291243c926a2ff1cd706c7f3eeb9b62144bf60f77c9fb9ff2fb26bd3/iniconfig-2.3.1.tar.gz", hash = "sha256:67f4b9c50da0dedf52af349e7749a80a9057a5031199791b906c3bb3ae878960", size = 21209, upload-time = "2026-10-06T22:48:38.076Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/56/43/4ca9e49d27a1fcf6bece6f6aec0ea46bb9112489b93d4b688fb415457bdb/iniconfig-2.3.1-py3-none-any.whl", hash = "sha256:9121e2c1fdb355232495be3194c8dfe87ccc2d5dee45947b78e68f499790d7a7", size = 7552, upload-time = "2026-10-06T22:48:36.959Z" }, +] + +[[package]] +name = "isodate" +version = "0.7.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/54/4d/e940025e2ce31a8ce1202635910747e5a87cc3a6a6bb2d00973375014749/isodate-0.7.2.tar.gz", hash = "sha256:4cd1aa0f43ca76f4a6c6c0292a85f40b35ec2e43e315b59f06e6d32171a953e6", size = 29705, upload-time = "2024-10-08T23:04:11.5Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/15/aa/0aca39a37d3c7eb941ba736ede56d689e7be91cab5d9ca846bde3999eba6/isodate-0.7.2-py3-none-any.whl", hash = "sha256:28009937d8031054830160fce6d409ed342816b543597cece116d966c6d99e15", size = 22320, upload-time = "2024-10-08T23:04:09.501Z" }, +] + +[[package]] +name = "jaraco-classes" +version = "3.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "more-itertools" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/c0/ed4a27bc5571b99e3cff68f8a9fa5b56ff7df1c2251cc715a652ddd26402/jaraco.classes-3.4.0.tar.gz", hash = "sha256:47a024b51d0239c0dd8c8540c6c7f484be3b8fcf0b2d85c13825780d3b3f3acd", size = 11780, upload-time = "2024-03-31T07:27:36.643Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7f/66/b15ce62552d84bbfcec9a4873ab79d993a1dd4edb922cbfccae192bd5b5f/jaraco.classes-3.4.0-py3-none-any.whl", hash = "sha256:f662826b6bed8cace05e7ff873ce0f9283b5c924470fe664fff1c2f00f581790", size = 6777, upload-time = "2024-03-31T07:27:34.792Z" }, +] + +[[package]] +name = "jaraco-context" +version = "6.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/af/50/4763cd07e722bb6285316d390a164bc7e479db9d90daa769f22578f698b4/jaraco_context-6.1.2.tar.gz", hash = "sha256:f1a6c9d391e661cc5b8d39861ff077a7dc24dc23833ccee564b234b81c82dfe3", size = 16801, upload-time = "2026-03-20T22:13:33.922Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f2/58/bc8954bda5fcda97bd7c19be11b85f91973d67a706ed4a3aec33e7de22db/jaraco_context-6.1.2-py3-none-any.whl", hash = "sha256:bf8150b79a2d5d91ae48629d8b427a8f7ba0e1097dd6202a9059f29a36379535", size = 7871, upload-time = "2026-03-20T22:13:32.808Z" }, +] + +[[package]] +name = "jaraco-functools" +version = "4.6.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "more-itertools" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6c/1f/c23395957d41ccf27c4e535c3d334c4051e5395b3752057ba4cbaec35c56/jaraco_functools-4.6.0.tar.gz", hash = "sha256:880c577ec9720b3a052d5bc611fb9f2269b3d87902ef42440df443b88e443280", size = 20837, upload-time = "2026-07-14T01:28:02.544Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/02/36/ecc85bc96c273dc8a11273ed4782272975e6338d4a3e9228621175edf0e3/jaraco_functools-4.6.0-py3-none-any.whl", hash = "sha256:99e3dc0060c5cbe8fcd1cdb36258e2a65ca40f1566b2033b12abb1bb44dd3c30", size = 11677, upload-time = "2026-07-14T01:28:01.59Z" }, +] + +[[package]] +name = "jeepney" +version = "0.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7b/6f/357efd7602486741aa73ffc0617fb310a29b588ed0fd69c2399acbb85b0c/jeepney-0.9.0.tar.gz", hash = "sha256:cf0e9e845622b81e4a28df94c40345400256ec608d0e55bb8a3feaa9163f5732", size = 106758, upload-time = "2025-02-27T18:51:01.684Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b2/a3/e137168c9c44d18eff0376253da9f1e9234d0239e0ee230d2fee6cea8e55/jeepney-0.9.0-py3-none-any.whl", hash = "sha256:97e5714520c16fc0a45695e5365a2e11b81ea79bba796e26f9f1d178cb182683", size = 49010, upload-time = "2025-02-27T18:51:00.104Z" }, +] + +[[package]] +name = "jinja2" +version = "3.1.6" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "markupsafe" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/df/bf/f7da0350254c0ed7c72f3e33cef02e048281fec7ecec5f032d4aac52226b/jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d", size = 245115, upload-time = "2025-03-05T20:05:02.478Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/62/a1/3d680cbfd5f4b8f15abc1d571870c5fc3e594bb582bc3b64ea099db13e56/jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67", size = 134899, upload-time = "2025-03-05T20:05:00.369Z" }, +] + +[[package]] +name = "jmespath" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d3/59/322338183ecda247fb5d1763a6cbe46eff7222eaeebafd9fa65d4bf5cb11/jmespath-1.1.0.tar.gz", hash = "sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d", size = 27377, upload-time = "2026-01-22T16:35:26.279Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/14/2f/967ba146e6d58cf6a652da73885f52fc68001525b4197effc174321d70b4/jmespath-1.1.0-py3-none-any.whl", hash = "sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64", size = 20419, upload-time = "2026-01-22T16:35:24.919Z" }, +] + +[[package]] +name = "jsonschema" +version = "4.26.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "jsonschema-specifications" }, + { name = "referencing" }, + { name = "rpds-py" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b3/fc/e067678238fa451312d4c62bf6e6cf5ec56375422aee02f9cb5f909b3047/jsonschema-4.26.0.tar.gz", hash = "sha256:0c26707e2efad8aa1bfc5b7ce170f3fccc2e4918ff85989ba9ffa9facb2be326", size = 366583, upload-time = "2026-01-07T13:41:07.246Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/69/90/f63fb5873511e014207a475e2bb4e8b2e570d655b00ac19a9a0ca0a385ee/jsonschema-4.26.0-py3-none-any.whl", hash = "sha256:d489f15263b8d200f8387e64b4c3a75f06629559fb73deb8fdfb525f2dab50ce", size = 90630, upload-time = "2026-01-07T13:41:05.306Z" }, +] + +[[package]] +name = "jsonschema-specifications" +version = "2025.9.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "referencing" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/19/74/a633ee74eb36c44aa6d1095e7cc5569bebf04342ee146178e2d36600708b/jsonschema_specifications-2025.9.1.tar.gz", hash = "sha256:b540987f239e745613c7a9176f3edb72b832a4ac465cf02712288397832b5e8d", size = 32855, upload-time = "2025-09-08T01:34:59.186Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/41/45/1a4ed80516f02155c51f51e8cedb3c1902296743db0bbc66608a0db2814f/jsonschema_specifications-2025.9.1-py3-none-any.whl", hash = "sha256:98802fee3a11ee76ecaca44429fda8a41bff98b00a0f2838151b113f210cc6fe", size = 18437, upload-time = "2025-09-08T01:34:57.871Z" }, +] + +[[package]] +name = "keyring" +version = "25.7.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "jaraco-classes" }, + { name = "jaraco-context" }, + { name = "jaraco-functools" }, + { name = "jeepney", marker = "sys_platform == 'linux'" }, + { name = "pywin32-ctypes", marker = "sys_platform == 'win32'" }, + { name = "secretstorage", marker = "sys_platform == 'linux'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/43/4b/674af6ef2f97d56f0ab5153bf0bfa28ccb6c3ed4d1babf4305449668807b/keyring-25.7.0.tar.gz", hash = "sha256:fe01bd85eb3f8fb3dd0405defdeac9a5b4f6f0439edbb3149577f244a2e8245b", size = 63516, upload-time = "2025-11-16T16:26:09.482Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/81/db/e655086b7f3a705df045bf0933bdd9c2f79bb3c97bfef1384598bb79a217/keyring-25.7.0-py3-none-any.whl", hash = "sha256:be4a0b195f149690c166e850609a477c532ddbfbaed96a404d4e43f8d5e2689f", size = 39160, upload-time = "2025-11-16T16:26:08.402Z" }, +] + +[[package]] +name = "leather" +version = "0.4.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/9e/09/849cf129d7eae1e42f873f2dbd60323267c738390b686a7384fb3fb289ad/leather-0.4.1.tar.gz", hash = "sha256:67119c2aee93be821f077193bd8534e296c05b38bd174d9c5a80c4aa31d1a4d3", size = 44072, upload-time = "2025-12-15T19:01:42.224Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1a/d4/c4dcb02ed11f8884e169b3350fc40aa4c08edf8bed77a8f0f267542e6452/leather-0.4.1-py3-none-any.whl", hash = "sha256:ec61cba1ca3ccb96ed90e38b116fc58757d97d352171006b3288c47ce3fbd183", size = 30340, upload-time = "2025-12-15T19:01:40.823Z" }, +] + +[[package]] +name = "markupsafe" +version = "3.0.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/38/9b/e422a865e1d5d57d0e509b4e0bf1c1a70a7f6382c29a5aa428df994c8bc8/markupsafe-3.0.4.tar.gz", hash = "sha256:2e9ad7dd851bf45fab9f75cbff4cb493fee9979e8d8c7c9c3ee119022518edd6", size = 153777, upload-time = "2026-10-02T23:07:22.29Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/81/09/4c59d56b8461ae8eb0d8ba34bb25b7e618547044679d58a82ef9b2479fc1/markupsafe-3.0.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:61631e08084be9e21a8967ec3139c7616ed7c5e9368e05c86d1b39562c8a57b6", size = 11658, upload-time = "2026-10-02T23:04:51.876Z" }, + { url = "https://files.pythonhosted.org/packages/a2/f0/d6613774d86fbf6d145751d43c59875e47a6f9f17daee0aef173bd36d90e/markupsafe-3.0.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0930db9bdc62d22944e10b066448bb65dc9abe9112880c7cab8da54db4284d5f", size = 12050, upload-time = "2026-10-02T23:04:52.931Z" }, + { url = "https://files.pythonhosted.org/packages/0d/f2/8f18e0b806eb13c1f8d07d917a720831ead54253a6dec011fbc78098a6f8/markupsafe-3.0.4-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6a45c3d514f2436064db00d7fc8778d888f0236ebfed649b53d13a59e69ad51b", size = 24363, upload-time = "2026-10-02T23:04:53.895Z" }, + { url = "https://files.pythonhosted.org/packages/60/ce/fa07dbe8a5675558fa36dea033e19995bc783de2dec5f540ccb9030b06aa/markupsafe-3.0.4-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:1e1451fab512d1bcc3dc26988ec1edb0b82c2db909132872cd9356070a6b63df", size = 28525, upload-time = "2026-10-02T23:04:54.905Z" }, + { url = "https://files.pythonhosted.org/packages/85/40/be87c01f3868ec217f8a2015089d71c22c8c5a75324822e5ed1cdd87210d/markupsafe-3.0.4-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:bd3ce56ae2cbae3ba82b683bc425cd7e48d2ed8b10f3e818186b6f5646d9271c", size = 24733, upload-time = "2026-10-02T23:04:56.229Z" }, + { url = "https://files.pythonhosted.org/packages/4f/a7/aeedb5140afa41fc74c225e9184ab96723a6e873b6ee1c9fede7283456d8/markupsafe-3.0.4-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:8e124f974786f831d6043728e38296969d3579db8896fe004682f5758e613581", size = 22985, upload-time = "2026-10-02T23:04:57.521Z" }, + { url = "https://files.pythonhosted.org/packages/c3/fc/e91352bb08c6a59da3ef0909d457bf95a5f5908fbf151b30a06d9dbcfbb4/markupsafe-3.0.4-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:c02e8f18bdedba082cef725942ac823b9b60656db07f7e265cb31618dfd00d77", size = 22001, upload-time = "2026-10-02T23:04:58.597Z" }, + { url = "https://files.pythonhosted.org/packages/5d/f8/bffee5e7d2a3deb59748a797650a48af7e672025cf641a79344a771ad106/markupsafe-3.0.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:9f098115c247e11d138ab83a28fa0323c77015007ea2df73ba5fd714dfefd67c", size = 23793, upload-time = "2026-10-02T23:04:59.686Z" }, + { url = "https://files.pythonhosted.org/packages/ed/59/b853d6628ecb4d658e1d637224846d5e9bb4adf4f8df97f3be9f29dce2ec/markupsafe-3.0.4-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:d5f93ebbeb8032d47e349328ec8662d973d9b05a70b3c35df1f91fe419b84749", size = 22640, upload-time = "2026-10-02T23:05:00.768Z" }, + { url = "https://files.pythonhosted.org/packages/09/b2/1506df394f0f075797c418d0301498f49e43be194e3ffcb49e6fe6ccf022/markupsafe-3.0.4-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:64511c54db4e4987aef4c41923235927428729e8174c5dba488429be70a998ed", size = 24074, upload-time = "2026-10-02T23:05:01.813Z" }, + { url = "https://files.pythonhosted.org/packages/c7/81/5ed69cda630ac69ef60d06c09ba5a7f84ff66a2e28cf986fd5614ab3c6e6/markupsafe-3.0.4-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:e1a622f13970d81f95d0c72f9dc090dce9085fccfa4c9f2174377ee32bd15786", size = 21563, upload-time = "2026-10-02T23:05:03.239Z" }, + { url = "https://files.pythonhosted.org/packages/0c/fe/fb1e79be0fea60aa32602ebefc9c35a82bb42b4df157285ab7dfec12341a/markupsafe-3.0.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:c9a7f43c0b202b334cc9184af09bb8f21d3a209e038efaf106936fb69e6b026e", size = 23048, upload-time = "2026-10-02T23:05:04.479Z" }, + { url = "https://files.pythonhosted.org/packages/c8/52/7632a53360671a9b750cdbabaf9cdd89f18b42248b8e4cb42c0b0296e459/markupsafe-3.0.4-cp312-cp312-win32.whl", hash = "sha256:f0ec3b750b59375eab5b0fb2b9254810c00a3375be6d789899f1055a1d556237", size = 14108, upload-time = "2026-10-02T23:05:05.513Z" }, + { url = "https://files.pythonhosted.org/packages/3f/bf/62495e180b7000aaf30000fff849e933f74264638057176cf46852500adc/markupsafe-3.0.4-cp312-cp312-win_amd64.whl", hash = "sha256:11935df9bf455ed0c04eb87bcd720f02b1fe5e02128a9430f23aed6f93336fc7", size = 14303, upload-time = "2026-10-02T23:05:06.538Z" }, + { url = "https://files.pythonhosted.org/packages/c5/8e/4c24208776a65878d656996945aacfbfe010d3720d1a98fc0eb8491fc03b/markupsafe-3.0.4-cp312-cp312-win_arm64.whl", hash = "sha256:a4bbd2d87dd233b9fc5812160c3d0ffbe42edc22a26ce0469f58479ede633fe9", size = 14174, upload-time = "2026-10-02T23:05:07.617Z" }, +] + +[[package]] +name = "mashumaro" +version = "3.14" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/eb/47/0a450b281bef2d7e97ec02c8e1168d821e283f58e02e6c403b2bb4d73c1c/mashumaro-3.14.tar.gz", hash = "sha256:5ef6f2b963892cbe9a4ceb3441dfbea37f8c3412523f25d42e9b3a7186555f1d", size = 166160, upload-time = "2024-10-23T21:48:40.164Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1b/35/8d63733a2c12149d0c7663c29bf626bdbeea5f0ff963afe58a42b4810981/mashumaro-3.14-py3-none-any.whl", hash = "sha256:c12a649599a8f7b1a0b35d18f12e678423c3066189f7bc7bd8dd431c5c8132c3", size = 92183, upload-time = "2024-10-23T21:48:38.334Z" }, +] + +[package.optional-dependencies] +msgpack = [ + { name = "msgpack" }, +] + +[[package]] +name = "more-itertools" +version = "10.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ea/5d/38b681d3fce7a266dd9ab73c66959406d565b3e85f21d5e66e1181d93721/more_itertools-10.8.0.tar.gz", hash = "sha256:f638ddf8a1a0d134181275fb5d58b086ead7c6a72429ad725c67503f13ba30bd", size = 137431, upload-time = "2025-09-02T15:23:11.018Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a4/8e/469e5a4a2f5855992e425f3cb33804cc07bf18d48f2db061aec61ce50270/more_itertools-10.8.0-py3-none-any.whl", hash = "sha256:52d4362373dcf7c52546bc4af9a86ee7c4579df9a8dc268be0a2f949d376cc9b", size = 69667, upload-time = "2025-09-02T15:23:09.635Z" }, +] + +[[package]] +name = "msgpack" +version = "1.2.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/0a/e7/bb605a7bab2d8425a64b3fa762b39dc1bf1c7e3f11ba6fb5413d6db0ff8c/msgpack-1.2.3.tar.gz", hash = "sha256:32edb81a2b5eb7cd7c9d941b2bfbbb082fd2cd09e0e725930316af6b708db186", size = 196517, upload-time = "2026-09-29T02:33:52.276Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/af/12/4d7c6d6203416d9fbf0f59ebaa805e70fb929b93a41b611bc821ec5964a0/msgpack-1.2.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:89c930aece4e972b208ba589c8410b4167b05e411a5ea2cb25fd96f8bc47ee43", size = 91577, upload-time = "2026-09-29T02:32:02.141Z" }, + { url = "https://files.pythonhosted.org/packages/eb/c7/8576ad39f4ca42ddad26f68eb8621d2d0a60501193d480f504bd9d7f36c4/msgpack-1.2.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:905a189853d6bdb204c7ae5f4ab77fb857448abfff574d3d93c62e2815b24b4f", size = 90027, upload-time = "2026-09-29T02:32:03.508Z" }, + { url = "https://files.pythonhosted.org/packages/0a/3a/aa9c580aea1314529a0f3562461479780b0d254b064f0880956bfbcc74a8/msgpack-1.2.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f3d7b3d0018746b5997dd6b14a1870b07cc4c327d9101145d94a1fc264a51a06", size = 460343, upload-time = "2026-09-29T02:32:04.906Z" }, + { url = "https://files.pythonhosted.org/packages/3a/cf/9c2e4d6c179529d5bf4a64cff76fa581486569e9fbdd35bd98f51cb624bf/msgpack-1.2.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ede33b2892ceb976283e009ad12fa1834cfdf1f9c43ee9c97849fc588d00a618", size = 472998, upload-time = "2026-09-29T02:32:06.69Z" }, + { url = "https://files.pythonhosted.org/packages/7b/41/915c81fe6df2d3cbdb0dece4f1a5cd313e1cd2abd9f501d0f50c0582517e/msgpack-1.2.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:666ef5601ab0e6e345e47febc96aa81143cc932201543480cbb9499164f05ffb", size = 423216, upload-time = "2026-09-29T02:32:08.739Z" }, + { url = "https://files.pythonhosted.org/packages/a2/e7/7dda8b1039abfd9bba4c5068172c67135c9e33089f503512db9226f23c24/msgpack-1.2.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:87cf2ef05ff2f2493ba29fcdaef27e960ca64dacfd13460ae29e6f92e0ed05bb", size = 451218, upload-time = "2026-09-29T02:32:10.517Z" }, + { url = "https://files.pythonhosted.org/packages/16/5b/ce995c1ed4a0522b7f2d034bc2034fd63005f240b945961b70fb56fbaf3d/msgpack-1.2.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:b774ff994d844e541439ac5d2d49a14def4104830c3465e9394c153f86200ffb", size = 422453, upload-time = "2026-09-29T02:32:11.956Z" }, + { url = "https://files.pythonhosted.org/packages/d2/3f/ce191fb87e2650d0166b34c437e499ee4a7f9db9c1eb164f41725eb6160e/msgpack-1.2.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:eaf7e82249837e3aa97297b34a0bb9ff562027381631e057cea6e1367f10b438", size = 469003, upload-time = "2026-09-29T02:32:13.663Z" }, + { url = "https://files.pythonhosted.org/packages/42/35/539123407fe200fb16609c835675496fbeb6017ace9fc93909f0613223ae/msgpack-1.2.3-cp312-cp312-win32.whl", hash = "sha256:7c047250096f9fc19dba26e3d1639b5e7a84114003605c94def667149a70ced1", size = 68303, upload-time = "2026-09-29T02:32:15.02Z" }, + { url = "https://files.pythonhosted.org/packages/6f/4c/331b45f9b86fbda6b9e103244d189068e51f726d8c40021ed66e1f2c415e/msgpack-1.2.3-cp312-cp312-win_amd64.whl", hash = "sha256:3ec409b0d6aa8e9eec6eaf881b893caa215dbe68c5319ca96e8a271d81bb111d", size = 76744, upload-time = "2026-09-29T02:32:16.344Z" }, + { url = "https://files.pythonhosted.org/packages/13/9f/fb572dc42b9fac06c7ea848aaee6e140d84469743bd1402bc07089fc4566/msgpack-1.2.3-cp312-cp312-win_arm64.whl", hash = "sha256:59612b4ed48a04cf024584218e813562f3b30a3bafa5f55abe300b15da314751", size = 71580, upload-time = "2026-09-29T02:32:17.617Z" }, +] + +[[package]] +name = "networkx" +version = "3.7" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/dc/76/3af777226b63a5e64a6b36b1ec5855c14e2b94a37096d4760e595fc43511/networkx-3.7.tar.gz", hash = "sha256:fd77a511bd90f39f3d016351345b52cf5319b813bdca01de3f755d3cca62e96a", size = 1866482, upload-time = "2026-09-21T16:45:16.974Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/cd/fe58041e9011f307c490e3e17dd48cc516448f7c698a3f2d9d9d65d7e6a8/networkx-3.7-py3-none-any.whl", hash = "sha256:e3fd2c13a7814cee3746340d8d7f8598a67f16a58bf47fb7f8793fab6efca1b0", size = 2142205, upload-time = "2026-09-21T16:45:14.609Z" }, +] + +[[package]] +name = "numpy" +version = "2.5.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/95/b0/c7453d0b6e2073c3264468b106ee1563750cecc910965e67357e3698c83e/numpy-2.5.4.tar.gz", hash = "sha256:9a94cf751c9ad8ebaa835bcd3d40dacf8534ad086b88c38029b65123c7999d2a", size = 20866315, upload-time = "2026-10-10T20:05:31.422Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d0/97/ba2074e92b7befea137e77ea8471e768bbd87c339b7e8c9f5a931949f977/numpy-2.5.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c6342f54c67093cae5c0227eb0eb772fdb79f2a2c37a6eb278b9909ee06aa356", size = 17001609, upload-time = "2026-10-10T20:02:40.843Z" }, + { url = "https://files.pythonhosted.org/packages/ff/a9/bac826765e971d8e16e2064e9ac7525fd69b40ac17c905033a7f5442023f/numpy-2.5.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b11e8fda06a7d69f15ebf542660b74466c2e51094800c1fb794f47ad4faeef17", size = 12015718, upload-time = "2026-10-10T20:02:43.45Z" }, + { url = "https://files.pythonhosted.org/packages/31/2f/5ea3570fcb8ccd0882bea99436a513b2c85dad8f774a2057849130a8fb99/numpy-2.5.4-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:9cb18a327b49c5c337f972b03682f6a49855525faaf3c0d3e9c96cd0fd8880a8", size = 5451717, upload-time = "2026-10-10T20:02:46.169Z" }, + { url = "https://files.pythonhosted.org/packages/34/f2/b4fc1bafca03868220b5eaf729d2f21ebd7d7b151c0f9e144fe212bbca35/numpy-2.5.4-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:aec3fc4b32ff82421274f5d205c559c51c840c8df66a78efd7f3612dd005a26a", size = 6789926, upload-time = "2026-10-10T20:02:48.139Z" }, + { url = "https://files.pythonhosted.org/packages/dc/96/8319e2457ae4333c62c815c7006b869a4f60985c1e01024c2f8c6c040fe5/numpy-2.5.4-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fe4d21ab149f15e4e6043dfb0de87e6e5f34ac176cde83060e9802981fca2ac2", size = 15695312, upload-time = "2026-10-10T20:02:50.115Z" }, + { url = "https://files.pythonhosted.org/packages/43/a3/c799c62e19c337e6d3770b08e475887fb30ce8477d3c09efca6b2f0228a6/numpy-2.5.4-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fbde6962867ee75b48b0ee29b2b9372ec5d617799dbaf38e82dc0596f2f7738a", size = 16727283, upload-time = "2026-10-10T20:02:53.186Z" }, + { url = "https://files.pythonhosted.org/packages/39/6b/3604e53fb00314d0dc1b94ec9125a1484f649c0a17480b1f0f0c7a9d6250/numpy-2.5.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:381a7a3d2e65e64c0ec302795ab9dc12bb1e73f150904699c153716177eebdaf", size = 17047890, upload-time = "2026-10-10T20:02:56.038Z" }, + { url = "https://files.pythonhosted.org/packages/4a/7a/e8b58a5289a0d464c52885de47c35a935cdd70c03a4c3ab94a5126416dd0/numpy-2.5.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b89d0aaae2fe498c648f4c4795c084db535af5bd98ef942b2a3681fb74ce8645", size = 18485839, upload-time = "2026-10-10T20:02:59.018Z" }, + { url = "https://files.pythonhosted.org/packages/6f/c9/47094f597015009f310b8c900def59065ef1ff5a6fe7b51fc65ec58ec2c6/numpy-2.5.4-cp312-cp312-win32.whl", hash = "sha256:9968ab7e49b93ac6e1c3b2239732183152c9150f16308d30b66a372cffe3483c", size = 6138936, upload-time = "2026-10-10T20:03:01.626Z" }, + { url = "https://files.pythonhosted.org/packages/12/33/fefe62073dc8acfd0f2b9ed7c003af2f50aa61555e113e6db02b8f79f145/numpy-2.5.4-cp312-cp312-win_amd64.whl", hash = "sha256:a7b1b6353e36a7e50de2973a38d705c88ee93adcf120673cee7f45a4a3fa223a", size = 12573091, upload-time = "2026-10-10T20:03:04.349Z" }, + { url = "https://files.pythonhosted.org/packages/1a/07/161270b0c2eec56e4c905f6d6d22e1b836887b2cb189d3f5820aa588e9dd/numpy-2.5.4-cp312-cp312-win_arm64.whl", hash = "sha256:aa1cce2ff3f8d953de38b76bf44602caeb69f101430208f64a10067f7cb4b1d3", size = 10521630, upload-time = "2026-10-10T20:03:06.767Z" }, +] + +[[package]] +name = "opentelemetry-api" +version = "1.45.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/2e/02/6e0ae9cc61bd3169d401077b507b3ebc344745171e1051ab430be012dcd9/opentelemetry_api-1.45.1.tar.gz", hash = "sha256:aa38ed19bcc084ba42782a73255b3582283eced7ad6dddbd6695189e69adfb75", size = 72804, upload-time = "2026-10-06T17:32:58.133Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/41/f7dcf80b81ee8e71c1a2b59f14208bc723edbd89ed027a73b175abf6348e/opentelemetry_api-1.45.1-py3-none-any.whl", hash = "sha256:b31553efa588ae44bc306f863c785c5333a9ecc091248c6ee68b4b6c87fdedfb", size = 60256, upload-time = "2026-10-06T17:32:33.506Z" }, +] + +[[package]] +name = "orderly-set" +version = "5.5.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/4a/88/39c83c35d5e97cc203e9e77a4f93bf87ec89cf6a22ac4818fdcc65d66584/orderly_set-5.5.0.tar.gz", hash = "sha256:e87185c8e4d8afa64e7f8160ee2c542a475b738bc891dc3f58102e654125e6ce", size = 27414, upload-time = "2025-07-10T20:10:55.885Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/12/27/fb8d7338b4d551900fa3e580acbe7a0cf655d940e164cb5c00ec31961094/orderly_set-5.5.0-py3-none-any.whl", hash = "sha256:46f0b801948e98f427b412fcabb831677194c05c3b699b80de260374baa0b1e7", size = 13068, upload-time = "2025-07-10T20:10:54.377Z" }, +] + +[[package]] +name = "packaging" +version = "26.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/fa/3944b40b07da9ce895c0e6303a5ab7d53da063554f534556b134a54d6093/packaging-26.3.tar.gz", hash = "sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79", size = 313412, upload-time = "2026-08-04T18:15:28.737Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", hash = "sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", size = 129956, upload-time = "2026-08-04T18:15:27.159Z" }, +] + +[[package]] +name = "pandas" +version = "3.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, + { name = "python-dateutil" }, + { name = "tzdata", marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/55/b1/a10c0d1a7e3ce44eb42635771004aa503e3b9ddde0bd39c524dda65f7eff/pandas-3.1.0.tar.gz", hash = "sha256:ecb9b2d8a817b1e1ba1fdb90c0abe9eab31b4fdf2562b7a56b86ac66ace17e49", size = 5933421, upload-time = "2026-10-22T11:57:03.087Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e9/12/b94c28870f351e9650c724e6daedf661455b348a9dca6d717c42c30fa216/pandas-3.1.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:f4772320b7735d96dbbef437129f840dec4175f8a18a2391bd1222ed8943825c", size = 11587014, upload-time = "2026-10-22T11:55:07.208Z" }, + { url = "https://files.pythonhosted.org/packages/1b/4b/af8ca15261db52ab07f5d0c2d4c1bc28d822ea4c2a79756bc2606c012e7c/pandas-3.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:cc5e4e12d287ea77bbf4c4d4bf47ed152d82a990d3a6794dee7c350f887532b5", size = 11167712, upload-time = "2026-10-22T11:55:09.523Z" }, + { url = "https://files.pythonhosted.org/packages/49/c4/cd5127751e2bef3d10f07e0d856e120916a3df0df6450f759ddb48c78b1c/pandas-3.1.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9b4e696ff70ed06e986feeec09356c491ce878d0df69a36c8e591b6a38afa29b", size = 11476964, upload-time = "2026-10-22T11:55:11.889Z" }, + { url = "https://files.pythonhosted.org/packages/c5/a6/dc89cdd95e12093bc813e9b814a09f1a056be3e1304edaef9bf9dbee108a/pandas-3.1.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9433b5dc2f26ef9d9b56daaf4d41d078bf4372c5d88c42ed5b397a6eea80de16", size = 12019688, upload-time = "2026-10-22T11:55:14.246Z" }, + { url = "https://files.pythonhosted.org/packages/36/87/da14aa671da792612f15847451e731d5dc0f06f6747b76b15f197ffcce13/pandas-3.1.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3cba20fd269808e224d6212273008173978033a1dee0f3f217bfdbd58775a72c", size = 12490149, upload-time = "2026-10-22T11:55:16.675Z" }, + { url = "https://files.pythonhosted.org/packages/5e/a0/1ce996c7354ad37983955571058ba0dde585663163ab067275197183868f/pandas-3.1.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:743554ed96755ad1963397ccadbe9965d92db8777ca2ae8c1fe58bd9549d7f28", size = 13090624, upload-time = "2026-10-22T11:55:18.854Z" }, + { url = "https://files.pythonhosted.org/packages/92/72/d235f82b51a22478938a3583b68e8dc05811937543adf410f34bab76004a/pandas-3.1.0-cp312-cp312-pyemscripten_2024_0_wasm32.whl", hash = "sha256:ac2f833557874a79fc0b5fac4c86e402ea6498beba8753dc29ca23d08481689a", size = 8091628, upload-time = "2026-10-22T11:55:21.285Z" }, + { url = "https://files.pythonhosted.org/packages/10/72/06aefff8fe5efc5e1ca8bfb2695b648a435358f6dc224d4e40159314181a/pandas-3.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:b8c75dbb27c7d170b25c13dd35515da4bb92d40fe8bb3bd2c69a95de2218889b", size = 10748836, upload-time = "2026-10-22T11:55:23.47Z" }, + { url = "https://files.pythonhosted.org/packages/f6/b6/4c86f659f5b14790d14e17253becc93c4e3d1511b73f0ba2184f06b164f8/pandas-3.1.0-cp312-cp312-win_arm64.whl", hash = "sha256:73b6cea22090b7123c78ee662a5108ff36616b0bec4fe47026f86fb04a49bce7", size = 10128898, upload-time = "2026-10-22T11:55:25.995Z" }, +] + +[[package]] +name = "parsedatetime" +version = "2.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/a8/20/cb587f6672dbe585d101f590c3871d16e7aec5a576a1694997a3777312ac/parsedatetime-2.6.tar.gz", hash = "sha256:4cb368fbb18a0b7231f4d76119165451c8d2e35951455dfee97c62a87b04d455", size = 60114, upload-time = "2020-05-31T23:50:57.443Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9d/a4/3dd804926a42537bf69fb3ebb9fd72a50ba84f807d95df5ae016606c976c/parsedatetime-2.6-py3-none-any.whl", hash = "sha256:cb96edd7016872f58479e35879294258c71437195760746faffedb692aef000b", size = 42548, upload-time = "2020-05-31T23:50:56.315Z" }, +] + +[[package]] +name = "pathspec" +version = "0.12.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ca/bc/f35b8446f4531a7cb215605d100cd88b7ac6f44ab3fc94870c120ab3adbf/pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712", size = 51043, upload-time = "2023-12-10T22:30:45Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/cc/20/ff623b09d963f88bfde16306a54e12ee5ea43e9b597108672ff3a408aad6/pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08", size = 31191, upload-time = "2023-12-10T22:30:43.14Z" }, +] + +[[package]] +name = "platformdirs" +version = "4.13.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/58/e6/d0abbb256b44e6a05748c6942511ff67db8dbb068de0c35b24bf1b395b09/platformdirs-4.13.5.tar.gz", hash = "sha256:05831adf87d392760f4b5c3a0a95579ab5a10b18b9b9bd43680dcb47544788a0", size = 62362, upload-time = "2026-10-19T01:02:11.778Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/6c/8a/1d8cb3842909dc193a73835adedf412e32e18f8ffd59f43983949622fe6f/platformdirs-4.13.5-py3-none-any.whl", hash = "sha256:53d3ec4280b815c1e0b02edad7f9ee8fda1ba9c0865dd9c2e6e63849ce82fc1e", size = 32779, upload-time = "2026-10-19T01:02:10.17Z" }, +] + +[[package]] +name = "pluggy" +version = "1.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bf/db/7fc19e6f2dc92a966727031389fc2e08b558f0f25eb7403c1119ad4713cd/pluggy-1.7.0.tar.gz", hash = "sha256:d1eaa46ebb595891b860ab086b4d09c8588af65ebd4361b8e8f4bb8920b90ba8", size = 123304, upload-time = "2026-10-15T09:50:58.343Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/40/9e/2b38731e0fc536806f16490e1a12d7f0dc2a1235aa8cc07bcc75416a7daa/pluggy-1.7.0-py3-none-any.whl", hash = "sha256:7dd7b0d8832ba3cb632c306926ded123429211b83641b35dc5c41ad2d34f9bec", size = 27082, upload-time = "2026-10-15T09:50:56.808Z" }, +] + +[[package]] +name = "protobuf" +version = "6.33.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/66/70/e908e9c5e52ef7c3a6c7902c9dfbb34c7e29c25d2f81ade3856445fd5c94/protobuf-6.33.6.tar.gz", hash = "sha256:a6768d25248312c297558af96a9f9c929e8c4cee0659cb07e780731095f38135", size = 444531, upload-time = "2026-03-18T19:05:00.988Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fc/9f/2f509339e89cfa6f6a4c4ff50438db9ca488dec341f7e454adad60150b00/protobuf-6.33.6-cp310-abi3-win32.whl", hash = "sha256:7d29d9b65f8afef196f8334e80d6bc1d5d4adedb449971fefd3723824e6e77d3", size = 425739, upload-time = "2026-03-18T19:04:48.373Z" }, + { url = "https://files.pythonhosted.org/packages/76/5d/683efcd4798e0030c1bab27374fd13a89f7c2515fb1f3123efdfaa5eab57/protobuf-6.33.6-cp310-abi3-win_amd64.whl", hash = "sha256:0cd27b587afca21b7cfa59a74dcbd48a50f0a6400cfb59391340ad729d91d326", size = 437089, upload-time = "2026-03-18T19:04:50.381Z" }, + { url = "https://files.pythonhosted.org/packages/5c/01/a3c3ed5cd186f39e7880f8303cc51385a198a81469d53d0fdecf1f64d929/protobuf-6.33.6-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:9720e6961b251bde64edfdab7d500725a2af5280f3f4c87e57c0208376aa8c3a", size = 427737, upload-time = "2026-03-18T19:04:51.866Z" }, + { url = "https://files.pythonhosted.org/packages/ee/90/b3c01fdec7d2f627b3a6884243ba328c1217ed2d978def5c12dc50d328a3/protobuf-6.33.6-cp39-abi3-manylinux2014_aarch64.whl", hash = "sha256:e2afbae9b8e1825e3529f88d514754e094278bb95eadc0e199751cdd9a2e82a2", size = 324610, upload-time = "2026-03-18T19:04:53.096Z" }, + { url = "https://files.pythonhosted.org/packages/9b/ca/25afc144934014700c52e05103c2421997482d561f3101ff352e1292fb81/protobuf-6.33.6-cp39-abi3-manylinux2014_s390x.whl", hash = "sha256:c96c37eec15086b79762ed265d59ab204dabc53056e3443e702d2681f4b39ce3", size = 339381, upload-time = "2026-03-18T19:04:54.616Z" }, + { url = "https://files.pythonhosted.org/packages/16/92/d1e32e3e0d894fe00b15ce28ad4944ab692713f2e7f0a99787405e43533a/protobuf-6.33.6-cp39-abi3-manylinux2014_x86_64.whl", hash = "sha256:e9db7e292e0ab79dd108d7f1a94fe31601ce1ee3f7b79e0692043423020b0593", size = 323436, upload-time = "2026-03-18T19:04:55.768Z" }, + { url = "https://files.pythonhosted.org/packages/c4/72/02445137af02769918a93807b2b7890047c32bfb9f90371cbc12688819eb/protobuf-6.33.6-py3-none-any.whl", hash = "sha256:77179e006c476e69bf8e8ce866640091ec42e1beb80b213c3900006ecfba6901", size = 170656, upload-time = "2026-03-18T19:04:59.826Z" }, +] + +[[package]] +name = "pyarrow" +version = "26.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ec/34/17c34cb38e5d940e38f0f0d9fdfa0e8a506676409ea9b85aff7e3079f831/pyarrow-26.0.0.tar.gz", hash = "sha256:0cccd36e00ea3afeb52ded61f2721ce71f604853d70c45365c58324eb773d6ae", size = 1239433, upload-time = "2026-10-09T08:26:25.315Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/60/6793778f2617cce469383dac0ba08c4f2401cf342df0c7b9ca53939d9b46/pyarrow-26.0.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:90ddaf7c625307ad52f31a9b25c34fe5e4897c7529ee3481135822b2b6842ff1", size = 36333953, upload-time = "2026-10-09T08:14:00.387Z" }, + { url = "https://files.pythonhosted.org/packages/db/81/f944cc63ce8a753e5fbff25de6d1d475ebd7fffdf9cf98c65130294fc896/pyarrow-26.0.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:ee341973f78a0b46e073d065e88e75026a9c584051e97f98a0d05d96c6bac7dd", size = 38688456, upload-time = "2026-10-09T08:14:04.344Z" }, + { url = "https://files.pythonhosted.org/packages/f5/2d/7e5c722fa5d5d9f3b75e62fe11694b34217664d4f05ac88031197166b277/pyarrow-26.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:01c863a18bd9c8412453dd0d92de6d0ee7b2b3d6fb079d9734a4b2a3c8bd4453", size = 50867603, upload-time = "2026-10-09T08:14:09.115Z" }, + { url = "https://files.pythonhosted.org/packages/88/e4/9cd356d906e71bd79b0c3fc5c9a54e01a0020dcf14c152ccfbcb503c7298/pyarrow-26.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:6a628922ba20705fa964ca73e4ef959c2fb2f14b9bbec5589a6a1e68e6257c85", size = 53931932, upload-time = "2026-10-09T08:14:24.051Z" }, + { url = "https://files.pythonhosted.org/packages/bb/e4/5bae3133b7fe04c24907a20f3bc1fba388cbbde659199e7b76445982047a/pyarrow-26.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:954d971b363b16ee41f89389a4053315dc71265f2ce5c2468eb0a910b1166268", size = 54444720, upload-time = "2026-10-09T08:14:31.214Z" }, + { url = "https://files.pythonhosted.org/packages/ba/b4/ee422493bb6dafdbef776cfe2c2a73106a1063a79bf4e78d1e5f51176885/pyarrow-26.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5d5768d03426abe6526d5274adefa00abf00a7f81118c46e98b5a46390f5549e", size = 57388949, upload-time = "2026-10-09T08:14:38.964Z" }, + { url = "https://files.pythonhosted.org/packages/54/3c/1783aab1dac28e175dcf26dfc7123725efc474caecaed91e8a34cb89cad0/pyarrow-26.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:cc903e1069e9dd5e9dcf780324c0112e27e051e422ecfaff574fb33ed65d9160", size = 28567581, upload-time = "2026-10-09T08:14:44.279Z" }, +] + +[[package]] +name = "pycparser" +version = "3.12" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cb/27/eeb9f307bf07060f3d9f9aa0fb8c15b01bdfded0a941e4b2719c7f94cacd/pycparser-3.12.tar.gz", hash = "sha256:f03387877eab82ce01bbdfd0ba7252ee55a90b1c708d5129e2ce915475cf1db6", size = 158785, upload-time = "2026-10-22T13:40:03.56Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b6/ae/315bbeee126a769cbde43cf4908acb4b40408d8bf8bbf334dd7368d8dadf/pycparser-3.12-py3-none-any.whl", hash = "sha256:a076f3f489b39b43ed13563a1dafe42447445bec2e294154ec5c57fcdda4c85e", size = 62600, upload-time = "2026-10-22T13:40:02.021Z" }, +] + +[[package]] +name = "pydantic" +version = "2.14.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-types" }, + { name = "pydantic-core" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7c/0b/8e10b2e693af8ec54346a14caf36221334775975a747c9ceaa3f8371d96d/pydantic-2.14.1.tar.gz", hash = "sha256:94f478203dd03404682a1ada216965651dd74b1d2d5ffd62e00e0837caab5c26", size = 900362, upload-time = "2026-10-11T18:37:55.396Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/35/ea/a56b9fe5066f3537b7882f77e9c5dfb26d8c2eefdaed9b5fc73d57b4dc22/pydantic-2.14.1-py3-none-any.whl", hash = "sha256:9195d967ec791692a04438115466764fb8b9a27b31f14a760437694f40d6b454", size = 488106, upload-time = "2026-10-11T18:37:53.437Z" }, +] + +[[package]] +name = "pydantic-core" +version = "2.50.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a6/24/af4ec4be49fbc810f35b0bdcacc3d433b56bbfa469fd3bfd4ae116cd9bf1/pydantic_core-2.50.1.tar.gz", hash = "sha256:e50d7b94baac6c7d09927fa5ca5800a0c7ee5015c7fcff65beb3a1931b5a6e09", size = 513515, upload-time = "2026-10-11T18:35:44.82Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ee/94/101b49a6c63f99586755690696eae14532772587d9e39e93f7b8d28a49da/pydantic_core-2.50.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:704075d10b74f2f3c6e15407c696d88701df35fc8953f434a431add0d0074db0", size = 2222428, upload-time = "2026-10-11T18:32:20.228Z" }, + { url = "https://files.pythonhosted.org/packages/db/a3/f8b1b39349c33037c73d7a1b794c9c29d8f07d5bd2b74399c1375a529cc6/pydantic_core-2.50.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e8e1d6ce820aa23317e8209a86bd65a540973c12dc7552b48a4f6c8e9926815e", size = 2025779, upload-time = "2026-10-11T18:32:21.723Z" }, + { url = "https://files.pythonhosted.org/packages/d2/2d/862a7d115305cc621358e51ba13f296e0ffdf4fbd984be9f63bc7862730b/pydantic_core-2.50.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c18db21573bd2c6489f9a544b7499f0df2853958c568e5e783536ee1f690af41", size = 2048429, upload-time = "2026-10-11T18:32:23.3Z" }, + { url = "https://files.pythonhosted.org/packages/c0/7d/fed95d18bf2b5abe2d5ab2bbec4184bfc22b6805994a33ccdae8edffd25a/pydantic_core-2.50.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:cb57f304525a5e3c13333b772bf9a473f36326e9c821b2e8e1b2fd36f80ae2c3", size = 2173274, upload-time = "2026-10-11T18:32:24.697Z" }, + { url = "https://files.pythonhosted.org/packages/93/1e/c6f22ceac65dc3d291ff79bea0eb74cbd1de8b128f01fee48af03bcea1e1/pydantic_core-2.50.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a27c09d86600f1bf2fe3f37e1ae697faf3143931c09322cd799da94deee923b5", size = 2360403, upload-time = "2026-10-11T18:32:26.281Z" }, + { url = "https://files.pythonhosted.org/packages/2c/97/190b5b6bfe4c72f265feb657913eb30749193927a5ad7b83cfa7866a25a2/pydantic_core-2.50.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:46b3301d3b5c886f77de7546e47274a5842c622ea2020b8c6524c6b66913b4a6", size = 2439799, upload-time = "2026-10-11T18:32:27.844Z" }, + { url = "https://files.pythonhosted.org/packages/90/24/48cd98388e4ac2c08d0800af36d15413db15e20387f1038123a2c86cb2f6/pydantic_core-2.50.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:93ba4e9d8210d941c200431a56b2c0400b131865947903937ed3ec5404307d2e", size = 2160157, upload-time = "2026-10-11T18:32:29.553Z" }, + { url = "https://files.pythonhosted.org/packages/00/86/d909133d0b9979ed14c20f110f21124a0e346caf071bd66ac5af3009062b/pydantic_core-2.50.1-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:e5faeaee74a57d32b3ab3aebad2e348f06d3ba946fc5d28c1728455f00a3d13a", size = 2229523, upload-time = "2026-10-11T18:32:31.027Z" }, + { url = "https://files.pythonhosted.org/packages/90/31/4b43f08131b2098e73a852fd932063385f3103b33839fd9d1abdb89e673d/pydantic_core-2.50.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a3cda0e538208e5d722bbf3698b24f19c0a7d05bc8d5f8a7f9b121ea7fa243d9", size = 2242700, upload-time = "2026-10-11T18:32:32.622Z" }, + { url = "https://files.pythonhosted.org/packages/88/77/de39658569dc7a4c293d110a89629f61145e7538f00d1e4280c28debefd3/pydantic_core-2.50.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:57f51b31ff826e2859120cf4737c5a758a48d96f3e97da40ccee1796d58078ff", size = 2278452, upload-time = "2026-10-11T18:32:34.165Z" }, + { url = "https://files.pythonhosted.org/packages/46/bd/1eeb9e4e791e81210ef9096f58080b32ba15b5b13c108d50aaa3f8ce9f73/pydantic_core-2.50.1-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:8daa7ee75245d43ad7d747e5c9ecc1b1d06552f72b14887e9276f787d57375f4", size = 2450098, upload-time = "2026-10-11T18:32:35.547Z" }, + { url = "https://files.pythonhosted.org/packages/1c/e6/b2accfdff17e5a88d6358536702146de00f9339adaa4d5bbe1ee93cb4832/pydantic_core-2.50.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:acbf31f37c53a5ac0c34706c80b4f5107ba20b05fdd3816124bf236ef0c57dd2", size = 2476839, upload-time = "2026-10-11T18:32:37.119Z" }, + { url = "https://files.pythonhosted.org/packages/3b/7e/c8a83eb5d2cd42c2e13507150fb777b1edea5a018adc9cbf2858a1d08285/pydantic_core-2.50.1-cp312-cp312-win32.whl", hash = "sha256:45b11cac094aa25725581d9304eee93c9028516b9ea80dd9e175e13a5a2c840e", size = 2101425, upload-time = "2026-10-11T18:32:38.593Z" }, + { url = "https://files.pythonhosted.org/packages/c5/17/8bbd530b8659e9d963f5b16f6cb8e159f4cd174b4d6304e48969ab12dba8/pydantic_core-2.50.1-cp312-cp312-win_amd64.whl", hash = "sha256:132529c83901437ff642f585216831bf5fd7a91df66829907e155192ead62498", size = 2172522, upload-time = "2026-10-11T18:32:40.112Z" }, + { url = "https://files.pythonhosted.org/packages/af/9f/a79d690b127e4e3cff143d031538ab9aa715f6b2fbeecbd07cd6f2bf9fb7/pydantic_core-2.50.1-cp312-cp312-win_arm64.whl", hash = "sha256:4e834f6a8e4ff772dcc34f58ef5504147a3ea5b0f4eeb13b0f8eb2ca75ac57f1", size = 2161366, upload-time = "2026-10-11T18:32:41.606Z" }, + { url = "https://files.pythonhosted.org/packages/a5/09/1bcf160f3cd6333e2a76982fae5963e5032de0bcaaba38464223dc266bfe/pydantic_core-2.50.1-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:76e2e83fa6ec8cdc972d438dafc2522b3a47bee4ec0ae668b29cfb1977ab5242", size = 2217070, upload-time = "2026-10-11T18:35:15.688Z" }, + { url = "https://files.pythonhosted.org/packages/3d/81/f02de95eec7794dfc7a5bc69a834f0e834b31b3009014ce6de903855f775/pydantic_core-2.50.1-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:a51eee75939cf811ac09b278745a6cee7dc873ccfbc8b9af3cc88fe4b7ce25b5", size = 2039027, upload-time = "2026-10-11T18:35:18.07Z" }, + { url = "https://files.pythonhosted.org/packages/4d/16/227746ed3771d9bf2304568b55134849ece34e13b38b1d76de763015c77f/pydantic_core-2.50.1-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ae28183297fb0d2b8dc46a1f01d51f5e45825fc5afe76a835a6cb7fb34821295", size = 2109009, upload-time = "2026-10-11T18:35:20.218Z" }, + { url = "https://files.pythonhosted.org/packages/e5/b9/7664d1592a0e5a74903f357cd5b28ceb7336f3473256c5a4e8c8432f1881/pydantic_core-2.50.1-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:88e492e8b9d0312e7dc13667c30222abf284dc3b79b5302b3607b41a5784ce61", size = 2269883, upload-time = "2026-10-11T18:35:22.48Z" }, +] + +[[package]] +name = "pygments" +version = "2.21.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/49/2e/ced460408999b33da6b31b0021b0f37d329e202d4169aeb164493778f25b/pygments-2.21.0.tar.gz", hash = "sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c", size = 5005329, upload-time = "2026-08-17T08:02:48.824Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/46/17f022dd3e953bf20a04a028a21ec746d942f8d2af30fa0f124fa0e6a684/pygments-2.21.0-py3-none-any.whl", hash = "sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9", size = 1250147, upload-time = "2026-08-17T08:02:44.912Z" }, +] + +[[package]] +name = "pyjwt" +version = "2.15.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/43/ea/5194e52748b0da83d71e082d75496eaec6e58f419f5e184786ded517e6a9/pyjwt-2.15.1.tar.gz", hash = "sha256:4f259e80cdfb6b3fc18a7de51fd1ef9ec79652f25019bae68975ca2468a34df8", size = 121252, upload-time = "2026-09-28T18:40:42.598Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/50/ca/44de4e75f8aadc457f0634be3b542815078ded46dca30efb960edeecad6e/pyjwt-2.15.1-py3-none-any.whl", hash = "sha256:42d59d631f7768a1028a64c7ff581a9bf7519804daf91fc5b6c56e30eec5e193", size = 33860, upload-time = "2026-09-28T18:40:41.429Z" }, +] + +[[package]] +name = "pyopenssl" +version = "26.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cryptography" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3f/e8/7325d258199b159eb2c03fe32107533e2832e70e63f4fb88a6aa00023201/pyopenssl-26.4.0.tar.gz", hash = "sha256:28dfcce0162b9211413e26dfbfdf1d24317fbeba18fc93c12400a1856b2a0bc7", size = 182046, upload-time = "2026-08-01T19:50:50.512Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/51/ad/2cf6d3fa2fae5c79e1ed9960c0d42badd0f94d81dd12b50604cdc839e648/pyopenssl-26.4.0-py3-none-any.whl", hash = "sha256:f0eb0cb2d581d3ad2b9c489468485e7f2ab6727d08401bcf9d824c3caddf3c1c", size = 56026, upload-time = "2026-08-01T19:50:48.94Z" }, +] + +[[package]] +name = "pytest" +version = "9.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e4/47/b9efed96c114afcfa3c9d3fe98a76a1d14c74a9e266d397cf6eb64be5e01/pytest-9.1.1.tar.gz", hash = "sha256:1088fbde8f2b49d95a549a195707afa7a76a3ce9bcadc26b6d71f0ffda5fe313", size = 1636369, upload-time = "2026-06-19T10:58:32.857Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/24/25/1de2678b631f5a49215c6c96fff41ba892b0a34df68d6d80292b1b48aa7f/pytest-9.1.1-py3-none-any.whl", hash = "sha256:37a86b45efb9a47a61a36449063e8e18d0cab3161329fc099eb21783169c4f0c", size = 386536, upload-time = "2026-06-19T10:58:31.347Z" }, +] + +[[package]] +name = "python-dateutil" +version = "2.9.0.post0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, +] + +[[package]] +name = "python-slugify" +version = "9.1.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "text-unidecode" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/bd/e8/26b1af09d728d604dc16427a39f53985b22a170dbd61addac3f48db73f03/python_slugify-9.1.3.tar.gz", hash = "sha256:90e997f2e0987239ce95e12f700086eb18e1d1d3ee22624fbbdbd095afca42b6", size = 66093, upload-time = "2026-10-07T01:06:20.101Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/47/67/5598c98725332a6a4084c7b622d5a1ebe7d6745c1d2f434c9e4e89e04cba/python_slugify-9.1.3-py3-none-any.whl", hash = "sha256:9aced0670e54c5603e2335c0fa9a4011ad4bc41ecb3c46e8097294c46fd60ed4", size = 16032, upload-time = "2026-10-07T01:06:18.891Z" }, +] + +[[package]] +name = "pytimeparse" +version = "1.1.10" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/92/bf/87bd96fd60abf241cae10e6f7fc3882c8a81ebdbe76f49f9c3116b5b38ab/pytimeparse-1.1.10.tar.gz", hash = "sha256:ecf0ee613a72cb0b1d3d5404bd398a64266576fc36ccf3149178af6f2ec6f736", size = 11613, upload-time = "2026-10-17T12:38:58.437Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b1/e4/507b557e0f2a8dcddb14bb951bcf2ac04afc229e73c7256ccbc3a930ca58/pytimeparse-1.1.10-py3-none-any.whl", hash = "sha256:a9f9117a02eaa32efd268e70af29bc3ec1e84f473d3edef6d9809b36793b1c5d", size = 10661, upload-time = "2026-10-17T12:38:57.254Z" }, +] + +[[package]] +name = "pytz" +version = "2026.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/14/21/d83d6ef28c4c912c4bb4d1dcf591f7b8c6bde87b9c66f9f454677314e16d/pytz-2026.5.tar.gz", hash = "sha256:fa23724b9c486543b9ff54a327ee7569ac83ade54bb9afd0fc18676620401c86", size = 318572, upload-time = "2026-10-04T02:37:58.719Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4f/ef/c66110d46fb800dda0bf33164182dfadabe26a90e4476844d502a23dca8e/pytz-2026.5-py2.py3-none-any.whl", hash = "sha256:e658af3757f9e26a9d25dd2aff38335acd92bc9104f890a894b2c1ba28311b03", size = 506342, upload-time = "2026-10-04T02:37:56.814Z" }, +] + +[[package]] +name = "pywin32-ctypes" +version = "0.2.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/85/9f/01a1a99704853cb63f253eea009390c88e7131c67e66a0a02099a8c917cb/pywin32-ctypes-0.2.3.tar.gz", hash = "sha256:d162dc04946d704503b2edc4d55f3dba5c1d539ead017afa00142c38b9885755", size = 29471, upload-time = "2024-08-14T10:15:34.626Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/de/3d/8161f7711c017e01ac9f008dfddd9410dff3674334c233bde66e7ba65bbf/pywin32_ctypes-0.2.3-py3-none-any.whl", hash = "sha256:8a1513379d709975552d202d942d9837758905c8d01eb82b8bcc30918929e7b8", size = 30756, upload-time = "2024-08-14T10:15:33.187Z" }, +] + +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, + { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, + { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, +] + +[[package]] +name = "referencing" +version = "0.37.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "attrs" }, + { name = "rpds-py" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/22/f5/df4e9027acead3ecc63e50fe1e36aca1523e1719559c499951bb4b53188f/referencing-0.37.0.tar.gz", hash = "sha256:44aefc3142c5b842538163acb373e24cce6632bd54bdb01b21ad5863489f50d8", size = 78036, upload-time = "2025-10-13T15:30:48.871Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2c/58/ca301544e1fa93ed4f80d724bf5b194f6e4b945841c5bfd555878eea9fcb/referencing-0.37.0-py3-none-any.whl", hash = "sha256:381329a9f99628c9069361716891d34ad94af76e461dcb0335825aecc7692231", size = 26766, upload-time = "2025-10-13T15:30:47.625Z" }, +] + +[[package]] +name = "requests" +version = "2.34.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "idna" }, + { name = "urllib3" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ac/c3/e2a2b89f2d3e2179abd6d00ebd70bff6273f37fb3e0cc209f48b39d00cbf/requests-2.34.2.tar.gz", hash = "sha256:f288924cae4e29463698d6d60bc6a4da69c89185ad1e0bcc4104f584e960b9ed", size = 142856, upload-time = "2026-05-14T19:25:27.735Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, +] + +[[package]] +name = "rpds-py" +version = "2026.9.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/42/68/3bd46b8a5e01d3c2ebdf9c5e9497912e3fe0cde02bac21a7130ca866e403/rpds_py-2026.9.1.tar.gz", hash = "sha256:4793ef7f78268b124b73fa933440f01d258bbae01de9fa53e9080c9ab0425a12", size = 63948, upload-time = "2026-10-04T16:32:36.469Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5d/34/a828586ea3329fbb50895b50e9cf98ca3d924a9f41a0b26d443bff1b3794/rpds_py-2026.9.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:50906f5aea24b5a865cbd0a589698288631d9f3a54c3a937c83aefa95a0d14af", size = 346822, upload-time = "2026-10-04T16:29:16.258Z" }, + { url = "https://files.pythonhosted.org/packages/90/81/ac6a0d064982251856ce009c9e1dd51a34110b3c055aa7d1aad18b4899a3/rpds_py-2026.9.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e21c1429e205828ea886a2293a4a2c8e01f4c25d9893ca330e97a6cf73f52e7b", size = 341380, upload-time = "2026-10-04T16:29:17.706Z" }, + { url = "https://files.pythonhosted.org/packages/42/ff/bf7d54f362748fd6a49277b9d6531c394074110fedf50ad791ec59133fbb/rpds_py-2026.9.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2693b2728bbcc48d09a981a356954b0c47c53ff25b545856f28a889ea619f69a", size = 373053, upload-time = "2026-10-04T16:29:19.063Z" }, + { url = "https://files.pythonhosted.org/packages/60/de/74b0ccdbbd28687b9b5fdb34c1cabed88352182facaced9d6f5486b8b9ed/rpds_py-2026.9.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8601470267d938bcb7f3ab1a336100af51a4fd5b6ed030ef52461bb3ef5e7e07", size = 377742, upload-time = "2026-10-04T16:29:20.543Z" }, + { url = "https://files.pythonhosted.org/packages/c2/6d/b979775a3057b2a5c26d75ecbff60a20a24ef081db6dd76836fca3f20247/rpds_py-2026.9.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3890a6aa36e6baa53d5258a2a25d3ef8b37ad165a6ab27a892d7c3e3a432cd69", size = 487289, upload-time = "2026-10-04T16:29:21.958Z" }, + { url = "https://files.pythonhosted.org/packages/1d/5d/7c34734ce3ece943d9d6ee0122a74a21bbc75b47acb5b7053c83f6efb1ed/rpds_py-2026.9.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6b5b393eda5ea42cca1c1a6665f2a4882b4fd5d1777e41ce0545a107fb008c9d", size = 390710, upload-time = "2026-10-04T16:29:23.378Z" }, + { url = "https://files.pythonhosted.org/packages/1d/6d/b26eb1e75395925b3a142ed351cbed2ec8212f5c9d937aee3df32c701baa/rpds_py-2026.9.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:addeda51556dac7c1a2f14cda62db8b621cd12afba3091d03a96c72932387eab", size = 371261, upload-time = "2026-10-04T16:29:25.1Z" }, + { url = "https://files.pythonhosted.org/packages/7c/98/b2fdfe10301a9e27af21e634337fbba7baac0bc17fe44619a17c26bba2cb/rpds_py-2026.9.1-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:d9edf30457d74eebfd76b045535e36f1cd89062566a128a0db2145ca042d787e", size = 376075, upload-time = "2026-10-04T16:29:26.748Z" }, + { url = "https://files.pythonhosted.org/packages/c5/b1/c4b8d954e49c3c69cf8063c0c9e0919c99f3d2cecc46311606a1cbe835dc/rpds_py-2026.9.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:815d26356930846a40c7bc1366e7b1b0320ab8a063e66c11298a208bed0fd237", size = 399358, upload-time = "2026-10-04T16:29:28.282Z" }, + { url = "https://files.pythonhosted.org/packages/b3/59/9559a7293c97dff0cbb293efffbd36644103883ecc27741af8ef90c84ca8/rpds_py-2026.9.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:3b5a6f40f0a1486b4b36c888123afc67acdbd9f33235927acf5ff295429a0ba3", size = 550267, upload-time = "2026-10-04T16:29:29.851Z" }, + { url = "https://files.pythonhosted.org/packages/6c/9d/6dc60e49511de4c8b00e74f9c1d43aed4d27f712cdb1ade92f2c199cbf64/rpds_py-2026.9.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:b5b8b0753718d258fd454283fbd57e14545d3b40583fa672e27cb4f987626bcc", size = 614018, upload-time = "2026-10-04T16:29:31.54Z" }, + { url = "https://files.pythonhosted.org/packages/fb/bc/92a4ecf27301b886232a413360f6b348a81d55ffef654f2ebda5970ebc3d/rpds_py-2026.9.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:46d80bc76b51a6c24f9944368c28d38b8bcbcea1da4f2f8d3ebc31a67e8c6ec6", size = 578509, upload-time = "2026-10-04T16:29:33.01Z" }, + { url = "https://files.pythonhosted.org/packages/08/53/efb97f2e6589b7ab8394385a73870e0b2ab2a195281c117b21fd6d7d4855/rpds_py-2026.9.1-cp312-cp312-win32.whl", hash = "sha256:befc2d6a953e563f8a7bfd87a42c22ebf8a3e980dcb7b6a4d17b70b0e914e8a3", size = 205401, upload-time = "2026-10-04T16:29:34.451Z" }, + { url = "https://files.pythonhosted.org/packages/1c/84/1700cc748d0eaa747486e4e82882d2575224e6e9a3148df583058850c629/rpds_py-2026.9.1-cp312-cp312-win_amd64.whl", hash = "sha256:5ce8943f79c2210f7abcc28e86367b03b28d95027fd01c46d2472373ae70c86f", size = 222923, upload-time = "2026-10-04T16:29:35.709Z" }, + { url = "https://files.pythonhosted.org/packages/c3/89/0302215373c2f8b4408b0fdfee955368cc378a98ff59508d47a7e0dc2dbf/rpds_py-2026.9.1-cp312-cp312-win_arm64.whl", hash = "sha256:501909f2e4a1e2dee528ef766fe3c469060ebc17e54a8383d404ba07a81a6f02", size = 218415, upload-time = "2026-10-04T16:29:37.263Z" }, + { url = "https://files.pythonhosted.org/packages/a3/90/a9ba81408369d34c1aca73319c15adc91ed45c1d21e9e8ee4832ffb7fe0b/rpds_py-2026.9.1-pp312-pypy312_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6cdc537c8633d7fd92a82e2e0d2ab74320a3f63d5e59fb9cf08711e08fe151c4", size = 381109, upload-time = "2026-10-04T16:32:08.994Z" }, + { url = "https://files.pythonhosted.org/packages/b3/44/5192a0bed94cec86bd2a5e1cc5a1bb8f7eec3aec907eacd2deb13e87e326/rpds_py-2026.9.1-pp312-pypy312_pp73-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:10e208f2425d973938afcd56e28a7c4be32e27b6a60b5d381f49fb9d8acf9759", size = 388222, upload-time = "2026-10-04T16:32:11.321Z" }, + { url = "https://files.pythonhosted.org/packages/92/cd/5356549711448f18b52f7a11ffbe90f1228774996fbf3c6cd1b18d22abaf/rpds_py-2026.9.1-pp312-pypy312_pp73-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:6c0dbbcc19735fe5f8b0a54c07659d154a9e69f47e15d0a6ab7299215daf62cb", size = 500384, upload-time = "2026-10-04T16:32:13.478Z" }, + { url = "https://files.pythonhosted.org/packages/7a/88/ddda9d28adfe33119c75b2e733d4ba7e326f41d7e1b1093951771768ca48/rpds_py-2026.9.1-pp312-pypy312_pp73-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:684fd492fff4fead00587544e059be2bbcb6f93454f21fa2a91b66fc7508be82", size = 401661, upload-time = "2026-10-04T16:32:15.653Z" }, + { url = "https://files.pythonhosted.org/packages/32/c3/bb59ba16a6b4a57995d15b8c04c00f28df3b938c4dcdde48fbe0f73edb0c/rpds_py-2026.9.1-pp312-pypy312_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d1028417bb44037eb3069c1009bd7b7277212876cda22fbe565b0bca9fab6d2c", size = 380748, upload-time = "2026-10-04T16:32:17.727Z" }, + { url = "https://files.pythonhosted.org/packages/f5/b8/0580faa1c5a32ddc160130dd7ae54d2d007272a89a40a6b5db3e50f9f585/rpds_py-2026.9.1-pp312-pypy312_pp73-manylinux_2_31_riscv64.whl", hash = "sha256:492e5e428cbe126221611f47e068f01660352feec4ad18bc0f5ea9b2ae88fb14", size = 385731, upload-time = "2026-10-04T16:32:19.855Z" }, + { url = "https://files.pythonhosted.org/packages/bf/70/f73564642bbe3322c7eeef2c2f258cf040b71c41a430b532e59d04a1b838/rpds_py-2026.9.1-pp312-pypy312_pp73-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:88b5268892fde430d5531f95bc560b6efbbd67c929662c586afd729a96e7461c", size = 412131, upload-time = "2026-10-04T16:32:22.285Z" }, + { url = "https://files.pythonhosted.org/packages/8c/5d/17fff2e1f8f68721a2afb5cb48f7e442c751bb6b4883157c5abaf618cd08/rpds_py-2026.9.1-pp312-pypy312_pp73-musllinux_1_2_aarch64.whl", hash = "sha256:01445c8d194aa032a08e944f16567672da1c62dbdbefd8b6d0693032e290cf68", size = 558193, upload-time = "2026-10-04T16:32:24.4Z" }, + { url = "https://files.pythonhosted.org/packages/2a/d0/869ab6fb08531f97aae4a780b4d61b4190d8aef48fe054ee28bcf7114466/rpds_py-2026.9.1-pp312-pypy312_pp73-musllinux_1_2_i686.whl", hash = "sha256:eef6a03b0b6d08d0835ccfa8ec8d1bc70525e3801387567137b50c557695e6da", size = 622448, upload-time = "2026-10-04T16:32:26.844Z" }, + { url = "https://files.pythonhosted.org/packages/ba/a5/9672e532fe02cd3b92c78cfd8743abca939d96853006c63835009a1ecb6f/rpds_py-2026.9.1-pp312-pypy312_pp73-musllinux_1_2_x86_64.whl", hash = "sha256:6b9bf3135b4ad5981df9a73d71a35272d650a2985ae9c2746357b24d59de2448", size = 591126, upload-time = "2026-10-04T16:32:29.274Z" }, + { url = "https://files.pythonhosted.org/packages/80/ef/3a9f8e4279c7920af561deed4cb7ebebed2794a8f608cf404d5eef14a105/rpds_py-2026.9.1-pp312-pypy312_pp80-macosx_10_12_x86_64.whl", hash = "sha256:56c6952a9b15047466d0c2347c446a761d4527f89976156341e68f0ce5cc08b0", size = 357108, upload-time = "2026-10-04T16:32:31.641Z" }, + { url = "https://files.pythonhosted.org/packages/b7/55/4b2fa381a583760aea5c92841e4e928a358e1c6511b129219e9c2826a226/rpds_py-2026.9.1-pp312-pypy312_pp80-macosx_11_0_arm64.whl", hash = "sha256:b242c27c8f836305a4a72df9cdd564386ac57b807bd252a063223331c9316b37", size = 346877, upload-time = "2026-10-04T16:32:34.061Z" }, +] + +[[package]] +name = "s3transfer" +version = "0.19.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "botocore" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/49/c5/0a5ce655109cb96b8dfad36908a4e108d15f5dc2eff8fbc3146c4312e704/s3transfer-0.19.3.tar.gz", hash = "sha256:8326acc3d056d7f7b8a37110c7d2aea41f67e83f6ae79cd3b5fbcca34c726406", size = 166087, upload-time = "2026-10-19T19:19:49.751Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e2/a4/88f9e10498759121060b8050a1554753dfc04df39bbbcc7aad9ecc8d9440/s3transfer-0.19.3-py3-none-any.whl", hash = "sha256:24eb7535804306355e31c952a90022bf5b2f9d760f9a11062c7493a217c08553", size = 90252, upload-time = "2026-10-19T19:19:48.309Z" }, +] + +[[package]] +name = "secretstorage" +version = "3.5.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cryptography", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, + { name = "jeepney", marker = "sys_platform != 'emscripten' and sys_platform != 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/1c/03/e834bcd866f2f8a49a85eaff47340affa3bfa391ee9912a952a1faa68c7b/secretstorage-3.5.0.tar.gz", hash = "sha256:f04b8e4689cbce351744d5537bf6b1329c6fc68f91fa666f60a380edddcd11be", size = 19884, upload-time = "2025-11-23T19:02:53.191Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/46/f5af3402b579fd5e11573ce652019a67074317e18c1935cc0b4ba9b35552/secretstorage-3.5.0-py3-none-any.whl", hash = "sha256:0ce65888c0725fcb2c5bc0fdb8e5438eece02c523557ea40ce0703c266248137", size = 15554, upload-time = "2025-11-23T19:02:51.545Z" }, +] + +[[package]] +name = "six" +version = "1.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, +] + +[[package]] +name = "slim-ci-lab" +version = "0.1.0" +source = { editable = "." } +dependencies = [ + { name = "dbt-core" }, + { name = "dbt-snowflake" }, + { name = "pandas" }, + { name = "pyarrow" }, + { name = "requests" }, +] + +[package.dev-dependencies] +dev = [ + { name = "pytest" }, +] + +[package.metadata] +requires-dist = [ + { name = "dbt-core", specifier = ">=1.10,<1.11" }, + { name = "dbt-snowflake", specifier = ">=1.10,<1.11" }, + { name = "pandas", specifier = ">=2.2" }, + { name = "pyarrow", specifier = ">=17" }, + { name = "requests", specifier = ">=2.32" }, +] + +[package.metadata.requires-dev] +dev = [{ name = "pytest", specifier = ">=8.3" }] + +[[package]] +name = "snowflake-connector-python" +version = "4.8.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "asn1crypto" }, + { name = "boto3" }, + { name = "botocore" }, + { name = "certifi" }, + { name = "charset-normalizer" }, + { name = "cryptography" }, + { name = "filelock" }, + { name = "idna" }, + { name = "packaging" }, + { name = "platformdirs" }, + { name = "pyjwt" }, + { name = "pyopenssl" }, + { name = "pytz" }, + { name = "requests" }, + { name = "sortedcontainers" }, + { name = "tomlkit" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/c2/9e/5e54a9ab5a9e02caa777750e0054b14cb5007be75505815084f97a067ab1/snowflake_connector_python-4.8.0.tar.gz", hash = "sha256:1bf40aa8684eb0769fedeb53a702b274472571812835d5a03fb9dcf9254333b9", size = 992670, upload-time = "2026-09-30T16:31:09.818Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/51/50/1c4471bccde09971136884be0ea099a3206e320ed24b65b5c916126e316a/snowflake_connector_python-4.8.0-cp312-cp312-macosx_15_0_arm64.whl", hash = "sha256:2fde1216c4c1e4015ad7e39b7b39b6a44a29551f4d7a3dfc64b0905e66dc24ac", size = 1220915, upload-time = "2026-09-30T16:31:17.723Z" }, + { url = "https://files.pythonhosted.org/packages/6f/8a/21dff856ce456874a93816b9e07417f5c253ca0e9b5f65111bcc8164ffdc/snowflake_connector_python-4.8.0-cp312-cp312-macosx_15_0_x86_64.whl", hash = "sha256:c40a058f87dfbca1e65122932afb41c42cb9c3b7cb12826204527b41dd0f39ef", size = 1227694, upload-time = "2026-09-30T16:31:19.093Z" }, + { url = "https://files.pythonhosted.org/packages/9f/1a/e225f7c901550c4eb06430f6ac7b2c0847a885492e49ce97f2fc56ddc558/snowflake_connector_python-4.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f2a3cbcd3e6c4a449165f78bb331e1570ce6026cc21a2673313b4912d80745c0", size = 2953261, upload-time = "2026-09-30T16:30:57.611Z" }, + { url = "https://files.pythonhosted.org/packages/67/60/d77cfe97234be726c1635de98119373b9ebc6ca052f5cf9ce16b2bab77dc/snowflake_connector_python-4.8.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1c80df2fc5e16f3a49e6d16b1cdca0358308c1db2eae22d82aa7c038c80cc913", size = 2986741, upload-time = "2026-09-30T16:30:59.131Z" }, + { url = "https://files.pythonhosted.org/packages/8d/66/34f0d29c37c4bd25ca325ab4e8a11df75866abc4180aa0a0a044a0139bec/snowflake_connector_python-4.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:b9bddddc713aaa20dc5ff5635896f66e50c0fed88b6a905b088e727d046e361d", size = 5452920, upload-time = "2026-09-30T16:31:34.679Z" }, +] + +[package.optional-dependencies] +secure-local-storage = [ + { name = "keyring" }, +] + +[[package]] +name = "snowplow-tracker" +version = "1.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "requests" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ff/77/1ab6e5bafb9c80d8128f065a355377a04ac5b3c38eb719d920a9909d346e/snowplow_tracker-1.1.0.tar.gz", hash = "sha256:95d8fdc8bd542fd12a0b9a076852239cbaf0599eda8721deaf5f93f7138fe755", size = 34135, upload-time = "2025-02-21T10:58:48.112Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/78/10/1c76269cbf2d6e127f4415044d9ddb0295858230678bbf4bfba905593c82/snowplow_tracker-1.1.0-py3-none-any.whl", hash = "sha256:24ea32ddac9cca547421bf9ab162f5f33c00711c6ef118ad5f78093cee962224", size = 44128, upload-time = "2025-02-21T10:58:45.818Z" }, +] + +[[package]] +name = "sortedcontainers" +version = "2.4.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e8/c4/ba2f8066cceb6f23394729afe52f3bf7adec04bf9ed2c820b39e19299111/sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88", size = 30594, upload-time = "2021-05-16T22:03:42.897Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0", size = 29575, upload-time = "2021-05-16T22:03:41.177Z" }, +] + +[[package]] +name = "sqlparse" +version = "0.5.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/18/67/701f86b28d63b2086de47c942eccf8ca2208b3be69715a1119a4e384415a/sqlparse-0.5.4.tar.gz", hash = "sha256:4396a7d3cf1cd679c1be976cf3dc6e0a51d0111e87787e7a8d780e7d5a998f9e", size = 120112, upload-time = "2025-11-28T07:10:18.377Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/25/70/001ee337f7aa888fb2e3f5fd7592a6afc5283adb1ed44ce8df5764070f22/sqlparse-0.5.4-py3-none-any.whl", hash = "sha256:99a9f0314977b76d776a0fcb8554de91b9bb8a18560631d6bc48721d07023dcb", size = 45933, upload-time = "2025-11-28T07:10:19.73Z" }, +] + +[[package]] +name = "text-unidecode" +version = "1.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ab/e2/e9a00f0ccb71718418230718b3d900e71a5d16e701a3dae079a21e9cd8f8/text-unidecode-1.3.tar.gz", hash = "sha256:bad6603bb14d279193107714b288be206cac565dfa49aa5b105294dd5c4aab93", size = 76885, upload-time = "2019-08-30T21:36:45.405Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a6/a5/c0b6468d3824fe3fde30dbb5e1f687b291608f9473681bbf7dabbf5a87d7/text_unidecode-1.3-py2.py3-none-any.whl", hash = "sha256:1311f10e8b895935241623731c2ba64f4c455287888b18189350b67134a822e8", size = 78154, upload-time = "2019-08-30T21:37:03.543Z" }, +] + +[[package]] +name = "tomlkit" +version = "0.15.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/08/72/fc01563df95c189a9f65850096bdc205bc2fe37da6c740cd64afaf18d07e/tomlkit-0.15.2.tar.gz", hash = "sha256:83e32755ba1d20d9340bd069ee97e9f6232ba2346b63ffcba425de01f7e327a6", size = 190522, upload-time = "2026-10-16T04:07:08.166Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/da/46/7d84138a60a8bd796825b2d38ebe92d0d16501ca57d501b7c5d677401bbc/tomlkit-0.15.2-py3-none-any.whl", hash = "sha256:05562412163a3037b94497385fb7f9204c637bc605af7a48199e41cf2272f437", size = 52225, upload-time = "2026-10-16T04:07:09.216Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, +] + +[[package]] +name = "typing-inspection" +version = "0.4.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a3/26/b09b8010994eccc3c09092e6b34058f36a460eea2d4c3e8b910c695975a0/typing_inspection-0.4.4.tar.gz", hash = "sha256:547274fa6b0a561ccf549cc9524b999a578e737d015d8709d021f9d0d13bea47", size = 76928, upload-time = "2026-08-12T12:37:25.997Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/67/81/4add07e5172b7ac40d8ed5ff580409a7801a4fe26d529bdd915401dabfbe/typing_inspection-0.4.4-py3-none-any.whl", hash = "sha256:65b8397ba37ccbce054456aaccddfc91e6e3083c92824df348d96ca832f3f147", size = 14750, upload-time = "2026-08-12T12:37:24.648Z" }, +] + +[[package]] +name = "tzdata" +version = "2026.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d9/68/f1b440335057bfce71b6e50a9d09445aa2ecbd08359a337976627b8409e7/tzdata-2026.5.tar.gz", hash = "sha256:8cc73c0a0bfca7dbfa59235d60b2eff82231dee33f53d206db1acd9173cfc0a7", size = 200404, upload-time = "2026-10-03T09:23:14.143Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/94/21/1e5995a1c920cce14e4bffae20c665ec10e7ed03ab25e006cd741092b718/tzdata-2026.5-py2.py3-none-any.whl", hash = "sha256:b683bd1b6659ddcd810ff02ad09ba821d4bf1065072805063eb35c49617905ac", size = 347996, upload-time = "2026-10-03T09:23:12.535Z" }, +] + +[[package]] +name = "urllib3" +version = "2.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e3/05/b17359e1cefb4f909b5e40b1b90a496d987258916dbbf88e842c729f510e/urllib3-2.8.0.tar.gz", hash = "sha256:63bf2ead4c879426ebf22ef2a781eeb4aa3b4ae798a0435506f8687fd5bb9b63", size = 458972, upload-time = "2026-09-15T19:29:36.253Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/92/9d/c4e665119135114480843e7ab388fa94d8480650450e6f8e26b70d323a4c/urllib3-2.8.0-py3-none-any.whl", hash = "sha256:0cf3cae568d36aa9576b28dfb35f11328f1cb974ca7647d9475ebb86c75ac6e3", size = 135717, upload-time = "2026-09-15T19:29:34.577Z" }, +] + +[[package]] +name = "zipp" +version = "4.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/18/05/ba1458dee0ac485b2a01071c1b88f5199fe878a485df482e10e1ef2205cc/zipp-4.1.2.tar.gz", hash = "sha256:cf474b90b07ea932637e17c6b9479de2b2882900b0d311bdaf4f5c000a15a97f", size = 27864, upload-time = "2026-10-21T18:57:04.743Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b8/44/7bd2007260da11cccca29ba2d4c345f1287aa663db38ee7cefeabf6386a5/zipp-4.1.2-py3-none-any.whl", hash = "sha256:b30cb6cd253122169c47de245ca24a86bac2a5d3802ba15bf7ba6414eb9e144d", size = 10581, upload-time = "2026-10-21T18:57:03.79Z" }, +] diff --git a/tenantboard/.env.example b/tenantboard/.env.example new file mode 100644 index 0000000..59ccf90 --- /dev/null +++ b/tenantboard/.env.example @@ -0,0 +1,41 @@ +# Every variable Tenantboard reads, with its default. Nothing here is required +# for the emulator path except the LocalStack token. The Makefile does not load +# this file: export what you change (or `set -a; . ./.env; set +a`). + +# --- Emulator -------------------------------------------------------------- +# LocalStack auth token. The Makefile also reads ~/.localstack/default.env. +# LOCALSTACK_AUTH_TOKEN= +SNOWFLAKE_IMAGE=localstack/snowflake-next:latest +SNOWFLAKE_PORT=4566 + +# --- Shared ---------------------------------------------------------------- +# EMULATOR (default) or SNOWFLAKE_CLOUD: picks the connection used by the API, +# the seed and demo scripts, and the tests. +TEST_TARGET=EMULATOR +# Key pair for SNOWFLAKE_JWT auth, used by both identities. `make up` generates a throwaway one. +SNOWFLAKE_PRIVATE_KEY_PATH=.secrets/rsa_key.p8 +SNOWFLAKE_WAREHOUSE=TB_WH + +# Admin identity: deploys, provisions tenant roles, seeds. Never used by the API. +SNOWFLAKE_USER=SAMPLE_SVC +# Emulator default ACCOUNTADMIN; required on real Snowflake (needs CREATE DATABASE, +# CREATE ROLE and MANAGE GRANTS, e.g. a role holding SYSADMIN and SECURITYADMIN). +SNOWFLAKE_ROLE= + +# App identity: the API's service user. Each request runs under a tenant role. +TB_APP_USER=TENANTBOARD_SVC +TB_DATABASE=TENANTBOARD +TB_SCHEMA=APP +# Open per-role Snowflake sessions kept by the API (least recently used are closed). +TB_MAX_ROLE_CONNECTIONS=16 +TB_MAX_UPLOAD_BYTES=1000000 +TB_CORS_ORIGINS=["http://localhost:5173","http://127.0.0.1:5173"] + +# make run +API_PORT=8000 +WEB_PORT=5173 + +# --- Real Snowflake (TEST_TARGET=SNOWFLAKE_CLOUD) --------------------------- +# Read only on the cloud path. +SNOWFLAKE_ACCOUNT= +SNOWFLAKE_HOST= diff --git a/tenantboard/.gitignore b/tenantboard/.gitignore new file mode 100644 index 0000000..bc17946 --- /dev/null +++ b/tenantboard/.gitignore @@ -0,0 +1,13 @@ +# Throwaway key pair generated by `make up` +.secrets/ +.env + +# Python +.venv/ +__pycache__/ +.pytest_cache/ +.ruff_cache/ + +# Dashboard +web/node_modules/ +web/dist/ diff --git a/tenantboard/Makefile b/tenantboard/Makefile new file mode 100644 index 0000000..1317677 --- /dev/null +++ b/tenantboard/Makefile @@ -0,0 +1,77 @@ +# Tenantboard: a multi-tenant analytics API whose tenant isolation and PII +# masking are enforced by Snowflake policies. Run `make help` for the targets. + +-include $(HOME)/.localstack/default.env +export LOCALSTACK_AUTH_TOKEN +SNOWFLAKE_PORT ?= 4566 +export SNOWFLAKE_PORT +COMPOSE := docker compose -p snfs-tenantboard + +SNOWFLAKE_PRIVATE_KEY_PATH ?= $(CURDIR)/.secrets/rsa_key.p8 +export SNOWFLAKE_PRIVATE_KEY_PATH + +UV_RUN := uv run --frozen +# At most 4 workers: each one clones its own database from the template. +PYTEST_ARGS ?= -n 4 +API_PORT ?= 8000 +WEB_PORT ?= 5173 +export API_PORT WEB_PORT + +.DEFAULT_GOAL := help +.PHONY: help keys up seed test demo run web web-build down test-cloud logs install start stop + +help: ## List targets + @grep -hE '^[a-z-]+:.*## ' $(firstword $(MAKEFILE_LIST)) | awk -F':.*## ' '{printf " %-12s %s\n", $$1, $$2}' + +keys: .secrets/rsa_key.p8 ## Generate the throwaway key pair used for SNOWFLAKE_JWT auth + +.secrets/rsa_key.p8: + @mkdir -p .secrets && chmod 700 .secrets + @openssl genpkey -algorithm RSA -pkeyopt rsa_keygen_bits:2048 -out .secrets/rsa_key.p8 2>/dev/null + @openssl rsa -in .secrets/rsa_key.p8 -pubout -out .secrets/rsa_key.pub 2>/dev/null + @chmod 600 .secrets/rsa_key.p8 + @echo "Generated .secrets/rsa_key.p8 (throwaway, for local development)" + +up: keys ## Start the emulator, install the Python env, wait until SQL answers + $(COMPOSE) up -d + uv sync --frozen --quiet + ./scripts/wait-for-emulator.sh 180 + $(UV_RUN) python -m scripts.wait_for_init + +seed: ## Load the synthetic merchants, customers and orders into TENANTBOARD (idempotent) + $(UV_RUN) python -m scripts.seed + +test: ## Run the suite against the running emulator (each worker clones its own database) + $(UV_RUN) pytest $(PYTEST_ARGS) + +demo: ## Headline demo: a raw SELECT * FROM orders as tenant ACME, with and without the policy + $(UV_RUN) python -m scripts.demo + +web/node_modules: web/package-lock.json + cd web && npm ci --no-audit --no-fund + @touch $@ + +web: web/node_modules ## Install the dashboard's npm dependencies + +web-build: web ## Type-check and build the dashboard into web/dist + cd web && npm run build + +run: web ## Serve the API on :8000 and the dashboard on :5173 (Ctrl-C stops both) + ./scripts/run.sh + +down: ## Stop the emulator and remove its volumes + $(COMPOSE) down -v --remove-orphans + +test-cloud: ## Run the same suite against real Snowflake (billable; needs SNOWFLAKE_* env) + TEST_TARGET=SNOWFLAKE_CLOUD $(UV_RUN) pytest $(PYTEST_ARGS) + +logs: ## Tail the emulator log (init hook results included) + $(COMPOSE) logs --tail=200 snowflake + +# The targets the repository CI runs for every sample (see the root README) +install: ## Nothing to install up front: make up and make test fetch the dependencies + @echo "Nothing to install, make up and make test fetch the dependencies" + +start: up ## Same as make up + +stop: down ## Same as make down diff --git a/tenantboard/README.md b/tenantboard/README.md new file mode 100644 index 0000000..2516fb2 --- /dev/null +++ b/tenantboard/README.md @@ -0,0 +1,135 @@ +# Tenantboard + +A multi-tenant analytics API (FastAPI + React) for a B2B commerce platform, where tenant isolation and PII masking are enforced by Snowflake row access and masking policies, and a leak test proves it on every run. It is for backend engineers who embed analytics in a SaaS product and want to test RBAC and policies locally, without a Snowflake account. + +## Architecture + +```mermaid +flowchart LR + ui["React dashboard
(Vite, :5173)"] -- "/api/* + X-Tenant-Role" --> api["FastAPI
(:8000)"] + api --> cache["RoleSessionCache
one session per role"] + api --> repo["Repository
WHERE TENANT_ID = ?"] + cache -- "key-pair JWT, role TENANT_ACME / TENANT_ACME_ADMIN / PLATFORM_ADMIN" --> sf + repo -- "qmark binds, QUERY_TAG per endpoint" --> sf + subgraph sf["Snowflake: TENANTBOARD.APP"] + rap["Row access policy
TENANT_ISOLATION"] --> tables[("MERCHANTS, CUSTOMERS,
ORDERS, LINE_ITEMS")] + mask["Masking policy
EMAIL_MASK"] --> tables + end + tests["pytest -n 4"] -. "CLONE per worker" .-> sf +``` + +Isolation has two independent layers. The application binds the tenant in every query (`WHERE TENANT_ID = ?`), and each request runs in a Snowflake session under the tenant's own role, where the `TENANT_ISOLATION` policy shows only rows whose `TENANT_` role is in the session. A forgotten `WHERE` clause or a hand-written support query still cannot cross tenants. The test suite proves each layer on its own. + +## How it works + +1. **Roles are the tenant identity.** Provisioning (`app/provisioning.py`) creates `TENANT_` (viewer) and `TENANT__ADMIN` (inherits the viewer role and `TENANT_ADMIN`) per merchant and grants both to the API's service user. +2. **Policies are attached at creation.** Every tenant table is created `WITH ROW ACCESS POLICY TENANT_ISOLATION ON (TENANT_ID)` (`init/03_tables.sf.sql`), so no table ever exists without its policy. +3. **One session per role.** `RoleSessionCache` (`app/db.py`) opens an async session per role on first use, shares it between requests and closes the least recently used one beyond a bound. The API never switches roles inside a session. +4. **Masking follows the role.** The same SQL returns `***MASKED***` for viewers and the email for admins; the API has no masking code. +5. **Uploads.** An admin's CSV is validated in pandas, staged with `write_pandas` into a temporary table and applied with one tenant-bound `MERGE`. +6. **Errors** are RFC 9457 problem details that keep the Snowflake error code, SQL state and query id. + +```mermaid +sequenceDiagram + autonumber + participant B as Browser + participant A as FastAPI + participant P as Session PLATFORM_ADMIN + participant T as Session TENANT_ACME + participant S as Policies + B->>A: GET /api/tenants/acme/orders (X-Tenant-Role: viewer) + A->>P: SELECT ... FROM MERCHANTS WHERE SLUG = ? + P-->>A: merchant ACME (or 404 problem) + A->>T: SELECT ... FROM ORDERS WHERE TENANT_ID = ? LIMIT ? OFFSET ? + T->>S: apply TENANT_ISOLATION and EMAIL_MASK + S-->>T: ACME rows only, EMAIL masked + T-->>A: page + COUNT(*) OVER () + A-->>B: 200 OrderPage +``` + +## Prerequisites + +- Docker with Compose v2 +- [`uv`](https://docs.astral.sh/uv/) (installs Python 3.12 if needed) +- Node.js 22 with npm (for the dashboard) +- `openssl`, `make`, `curl` +- A LocalStack auth token, exported as `LOCALSTACK_AUTH_TOKEN` or stored in `~/.localstack/default.env` + +## Quickstart + +```sh +make up # dev key pair, emulator, Python env; waits until tables, policies and grants exist +make seed # 50 merchants: tenant roles, customers, orders, line items +make demo # the leak test, narrated, on a throwaway clone +make run # API on http://localhost:8000 (docs at /docs), dashboard on http://localhost:5173 +make test # the full suite, 4 workers, each on its own clone +make down # stop and remove everything +``` + +If port 4566 is taken, set `SNOWFLAKE_PORT=14566` (or any free port) for every target. + +With `make run` going, call the API directly. Requests run as the tenant's viewer role unless `X-Tenant-Role: admin` is sent: + +```sh +curl -s localhost:8000/api/tenants/acme/summary +# {"tenant_id":"ACME","role":"viewer","snowflake_role":"TENANT_ACME","orders":24,"revenue":"12227.22",...} + +curl -s "localhost:8000/api/tenants/acme/orders?page_size=1" +# {"items":[{"order_id":"ORD-000013","tenant_id":"ACME",...,"customer_email":"***MASKED***"}],"page":1,"page_size":1,"total":24} + +curl -s -H 'X-Tenant-Role: admin' "localhost:8000/api/tenants/acme/orders?page_size=1" +# {"items":[{"order_id":"ORD-000013",...,"customer_email":"tara.chen.11@acme-buyers.example"}],...} + +curl -s localhost:8000/api/tenants/nope/summary +# 404 {"type":"urn:tenantboard:problem:tenant-not-found","title":"Unknown tenant",...} +``` + +`make demo` runs a raw `SELECT * FROM ORDERS`, with no `WHERE` clause, under each role, then detaches the policy: + +```text +[2/5] SELECT * FROM ORDERS -- no WHERE clause, run under each role + as TENANT_ACME 24 rows from 1 tenant(s): ACME + as TENANT_GLOBEX 10 rows from 1 tenant(s): GLOBEX + as TENANT_ACME_ADMIN 24 rows from 1 tenant(s): ACME + as PLATFORM_ADMIN 698 rows from 50 tenant(s): ACME, AMBER_DEPOT, AMBER_STUDIO, ... +... +[4/5] ALTER TABLE ORDERS DROP ROW ACCESS POLICY TENANT_ISOLATION -- simulate a missing policy + as TENANT_ACME 698 rows from 50 tenant(s): ACME, AMBER_DEPOT, AMBER_STUDIO, ... <-- LEAK: every tenant's orders +``` + +In the dashboard, pick a merchant and toggle **Viewer** / **Admin**: the orders and totals change with the tenant, and `customer_email` switches between `***MASKED***` and the address. `X-Tenant-Role` stands in for authentication; in production the tenant and role come from verified token claims. + +## Tests + +`make test` runs `pytest -n 4`. The controller builds a template database from `init/*.sf.sql` and the seeded dataset; each worker clones it, runs its own API instance against the clone and drops it at the end. Expected values are recomputed from the seed in plain pandas (`tests/helpers.py`), never read back from the API. + +- **Leak test:** a raw `SELECT * FROM ORDERS` as a tenant role returns exactly that tenant's orders, and the same `SELECT` returns every tenant once the policy is detached. +- Every tenant table is filtered; the admin role inherits the isolation; `PLATFORM_ADMIN` sees all tenants; the app-layer filter isolates on its own. +- Email masking per role, summary, gap-filled daily revenue, top customers and pagination against the seed. +- CSV upload: inserts and updates only the caller's orders, is idempotent, and rolls back entirely on a `NOT NULL` violation. +- Error paths as problem details: unknown tenant (404), invalid input (422), viewer upload (403), unprovisioned tenant role (403, Snowflake `390189`). +- The per-role session cache: shared per role, bounded, never closes a leased session. + +To run the same suite on a real account, run the account-level part of each `init/*.sf.sql` (above the `-- @deploy:` marker) once, then set `SNOWFLAKE_ACCOUNT`, `SNOWFLAKE_USER`, `SNOWFLAKE_ROLE` (with `CREATE DATABASE`, `CREATE ROLE` and `MANAGE GRANTS`), `SNOWFLAKE_WAREHOUSE` and `SNOWFLAKE_PRIVATE_KEY_PATH`, and run `make test-cloud`. This consumes credits. + +## Project structure + +``` +tenantboard/ + Makefile, docker-compose.yml + init/ platform objects, policies, tables, grants (*.sf.sql) + app/ FastAPI app, session cache, repository, provisioning, uploads + scripts/ seed, demo, run and readiness helpers + web/ React + Vite + TypeScript dashboard + tests/ fixtures (template + clone per worker), oracle, test modules + docs/architecture.md data model, role model, test isolation, error catalog +``` + +## Learn more + +- [Row access policies](https://docs.snowflake.com/en/user-guide/security-row-intro) +- [Masking policies](https://docs.snowflake.com/en/user-guide/security-column-ddm-intro) +- [`IS_ROLE_IN_SESSION`](https://docs.snowflake.com/en/sql-reference/functions/is_role_in_session) +- [Cloning considerations](https://docs.snowflake.com/en/user-guide/object-clone) +- [Python connector and `write_pandas`](https://docs.snowflake.com/en/developer-guide/python-connector/python-connector-pandas) +- [Key-pair authentication](https://docs.snowflake.com/en/user-guide/key-pair-auth) diff --git a/tenantboard/app/__init__.py b/tenantboard/app/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tenantboard/app/db.py b/tenantboard/app/db.py new file mode 100644 index 0000000..6e4d435 --- /dev/null +++ b/tenantboard/app/db.py @@ -0,0 +1,197 @@ +"""Snowflake connections: key-pair auth, the per-role session cache, query helpers. + +The API keeps one Snowflake session per *role* it serves (TENANT_ACME, +TENANT_ACME_ADMIN, PLATFORM_ADMIN, ...), because Snowflake evaluates row access +and masking policies against the session's role. Sessions are opened lazily, +shared by concurrent requests for the same role, and bounded: the least +recently used one is closed once more than `max_size` are open. +""" + +from __future__ import annotations + +import asyncio +import contextlib +import logging +from collections import OrderedDict +from collections.abc import AsyncIterator, Awaitable, Callable, Sequence +from typing import Any + +import pandas as pd +import snowflake.connector +from cryptography.hazmat.primitives import serialization +from snowflake.connector.aio import DictCursor, SnowflakeConnection + +from app.settings import Settings + +log = logging.getLogger(__name__) + +APP_NAME = "tenantboard" + + +def private_key_der(settings: Settings) -> bytes: + """The unencrypted PKCS#8 key as DER bytes. + + Loaded here rather than passed as private_key_file: connector 5.x refuses + key files that are group- or world-writable, which is every file on some + mounted filesystems (WSL /mnt/c, Docker Desktop bind mounts). + """ + key = serialization.load_pem_private_key(settings.snowflake_private_key_path.read_bytes(), password=None) + return key.private_bytes( + encoding=serialization.Encoding.DER, + format=serialization.PrivateFormat.PKCS8, + encryption_algorithm=serialization.NoEncryption(), + ) + + +def _connect_kwargs(settings: Settings, *, user: str, role: str, database: str | None) -> dict[str, Any]: + return { + "account": settings.account, + "host": settings.host, + "port": settings.port, + "protocol": "https", + "user": user, + "authenticator": "SNOWFLAKE_JWT", + "private_key": private_key_der(settings), + "role": role, + "warehouse": settings.snowflake_warehouse, + "database": database, + "schema": settings.tb_schema if database else None, + # Every bind in this sample is a qmark bind (server-side binding). + "paramstyle": "qmark", + "application": APP_NAME, + # Always log out on close; no server-side async queries outlive a session here. + "enable_server_session_keep_alive_auto_detection": False, + } + + +async def open_role_session(settings: Settings, role: str, *, database: str | None = None) -> SnowflakeConnection: + """Open an async session for the app's service user under `role`.""" + conn = SnowflakeConnection( + **_connect_kwargs(settings, user=settings.tb_app_user, role=role, database=database or settings.tb_database), + session_parameters={ + "QUERY_TAG": APP_NAME, + # Dashboards re-run identical aggregate queries; the result cache + # serves repeat loads without warehouse time. Each session runs as + # one tenant role, so a reused result is that tenant's own. Set + # explicitly so an account-level default cannot turn it off. + "USE_CACHED_RESULT": True, + "TIMEZONE": "UTC", + }, + client_session_keep_alive=True, + ) + await conn.connect() + return conn + + +def connect_sync( + settings: Settings, *, user: str, role: str, database: str | None = None, **session_parameters: Any +) -> snowflake.connector.SnowflakeConnection: + """Synchronous session for tooling and tests (never used on the request path).""" + return snowflake.connector.connect( + **_connect_kwargs(settings, user=user, role=role, database=database), + session_parameters={"QUERY_TAG": f"{APP_NAME}:tooling", "TIMEZONE": "UTC", **session_parameters}, + client_session_keep_alive=False, + ) + + +def connect_admin(settings: Settings, database: str | None = None) -> snowflake.connector.SnowflakeConnection: + """Admin session for deployment, provisioning and seeding.""" + return connect_sync(settings, user=settings.snowflake_user, role=settings.admin_role, database=database) + + +class _Entry: + __slots__ = ("conn", "leases", "retired") + + def __init__(self, conn: SnowflakeConnection) -> None: + self.conn = conn + self.leases = 0 + self.retired = False + + +class RoleSessionCache: + """Bounded LRU of role -> session, with leases so eviction never closes a busy session.""" + + def __init__(self, factory: Callable[[str], Awaitable[SnowflakeConnection]], max_size: int) -> None: + self._factory = factory + self._max_size = max_size + self._entries: OrderedDict[str, _Entry] = OrderedDict() + self._opening: dict[str, asyncio.Lock] = {} + self._closed = False + + @property + def open_roles(self) -> list[str]: + return list(self._entries) + + async def _get(self, role: str) -> _Entry: + entry = self._entries.get(role) + if entry is None or entry.conn.is_closed(): + lock = self._opening.setdefault(role, asyncio.Lock()) + async with lock: + entry = self._entries.get(role) + if entry is None or entry.conn.is_closed(): + entry = _Entry(await self._factory(role)) + self._entries[role] = entry + log.info("opened Snowflake session for role %s (%d open)", role, len(self._entries)) + self._entries.move_to_end(role) + return entry + + def _evict(self) -> list[_Entry]: + idle: list[_Entry] = [] + while len(self._entries) > self._max_size: + _, entry = self._entries.popitem(last=False) + entry.retired = True + if entry.leases == 0: + idle.append(entry) + return idle + + @contextlib.asynccontextmanager + async def lease(self, role: str) -> AsyncIterator[SnowflakeConnection]: + if self._closed: + raise RuntimeError("session cache is closed") + entry = await self._get(role) + entry.leases += 1 + for stale in self._evict(): + await stale.conn.close() + try: + yield entry.conn + finally: + entry.leases -= 1 + if entry.retired and entry.leases == 0: + await entry.conn.close() + + async def aclose(self) -> None: + self._closed = True + entries, self._entries = list(self._entries.values()), OrderedDict() + await asyncio.gather(*(entry.conn.close() for entry in entries), return_exceptions=True) + + +def query_tag(endpoint: str) -> str: + """Per-endpoint query tag, so QUERY_HISTORY attributes warehouse time to API routes.""" + return f"{APP_NAME}:{endpoint}" + + +async def fetch_dicts( + conn: SnowflakeConnection, sql: str, params: Sequence[Any] | None = None, *, tag: str +) -> list[dict[str, Any]]: + async with conn.cursor(DictCursor) as cur: + # _statement_params sets QUERY_TAG for this statement only, so requests + # for different endpoints can share one session without ALTER SESSION races. + await cur.execute(sql, params, _statement_params={"QUERY_TAG": tag}) + return list(await cur.fetchall()) + + +async def fetch_frame( + conn: SnowflakeConnection, sql: str, params: Sequence[Any] | None = None, *, tag: str +) -> pd.DataFrame: + """Run a query and fetch the result as Arrow batches straight into pandas.""" + async with conn.cursor() as cur: + await cur.execute(sql, params, _statement_params={"QUERY_TAG": tag}) + return await cur.fetch_pandas_all() + + +async def execute( + conn: SnowflakeConnection, sql: str, params: Sequence[Any] | None = None, *, tag: str +) -> tuple[Any, ...] | None: + async with conn.cursor() as cur: + await cur.execute(sql, params, _statement_params={"QUERY_TAG": tag}) + return await cur.fetchone() if cur.description else None diff --git a/tenantboard/app/errors.py b/tenantboard/app/errors.py new file mode 100644 index 0000000..fa3f03d --- /dev/null +++ b/tenantboard/app/errors.py @@ -0,0 +1,102 @@ +"""RFC 9457 problem details for every error the API returns. + +Snowflake errors keep their identity on the wire: the response carries the +Snowflake error code, SQLSTATE and query id, so a client (or an on-call +engineer) can find the failing statement in QUERY_HISTORY. +""" + +from __future__ import annotations + +import logging +from http import HTTPStatus +from typing import Any + +from fastapi import FastAPI, Request +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse +from snowflake.connector.errors import Error as SnowflakeError +from starlette.exceptions import HTTPException as StarletteHTTPException + +log = logging.getLogger(__name__) + +PROBLEM_JSON = "application/problem+json" +PROBLEM_BASE = "urn:tenantboard:problem" + +# Snowflake error code -> (HTTP status, problem type slug, title) +SNOWFLAKE_ERRORS: dict[int, tuple[int, str, str]] = { + # Role in the connect string does not exist or is not granted to the user. + 390189: (403, "tenant-role-unavailable", "Tenant role is not available to the service"), + # NULL in a NOT NULL column during DML. + 100072: (422, "constraint-violation", "Uploaded rows violate a table constraint"), + # MERGE matched one target row to several source rows. + 100090: (409, "duplicate-merge-source", "Several uploaded rows target the same order"), + # Object does not exist or not authorized / insufficient privileges. + 2003: (403, "not-authorized", "Object does not exist or not authorized"), + 3001: (403, "not-authorized", "Insufficient privileges"), +} + + +class ProblemError(Exception): + """An error the API reports as problem details.""" + + def __init__(self, status: int, slug: str, title: str, detail: str, **extensions: Any) -> None: + super().__init__(detail) + self.status, self.slug, self.title, self.detail, self.extensions = status, slug, title, detail, extensions + + +def tenant_not_found(slug: str) -> ProblemError: + return ProblemError(404, "tenant-not-found", "Unknown tenant", f"No merchant with slug {slug!r}.", tenant=slug) + + +def problem_response( + request: Request, status: int, slug: str, title: str, detail: str, **extensions: Any +) -> JSONResponse: + body = { + "type": f"{PROBLEM_BASE}:{slug}", + "title": title, + "status": status, + "detail": detail, + "instance": request.url.path, + **extensions, + } + return JSONResponse(body, status_code=status, media_type=PROBLEM_JSON) + + +def snowflake_problem(exc: SnowflakeError) -> ProblemError: + errno = exc.errno or 0 + status, slug, title = SNOWFLAKE_ERRORS.get(errno, (502, "snowflake-error", "Snowflake rejected the statement")) + # The connector's message starts with " (): "; keep the text. + message = (exc.raw_msg or exc.msg or str(exc)).strip() + return ProblemError( + status, + slug, + title, + message, + snowflake={"error_code": f"{errno:06d}", "sql_state": exc.sqlstate, "query_id": exc.sfqid}, + ) + + +def install(app: FastAPI) -> None: + @app.exception_handler(ProblemError) + async def _problem(request: Request, exc: ProblemError) -> JSONResponse: + return problem_response(request, exc.status, exc.slug, exc.title, exc.detail, **exc.extensions) + + @app.exception_handler(SnowflakeError) + async def _snowflake(request: Request, exc: SnowflakeError) -> JSONResponse: + problem = snowflake_problem(exc) + log.warning("Snowflake error %s on %s: %s", exc.errno, request.url.path, problem.detail) + return problem_response( + request, problem.status, problem.slug, problem.title, problem.detail, **problem.extensions + ) + + @app.exception_handler(RequestValidationError) + async def _validation(request: Request, exc: RequestValidationError) -> JSONResponse: + errors = [{"loc": list(e["loc"]), "msg": e["msg"], "type": e["type"]} for e in exc.errors()] + return problem_response( + request, 422, "invalid-request", "Invalid request", "The request did not validate.", errors=errors + ) + + @app.exception_handler(StarletteHTTPException) + async def _http(request: Request, exc: StarletteHTTPException) -> JSONResponse: + title = HTTPStatus(exc.status_code).phrase + return problem_response(request, exc.status_code, title.lower().replace(" ", "-"), title, str(exc.detail)) diff --git a/tenantboard/app/main.py b/tenantboard/app/main.py new file mode 100644 index 0000000..5411f32 --- /dev/null +++ b/tenantboard/app/main.py @@ -0,0 +1,175 @@ +"""Tenantboard API: customer-facing analytics for the merchants of a B2B commerce platform. + + uv run uvicorn app.main:app --port 8000 + +Every tenant request runs on a Snowflake session whose role is the tenant's +own (TENANT_ for viewers, TENANT__ADMIN for admins), so Snowflake's +row access and masking policies apply to it. The caller's tenant and role come +from the URL and the X-Tenant-Role header here; in production they come from +the verified claims of the caller's token, never from the request body. +""" + +from __future__ import annotations + +import logging +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from dataclasses import dataclass +from datetime import date +from typing import Annotated + +from fastapi import Depends, FastAPI, File, Header, Path, Query, Request, UploadFile +from fastapi.middleware.cors import CORSMiddleware + +from app import db, errors, tenancy +from app.errors import ProblemError, tenant_not_found +from app.models import Merchant, OrderPage, RevenueSeries, Summary, TenantRole, TopCustomer, UploadResult +from app.repository import PlatformRepository, TenantRepository +from app.settings import Settings, get_settings +from app.uploads import parse_orders_csv + +log = logging.getLogger(__name__) + +SLUG_PATTERN = r"^[a-z0-9][a-z0-9-]{0,47}$" + + +@dataclass(frozen=True) +class AppState: + settings: Settings + sessions: db.RoleSessionCache + + +def get_state(request: Request) -> AppState: + return request.app.state.tenantboard + + +def endpoint_tag(request: Request) -> str: + """QUERY_TAG for the route (its path template, so tags group by endpoint, not by tenant).""" + route = request.scope.get("route") + return db.query_tag(getattr(route, "path", request.url.path)) + + +StateDep = Annotated[AppState, Depends(get_state)] +TagDep = Annotated[str, Depends(endpoint_tag)] + + +async def platform_repository(state: StateDep, tag: TagDep) -> AsyncIterator[PlatformRepository]: + async with state.sessions.lease(tenancy.PLATFORM_ROLE) as conn: + yield PlatformRepository(conn, tag) + + +PlatformDep = Annotated[PlatformRepository, Depends(platform_repository)] + + +async def tenant_repository( + slug: Annotated[str, Path(pattern=SLUG_PATTERN, description="Merchant slug, e.g. acme")], + state: StateDep, + tag: TagDep, + platform: PlatformDep, + role: Annotated[TenantRole, Header(alias="X-Tenant-Role")] = TenantRole.viewer, +) -> AsyncIterator[TenantRepository]: + merchant = await platform.get_merchant(slug) + if merchant is None: + raise tenant_not_found(slug) + snowflake_role = (tenancy.admin_role if role is TenantRole.admin else tenancy.viewer_role)(merchant.tenant_id) + async with state.sessions.lease(snowflake_role) as conn: + yield TenantRepository(conn, merchant, role, snowflake_role, tag) + + +TenantDep = Annotated[TenantRepository, Depends(tenant_repository)] + + +def create_app(settings: Settings | None = None) -> FastAPI: + settings = settings or get_settings() + + @asynccontextmanager + async def lifespan(app: FastAPI) -> AsyncIterator[None]: + sessions = db.RoleSessionCache( + lambda role: db.open_role_session(settings, role), max_size=settings.tb_max_role_connections + ) + app.state.tenantboard = AppState(settings=settings, sessions=sessions) + try: + yield + finally: + await sessions.aclose() + + app = FastAPI( + title="Tenantboard API", + version="0.1.0", + summary="Multi-tenant analytics whose isolation and masking are enforced by Snowflake policies", + lifespan=lifespan, + ) + app.add_middleware( + CORSMiddleware, + allow_origins=settings.tb_cors_origins, + allow_methods=["GET", "POST"], + allow_headers=["X-Tenant-Role"], + ) + errors.install(app) + + @app.get("/api/health", tags=["platform"]) + async def health(state: StateDep, tag: TagDep) -> dict[str, str]: + async with state.sessions.lease(tenancy.PLATFORM_ROLE) as conn: + row = await db.execute(conn, "SELECT CURRENT_VERSION()", tag=tag) + return { + "status": "ok", + "snowflake_version": str(row[0]) if row else "unknown", + "database": settings.tb_database, + } + + @app.get("/api/tenants", tags=["platform"]) + async def list_tenants(platform: PlatformDep) -> list[Merchant]: + """The tenant directory (the dashboard's tenant picker).""" + return await platform.list_merchants() + + @app.get("/api/tenants/{slug}/summary", tags=["analytics"]) + async def summary(repo: TenantDep) -> Summary: + return await repo.summary() + + @app.get("/api/tenants/{slug}/revenue", tags=["analytics"]) + async def revenue( + repo: TenantDep, + start: Annotated[date | None, Query(description="First day (inclusive); default: first order")] = None, + end: Annotated[date | None, Query(description="Last day (inclusive); default: last order")] = None, + ) -> RevenueSeries: + if start and end and start > end: + raise ProblemError(422, "invalid-request", "Invalid request", "start must not be after end.") + return await repo.revenue_by_day(start, end) + + @app.get("/api/tenants/{slug}/top-customers", tags=["analytics"]) + async def top_customers(repo: TenantDep, limit: Annotated[int, Query(ge=1, le=50)] = 5) -> list[TopCustomer]: + return await repo.top_customers(limit) + + @app.get("/api/tenants/{slug}/orders", tags=["orders"]) + async def list_orders( + repo: TenantDep, + page: Annotated[int, Query(ge=1, le=10_000)] = 1, + page_size: Annotated[int, Query(ge=1, le=200)] = 25, + ) -> OrderPage: + return await repo.list_orders(page, page_size) + + @app.post("/api/tenants/{slug}/orders/upload", tags=["orders"], status_code=200) + async def upload_orders( + repo: TenantDep, file: Annotated[UploadFile, File(description="CSV of offline orders")] + ) -> UploadResult: + """Insert or update offline orders from a CSV (admin role only).""" + if repo.role is not TenantRole.admin: + raise ProblemError( + 403, "admin-role-required", "Admin role required", "Uploading orders requires X-Tenant-Role: admin." + ) + payload = await file.read(settings.tb_max_upload_bytes + 1) + if len(payload) > settings.tb_max_upload_bytes: + raise ProblemError( + 413, "upload-too-large", "Upload too large", f"At most {settings.tb_max_upload_bytes} bytes." + ) + frame = parse_orders_csv(payload, repo.merchant.currency) + inserted, updated = await repo.upload_orders(frame) + log.info( + "tenant %s uploaded %d orders (%d inserted, %d updated)", repo.tenant_id, len(frame), inserted, updated + ) + return UploadResult(tenant_id=repo.tenant_id, rows=len(frame), inserted=inserted, updated=updated) + + return app + + +app = create_app() diff --git a/tenantboard/app/models.py b/tenantboard/app/models.py new file mode 100644 index 0000000..84c2886 --- /dev/null +++ b/tenantboard/app/models.py @@ -0,0 +1,91 @@ +"""Request and response models of the public API.""" + +from __future__ import annotations + +from datetime import date, datetime +from decimal import Decimal +from enum import StrEnum + +from pydantic import BaseModel, ConfigDict, Field + + +class TenantRole(StrEnum): + """Which of the tenant's two Snowflake roles a request runs under.""" + + viewer = "viewer" # TENANT_: customer email masked + admin = "admin" # TENANT__ADMIN: email clear, may upload + + +class _Model(BaseModel): + model_config = ConfigDict(frozen=True) + + +class Merchant(_Model): + tenant_id: str + slug: str + name: str + plan: str + region: str + currency: str + + +class Summary(_Model): + tenant_id: str + role: TenantRole + snowflake_role: str = Field(description="Role the request's Snowflake session ran under") + orders: int + completed_orders: int + customers: int + revenue: Decimal = Field(description="Sum of TOTAL_AMOUNT over completed orders") + average_order_value: Decimal + currency: str + first_order_at: datetime | None + last_order_at: datetime | None + + +class DailyRevenue(_Model): + day: date + orders: int + revenue: float + revenue_7d_avg: float = Field(description="Trailing 7-day mean of daily revenue, days without orders count as 0") + + +class RevenueSeries(_Model): + tenant_id: str + currency: str + days: list[DailyRevenue] + + +class TopCustomer(_Model): + customer_id: int + name: str + email: str = Field(description="***MASKED*** unless the request runs as the tenant's admin role") + orders: int + revenue: Decimal + + +class Order(_Model): + order_id: str + tenant_id: str + ordered_at: datetime + status: str + channel: str + currency: str + total_amount: Decimal + customer_id: int + customer_name: str | None + customer_email: str | None + + +class OrderPage(_Model): + items: list[Order] + page: int + page_size: int + total: int + + +class UploadResult(_Model): + tenant_id: str + rows: int + inserted: int + updated: int diff --git a/tenantboard/app/provisioning.py b/tenantboard/app/provisioning.py new file mode 100644 index 0000000..c346b9d --- /dev/null +++ b/tenantboard/app/provisioning.py @@ -0,0 +1,206 @@ +"""Deployment and tenant provisioning, run with the admin identity. + +Used by `make seed`, `make demo` and the test suite; the API never imports it. +Everything here is synchronous: it is batch tooling, not the request path. +""" + +from __future__ import annotations + +import json +import re +from collections.abc import Iterable, Sequence +from dataclasses import dataclass +from typing import Any + +import pandas as pd +from snowflake.connector import SnowflakeConnection +from snowflake.connector.pandas_tools import write_pandas + +from app.settings import SAMPLE_ROOT, Settings +from app.tenancy import admin_role, viewer_role + +INIT_DIR = SAMPLE_ROOT / "init" +# Init files keep account-level setup above this marker and database-relative +# DDL below it; the test suite deploys the part below into its own databases. +DEPLOY_MARKER = "-- @deploy:" +SCHEMA = "APP" +GRANTS_FILE = "04_grants.sf.sql" +# Load order matters only for readability; there are no enforced foreign keys. +TENANT_TABLES = ("MERCHANTS", "CUSTOMERS", "ORDERS", "LINE_ITEMS") + +IDENTIFIER_PATTERN = re.compile(r"^[A-Z][A-Z0-9_]{0,254}$") + + +def _identifier(name: str) -> str: + """Database names come from code and fixtures, never from requests; check anyway.""" + if not IDENTIFIER_PATTERN.fullmatch(name): + raise ValueError(f"unsafe identifier {name!r}") + return name + + +def query(conn: SnowflakeConnection, sql: str, params: Sequence[Any] | None = None) -> list[tuple[Any, ...]]: + with conn.cursor() as cur: + cur.execute(sql, params) + return cur.fetchall() if cur.description else [] + + +def run_script(conn: SnowflakeConnection, sql: str) -> None: + for cur in conn.execute_string(sql, remove_comments=True): + cur.close() + + +def deployable_sql(*, only: str | None = None) -> str: + """Database-relative DDL from init/*.sf.sql (everything below the marker), in file order.""" + parts = [] + for path in sorted(INIT_DIR.glob("*.sf.sql")): + if only is not None and path.name != only: + continue + text = path.read_text(encoding="utf-8") + if DEPLOY_MARKER not in text: + raise RuntimeError(f"{path.name} has no '{DEPLOY_MARKER}' marker") + parts.append(text.split(DEPLOY_MARKER, 1)[1].split("\n", 1)[1]) + return "\n".join(parts) + + +def apply_grants(conn: SnowflakeConnection, database: str) -> None: + """Database-level grants (a clone never copies them) plus the grants file's schema/table grants.""" + database = _identifier(database) + query(conn, f"USE DATABASE {database}") + for role in ("TENANT_READER", "PLATFORM_ADMIN"): + query(conn, f"GRANT USAGE ON DATABASE {database} TO ROLE {role}") + run_script(conn, deployable_sql(only=GRANTS_FILE)) + + +def deploy_database(conn: SnowflakeConnection, database: str) -> None: + """Create `database` from scratch with every table, policy and grant in it.""" + database = _identifier(database) + query(conn, f"CREATE OR REPLACE DATABASE {database}") + query(conn, f"USE DATABASE {database}") + run_script(conn, deployable_sql()) + apply_grants(conn, database) + + +def drop_database(conn: SnowflakeConnection, database: str) -> None: + query(conn, f"DROP DATABASE IF EXISTS {_identifier(database)}") + + +def roles_granted_to_user(conn: SnowflakeConnection, user: str) -> set[str]: + with conn.cursor() as cur: + cur.execute(f"SHOW GRANTS TO USER {_identifier(user)}") + columns = [d.name.lower() for d in cur.description] + index = next(columns.index(c) for c in ("role", "name") if c in columns) + return {row[index] for row in cur.fetchall()} + + +def provision_tenants(conn: SnowflakeConnection, settings: Settings, tenant_ids: Iterable[str]) -> int: + """Create each tenant's viewer and admin roles and grant both to the app's service user. + + Idempotent and incremental: tenants whose admin role is already granted to + the service user (the last step) are skipped. The rest are sent as + multi-statement requests, a batch of tenants per round trip. Returns the + number of tenants provisioned. + """ + app_user = _identifier(settings.tb_app_user) + granted = roles_granted_to_user(conn, app_user) + pending = [t for t in tenant_ids if admin_role(t) not in granted or viewer_role(t) not in granted] + statements: list[str] = [] + for tenant_id in pending: + viewer, admin = viewer_role(tenant_id), admin_role(tenant_id) + statements += [ + f"CREATE ROLE IF NOT EXISTS {viewer}", + f"CREATE ROLE IF NOT EXISTS {admin}", + f"GRANT ROLE TENANT_READER TO ROLE {viewer}", + f"GRANT ROLE {viewer} TO ROLE {admin}", + f"GRANT ROLE TENANT_ADMIN TO ROLE {admin}", + f"GRANT ROLE {viewer} TO USER {app_user}", + f"GRANT ROLE {admin} TO USER {app_user}", + ] + batch = 70 + with conn.cursor() as cur: + for start in range(0, len(statements), batch): + chunk = statements[start : start + batch] + cur.execute(";\n".join(chunk), num_statements=len(chunk)) + while cur.nextset(): + pass + return len(pending) + + +def load_tables(conn: SnowflakeConnection, database: str, frames: dict[str, pd.DataFrame]) -> dict[str, int]: + """Replace the contents of each table with its frame (TRUNCATE, then write_pandas).""" + database = _identifier(database) + loaded = {} + for table, frame in frames.items(): + table = _identifier(table) + query(conn, f"TRUNCATE TABLE {database}.{SCHEMA}.{table}") + result = write_pandas( + conn, frame, table, database=database, schema=SCHEMA, use_logical_type=True, quote_identifiers=True + ) + if not result.success: + raise RuntimeError(f"write_pandas into {table} failed: {result.copy_results}") + loaded[table] = result.nrows + return loaded + + +@dataclass(frozen=True) +class PolicyRef: + kind: str # ROW_ACCESS_POLICY | MASKING_POLICY + database: str + schema: str + name: str + column: str | None # masking: the protected column + arg_columns: tuple[str, ...] # row access: the policy's argument columns + + +def policy_refs(conn: SnowflakeConnection, database: str, table: str) -> list[PolicyRef]: + database, table = _identifier(database), _identifier(table) + rows = query( + conn, + f"SELECT POLICY_KIND, POLICY_DB, POLICY_SCHEMA, POLICY_NAME, REF_COLUMN_NAME, REF_ARG_COLUMN_NAMES " + f"FROM TABLE({database}.INFORMATION_SCHEMA.POLICY_REFERENCES(REF_ENTITY_NAME => ?, REF_ENTITY_DOMAIN => 'table'))", + [f"{database}.{SCHEMA}.{table}"], + ) + return [ + PolicyRef(kind, db, schema, name, column, tuple(json.loads(args)) if args else ()) + for kind, db, schema, name, column, args in rows + ] + + +def clone_database(conn: SnowflakeConnection, source: str, target: str) -> None: + """CREATE DATABASE target CLONE source, provisioned with the source's policies and grants. + + Every policy reference the source's tables carry (POLICY_REFERENCES) is + ensured on the clone, pointing at the clone's own copy of the policy, and + the grants are applied (a clone never copies grants on the database + itself). Each test database therefore matches production exactly. + + The step is driven by the *source's* references, not by a list in code: a + source table without a policy yields a clone without one, so the leak test + fails when the policy is removed from init/. + """ + source, target = _identifier(source), _identifier(target) + query(conn, f"CREATE OR REPLACE DATABASE {target} CLONE {source}") + query(conn, f"USE DATABASE {target}") + # The clone's own copy of every policy (CREATE ... IF NOT EXISTS). + run_script(conn, deployable_sql(only="02_policies.sf.sql")) + statements: list[str] = [] + for table in TENANT_TABLES: + have = policy_refs(conn, target, table) + for ref in policy_refs(conn, source, table): + own = PolicyRef(ref.kind, target, ref.schema, ref.name, ref.column, ref.arg_columns) + if own in have: + continue + policy = f"{target}.{_identifier(ref.schema)}.{_identifier(ref.name)}" + fqtn = f"{target}.{SCHEMA}.{table}" + if ref.kind == "ROW_ACCESS_POLICY": + cols = ", ".join(_identifier(c) for c in ref.arg_columns) + statements.append(f"ALTER TABLE {fqtn} ADD ROW ACCESS POLICY {policy} ON ({cols})") + elif ref.kind == "MASKING_POLICY" and ref.column: + column = _identifier(ref.column) + if any(h.kind == ref.kind and h.column == ref.column for h in have): + statements.append(f"ALTER TABLE {fqtn} MODIFY COLUMN {column} UNSET MASKING POLICY") + statements.append(f"ALTER TABLE {fqtn} MODIFY COLUMN {column} SET MASKING POLICY {policy}") + else: + raise RuntimeError(f"unexpected policy reference {ref}") + for statement in statements: + query(conn, statement) + apply_grants(conn, target) diff --git a/tenantboard/app/repository.py b/tenantboard/app/repository.py new file mode 100644 index 0000000..d38c429 --- /dev/null +++ b/tenantboard/app/repository.py @@ -0,0 +1,235 @@ +"""Every SQL statement the API runs. + +Layer 1 of tenant isolation lives here: each tenant query binds TENANT_ID as +a qmark parameter (never formatted into the SQL text). Layer 2 is the +TENANT_ISOLATION row access policy, evaluated against the session's role. Each +layer is sufficient on its own; tests/test_isolation.py proves both. +""" + +from __future__ import annotations + +import uuid +from datetime import date +from decimal import Decimal +from typing import Any + +import pandas as pd +from snowflake.connector.aio import SnowflakeConnection, write_pandas + +from app import db +from app.models import DailyRevenue, Merchant, Order, OrderPage, RevenueSeries, Summary, TenantRole, TopCustomer + +COMPLETED = "completed" + + +def _lower_keys(row: dict[str, Any]) -> dict[str, Any]: + return {k.lower(): v for k, v in row.items()} + + +class PlatformRepository: + """Tenant directory, read under PLATFORM_ADMIN (the only cross-tenant query in the API).""" + + def __init__(self, conn: SnowflakeConnection, tag: str) -> None: + self._conn, self._tag = conn, tag + + async def list_merchants(self) -> list[Merchant]: + rows = await db.fetch_dicts( + self._conn, + "SELECT TENANT_ID, SLUG, NAME, PLAN, REGION, CURRENCY FROM MERCHANTS ORDER BY NAME", + tag=self._tag, + ) + return [Merchant(**_lower_keys(row)) for row in rows] + + async def get_merchant(self, slug: str) -> Merchant | None: + rows = await db.fetch_dicts( + self._conn, + "SELECT TENANT_ID, SLUG, NAME, PLAN, REGION, CURRENCY FROM MERCHANTS WHERE SLUG = ?", + [slug], + tag=self._tag, + ) + return Merchant(**_lower_keys(rows[0])) if rows else None + + +class TenantRepository: + """Queries for one tenant, on a session that runs as one of that tenant's roles.""" + + def __init__( + self, conn: SnowflakeConnection, merchant: Merchant, role: TenantRole, snowflake_role: str, tag: str + ) -> None: + self._conn = conn + self.merchant = merchant + self.role = role + self.snowflake_role = snowflake_role + self._tag = tag + + @property + def tenant_id(self) -> str: + return self.merchant.tenant_id + + async def summary(self) -> Summary: + rows = await db.fetch_dicts( + self._conn, + """ + SELECT COUNT(*) AS ORDERS, + COUNT_IF(STATUS = ?) AS COMPLETED_ORDERS, + COUNT(DISTINCT CUSTOMER_ID) AS CUSTOMERS, + COALESCE(SUM(IFF(STATUS = ?, TOTAL_AMOUNT, 0)), 0) AS REVENUE, + MIN(ORDERED_AT) AS FIRST_ORDER_AT, + MAX(ORDERED_AT) AS LAST_ORDER_AT + FROM ORDERS + WHERE TENANT_ID = ? + """, + [COMPLETED, COMPLETED, self.tenant_id], + tag=self._tag, + ) + row = rows[0] + revenue = Decimal(row["REVENUE"]) + completed = int(row["COMPLETED_ORDERS"]) + return Summary( + tenant_id=self.tenant_id, + role=self.role, + snowflake_role=self.snowflake_role, + orders=int(row["ORDERS"]), + completed_orders=completed, + customers=int(row["CUSTOMERS"]), + revenue=revenue, + average_order_value=(revenue / completed).quantize(Decimal("0.01")) if completed else Decimal("0.00"), + currency=self.merchant.currency, + first_order_at=row["FIRST_ORDER_AT"], + last_order_at=row["LAST_ORDER_AT"], + ) + + async def revenue_by_day(self, start: date | None, end: date | None) -> RevenueSeries: + """Daily completed revenue, gap-filled and smoothed in pandas. + + The aggregate comes back as Arrow batches (fetch_pandas_all), so the + series lands in a DataFrame without per-row Python conversion. + """ + frame = await db.fetch_frame( + self._conn, + """ + SELECT TO_DATE(ORDERED_AT) AS DAY, COUNT(*) AS ORDERS, SUM(TOTAL_AMOUNT) AS REVENUE + FROM ORDERS + WHERE TENANT_ID = ? + AND STATUS = ? + AND TO_DATE(ORDERED_AT) BETWEEN COALESCE(?::DATE, '0001-01-01'::DATE) AND COALESCE(?::DATE, '9999-12-31'::DATE) + GROUP BY 1 + ORDER BY 1 + """, + [self.tenant_id, COMPLETED, start.isoformat() if start else None, end.isoformat() if end else None], + tag=self._tag, + ) + if frame.empty and (start is None or end is None): + return RevenueSeries(tenant_id=self.tenant_id, currency=self.merchant.currency, days=[]) + frame["DAY"] = pd.to_datetime(frame["DAY"]) + index = pd.date_range(start or frame["DAY"].min(), end or frame["DAY"].max(), freq="D", name="DAY") + series = ( + frame.set_index("DAY")[["ORDERS", "REVENUE"]] + .astype({"ORDERS": "int64", "REVENUE": "float64"}) + .reindex(index, fill_value=0) + ) + series["AVG7"] = series["REVENUE"].rolling(7, min_periods=1).mean() + days = [ + DailyRevenue( + day=day.date(), + orders=int(r.ORDERS), + revenue=round(float(r.REVENUE), 2), + revenue_7d_avg=round(float(r.AVG7), 2), + ) + for day, r in series.iterrows() + ] + return RevenueSeries(tenant_id=self.tenant_id, currency=self.merchant.currency, days=days) + + async def top_customers(self, limit: int) -> list[TopCustomer]: + rows = await db.fetch_dicts( + self._conn, + """ + SELECT c.CUSTOMER_ID, c.NAME, c.EMAIL, COUNT(*) AS ORDERS, SUM(o.TOTAL_AMOUNT) AS REVENUE + FROM ORDERS o + JOIN CUSTOMERS c ON c.TENANT_ID = o.TENANT_ID AND c.CUSTOMER_ID = o.CUSTOMER_ID + WHERE o.TENANT_ID = ? + AND o.STATUS = ? + GROUP BY c.CUSTOMER_ID, c.NAME, c.EMAIL + ORDER BY REVENUE DESC, c.CUSTOMER_ID + LIMIT ? + """, + [self.tenant_id, COMPLETED, limit], + tag=self._tag, + ) + return [ + TopCustomer( + customer_id=r["CUSTOMER_ID"], name=r["NAME"], email=r["EMAIL"], orders=r["ORDERS"], revenue=r["REVENUE"] + ) + for r in rows + ] + + async def list_orders(self, page: int, page_size: int) -> OrderPage: + """One page of orders, newest first, with the total from the same scan (COUNT(*) OVER ()).""" + rows = await db.fetch_dicts( + self._conn, + """ + SELECT o.ORDER_ID, o.TENANT_ID, o.ORDERED_AT, o.STATUS, o.CHANNEL, o.CURRENCY, o.TOTAL_AMOUNT, + o.CUSTOMER_ID, c.NAME AS CUSTOMER_NAME, c.EMAIL AS CUSTOMER_EMAIL, + COUNT(*) OVER () AS TOTAL + FROM ORDERS o + LEFT JOIN CUSTOMERS c ON c.TENANT_ID = o.TENANT_ID AND c.CUSTOMER_ID = o.CUSTOMER_ID + WHERE o.TENANT_ID = ? + ORDER BY o.ORDERED_AT DESC, o.ORDER_ID DESC + LIMIT ? OFFSET ? + """, + [self.tenant_id, page_size, (page - 1) * page_size], + tag=self._tag, + ) + if rows: + total = int(rows[0]["TOTAL"]) + else: # past the last page: the window had no rows to count + counted = await db.fetch_dicts( + self._conn, "SELECT COUNT(*) AS N FROM ORDERS WHERE TENANT_ID = ?", [self.tenant_id], tag=self._tag + ) + total = int(counted[0]["N"]) + items = [Order(**_lower_keys({k: v for k, v in row.items() if k != "TOTAL"})) for row in rows] + return OrderPage(items=items, page=page, page_size=page_size, total=total) + + async def upload_orders(self, frame: pd.DataFrame) -> tuple[int, int]: + """Stage the rows with write_pandas into a temporary table, then MERGE them into ORDERS. + + The MERGE is one statement, so it applies all rows or none. Its ON + clause binds the tenant: an upload can only ever match this tenant's + orders, even though every tenant has an ORD-000001. + """ + staging = f"ORDERS_UPLOAD_{uuid.uuid4().hex.upper()}" + result = await write_pandas( + self._conn, + frame, + staging, + auto_create_table=True, + table_type="temp", + use_logical_type=True, + quote_identifiers=True, + ) + if not result.success: + raise RuntimeError(f"write_pandas staging failed: {result.copy_results}") + try: + row = await db.execute( + self._conn, + """ + MERGE INTO ORDERS t + USING ( + SELECT ? AS TENANT_ID, "ORDER_ID", "CUSTOMER_ID", "ORDERED_AT", "STATUS", "CHANNEL", "CURRENCY", "TOTAL_AMOUNT" + FROM IDENTIFIER(?) + ) s + ON t.TENANT_ID = ? AND t.TENANT_ID = s.TENANT_ID AND t.ORDER_ID = s.ORDER_ID + WHEN MATCHED THEN UPDATE SET + CUSTOMER_ID = s.CUSTOMER_ID, ORDERED_AT = s.ORDERED_AT, STATUS = s.STATUS, + CHANNEL = s.CHANNEL, CURRENCY = s.CURRENCY, TOTAL_AMOUNT = s.TOTAL_AMOUNT + WHEN NOT MATCHED THEN INSERT + (TENANT_ID, ORDER_ID, CUSTOMER_ID, ORDERED_AT, STATUS, CHANNEL, CURRENCY, TOTAL_AMOUNT) + VALUES (s.TENANT_ID, s.ORDER_ID, s.CUSTOMER_ID, s.ORDERED_AT, s.STATUS, s.CHANNEL, s.CURRENCY, s.TOTAL_AMOUNT) + """, + [self.tenant_id, staging, self.tenant_id], + tag=self._tag, + ) + finally: + await db.execute(self._conn, "DROP TABLE IF EXISTS IDENTIFIER(?)", [staging], tag=self._tag) + inserted, updated = (int(row[0]), int(row[1])) if row else (0, 0) + return inserted, updated diff --git a/tenantboard/app/settings.py b/tenantboard/app/settings.py new file mode 100644 index 0000000..7191e63 --- /dev/null +++ b/tenantboard/app/settings.py @@ -0,0 +1,97 @@ +"""Connection and app settings, read from the environment in one place. + +TEST_TARGET picks the defaults: EMULATOR (the local Snowflake-Next container) +or SNOWFLAKE_CLOUD (the SNOWFLAKE_* variables). Nothing else in the sample +branches on the target. + +Two identities are configured: + +* the *admin* identity (SNOWFLAKE_USER / SNOWFLAKE_ROLE) deploys schema, + provisions tenants and seeds data; the API never uses it; +* the *app* identity (TB_APP_USER) is the API's service user. It never runs + with an administrative role: each request runs under the tenant's own role. +""" + +from __future__ import annotations + +from functools import lru_cache +from pathlib import Path +from typing import Literal + +from pydantic import Field, field_validator, model_validator +from pydantic_settings import BaseSettings, SettingsConfigDict + +SAMPLE_ROOT = Path(__file__).resolve().parent.parent +EMULATOR_HOST = "snowflake.localhost.localstack.cloud" + + +class Settings(BaseSettings): + model_config = SettingsConfigDict(extra="ignore", frozen=True) + + test_target: Literal["EMULATOR", "SNOWFLAKE_CLOUD"] = "EMULATOR" + + snowflake_account: str | None = None + snowflake_host: str | None = None + snowflake_port: int = 4566 + snowflake_user: str = "SAMPLE_SVC" + snowflake_role: str | None = None + snowflake_warehouse: str = "TB_WH" + snowflake_private_key_path: Path = SAMPLE_ROOT / ".secrets" / "rsa_key.p8" + + tb_app_user: str = "TENANTBOARD_SVC" + tb_database: str = "TENANTBOARD" + tb_schema: str = "APP" + # Upper bound on open per-role sessions; least recently used ones are closed. + tb_max_role_connections: int = Field(default=16, ge=1, le=256) + tb_cors_origins: list[str] = ["http://localhost:5173", "http://127.0.0.1:5173"] + tb_max_upload_bytes: int = 1_000_000 + + @field_validator("test_target", mode="before") + @classmethod + def _upper(cls, value: object) -> object: + return value.upper() if isinstance(value, str) else value + + @field_validator("snowflake_private_key_path") + @classmethod + def _absolute(cls, value: Path) -> Path: + return value if value.is_absolute() else SAMPLE_ROOT / value + + @model_validator(mode="after") + def _cloud_requirements(self) -> Settings: + if self.test_target == "SNOWFLAKE_CLOUD": + missing = [name for name in ("snowflake_account", "snowflake_role") if not getattr(self, name)] + if missing: + names = ", ".join(name.upper() for name in missing) + raise ValueError(f"{names} must be set when TEST_TARGET=SNOWFLAKE_CLOUD") + return self + + @property + def is_emulator(self) -> bool: + return self.test_target == "EMULATOR" + + @property + def account(self) -> str: + if self.is_emulator: + # Init hooks run in the emulator's SNOWFLAKE account; accounts are isolated. + return "snowflake" + assert self.snowflake_account is not None # checked by _cloud_requirements + return self.snowflake_account + + @property + def host(self) -> str: + if self.is_emulator: + return EMULATOR_HOST + return self.snowflake_host or f"{self.account}.snowflakecomputing.com" + + @property + def port(self) -> int: + return self.snowflake_port if self.is_emulator else 443 + + @property + def admin_role(self) -> str: + return self.snowflake_role or "ACCOUNTADMIN" + + +@lru_cache +def get_settings() -> Settings: + return Settings() diff --git a/tenantboard/app/tenancy.py b/tenantboard/app/tenancy.py new file mode 100644 index 0000000..0713f69 --- /dev/null +++ b/tenantboard/app/tenancy.py @@ -0,0 +1,38 @@ +"""Tenant ids and the Snowflake role names derived from them. + +The row access policy matches 'TENANT_' || TENANT_ID against the session's +roles, so the role name *is* the tenant's identity. That makes the naming rule +a security invariant: a tenant called ADMIN or READER, or one whose id ends in +_ADMIN, would collide with another role (TENANT_ACME_ADMIN would see the rows +of a tenant ACME_ADMIN), so such ids are refused at provisioning time. +""" + +from __future__ import annotations + +import re + +TENANT_ID_PATTERN = re.compile(r"^[A-Z][A-Z0-9]*(?:_[A-Z0-9]+)*$") +RESERVED_TENANT_IDS = frozenset({"ADMIN", "READER"}) +MAX_TENANT_ID_LENGTH = 48 +PLATFORM_ROLE = "PLATFORM_ADMIN" + + +def validate_tenant_id(tenant_id: str) -> str: + if ( + len(tenant_id) > MAX_TENANT_ID_LENGTH + or not TENANT_ID_PATTERN.fullmatch(tenant_id) + or tenant_id in RESERVED_TENANT_IDS + or tenant_id.endswith("_ADMIN") + ): + raise ValueError(f"invalid tenant id {tenant_id!r}") + return tenant_id + + +def viewer_role(tenant_id: str) -> str: + """TENANT_: sees the tenant's rows, customer email masked.""" + return f"TENANT_{validate_tenant_id(tenant_id)}" + + +def admin_role(tenant_id: str) -> str: + """TENANT__ADMIN: inherits TENANT_ and TENANT_ADMIN, so email is clear.""" + return f"TENANT_{validate_tenant_id(tenant_id)}_ADMIN" diff --git a/tenantboard/app/uploads.py b/tenantboard/app/uploads.py new file mode 100644 index 0000000..0c4390d --- /dev/null +++ b/tenantboard/app/uploads.py @@ -0,0 +1,100 @@ +"""Parsing and validating a CSV of offline orders. + +The API validates shape and types here and leaves required-ness to the table: +ORDERS' NOT NULL constraints are the single source of truth, and a violation +comes back from Snowflake as 100072, reported as 422 with the Snowflake code. +""" + +from __future__ import annotations + +import io + +import pandas as pd + +from app.errors import ProblemError + +REQUIRED_COLUMNS = ("order_id", "customer_id", "ordered_at", "status", "total_amount") +OPTIONAL_COLUMNS = ("currency",) +STATUSES = frozenset({"completed", "refunded", "cancelled"}) +MAX_ROWS = 5_000 +ORDER_ID_PATTERN = r"^[A-Z0-9][A-Z0-9-]{0,31}$" + + +def _invalid(detail: str, **extensions: object) -> ProblemError: + return ProblemError(422, "invalid-upload", "Invalid upload", detail, **extensions) + + +def parse_orders_csv(payload: bytes, default_currency: str) -> pd.DataFrame: + """Return the upload as a frame with ORDERS' column names (TENANT_ID is added by the MERGE).""" + try: + raw = pd.read_csv(io.BytesIO(payload), dtype=str, keep_default_na=False, na_values=[""]) + except (pd.errors.ParserError, pd.errors.EmptyDataError, UnicodeDecodeError) as exc: + raise _invalid(f"Not a readable CSV file: {exc}") from exc + raw.columns = [str(c).strip().lower() for c in raw.columns] + + missing = [c for c in REQUIRED_COLUMNS if c not in raw.columns] + unexpected = [c for c in raw.columns if c not in REQUIRED_COLUMNS + OPTIONAL_COLUMNS] + if missing or unexpected: + raise _invalid("CSV header does not match the offline order format.", missing=missing, unexpected=unexpected) + if raw.empty: + raise _invalid("CSV has a header but no rows.") + if len(raw) > MAX_ROWS: + raise _invalid(f"At most {MAX_ROWS} rows per upload, got {len(raw)}.") + + problems: list[dict[str, object]] = [] + + def check(column: str, bad: pd.Series, message: str) -> None: + # CSV line numbers: header is line 1. + problems.extend({"line": int(i) + 2, "column": column, "msg": message} for i in raw.index[bad]) + + order_id = raw["order_id"].str.strip().str.upper() + check( + "order_id", + order_id.notna() & ~order_id.str.fullmatch(ORDER_ID_PATTERN, na=False), + "expected letters, digits and dashes", + ) + check("order_id", order_id.notna() & order_id.duplicated(keep=False), "duplicate order_id in this file") + + customer_id = pd.to_numeric(raw["customer_id"], errors="coerce") + check( + "customer_id", + raw["customer_id"].notna() & (customer_id.isna() | (customer_id % 1 != 0) | (customer_id < 1)), + "expected a positive integer", + ) + + ordered_at = pd.to_datetime(raw["ordered_at"], errors="coerce", format="ISO8601") + check("ordered_at", raw["ordered_at"].notna() & ordered_at.isna(), "expected an ISO 8601 timestamp") + + status = raw["status"].str.strip().str.lower() + check("status", status.notna() & ~status.isin(STATUSES), f"expected one of {sorted(STATUSES)}") + + amount = pd.to_numeric(raw["total_amount"], errors="coerce") + check( + "total_amount", + raw["total_amount"].notna() & (amount.isna() | (amount < 0) | (amount.round(2) != amount)), + "expected a non-negative amount with at most 2 decimals", + ) + + currency = ( + raw["currency"].str.strip().str.upper() + if "currency" in raw.columns + else pd.Series(default_currency, index=raw.index) + ) + check("currency", currency.notna() & ~currency.str.fullmatch(r"^[A-Z]{3}$", na=False), "expected an ISO 4217 code") + + if problems: + raise _invalid(f"{len(problems)} invalid value(s).", errors=problems[:50]) + + if ordered_at.dt.tz is not None: + ordered_at = ordered_at.dt.tz_convert("UTC").dt.tz_localize(None) + return pd.DataFrame( + { + "ORDER_ID": order_id, + "CUSTOMER_ID": customer_id.astype("Int64"), + "ORDERED_AT": ordered_at, + "STATUS": status, + "CHANNEL": "offline", + "CURRENCY": currency.fillna(default_currency), + "TOTAL_AMOUNT": amount, + } + ) diff --git a/tenantboard/docker-compose.yml b/tenantboard/docker-compose.yml new file mode 100644 index 0000000..6efcbb6 --- /dev/null +++ b/tenantboard/docker-compose.yml @@ -0,0 +1,18 @@ +name: snfs-tenantboard + +services: + snowflake: + image: ${SNOWFLAKE_IMAGE:-localstack/snowflake-next:latest} + pull_policy: always + ports: + - "127.0.0.1:${SNOWFLAKE_PORT:-4566}:4566" + environment: + LOCALSTACK_AUTH_TOKEN: ${LOCALSTACK_AUTH_TOKEN:?set LOCALSTACK_AUTH_TOKEN} + volumes: + - ./init:/etc/localstack/init/ready.d:ro + networks: + default: + aliases: + - snowflake.localhost.localstack.cloud + - stages.s3.localhost.localstack.cloud + - stages-enc.s3.localhost.localstack.cloud diff --git a/tenantboard/docs/architecture.md b/tenantboard/docs/architecture.md new file mode 100644 index 0000000..222e8aa --- /dev/null +++ b/tenantboard/docs/architecture.md @@ -0,0 +1,118 @@ +# Tenantboard architecture + +Design notes behind the [README](../README.md): the data model, the role model the policies depend on, the request path, test isolation and the error contract. + +## Data model + +All tenant tables live in `TENANTBOARD.APP`, carry `TENANT_ID` and are created `WITH ROW ACCESS POLICY TENANT_ISOLATION ON (TENANT_ID)`. Keys are composite with `TENANT_ID` first: order ids are unique per tenant only (every merchant has an `ORD-000001`), which is why the upload's `MERGE` condition binds the tenant. + +```mermaid +erDiagram + MERCHANTS ||--o{ CUSTOMERS : "has" + MERCHANTS ||--o{ ORDERS : "has" + CUSTOMERS ||--o{ ORDERS : "places" + ORDERS ||--|{ LINE_ITEMS : "contains" + MERCHANTS { + varchar TENANT_ID PK + varchar SLUG + varchar NAME + varchar PLAN + } + CUSTOMERS { + varchar TENANT_ID PK + number CUSTOMER_ID PK + varchar EMAIL "EMAIL_MASK" + } + ORDERS { + varchar TENANT_ID PK + varchar ORDER_ID PK + number CUSTOMER_ID + timestamp ORDERED_AT + varchar STATUS + number TOTAL_AMOUNT + } + LINE_ITEMS { + varchar TENANT_ID PK + varchar ORDER_ID PK + number LINE_NO PK + varchar SKU + } +``` + +The seed (`scripts/seed.py`) is a pure function of `(seed, merchant count)`: 50 merchants, about 550 customers, 700 orders and 1,400 line items, so the tests can regenerate it and compute every expected value independently. + +## Role model + +```mermaid +flowchart BT + svc(["user TENANTBOARD_SVC"]) + reader["TENANT_READER
SELECT on tenant tables"] + tadmin["TENANT_ADMIN
INSERT, UPDATE on ORDERS"] + acme["TENANT_ACME"] + acmeadmin["TENANT_ACME_ADMIN"] + platform["PLATFORM_ADMIN
all rows, emails masked"] + reader --> acme + acme --> acmeadmin + tadmin --> acmeadmin + acme --> svc + acmeadmin --> svc + platform --> svc +``` + +Arrows point from a role to the role or user it is granted to. The policies key on role names, so the role is the entitlement and no lookup table is needed: + +| Session role | Rows (`TENANT_ISOLATION`) | `EMAIL` (`EMAIL_MASK`) | +| --- | --- | --- | +| `TENANT_ACME` | ACME | `***MASKED***` | +| `TENANT_ACME_ADMIN` | ACME (inherited) | clear | +| `PLATFORM_ADMIN` | all tenants | `***MASKED***` | + +**Naming invariant.** Because the policy derives the role name from the row's `TENANT_ID`, `app/tenancy.py` refuses tenant ids that would collide with another role: `ADMIN`, `READER` and anything ending in `_ADMIN`. + +**Onboarding.** `provisioning.provision_tenants` creates both roles, wires the hierarchy and grants them to the service user in batched multi-statement requests. It is incremental, so a re-seed costs nothing. + +## Request path + +- **Identity.** A FastAPI dependency resolves the slug to a merchant through a `PLATFORM_ADMIN` session (the only cross-tenant query), maps `X-Tenant-Role` to `TENANT_` or `TENANT__ADMIN`, and leases that role's session for the request. +- **Sessions.** `RoleSessionCache` keeps at most `TB_MAX_ROLE_CONNECTIONS` sessions (default 16), one per role, evicting the least recently used; a leased session is closed only when its lease ends. One session per role instead of `USE ROLE` per request means a failed request can never leave a shared session on the wrong tenant. +- **Statements.** All parameters are qmark binds. `QUERY_TAG` is set per statement (`tenantboard:`), so concurrent requests share a session without `ALTER SESSION` races. +- **Result cache.** Sessions set `USE_CACHED_RESULT = TRUE`; each session is a single tenant role, so a reused result is always that tenant's own. + +## Test isolation + +```mermaid +sequenceDiagram + autonumber + participant C as pytest controller + participant W as xdist worker + participant S as Snowflake + C->>S: provision tenant roles (incremental) + C->>S: CREATE DATABASE TB_TEMPLATE from init/*.sf.sql, load the seed + C->>W: start 4 workers + W->>S: CREATE DATABASE TB_TEST_GWn CLONE TB_TEMPLATE + W->>S: apply the template's policy references and grants + W->>S: tests (API, raw role sessions, scratch clones) + W->>S: DROP DATABASE TB_TEST_GWn + C->>S: DROP DATABASE TB_TEMPLATE +``` + +Each worker database is provisioned from the template's `POLICY_REFERENCES` and grants file, so every test database carries exactly the protection production has. The step is driven by the template, not by a list in code: a policy missing from `init/` is missing from every clone, and the leak test fails. + +Uploads write only to tenant `INITECH` with order ids unique per test, so tests are order-independent within a worker. Tests that modify policies work on their own scratch clones. + +## Error catalog + +Every error is `application/problem+json` with `type`, `title`, `status`, `detail` and `instance`. Snowflake errors add `snowflake: {error_code, sql_state, query_id}`. + +| Status | `type` suffix | When | +| --- | --- | --- | +| 404 | `tenant-not-found` | No merchant with that slug | +| 422 | `invalid-request` | Path, query or header validation | +| 422 | `invalid-upload` | CSV header, types or duplicates | +| 422 | `constraint-violation` | Snowflake `100072`: `NULL` in a `NOT NULL` column; nothing applied | +| 403 | `admin-role-required` | Upload as viewer | +| 403 | `tenant-role-unavailable` | Snowflake `390189`: the tenant's role is not available to the service user | +| 403 | `not-authorized` | Snowflake `002003` / `003001` | +| 409 | `duplicate-merge-source` | Snowflake `100090`: several rows target one order | +| 413 | `upload-too-large` | Above `TB_MAX_UPLOAD_BYTES` | +| 502 | `snowflake-error` | Any other Snowflake error | diff --git a/tenantboard/init/01_platform.sf.sql b/tenantboard/init/01_platform.sf.sql new file mode 100644 index 0000000..131944c --- /dev/null +++ b/tenantboard/init/01_platform.sf.sql @@ -0,0 +1,42 @@ +-- Tenantboard: account objects (warehouse, shared roles, service user) and the +-- TENANTBOARD database. +-- +-- Account-level objects are created as ACCOUNTADMIN. +USE ROLE ACCOUNTADMIN; + +CREATE WAREHOUSE IF NOT EXISTS TB_WH + WAREHOUSE_SIZE = XSMALL + AUTO_SUSPEND = 60 + INITIALLY_SUSPENDED = TRUE; + +-- Role model. Per-tenant roles are created when a tenant is provisioned +-- (app/provisioning.py), never by hand: +-- TENANT_ viewer role of one tenant; the row access policy keys on its name +-- TENANT__ADMIN that tenant's admin: inherits TENANT_ and TENANT_ADMIN +-- TENANT_READER shared read grants, inherited by every TENANT_ +-- TENANT_ADMIN shared write grants; the email masking policy checks it +-- PLATFORM_ADMIN platform operators: every tenant's rows, emails still masked +CREATE ROLE IF NOT EXISTS TENANT_READER; +CREATE ROLE IF NOT EXISTS TENANT_ADMIN; +CREATE ROLE IF NOT EXISTS PLATFORM_ADMIN; +-- Custom roles roll up to SYSADMIN, Snowflake's recommended hierarchy. +GRANT ROLE PLATFORM_ADMIN TO ROLE SYSADMIN; + +-- The API's service user. It holds PLATFORM_ADMIN (tenant directory only) and +-- each tenant's two roles, and opens one session per role it serves. On real +-- Snowflake, register the public key with: +-- ALTER USER TENANTBOARD_SVC SET RSA_PUBLIC_KEY = ''; +CREATE USER IF NOT EXISTS TENANTBOARD_SVC + DEFAULT_WAREHOUSE = TB_WH + COMMENT = 'Tenantboard API service user (key-pair auth)'; +GRANT ROLE PLATFORM_ADMIN TO USER TENANTBOARD_SVC; +GRANT USAGE ON WAREHOUSE TB_WH TO ROLE TENANT_READER; +GRANT USAGE ON WAREHOUSE TB_WH TO ROLE PLATFORM_ADMIN; + +CREATE DATABASE IF NOT EXISTS TENANTBOARD; +USE DATABASE TENANTBOARD; + +-- @deploy: everything below this line is database-relative. The test suite +-- builds its template database from these files in order. + +CREATE SCHEMA IF NOT EXISTS APP; diff --git a/tenantboard/init/02_policies.sf.sql b/tenantboard/init/02_policies.sf.sql new file mode 100644 index 0000000..8a0675a --- /dev/null +++ b/tenantboard/init/02_policies.sf.sql @@ -0,0 +1,25 @@ +-- Tenantboard: tenant isolation and PII masking, enforced by Snowflake. +-- The policies exist before the tables, which attach them at creation +-- (03_tables.sf.sql), so no table is ever readable without them. +USE ROLE ACCOUNTADMIN; +USE DATABASE TENANTBOARD; + +-- @deploy: everything below this line is database-relative. + +USE SCHEMA APP; + +-- A session sees a row only when the tenant's own role is active in it +-- (directly or inherited, so TENANT__ADMIN qualifies), or when it is a +-- platform operator. The app's WHERE TENANT_ID = ? filter is a second, +-- independent layer: the leak test removes it and this policy alone holds. +CREATE ROW ACCESS POLICY IF NOT EXISTS TENANT_ISOLATION + AS (TENANT_ID VARCHAR) RETURNS BOOLEAN -> + IS_ROLE_IN_SESSION('PLATFORM_ADMIN') OR IS_ROLE_IN_SESSION('TENANT_' || TENANT_ID) + COMMENT = 'Row visible to TENANT_ (and roles inheriting it) and PLATFORM_ADMIN'; + +-- Customer email is clear only for tenant admins (TENANT__ADMIN inherits +-- TENANT_ADMIN). Viewers and platform operators see ***MASKED***. +CREATE MASKING POLICY IF NOT EXISTS EMAIL_MASK + AS (VAL VARCHAR) RETURNS VARCHAR -> + CASE WHEN IS_ROLE_IN_SESSION('TENANT_ADMIN') THEN VAL ELSE '***MASKED***' END + COMMENT = 'Clear email for TENANT_ADMIN only'; diff --git a/tenantboard/init/03_tables.sf.sql b/tenantboard/init/03_tables.sf.sql new file mode 100644 index 0000000..c45cfcd --- /dev/null +++ b/tenantboard/init/03_tables.sf.sql @@ -0,0 +1,59 @@ +-- Tenantboard: tenant tables, each created with its policies attached, so no +-- table ever exists, even briefly, without its row access and masking policies. +USE ROLE ACCOUNTADMIN; +USE DATABASE TENANTBOARD; + +-- @deploy: everything below this line is database-relative. + +USE SCHEMA APP; + +CREATE TABLE IF NOT EXISTS MERCHANTS ( + TENANT_ID VARCHAR(64) NOT NULL PRIMARY KEY, + SLUG VARCHAR(64) NOT NULL UNIQUE, + NAME VARCHAR(200) NOT NULL, + PLAN VARCHAR(20) NOT NULL, + REGION VARCHAR(20) NOT NULL, + CURRENCY VARCHAR(3) NOT NULL, + CREATED_AT TIMESTAMP_NTZ NOT NULL +) +WITH ROW ACCESS POLICY TENANT_ISOLATION ON (TENANT_ID) +COMMENT = 'One row per tenant (merchant) of the platform'; + +CREATE TABLE IF NOT EXISTS CUSTOMERS ( + TENANT_ID VARCHAR(64) NOT NULL, + CUSTOMER_ID NUMBER(10,0) NOT NULL, + NAME VARCHAR(200) NOT NULL, + EMAIL VARCHAR(320) NOT NULL WITH MASKING POLICY EMAIL_MASK, + COUNTRY VARCHAR(2) NOT NULL, + CREATED_AT TIMESTAMP_NTZ NOT NULL, + PRIMARY KEY (TENANT_ID, CUSTOMER_ID) +) +WITH ROW ACCESS POLICY TENANT_ISOLATION ON (TENANT_ID) +COMMENT = 'A merchant''s buyers; EMAIL is PII'; + +-- Order ids are unique per tenant only: every merchant has an ORD-000001. +CREATE TABLE IF NOT EXISTS ORDERS ( + TENANT_ID VARCHAR(64) NOT NULL, + ORDER_ID VARCHAR(32) NOT NULL, + CUSTOMER_ID NUMBER(10,0) NOT NULL, + ORDERED_AT TIMESTAMP_NTZ NOT NULL, + STATUS VARCHAR(20) NOT NULL, + CHANNEL VARCHAR(20) NOT NULL, + CURRENCY VARCHAR(3) NOT NULL, + TOTAL_AMOUNT NUMBER(12,2) NOT NULL, + PRIMARY KEY (TENANT_ID, ORDER_ID) +) +WITH ROW ACCESS POLICY TENANT_ISOLATION ON (TENANT_ID) +COMMENT = 'Orders of every merchant'; + +CREATE TABLE IF NOT EXISTS LINE_ITEMS ( + TENANT_ID VARCHAR(64) NOT NULL, + ORDER_ID VARCHAR(32) NOT NULL, + LINE_NO NUMBER(4,0) NOT NULL, + SKU VARCHAR(32) NOT NULL, + QUANTITY NUMBER(6,0) NOT NULL, + UNIT_PRICE NUMBER(12,2) NOT NULL, + PRIMARY KEY (TENANT_ID, ORDER_ID, LINE_NO) +) +WITH ROW ACCESS POLICY TENANT_ISOLATION ON (TENANT_ID) +COMMENT = 'Order lines'; diff --git a/tenantboard/init/04_grants.sf.sql b/tenantboard/init/04_grants.sf.sql new file mode 100644 index 0000000..118e562 --- /dev/null +++ b/tenantboard/init/04_grants.sf.sql @@ -0,0 +1,31 @@ +-- Tenantboard: least-privilege grants for the shared roles. +USE ROLE ACCOUNTADMIN; +USE DATABASE TENANTBOARD; +GRANT USAGE ON DATABASE TENANTBOARD TO ROLE TENANT_READER; +GRANT USAGE ON DATABASE TENANTBOARD TO ROLE PLATFORM_ADMIN; + +-- @deploy: everything below this line is database-relative. It is also applied +-- to every clone (app/provisioning.py), so each test database carries exactly +-- the grants production has. + +USE SCHEMA APP; + +-- Tenant viewers read the tenant tables; the row access policy decides which rows. +GRANT USAGE ON SCHEMA APP TO ROLE TENANT_READER; +GRANT SELECT ON TABLE MERCHANTS TO ROLE TENANT_READER; +GRANT SELECT ON TABLE CUSTOMERS TO ROLE TENANT_READER; +GRANT SELECT ON TABLE ORDERS TO ROLE TENANT_READER; +GRANT SELECT ON TABLE LINE_ITEMS TO ROLE TENANT_READER; + +-- Tenant admins upload offline orders: write_pandas stages a temporary table, +-- then one MERGE writes ORDERS (still subject to the row access policy). +GRANT CREATE TABLE, CREATE STAGE, CREATE FILE FORMAT ON SCHEMA APP TO ROLE TENANT_ADMIN; +GRANT INSERT, UPDATE ON TABLE ORDERS TO ROLE TENANT_ADMIN; + +-- Platform operators read every tenant's rows (the policy lets PLATFORM_ADMIN +-- through) for the tenant directory and support, with customer email masked. +GRANT USAGE ON SCHEMA APP TO ROLE PLATFORM_ADMIN; +GRANT SELECT ON TABLE MERCHANTS TO ROLE PLATFORM_ADMIN; +GRANT SELECT ON TABLE CUSTOMERS TO ROLE PLATFORM_ADMIN; +GRANT SELECT ON TABLE ORDERS TO ROLE PLATFORM_ADMIN; +GRANT SELECT ON TABLE LINE_ITEMS TO ROLE PLATFORM_ADMIN; diff --git a/tenantboard/pyproject.toml b/tenantboard/pyproject.toml new file mode 100644 index 0000000..a97d474 --- /dev/null +++ b/tenantboard/pyproject.toml @@ -0,0 +1,45 @@ +[project] +name = "tenantboard" +version = "0.1.0" +description = "A multi-tenant analytics API where tenant isolation and PII masking are enforced by Snowflake policies" +readme = "README.md" +license = "Apache-2.0" +requires-python = "==3.12.*" +dependencies = [ + "fastapi>=0.115,<1", + "uvicorn[standard]>=0.32,<1", + "pydantic>=2.9,<3", + "pydantic-settings>=2.6,<3", + "python-multipart>=0.0.12", + "snowflake-connector-python[aio,pandas]>=5.1,<6", + "cryptography>=43", +] + +[dependency-groups] +dev = [ + "pytest>=8.3,<9", + "pytest-xdist>=3.6,<4", + "httpx>=0.27,<1", +] + +[tool.uv] +package = false + +[tool.pytest.ini_options] +testpaths = ["tests"] +pythonpath = ["."] +addopts = "-ra" +filterwarnings = [ + # starlette's TestClient announces a future httpx successor; nothing to act on yet. + "ignore::DeprecationWarning:fastapi.testclient", + "ignore:Using `httpx` with `starlette.testclient` is deprecated", +] + +[tool.ruff] +line-length = 120 +target-version = "py312" + +[tool.ruff.lint] +select = ["E", "F", "W", "B", "UP", "I"] +# The formatter wraps code at line-length; long SQL strings stay whole. +ignore = ["E501"] diff --git a/tenantboard/scripts/__init__.py b/tenantboard/scripts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tenantboard/scripts/demo.py b/tenantboard/scripts/demo.py new file mode 100644 index 0000000..03ad2a4 --- /dev/null +++ b/tenantboard/scripts/demo.py @@ -0,0 +1,92 @@ +"""`make demo`: the leak test, narrated. + +Runs a raw `SELECT * FROM ORDERS` (no WHERE clause, no app code) under +tenant roles, shows customer email masking per role, then detaches the row +access policy and runs the same SELECT again. Everything happens in TB_DEMO, a +clone of TENANTBOARD, which is dropped at the end. +""" + +from __future__ import annotations + +import sys +import time +from collections import Counter + +from app import provisioning +from app.db import connect_admin, connect_sync +from app.settings import Settings, get_settings +from app.tenancy import PLATFORM_ROLE, admin_role, viewer_role +from scripts import seed + +DEMO_DB = "TB_DEMO" +RAW_SELECT = "SELECT * FROM ORDERS" + + +def tenants_seen(settings: Settings, role: str) -> Counter: + with connect_sync( + settings, user=settings.tb_app_user, role=role, database=DEMO_DB, USE_CACHED_RESULT=False + ) as conn: + with conn.cursor() as cur: + cur.execute(RAW_SELECT) + column = [d.name for d in cur.description].index("TENANT_ID") + return Counter(row[column] for row in cur.fetchall()) + + +def describe(seen: Counter) -> str: + tenants = ", ".join(sorted(seen)[:3]) + (", ..." if len(seen) > 3 else "") + return f"{sum(seen.values()):4d} rows from {len(seen):2d} tenant(s): {tenants}" + + +def emails(settings: Settings, role: str) -> list[str]: + with connect_sync(settings, user=settings.tb_app_user, role=role, database=DEMO_DB) as conn: + with conn.cursor() as cur: + cur.execute("SELECT EMAIL FROM CUSTOMERS ORDER BY CUSTOMER_ID LIMIT 2") + return [row[0] for row in cur.fetchall()] + + +def main() -> int: + settings = get_settings() + started = time.monotonic() + with connect_admin(settings) as admin: + with connect_sync( + settings, user=settings.tb_app_user, role=PLATFORM_ROLE, database=settings.tb_database + ) as platform: + merchants = provisioning.query(platform, "SELECT COUNT(*) FROM MERCHANTS")[0][0] + if merchants == 0: + print(f"{settings.tb_database} is empty; seeding it first (make seed)") + seed.seed_database(settings.tb_database, seed.generate()) + + print(f"[1/5] CREATE DATABASE {DEMO_DB} CLONE {settings.tb_database}") + provisioning.clone_database(admin, settings.tb_database, DEMO_DB) + try: + policy = f"{DEMO_DB}.APP.TENANT_ISOLATION" + print(f"\n[2/5] {RAW_SELECT} -- no WHERE clause, run under each role") + for role in (viewer_role("ACME"), viewer_role("GLOBEX"), admin_role("ACME"), PLATFORM_ROLE): + print(f" as {role:<18} {describe(tenants_seen(settings, role))}") + + print("\n[3/5] SELECT EMAIL FROM CUSTOMERS -- masking policy EMAIL_MASK") + for role in (viewer_role("ACME"), admin_role("ACME")): + print(f" as {role:<18} {', '.join(emails(settings, role))}") + + print("\n[4/5] ALTER TABLE ORDERS DROP ROW ACCESS POLICY TENANT_ISOLATION -- simulate a missing policy") + provisioning.query(admin, f"ALTER TABLE {DEMO_DB}.APP.ORDERS DROP ROW ACCESS POLICY {policy}") + leaked = tenants_seen(settings, viewer_role("ACME")) + print(f" as {viewer_role('ACME'):<18} {describe(leaked)} <-- LEAK: every tenant's orders") + + print("\n[5/5] ALTER TABLE ORDERS ADD ROW ACCESS POLICY TENANT_ISOLATION ON (TENANT_ID)") + provisioning.query(admin, f"ALTER TABLE {DEMO_DB}.APP.ORDERS ADD ROW ACCESS POLICY {policy} ON (TENANT_ID)") + restored = tenants_seen(settings, viewer_role("ACME")) + print(f" as {viewer_role('ACME'):<18} {describe(restored)}") + finally: + provisioning.drop_database(admin, DEMO_DB) + + isolated = set(restored) == {"ACME"} and len(leaked) > 1 + print( + f"\n{'Isolation holds' if isolated else 'UNEXPECTED RESULT'}: the policy, not the app, decides which rows a tenant session can read." + f" ({time.monotonic() - started:.1f}s, {DEMO_DB} dropped)" + ) + return 0 if isolated else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tenantboard/scripts/run.sh b/tenantboard/scripts/run.sh new file mode 100755 index 0000000..0d62c21 --- /dev/null +++ b/tenantboard/scripts/run.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# `make run`: the API on :API_PORT (default 8000) and the dashboard's Vite dev +# server on :WEB_PORT (default 5173), which proxies /api to the API. +# Ctrl-C (or either process exiting) stops both. +set -euo pipefail +cd "$(dirname "${BASH_SOURCE[0]}")/.." + +api_port="${API_PORT:-8000}" +web_port="${WEB_PORT:-5173}" +pids=() + +cleanup() { + trap - EXIT INT TERM + for pid in "${pids[@]}"; do + kill "$pid" 2>/dev/null || true + done + wait 2>/dev/null || true +} +trap cleanup EXIT INT TERM + +uv run --frozen uvicorn app.main:app --host 127.0.0.1 --port "$api_port" & +pids+=($!) + +deadline=$((SECONDS + 60)) +until curl -fsS -o /dev/null "http://127.0.0.1:${api_port}/api/health" 2>/dev/null; do + if (( SECONDS >= deadline )); then + echo "API did not answer /api/health within 60s (is the emulator up? run 'make up' first)." >&2 + exit 1 + fi + sleep 1 +done + +(cd web && exec npx vite --port "$web_port") & +pids+=($!) + +echo +echo " API http://localhost:${api_port}/docs" +echo " Dashboard http://localhost:${web_port}" +echo + +wait -n diff --git a/tenantboard/scripts/seed.py b/tenantboard/scripts/seed.py new file mode 100644 index 0000000..943c3fa --- /dev/null +++ b/tenantboard/scripts/seed.py @@ -0,0 +1,271 @@ +"""Seeded synthetic data for Tenantboard: merchants, customers, orders and line items. + +Everything is a pure function of (seed, merchant count), so the test suite can +regenerate the exact dataset it loaded and compute expected values from it +independently of the API and of Snowflake. + + uv run python -m scripts.seed # load into TENANTBOARD (idempotent) + uv run python -m scripts.seed --merchants 10 +""" + +from __future__ import annotations + +import argparse +import random +import time +from dataclasses import dataclass +from datetime import datetime, timedelta + +import pandas as pd + +from app import provisioning, tenancy +from app.db import connect_admin +from app.settings import get_settings + +SEED = 42 +MERCHANTS = 50 +# Order timestamps fall in [WINDOW_START, WINDOW_START + WINDOW_DAYS). +WINDOW_START = datetime(2026, 9, 1) +WINDOW_DAYS = 30 +# Volumes per merchant, by plan: (customers, orders). Small enough that the +# whole suite, one template plus a clone per worker, runs in about a minute. +VOLUME = {"starter": ((4, 8), (5, 9)), "growth": ((8, 14), (10, 16)), "enterprise": ((14, 22), (18, 28))} +STATUSES = (("completed", 82), ("refunded", 6), ("cancelled", 12)) +CHANNELS = (("web", 55), ("mobile", 30), ("marketplace", 15)) +PLANS = ("starter", "growth", "enterprise") +REGIONS = {"us-east": "USD", "eu-west": "EUR", "uk-south": "GBP"} + +# The first merchants are fixed (slug, name, plan) so demos and tests can name them. +FIXED_MERCHANTS = ( + ("acme", "Acme Corporation", "enterprise"), + ("globex", "Globex Corporation", "growth"), + ("initech", "Initech", "starter"), +) +_ADJECTIVES = ( + "Amber", + "Birch", + "Blue", + "Cedar", + "Copper", + "Coral", + "Crimson", + "Delta", + "Ember", + "Fable", + "Granite", + "Harbor", + "Indigo", + "Juniper", + "Lumen", + "Maple", + "Meadow", + "North", + "Oak", + "Onyx", +) +_NOUNS = ( + "Goods", + "Supply", + "Outfitters", + "Market", + "Works", + "Trading", + "Provisions", + "Mercantile", + "Collective", + "Studio", + "Depot", + "Atelier", +) +_FIRST = ( + "ada", + "ben", + "chloe", + "dev", + "elena", + "farid", + "grace", + "hiro", + "ines", + "jonas", + "kai", + "lena", + "mateo", + "nora", + "omar", + "priya", + "quinn", + "rosa", + "sam", + "tara", + "uma", + "viktor", + "wen", + "yara", +) +_LAST = ( + "adams", + "baker", + "chen", + "diaz", + "evans", + "fischer", + "garcia", + "haddad", + "ito", + "jensen", + "khan", + "larsen", + "moreau", + "nakamura", + "okafor", + "patel", + "rossi", + "silva", + "tanaka", + "weber", +) +_COUNTRIES = ("US", "CA", "GB", "DE", "FR", "NL", "ES", "IT", "SE", "JP") + + +@dataclass(frozen=True) +class Dataset: + merchants: pd.DataFrame + customers: pd.DataFrame + orders: pd.DataFrame + line_items: pd.DataFrame + + @property + def tenant_ids(self) -> list[str]: + return self.merchants["TENANT_ID"].tolist() + + def frames(self) -> dict[str, pd.DataFrame]: + return { + "MERCHANTS": self.merchants, + "CUSTOMERS": self.customers, + "ORDERS": self.orders, + "LINE_ITEMS": self.line_items, + } + + +def tenant_id_for(slug: str) -> str: + return slug.upper().replace("-", "_") + + +def _weighted(rng: random.Random, choices: tuple[tuple[str, int], ...]) -> str: + return rng.choices([c for c, _ in choices], weights=[w for _, w in choices])[0] + + +def _merchants(rng: random.Random, count: int) -> list[tuple[str, str, str]]: + merchants = list(FIXED_MERCHANTS[:count]) + combos = [(a, n) for a in _ADJECTIVES for n in _NOUNS] + rng.shuffle(combos) + for adjective, noun in combos[: max(0, count - len(merchants))]: + merchants.append((f"{adjective}-{noun}".lower(), f"{adjective} {noun}", rng.choice(PLANS))) + return merchants + + +def generate(seed: int = SEED, merchants: int = MERCHANTS) -> Dataset: + if not 3 <= merchants <= len(_ADJECTIVES) * len(_NOUNS): + raise ValueError("merchants must be between 3 and 240") + rng = random.Random(seed) + merchant_rows, customer_rows, order_rows, line_rows = [], [], [], [] + for index, (slug, name, plan) in enumerate(_merchants(rng, merchants)): + tenant_id = tenancy.validate_tenant_id(tenant_id_for(slug)) + region = rng.choice(list(REGIONS)) + currency = REGIONS[region] + created_at = datetime(2025, 1, 6) + timedelta(days=index * 5) + merchant_rows.append((tenant_id, slug, name, plan, region, currency, created_at)) + + (min_customers, max_customers), (min_orders, max_orders) = VOLUME[plan] + customer_count = rng.randint(min_customers, max_customers) + for customer_id in range(1, customer_count + 1): + first, last = rng.choice(_FIRST), rng.choice(_LAST) + customer_rows.append( + ( + tenant_id, + customer_id, + f"{first.title()} {last.title()}", + f"{first}.{last}.{customer_id}@{slug}-buyers.example", + rng.choice(_COUNTRIES), + created_at + timedelta(days=rng.randrange(180)), + ) + ) + + for number in range(1, rng.randint(min_orders, max_orders) + 1): + order_id = f"ORD-{number:06d}" + ordered_at = WINDOW_START + timedelta(seconds=rng.randrange(WINDOW_DAYS * 86_400)) + total_cents = 0 + for line_no in range(1, rng.randint(1, 3) + 1): + quantity = rng.randint(1, 5) + unit_cents = rng.randrange(299, 19_999) + total_cents += quantity * unit_cents + line_rows.append( + (tenant_id, order_id, line_no, f"SKU-{rng.randrange(1, 400):04d}", quantity, unit_cents / 100) + ) + order_rows.append( + ( + tenant_id, + order_id, + rng.randint(1, customer_count), + ordered_at, + _weighted(rng, STATUSES), + _weighted(rng, CHANNELS), + currency, + total_cents / 100, + ) + ) + + return Dataset( + merchants=pd.DataFrame( + merchant_rows, columns=["TENANT_ID", "SLUG", "NAME", "PLAN", "REGION", "CURRENCY", "CREATED_AT"] + ), + customers=pd.DataFrame( + customer_rows, columns=["TENANT_ID", "CUSTOMER_ID", "NAME", "EMAIL", "COUNTRY", "CREATED_AT"] + ), + orders=pd.DataFrame( + order_rows, + columns=[ + "TENANT_ID", + "ORDER_ID", + "CUSTOMER_ID", + "ORDERED_AT", + "STATUS", + "CHANNEL", + "CURRENCY", + "TOTAL_AMOUNT", + ], + ), + line_items=pd.DataFrame( + line_rows, columns=["TENANT_ID", "ORDER_ID", "LINE_NO", "SKU", "QUANTITY", "UNIT_PRICE"] + ), + ) + + +def seed_database(database: str, dataset: Dataset) -> tuple[int, dict[str, int]]: + """Provision every tenant's roles and replace the tables' contents with `dataset`.""" + settings = get_settings() + with connect_admin(settings) as conn: + provisioned = provisioning.provision_tenants(conn, settings, dataset.tenant_ids) + return provisioned, provisioning.load_tables(conn, database, dataset.frames()) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--merchants", type=int, default=MERCHANTS) + parser.add_argument("--seed", type=int, default=SEED) + args = parser.parse_args() + settings = get_settings() + started = time.monotonic() + dataset = generate(args.seed, args.merchants) + provisioned, loaded = seed_database(settings.tb_database, dataset) + summary = ", ".join(f"{table} {rows}" for table, rows in loaded.items()) + print(f"Seeded {settings.tb_database}: {summary} ({time.monotonic() - started:.1f}s)") + print( + f"Tenant roles TENANT_ and TENANT__ADMIN: {len(dataset.tenant_ids)} merchants, " + f"{provisioned} newly provisioned, granted to {settings.tb_app_user}" + ) + + +if __name__ == "__main__": + main() diff --git a/tenantboard/scripts/wait-for-emulator.sh b/tenantboard/scripts/wait-for-emulator.sh new file mode 100755 index 0000000..501c7e2 --- /dev/null +++ b/tenantboard/scripts/wait-for-emulator.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# Wait until the Snowflake-Next emulator answers a SQL statement. +# +# Copy this file into a sample as scripts/wait-for-emulator.sh (samples stay +# self-contained). Usage: scripts/wait-for-emulator.sh [timeout-seconds] +# Reads SNOWFLAKE_PORT (default 4566). Readiness is checked from the host. +set -euo pipefail + +port="${SNOWFLAKE_PORT:-4566}" +timeout="${1:-120}" +url="https://snowflake.localhost.localstack.cloud:${port}" +deadline=$((SECONDS + timeout)) + +until curl -fsS --compressed -o /dev/null -X POST "${url}/api/v2/statements" \ + -H 'Authorization: Bearer local' -H 'Content-Type: application/json' \ + -H 'Accept: application/json' -d '{"statement":"SELECT 1"}' 2>/dev/null; do + if (( SECONDS >= deadline )); then + echo "Emulator at ${url} did not become ready within ${timeout}s." >&2 + echo "Check 'docker compose logs snowflake' (is LOCALSTACK_AUTH_TOKEN valid? is port ${port} free?)." >&2 + exit 1 + fi + sleep 2 +done +echo "Emulator ready at ${url}" diff --git a/tenantboard/scripts/wait_for_init.py b/tenantboard/scripts/wait_for_init.py new file mode 100644 index 0000000..78a24da --- /dev/null +++ b/tenantboard/scripts/wait_for_init.py @@ -0,0 +1,53 @@ +"""Wait until the init hooks have created TENANTBOARD with its policies and grants. + +The emulator accepts connections before it has finished applying +init/*.sf.sql, so `make up` waits for the end result: ORDERS with its row +access policy and its reader grant. +""" + +from __future__ import annotations + +import sys +import time + +from snowflake.connector import DictCursor, SnowflakeConnection +from snowflake.connector.errors import Error + +from app import provisioning +from app.db import connect_admin +from app.settings import get_settings + +TIMEOUT_SECONDS = 90 + + +def ready(conn: SnowflakeConnection) -> bool: + """ORDERS exists with its row access policy (03_tables) and its reader grant (04_grants, the last file).""" + database = get_settings().tb_database + try: + refs = provisioning.policy_refs(conn, database, "ORDERS") + with conn.cursor(DictCursor) as cur: + cur.execute(f"SHOW GRANTS ON TABLE {database}.APP.ORDERS") + grants = {(g["privilege"], g["grantee_name"]) for g in cur.fetchall()} + except Error: + return False + return any(r.kind == "ROW_ACCESS_POLICY" for r in refs) and ("SELECT", "TENANT_READER") in grants + + +def main() -> int: + deadline = time.monotonic() + TIMEOUT_SECONDS + with connect_admin(get_settings()) as conn: + while time.monotonic() < deadline: + if ready(conn): + print("Init hooks applied: TENANTBOARD.APP tables, policies and grants are in place") + return 0 + time.sleep(1) + print( + f"TENANTBOARD was not fully initialised within {TIMEOUT_SECONDS}s. Check the init hook output:\n" + " docker compose -p snfs-tenantboard logs snowflake | grep init_hooks", + file=sys.stderr, + ) + return 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tenantboard/tests/__init__.py b/tenantboard/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tenantboard/tests/conftest.py b/tenantboard/tests/conftest.py new file mode 100644 index 0000000..e57ed2e --- /dev/null +++ b/tenantboard/tests/conftest.py @@ -0,0 +1,102 @@ +"""Fixtures: one template database per run, one CLONE of it per xdist worker. + +The controller process (or the only process, without xdist) builds +TB_TEMPLATE from init/*.sf.sql and the seeded dataset before any worker +starts, and drops it at the end. Each worker then clones it into +TB_TEST_, points its own API instance at the clone, and drops the clone +when its session ends. Expected values come from regenerating the same seeded +dataset in Python, never from the code under test. +""" + +from __future__ import annotations + +from collections.abc import Callable, Iterator + +import pytest +import snowflake.connector +from fastapi.testclient import TestClient + +from app import provisioning +from app.db import connect_admin, connect_sync +from app.main import create_app +from app.settings import Settings +from scripts import seed + +TEMPLATE_DB = "TB_TEMPLATE" + + +def _is_controller(config: pytest.Config) -> bool: + return not hasattr(config, "workerinput") + + +def pytest_sessionstart(session: pytest.Session) -> None: + if not _is_controller(session.config): + return + settings = Settings() + dataset = seed.generate() + with connect_admin(settings) as conn: + provisioning.provision_tenants(conn, settings, dataset.tenant_ids) + provisioning.deploy_database(conn, TEMPLATE_DB) + provisioning.load_tables(conn, TEMPLATE_DB, dataset.frames()) + + +def pytest_sessionfinish(session: pytest.Session) -> None: + if not _is_controller(session.config): + return + with connect_admin(Settings()) as conn: + provisioning.drop_database(conn, TEMPLATE_DB) + + +@pytest.fixture(scope="session") +def dataset() -> seed.Dataset: + return seed.generate() + + +@pytest.fixture(scope="session") +def worker_db(request: pytest.FixtureRequest) -> Iterator[str]: + """This worker's own database: a CLONE of the template, with the template's policies and grants.""" + worker = getattr(request.config, "workerinput", {}).get("workerid", "main") + name = f"TB_TEST_{worker.upper()}" + with connect_admin(Settings()) as conn: + provisioning.clone_database(conn, TEMPLATE_DB, name) + yield name + with connect_admin(Settings()) as conn: + provisioning.drop_database(conn, name) + + +@pytest.fixture(scope="session") +def settings(worker_db: str) -> Settings: + return Settings(tb_database=worker_db) + + +@pytest.fixture(scope="session") +def admin(settings: Settings) -> Iterator[snowflake.connector.SnowflakeConnection]: + with connect_admin(settings, database=settings.tb_database) as conn: + yield conn + + +@pytest.fixture(scope="session") +def client(settings: Settings) -> Iterator[TestClient]: + with TestClient(create_app(settings)) as test_client: + yield test_client + + +@pytest.fixture(scope="session") +def connect_as(settings: Settings) -> Iterator[Callable[[str], snowflake.connector.SnowflakeConnection]]: + """Raw sessions of the app's service user under a given role, outside the API. + + USE_CACHED_RESULT is off so every assertion is computed from storage under + the session's own role, not served from a result an earlier test cached. + """ + opened: dict[str, snowflake.connector.SnowflakeConnection] = {} + + def factory(role: str) -> snowflake.connector.SnowflakeConnection: + if role not in opened: + opened[role] = connect_sync( + settings, user=settings.tb_app_user, role=role, database=settings.tb_database, USE_CACHED_RESULT=False + ) + return opened[role] + + yield factory + for conn in opened.values(): + conn.close() diff --git a/tenantboard/tests/helpers.py b/tenantboard/tests/helpers.py new file mode 100644 index 0000000..5349bd1 --- /dev/null +++ b/tenantboard/tests/helpers.py @@ -0,0 +1,75 @@ +"""Shared test helpers and the independent oracle. + +Expected values are computed here, in plain pandas and Decimal arithmetic over +the seeded dataset, without touching the API or Snowflake. +""" + +from __future__ import annotations + +from datetime import date +from decimal import Decimal +from typing import Any + +import pandas as pd +import snowflake.connector + +from scripts.seed import Dataset + + +def dict_rows( + conn: snowflake.connector.SnowflakeConnection, sql: str, params: list[Any] | None = None +) -> list[dict[str, Any]]: + with conn.cursor(snowflake.connector.DictCursor) as cur: + cur.execute(sql, params) + return list(cur.fetchall()) + + +def money(value: float) -> Decimal: + """Amounts are generated as cents / 100; str() of such a float is its exact 2-decimal form.""" + return Decimal(str(value)).quantize(Decimal("0.01")) + + +def orders_of(ds: Dataset, tenant_id: str) -> pd.DataFrame: + return ds.orders[ds.orders["TENANT_ID"] == tenant_id] + + +def completed_of(ds: Dataset, tenant_id: str) -> pd.DataFrame: + orders = orders_of(ds, tenant_id) + return orders[orders["STATUS"] == "completed"] + + +def expected_summary(ds: Dataset, tenant_id: str) -> dict[str, Any]: + orders, completed = orders_of(ds, tenant_id), completed_of(ds, tenant_id) + revenue = sum((money(v) for v in completed["TOTAL_AMOUNT"]), Decimal("0.00")) + return { + "orders": len(orders), + "completed_orders": len(completed), + "customers": orders["CUSTOMER_ID"].nunique(), + "revenue": revenue, + "average_order_value": (revenue / len(completed)).quantize(Decimal("0.01")), + } + + +def expected_daily_revenue(ds: Dataset, tenant_id: str) -> dict[date, tuple[int, Decimal]]: + out: dict[date, tuple[int, Decimal]] = {} + for _, row in completed_of(ds, tenant_id).iterrows(): + day = row["ORDERED_AT"].date() + count, total = out.get(day, (0, Decimal("0.00"))) + out[day] = (count + 1, total + money(row["TOTAL_AMOUNT"])) + return out + + +def expected_top_customers(ds: Dataset, tenant_id: str, limit: int) -> list[tuple[int, int, Decimal]]: + """(customer_id, orders, revenue), highest revenue first, ties by customer id.""" + totals: dict[int, tuple[int, Decimal]] = {} + for _, row in completed_of(ds, tenant_id).iterrows(): + count, total = totals.get(int(row["CUSTOMER_ID"]), (0, Decimal("0.00"))) + totals[int(row["CUSTOMER_ID"])] = (count + 1, total + money(row["TOTAL_AMOUNT"])) + ranked = sorted(totals.items(), key=lambda item: (-item[1][1], item[0])) + return [(cid, count, total) for cid, (count, total) in ranked[:limit]] + + +def email_of(ds: Dataset, tenant_id: str, customer_id: int) -> str: + customers = ds.customers + match = customers[(customers["TENANT_ID"] == tenant_id) & (customers["CUSTOMER_ID"] == customer_id)] + return str(match["EMAIL"].iloc[0]) diff --git a/tenantboard/tests/test_api.py b/tenantboard/tests/test_api.py new file mode 100644 index 0000000..60baa2e --- /dev/null +++ b/tenantboard/tests/test_api.py @@ -0,0 +1,194 @@ +"""The HTTP API: values against the independent oracle, tenant switching, masking, errors.""" + +from __future__ import annotations + +from datetime import date, timedelta +from decimal import Decimal + +import pandas as pd +import pytest + +from app import provisioning +from tests.helpers import ( + dict_rows, + email_of, + expected_daily_revenue, + expected_summary, + expected_top_customers, + orders_of, +) + +ADMIN = {"X-Tenant-Role": "admin"} +VIEWER = {"X-Tenant-Role": "viewer"} +PROBLEM_JSON = "application/problem+json" + + +def test_health_reports_the_snowflake_version(client): + response = client.get("/api/health") + + assert response.status_code == 200 + body = response.json() + assert body["status"] == "ok" + assert body["snowflake_version"][0].isdigit() + + +def test_tenant_directory_lists_every_seeded_merchant(client, dataset): + response = client.get("/api/tenants") + + assert response.status_code == 200 + merchants = response.json() + assert [m["slug"] for m in merchants] == dataset.merchants.sort_values("NAME")["SLUG"].tolist() + assert {"acme", "globex"} <= {m["slug"] for m in merchants} + + +@pytest.mark.parametrize("slug", ["acme", "globex"]) +def test_summary_matches_the_seeded_orders(client, dataset, slug): + tenant = slug.upper() + response = client.get(f"/api/tenants/{slug}/summary") + + assert response.status_code == 200 + body = response.json() + expected = expected_summary(dataset, tenant) + assert body["tenant_id"] == tenant + assert body["snowflake_role"] == f"TENANT_{tenant}" + assert (body["orders"], body["completed_orders"], body["customers"]) == ( + expected["orders"], + expected["completed_orders"], + expected["customers"], + ) + assert Decimal(body["revenue"]) == expected["revenue"] + assert Decimal(body["average_order_value"]) == expected["average_order_value"] + + +def test_switching_tenant_changes_totals_and_never_shows_the_other_tenants_orders(client, dataset): + """Through the API the dashboard uses.""" + acme = client.get("/api/tenants/acme/orders", params={"page_size": 200}).json() + globex = client.get("/api/tenants/globex/orders", params={"page_size": 200}).json() + + assert {o["tenant_id"] for o in acme["items"]} == {"ACME"} + assert {o["tenant_id"] for o in globex["items"]} == {"GLOBEX"} + assert (acme["total"], globex["total"]) == (len(orders_of(dataset, "ACME")), len(orders_of(dataset, "GLOBEX"))) + assert acme["total"] != globex["total"] + + +def test_revenue_by_day_is_gap_filled_and_matches_the_seeded_orders(client, dataset): + response = client.get("/api/tenants/acme/revenue") + + assert response.status_code == 200 + days = response.json()["days"] + expected = expected_daily_revenue(dataset, "ACME") + first, last = min(expected), max(expected) + # One point per calendar day from the first to the last completed order, zeros included. + assert [d["day"] for d in days] == [(first + timedelta(n)).isoformat() for n in range((last - first).days + 1)] + for point in days: + count, revenue = expected.get(date.fromisoformat(point["day"]), (0, Decimal("0.00"))) + assert (point["orders"], Decimal(str(point["revenue"]))) == (count, revenue), point["day"] + # Trailing 7-day mean, computed independently for the last day. + window = [expected.get(last - timedelta(n), (0, Decimal(0)))[1] for n in range(7)] + assert abs(Decimal(str(days[-1]["revenue_7d_avg"])) - sum(window) / 7) <= Decimal("0.005") + + +def test_revenue_window_includes_days_without_orders(client, dataset): + expected = expected_daily_revenue(dataset, "GLOBEX") + quiet = next(d for d in (date(2026, 9, 1) + timedelta(n) for n in range(30)) if d not in expected) + + response = client.get("/api/tenants/globex/revenue", params={"start": quiet.isoformat(), "end": quiet.isoformat()}) + + assert response.json()["days"] == [{"day": quiet.isoformat(), "orders": 0, "revenue": 0.0, "revenue_7d_avg": 0.0}] + + +def test_top_customers_match_the_seeded_orders_with_email_masked_for_viewers(client, dataset): + response = client.get("/api/tenants/acme/top-customers", params={"limit": 5}) + + assert response.status_code == 200 + got = [(c["customer_id"], c["orders"], Decimal(c["revenue"]), c["email"]) for c in response.json()] + assert got == [(cid, n, revenue, "***MASKED***") for cid, n, revenue in expected_top_customers(dataset, "ACME", 5)] + + +def test_viewer_sees_masked_email_and_admin_sees_it_clear(client, dataset): + """The same orders page under the tenant's two roles.""" + viewer = client.get("/api/tenants/acme/orders", params={"page_size": 10}, headers=VIEWER).json()["items"] + admin = client.get("/api/tenants/acme/orders", params={"page_size": 10}, headers=ADMIN).json()["items"] + + assert [o["order_id"] for o in viewer] == [o["order_id"] for o in admin] + assert {o["customer_email"] for o in viewer} == {"***MASKED***"} + assert [o["customer_email"] for o in admin] == [email_of(dataset, "ACME", o["customer_id"]) for o in admin] + + +def test_orders_pages_walk_every_order_once_newest_first(client, dataset): + expected = orders_of(dataset, "ACME").sort_values(["ORDERED_AT", "ORDER_ID"], ascending=False)["ORDER_ID"].tolist() + seen: list[str] = [] + for page in range(1, len(expected) // 10 + 2): + body = client.get("/api/tenants/acme/orders", params={"page": page, "page_size": 10}).json() + assert (body["page"], body["page_size"], body["total"]) == (page, 10, len(expected)) + seen += [o["order_id"] for o in body["items"]] + + assert seen == expected + + +def test_queries_are_tagged_per_endpoint(client, admin, settings): + client.get("/api/tenants/globex/top-customers") + + tags = dict_rows( + admin, + "SELECT QUERY_TAG, USER_NAME FROM TABLE(INFORMATION_SCHEMA.QUERY_HISTORY(RESULT_LIMIT => 1000)) " + "WHERE QUERY_TAG = ?", + ["tenantboard:/api/tenants/{slug}/top-customers"], + ) + assert tags, "no query carried the endpoint's QUERY_TAG" + assert {t["USER_NAME"] for t in tags} == {settings.tb_app_user} + + +def test_unknown_tenant_is_a_404_problem(client): + response = client.get("/api/tenants/no-such-merchant/summary") + + assert response.status_code == 404 + assert response.headers["content-type"] == PROBLEM_JSON + body = response.json() + assert (body["title"], body["status"], body["tenant"]) == ("Unknown tenant", 404, "no-such-merchant") + assert body["type"].endswith("tenant-not-found") + + +@pytest.mark.parametrize( + ("path", "headers", "field"), + [ + ("/api/tenants/Not_A_Slug/summary", {}, "slug"), + ("/api/tenants/acme/summary", {"X-Tenant-Role": "superuser"}, "X-Tenant-Role"), + ("/api/tenants/acme/orders?page_size=1000", {}, "page_size"), + ], +) +def test_invalid_request_is_a_422_problem(client, path, headers, field): + response = client.get(path, headers=headers) + + assert response.status_code == 422 + assert response.headers["content-type"] == PROBLEM_JSON + assert field in {str(e["loc"][-1]) for e in response.json()["errors"]} + + +def test_revenue_rejects_a_window_that_ends_before_it_starts(client): + response = client.get("/api/tenants/acme/revenue", params={"start": "2026-09-10", "end": "2026-09-01"}) + + assert response.status_code == 422 + assert response.json()["detail"] == "start must not be after end." + + +def test_tenant_without_provisioned_roles_surfaces_snowflake_390189(client, admin, dataset, settings): + """Error path: a merchant row exists but onboarding never created its roles. + + Snowflake refuses the session (390189); the API reports it as a 403 problem + carrying the Snowflake error code instead of a bare 500. + """ + ghost = dataset.merchants.iloc[[0]].assign(TENANT_ID="GHOST", SLUG="ghost", NAME="Ghost Merchant") + provisioning.load_tables( + admin, settings.tb_database, {"MERCHANTS": pd.concat([dataset.merchants, ghost], ignore_index=True)} + ) + try: + response = client.get("/api/tenants/ghost/summary") + finally: + provisioning.load_tables(admin, settings.tb_database, {"MERCHANTS": dataset.merchants}) + + assert response.status_code == 403 + body = response.json() + assert body["title"] == "Tenant role is not available to the service" + assert body["snowflake"]["error_code"] == "390189" + assert "TENANT_GHOST" in body["detail"] diff --git a/tenantboard/tests/test_isolation.py b/tenantboard/tests/test_isolation.py new file mode 100644 index 0000000..c4a3f2f --- /dev/null +++ b/tenantboard/tests/test_isolation.py @@ -0,0 +1,124 @@ +"""Tenant isolation and masking, proven below the API. + +The headline leak test bypasses the app-layer filter entirely: it runs a raw +`SELECT * FROM ORDERS` (no WHERE clause) as one tenant's role and must get +that tenant's rows and nothing else. Only the row access policy can make that +true. Removing the policy from init/03_tables.sf.sql makes it fail; +test_detaching_the_policy_exposes_every_tenant checks the same sensitivity on +every run. +""" + +from __future__ import annotations + +import asyncio +from collections import Counter + +import pytest + +from app import provisioning +from app.db import connect_admin, connect_sync, open_role_session +from app.models import Merchant, TenantRole +from app.repository import TenantRepository +from app.settings import Settings +from app.tenancy import PLATFORM_ROLE, admin_role, viewer_role +from tests.conftest import TEMPLATE_DB +from tests.helpers import dict_rows, email_of, expected_summary, orders_of + +TENANT_TABLES = ("MERCHANTS", "CUSTOMERS", "ORDERS", "LINE_ITEMS") + + +def _expected_rows_per_tenant(dataset, table: str) -> Counter: + frame = { + "MERCHANTS": dataset.merchants, + "CUSTOMERS": dataset.customers, + "ORDERS": dataset.orders, + "LINE_ITEMS": dataset.line_items, + }[table] + return Counter(frame["TENANT_ID"]) + + +@pytest.mark.parametrize("tenant", ["ACME", "GLOBEX"]) +def test_leak_raw_select_star_from_orders_returns_only_the_tenants_rows(connect_as, dataset, tenant): + """Headline: no WHERE clause, no app code. The row access policy alone isolates the tenant.""" + got = dict_rows(connect_as(viewer_role(tenant)), "SELECT * FROM ORDERS") + + assert {row["TENANT_ID"] for row in got} == {tenant} + assert sorted(row["ORDER_ID"] for row in got) == sorted(orders_of(dataset, tenant)["ORDER_ID"]) + + +@pytest.mark.parametrize("table", TENANT_TABLES) +def test_every_tenant_table_is_filtered_to_the_sessions_tenant(connect_as, dataset, table): + got = dict_rows( + connect_as(viewer_role("GLOBEX")), f"SELECT TENANT_ID, COUNT(*) AS N FROM {table} GROUP BY TENANT_ID" + ) + + assert {row["TENANT_ID"]: row["N"] for row in got} == { + "GLOBEX": _expected_rows_per_tenant(dataset, table)["GLOBEX"] + } + + +def test_tenant_admin_role_inherits_the_same_isolation(connect_as, dataset): + got = dict_rows(connect_as(admin_role("ACME")), "SELECT TENANT_ID, COUNT(*) AS N FROM ORDERS GROUP BY TENANT_ID") + + assert {row["TENANT_ID"]: row["N"] for row in got} == {"ACME": len(orders_of(dataset, "ACME"))} + + +def test_platform_admin_sees_every_tenant(connect_as, dataset): + got = dict_rows(connect_as(PLATFORM_ROLE), "SELECT TENANT_ID, COUNT(*) AS N FROM ORDERS GROUP BY TENANT_ID") + + assert {row["TENANT_ID"]: row["N"] for row in got} == dict(_expected_rows_per_tenant(dataset, "ORDERS")) + + +@pytest.mark.parametrize( + ("role", "clear"), + [("viewer", False), ("admin", True), ("platform", False)], +) +def test_customer_email_is_clear_only_for_the_tenants_admin(connect_as, dataset, role, clear): + session_role = {"viewer": viewer_role("ACME"), "admin": admin_role("ACME"), "platform": PLATFORM_ROLE}[role] + got = dict_rows(connect_as(session_role), "SELECT CUSTOMER_ID, EMAIL FROM CUSTOMERS WHERE TENANT_ID = ?", ["ACME"]) + + assert len(got) == int((dataset.customers["TENANT_ID"] == "ACME").sum()) + expected = [email_of(dataset, "ACME", r["CUSTOMER_ID"]) if clear else "***MASKED***" for r in got] + assert [r["EMAIL"] for r in got] == expected + + +def test_app_layer_filter_alone_isolates_the_tenant(settings: Settings, dataset): + """Layer 1 on its own: the repository under PLATFORM_ADMIN (which the policy lets see everything) + still returns only ACME's orders, because every query binds TENANT_ID.""" + merchant = Merchant( + tenant_id="ACME", slug="acme", name="Acme Corporation", plan="enterprise", region="uk-south", currency="GBP" + ) + + async def run(): + conn = await open_role_session(settings, PLATFORM_ROLE) + try: + repo = TenantRepository(conn, merchant, TenantRole.viewer, PLATFORM_ROLE, tag="tenantboard:test") + return await repo.list_orders(page=1, page_size=200), await repo.summary() + finally: + await conn.close() + + page, summary = asyncio.run(run()) + + assert {order.tenant_id for order in page.items} == {"ACME"} + assert page.total == len(orders_of(dataset, "ACME")) + assert summary.revenue == expected_summary(dataset, "ACME")["revenue"] + + +def test_detaching_the_policy_exposes_every_tenant(settings: Settings, dataset, worker_db): + """The control for the leak test: on a scratch clone, drop the policy and the same raw SELECT leaks.""" + scratch = f"{worker_db}_LEAK" + with connect_admin(settings) as admin: + provisioning.clone_database(admin, TEMPLATE_DB, scratch) + try: + policy = f"{scratch}.APP.TENANT_ISOLATION" + with connect_sync( + settings, user=settings.tb_app_user, role=viewer_role("ACME"), database=scratch, USE_CACHED_RESULT=False + ) as tenant: + before = {r["TENANT_ID"] for r in dict_rows(tenant, "SELECT DISTINCT TENANT_ID FROM ORDERS")} + provisioning.query(admin, f"ALTER TABLE {scratch}.APP.ORDERS DROP ROW ACCESS POLICY {policy}") + after = {r["TENANT_ID"] for r in dict_rows(tenant, "SELECT DISTINCT TENANT_ID FROM ORDERS")} + finally: + provisioning.drop_database(admin, scratch) + + assert before == {"ACME"} + assert after == set(dataset.tenant_ids) diff --git a/tenantboard/tests/test_session_cache.py b/tenantboard/tests/test_session_cache.py new file mode 100644 index 0000000..1f52e8b --- /dev/null +++ b/tenantboard/tests/test_session_cache.py @@ -0,0 +1,93 @@ +"""The per-role session cache's contract: bounded, shared per role, never closes a busy session. + +Uses stand-in sessions (no Snowflake): the contract is about lifecycle, and a +real login per case would only add latency. +""" + +from __future__ import annotations + +import asyncio + +from app.db import RoleSessionCache + + +class FakeSession: + def __init__(self, role: str) -> None: + self.role = role + self.closed = False + + def is_closed(self) -> bool: + return self.closed + + async def close(self) -> None: + self.closed = True + + +def make_cache(max_size: int): + opened: list[FakeSession] = [] + + async def factory(role: str) -> FakeSession: + await asyncio.sleep(0.01) # a login takes a while; concurrent callers must not open twice + session = FakeSession(role) + opened.append(session) + return session + + return RoleSessionCache(factory, max_size=max_size), opened + + +def test_concurrent_requests_for_one_role_share_one_session(): + async def run(): + cache, opened = make_cache(max_size=4) + + async def use(role: str) -> FakeSession: + async with cache.lease(role) as session: + return session + + sessions = await asyncio.gather(*(use("TENANT_ACME") for _ in range(10))) + return sessions, opened + + sessions, opened = asyncio.run(run()) + + assert len(opened) == 1 + assert {id(s) for s in sessions} == {id(opened[0])} + + +def test_least_recently_used_session_is_closed_beyond_the_bound(): + async def run(): + cache, opened = make_cache(max_size=2) + for role in ("TENANT_A", "TENANT_B", "TENANT_A", "TENANT_C"): + async with cache.lease(role): + pass + return cache, opened + + cache, opened = asyncio.run(run()) + + assert cache.open_roles == ["TENANT_A", "TENANT_C"] + assert [(s.role, s.closed) for s in opened] == [("TENANT_A", False), ("TENANT_B", True), ("TENANT_C", False)] + + +def test_an_evicted_session_stays_open_until_its_lease_ends(): + async def run(): + cache, opened = make_cache(max_size=1) + async with cache.lease("TENANT_A") as busy: + async with cache.lease("TENANT_B"): + still_open = not busy.closed + return still_open, busy.closed, cache.open_roles + + still_open, closed_after, roles = asyncio.run(run()) + + assert still_open is True + assert closed_after is True + assert roles == ["TENANT_B"] + + +def test_aclose_closes_every_session(): + async def run(): + cache, opened = make_cache(max_size=4) + for role in ("TENANT_A", "TENANT_B"): + async with cache.lease(role): + pass + await cache.aclose() + return opened + + assert all(s.closed for s in asyncio.run(run())) diff --git a/tenantboard/tests/test_uploads.py b/tenantboard/tests/test_uploads.py new file mode 100644 index 0000000..085412b --- /dev/null +++ b/tenantboard/tests/test_uploads.py @@ -0,0 +1,119 @@ +"""CSV upload of offline orders: write_pandas into a temporary table, then one MERGE. + +All uploads go to INITECH, which no other test file reads, with order ids +unique per test, so tests stay independent within a worker's database. +""" + +from __future__ import annotations + +from decimal import Decimal + +import pytest + +from app.tenancy import admin_role, viewer_role +from tests.helpers import dict_rows, money, orders_of + +ADMIN = {"X-Tenant-Role": "admin"} +HEADER = "order_id,customer_id,ordered_at,status,total_amount\n" +URL = "/api/tenants/initech/orders/upload" + + +def upload(client, body: str, headers=ADMIN): + return client.post(URL, files={"file": ("offline.csv", (HEADER + body).encode(), "text/csv")}, headers=headers) + + +def order(connect_as, tenant: str, order_id: str) -> dict | None: + found = dict_rows( + connect_as(admin_role(tenant)), "SELECT * FROM ORDERS WHERE TENANT_ID = ? AND ORDER_ID = ?", [tenant, order_id] + ) + return found[0] if found else None + + +def test_upload_inserts_new_and_updates_existing_orders_of_this_tenant_only(client, connect_as, dataset): + """ORD-000001 exists for every merchant; the upload may only touch INITECH's.""" + acme_before = order(connect_as, "ACME", "ORD-000001") + + response = upload( + client, "ORD-910001,2,2026-09-30T12:00:00,completed,99.95\nORD-000001,1,2026-09-02T08:30:00,refunded,10.00\n" + ) + + assert response.status_code == 200 + assert response.json() == {"tenant_id": "INITECH", "rows": 2, "inserted": 1, "updated": 1} + new, updated = order(connect_as, "INITECH", "ORD-910001"), order(connect_as, "INITECH", "ORD-000001") + assert (new["CHANNEL"], new["TOTAL_AMOUNT"], new["CURRENCY"]) == ("offline", Decimal("99.95"), "EUR") + assert (updated["STATUS"], updated["TOTAL_AMOUNT"]) == ("refunded", Decimal("10.00")) + assert order(connect_as, "ACME", "ORD-000001") == acme_before + seeded = orders_of(dataset, "ACME").set_index("ORDER_ID").loc["ORD-000001"] + assert acme_before["TOTAL_AMOUNT"] == money(seeded["TOTAL_AMOUNT"]) + + +def test_uploading_the_same_file_twice_updates_and_inserts_nothing_new(client, connect_as): + body = "ORD-911001,3,2026-09-15T10:00:00,completed,25.00\nORD-911002,3,2026-09-15T11:00:00,cancelled,5.50\n" + + first, second = upload(client, body), upload(client, body) + + assert (first.json()["inserted"], first.json()["updated"]) == (2, 0) + assert (second.json()["inserted"], second.json()["updated"]) == (0, 2) + count = dict_rows( + connect_as(viewer_role("INITECH")), "SELECT COUNT(*) AS N FROM ORDERS WHERE ORDER_ID LIKE 'ORD-9110%'" + ) + assert count == [{"N": 2}] + + +def test_viewer_role_cannot_upload(client, connect_as): + response = upload(client, "ORD-912001,1,2026-09-15T10:00:00,completed,1.00\n", headers={"X-Tenant-Role": "viewer"}) + + assert response.status_code == 403 + assert response.json()["title"] == "Admin role required" + assert order(connect_as, "INITECH", "ORD-912001") is None + + +def test_null_in_a_required_column_rejects_the_whole_upload_with_snowflake_100072(client, connect_as): + """Error path: the table's NOT NULL constraint, not the API, rejects the row, and the + single MERGE statement leaves the valid row unapplied too.""" + response = upload( + client, "ORD-913001,1,2026-09-15T10:00:00,completed,12.00\nORD-913002,,2026-09-15T11:00:00,completed,8.00\n" + ) + + assert response.status_code == 422 + body = response.json() + assert body["snowflake"]["error_code"] == "100072" + assert body["snowflake"]["query_id"] + assert order(connect_as, "INITECH", "ORD-913001") is None + assert order(connect_as, "INITECH", "ORD-913002") is None + + +@pytest.mark.parametrize( + ("csv", "expected"), + [ + ( + "order_id,customer\nORD-1,2\n", + {"missing": ["customer_id", "ordered_at", "status", "total_amount"], "unexpected": ["customer"]}, + ), + (HEADER, None), + ( + HEADER + "ORD-914001,x,yesterday,shipped,-1\n", + {"columns": {"customer_id", "ordered_at", "status", "total_amount"}}, + ), + ( + HEADER + "ORD-914002,1,2026-09-15,completed,1\nORD-914002,1,2026-09-15,completed,1\n", + {"columns": {"order_id"}}, + ), + ], + ids=["wrong-header", "no-rows", "bad-values", "duplicate-ids"], +) +def test_malformed_csv_is_a_422_problem_and_writes_nothing(client, connect_as, csv, expected): + response = client.post(URL, files={"file": ("offline.csv", csv.encode(), "text/csv")}, headers=ADMIN) + + assert response.status_code == 422 + body = response.json() + assert body["title"] == "Invalid upload" + if expected and "missing" in expected: + assert (body["missing"], body["unexpected"]) == (expected["missing"], expected["unexpected"]) + if expected and "columns" in expected: + assert {e["column"] for e in body["errors"]} == expected["columns"] + assert {e["line"] for e in body["errors"]} <= {2, 3} + count = dict_rows( + connect_as(viewer_role("INITECH")), "SELECT COUNT(*) AS N FROM ORDERS WHERE ORDER_ID LIKE 'ORD-914%'" + ) + assert count == [{"N": 0}] diff --git a/tenantboard/uv.lock b/tenantboard/uv.lock new file mode 100644 index 0000000..5ac59a4 --- /dev/null +++ b/tenantboard/uv.lock @@ -0,0 +1,724 @@ +version = 1 +revision = 3 +requires-python = "==3.12.*" + +[[package]] +name = "annotated-doc" +version = "0.0.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5a/8e/38aa427ed5402449e226975b649c5dc73ccadfefeb95e6aecb8f8ea4b6b6/annotated_doc-0.0.5.tar.gz", hash = "sha256:c7e58ce09192557605d8bbd92836d7e1d520ac9580096042c0bfd197efacf1bb", size = 10758, upload-time = "2026-07-28T13:50:58.129Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3e/30/e900b21425a860e195f32e37657aa1f7c7f2b1bfb26f03ca209b90933c06/annotated_doc-0.0.5-py3-none-any.whl", hash = "sha256:117bac03a25ede5df5440e855b32d556049ca169ead221505badf432fed4b101", size = 5302, upload-time = "2026-07-28T13:50:57.239Z" }, +] + +[[package]] +name = "annotated-types" +version = "0.8.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5f/56/a8120250d128bed162cd73c76d45f6ef9991f3e068f62a8ee060afa3104a/annotated_types-0.8.0.tar.gz", hash = "sha256:13b2beaad985e05e2d6407ee4c4f35590b11f8d693a258a561055cac8f64cab7", size = 15893, upload-time = "2026-07-23T20:16:13.995Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/99/91/8acff4f5e50511b911bbccb72b8628a49c68ce14148cd9f6431094859a90/annotated_types-0.8.0-py3-none-any.whl", hash = "sha256:f072f4d804ea359e4eaf198b1af7a8b0943881a87f31bb764f8bf219bb9419e0", size = 13427, upload-time = "2026-07-23T20:16:12.938Z" }, +] + +[[package]] +name = "anyio" +version = "4.15.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "idna" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a9/d2/f4d173e22df740bc37b1db102b386ba719b66e95b0f0d751f556b387e6d2/anyio-4.15.1.tar.gz", hash = "sha256:9f28306018cbd6d329e64a36d58256edff76dd996fe423bc957326e578b82a94", size = 276966, upload-time = "2026-09-05T10:42:39.44Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/12/b8/4bd346e22b28902df4d651910f5242c28d84e4a5c2435ca5c3f797ed7e2e/anyio-4.15.1-py3-none-any.whl", hash = "sha256:6152fdbbf9a77fdec97731721bebf7c4c44f7c29b424b0065826173efc7ed101", size = 132079, upload-time = "2026-09-05T10:42:37.923Z" }, +] + +[[package]] +name = "certifi" +version = "2026.10.23" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fd/11/b524cbd9bbdaca58608e4cf4053fd463417549e1959fa8d3aa18319f9555/certifi-2026.10.23.tar.gz", hash = "sha256:a9895e6e2758b371e0545e321a3a8fd1fcfd9829ada8824a936dfb16aaaf36cd", size = 136225, upload-time = "2026-10-23T11:36:16.478Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/d1/a3652fe2c19134d0d15922177821d863cbd7e842d53a21bbe7e8e3a1b5ac/certifi-2026.10.23-py3-none-any.whl", hash = "sha256:e9b54de76e530bb6cf0ccd036787103acb58fd42ba930ea8bf948ddbaa2f61d1", size = 135082, upload-time = "2026-10-23T11:36:14.763Z" }, +] + +[[package]] +name = "cffi" +version = "2.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pycparser", marker = "implementation_name != 'PyPy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9e/ef/008a1939e372c06329a3fce4279c02f328488f3526744906eeec3da7ad5f/cffi-2.1.1.tar.gz", hash = "sha256:dd31f52ea1086513bb9df30f8fcee9b8918323ae067a3d5b78bc826a000712be", size = 530807, upload-time = "2026-08-03T21:21:18.939Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/10/69/43965eccfdead3b9220015fd1320e117be8c6ed01a62ffab76eeb752f5d5/cffi-2.1.1-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:c8c69575568085ba0b1b10c0249d779a214aea6f6522e949a0fc9fb0fcb449d0", size = 184821, upload-time = "2026-08-03T21:19:44.887Z" }, + { url = "https://files.pythonhosted.org/packages/54/7d/16e5a096677b5e313ca80cd5e5170efa3ea44624a82bb111925522da64b1/cffi-2.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f81b3b8f3d4e343550fa4baa0e479bba9f2d29ce9c2e9b51d1ce1718d7442fcf", size = 184719, upload-time = "2026-08-03T21:19:46.129Z" }, + { url = "https://files.pythonhosted.org/packages/56/e6/8941622732edec876dd17d0453dce07317ae96db34f2ec1436c9d3785986/cffi-2.1.1-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:811bd1e21d32de12efca32393a0ab3f5133b54fce9bd44b8bd77ab07da14bf6a", size = 214799, upload-time = "2026-08-03T21:19:47.218Z" }, + { url = "https://files.pythonhosted.org/packages/44/de/f98430906df1545ffde0d543dd124a7a439bc2cd32b36b9c53f805df7333/cffi-2.1.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:68e62fe11f30d5ca8289242866f0a5291402d8529ca2178ab8afc5c9694ae890", size = 222389, upload-time = "2026-08-03T21:19:48.331Z" }, + { url = "https://files.pythonhosted.org/packages/6a/5b/717f1526b9957b34456313c31645c5b82b8fb5c3fe9e4752999be7128bfc/cffi-2.1.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:4a7c934f7360e8cd64fe9efadcbd10c7c6364f531e432b9a4bf5ccbc9e0e8b50", size = 210249, upload-time = "2026-08-03T21:19:49.543Z" }, + { url = "https://files.pythonhosted.org/packages/64/b3/f8aa4f3e34986c7e4ec45072d1b1b9dd295b6b18007b45518d79726dd725/cffi-2.1.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:3143d81e29e1e20a9ce10901ec369012947876596f75a222235965f2b7ae832e", size = 208775, upload-time = "2026-08-03T21:19:50.918Z" }, + { url = "https://files.pythonhosted.org/packages/b1/db/dceb9dd5b231e1da801793f8acc9f3c52a7e1afe40bb1aae37e02b0faad5/cffi-2.1.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c1453022f490d2459a11819d83ad1d586e9ff65a12ac3e705ffebd46d3685dcf", size = 221822, upload-time = "2026-08-03T21:19:52.054Z" }, + { url = "https://files.pythonhosted.org/packages/a0/d2/6cd24ae3be000a634109c247d1475d62e5616d0dc78c82770942ec384248/cffi-2.1.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:208f941bb9d18e768138677f0a6d2ce01f590df56043dda1df1535ac57c88517", size = 225232, upload-time = "2026-08-03T21:19:53.109Z" }, + { url = "https://files.pythonhosted.org/packages/cb/52/3fa190537004dd7f0ab860a6dc7c0175b8667f68d1e618a46f5498d30250/cffi-2.1.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:210019b6c7cf07f081b4c54635c8cf744377001350e29cc0f81c4377b4797735", size = 223597, upload-time = "2026-08-03T21:19:54.515Z" }, + { url = "https://files.pythonhosted.org/packages/80/fb/0bb75b7039588c074b37ae99f40d9bfddf990ecb2fbc346ebccd2e56b9be/cffi-2.1.1-cp312-cp312-win32.whl", hash = "sha256:046bfc24911b37851ee1b51aab8bffe713d89c68c6a057b09484ce9fd5f69b4e", size = 175292, upload-time = "2026-08-03T21:19:55.566Z" }, + { url = "https://files.pythonhosted.org/packages/d9/79/615cc094e2fb508cade7de88d3b4f6c4ec2bab695c97bce9153dc65aadf5/cffi-2.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:f53e442b08449d42821fa4a4fba000095af9f62742a500f978a9f557ec44339a", size = 185919, upload-time = "2026-08-03T21:19:56.89Z" }, + { url = "https://files.pythonhosted.org/packages/70/c6/d0ea84713fe46b243a436a18fcd47d639732747e21635c8a27191b06dc30/cffi-2.1.1-cp312-cp312-win_arm64.whl", hash = "sha256:7bde5e4cc5c10140859842b9d383af292b22639a4dffb725314baf45968cef80", size = 180093, upload-time = "2026-08-03T21:19:58.155Z" }, +] + +[[package]] +name = "click" +version = "8.5.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c7/0e/7fa0ef50764b67090eca4114772a2abf8b6148198475e54c660b97caeee6/click-8.5.0.tar.gz", hash = "sha256:ba0d2089de75ea0310e2dde03160e6ca10009947fb95a182f9b54021bb272e34", size = 382235, upload-time = "2026-08-26T13:33:14.56Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/58/50/6c0d534c5f134586a8e1ba4e330569e32f057e33372ae556463212fb4cd3/click-8.5.0-py3-none-any.whl", hash = "sha256:255bc9599cf7748b4b1a446ccc735421bd08a2ae529a8b88597d3de5664ee360", size = 125251, upload-time = "2026-08-26T13:33:12.928Z" }, +] + +[[package]] +name = "colorama" +version = "0.4.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d8/53/6f443c9a4a8358a93a6792e2acffb9d9d5cb0a5cfd8802644b7b1c9a02e4/colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", size = 27697, upload-time = "2022-10-25T02:36:22.414Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, +] + +[[package]] +name = "cryptography" +version = "50.0.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi", marker = "platform_python_implementation != 'PyPy'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9d/af/182eb91b0df3fe75c4d9f26fe70684569566745f6ba7e5c9c73a862c5252/cryptography-50.0.2.tar.gz", hash = "sha256:7b46165bb56eb4704e2eaaf86f3c940d19154535d9b0ca7d6d590b04060e00d5", size = 880623, upload-time = "2026-09-30T15:30:04.884Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/56/d194340cc4a57535e82e1bee9e89667ac4b7c13b5d3f59686deae3094dd5/cryptography-50.0.2-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:fa8f5efb344d6908a1ce62f4a24e2e5780f825d6f53f5f50ec5ffacac72936cb", size = 3914904, upload-time = "2026-09-30T14:43:44.339Z" }, + { url = "https://files.pythonhosted.org/packages/d9/69/c9bd862c3bf43d6399c433caf002df16e2dffd4be49bdf515cda38038711/cryptography-50.0.2-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:79def8d059362e7831389ed3be0ecdf58a89386e1271e35dd9f5af84e81bffd0", size = 4731146, upload-time = "2026-09-30T14:43:47.113Z" }, + { url = "https://files.pythonhosted.org/packages/21/69/64cef1f702bf6657e0cc186ed1a2891d50d29fb41586b254e1c07adea261/cryptography-50.0.2-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:630ebfea3bf689d075f82316324ff7433dc447fe6bc1bfc76524b74b4a9567d2", size = 4719841, upload-time = "2026-09-30T14:43:49.01Z" }, + { url = "https://files.pythonhosted.org/packages/38/6b/61a3f8d8c5e1e49a6cddccafc4015cc1c0021360ab0acb4080e7a423644a/cryptography-50.0.2-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:f9f6143a8c75945eb960d9eb98905a441394abfa24afaae239d514ffb2586480", size = 4738340, upload-time = "2026-09-30T14:43:50.932Z" }, + { url = "https://files.pythonhosted.org/packages/7b/2e/7212ca32fd43dc91f2f41db20160b268098874b4c9a0e7be94d6835f5b2e/cryptography-50.0.2-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:a582ab2ae1d34f67112cadc86702774c9ea4374df6bca6afe672817203c99134", size = 5367029, upload-time = "2026-09-30T14:43:52.911Z" }, + { url = "https://files.pythonhosted.org/packages/1a/f1/b474e930c4d910328780e3940da76f5aa5cbc48ce1fc14e44d239d9ea9db/cryptography-50.0.2-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:4061c0079120205fb760c58acab6443e217307dcf05e3702cf970e0689972856", size = 4753050, upload-time = "2026-09-30T14:43:55.272Z" }, + { url = "https://files.pythonhosted.org/packages/7c/52/9af10e80ac16b0fcc2123f9cbd5e7afbd0fd5075bb7a607c592258a39cda/cryptography-50.0.2-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:ac9ed99d81760c62fe89d5f0815cdfa1ba9a35141cf30f1c2d044f04b4803d2e", size = 4376724, upload-time = "2026-09-30T14:43:57.24Z" }, + { url = "https://files.pythonhosted.org/packages/71/37/6202e488cc1eb625ea110c292c6bda92823176e023f427d8d5660ce8d632/cryptography-50.0.2-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:87e9ce85beb6b328ba370cc6e6aea483c92617b4c95b1d33a49297eb662bfb04", size = 4737859, upload-time = "2026-09-30T14:43:59.541Z" }, + { url = "https://files.pythonhosted.org/packages/8f/30/e86d7d518489b0ae2497091a35287abcb1a2ce4037837a34afbe9b1d6964/cryptography-50.0.2-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:f265528741e048bce55c3463ed721fb0aa45a5888d8add8cfeccb3035451bbdc", size = 5324103, upload-time = "2026-09-30T14:44:01.901Z" }, + { url = "https://files.pythonhosted.org/packages/d3/69/2c833a049475e0a3444e94c7d0aca0aa51d166374a449b09e92ac98138de/cryptography-50.0.2-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:9dab55f57c74c3cad24c323bacbbd04be4705ba6eb0d92e920b1fc4837ed5079", size = 4752576, upload-time = "2026-09-30T14:44:04.545Z" }, + { url = "https://files.pythonhosted.org/packages/6c/5d/906970b83bbfc1f5bbfb677a143c181f2801f23b6a7204a3b47c42c97e65/cryptography-50.0.2-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:25784ce8b9621c90c643efb9e1e2162ab3b0224cae446ad5e70e7fcb1ce18b51", size = 4870819, upload-time = "2026-09-30T14:44:06.884Z" }, + { url = "https://files.pythonhosted.org/packages/68/e3/f2298d3bb55e0c4a91841ec4d01b3f020ba8c5fbf15ccdcc6dcf03f97025/cryptography-50.0.2-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:85d0d9a31b9098e98534226d5686b47264b95e62ce459dc2e62fdfc809f9fe93", size = 5030152, upload-time = "2026-09-30T14:44:09.443Z" }, + { url = "https://files.pythonhosted.org/packages/9a/4f/adfc442765721292fff86d314ce385d3249d22db42295c0dd057727b60f3/cryptography-50.0.2-cp311-abi3-win_amd64.whl", hash = "sha256:7afa5a6602a9f29af1f3a2965f831bae7c9d5d597b7cbb716d41ab3b7d89879c", size = 3824692, upload-time = "2026-09-30T14:44:11.671Z" }, + { url = "https://files.pythonhosted.org/packages/2d/49/93f6a6e7a87c9aa68d44d3e1cdb5fe8f60c90d5d2f46acae9a56892816b8/cryptography-50.0.2-cp315-abi3.abi3t-macosx_11_0_arm64.whl", hash = "sha256:edc3342adf8f697fc5f59c887a304356f147b397809440ed64e2fa6af2f50f37", size = 4133708, upload-time = "2026-09-30T14:44:41.807Z" }, + { url = "https://files.pythonhosted.org/packages/8c/75/32ac2a56243d778805c16ca6a32b8f74fb757df7e28d7ecb560afafb59cf/cryptography-50.0.2-cp315-abi3.abi3t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d370b8d1dfcdf7130178137f6fbee6140774a1acc6cacefc4b42643ec11d0a3a", size = 4956267, upload-time = "2026-09-30T14:44:43.693Z" }, + { url = "https://files.pythonhosted.org/packages/aa/a4/2c8d734e43d97f0842ee9f1b7b4bfb3d0cf5e19edebf43c2afe6675c2320/cryptography-50.0.2-cp315-abi3.abi3t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f2f9bd7f90c64fe89253f0a2c05e3c4856072660429ce8831b4235bf29403a67", size = 4966465, upload-time = "2026-09-30T14:44:45.769Z" }, + { url = "https://files.pythonhosted.org/packages/c2/58/ee288c829a6f41f6235ae9dd33d82fd19b45442b65b4c8a3da36963d9f7a/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_28_aarch64.whl", hash = "sha256:e275096ea1e60cc595cda2836fd4a6c725d1125108b868be17f53684d164e2cc", size = 4959356, upload-time = "2026-09-30T14:44:48.211Z" }, + { url = "https://files.pythonhosted.org/packages/92/20/9ded6d51ddd9897f6b6e81fb9ebea7951d7cc5d6c890b0ed8abf77a51a80/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_28_ppc64le.whl", hash = "sha256:b13478603dcd0a2479ff8e87e2c19a7d525734686fe3c49542472293a204212d", size = 5548822, upload-time = "2026-09-30T14:44:50.86Z" }, + { url = "https://files.pythonhosted.org/packages/02/a8/8df951850d6b31d2a00218f19e2b3f999523437ed7a819df7fa427942fca/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_28_x86_64.whl", hash = "sha256:58a0c478eeca76fe5e07993c5a0703def34a6dc6a0cda4f5564639b33112ffe7", size = 5001199, upload-time = "2026-09-30T14:44:53.379Z" }, + { url = "https://files.pythonhosted.org/packages/8b/f9/36b3022218ce75b7cdf068fb95f809f9bd0d820e4955ef43b90c255cc7ac/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_31_armv7l.whl", hash = "sha256:d38cdff612d06fa6a32840d5e1b1f7a27cee4a349aa9085d94a67789d6bfd408", size = 4629333, upload-time = "2026-09-30T14:44:55.635Z" }, + { url = "https://files.pythonhosted.org/packages/8c/72/20f99a219f6af47cdd1cbd978c243b92d71496e168a746138af44ded4f29/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_34_aarch64.whl", hash = "sha256:fdd28f912fccfec1846a94e2e1e8f9b0012f557f0c46fe4f3eb0d7a87afcf90b", size = 4958822, upload-time = "2026-09-30T14:44:59.639Z" }, + { url = "https://files.pythonhosted.org/packages/f2/20/196f112617fb08eb4d608a2a6c422373d46f9cc2857f38fc0667033c0899/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_34_ppc64le.whl", hash = "sha256:cbc8738fd8526d80f35cb3a40d41f41a2e7030bb3b18b09a6778ef63d291c2fd", size = 5506351, upload-time = "2026-09-30T14:45:02.267Z" }, + { url = "https://files.pythonhosted.org/packages/24/95/83378121ef3eaaaf71d4b781577ff794acb39b9e1b87a3f156898c8497ed/cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_34_x86_64.whl", hash = "sha256:e105ab60406787da31fccc883fc0f733af1efd78f0136a4599692c4083a73d0c", size = 5000859, upload-time = "2026-09-30T14:45:05.009Z" }, + { url = "https://files.pythonhosted.org/packages/22/f7/70fd7ae4d1dbfa7ba29b02e1b9068771519a86027756510b700ce81086a8/cryptography-50.0.2-cp315-abi3.abi3t-musllinux_1_2_aarch64.whl", hash = "sha256:6f8700550aa1474a91e5dc07049c46f98b423b5b1ddd0483e0b51362eeeaf5be", size = 5092151, upload-time = "2026-09-30T15:29:15.932Z" }, + { url = "https://files.pythonhosted.org/packages/d4/be/688367b74de86984bd58d8efacfc7c9e68b89a6a22ced0fb4f38db50254a/cryptography-50.0.2-cp315-abi3.abi3t-musllinux_1_2_x86_64.whl", hash = "sha256:c71be1cbfa5cd9a41ee452acf1eccd82b2c05950358b106ec8ceb83411d1a020", size = 5286120, upload-time = "2026-09-30T15:29:18.309Z" }, + { url = "https://files.pythonhosted.org/packages/39/d1/55f8a3f2ef5d1529e16835ef10cf0fe3d559ce237b46dddc440c0bba3649/cryptography-50.0.2-cp315-abi3.abi3t-win_amd64.whl", hash = "sha256:c423ab384a46c4dff7217b2ea5ba2e11cffdeab6441acd04cf65a369caf0366c", size = 4111557, upload-time = "2026-09-30T15:29:20.155Z" }, + { url = "https://files.pythonhosted.org/packages/23/ad/ac987755d00e1e64273760228d2635ae38dae2be83e3c6e0d3289d91dec3/cryptography-50.0.2-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:0ec5f09541743261e66e291b4a0cbf0fb2997aeaab6d9e9c740b9dba1b58d1c2", size = 3943588, upload-time = "2026-09-30T15:29:22.265Z" }, + { url = "https://files.pythonhosted.org/packages/d5/8d/6d585339bedf85d45044c85d8412dac53f2bb6f918e8b7777efba1787844/cryptography-50.0.2-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c5e67125c7dca78d199ec4e116aa93dbb83494808ecbb8211a2cb09b1bf41dbd", size = 4756166, upload-time = "2026-09-30T15:29:24.58Z" }, + { url = "https://files.pythonhosted.org/packages/bf/f1/1c1f6874e8550cfddd4b688ceb38cefb6ed15ceed224d56f133f3d88c214/cryptography-50.0.2-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ee247f5c245c9a2fe7c8e2214e295918838e44e00a45a6718451e4004219e767", size = 4749145, upload-time = "2026-09-30T15:29:26.807Z" }, + { url = "https://files.pythonhosted.org/packages/c1/63/61b15dc1a8de03fe0adbe3fd7608b3ad5c73bf50993bbcb1faaa930afe33/cryptography-50.0.2-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:dfe9763530994147d9af1def057a5b9658b00e8f8fe8743d144d1e0911c2e454", size = 4763638, upload-time = "2026-09-30T15:29:28.588Z" }, + { url = "https://files.pythonhosted.org/packages/fc/35/b345bdfa40c9126df1a9d33236aa98418367931b8725f84fc3ae2b98dc59/cryptography-50.0.2-cp39-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:58ddb5a8e3179d12f19e4ea34d2d32e9d63a4baa142c875c1eb59f41b7243acd", size = 5382217, upload-time = "2026-09-30T15:29:30.589Z" }, + { url = "https://files.pythonhosted.org/packages/4f/87/ef344a9e616871f2519c22d6afcda79ddd5d35e9592d95eb6e677608d055/cryptography-50.0.2-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:f21e8a22c8605750c7af886bab299a363721264061b4ac0a30efb73cfd58efc5", size = 4781387, upload-time = "2026-09-30T15:29:32.605Z" }, + { url = "https://files.pythonhosted.org/packages/90/5b/f2fdb13cd0b96f6f932c8627bb292a45f11c64d21620a8e120aee9a3b848/cryptography-50.0.2-cp39-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:9c8402a82ea0dc4ceeab793db05f0fafa8ca139ca34fcde5df0f596103c74107", size = 4403790, upload-time = "2026-09-30T15:29:34.374Z" }, + { url = "https://files.pythonhosted.org/packages/bc/ce/7e4f662b1e3c393513569e402cfc85ac7da0bd3d5435e122a3140219eb2d/cryptography-50.0.2-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:0ddc924c04591c2811ca024d62ecad4f7f6f08af8939c211438f48a16bd23602", size = 4764319, upload-time = "2026-09-30T15:29:36.149Z" }, + { url = "https://files.pythonhosted.org/packages/3c/3f/86ff33ce34cc0de6847fb96e035a1a760d81652e38643f617c02ad32ef7a/cryptography-50.0.2-cp39-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:a6557e5f38e065ca9fbdaf7cfc7435ecb1d113aa81a022d1b51921ee7432e227", size = 5338560, upload-time = "2026-09-30T15:29:39.053Z" }, + { url = "https://files.pythonhosted.org/packages/40/cf/6b5c8e2fd9202d98988ab7cb5cc5c991704c4ad55f492ff408e4969f83f1/cryptography-50.0.2-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:1981f1db4630889b9ef7803fadef12b056f428cb6b85c27ba57b774793b6093c", size = 4780973, upload-time = "2026-09-30T15:29:41.251Z" }, + { url = "https://files.pythonhosted.org/packages/10/bf/8d6ebc7dded797bd0f0160d52188021211f011a2b164ef0ae1dac4587465/cryptography-50.0.2-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:7a8701d6b584d76e909e3d305b7d126b41439876a5aaf76cddc67fc230eafa2e", size = 4897738, upload-time = "2026-09-30T15:29:43.106Z" }, + { url = "https://files.pythonhosted.org/packages/d4/aa/f3f6e0de7e6253b8baa8b2d8fb9d50924fa75cee3d4624bd4bc1208ee923/cryptography-50.0.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:ce47f66801c20ec6c6632453bb5960fe38939e9306970b48b3a5a26de7745d94", size = 5058280, upload-time = "2026-09-30T15:29:44.827Z" }, + { url = "https://files.pythonhosted.org/packages/f6/b6/a1faf3a27ae9405fb34b1713cc73b2d8a26b04d5c561578fa2e6ef3e5bb9/cryptography-50.0.2-cp39-abi3-win_amd64.whl", hash = "sha256:4e81d95e5bafc2d6e34e4bed780e53e4d5b9a2f928573428aa4d35fbec1eb0de", size = 3854095, upload-time = "2026-09-30T15:29:46.782Z" }, +] + +[[package]] +name = "execnet" +version = "2.1.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bf/89/780e11f9588d9e7128a3f87788354c7946a9cbb1401ad38a48c4db9a4f07/execnet-2.1.2.tar.gz", hash = "sha256:63d83bfdd9a23e35b9c6a3261412324f964c2ec8dcd8d3c6916ee9373e0befcd", size = 166622, upload-time = "2025-11-12T09:56:37.75Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ab/84/02fc1827e8cdded4aa65baef11296a9bbe595c474f0d6d758af082d849fd/execnet-2.1.2-py3-none-any.whl", hash = "sha256:67fba928dd5a544b783f6056f449e5e3931a5c378b128bc18501f7ea79e296ec", size = 40708, upload-time = "2025-11-12T09:56:36.333Z" }, +] + +[[package]] +name = "fastapi" +version = "0.162.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-doc" }, + { name = "opentelemetry-api" }, + { name = "pydantic" }, + { name = "starlette" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/e9/eb/2cf8c80abf09b2800aef0a83b41066fbfda8ee5495b7a08b9d1af40f7046/fastapi-0.162.0.tar.gz", hash = "sha256:6a22c1e2e6e32e0633ee7e170f00009a29b8228b16feb219ee42f32fd8d88a8f", size = 527867, upload-time = "2026-10-23T18:47:33.264Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b4/4f/3f2c25c9b43970b5e9a08beb5e04776247edea684ddc0930f4f02f0bb6cb/fastapi-0.162.0-py3-none-any.whl", hash = "sha256:3785963de305dd09912426cb81f2d230fce98ab3521cdfe97265c8d14983128c", size = 148912, upload-time = "2026-10-23T18:47:31.734Z" }, +] + +[[package]] +name = "h11" +version = "0.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/ee/02a2c011bdab74c6fb3c75474d40b3052059d95df7e73351460c8588d963/h11-0.16.0.tar.gz", hash = "sha256:4e35b956cf45792e4caa5885e69fba00bdbc6ffafbfa020300e549b208ee5ff1", size = 101250, upload-time = "2025-04-24T03:35:25.427Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, +] + +[[package]] +name = "httpcore" +version = "1.0.9" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "certifi" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/06/94/82699a10bca87a5556c9c59b5963f2d039dbd239f25bc2a63907a05a14cb/httpcore-1.0.9.tar.gz", hash = "sha256:6e34463af53fd2ab5d807f399a9b45ea31c3dfa2276f15a2c3f00afff6e176e8", size = 85484, upload-time = "2025-04-24T22:06:22.219Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, +] + +[[package]] +name = "httptools" +version = "0.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3a/ec/deed52912ab7ca6c0b12859330c571c60c61d7267b341b28951fcbf13694/httptools-0.9.0.tar.gz", hash = "sha256:d484ebb7e3a3f3597b0f645fbd1b85633674ca808c1f5ba11c2caf7c66f5c8b6", size = 282523, upload-time = "2026-10-09T19:57:04.301Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/da/ed/0916b8b7ebd1deeaf22acba71b68c57b4b6b69aa1918f3812dea208b4276/httptools-0.9.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f9ccc9884241efceb4547a92955d128574c864681f11b7ea3ecbde295fafbe8b", size = 119039, upload-time = "2026-10-09T19:54:32.556Z" }, + { url = "https://files.pythonhosted.org/packages/c2/0b/9b6de4a01a563a904d0826c9069c824b330e1816df26c9bdf93f60b50857/httptools-0.9.0-cp312-cp312-macosx_11_0_x86_64.whl", hash = "sha256:45b3002392948dcf578029c89f6318e1289a993a1a5ec38a4161560fab60f811", size = 115051, upload-time = "2026-10-09T19:54:33.908Z" }, + { url = "https://files.pythonhosted.org/packages/85/3f/642113e9882f53158ecddf58003d25f18ded2c210ed23bf6eb663d4d51c3/httptools-0.9.0-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:3e3201fe4d46e0d15d7ff9fafc94a605da9eb82d2c5b9837f0368acb325481f1", size = 552848, upload-time = "2026-10-09T19:54:35.434Z" }, + { url = "https://files.pythonhosted.org/packages/95/4c/3ecc59c99c28652d8d08d9b5be65770a14d2cadc616dad94224cee2b0e7e/httptools-0.9.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:58a1b0ec4cbb930e69669f9771715b2c7898d3cdf064d9811f7a66afef96b544", size = 549181, upload-time = "2026-10-09T19:54:37.099Z" }, + { url = "https://files.pythonhosted.org/packages/43/ce/21f5b2759590b7054e38d3b704a3c6b853c3395c370b3d6f16c45aea0fc0/httptools-0.9.0-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4c58dc91aefb31adad500aa68054334f429b840b36dd29e34e834101044cb2ef", size = 569121, upload-time = "2026-10-09T19:54:38.772Z" }, + { url = "https://files.pythonhosted.org/packages/52/c3/7c523aa8d0fa7a57010a3e1bbdebc209585009076465f3d1ae6a3f54b814/httptools-0.9.0-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:6b900073e7b8481ef1aaf4f6c1789d210a1db01a9da8789821578cfeb4c2d540", size = 487649, upload-time = "2026-10-09T19:54:40.398Z" }, + { url = "https://files.pythonhosted.org/packages/94/e2/d90d60002692b8afcbc06fb49ca3a4365b32abed6c40fb2b612c67721a00/httptools-0.9.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:6c12d0393a903b58bc5f5a7406d6c5290acfb8284290d68547ce620c06f7d133", size = 529583, upload-time = "2026-10-09T19:54:42.296Z" }, + { url = "https://files.pythonhosted.org/packages/46/c0/19172874cde0344a20c85877a0b2d0dcfca31111729ad8a79e8b4ac4e207/httptools-0.9.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:29b0d823e3c1e7cd1093a5dc889245db693ef13ada624cd66e2262421ef38867", size = 552414, upload-time = "2026-10-09T19:54:44.19Z" }, + { url = "https://files.pythonhosted.org/packages/1b/b8/02ea7910f69e5371986b025fb3b410592106df54e977a5732fd1d95917b5/httptools-0.9.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:6ebd39ee26db460cfe5ab8b71a15d1149b289139a0d3981522757d6af620887e", size = 482400, upload-time = "2026-10-09T19:54:46.064Z" }, + { url = "https://files.pythonhosted.org/packages/de/97/f05eac916d44cbbfe43668a6a40ab93e7fd8f94d5120d1ce2d8e55c69871/httptools-0.9.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:4efbee349138a3fee7a4cc3a95abd2d499fae70dd5bff9fed9138d6f570f4283", size = 536148, upload-time = "2026-10-09T19:54:47.695Z" }, + { url = "https://files.pythonhosted.org/packages/b6/e9/9435dfcb7f1a1d6ebdc79a902164dfca70e33774c80bb60d25c630c31ef1/httptools-0.9.0-cp312-cp312-win32.whl", hash = "sha256:36fac804b8cfd6b935ae64f71349f833d2b6298404626d017a2c57bb942bc643", size = 86274, upload-time = "2026-10-09T19:54:49.1Z" }, + { url = "https://files.pythonhosted.org/packages/8b/69/813f1bf90be507d4166c437be1a413574d0e0abf36e2fec10c266661b0ee/httptools-0.9.0-cp312-cp312-win_amd64.whl", hash = "sha256:7e32b83bd8c2f8b6fa726ef34e63e21c4d7eddc277d40d4ef7245ea3ed28e5b6", size = 92446, upload-time = "2026-10-09T19:54:50.498Z" }, + { url = "https://files.pythonhosted.org/packages/ae/e0/1d29e328c4cafe843403341e1455e0aec18b0e6910fbb14f12b36b563f19/httptools-0.9.0-cp312-cp312-win_arm64.whl", hash = "sha256:813a32f94991b9627795528053c73a57d2ce3eb98ede89f0e1c7a31095938e81", size = 88676, upload-time = "2026-10-09T19:54:51.844Z" }, +] + +[[package]] +name = "httpx" +version = "0.28.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "certifi" }, + { name = "httpcore" }, + { name = "idna" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b1/df/48c586a5fe32a0f01324ee087459e112ebb7224f646c0b5023f5e79e9956/httpx-0.28.1.tar.gz", hash = "sha256:75e98c5f16b0f35b567856f597f06ff2270a374470a5c2392242528e3e3e42fc", size = 141406, upload-time = "2024-12-06T15:37:23.222Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, +] + +[[package]] +name = "idna" +version = "3.20" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f5/08/8eea9d4b8302028f3abb2c0813953f7aec26d33b7a8960ed760e65ff29fa/idna-3.20.tar.gz", hash = "sha256:a7db850025b95ded1eae8a46181a1a6c56c92c96f0e2b005d9ff8dc0210cab44", size = 216463, upload-time = "2026-09-17T14:11:04.752Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/58/a2/bb081bab032533a855d44de1d56f8e8426114ff1ba5d1f07a438a0a654f8/idna-3.20-py3-none-any.whl", hash = "sha256:ab7ae7122974553370f0bdb919e1a960b2cd1bc1ef0276416d896db81c14582c", size = 69583, upload-time = "2026-09-17T14:11:03.168Z" }, +] + +[[package]] +name = "iniconfig" +version = "2.3.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/01/e1/2069291243c926a2ff1cd706c7f3eeb9b62144bf60f77c9fb9ff2fb26bd3/iniconfig-2.3.1.tar.gz", hash = "sha256:67f4b9c50da0dedf52af349e7749a80a9057a5031199791b906c3bb3ae878960", size = 21209, upload-time = "2026-10-06T22:48:38.076Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/56/43/4ca9e49d27a1fcf6bece6f6aec0ea46bb9112489b93d4b688fb415457bdb/iniconfig-2.3.1-py3-none-any.whl", hash = "sha256:9121e2c1fdb355232495be3194c8dfe87ccc2d5dee45947b78e68f499790d7a7", size = 7552, upload-time = "2026-10-06T22:48:36.959Z" }, +] + +[[package]] +name = "numpy" +version = "2.5.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/95/b0/c7453d0b6e2073c3264468b106ee1563750cecc910965e67357e3698c83e/numpy-2.5.4.tar.gz", hash = "sha256:9a94cf751c9ad8ebaa835bcd3d40dacf8534ad086b88c38029b65123c7999d2a", size = 20866315, upload-time = "2026-10-10T20:05:31.422Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d0/97/ba2074e92b7befea137e77ea8471e768bbd87c339b7e8c9f5a931949f977/numpy-2.5.4-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c6342f54c67093cae5c0227eb0eb772fdb79f2a2c37a6eb278b9909ee06aa356", size = 17001609, upload-time = "2026-10-10T20:02:40.843Z" }, + { url = "https://files.pythonhosted.org/packages/ff/a9/bac826765e971d8e16e2064e9ac7525fd69b40ac17c905033a7f5442023f/numpy-2.5.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:b11e8fda06a7d69f15ebf542660b74466c2e51094800c1fb794f47ad4faeef17", size = 12015718, upload-time = "2026-10-10T20:02:43.45Z" }, + { url = "https://files.pythonhosted.org/packages/31/2f/5ea3570fcb8ccd0882bea99436a513b2c85dad8f774a2057849130a8fb99/numpy-2.5.4-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:9cb18a327b49c5c337f972b03682f6a49855525faaf3c0d3e9c96cd0fd8880a8", size = 5451717, upload-time = "2026-10-10T20:02:46.169Z" }, + { url = "https://files.pythonhosted.org/packages/34/f2/b4fc1bafca03868220b5eaf729d2f21ebd7d7b151c0f9e144fe212bbca35/numpy-2.5.4-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:aec3fc4b32ff82421274f5d205c559c51c840c8df66a78efd7f3612dd005a26a", size = 6789926, upload-time = "2026-10-10T20:02:48.139Z" }, + { url = "https://files.pythonhosted.org/packages/dc/96/8319e2457ae4333c62c815c7006b869a4f60985c1e01024c2f8c6c040fe5/numpy-2.5.4-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fe4d21ab149f15e4e6043dfb0de87e6e5f34ac176cde83060e9802981fca2ac2", size = 15695312, upload-time = "2026-10-10T20:02:50.115Z" }, + { url = "https://files.pythonhosted.org/packages/43/a3/c799c62e19c337e6d3770b08e475887fb30ce8477d3c09efca6b2f0228a6/numpy-2.5.4-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fbde6962867ee75b48b0ee29b2b9372ec5d617799dbaf38e82dc0596f2f7738a", size = 16727283, upload-time = "2026-10-10T20:02:53.186Z" }, + { url = "https://files.pythonhosted.org/packages/39/6b/3604e53fb00314d0dc1b94ec9125a1484f649c0a17480b1f0f0c7a9d6250/numpy-2.5.4-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:381a7a3d2e65e64c0ec302795ab9dc12bb1e73f150904699c153716177eebdaf", size = 17047890, upload-time = "2026-10-10T20:02:56.038Z" }, + { url = "https://files.pythonhosted.org/packages/4a/7a/e8b58a5289a0d464c52885de47c35a935cdd70c03a4c3ab94a5126416dd0/numpy-2.5.4-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:b89d0aaae2fe498c648f4c4795c084db535af5bd98ef942b2a3681fb74ce8645", size = 18485839, upload-time = "2026-10-10T20:02:59.018Z" }, + { url = "https://files.pythonhosted.org/packages/6f/c9/47094f597015009f310b8c900def59065ef1ff5a6fe7b51fc65ec58ec2c6/numpy-2.5.4-cp312-cp312-win32.whl", hash = "sha256:9968ab7e49b93ac6e1c3b2239732183152c9150f16308d30b66a372cffe3483c", size = 6138936, upload-time = "2026-10-10T20:03:01.626Z" }, + { url = "https://files.pythonhosted.org/packages/12/33/fefe62073dc8acfd0f2b9ed7c003af2f50aa61555e113e6db02b8f79f145/numpy-2.5.4-cp312-cp312-win_amd64.whl", hash = "sha256:a7b1b6353e36a7e50de2973a38d705c88ee93adcf120673cee7f45a4a3fa223a", size = 12573091, upload-time = "2026-10-10T20:03:04.349Z" }, + { url = "https://files.pythonhosted.org/packages/1a/07/161270b0c2eec56e4c905f6d6d22e1b836887b2cb189d3f5820aa588e9dd/numpy-2.5.4-cp312-cp312-win_arm64.whl", hash = "sha256:aa1cce2ff3f8d953de38b76bf44602caeb69f101430208f64a10067f7cb4b1d3", size = 10521630, upload-time = "2026-10-10T20:03:06.767Z" }, +] + +[[package]] +name = "opentelemetry-api" +version = "1.45.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/2e/02/6e0ae9cc61bd3169d401077b507b3ebc344745171e1051ab430be012dcd9/opentelemetry_api-1.45.1.tar.gz", hash = "sha256:aa38ed19bcc084ba42782a73255b3582283eced7ad6dddbd6695189e69adfb75", size = 72804, upload-time = "2026-10-06T17:32:58.133Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/41/f7dcf80b81ee8e71c1a2b59f14208bc723edbd89ed027a73b175abf6348e/opentelemetry_api-1.45.1-py3-none-any.whl", hash = "sha256:b31553efa588ae44bc306f863c785c5333a9ecc091248c6ee68b4b6c87fdedfb", size = 60256, upload-time = "2026-10-06T17:32:33.506Z" }, +] + +[[package]] +name = "packaging" +version = "26.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/7d/fa/3944b40b07da9ce895c0e6303a5ab7d53da063554f534556b134a54d6093/packaging-26.3.tar.gz", hash = "sha256:94edc256424af38762eb31306eed28beb9f0efc50a8837492c9d6fd6004aed79", size = 313412, upload-time = "2026-08-04T18:15:28.737Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/63/34/ba1c580383c9eada3711951fef0795c80b829a078d72188184bcab9dd527/packaging-26.3-py3-none-any.whl", hash = "sha256:d7193f7c8e4e93f444fde0262bf90af30e16fa0ad0ad44cb553c87339b23cd1c", size = 129956, upload-time = "2026-08-04T18:15:27.159Z" }, +] + +[[package]] +name = "pandas" +version = "2.3.3" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "numpy" }, + { name = "python-dateutil" }, + { name = "pytz" }, + { name = "tzdata" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/33/01/d40b85317f86cf08d853a4f495195c73815fdf205eef3993821720274518/pandas-2.3.3.tar.gz", hash = "sha256:e05e1af93b977f7eafa636d043f9f94c7ee3ac81af99c13508215942e64c993b", size = 4495223, upload-time = "2025-09-29T23:34:51.853Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9c/fb/231d89e8637c808b997d172b18e9d4a4bc7bf31296196c260526055d1ea0/pandas-2.3.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:6d21f6d74eb1725c2efaa71a2bfc661a0689579b58e9c0ca58a739ff0b002b53", size = 11597846, upload-time = "2025-09-29T23:19:48.856Z" }, + { url = "https://files.pythonhosted.org/packages/5c/bd/bf8064d9cfa214294356c2d6702b716d3cf3bb24be59287a6a21e24cae6b/pandas-2.3.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:3fd2f887589c7aa868e02632612ba39acb0b8948faf5cc58f0850e165bd46f35", size = 10729618, upload-time = "2025-09-29T23:39:08.659Z" }, + { url = "https://files.pythonhosted.org/packages/57/56/cf2dbe1a3f5271370669475ead12ce77c61726ffd19a35546e31aa8edf4e/pandas-2.3.3-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ecaf1e12bdc03c86ad4a7ea848d66c685cb6851d807a26aa245ca3d2017a1908", size = 11737212, upload-time = "2025-09-29T23:19:59.765Z" }, + { url = "https://files.pythonhosted.org/packages/e5/63/cd7d615331b328e287d8233ba9fdf191a9c2d11b6af0c7a59cfcec23de68/pandas-2.3.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b3d11d2fda7eb164ef27ffc14b4fcab16a80e1ce67e9f57e19ec0afaf715ba89", size = 12362693, upload-time = "2025-09-29T23:20:14.098Z" }, + { url = "https://files.pythonhosted.org/packages/a6/de/8b1895b107277d52f2b42d3a6806e69cfef0d5cf1d0ba343470b9d8e0a04/pandas-2.3.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:a68e15f780eddf2b07d242e17a04aa187a7ee12b40b930bfdd78070556550e98", size = 12771002, upload-time = "2025-09-29T23:20:26.76Z" }, + { url = "https://files.pythonhosted.org/packages/87/21/84072af3187a677c5893b170ba2c8fbe450a6ff911234916da889b698220/pandas-2.3.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:371a4ab48e950033bcf52b6527eccb564f52dc826c02afd9a1bc0ab731bba084", size = 13450971, upload-time = "2025-09-29T23:20:41.344Z" }, + { url = "https://files.pythonhosted.org/packages/86/41/585a168330ff063014880a80d744219dbf1dd7a1c706e75ab3425a987384/pandas-2.3.3-cp312-cp312-win_amd64.whl", hash = "sha256:a16dcec078a01eeef8ee61bf64074b4e524a2a3f4b3be9326420cabe59c4778b", size = 10992722, upload-time = "2025-09-29T23:20:54.139Z" }, +] + +[[package]] +name = "pluggy" +version = "1.7.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bf/db/7fc19e6f2dc92a966727031389fc2e08b558f0f25eb7403c1119ad4713cd/pluggy-1.7.0.tar.gz", hash = "sha256:d1eaa46ebb595891b860ab086b4d09c8588af65ebd4361b8e8f4bb8920b90ba8", size = 123304, upload-time = "2026-10-15T09:50:58.343Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/40/9e/2b38731e0fc536806f16490e1a12d7f0dc2a1235aa8cc07bcc75416a7daa/pluggy-1.7.0-py3-none-any.whl", hash = "sha256:7dd7b0d8832ba3cb632c306926ded123429211b83641b35dc5c41ad2d34f9bec", size = 27082, upload-time = "2026-10-15T09:50:56.808Z" }, +] + +[[package]] +name = "protobuf" +version = "6.33.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/66/70/e908e9c5e52ef7c3a6c7902c9dfbb34c7e29c25d2f81ade3856445fd5c94/protobuf-6.33.6.tar.gz", hash = "sha256:a6768d25248312c297558af96a9f9c929e8c4cee0659cb07e780731095f38135", size = 444531, upload-time = "2026-03-18T19:05:00.988Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/fc/9f/2f509339e89cfa6f6a4c4ff50438db9ca488dec341f7e454adad60150b00/protobuf-6.33.6-cp310-abi3-win32.whl", hash = "sha256:7d29d9b65f8afef196f8334e80d6bc1d5d4adedb449971fefd3723824e6e77d3", size = 425739, upload-time = "2026-03-18T19:04:48.373Z" }, + { url = "https://files.pythonhosted.org/packages/76/5d/683efcd4798e0030c1bab27374fd13a89f7c2515fb1f3123efdfaa5eab57/protobuf-6.33.6-cp310-abi3-win_amd64.whl", hash = "sha256:0cd27b587afca21b7cfa59a74dcbd48a50f0a6400cfb59391340ad729d91d326", size = 437089, upload-time = "2026-03-18T19:04:50.381Z" }, + { url = "https://files.pythonhosted.org/packages/5c/01/a3c3ed5cd186f39e7880f8303cc51385a198a81469d53d0fdecf1f64d929/protobuf-6.33.6-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:9720e6961b251bde64edfdab7d500725a2af5280f3f4c87e57c0208376aa8c3a", size = 427737, upload-time = "2026-03-18T19:04:51.866Z" }, + { url = "https://files.pythonhosted.org/packages/ee/90/b3c01fdec7d2f627b3a6884243ba328c1217ed2d978def5c12dc50d328a3/protobuf-6.33.6-cp39-abi3-manylinux2014_aarch64.whl", hash = "sha256:e2afbae9b8e1825e3529f88d514754e094278bb95eadc0e199751cdd9a2e82a2", size = 324610, upload-time = "2026-03-18T19:04:53.096Z" }, + { url = "https://files.pythonhosted.org/packages/9b/ca/25afc144934014700c52e05103c2421997482d561f3101ff352e1292fb81/protobuf-6.33.6-cp39-abi3-manylinux2014_s390x.whl", hash = "sha256:c96c37eec15086b79762ed265d59ab204dabc53056e3443e702d2681f4b39ce3", size = 339381, upload-time = "2026-03-18T19:04:54.616Z" }, + { url = "https://files.pythonhosted.org/packages/16/92/d1e32e3e0d894fe00b15ce28ad4944ab692713f2e7f0a99787405e43533a/protobuf-6.33.6-cp39-abi3-manylinux2014_x86_64.whl", hash = "sha256:e9db7e292e0ab79dd108d7f1a94fe31601ce1ee3f7b79e0692043423020b0593", size = 323436, upload-time = "2026-03-18T19:04:55.768Z" }, + { url = "https://files.pythonhosted.org/packages/c4/72/02445137af02769918a93807b2b7890047c32bfb9f90371cbc12688819eb/protobuf-6.33.6-py3-none-any.whl", hash = "sha256:77179e006c476e69bf8e8ce866640091ec42e1beb80b213c3900006ecfba6901", size = 170656, upload-time = "2026-03-18T19:04:59.826Z" }, +] + +[[package]] +name = "pyarrow" +version = "26.0.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/ec/34/17c34cb38e5d940e38f0f0d9fdfa0e8a506676409ea9b85aff7e3079f831/pyarrow-26.0.0.tar.gz", hash = "sha256:0cccd36e00ea3afeb52ded61f2721ce71f604853d70c45365c58324eb773d6ae", size = 1239433, upload-time = "2026-10-09T08:26:25.315Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/60/6793778f2617cce469383dac0ba08c4f2401cf342df0c7b9ca53939d9b46/pyarrow-26.0.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:90ddaf7c625307ad52f31a9b25c34fe5e4897c7529ee3481135822b2b6842ff1", size = 36333953, upload-time = "2026-10-09T08:14:00.387Z" }, + { url = "https://files.pythonhosted.org/packages/db/81/f944cc63ce8a753e5fbff25de6d1d475ebd7fffdf9cf98c65130294fc896/pyarrow-26.0.0-cp312-cp312-macosx_12_0_x86_64.whl", hash = "sha256:ee341973f78a0b46e073d065e88e75026a9c584051e97f98a0d05d96c6bac7dd", size = 38688456, upload-time = "2026-10-09T08:14:04.344Z" }, + { url = "https://files.pythonhosted.org/packages/f5/2d/7e5c722fa5d5d9f3b75e62fe11694b34217664d4f05ac88031197166b277/pyarrow-26.0.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:01c863a18bd9c8412453dd0d92de6d0ee7b2b3d6fb079d9734a4b2a3c8bd4453", size = 50867603, upload-time = "2026-10-09T08:14:09.115Z" }, + { url = "https://files.pythonhosted.org/packages/88/e4/9cd356d906e71bd79b0c3fc5c9a54e01a0020dcf14c152ccfbcb503c7298/pyarrow-26.0.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:6a628922ba20705fa964ca73e4ef959c2fb2f14b9bbec5589a6a1e68e6257c85", size = 53931932, upload-time = "2026-10-09T08:14:24.051Z" }, + { url = "https://files.pythonhosted.org/packages/bb/e4/5bae3133b7fe04c24907a20f3bc1fba388cbbde659199e7b76445982047a/pyarrow-26.0.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:954d971b363b16ee41f89389a4053315dc71265f2ce5c2468eb0a910b1166268", size = 54444720, upload-time = "2026-10-09T08:14:31.214Z" }, + { url = "https://files.pythonhosted.org/packages/ba/b4/ee422493bb6dafdbef776cfe2c2a73106a1063a79bf4e78d1e5f51176885/pyarrow-26.0.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:5d5768d03426abe6526d5274adefa00abf00a7f81118c46e98b5a46390f5549e", size = 57388949, upload-time = "2026-10-09T08:14:38.964Z" }, + { url = "https://files.pythonhosted.org/packages/54/3c/1783aab1dac28e175dcf26dfc7123725efc474caecaed91e8a34cb89cad0/pyarrow-26.0.0-cp312-cp312-win_amd64.whl", hash = "sha256:cc903e1069e9dd5e9dcf780324c0112e27e051e422ecfaff574fb33ed65d9160", size = 28567581, upload-time = "2026-10-09T08:14:44.279Z" }, +] + +[[package]] +name = "pycparser" +version = "3.12" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/cb/27/eeb9f307bf07060f3d9f9aa0fb8c15b01bdfded0a941e4b2719c7f94cacd/pycparser-3.12.tar.gz", hash = "sha256:f03387877eab82ce01bbdfd0ba7252ee55a90b1c708d5129e2ce915475cf1db6", size = 158785, upload-time = "2026-10-22T13:40:03.56Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b6/ae/315bbeee126a769cbde43cf4908acb4b40408d8bf8bbf334dd7368d8dadf/pycparser-3.12-py3-none-any.whl", hash = "sha256:a076f3f489b39b43ed13563a1dafe42447445bec2e294154ec5c57fcdda4c85e", size = 62600, upload-time = "2026-10-22T13:40:02.021Z" }, +] + +[[package]] +name = "pydantic" +version = "2.14.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "annotated-types" }, + { name = "pydantic-core" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7c/0b/8e10b2e693af8ec54346a14caf36221334775975a747c9ceaa3f8371d96d/pydantic-2.14.1.tar.gz", hash = "sha256:94f478203dd03404682a1ada216965651dd74b1d2d5ffd62e00e0837caab5c26", size = 900362, upload-time = "2026-10-11T18:37:55.396Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/35/ea/a56b9fe5066f3537b7882f77e9c5dfb26d8c2eefdaed9b5fc73d57b4dc22/pydantic-2.14.1-py3-none-any.whl", hash = "sha256:9195d967ec791692a04438115466764fb8b9a27b31f14a760437694f40d6b454", size = 488106, upload-time = "2026-10-11T18:37:53.437Z" }, +] + +[[package]] +name = "pydantic-core" +version = "2.50.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a6/24/af4ec4be49fbc810f35b0bdcacc3d433b56bbfa469fd3bfd4ae116cd9bf1/pydantic_core-2.50.1.tar.gz", hash = "sha256:e50d7b94baac6c7d09927fa5ca5800a0c7ee5015c7fcff65beb3a1931b5a6e09", size = 513515, upload-time = "2026-10-11T18:35:44.82Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ee/94/101b49a6c63f99586755690696eae14532772587d9e39e93f7b8d28a49da/pydantic_core-2.50.1-cp312-cp312-macosx_10_12_x86_64.whl", hash = "sha256:704075d10b74f2f3c6e15407c696d88701df35fc8953f434a431add0d0074db0", size = 2222428, upload-time = "2026-10-11T18:32:20.228Z" }, + { url = "https://files.pythonhosted.org/packages/db/a3/f8b1b39349c33037c73d7a1b794c9c29d8f07d5bd2b74399c1375a529cc6/pydantic_core-2.50.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:e8e1d6ce820aa23317e8209a86bd65a540973c12dc7552b48a4f6c8e9926815e", size = 2025779, upload-time = "2026-10-11T18:32:21.723Z" }, + { url = "https://files.pythonhosted.org/packages/d2/2d/862a7d115305cc621358e51ba13f296e0ffdf4fbd984be9f63bc7862730b/pydantic_core-2.50.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c18db21573bd2c6489f9a544b7499f0df2853958c568e5e783536ee1f690af41", size = 2048429, upload-time = "2026-10-11T18:32:23.3Z" }, + { url = "https://files.pythonhosted.org/packages/c0/7d/fed95d18bf2b5abe2d5ab2bbec4184bfc22b6805994a33ccdae8edffd25a/pydantic_core-2.50.1-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:cb57f304525a5e3c13333b772bf9a473f36326e9c821b2e8e1b2fd36f80ae2c3", size = 2173274, upload-time = "2026-10-11T18:32:24.697Z" }, + { url = "https://files.pythonhosted.org/packages/93/1e/c6f22ceac65dc3d291ff79bea0eb74cbd1de8b128f01fee48af03bcea1e1/pydantic_core-2.50.1-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a27c09d86600f1bf2fe3f37e1ae697faf3143931c09322cd799da94deee923b5", size = 2360403, upload-time = "2026-10-11T18:32:26.281Z" }, + { url = "https://files.pythonhosted.org/packages/2c/97/190b5b6bfe4c72f265feb657913eb30749193927a5ad7b83cfa7866a25a2/pydantic_core-2.50.1-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:46b3301d3b5c886f77de7546e47274a5842c622ea2020b8c6524c6b66913b4a6", size = 2439799, upload-time = "2026-10-11T18:32:27.844Z" }, + { url = "https://files.pythonhosted.org/packages/90/24/48cd98388e4ac2c08d0800af36d15413db15e20387f1038123a2c86cb2f6/pydantic_core-2.50.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:93ba4e9d8210d941c200431a56b2c0400b131865947903937ed3ec5404307d2e", size = 2160157, upload-time = "2026-10-11T18:32:29.553Z" }, + { url = "https://files.pythonhosted.org/packages/00/86/d909133d0b9979ed14c20f110f21124a0e346caf071bd66ac5af3009062b/pydantic_core-2.50.1-cp312-cp312-manylinux_2_31_riscv64.whl", hash = "sha256:e5faeaee74a57d32b3ab3aebad2e348f06d3ba946fc5d28c1728455f00a3d13a", size = 2229523, upload-time = "2026-10-11T18:32:31.027Z" }, + { url = "https://files.pythonhosted.org/packages/90/31/4b43f08131b2098e73a852fd932063385f3103b33839fd9d1abdb89e673d/pydantic_core-2.50.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:a3cda0e538208e5d722bbf3698b24f19c0a7d05bc8d5f8a7f9b121ea7fa243d9", size = 2242700, upload-time = "2026-10-11T18:32:32.622Z" }, + { url = "https://files.pythonhosted.org/packages/88/77/de39658569dc7a4c293d110a89629f61145e7538f00d1e4280c28debefd3/pydantic_core-2.50.1-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:57f51b31ff826e2859120cf4737c5a758a48d96f3e97da40ccee1796d58078ff", size = 2278452, upload-time = "2026-10-11T18:32:34.165Z" }, + { url = "https://files.pythonhosted.org/packages/46/bd/1eeb9e4e791e81210ef9096f58080b32ba15b5b13c108d50aaa3f8ce9f73/pydantic_core-2.50.1-cp312-cp312-musllinux_1_1_armv7l.whl", hash = "sha256:8daa7ee75245d43ad7d747e5c9ecc1b1d06552f72b14887e9276f787d57375f4", size = 2450098, upload-time = "2026-10-11T18:32:35.547Z" }, + { url = "https://files.pythonhosted.org/packages/1c/e6/b2accfdff17e5a88d6358536702146de00f9339adaa4d5bbe1ee93cb4832/pydantic_core-2.50.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:acbf31f37c53a5ac0c34706c80b4f5107ba20b05fdd3816124bf236ef0c57dd2", size = 2476839, upload-time = "2026-10-11T18:32:37.119Z" }, + { url = "https://files.pythonhosted.org/packages/3b/7e/c8a83eb5d2cd42c2e13507150fb777b1edea5a018adc9cbf2858a1d08285/pydantic_core-2.50.1-cp312-cp312-win32.whl", hash = "sha256:45b11cac094aa25725581d9304eee93c9028516b9ea80dd9e175e13a5a2c840e", size = 2101425, upload-time = "2026-10-11T18:32:38.593Z" }, + { url = "https://files.pythonhosted.org/packages/c5/17/8bbd530b8659e9d963f5b16f6cb8e159f4cd174b4d6304e48969ab12dba8/pydantic_core-2.50.1-cp312-cp312-win_amd64.whl", hash = "sha256:132529c83901437ff642f585216831bf5fd7a91df66829907e155192ead62498", size = 2172522, upload-time = "2026-10-11T18:32:40.112Z" }, + { url = "https://files.pythonhosted.org/packages/af/9f/a79d690b127e4e3cff143d031538ab9aa715f6b2fbeecbd07cd6f2bf9fb7/pydantic_core-2.50.1-cp312-cp312-win_arm64.whl", hash = "sha256:4e834f6a8e4ff772dcc34f58ef5504147a3ea5b0f4eeb13b0f8eb2ca75ac57f1", size = 2161366, upload-time = "2026-10-11T18:32:41.606Z" }, + { url = "https://files.pythonhosted.org/packages/a5/09/1bcf160f3cd6333e2a76982fae5963e5032de0bcaaba38464223dc266bfe/pydantic_core-2.50.1-graalpy312-graalpy250_312_native-macosx_10_12_x86_64.whl", hash = "sha256:76e2e83fa6ec8cdc972d438dafc2522b3a47bee4ec0ae668b29cfb1977ab5242", size = 2217070, upload-time = "2026-10-11T18:35:15.688Z" }, + { url = "https://files.pythonhosted.org/packages/3d/81/f02de95eec7794dfc7a5bc69a834f0e834b31b3009014ce6de903855f775/pydantic_core-2.50.1-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:a51eee75939cf811ac09b278745a6cee7dc873ccfbc8b9af3cc88fe4b7ce25b5", size = 2039027, upload-time = "2026-10-11T18:35:18.07Z" }, + { url = "https://files.pythonhosted.org/packages/4d/16/227746ed3771d9bf2304568b55134849ece34e13b38b1d76de763015c77f/pydantic_core-2.50.1-graalpy312-graalpy250_312_native-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ae28183297fb0d2b8dc46a1f01d51f5e45825fc5afe76a835a6cb7fb34821295", size = 2109009, upload-time = "2026-10-11T18:35:20.218Z" }, + { url = "https://files.pythonhosted.org/packages/e5/b9/7664d1592a0e5a74903f357cd5b28ceb7336f3473256c5a4e8c8432f1881/pydantic_core-2.50.1-graalpy312-graalpy250_312_native-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:88e492e8b9d0312e7dc13667c30222abf284dc3b79b5302b3607b41a5784ce61", size = 2269883, upload-time = "2026-10-11T18:35:22.48Z" }, +] + +[[package]] +name = "pydantic-settings" +version = "2.16.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "pydantic" }, + { name = "python-dotenv" }, + { name = "typing-extensions" }, + { name = "typing-inspection" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/05/ca/7817b4bf0c4b15c6a6ab8cdf4cb96c268335a8ca2f525de93b6ebe4c7d28/pydantic_settings-2.16.1.tar.gz", hash = "sha256:aff602a1886338c2df3d1e958e2193a8dc33cec75b91f202e83fd98484b4b0da", size = 303436, upload-time = "2026-10-20T11:37:19.024Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/3c/74/fdf926afe145d8629c4b2ff57f4ea1d64ed7abc4d84d97463b2517a2547e/pydantic_settings-2.16.1-py3-none-any.whl", hash = "sha256:30f5313340da4e8f1b440d1910b92a5bc3ccbb6b4ba70ec2b8395bc921f418f5", size = 80771, upload-time = "2026-10-20T11:37:17.298Z" }, +] + +[[package]] +name = "pygments" +version = "2.21.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/49/2e/ced460408999b33da6b31b0021b0f37d329e202d4169aeb164493778f25b/pygments-2.21.0.tar.gz", hash = "sha256:610ca751c9bc2492b38eb9a38a7fbc93edbbb2d7182edaf34e66ae493dee5c8c", size = 5005329, upload-time = "2026-08-17T08:02:48.824Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/71/46/17f022dd3e953bf20a04a028a21ec746d942f8d2af30fa0f124fa0e6a684/pygments-2.21.0-py3-none-any.whl", hash = "sha256:2363c69b61c4a97c838da3b130dcd6468f4848992b21a82f2a63ec34377137d9", size = 1250147, upload-time = "2026-08-17T08:02:44.912Z" }, +] + +[[package]] +name = "pytest" +version = "8.4.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "iniconfig" }, + { name = "packaging" }, + { name = "pluggy" }, + { name = "pygments" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a3/5c/00a0e072241553e1a7496d638deababa67c5058571567b92a7eaa258397c/pytest-8.4.2.tar.gz", hash = "sha256:86c0d0b93306b961d58d62a4db4879f27fe25513d4b969df351abdddb3c30e01", size = 1519618, upload-time = "2025-09-04T14:34:22.711Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a8/a4/20da314d277121d6534b3a980b29035dcd51e6744bd79075a6ce8fa4eb8d/pytest-8.4.2-py3-none-any.whl", hash = "sha256:872f880de3fc3a5bdc88a11b39c9710c3497a547cfa9320bc3c5e62fbf272e79", size = 365750, upload-time = "2025-09-04T14:34:20.226Z" }, +] + +[[package]] +name = "pytest-xdist" +version = "3.8.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "execnet" }, + { name = "pytest" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/78/b4/439b179d1ff526791eb921115fca8e44e596a13efeda518b9d845a619450/pytest_xdist-3.8.0.tar.gz", hash = "sha256:7e578125ec9bc6050861aa93f2d59f1d8d085595d6551c2c90b6f4fad8d3a9f1", size = 88069, upload-time = "2025-07-01T13:30:59.346Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ca/31/d4e37e9e550c2b92a9cbc2e4d0b7420a27224968580b5a447f420847c975/pytest_xdist-3.8.0-py3-none-any.whl", hash = "sha256:202ca578cfeb7370784a8c33d6d05bc6e13b4f25b5053c30a152269fd10f0b88", size = 46396, upload-time = "2025-07-01T13:30:56.632Z" }, +] + +[[package]] +name = "python-dateutil" +version = "3.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "six" }, + { name = "tzdata" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0e/df/f15f80383f152452d8985b36d95cb8751f901a3e0ff4cd27f7e118f168e2/python_dateutil-3.0.0.tar.gz", hash = "sha256:7ebcb3c29a92f3ceafe79a7a48a5c45677e9e004bbeec5742ac0bbb00560b7a2", size = 271066, upload-time = "2026-10-21T23:41:45.075Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/5a/17/cebee01d1c0c1441174cf0456ddc91b280d79acb65340c05e41d77866ee8/python_dateutil-3.0.0-py2.py3-none-any.whl", hash = "sha256:3c79630202e4152c6add678446984236efe4f57e84d8e892aab0637bab82e9fd", size = 94659, upload-time = "2026-10-21T23:41:43.431Z" }, +] + +[[package]] +name = "python-dotenv" +version = "1.2.4" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/74/26/2fbeedb218a787a5eea551c7532cac4e009f83d689dd2faa0d0353473f86/python_dotenv-1.2.4.tar.gz", hash = "sha256:f0d53e69935a851c0dcc78f3ab7aaccd8cabef0b92382b576b824212902873c0", size = 60824, upload-time = "2026-10-01T05:36:10Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/60/d1/38f3a3405989a89ac18390803e70c6ad7c7760da4f9b83cbeca0c44a0c72/python_dotenv-1.2.4-py3-none-any.whl", hash = "sha256:42269a8a5b3fd54ffa6f3d84b18abed50064717576b4ecf03dc4a55d8aa04fdc", size = 23266, upload-time = "2026-10-01T05:36:08.633Z" }, +] + +[[package]] +name = "python-multipart" +version = "0.0.32" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/5b/42/55c32bb9b12693c092ad250a0e82edb5b31ddeda6eb772de5f308b3804ad/python_multipart-0.0.32.tar.gz", hash = "sha256:be54b7f3fa167bb83e4fcd936b887b708f4e57fe75911c02aebf53efaf8d938e", size = 46881, upload-time = "2026-06-04T16:18:58.647Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e1/04/e8135ebd1ad02c56ec633277529b2602ff99ff634be76cdba5744cf554fd/python_multipart-0.0.32-py3-none-any.whl", hash = "sha256:ff6d3f776f16878c894e52e107296ffc890e913c611b1a4ec6c44e2821fe2e23", size = 30042, upload-time = "2026-06-04T16:18:57.319Z" }, +] + +[[package]] +name = "pytz" +version = "2026.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/14/21/d83d6ef28c4c912c4bb4d1dcf591f7b8c6bde87b9c66f9f454677314e16d/pytz-2026.5.tar.gz", hash = "sha256:fa23724b9c486543b9ff54a327ee7569ac83ade54bb9afd0fc18676620401c86", size = 318572, upload-time = "2026-10-04T02:37:58.719Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4f/ef/c66110d46fb800dda0bf33164182dfadabe26a90e4476844d502a23dca8e/pytz-2026.5-py2.py3-none-any.whl", hash = "sha256:e658af3757f9e26a9d25dd2aff38335acd92bc9104f890a894b2c1ba28311b03", size = 506342, upload-time = "2026-10-04T02:37:56.814Z" }, +] + +[[package]] +name = "pyyaml" +version = "6.0.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/05/8e/961c0007c59b8dd7729d542c61a4d537767a59645b82a0b521206e1e25c2/pyyaml-6.0.3.tar.gz", hash = "sha256:d76623373421df22fb4cf8817020cbb7ef15c725b9d5e45f17e189bfc384190f", size = 130960, upload-time = "2025-09-25T21:33:16.546Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/33/422b98d2195232ca1826284a76852ad5a86fe23e31b009c9886b2d0fb8b2/pyyaml-6.0.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:7f047e29dcae44602496db43be01ad42fc6f1cc0d8cd6c83d342306c32270196", size = 182063, upload-time = "2025-09-25T21:32:11.445Z" }, + { url = "https://files.pythonhosted.org/packages/89/a0/6cf41a19a1f2f3feab0e9c0b74134aa2ce6849093d5517a0c550fe37a648/pyyaml-6.0.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:fc09d0aa354569bc501d4e787133afc08552722d3ab34836a80547331bb5d4a0", size = 173973, upload-time = "2025-09-25T21:32:12.492Z" }, + { url = "https://files.pythonhosted.org/packages/ed/23/7a778b6bd0b9a8039df8b1b1d80e2e2ad78aa04171592c8a5c43a56a6af4/pyyaml-6.0.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9149cad251584d5fb4981be1ecde53a1ca46c891a79788c0df828d2f166bda28", size = 775116, upload-time = "2025-09-25T21:32:13.652Z" }, + { url = "https://files.pythonhosted.org/packages/65/30/d7353c338e12baef4ecc1b09e877c1970bd3382789c159b4f89d6a70dc09/pyyaml-6.0.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:5fdec68f91a0c6739b380c83b951e2c72ac0197ace422360e6d5a959d8d97b2c", size = 844011, upload-time = "2025-09-25T21:32:15.21Z" }, + { url = "https://files.pythonhosted.org/packages/8b/9d/b3589d3877982d4f2329302ef98a8026e7f4443c765c46cfecc8858c6b4b/pyyaml-6.0.3-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:ba1cc08a7ccde2d2ec775841541641e4548226580ab850948cbfda66a1befcdc", size = 807870, upload-time = "2025-09-25T21:32:16.431Z" }, + { url = "https://files.pythonhosted.org/packages/05/c0/b3be26a015601b822b97d9149ff8cb5ead58c66f981e04fedf4e762f4bd4/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:8dc52c23056b9ddd46818a57b78404882310fb473d63f17b07d5c40421e47f8e", size = 761089, upload-time = "2025-09-25T21:32:17.56Z" }, + { url = "https://files.pythonhosted.org/packages/be/8e/98435a21d1d4b46590d5459a22d88128103f8da4c2d4cb8f14f2a96504e1/pyyaml-6.0.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:41715c910c881bc081f1e8872880d3c650acf13dfa8214bad49ed4cede7c34ea", size = 790181, upload-time = "2025-09-25T21:32:18.834Z" }, + { url = "https://files.pythonhosted.org/packages/74/93/7baea19427dcfbe1e5a372d81473250b379f04b1bd3c4c5ff825e2327202/pyyaml-6.0.3-cp312-cp312-win32.whl", hash = "sha256:96b533f0e99f6579b3d4d4995707cf36df9100d67e0c8303a0c55b27b5f99bc5", size = 137658, upload-time = "2025-09-25T21:32:20.209Z" }, + { url = "https://files.pythonhosted.org/packages/86/bf/899e81e4cce32febab4fb42bb97dcdf66bc135272882d1987881a4b519e9/pyyaml-6.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:5fcd34e47f6e0b794d17de1b4ff496c00986e1c83f7ab2fb8fcfe9616ff7477b", size = 154003, upload-time = "2025-09-25T21:32:21.167Z" }, + { url = "https://files.pythonhosted.org/packages/1a/08/67bd04656199bbb51dbed1439b7f27601dfb576fb864099c7ef0c3e55531/pyyaml-6.0.3-cp312-cp312-win_arm64.whl", hash = "sha256:64386e5e707d03a7e172c0701abfb7e10f0fb753ee1d773128192742712a98fd", size = 140344, upload-time = "2025-09-25T21:32:22.617Z" }, +] + +[[package]] +name = "six" +version = "1.17.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, +] + +[[package]] +name = "snowflake-connector-python" +version = "5.1.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cryptography" }, + { name = "protobuf" }, + { name = "pytz" }, + { name = "tomlkit" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/7a/e4/05727a3af8c240ff6880c0becf729968a34ef691f1bb9f6e0d983bb685e6/snowflake_connector_python-5.1.0.tar.gz", hash = "sha256:2d9eec7fbaa329b4013b9d67e01e2120fe2878854f6de421cc9d07c620fa270d", size = 1950546, upload-time = "2026-10-21T21:33:05.19Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/32/e7/0519ab883f7073f356b99ad84475011759578949abfaf9ac0325cbd899d7/snowflake_connector_python-5.1.0-cp312-cp312-macosx_10_14_x86_64.whl", hash = "sha256:86fb60cace989dd6d27769a2bfffc0b5e894095776140006e6c57fd4b5065070", size = 11229627, upload-time = "2026-10-21T21:32:09.051Z" }, + { url = "https://files.pythonhosted.org/packages/14/d3/c7bd1b127d2b2fe57d01e1ba29df24ca4a400f2a9fb4da2400185af597e8/snowflake_connector_python-5.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:525eca4af24aed27fed48c8500c2a2c6490b613b3813260a867d4430fc08dcbe", size = 10581308, upload-time = "2026-10-21T21:31:57.59Z" }, + { url = "https://files.pythonhosted.org/packages/29/22/0dc1479ff11662e6e2e65aeab6953c65efeaceb444cb2d95c339f00615c6/snowflake_connector_python-5.1.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:7eb533a075dae3b76b9567ecdcb6c66d57b97156cc4366f349843e49f295746a", size = 14241926, upload-time = "2026-10-21T21:32:33.682Z" }, + { url = "https://files.pythonhosted.org/packages/f4/ac/fb606e7be6c58d01175cf6f7e05b84abeaddccccbc5cde93e846aa2ea7eb/snowflake_connector_python-5.1.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:e899d9d69bc7fb40ab531c3f34049d064dae57fcc4805d0c286aa35570e683de", size = 15249669, upload-time = "2026-10-21T21:32:20.298Z" }, + { url = "https://files.pythonhosted.org/packages/82/75/4d20ce4e4f0704d35ebfbc17fcc62c3a9f5bca2a5345509174590686fdb2/snowflake_connector_python-5.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:ebbdfa1a53548bb308460d75f132cc85f45dc618c16ba450b6e1486d4d08050b", size = 11567901, upload-time = "2026-10-21T21:32:48.965Z" }, + { url = "https://files.pythonhosted.org/packages/e7/37/c609d9599e6831854cca14e00122f586e14a09cacb5d223744563cdc80f2/snowflake_connector_python-5.1.0-cp312-cp312-win_arm64.whl", hash = "sha256:64c15ed531ff03d42c383250ffd87b7a81ff9ba78cb6b4afaaefae36e8109851", size = 10880368, upload-time = "2026-10-21T21:33:02.974Z" }, +] + +[package.optional-dependencies] +pandas = [ + { name = "pandas" }, + { name = "pyarrow" }, +] + +[[package]] +name = "starlette" +version = "1.9.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/ad/eb/4e197bf26b023abafc5e9043ec8710d7de0fcb032d0969045236d72aa0b1/starlette-1.9.0.tar.gz", hash = "sha256:3d931de246ed870e35ce605974440816ef895e592f255d1396cfdf191d4c4318", size = 2732466, upload-time = "2026-10-18T15:50:33.189Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f3/93/181df62699d9b5aca1d87e5cfda131dac87e3cdaebf668662f7206ca3470/starlette-1.9.0-py3-none-any.whl", hash = "sha256:fd3f159936e882d06c1108f84688f7cda2cf85fc6a592a50edbd175e882c2ddc", size = 80335, upload-time = "2026-10-18T15:50:31.526Z" }, +] + +[[package]] +name = "tenantboard" +version = "0.1.0" +source = { virtual = "." } +dependencies = [ + { name = "cryptography" }, + { name = "fastapi" }, + { name = "pydantic" }, + { name = "pydantic-settings" }, + { name = "python-multipart" }, + { name = "snowflake-connector-python", extra = ["pandas"] }, + { name = "uvicorn", extra = ["standard"] }, +] + +[package.dev-dependencies] +dev = [ + { name = "httpx" }, + { name = "pytest" }, + { name = "pytest-xdist" }, +] + +[package.metadata] +requires-dist = [ + { name = "cryptography", specifier = ">=43" }, + { name = "fastapi", specifier = ">=0.115,<1" }, + { name = "pydantic", specifier = ">=2.9,<3" }, + { name = "pydantic-settings", specifier = ">=2.6,<3" }, + { name = "python-multipart", specifier = ">=0.0.12" }, + { name = "snowflake-connector-python", extras = ["aio", "pandas"], specifier = ">=5.1,<6" }, + { name = "uvicorn", extras = ["standard"], specifier = ">=0.32,<1" }, +] + +[package.metadata.requires-dev] +dev = [ + { name = "httpx", specifier = ">=0.27,<1" }, + { name = "pytest", specifier = ">=8.3,<9" }, + { name = "pytest-xdist", specifier = ">=3.6,<4" }, +] + +[[package]] +name = "tomlkit" +version = "0.15.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/08/72/fc01563df95c189a9f65850096bdc205bc2fe37da6c740cd64afaf18d07e/tomlkit-0.15.2.tar.gz", hash = "sha256:83e32755ba1d20d9340bd069ee97e9f6232ba2346b63ffcba425de01f7e327a6", size = 190522, upload-time = "2026-10-16T04:07:08.166Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/da/46/7d84138a60a8bd796825b2d38ebe92d0d16501ca57d501b7c5d677401bbc/tomlkit-0.15.2-py3-none-any.whl", hash = "sha256:05562412163a3037b94497385fb7f9204c637bc605af7a48199e41cf2272f437", size = 52225, upload-time = "2026-10-16T04:07:09.216Z" }, +] + +[[package]] +name = "typing-extensions" +version = "4.16.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/f6/cc/6253133b5bb138fc3306cebfbda2c520f545d36b5be2c7255cc528bb45d6/typing_extensions-4.16.0.tar.gz", hash = "sha256:dc983d19a509c94dba722ee6abd33940f7c05a89e243c47e907eb4db6f1a43e5", size = 113555, upload-time = "2026-07-02T08:40:05.92Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/49/d3/b8441a820a491ddfc024b0b0cf0393375b75ea13866d9c66727e54c2fc80/typing_extensions-4.16.0-py3-none-any.whl", hash = "sha256:481caa481374e813c1b176ada14e97f1f67a4539ce9cfeb3f350d78d6370c2e8", size = 45571, upload-time = "2026-07-02T08:40:04.659Z" }, +] + +[[package]] +name = "typing-inspection" +version = "0.4.4" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/a3/26/b09b8010994eccc3c09092e6b34058f36a460eea2d4c3e8b910c695975a0/typing_inspection-0.4.4.tar.gz", hash = "sha256:547274fa6b0a561ccf549cc9524b999a578e737d015d8709d021f9d0d13bea47", size = 76928, upload-time = "2026-08-12T12:37:25.997Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/67/81/4add07e5172b7ac40d8ed5ff580409a7801a4fe26d529bdd915401dabfbe/typing_inspection-0.4.4-py3-none-any.whl", hash = "sha256:65b8397ba37ccbce054456aaccddfc91e6e3083c92824df348d96ca832f3f147", size = 14750, upload-time = "2026-08-12T12:37:24.648Z" }, +] + +[[package]] +name = "tzdata" +version = "2026.5" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/d9/68/f1b440335057bfce71b6e50a9d09445aa2ecbd08359a337976627b8409e7/tzdata-2026.5.tar.gz", hash = "sha256:8cc73c0a0bfca7dbfa59235d60b2eff82231dee33f53d206db1acd9173cfc0a7", size = 200404, upload-time = "2026-10-03T09:23:14.143Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/94/21/1e5995a1c920cce14e4bffae20c665ec10e7ed03ab25e006cd741092b718/tzdata-2026.5-py2.py3-none-any.whl", hash = "sha256:b683bd1b6659ddcd810ff02ad09ba821d4bf1065072805063eb35c49617905ac", size = 347996, upload-time = "2026-10-03T09:23:12.535Z" }, +] + +[[package]] +name = "uvicorn" +version = "0.54.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "click" }, + { name = "h11" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/da/34/30e9280707135d2cfc589dfff3cb796bd07a3aeb1a3e415ba09dd89d7bb4/uvicorn-0.54.0.tar.gz", hash = "sha256:a2e33cbfaa0306f8e6b0c13e0cb89d7d7a2da3e62b90c66e18c33d9807b28620", size = 112283, upload-time = "2026-09-25T06:52:37.601Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/38/0c/b54a4fdd7f90a3af8b02ebc9ce6712c2c208b7926a2f7bad95c33ebbe943/uvicorn-0.54.0-py3-none-any.whl", hash = "sha256:505bdb0f318731d45f1f712071fc781a8981f6847a31c902c9f5e652d4f67faf", size = 87427, upload-time = "2026-09-25T06:52:35.829Z" }, +] + +[package.optional-dependencies] +standard = [ + { name = "httptools" }, + { name = "python-dotenv" }, + { name = "pyyaml" }, + { name = "uvloop", marker = "platform_python_implementation != 'PyPy' and sys_platform != 'cygwin' and sys_platform != 'win32'" }, + { name = "watchfiles" }, + { name = "websockets" }, +] + +[[package]] +name = "uvloop" +version = "0.23.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fa/42/02c739ce85fb2ee8d99212c61417da8140c6b87e9d97c430bea520d76044/uvloop-0.23.0.tar.gz", hash = "sha256:28d160f51ab4da3b187063652e643dea6831072add4adc1e6d62afbe73b6be27", size = 2559185, upload-time = "2026-10-01T03:17:04.4Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/05/98/04e766a6de99e6f7f955ecb7829e8d5a557de3427cb85be2236de54dda0c/uvloop-0.23.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:93935ab27b6eaef4c3e5489aebc84284f0644592f7ab516df60ee1b27eaf5eb3", size = 1393055, upload-time = "2026-10-01T03:15:42.526Z" }, + { url = "https://files.pythonhosted.org/packages/33/8a/499e7b863a848ede009539bce39806b66205da5f8779354228e785601144/uvloop-0.23.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4448e9124537620f9c25d004c227bb5104440b58955c19bbd312d910af919a63", size = 768909, upload-time = "2026-10-01T03:15:43.974Z" }, + { url = "https://files.pythonhosted.org/packages/3d/95/a880f8ce3b87ac5b307c354e8ee480be4658d24bf01f87921d57e3530b4a/uvloop-0.23.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f7548ede3ee908cfabc0d068106e303a9a2d811af959cdf6ab85676344cedcda", size = 4419106, upload-time = "2026-10-01T03:15:45.551Z" }, + { url = "https://files.pythonhosted.org/packages/51/27/c1d2f9fa977f8f42ea294604166df10e0027e6dc6cd17f85ede386c9bf36/uvloop-0.23.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:090865d8ce7a03986755a3ce711b7dd0d4b44eb14ab74368b717f3fad1180208", size = 4532597, upload-time = "2026-10-01T03:15:47.258Z" }, + { url = "https://files.pythonhosted.org/packages/42/dd/2cb6a2c8a30ca55c07a882dd4ae4ceae0fa7d8c15b25b3b7cb9a4b6cf4ca/uvloop-0.23.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:bd6f2f81c7b9da99d301c0b16b82044e76fe887086e42e1590ecf520b94dbdac", size = 4230048, upload-time = "2026-10-01T03:15:49.119Z" }, + { url = "https://files.pythonhosted.org/packages/f4/52/29989cbaa4022dc4ef35c1dd60a4ab989e4c2065f341ed483ae71d2bd950/uvloop-0.23.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:a6ac96da66c35bf789bdcde78a88dc7d56b7907d8379648c54adc1c61594575d", size = 4394152, upload-time = "2026-10-01T03:15:50.829Z" }, +] + +[[package]] +name = "watchfiles" +version = "1.3.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "anyio" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/b3/68/e6aa0b77d217b31f8f486ec0cdfe5e00e6e38dc0be657e7d85819b9faf0a/watchfiles-1.3.0.tar.gz", hash = "sha256:99aee4a07847c06820765fd7b1b49ceac4f3f711ccb7d104655a33231de1c207", size = 108119, upload-time = "2026-09-21T09:08:55.664Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/68/fa/c0b840d5d8bafe640925408ce11095948c8945bb61be1e67d0ab629b872a/watchfiles-1.3.0-cp310-abi3-macosx_10_12_x86_64.whl", hash = "sha256:000b9688fc8133037a8b075c8ebf98f32844ff8964dda61e85c1db547dafc441", size = 399703, upload-time = "2026-09-21T09:08:13.518Z" }, + { url = "https://files.pythonhosted.org/packages/c7/8a/894799b485fe9473ad10422a0d9668e53fb78e3a2b5cc6061159572844ad/watchfiles-1.3.0-cp310-abi3-macosx_11_0_arm64.whl", hash = "sha256:bbc1198edfdc90fda0600f825aa94150f428dfcbf8138746f55998e0e660d64c", size = 397647, upload-time = "2026-09-21T09:08:15.16Z" }, + { url = "https://files.pythonhosted.org/packages/5a/61/f277cd5ead05f5d37b98b41fbdd0c943ee0c04d49738f62e902cb291cf43/watchfiles-1.3.0-cp310-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4af9c464d410e8c44b58ddf1cbec7c3a02c1cb2c6c80df9c74659ba2fbda0ab5", size = 454994, upload-time = "2026-09-21T09:08:16.587Z" }, + { url = "https://files.pythonhosted.org/packages/46/66/ea4c01382975e53b1ac1b0a0ea1b353416b8127fb6e04f5c1c313119aab6/watchfiles-1.3.0-cp310-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:eb94e40b9a0db19636b5e3fed0a9803452ecc7381df13b385272abb779ee83ca", size = 460105, upload-time = "2026-09-21T09:08:17.961Z" }, + { url = "https://files.pythonhosted.org/packages/31/c3/1a501b597817096c9831c255f6270c124ce0e2faeb74124be24dcc93b86d/watchfiles-1.3.0-cp310-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7398930a2b76b9dc67a4e6b8bfc7baf30946892a86f417a3920c07bdd6febc30", size = 493761, upload-time = "2026-09-21T09:08:19.327Z" }, + { url = "https://files.pythonhosted.org/packages/5a/16/77352d05e152c7786a016e9f1decdb4507e8aa5bcf3343c72142a0413d14/watchfiles-1.3.0-cp310-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:025b108f2d5cc2799cb941f79380df3e8b22bfb589c218e801767fe78d598c95", size = 577500, upload-time = "2026-09-21T09:08:20.723Z" }, + { url = "https://files.pythonhosted.org/packages/95/fe/81001c55b24466fe9911135e50269fd3a29f43e446627a92adfab54553b6/watchfiles-1.3.0-cp310-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:be9ef3cd403d756a304a0a08a112163c63da7ec72c8c9d083c98818291ff8f29", size = 469209, upload-time = "2026-09-21T09:08:22.346Z" }, + { url = "https://files.pythonhosted.org/packages/8a/cf/e3ad894ed5909a7c44b0707e89bc25d19a3d78b3a7aec846119443b6ae56/watchfiles-1.3.0-cp310-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b5768b49e426fd5b550b012c866db347cdf15c398ef98dc557b6e6b72fa74cd1", size = 458353, upload-time = "2026-09-21T09:08:23.7Z" }, + { url = "https://files.pythonhosted.org/packages/7f/f5/2f106829e6e2bc91ad74614730f6df484202b9a4d20d927dbf2f08d37d44/watchfiles-1.3.0-cp310-abi3-manylinux_2_31_riscv64.whl", hash = "sha256:d36c72fcdc08f143d87316a4c4c3b064e340da2c7383d39803950740ed418935", size = 468707, upload-time = "2026-09-21T09:08:25.167Z" }, + { url = "https://files.pythonhosted.org/packages/15/37/2741ab776b978669c1f3bba1e573f1a184f727ab8b4d3a6d9c61090800ed/watchfiles-1.3.0-cp310-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:f0c9865240e065a2247f4b5248528ae3d01e0e8ae071462c250c884dc16e3ee3", size = 632510, upload-time = "2026-09-21T09:08:26.532Z" }, + { url = "https://files.pythonhosted.org/packages/48/64/db4a4684275fe9b696ee5118c194705c1423b3d42181c3f8ee70194d5c73/watchfiles-1.3.0-cp310-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:380f513d26cc2e598266b88d66456e07d67ef4be8f0f435a1154d9c44b43d509", size = 661723, upload-time = "2026-09-21T09:08:28.004Z" }, + { url = "https://files.pythonhosted.org/packages/52/80/95ddb24f6a2f595c031c24432ab7b8aa7d6d1ada0c6d63a552166e06e8d1/watchfiles-1.3.0-cp310-abi3-win32.whl", hash = "sha256:509d9f74d2bec5c1f4cd868ddcfe0c9601f7ea53066dfcb19d8ede742c4de661", size = 277761, upload-time = "2026-09-21T09:08:29.335Z" }, + { url = "https://files.pythonhosted.org/packages/48/a1/1c4a1b3c8030cd03206232612be8a6e8799093490c220b0fcb935d02b75a/watchfiles-1.3.0-cp310-abi3-win_amd64.whl", hash = "sha256:1acabde19b67e673274e89a04d613b7fb1f122e2c4b8ec9a581e59c149264b61", size = 290643, upload-time = "2026-09-21T09:08:30.45Z" }, + { url = "https://files.pythonhosted.org/packages/82/e0/d2db577c632d26d08078c4390dd870b8555c4e558df25d22270f0166c71f/watchfiles-1.3.0-cp310-abi3-win_arm64.whl", hash = "sha256:d978cc1dd7ba44f5590d7de74f7a9c8be7262d9e469c2c0efb8bbfc2574321dc", size = 285510, upload-time = "2026-09-21T09:08:31.561Z" }, +] + +[[package]] +name = "websockets" +version = "17.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/30/4b/4692be6d5bf739ed049b3813901c42091f8c054df89d5c089343823dea0d/websockets-17.3.tar.gz", hash = "sha256:1c7dd93a1c9f89297471493674b1fc1d9467cafd45374f2172213046bdb3a329", size = 188878, upload-time = "2026-10-19T16:58:05.813Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/0d/29/902ffe19c0d35603102f31490dea342f083bc77d8f8bcdc7d8b16f5810f9/websockets-17.3-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:41665c7ea01d3ae497a0c61fd22d3424230d18782a5fceb79f04db9ccb2042c9", size = 218250, upload-time = "2026-10-19T16:54:54.718Z" }, + { url = "https://files.pythonhosted.org/packages/f0/66/f56a2637b1aac263205b0e3cd7b2ff2e2f8b404b8118c991f4d122fd381f/websockets-17.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:8f7469408740cf06a79b01605f2ae0b2acc8839c03fcbbf05da49baee12d3302", size = 215927, upload-time = "2026-10-19T16:54:56.124Z" }, + { url = "https://files.pythonhosted.org/packages/22/7d/a27f25451c4cc62141992b51005688481210f3d3e63de3dc66f9e882ba59/websockets-17.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d68da281ac208b6236ffa8beef23fe7c7c956d8bf09b421d447232fd94441b35", size = 216196, upload-time = "2026-10-19T16:54:57.538Z" }, + { url = "https://files.pythonhosted.org/packages/9f/77/40bbc1a5c016cf9f598dea21d1250a93245c995eafaec6401d0e4eb4e647/websockets-17.3-cp312-cp312-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:078d1d41f45ee34eed5a4f3bbe7c588caecbdd6e12dfca73e8c1b0e803773793", size = 225516, upload-time = "2026-10-19T16:54:59.104Z" }, + { url = "https://files.pythonhosted.org/packages/3d/9c/91889c8c62fb7d16cd58e3074473a259303d6229bf0ce496aec87e66f3dc/websockets-17.3-cp312-cp312-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:a7f391838e8adcf1f2fe71561d9e5a9f74ea7ffa5052eb2db329aac677383136", size = 225795, upload-time = "2026-10-19T16:55:00.438Z" }, + { url = "https://files.pythonhosted.org/packages/b8/29/2e5e2158e9920f806cd45f3cbc81cf5f3eef2b9e5a74b9ef049e2a3578e3/websockets-17.3-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c15a165673bb3fb57facbf040edf5c0556d20ccd2e0ccc5f23ebb69fb715fdc1", size = 227034, upload-time = "2026-10-19T16:55:02.065Z" }, + { url = "https://files.pythonhosted.org/packages/b6/af/fbab5c4098b26026b877bfc139b1ea0d7c6eaf7a86b84544933b4c445f82/websockets-17.3-cp312-cp312-manylinux2014_armv7l.manylinux_2_17_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:28d1935c16782a1e3650c9baa79069b016fff739e292e4cb14bbb830d6930491", size = 229863, upload-time = "2026-10-19T16:55:03.415Z" }, + { url = "https://files.pythonhosted.org/packages/ac/72/d6e460a991607794072fe3fdfcb3221a0342af61586f5bd7ea40bddcfd21/websockets-17.3-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c534c9524a2acd0b97b7022074bec3784ac273c683cf9535087e437672ea8531", size = 227669, upload-time = "2026-10-19T16:55:04.763Z" }, + { url = "https://files.pythonhosted.org/packages/ec/8f/1c7da1cfb632e141e019cf2e0bbbd4e6dc8c198c8cc29544e4acf1a89f56/websockets-17.3-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:b459d29c8b82a420a98fa0ce8e9cf787ce0062d98cc90604ef9574fb3cc26ff8", size = 226429, upload-time = "2026-10-19T16:55:06.112Z" }, + { url = "https://files.pythonhosted.org/packages/3e/63/c48ef8bbfbd982f2cfae1cfcf9e63f5bfbb8468fd37ca700507efc0b0007/websockets-17.3-cp312-cp312-manylinux_2_31_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:53209dffd91b0aefed9ea630fa6ad5c2944d6210ba3a8a3a8664dc1575289b14", size = 223653, upload-time = "2026-10-19T16:55:07.473Z" }, + { url = "https://files.pythonhosted.org/packages/cb/37/7bedaad48ae8e89eaa4f3942c11be0c41de77ce2118a475f3ba05e9b5a0f/websockets-17.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:9662a9fe457820c60365b3209e03cbd8e4a817195d55de65a80fa680b4bea136", size = 226465, upload-time = "2026-10-19T16:55:08.791Z" }, + { url = "https://files.pythonhosted.org/packages/33/cf/ab9a8a0de4769ca1dce392021b6a922e21b4d3687fe9e1ab4db585a088f8/websockets-17.3-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:5670a762c7ae388bf8aa04199fd436180a4d28318dca9c1eae7b38604676dbfd", size = 225071, upload-time = "2026-10-19T16:55:10.127Z" }, + { url = "https://files.pythonhosted.org/packages/89/96/64d8460cec95f873c61c6d13a09b435915150640525ae56d5871400aec09/websockets-17.3-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:ed59cb4ee258674a6ed9b8515e42079db02cb3be94f133fdd3f200183a40c49d", size = 225841, upload-time = "2026-10-19T16:55:11.519Z" }, + { url = "https://files.pythonhosted.org/packages/a6/98/2b83211ee3087db4ae66533782664c0e283cf4a08cbb14036018d0803e09/websockets-17.3-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:0f78bb2e82fe88912c9d7b25c98e57a9b08655579af89f164fd4c0657a075d62", size = 226945, upload-time = "2026-10-19T16:55:12.85Z" }, + { url = "https://files.pythonhosted.org/packages/9f/fe/6b2a37d422d7f4dfc0c92c95a14649d9bd662243ed8b0774523ed2703604/websockets-17.3-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:71250e010eede33c1d638e2c7a98288520d67f782fd5eceecaeca3129233a09a", size = 224502, upload-time = "2026-10-19T16:55:14.48Z" }, + { url = "https://files.pythonhosted.org/packages/0c/c5/364015c9311c89ea6085d54c70d654c5d27ba05014d44ef9fd9daa09462b/websockets-17.3-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:729989759feb81ff6493879a0d60a0bd7d2eee41001bbec0381a080318439382", size = 225483, upload-time = "2026-10-19T16:55:15.899Z" }, + { url = "https://files.pythonhosted.org/packages/c2/b3/75d0cae8398816ed34a16a78252bfc8651f2f883cc5a568e1198395f15df/websockets-17.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:90d8c4fa5ba293cf92de351fbc292cdaadb1a1d58d031a1a97ff9b8fb6250ec7", size = 225761, upload-time = "2026-10-19T16:55:17.28Z" }, + { url = "https://files.pythonhosted.org/packages/f0/af/9a7c4b2e87f47a2a6a066acdc2157e9b7cb6222a224b95a432f54ae786ce/websockets-17.3-cp312-cp312-win32.whl", hash = "sha256:0824e6ecee838cc28e6481062574c7fde49f9a2cedfacd36f0e91cb4b37fd6fe", size = 218722, upload-time = "2026-10-19T16:55:18.658Z" }, + { url = "https://files.pythonhosted.org/packages/f7/e8/e52953a8280a1e51cb9307218443f68c8f4dc4b9952db6021d5c12d03c0f/websockets-17.3-cp312-cp312-win_amd64.whl", hash = "sha256:6644eecd99c4e8cdf72e646e17b391bb70b054f44becde37dcdc48cbec7e1527", size = 219024, upload-time = "2026-10-19T16:55:19.987Z" }, + { url = "https://files.pythonhosted.org/packages/85/a4/005eb7c1f5c1a249c1231a6ed83a54ada0b6b8ffab33a608ca20e3034ea0/websockets-17.3-cp312-cp312-win_arm64.whl", hash = "sha256:4a9133dbaedc6f1af358127275676171d9a1bb3e61f47ee62b2ff9acb78af3d0", size = 218950, upload-time = "2026-10-19T16:55:22.161Z" }, + { url = "https://files.pythonhosted.org/packages/5a/1a/5582103e0362f86c4725051ded89e585ceaf750f26beace530b7e42c81c0/websockets-17.3-pp312-pypy312_pp80-macosx_10_15_x86_64.whl", hash = "sha256:0e65260a1b1d7b81651f47343d836de70b88b02358ae3658efa6382104ec37b1", size = 215872, upload-time = "2026-10-19T16:57:55.95Z" }, + { url = "https://files.pythonhosted.org/packages/ef/a7/f69d65ed3ff9ae73d428a918300c23a992e4d50683090b8dc33cc3341d5a/websockets-17.3-pp312-pypy312_pp80-macosx_11_0_arm64.whl", hash = "sha256:94c02b0702656f083c098bdbd93bce22a7aa7a2b34ba56d2472efb67dab7cf58", size = 216136, upload-time = "2026-10-19T16:57:57.251Z" }, + { url = "https://files.pythonhosted.org/packages/69/76/c64fda4ac18a3aeb32e66acec481011a766add480b6362fad80e2c3c1963/websockets-17.3-pp312-pypy312_pp80-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:ade76f8c84ade325bbed96806357fffd353fcc29a5166d6dc6d39381f6d9f76c", size = 217080, upload-time = "2026-10-19T16:57:58.593Z" }, + { url = "https://files.pythonhosted.org/packages/6b/6d/085a3f0e6152bbe79917973f3d33cd008437de6c04675e2f15253b921f21/websockets-17.3-pp312-pypy312_pp80-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:9a215749c9a88ef2842e880215193e56ae60b856773c99702b078236bc6d4adb", size = 216966, upload-time = "2026-10-19T16:57:59.924Z" }, + { url = "https://files.pythonhosted.org/packages/ed/b8/7eb904583ea2090b329f50487e6f8dc58eb558712b9f3c16f289f0be668e/websockets-17.3-pp312-pypy312_pp80-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4e7b5d23ef144b32645d97bd99f382c4394c443395f45fe1576786838d26e308", size = 217828, upload-time = "2026-10-19T16:58:01.822Z" }, + { url = "https://files.pythonhosted.org/packages/79/36/f54b4a1cd9456ca0a65a76d77d181621433793aca633dfeed214748ff4f0/websockets-17.3-pp312-pypy312_pp80-win_amd64.whl", hash = "sha256:13eace1b315345ebac5b27821f5dcce24e5cf9976ab18f79e00a59bdd67e5d19", size = 306301, upload-time = "2026-10-19T16:58:03.161Z" }, + { url = "https://files.pythonhosted.org/packages/d7/b0/3f01fcb3406662f6deb1e3e72b34dd0979e993991c0b7435a9e12bc14f02/websockets-17.3-py3-none-any.whl", hash = "sha256:dc2b495a92b3971966bf102e73589f950db3ec1ca6713265355ce7771d97b199", size = 212375, upload-time = "2026-10-19T16:58:04.483Z" }, +] diff --git a/tenantboard/web/index.html b/tenantboard/web/index.html new file mode 100644 index 0000000..b84926b --- /dev/null +++ b/tenantboard/web/index.html @@ -0,0 +1,13 @@ + + + + + + + Tenantboard + + +
+ + + diff --git a/tenantboard/web/package-lock.json b/tenantboard/web/package-lock.json new file mode 100644 index 0000000..e0eace5 --- /dev/null +++ b/tenantboard/web/package-lock.json @@ -0,0 +1,1794 @@ +{ + "name": "tenantboard-web", + "version": "0.1.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "tenantboard-web", + "version": "0.1.0", + "license": "Apache-2.0", + "dependencies": { + "react": "18.3.1", + "react-dom": "18.3.1" + }, + "devDependencies": { + "@types/react": "18.3.12", + "@types/react-dom": "18.3.1", + "@vitejs/plugin-react": "4.3.4", + "typescript": "5.6.3", + "vite": "6.0.7" + }, + "engines": { + "node": ">=22" + } + }, + "node_modules/@babel/code-frame": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.7.tgz", + "integrity": "sha512-Aup7aUOfpbAUg2ROOJN6Iw5f9DMBlzu0mIkm/malLQFN/YQgO48wCj0Kxa3sEHJvPVFg7siR+qRInwXd2qhQKw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-validator-identifier": "^7.29.7", + "js-tokens": "^4.0.0", + "picocolors": "^1.1.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/compat-data": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/compat-data/-/compat-data-7.29.7.tgz", + "integrity": "sha512-locTkQyKvwIEgBzVrn8693ebc97F2U8ZHjbXwDXJ5Fn2TCpNwTlKcaKLkdHop5c/icOFE7qt7Q9JC5hnKNa6Gg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/core": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/core/-/core-7.29.7.tgz", + "integrity": "sha512-RgHBCvtjbOK2gXSNBNIkNoEc9qoVEtau3hj8gEqKQuL3HZAibKarWFEI3Lfm6EYKkLalOh8eSrj9b+ch9H/VBA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.29.7", + "@babel/generator": "^7.29.7", + "@babel/helper-compilation-targets": "^7.29.7", + "@babel/helper-module-transforms": "^7.29.7", + "@babel/helpers": "^7.29.7", + "@babel/parser": "^7.29.7", + "@babel/template": "^7.29.7", + "@babel/traverse": "^7.29.7", + "@babel/types": "^7.29.7", + "@jridgewell/remapping": "^2.3.5", + "convert-source-map": "^2.0.0", + "debug": "^4.1.0", + "gensync": "^1.0.0-beta.2", + "json5": "^2.2.3", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/babel" + } + }, + "node_modules/@babel/generator": { + "version": "7.29.8", + "resolved": "https://registry.npmjs.org/@babel/generator/-/generator-7.29.8.tgz", + "integrity": "sha512-gZbepsdh3WDtgZKWL+vTPh71LSBrm/Y4/QDZBVCcYfmeTEEuoOYwlSy+G1StfJg+/Zy550u/3TATbm7qDbbMtg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.29.8", + "@babel/types": "^7.29.8", + "@jridgewell/gen-mapping": "^0.3.12", + "@jridgewell/trace-mapping": "^0.3.28", + "jsesc": "^3.0.2" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-compilation-targets": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-compilation-targets/-/helper-compilation-targets-7.29.7.tgz", + "integrity": "sha512-wem6WaBj4NaVYVdNhLPPVacES6ZJ+KBBfSkTMD3YZxbP3rm3Di85tJU5ljaUNhaOynt+Aj0xruhYuzQBt8n71g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/compat-data": "^7.29.7", + "@babel/helper-validator-option": "^7.29.7", + "browserslist": "^4.24.0", + "lru-cache": "^5.1.1", + "semver": "^6.3.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-globals": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-globals/-/helper-globals-7.29.7.tgz", + "integrity": "sha512-3nQVUAtvkKH9zahfWgw96Jc/uFOmjACE1kQz82E2lqWmHBgjzbNlsC22nuQTfahmWeQtTq5nQ/4Nnd2A1wj4zA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-module-imports": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-module-imports/-/helper-module-imports-7.29.7.tgz", + "integrity": "sha512-ejHwrQQYcm9xnTivShn2IDOlIzInN34AXskvq9QicvCtEzq1Vzclu/tKF8Jq1Cg8JG2GL6/EmjgsCT7lXepE3g==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/traverse": "^7.29.7", + "@babel/types": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-module-transforms": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-module-transforms/-/helper-module-transforms-7.29.7.tgz", + "integrity": "sha512-UPUVSyXbOh627KiCIGQSgwWzGeBKLkaJ9PJEdrngIwMSzxLR4jS4+f1f1jb7VzBbg8nFLaYotvVPFCTqdrmTAg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-module-imports": "^7.29.7", + "@babel/helper-validator-identifier": "^7.29.7", + "@babel/traverse": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0" + } + }, + "node_modules/@babel/helper-plugin-utils": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-plugin-utils/-/helper-plugin-utils-7.29.7.tgz", + "integrity": "sha512-G7sHYigPY17oO5SYWnfD/0MTBwVR781S/JI643e/JhUYgVgWE/61SoW3NH9KWUKyKq5LVh3npif99Wkt6j86Jw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-string-parser": { + "version": "7.29.11", + "resolved": "https://registry.npmjs.org/@babel/helper-string-parser/-/helper-string-parser-7.29.11.tgz", + "integrity": "sha512-snZbO7MEDOof+hvNa97EjEY7qD7h2kwvQr5AJC3ZhQudKW5dnej6k3kP9NQV88hJwaS8vDqRLFo9rivLEhle2g==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-identifier": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-identifier/-/helper-validator-identifier-7.29.7.tgz", + "integrity": "sha512-qehxGkRj55h/ff8EMaJ+cYhyaKlHIxqYDn682wQD7RNp9UujOQsHog2uS0r2vzr4pW+sXf90NeeayjcNaX3fFg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helper-validator-option": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/helper-validator-option/-/helper-validator-option-7.29.7.tgz", + "integrity": "sha512-N9ZErrD+yW5geCDtBqnOoxmR8+tNKiGuxKlDpuJxfsqpa2dFcexaziGAE/qoHLiDDreVNMupxGmSoNlyvsA3gw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/helpers": { + "version": "7.29.10", + "resolved": "https://registry.npmjs.org/@babel/helpers/-/helpers-7.29.10.tgz", + "integrity": "sha512-EMCx3AGZ3WfAl6ht7AyQoPm2Lh+BGYEMdc4Q6Hf7uSCDOZFuLq44qNwL+iaim8VlyYhRAo4X43+weadK8ZB4Yg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/template": "^7.29.7", + "@babel/types": "^7.29.8" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/parser": { + "version": "7.29.11", + "resolved": "https://registry.npmjs.org/@babel/parser/-/parser-7.29.11.tgz", + "integrity": "sha512-OZ6+oaK7GMyg3IfCpj8JOmn8n+w2rJbDT0LLfVDTLb1ZFeKDh3AiNELgCMUw175IJHSkLl74hG0M0VLsvx+gag==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.29.11" + }, + "bin": { + "parser": "bin/babel-parser.js" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@babel/plugin-transform-react-jsx-self": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-jsx-self/-/plugin-transform-react-jsx-self-7.29.7.tgz", + "integrity": "sha512-TL0hMc9xzy86VD31nUiwzd5otRAcyEPcsegCxolO0PvcXuH1v0kECe/UIznYFihpkvU5wg/jk4v0TTEFfm53fw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/plugin-transform-react-jsx-source": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/plugin-transform-react-jsx-source/-/plugin-transform-react-jsx-source-7.29.7.tgz", + "integrity": "sha512-06IyK09H3wi4cGbhDBwp5gUGo0IKtnYa8tyTiephirPCK6fbobVGiXMMI5zLQ4aKEYP3wZ3ArU44o+8KMrSG/Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-plugin-utils": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + }, + "peerDependencies": { + "@babel/core": "^7.0.0-0" + } + }, + "node_modules/@babel/template": { + "version": "7.29.7", + "resolved": "https://registry.npmjs.org/@babel/template/-/template-7.29.7.tgz", + "integrity": "sha512-puq+Gf35oI24FeN11LkoUQFqv9uwNeWpxXZi/Ji3rRIoKAzKnxRaZ+Gkj0vKS9ZCiTESfng1N9LyOyXvo+m+Gg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.29.7", + "@babel/parser": "^7.29.7", + "@babel/types": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/traverse": { + "version": "7.29.10", + "resolved": "https://registry.npmjs.org/@babel/traverse/-/traverse-7.29.10.tgz", + "integrity": "sha512-B3lYAs2nkKskbgQ1nfy8Ma9ao9Q1rntTAQHrzzyDuj+TSO+X3le6PfWwqDvodsb+OumE10eMFfuMCTQSQ5iuIQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/code-frame": "^7.29.7", + "@babel/generator": "^7.29.8", + "@babel/helper-globals": "^7.29.7", + "@babel/parser": "^7.29.9", + "@babel/template": "^7.29.7", + "@babel/types": "^7.29.8", + "debug": "^4.3.1" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@babel/types": { + "version": "7.29.11", + "resolved": "https://registry.npmjs.org/@babel/types/-/types-7.29.11.tgz", + "integrity": "sha512-WCsemW40Cpc6gouC1SrY6PBaZ+oLRlEVY9XBvZBI7il/HyUrb2nJ505wE1MHEFyDZNnrWDEuy4jL50WVHfTTSQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/helper-string-parser": "^7.29.11", + "@babel/helper-validator-identifier": "^7.29.7" + }, + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/@esbuild/aix-ppc64": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.24.2.tgz", + "integrity": "sha512-thpVCb/rhxE/BnMLQ7GReQLLN8q9qbHmI55F4489/ByVg2aQaQ6kbcLb6FHkocZzQhxc4gx0sCk0tJkKBFzDhA==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "aix" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm/-/android-arm-0.24.2.tgz", + "integrity": "sha512-tmwl4hJkCfNHwFB3nBa8z1Uy3ypZpxqxfTQOcHX+xRByyYgunVbZ9MzUUfb0RxaHIMnbHagwAxuTL+tnNM+1/Q==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-arm64": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-arm64/-/android-arm64-0.24.2.tgz", + "integrity": "sha512-cNLgeqCqV8WxfcTIOeL4OAtSmL8JjcN6m09XIgro1Wi7cF4t/THaWEa7eL5CMoMBdjoHOTh/vwTO/o2TRXIyzg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/android-x64": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/android-x64/-/android-x64-0.24.2.tgz", + "integrity": "sha512-B6Q0YQDqMx9D7rvIcsXfmJfvUYLoP722bgfBlO5cGvNVb5V/+Y7nhBE3mHV9OpxBf4eAS2S68KZztiPaWq4XYw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-arm64": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-arm64/-/darwin-arm64-0.24.2.tgz", + "integrity": "sha512-kj3AnYWc+CekmZnS5IPu9D+HWtUI49hbnyqk0FLEJDbzCIQt7hg7ucF1SQAilhtYpIujfaHr6O0UHlzzSPdOeA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/darwin-x64": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/darwin-x64/-/darwin-x64-0.24.2.tgz", + "integrity": "sha512-WeSrmwwHaPkNR5H3yYfowhZcbriGqooyu3zI/3GGpF8AyUdsrrP0X6KumITGA9WOyiJavnGZUwPGvxvwfWPHIA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-arm64": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-arm64/-/freebsd-arm64-0.24.2.tgz", + "integrity": "sha512-UN8HXjtJ0k/Mj6a9+5u6+2eZ2ERD7Edt1Q9IZiB5UZAIdPnVKDoG7mdTVGhHJIeEml60JteamR3qhsr1r8gXvg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/freebsd-x64": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/freebsd-x64/-/freebsd-x64-0.24.2.tgz", + "integrity": "sha512-TvW7wE/89PYW+IevEJXZ5sF6gJRDY/14hyIGFXdIucxCsbRmLUcjseQu1SyTko+2idmCw94TgyaEZi9HUSOe3Q==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm/-/linux-arm-0.24.2.tgz", + "integrity": "sha512-n0WRM/gWIdU29J57hJyUdIsk0WarGd6To0s+Y+LwvlC55wt+GT/OgkwoXCXvIue1i1sSNWblHEig00GBWiJgfA==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-arm64": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-arm64/-/linux-arm64-0.24.2.tgz", + "integrity": "sha512-7HnAD6074BW43YvvUmE/35Id9/NB7BeX5EoNkK9obndmZBUk8xmJJeU7DwmUeN7tkysslb2eSl6CTrYz6oEMQg==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ia32": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ia32/-/linux-ia32-0.24.2.tgz", + "integrity": "sha512-sfv0tGPQhcZOgTKO3oBE9xpHuUqguHvSo4jl+wjnKwFpapx+vUDcawbwPNuBIAYdRAvIDBfZVvXprIj3HA+Ugw==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-loong64": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-loong64/-/linux-loong64-0.24.2.tgz", + "integrity": "sha512-CN9AZr8kEndGooS35ntToZLTQLHEjtVB5n7dl8ZcTZMonJ7CCfStrYhrzF97eAecqVbVJ7APOEe18RPI4KLhwQ==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-mips64el": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-mips64el/-/linux-mips64el-0.24.2.tgz", + "integrity": "sha512-iMkk7qr/wl3exJATwkISxI7kTcmHKE+BlymIAbHO8xanq/TjHaaVThFF6ipWzPHryoFsesNQJPE/3wFJw4+huw==", + "cpu": [ + "mips64el" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-ppc64": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-ppc64/-/linux-ppc64-0.24.2.tgz", + "integrity": "sha512-shsVrgCZ57Vr2L8mm39kO5PPIb+843FStGt7sGGoqiiWYconSxwTiuswC1VJZLCjNiMLAMh34jg4VSEQb+iEbw==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-riscv64": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-riscv64/-/linux-riscv64-0.24.2.tgz", + "integrity": "sha512-4eSFWnU9Hhd68fW16GD0TINewo1L6dRrB+oLNNbYyMUAeOD2yCK5KXGK1GH4qD/kT+bTEXjsyTCiJGHPZ3eM9Q==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-s390x": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-s390x/-/linux-s390x-0.24.2.tgz", + "integrity": "sha512-S0Bh0A53b0YHL2XEXC20bHLuGMOhFDO6GN4b3YjRLK//Ep3ql3erpNcPlEFed93hsQAjAQDNsvcK+hV90FubSw==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/linux-x64": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/linux-x64/-/linux-x64-0.24.2.tgz", + "integrity": "sha512-8Qi4nQcCTbLnK9WoMjdC9NiTG6/E38RNICU6sUNqK0QFxCYgoARqVqxdFmWkdonVsvGqWhmm7MO0jyTqLqwj0Q==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-arm64": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-arm64/-/netbsd-arm64-0.24.2.tgz", + "integrity": "sha512-wuLK/VztRRpMt9zyHSazyCVdCXlpHkKm34WUyinD2lzK07FAHTq0KQvZZlXikNWkDGoT6x3TD51jKQ7gMVpopw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/netbsd-x64": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/netbsd-x64/-/netbsd-x64-0.24.2.tgz", + "integrity": "sha512-VefFaQUc4FMmJuAxmIHgUmfNiLXY438XrL4GDNV1Y1H/RW3qow68xTwjZKfj/+Plp9NANmzbH5R40Meudu8mmw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "netbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-arm64": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-arm64/-/openbsd-arm64-0.24.2.tgz", + "integrity": "sha512-YQbi46SBct6iKnszhSvdluqDmxCJA+Pu280Av9WICNwQmMxV7nLRHZfjQzwbPs3jeWnuAhE9Jy0NrnJ12Oz+0A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/openbsd-x64": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/openbsd-x64/-/openbsd-x64-0.24.2.tgz", + "integrity": "sha512-+iDS6zpNM6EnJyWv0bMGLWSWeXGN/HTaF/LXHXHwejGsVi+ooqDfMCCTerNFxEkM3wYVcExkeGXNqshc9iMaOA==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/sunos-x64": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/sunos-x64/-/sunos-x64-0.24.2.tgz", + "integrity": "sha512-hTdsW27jcktEvpwNHJU4ZwWFGkz2zRJUz8pvddmXPtXDzVKTTINmlmga3ZzwcuMpUvLw7JkLy9QLKyGpD2Yxig==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "sunos" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-arm64": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-arm64/-/win32-arm64-0.24.2.tgz", + "integrity": "sha512-LihEQ2BBKVFLOC9ZItT9iFprsE9tqjDjnbulhHoFxYQtQfai7qfluVODIYxt1PgdoyQkz23+01rzwNwYfutxUQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-ia32": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-ia32/-/win32-ia32-0.24.2.tgz", + "integrity": "sha512-q+iGUwfs8tncmFC9pcnD5IvRHAzmbwQ3GPS5/ceCyHdjXubwQWI12MKWSNSMYLJMq23/IUCvJMS76PDqXe1fxA==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@esbuild/win32-x64": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/@esbuild/win32-x64/-/win32-x64-0.24.2.tgz", + "integrity": "sha512-7VTgWzgMGvup6aSqDPLiW5zHaxYJGTO4OokMjIlrCtf+VpEL+cXKtCvg723iguPYI5oaUNdS+/V7OU2gvXVWEg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">=18" + } + }, + "node_modules/@jridgewell/gen-mapping": { + "version": "0.3.13", + "resolved": "https://registry.npmjs.org/@jridgewell/gen-mapping/-/gen-mapping-0.3.13.tgz", + "integrity": "sha512-2kkt/7niJ6MgEPxF0bYdQ6etZaA+fQvDcLKckhy1yIQOzaoKjBBjSj63/aLVjYE3qhRt5dvM+uUyfCg6UKCBbA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/sourcemap-codec": "^1.5.0", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/remapping": { + "version": "2.3.5", + "resolved": "https://registry.npmjs.org/@jridgewell/remapping/-/remapping-2.3.5.tgz", + "integrity": "sha512-LI9u/+laYG4Ds1TDKSJW2YPrIlcVYOwi2fUC6xB43lueCjgxV4lffOCZCtYFiH6TNOX+tQKXx97T4IKHbhyHEQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/gen-mapping": "^0.3.5", + "@jridgewell/trace-mapping": "^0.3.24" + } + }, + "node_modules/@jridgewell/resolve-uri": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/@jridgewell/resolve-uri/-/resolve-uri-3.1.2.tgz", + "integrity": "sha512-bRISgCIjP20/tbWSPWMEi54QVPRZExkuD9lJL+UIxUKtwVJA8wW1Trb1jMs1RFXo1CBTNZ/5hpC9QvmKWdopKw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/@jridgewell/sourcemap-codec": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/@jridgewell/sourcemap-codec/-/sourcemap-codec-1.6.0.tgz", + "integrity": "sha512-T7jf+5zgsZHwNJ4lvQ7/aezbyk0nNX+zJVWpmHA7VYsEx7a7qr5Rg5IbtJFqkgze5Y2sruq1RUY8Q837Od7iFw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@jridgewell/trace-mapping": { + "version": "0.3.31", + "resolved": "https://registry.npmjs.org/@jridgewell/trace-mapping/-/trace-mapping-0.3.31.tgz", + "integrity": "sha512-zzNR+SdQSDJzc8joaeP8QQoCQr8NuYx2dIIytl1QeBEZHJ9uW6hebsrYgbz8hJwUQao3TWCMtmfV8Nu1twOLAw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@jridgewell/resolve-uri": "^3.1.0", + "@jridgewell/sourcemap-codec": "^1.4.14" + } + }, + "node_modules/@napi-rs/lzma-linux-x64-gnu": { + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/@napi-rs/lzma-linux-x64-gnu/-/lzma-linux-x64-gnu-1.5.1.tgz", + "integrity": "sha512-oTXEIha4SsuXdTA4Iyskj0kpdx2yVXdhd75c2v3xGrHFfVMsbhTPZU/nMPL4sWKo4pBHm3aucLaqGlF696dTyQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": "^22.20 || ^24.12 || >=25" + } + }, + "node_modules/@rollup/rollup-android-arm-eabi": { + "version": "4.65.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm-eabi/-/rollup-android-arm-eabi-4.65.0.tgz", + "integrity": "sha512-t2AnVba25baFdq7gczzS13JyVcdubLVbFX1X04mxrggDrTQPUXNv3XM0k9H1hr5tqvg7AVvtQ720QHAj0iYqhg==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-android-arm64": { + "version": "4.65.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-android-arm64/-/rollup-android-arm64-4.65.0.tgz", + "integrity": "sha512-dNONUaEwPGhl/nuO2KV8VABWlcjYyU6KtZGN8dQv5OS5JU7JSCxPijVNKXbRJd0iuTzTiJ4TjPt/4bbvDRNiMQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "android" + ] + }, + "node_modules/@rollup/rollup-darwin-arm64": { + "version": "4.65.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-arm64/-/rollup-darwin-arm64-4.65.0.tgz", + "integrity": "sha512-zWQqUH+9zavr2Bxuzfpuv2SIF3X31cr58TvSQC9o1GhpOvkqRwr60GkZdQcudyT/BEgdpdSafjgd2I13MbTL7g==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-darwin-x64": { + "version": "4.65.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-darwin-x64/-/rollup-darwin-x64-4.65.0.tgz", + "integrity": "sha512-RQYreBFWtO4//hIuOaZkYUnF/lPU4lz35dveDimDshWCJvVisVtJ/shmT9bkXK8wFJY+FJF89X6F3Cq1qI3LCw==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ] + }, + "node_modules/@rollup/rollup-freebsd-arm64": { + "version": "4.65.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-arm64/-/rollup-freebsd-arm64-4.65.0.tgz", + "integrity": "sha512-A0NfMj6Cuh/9BE5k3awJKv6HoxFVXIcLAb6SZqAMoxUHPA1ZNNK4RsGuUPD/vL7nEAm9LG+ZzXWWTrnC0XtkpA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-freebsd-x64": { + "version": "4.65.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-freebsd-x64/-/rollup-freebsd-x64-4.65.0.tgz", + "integrity": "sha512-BI6/H7q3aP0I7S7GOrjyxTjgEJlSnovXE38+bdxBpuKN6z+d//uF+T+VQ3Q7ltJL/71p17ufartARX9Ja5gOGQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "freebsd" + ] + }, + "node_modules/@rollup/rollup-linux-arm-gnueabihf": { + "version": "4.65.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-gnueabihf/-/rollup-linux-arm-gnueabihf-4.65.0.tgz", + "integrity": "sha512-W1vUprf09RWaVWnsFP1vlUkEWJzTZlpObOEAYZXt9Fq0UVAOYEkeGr7wBX7McKqqIQskXdBnpiOMgwPujordgA==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm-musleabihf": { + "version": "4.65.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm-musleabihf/-/rollup-linux-arm-musleabihf-4.65.0.tgz", + "integrity": "sha512-s8QTcZWgrl9haFzEGIqyE5BWyo89pFDGLMBFZ752WYxk8DJwNU/O1X6pZgPw829d9/sm6CCHyrMP3aurt76p1Q==", + "cpu": [ + "arm" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-gnu": { + "version": "4.65.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-gnu/-/rollup-linux-arm64-gnu-4.65.0.tgz", + "integrity": "sha512-ke1f6dLRzkqveyjjF2fkEwQYZvxHtNNwjegY8nJrwcO0Bdqr8CzFO/pKE0Bns13cx0qz2Ba2Y0Hs3dSwMdnHxA==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-arm64-musl": { + "version": "4.65.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-arm64-musl/-/rollup-linux-arm64-musl-4.65.0.tgz", + "integrity": "sha512-926SMeUcy4y18Rz/w2aJiUKOnyAnm6czg72bXkQEaeemsll+Wr0hnsjb2BGRpHvWgwAMYy5ACJCotbvfLCfj0A==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loong64-gnu": { + "version": "4.65.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-gnu/-/rollup-linux-loong64-gnu-4.65.0.tgz", + "integrity": "sha512-6T7JNRP1hc4hqF2dUsEoVLOt9K8tmODoCLldtQRYnrddGGQY8Gqe2pM/qMMUo7EMowpP7oa2N95k0/8sYK/ipw==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-loong64-musl": { + "version": "4.65.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-loong64-musl/-/rollup-linux-loong64-musl-4.65.0.tgz", + "integrity": "sha512-SImGFl1Je5kpyAgOCPONVT7h6ASV8mAUwGRjY7eZvVjPg//PfHOXgI+P241RF/+HqsA4PP3qD527ysWFKNQyhQ==", + "cpu": [ + "loong64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-gnu": { + "version": "4.65.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-gnu/-/rollup-linux-ppc64-gnu-4.65.0.tgz", + "integrity": "sha512-XeDTbmPF1Yu5asfXGPrMkWoxTDhgryQbmw5Q1+R582O51VNCk23J6f2pDrsAp5YkneBVBwtExx6ZGOb9D2aQ6A==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-ppc64-musl": { + "version": "4.65.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-ppc64-musl/-/rollup-linux-ppc64-musl-4.65.0.tgz", + "integrity": "sha512-VS55JUI/1nsljmV/jfqNkLZXFw8uMO8nNlfoHmSevNf6fO3neuhyDvx9dv3c/Ytt8r/431SL8Qax1dpYvOMZTg==", + "cpu": [ + "ppc64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-gnu": { + "version": "4.65.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-gnu/-/rollup-linux-riscv64-gnu-4.65.0.tgz", + "integrity": "sha512-Vblw84TbX7RS4la1Y4+yQ7aore/oSyaNtFbppuBr5Vs/4LfBeXKq9dIfoExw6zOvRERqd2rRAjnZRaMhH/0x5w==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-riscv64-musl": { + "version": "4.65.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-riscv64-musl/-/rollup-linux-riscv64-musl-4.65.0.tgz", + "integrity": "sha512-7+gHjtONvKgwR9InTKFaLq53f6YzBJkgeMRBENdWTsQr+spahGqQm8HF5aXpq8i4JaRrOqYyv6Zn1WltDTxaZQ==", + "cpu": [ + "riscv64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-s390x-gnu": { + "version": "4.65.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-s390x-gnu/-/rollup-linux-s390x-gnu-4.65.0.tgz", + "integrity": "sha512-KvNDC0J6s4vVHhtJ6KrDXU0Er0au/hmRIBq/+1maCV8TseGupeKizq0bWcrbp2HE1T8xoknvN0Q/KZ/YF0qX3w==", + "cpu": [ + "s390x" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-gnu": { + "version": "4.65.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-gnu/-/rollup-linux-x64-gnu-4.65.0.tgz", + "integrity": "sha512-GExdD15WPvu/x/drELjVam7vX8EsPgvTMOa/iJL/p/4m/oOqzbb7JXZg8fXR4hlXtg/sF2zyTZ3bR0AEtEMlhQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-linux-x64-musl": { + "version": "4.65.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-linux-x64-musl/-/rollup-linux-x64-musl-4.65.0.tgz", + "integrity": "sha512-8sT9PemIZfIMED4Vd4KGoOXVDj5CMSWNcGdbheRUx9TYS03yOWdT2Q0lXPyEyNBB9YTilwewIQKNIBk6Nxi0Kg==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "linux" + ] + }, + "node_modules/@rollup/rollup-openbsd-x64": { + "version": "4.65.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openbsd-x64/-/rollup-openbsd-x64-4.65.0.tgz", + "integrity": "sha512-pO6Ehi0jo5yE0R6PltKhJhv9mlIiyzBJk5N5NUF2OF6xtVKI+1bdXxj/rb0jGXy3hck1ELVT0QPY3tzkAm360w==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openbsd" + ] + }, + "node_modules/@rollup/rollup-openharmony-arm64": { + "version": "4.65.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-openharmony-arm64/-/rollup-openharmony-arm64-4.65.0.tgz", + "integrity": "sha512-vo7N3/+cjjPDYr8vtxC+qSNgWa9sDvJ7g8N8R5SWqFY4WiNH1ULcAU70SKVyHeiIZOYYwNP9pTipmhN5sl1NPw==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "openharmony" + ] + }, + "node_modules/@rollup/rollup-win32-arm64-msvc": { + "version": "4.65.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-arm64-msvc/-/rollup-win32-arm64-msvc-4.65.0.tgz", + "integrity": "sha512-ArwvgFAbjKcDM7qqotZ4OVYf+UMT9xH2YzkCzK4+AuZSDihMlz1Ekoe4INwBVXzn2mnJzuIYOcSx07iTBijGRQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-ia32-msvc": { + "version": "4.65.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-ia32-msvc/-/rollup-win32-ia32-msvc-4.65.0.tgz", + "integrity": "sha512-QGMmK4/cxKE/36FoLMSHQBCyymv/eFTnsCvvsBF0rgmOvVbvX78VWZRerL3dHa77f4pM6Xcc2iGDhJ10w/svKQ==", + "cpu": [ + "ia32" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-gnu": { + "version": "4.65.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-gnu/-/rollup-win32-x64-gnu-4.65.0.tgz", + "integrity": "sha512-XfKYFWHIPs4HPODIMsGF1IKlS3H1e3WokUZIgpmn4+uNnZBlnRDDHEylNyazbQXwhMKZre68OjtRYbKqb5+j8w==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@rollup/rollup-win32-x64-msvc": { + "version": "4.65.0", + "resolved": "https://registry.npmjs.org/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.65.0.tgz", + "integrity": "sha512-wPWi++K4Bs1VR4XihOhfGi4IGX6Cub7FlerTf8X2v+2BtDHddgSqWCXwGD+h2yAe4/niIhNDwAltioOC+wEogQ==", + "cpu": [ + "x64" + ], + "dev": true, + "license": "MIT", + "optional": true, + "os": [ + "win32" + ] + }, + "node_modules/@types/babel__core": { + "version": "7.20.5", + "resolved": "https://registry.npmjs.org/@types/babel__core/-/babel__core-7.20.5.tgz", + "integrity": "sha512-qoQprZvz5wQFJwMDqeseRXWv3rqMvhgpbXFfVyWhbx9X47POIA6i/+dXefEmZKoAgOaTdaIgNSMqMIU61yRyzA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.20.7", + "@babel/types": "^7.20.7", + "@types/babel__generator": "*", + "@types/babel__template": "*", + "@types/babel__traverse": "*" + } + }, + "node_modules/@types/babel__generator": { + "version": "7.27.0", + "resolved": "https://registry.npmjs.org/@types/babel__generator/-/babel__generator-7.27.0.tgz", + "integrity": "sha512-ufFd2Xi92OAVPYsy+P4n7/U7e68fex0+Ee8gSG9KX7eo084CWiQ4sdxktvdl0bOPupXtVJPY19zk6EwWqUQ8lg==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.0.0" + } + }, + "node_modules/@types/babel__template": { + "version": "7.4.4", + "resolved": "https://registry.npmjs.org/@types/babel__template/-/babel__template-7.4.4.tgz", + "integrity": "sha512-h/NUaSyG5EyxBIp8YRxo4RMe2/qQgvyowRwVMzhYhBCONbW8PUsg4lkFMrhgZhUe5z3L3MiLDuvyJ/CaPa2A8A==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/parser": "^7.1.0", + "@babel/types": "^7.0.0" + } + }, + "node_modules/@types/babel__traverse": { + "version": "7.28.0", + "resolved": "https://registry.npmjs.org/@types/babel__traverse/-/babel__traverse-7.28.0.tgz", + "integrity": "sha512-8PvcXf70gTDZBgt9ptxJ8elBeBjcLOAcOtoO/mPJjtji1+CdGbHgm77om1GrsPxsiE+uXIpNSK64UYaIwQXd4Q==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/types": "^7.28.2" + } + }, + "node_modules/@types/estree": { + "version": "1.0.9", + "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.9.tgz", + "integrity": "sha512-GhdPgy1el4/ImP05X05Uw4cw2/M93BCUmnEvWZNStlCzEKME4Fkk+YpoA5OiHNQmoS7Cafb8Xa3Pya8m1Qrzeg==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/prop-types": { + "version": "15.7.15", + "resolved": "https://registry.npmjs.org/@types/prop-types/-/prop-types-15.7.15.tgz", + "integrity": "sha512-F6bEyamV9jKGAFBEmlQnesRPGOQqS2+Uwi0Em15xenOxHaf2hv6L8YCVn3rPdPJOiJfPiCnLIRyvwVaqMY3MIw==", + "dev": true, + "license": "MIT" + }, + "node_modules/@types/react": { + "version": "18.3.12", + "resolved": "https://registry.npmjs.org/@types/react/-/react-18.3.12.tgz", + "integrity": "sha512-D2wOSq/d6Agt28q7rSI3jhU7G6aiuzljDGZ2hTZHIkrTLUI+AF3WMeKkEZ9nN2fkBAlcktT6vcZjDFiIhMYEQw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/prop-types": "*", + "csstype": "^3.0.2" + } + }, + "node_modules/@types/react-dom": { + "version": "18.3.1", + "resolved": "https://registry.npmjs.org/@types/react-dom/-/react-dom-18.3.1.tgz", + "integrity": "sha512-qW1Mfv8taImTthu4KoXgDfLuk4bydU6Q/TkADnDWWHwi4NX4BR+LWfTp2sVmTqRrsHvyDDTelgelxJ+SsejKKQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/react": "*" + } + }, + "node_modules/@vitejs/plugin-react": { + "version": "4.3.4", + "resolved": "https://registry.npmjs.org/@vitejs/plugin-react/-/plugin-react-4.3.4.tgz", + "integrity": "sha512-SCCPBJtYLdE8PX/7ZQAs1QAZ8Jqwih+0VBLum1EGqmCCQal+MIUqLCzj3ZUy8ufbC0cAM4LRlSTm7IQJwWT4ug==", + "dev": true, + "license": "MIT", + "dependencies": { + "@babel/core": "^7.26.0", + "@babel/plugin-transform-react-jsx-self": "^7.25.9", + "@babel/plugin-transform-react-jsx-source": "^7.25.9", + "@types/babel__core": "^7.20.5", + "react-refresh": "^0.14.2" + }, + "engines": { + "node": "^14.18.0 || >=16.0.0" + }, + "peerDependencies": { + "vite": "^4.2.0 || ^5.0.0 || ^6.0.0" + } + }, + "node_modules/baseline-browser-mapping": { + "version": "2.11.32", + "resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.11.32.tgz", + "integrity": "sha512-UvuxFU9P3hM29LrQdwu0cD6w6/+SFr+UlB9s/knI7+O2rK9xFxKUwSDWxGdbbqQ9/V7BJ5s0NI5nQAE4uJp3gQ==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "baseline-browser-mapping": "dist/cli.cjs" + }, + "engines": { + "node": ">=6.0.0" + } + }, + "node_modules/browserslist": { + "version": "4.29.4", + "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.29.4.tgz", + "integrity": "sha512-2TcbbiwXEq+N0kD36tykpCfyyQigUXmuwNQbnPp4Xk4PX9JOqVIKjJlIDjpH7uY6tmRGoAd+apeT1p60T8wsdA==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "baseline-browser-mapping": "^2.11.29", + "caniuse-lite": "^1.0.30001816", + "electron-to-chromium": "^1.5.459", + "node-releases": "^2.0.59", + "update-browserslist-db": "^1.3.4" + }, + "bin": { + "browserslist": "cli.js" + }, + "engines": { + "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" + } + }, + "node_modules/caniuse-lite": { + "version": "1.0.30001819", + "resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001819.tgz", + "integrity": "sha512-LFUqIIAwpzsIuevnKnsR9qnStBcgzPShTX61lBwN9uFkkf8o8zLiebEjbW8AYtZiF+1YKDWfdt/pWDtMuG8Xwg==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/caniuse-lite" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "CC-BY-4.0" + }, + "node_modules/convert-source-map": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", + "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==", + "dev": true, + "license": "MIT" + }, + "node_modules/csstype": { + "version": "3.2.3", + "resolved": "https://registry.npmjs.org/csstype/-/csstype-3.2.3.tgz", + "integrity": "sha512-z1HGKcYy2xA8AGQfwrn0PAy+PB7X/GSj3UVJW9qKyn43xWa+gl5nXmU4qqLMRzWVLFC8KusUX8T/0kCiOYpAIQ==", + "dev": true, + "license": "MIT" + }, + "node_modules/debug": { + "version": "4.4.3", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.4.3.tgz", + "integrity": "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA==", + "dev": true, + "license": "MIT", + "dependencies": { + "ms": "^2.1.3" + }, + "engines": { + "node": ">=6.0" + }, + "peerDependenciesMeta": { + "supports-color": { + "optional": true + } + } + }, + "node_modules/electron-to-chromium": { + "version": "1.5.464", + "resolved": "https://registry.npmjs.org/electron-to-chromium/-/electron-to-chromium-1.5.464.tgz", + "integrity": "sha512-nRa65aFge40viPAKxMKULIY8tcXKwKboJWNn4RoKwyWXJ6SP7BnrDGKUTzc64818jIciH0PK7LSF8QzZes19GA==", + "dev": true, + "license": "ISC" + }, + "node_modules/esbuild": { + "version": "0.24.2", + "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.24.2.tgz", + "integrity": "sha512-+9egpBW8I3CD5XPe0n6BfT5fxLzxrlDzqydF3aviG+9ni1lDC/OvMHcxqEFV0+LANZG5R1bFMWfUrjVsdwxJvA==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "bin": { + "esbuild": "bin/esbuild" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "@esbuild/aix-ppc64": "0.24.2", + "@esbuild/android-arm": "0.24.2", + "@esbuild/android-arm64": "0.24.2", + "@esbuild/android-x64": "0.24.2", + "@esbuild/darwin-arm64": "0.24.2", + "@esbuild/darwin-x64": "0.24.2", + "@esbuild/freebsd-arm64": "0.24.2", + "@esbuild/freebsd-x64": "0.24.2", + "@esbuild/linux-arm": "0.24.2", + "@esbuild/linux-arm64": "0.24.2", + "@esbuild/linux-ia32": "0.24.2", + "@esbuild/linux-loong64": "0.24.2", + "@esbuild/linux-mips64el": "0.24.2", + "@esbuild/linux-ppc64": "0.24.2", + "@esbuild/linux-riscv64": "0.24.2", + "@esbuild/linux-s390x": "0.24.2", + "@esbuild/linux-x64": "0.24.2", + "@esbuild/netbsd-arm64": "0.24.2", + "@esbuild/netbsd-x64": "0.24.2", + "@esbuild/openbsd-arm64": "0.24.2", + "@esbuild/openbsd-x64": "0.24.2", + "@esbuild/sunos-x64": "0.24.2", + "@esbuild/win32-arm64": "0.24.2", + "@esbuild/win32-ia32": "0.24.2", + "@esbuild/win32-x64": "0.24.2" + } + }, + "node_modules/escalade": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/escalade/-/escalade-3.2.0.tgz", + "integrity": "sha512-WUj2qlxaQtO4g6Pq5c29GTcWGDyd8itL8zTlipgECz3JesAiiOKotd8JU6otB3PACgG6xkJUyVhboMS+bje/jA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6" + } + }, + "node_modules/fsevents": { + "version": "2.3.3", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", + "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", + "dev": true, + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" + } + }, + "node_modules/gensync": { + "version": "1.0.0-beta.2", + "resolved": "https://registry.npmjs.org/gensync/-/gensync-1.0.0-beta.2.tgz", + "integrity": "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=6.9.0" + } + }, + "node_modules/js-tokens": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", + "integrity": "sha512-RdJUflcE3cUzKiMqQgsCu06FPu9UdIJO0beYbPhHN4k6apgJtifcoCtT9bcxOpYBtpD2kCM6Sbzg4CausW/PKQ==", + "license": "MIT" + }, + "node_modules/jsesc": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-3.1.0.tgz", + "integrity": "sha512-/sM3dO2FOzXjKQhJuo0Q173wf2KOo8t4I8vHy6lF9poUp7bKT0/NHE8fPX23PwfhnykfqnC2xRxOnVw5XuGIaA==", + "dev": true, + "license": "MIT", + "bin": { + "jsesc": "bin/jsesc" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/json5": { + "version": "2.2.3", + "resolved": "https://registry.npmjs.org/json5/-/json5-2.2.3.tgz", + "integrity": "sha512-XmOWe7eyHYH14cLdVPoyg+GOH3rYX++KpzrylJwSW98t3Nk+U8XOl8FWKOgwtzdb8lXGf6zYwDUzeHMWfxasyg==", + "dev": true, + "license": "MIT", + "bin": { + "json5": "lib/cli.js" + }, + "engines": { + "node": ">=6" + } + }, + "node_modules/loose-envify": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/loose-envify/-/loose-envify-1.4.0.tgz", + "integrity": "sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==", + "license": "MIT", + "dependencies": { + "js-tokens": "^3.0.0 || ^4.0.0" + }, + "bin": { + "loose-envify": "cli.js" + } + }, + "node_modules/lru-cache": { + "version": "5.1.1", + "resolved": "https://registry.npmjs.org/lru-cache/-/lru-cache-5.1.1.tgz", + "integrity": "sha512-KpNARQA3Iwv+jTA0utUVVbrh+Jlrr1Fv0e56GGzAFOXN7dk/FviaDW8LHmK52DlcH4WP2n6gI8vN1aesBFgo9w==", + "dev": true, + "license": "ISC", + "dependencies": { + "yallist": "^3.0.2" + } + }, + "node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true, + "license": "MIT" + }, + "node_modules/nanoid": { + "version": "3.3.20", + "resolved": "https://registry.npmjs.org/nanoid/-/nanoid-3.3.20.tgz", + "integrity": "sha512-uKdg2G3GNCKQn9byYOpxbGqrT2fGO5KRt5J/8b3pok8rT6qxGWF6hxMyJiEYtAf+FVyYuD9hRaDqX5uPFYJ4ZQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "bin": { + "nanoid": "bin/nanoid.cjs" + }, + "engines": { + "node": "^10 || ^12 || ^13.7 || ^14 || >=15.0.1" + } + }, + "node_modules/node-releases": { + "version": "2.0.60", + "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.60.tgz", + "integrity": "sha512-OmO8+GM1JUZqXET720VrJCwkjUCXmCMQm0Z3UDDRbfNLHS4NEAvY60j5jiFv0tkKw2PNNlqOLhahVmwJIOLDIQ==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=18" + } + }, + "node_modules/picocolors": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/picocolors/-/picocolors-1.1.1.tgz", + "integrity": "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA==", + "dev": true, + "license": "ISC" + }, + "node_modules/postcss": { + "version": "8.5.31", + "resolved": "https://registry.npmjs.org/postcss/-/postcss-8.5.31.tgz", + "integrity": "sha512-YUbWYCgErwnMAVkjUnYa+KlMBzGpHwiIt/DE7f0G33AuX2LH9UcTk6KgWxd3zRU+pN3u/C+Xl3EnMLM4Hea7Sg==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/postcss/" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/postcss" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "nanoid": "^3.3.20", + "picocolors": "^1.1.1", + "source-map-js": "^1.2.2" + }, + "engines": { + "node": "^10 || ^12 || >=14" + } + }, + "node_modules/react": { + "version": "18.3.1", + "resolved": "https://registry.npmjs.org/react/-/react-18.3.1.tgz", + "integrity": "sha512-wS+hAgJShR0KhEvPJArfuPVN1+Hz1t0Y6n5jLrGQbkb4urgPE/0Rve+1kMB1v/oWgHgm4WIcV+i7F2pTVj+2iQ==", + "license": "MIT", + "dependencies": { + "loose-envify": "^1.1.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/react-dom": { + "version": "18.3.1", + "resolved": "https://registry.npmjs.org/react-dom/-/react-dom-18.3.1.tgz", + "integrity": "sha512-5m4nQKp+rZRb09LNH59GM4BxTh9251/ylbKIbpe7TpGxfJ+9kv6BLkLBXIjjspbgbnIBNqlI23tRnTWT0snUIw==", + "license": "MIT", + "dependencies": { + "loose-envify": "^1.1.0", + "scheduler": "^0.23.2" + }, + "peerDependencies": { + "react": "^18.3.1" + } + }, + "node_modules/react-refresh": { + "version": "0.14.2", + "resolved": "https://registry.npmjs.org/react-refresh/-/react-refresh-0.14.2.tgz", + "integrity": "sha512-jCvmsr+1IUSMUyzOkRcvnVbX3ZYC6g9TDrDbFuFmRDq7PD4yaGbLKNQL6k2jnArV8hjYxh7hVhAZB6s9HDGpZA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/rollup": { + "version": "4.65.0", + "resolved": "https://registry.npmjs.org/rollup/-/rollup-4.65.0.tgz", + "integrity": "sha512-QVCvQUTGDN6JRwi3jVGCY3H4aTHvi9rX0LyoBMkGUvdh/Z6O51CzcLz85BWZaTCDGhl6ySGqYZs+xI1jmPqGcw==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/estree": "1.0.9" + }, + "bin": { + "rollup": "dist/bin/rollup" + }, + "engines": { + "node": ">=18.0.0", + "npm": ">=8.0.0" + }, + "optionalDependencies": { + "@napi-rs/lzma-linux-x64-gnu": "1.5.1", + "@rollup/rollup-android-arm-eabi": "4.65.0", + "@rollup/rollup-android-arm64": "4.65.0", + "@rollup/rollup-darwin-arm64": "4.65.0", + "@rollup/rollup-darwin-x64": "4.65.0", + "@rollup/rollup-freebsd-arm64": "4.65.0", + "@rollup/rollup-freebsd-x64": "4.65.0", + "@rollup/rollup-linux-arm-gnueabihf": "4.65.0", + "@rollup/rollup-linux-arm-musleabihf": "4.65.0", + "@rollup/rollup-linux-arm64-gnu": "4.65.0", + "@rollup/rollup-linux-arm64-musl": "4.65.0", + "@rollup/rollup-linux-loong64-gnu": "4.65.0", + "@rollup/rollup-linux-loong64-musl": "4.65.0", + "@rollup/rollup-linux-ppc64-gnu": "4.65.0", + "@rollup/rollup-linux-ppc64-musl": "4.65.0", + "@rollup/rollup-linux-riscv64-gnu": "4.65.0", + "@rollup/rollup-linux-riscv64-musl": "4.65.0", + "@rollup/rollup-linux-s390x-gnu": "4.65.0", + "@rollup/rollup-linux-x64-gnu": "4.65.0", + "@rollup/rollup-linux-x64-musl": "4.65.0", + "@rollup/rollup-openbsd-x64": "4.65.0", + "@rollup/rollup-openharmony-arm64": "4.65.0", + "@rollup/rollup-win32-arm64-msvc": "4.65.0", + "@rollup/rollup-win32-ia32-msvc": "4.65.0", + "@rollup/rollup-win32-x64-gnu": "4.65.0", + "@rollup/rollup-win32-x64-msvc": "4.65.0", + "fsevents": "~2.3.2" + } + }, + "node_modules/scheduler": { + "version": "0.23.2", + "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.23.2.tgz", + "integrity": "sha512-UOShsPwz7NrMUqhR6t0hWjFduvOzbtv7toDH1/hIrfRNIDBnnBWd0CwJTGvTpngVlmwGCdP9/Zl/tVrDqcuYzQ==", + "license": "MIT", + "dependencies": { + "loose-envify": "^1.1.0" + } + }, + "node_modules/semver": { + "version": "6.3.1", + "resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz", + "integrity": "sha512-BR7VvDCVHO+q2xBEWskxS6DJE1qRnb7DxzUrogb71CWoSficBxYsiAGd+Kl0mmq/MprG9yArRkyrQxTO6XjMzA==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + } + }, + "node_modules/source-map-js": { + "version": "1.2.2", + "resolved": "https://registry.npmjs.org/source-map-js/-/source-map-js-1.2.2.tgz", + "integrity": "sha512-KGj/8Y43x35aZVDtt+J4mK1hoLGHULMYfSkODJNQjNDC3oW1PqPoxMwo0pLUsWM/UEGzON/NxeHywEfNXNP3Vw==", + "dev": true, + "license": "BSD-3-Clause", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/typescript": { + "version": "5.6.3", + "resolved": "https://registry.npmjs.org/typescript/-/typescript-5.6.3.tgz", + "integrity": "sha512-hjcS1mhfuyi4WW8IWtjP7brDrG2cuDZukyrYrSauoXGNgx0S7zceP07adYkJycEr56BOUTNPzbInooiN3fn1qw==", + "dev": true, + "license": "Apache-2.0", + "bin": { + "tsc": "bin/tsc", + "tsserver": "bin/tsserver" + }, + "engines": { + "node": ">=14.17" + } + }, + "node_modules/update-browserslist-db": { + "version": "1.3.4", + "resolved": "https://registry.npmjs.org/update-browserslist-db/-/update-browserslist-db-1.3.4.tgz", + "integrity": "sha512-t8wonIOdQyOjtPd2VktlBu6QjTVTks6v8bSi7kd0r2Vll8TK1JYDZtRRdbmCHOAMphkZgrzK+L+vXK4pyZ8t4A==", + "dev": true, + "funding": [ + { + "type": "opencollective", + "url": "https://opencollective.com/browserslist" + }, + { + "type": "tidelift", + "url": "https://tidelift.com/funding/github/npm/browserslist" + }, + { + "type": "github", + "url": "https://github.com/sponsors/ai" + } + ], + "license": "MIT", + "dependencies": { + "escalade": "^3.2.0", + "picocolors": "^1.1.1" + }, + "bin": { + "update-browserslist-db": "cli.js" + }, + "peerDependencies": { + "browserslist": ">= 4.21.0" + } + }, + "node_modules/vite": { + "version": "6.0.7", + "resolved": "https://registry.npmjs.org/vite/-/vite-6.0.7.tgz", + "integrity": "sha512-RDt8r/7qx9940f8FcOIAH9PTViRrghKaK2K1jY3RaAURrEUbm9Du1mJ72G+jlhtG3WwodnfzY8ORQZbBavZEAQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "esbuild": "^0.24.2", + "postcss": "^8.4.49", + "rollup": "^4.23.0" + }, + "bin": { + "vite": "bin/vite.js" + }, + "engines": { + "node": "^18.0.0 || ^20.0.0 || >=22.0.0" + }, + "funding": { + "url": "https://github.com/vitejs/vite?sponsor=1" + }, + "optionalDependencies": { + "fsevents": "~2.3.3" + }, + "peerDependencies": { + "@types/node": "^18.0.0 || ^20.0.0 || >=22.0.0", + "jiti": ">=1.21.0", + "less": "*", + "lightningcss": "^1.21.0", + "sass": "*", + "sass-embedded": "*", + "stylus": "*", + "sugarss": "*", + "terser": "^5.16.0", + "tsx": "^4.8.1", + "yaml": "^2.4.2" + }, + "peerDependenciesMeta": { + "@types/node": { + "optional": true + }, + "jiti": { + "optional": true + }, + "less": { + "optional": true + }, + "lightningcss": { + "optional": true + }, + "sass": { + "optional": true + }, + "sass-embedded": { + "optional": true + }, + "stylus": { + "optional": true + }, + "sugarss": { + "optional": true + }, + "terser": { + "optional": true + }, + "tsx": { + "optional": true + }, + "yaml": { + "optional": true + } + } + }, + "node_modules/yallist": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/yallist/-/yallist-3.1.1.tgz", + "integrity": "sha512-a4UGQaWPH59mOXUYnAG2ewncQS4i4F43Tv3JoAM+s2VDAmS9NsK8GpDMLrCHPksFT7h3K6TOoUNn2pb7RoXx4g==", + "dev": true, + "license": "ISC" + } + } +} diff --git a/tenantboard/web/package.json b/tenantboard/web/package.json new file mode 100644 index 0000000..d718834 --- /dev/null +++ b/tenantboard/web/package.json @@ -0,0 +1,27 @@ +{ + "name": "tenantboard-web", + "private": true, + "version": "0.1.0", + "type": "module", + "description": "Tenantboard dashboard: per-tenant revenue, top customers and orders", + "license": "Apache-2.0", + "engines": { + "node": ">=22" + }, + "scripts": { + "dev": "vite", + "build": "tsc --noEmit && vite build", + "preview": "vite preview" + }, + "dependencies": { + "react": "18.3.1", + "react-dom": "18.3.1" + }, + "devDependencies": { + "@types/react": "18.3.12", + "@types/react-dom": "18.3.1", + "@vitejs/plugin-react": "4.3.4", + "typescript": "5.6.3", + "vite": "6.0.7" + } +} diff --git a/tenantboard/web/src/App.tsx b/tenantboard/web/src/App.tsx new file mode 100644 index 0000000..66d7e87 --- /dev/null +++ b/tenantboard/web/src/App.tsx @@ -0,0 +1,287 @@ +import { useCallback, useEffect, useState, type FormEvent, type ReactNode } from "react"; +import { + api, + ApiError, + type Merchant, + type OrderPage, + type Problem, + type RevenueSeries, + type Summary, + type TenantRole, + type TopCustomer, + type UploadResult, +} from "./api"; +import { formatMoney, formatTimestamp } from "./format"; +import { RevenueChart } from "./RevenueChart"; + +const PAGE_SIZE = 10; + +type Load = { state: "loading" } | { state: "ready"; data: T } | { state: "error"; problem: Problem }; + +function toProblem(error: unknown): Problem { + if (error instanceof ApiError) return error.problem; + return { type: "about:blank", title: "Network error", status: 0, detail: String(error) }; +} + +/** Fetch whenever `deps` change; stale responses are aborted, never rendered. */ +function useLoad(fetcher: (signal: AbortSignal) => Promise, deps: unknown[]): [Load, () => void] { + const [load, setLoad] = useState>({ state: "loading" }); + const [nonce, setNonce] = useState(0); + useEffect(() => { + const controller = new AbortController(); + setLoad({ state: "loading" }); + fetcher(controller.signal) + .then((data) => setLoad({ state: "ready", data })) + .catch((error: unknown) => { + if (!controller.signal.aborted) setLoad({ state: "error", problem: toProblem(error) }); + }); + return () => controller.abort(); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [...deps, nonce]); + return [load, useCallback(() => setNonce((n) => n + 1), [])]; +} + +function ProblemNotice({ problem }: { problem: Problem }) { + return ( +
+ {problem.title} + {problem.detail} + {problem.snowflake && ( + Snowflake {problem.snowflake.error_code}{problem.snowflake.query_id ? ` · query ${problem.snowflake.query_id}` : ""} + )} + {problem.errors?.slice(0, 5).map((e, i) => ( + {e.line ? `line ${e.line}, ` : ""}{e.column ?? e.loc?.join(".")}: {e.msg} + ))} +
+ ); +} + +function Card({ title, subtitle, children, actions }: { title: string; subtitle?: string; children: ReactNode; actions?: ReactNode }) { + return ( +
+
+
+

{title}

+ {subtitle &&

{subtitle}

} +
+ {actions} +
+ {children} +
+ ); +} + +function Loading({ load, children }: { load: Load; children: (data: T) => ReactNode }) { + if (load.state === "loading") return

Loading...

; + if (load.state === "error") return ; + return <>{children(load.data)}; +} + +function Kpis({ summary }: { summary: Summary }) { + return ( +
+
+ Revenue (completed orders) + {formatMoney(summary.revenue, summary.currency)} +
+
Orders{summary.orders}
+
Customers{summary.customers}
+
+ Average order value + {formatMoney(summary.average_order_value, summary.currency)} +
+
+ ); +} + +function TopCustomers({ customers, currency }: { customers: TopCustomer[]; currency: string }) { + return ( +
+
+ + + + + {customers.map((c) => ( + + + + + + + ))} + +
CustomerEmailOrdersRevenue
{c.name}{c.orders}{formatMoney(c.revenue, currency)}
+ + ); +} + +function Email({ value }: { value: string | null }) { + if (value === "***MASKED***") return ***MASKED***; + return {value ?? "-"}; +} + +function Orders({ page, onPage }: { page: OrderPage; onPage: (page: number) => void }) { + const first = page.total ? (page.page - 1) * page.page_size + 1 : 0; + const last = Math.min(page.page * page.page_size, page.total); + return ( + <> +
+ + + + + + + + + {page.items.map((o) => ( + + + + + + + + + + + ))} + +
OrderTenantOrdered (UTC)StatusChannelCustomercustomer_emailTotal
{o.order_id}{o.tenant_id.toLowerCase().replace(/_/g, "-")}{formatTimestamp(o.ordered_at)}{o.status}{o.channel}{o.customer_name ?? "-"}{formatMoney(o.total_amount, o.currency)}
+
+ + + ); +} + +function Upload({ slug, onUploaded }: { slug: string; onUploaded: () => void }) { + const [file, setFile] = useState(null); + const [result, setResult] = useState<{ ok: UploadResult } | { problem: Problem } | null>(null); + const [busy, setBusy] = useState(false); + + const submit = async (event: FormEvent) => { + event.preventDefault(); + if (!file) return; + setBusy(true); + try { + setResult({ ok: await api.upload(slug, file) }); + onUploaded(); + } catch (error) { + setResult({ problem: toProblem(error) }); + } finally { + setBusy(false); + } + }; + + return ( +
+ + + {result && "ok" in result && ( +

{result.ok.rows} rows: {result.ok.inserted} inserted, {result.ok.updated} updated

+ )} + {result && "problem" in result && } + + ); +} + +export function App() { + const [tenants] = useLoad((signal) => api.tenants(signal), []); + const [slug, setSlug] = useState(() => new URLSearchParams(location.search).get("tenant") ?? "acme"); + const [role, setRole] = useState("viewer"); + const [page, setPage] = useState(1); + + useEffect(() => { + const url = new URL(location.href); + url.searchParams.set("tenant", slug); + history.replaceState(null, "", url); + }, [slug]); + + const [summary, reloadSummary] = useLoad((s) => api.summary(slug, role, s), [slug, role]); + const [revenue, reloadRevenue] = useLoad((s) => api.revenue(slug, role, s), [slug, role]); + const [top] = useLoad((s) => api.topCustomers(slug, role, s), [slug, role]); + const [orders, reloadOrders] = useLoad((s) => api.orders(slug, role, page, PAGE_SIZE, s), [slug, role, page]); + + const merchant: Merchant | undefined = tenants.state === "ready" ? tenants.data.find((t) => t.slug === slug) : undefined; + const currency = merchant?.currency ?? "USD"; + + return ( +
+
+
+ +
+

Tenantboard

+

Merchant analytics, isolated by Snowflake row access policies

+
+
+
+ +
+ Role +
+ {(["viewer", "admin"] as const).map((r) => ( + + ))} +
+
+
+
+ + {tenants.state === "error" && } + +
+
+ {summary.state === "ready" ? ( + <>Snowflake session role {summary.data.snowflake_role} · tenant {summary.data.tenant_id} + {merchant && <> · {merchant.plan} plan · {merchant.region}} + ) : ( + <>Opening a session for {slug}... + )} +
+ + {(data) => } + + + {(data) => } + + +
+ + {(data) => } + + + {role === "admin" ? ( + { reloadSummary(); reloadRevenue(); reloadOrders(); }} /> + ) : ( +

Viewers read masked data and cannot write. The API also refuses uploads from the viewer role (403).

+ )} + +
+ + + {(data) => } + +
+
+ ); +} diff --git a/tenantboard/web/src/RevenueChart.tsx b/tenantboard/web/src/RevenueChart.tsx new file mode 100644 index 0000000..4058b0c --- /dev/null +++ b/tenantboard/web/src/RevenueChart.tsx @@ -0,0 +1,129 @@ +import { useEffect, useMemo, useRef, useState } from "react"; +import type { DailyRevenue } from "./api"; +import { formatDay, formatMoney, formatMoneyCompact } from "./format"; + +// Daily completed revenue (columns, series slot 1) with its trailing 7-day mean +// (line, slot 2). One y-axis: both are the same currency. + +const HEIGHT = 240; +const MARGIN = { top: 16, right: 64, bottom: 28, left: 56 }; + +function niceMax(value: number): number { + if (value <= 0) return 1; + const magnitude = 10 ** Math.floor(Math.log10(value)); + const step = [1, 2, 2.5, 5, 10].find((s) => s * magnitude >= value / 4) ?? 10; + return Math.ceil(value / (step * magnitude)) * step * magnitude; +} + +/** Column path with a 4px rounded data-end and a square baseline. */ +function columnPath(x: number, y: number, width: number, baseline: number): string { + const r = Math.min(4, width / 2, baseline - y); + if (baseline - y <= 0) return ""; + return `M${x},${baseline} V${y + r} Q${x},${y} ${x + r},${y} H${x + width - r} Q${x + width},${y} ${x + width},${y + r} V${baseline} Z`; +} + +export function RevenueChart({ days, currency }: { days: DailyRevenue[]; currency: string }) { + const frame = useRef(null); + const [width, setWidth] = useState(720); + const [hover, setHover] = useState(null); + const [showTable, setShowTable] = useState(false); + + useEffect(() => { + const element = frame.current; + if (!element) return; + const observer = new ResizeObserver(([entry]) => setWidth(Math.max(320, Math.floor(entry.contentRect.width)))); + observer.observe(element); + return () => observer.disconnect(); + }, [showTable]); + + const geometry = useMemo(() => { + const plotWidth = width - MARGIN.left - MARGIN.right; + const plotHeight = HEIGHT - MARGIN.top - MARGIN.bottom; + const max = niceMax(Math.max(0, ...days.map((d) => Math.max(d.revenue, d.revenue_7d_avg)))); + const band = days.length ? plotWidth / days.length : plotWidth; + const barWidth = Math.max(2, Math.min(24, band - 2)); // 2px surface gap between neighbours + const x = (i: number) => MARGIN.left + i * band + band / 2; + const y = (v: number) => MARGIN.top + plotHeight - (v / max) * plotHeight; + const ticks = [0, 0.25, 0.5, 0.75, 1].map((t) => t * max); + const line = days.map((d, i) => `${i ? "L" : "M"}${x(i).toFixed(1)},${y(d.revenue_7d_avg).toFixed(1)}`).join(" "); + return { plotWidth, plotHeight, band, barWidth, x, y, ticks, line, baseline: y(0) }; + }, [days, width]); + + if (!days.length) return

No completed orders yet.

; + + const { band, barWidth, x, y, ticks, line, baseline } = geometry; + const last = days.length - 1; + const active = hover === null ? null : days[hover]; + const labelEvery = Math.ceil(days.length / Math.max(2, Math.floor(geometry.plotWidth / 90))); + + return ( +
+
+
    +
    • +
    • +
+ +
+ + {showTable ? ( +
+ + + + + + + {days.map((d) => ( + + + + + + + ))} + +
Daily revenue in {currency}
DayOrdersRevenue7-day average
{formatDay(d.day)}{d.orders}{formatMoney(d.revenue, currency)}{formatMoney(d.revenue_7d_avg, currency)}
+
+ ) : ( +
+ setHover(null)}> + {ticks.map((t) => ( + + + {formatMoneyCompact(t, currency)} + + ))} + {days.map((d, i) => (i % labelEvery === 0 ? ( + {formatDay(d.day)} + ) : null))} + {hover !== null && } + {days.map((d, i) => ( + + ))} + + + + {formatMoneyCompact(days[last].revenue_7d_avg, currency)} + + {days.map((d, i) => ( + setHover(i)} onFocus={() => setHover(i)} tabIndex={-1} /> + ))} + + {active && hover !== null && ( +
+ {formatDay(active.day)} + + +
+ )} +
+ )} +
+ ); +} diff --git a/tenantboard/web/src/api.ts b/tenantboard/web/src/api.ts new file mode 100644 index 0000000..3fb3e74 --- /dev/null +++ b/tenantboard/web/src/api.ts @@ -0,0 +1,123 @@ +// Typed client for the Tenantboard API. Every error comes back as RFC 9457 +// problem details; ApiError keeps them, including any Snowflake error code. + +export type TenantRole = "viewer" | "admin"; + +export interface Merchant { + tenant_id: string; + slug: string; + name: string; + plan: string; + region: string; + currency: string; +} + +export interface Summary { + tenant_id: string; + role: TenantRole; + snowflake_role: string; + orders: number; + completed_orders: number; + customers: number; + revenue: string; + average_order_value: string; + currency: string; + first_order_at: string | null; + last_order_at: string | null; +} + +export interface DailyRevenue { + day: string; + orders: number; + revenue: number; + revenue_7d_avg: number; +} + +export interface RevenueSeries { + tenant_id: string; + currency: string; + days: DailyRevenue[]; +} + +export interface TopCustomer { + customer_id: number; + name: string; + email: string; + orders: number; + revenue: string; +} + +export interface Order { + order_id: string; + tenant_id: string; + ordered_at: string; + status: string; + channel: string; + currency: string; + total_amount: string; + customer_id: number; + customer_name: string | null; + customer_email: string | null; +} + +export interface OrderPage { + items: Order[]; + page: number; + page_size: number; + total: number; +} + +export interface UploadResult { + tenant_id: string; + rows: number; + inserted: number; + updated: number; +} + +export interface Problem { + type: string; + title: string; + status: number; + detail: string; + snowflake?: { error_code: string; sql_state: string | null; query_id: string | null }; + errors?: { line?: number; column?: string; loc?: string[]; msg: string }[]; +} + +export class ApiError extends Error { + constructor(readonly problem: Problem) { + super(problem.detail || problem.title); + } +} + +async function request(path: string, role: TenantRole | null, init: RequestInit = {}, signal?: AbortSignal): Promise { + const headers = new Headers(init.headers); + if (role) headers.set("X-Tenant-Role", role); + const response = await fetch(path, { ...init, headers, signal }); + if (!response.ok) { + let problem: Problem; + try { + problem = (await response.json()) as Problem; + } catch { + problem = { type: "about:blank", title: response.statusText, status: response.status, detail: `HTTP ${response.status}` }; + } + throw new ApiError(problem); + } + return (await response.json()) as T; +} + +const tenant = (slug: string) => `/api/tenants/${encodeURIComponent(slug)}`; + +export const api = { + tenants: (signal?: AbortSignal) => request("/api/tenants", null, {}, signal), + summary: (slug: string, role: TenantRole, signal?: AbortSignal) => request(`${tenant(slug)}/summary`, role, {}, signal), + revenue: (slug: string, role: TenantRole, signal?: AbortSignal) => request(`${tenant(slug)}/revenue`, role, {}, signal), + topCustomers: (slug: string, role: TenantRole, signal?: AbortSignal) => + request(`${tenant(slug)}/top-customers?limit=5`, role, {}, signal), + orders: (slug: string, role: TenantRole, page: number, pageSize: number, signal?: AbortSignal) => + request(`${tenant(slug)}/orders?page=${page}&page_size=${pageSize}`, role, {}, signal), + upload: (slug: string, file: File) => { + const body = new FormData(); + body.append("file", file); + return request(`${tenant(slug)}/orders/upload`, "admin", { method: "POST", body }); + }, +}; diff --git a/tenantboard/web/src/format.ts b/tenantboard/web/src/format.ts new file mode 100644 index 0000000..342cd95 --- /dev/null +++ b/tenantboard/web/src/format.ts @@ -0,0 +1,29 @@ +const moneyFormats = new Map(); +const compactFormats = new Map(); + +function cached(map: Map, currency: string, options: Intl.NumberFormatOptions) { + let format = map.get(currency); + if (!format) { + format = new Intl.NumberFormat("en-US", { style: "currency", currency, ...options }); + map.set(currency, format); + } + return format; +} + +export function formatMoney(value: number | string, currency: string): string { + return cached(moneyFormats, currency, { minimumFractionDigits: 2 }).format(Number(value)); +} + +export function formatMoneyCompact(value: number | string, currency: string): string { + return cached(compactFormats, currency, { notation: "compact", maximumFractionDigits: 1 }).format(Number(value)); +} + +export function formatDay(day: string): string { + return new Date(`${day}T00:00:00Z`).toLocaleDateString("en-US", { month: "short", day: "numeric", timeZone: "UTC" }); +} + +export function formatTimestamp(timestamp: string): string { + return new Date(`${timestamp}Z`).toLocaleString("en-US", { + month: "short", day: "numeric", hour: "2-digit", minute: "2-digit", timeZone: "UTC", + }); +} diff --git a/tenantboard/web/src/main.tsx b/tenantboard/web/src/main.tsx new file mode 100644 index 0000000..573e24a --- /dev/null +++ b/tenantboard/web/src/main.tsx @@ -0,0 +1,12 @@ +import { StrictMode } from "react"; +import { createRoot } from "react-dom/client"; +import { App } from "./App"; +import "./styles.css"; + +const root = document.getElementById("root"); +if (!root) throw new Error("missing #root"); +createRoot(root).render( + + + , +); diff --git a/tenantboard/web/src/styles.css b/tenantboard/web/src/styles.css new file mode 100644 index 0000000..e71401d --- /dev/null +++ b/tenantboard/web/src/styles.css @@ -0,0 +1,177 @@ +:root { + color-scheme: light; + --page: #f9f9f7; + --surface: #fcfcfb; + --surface-2: #f0efec; + --text-primary: #0b0b0b; + --text-secondary: #52514e; + --text-muted: #6b6a65; + --grid: #e1e0d9; + --axis: #c3c2b7; + --border: rgba(11, 11, 11, 0.1); + --series-1: #2a78d6; + --series-2: #eb6834; + --accent: #2a78d6; + --accent-ink: #ffffff; + --good: #006300; + --critical: #b42f2f; + --warning-bg: #fdf3dc; + --critical-bg: #fbeaea; + --masked-bg: #eeeef8; + --masked-ink: #4a3aa7; + --radius: 10px; + font-family: system-ui, -apple-system, "Segoe UI", sans-serif; + font-size: 15px; + line-height: 1.45; +} + +@media (prefers-color-scheme: dark) { + :root:not([data-theme="light"]) { + color-scheme: dark; + --page: #0d0d0d; + --surface: #1a1a19; + --surface-2: #262624; + --text-primary: #f0efec; + --text-secondary: #c3c2b7; + --text-muted: #9a998f; + --grid: #2c2c2a; + --axis: #383835; + --border: rgba(255, 255, 255, 0.1); + --series-1: #3987e5; + --series-2: #d95926; + --accent: #3987e5; + --good: #0ca30c; + --critical: #e66767; + --warning-bg: #3a2f14; + --critical-bg: #3a1c1c; + --masked-bg: #25233a; + --masked-ink: #b5adf2; + } +} + +:root[data-theme="dark"] { + color-scheme: dark; + --page: #0d0d0d; + --surface: #1a1a19; + --surface-2: #262624; + --text-primary: #f0efec; + --text-secondary: #c3c2b7; + --text-muted: #9a998f; + --grid: #2c2c2a; + --axis: #383835; + --border: rgba(255, 255, 255, 0.1); + --series-1: #3987e5; + --series-2: #d95926; + --accent: #3987e5; + --good: #0ca30c; + --critical: #e66767; + --warning-bg: #3a2f14; + --critical-bg: #3a1c1c; + --masked-bg: #25233a; + --masked-ink: #b5adf2; +} + +* { box-sizing: border-box; } +body { margin: 0; background: var(--page); color: var(--text-primary); } +h1, h2 { margin: 0; font-weight: 600; } +h1 { font-size: 1.25rem; } +h2 { font-size: 1rem; } +p { margin: 0; } +code { font-family: ui-monospace, "SFMono-Regular", Menlo, monospace; font-size: 0.85em; } +.muted { color: var(--text-muted); font-size: 0.875rem; } +.sr-only { position: absolute; width: 1px; height: 1px; overflow: hidden; clip: rect(0 0 0 0); white-space: nowrap; } + +.app { max-width: 1200px; margin: 0 auto; padding: 16px; } +.topbar { display: flex; flex-wrap: wrap; gap: 16px; align-items: flex-end; justify-content: space-between; padding: 8px 0 16px; } +.brand { display: flex; gap: 12px; align-items: center; } +.logo { + display: grid; place-items: center; width: 40px; height: 40px; border-radius: 10px; + background: var(--accent); color: var(--accent-ink); font-weight: 700; letter-spacing: -0.02em; +} +.controls { display: flex; flex-wrap: wrap; gap: 12px; align-items: flex-end; } +.field { display: flex; flex-direction: column; gap: 4px; font-size: 0.8rem; color: var(--text-secondary); } +select, button, input[type="file"] { font: inherit; color: inherit; } +select { + min-width: 260px; max-width: 100%; padding: 8px 10px; border-radius: 8px; + border: 1px solid var(--border); background: var(--surface); +} +button { + padding: 8px 14px; border-radius: 8px; border: 1px solid var(--border); background: var(--surface); cursor: pointer; +} +button:hover:not(:disabled) { background: var(--surface-2); } +button:disabled { opacity: 0.5; cursor: default; } +button:focus-visible, select:focus-visible, input:focus-visible { outline: 2px solid var(--accent); outline-offset: 2px; } +button.link { border: none; background: none; padding: 4px 0; color: var(--accent); } +button.link:hover { background: none; text-decoration: underline; } +.segmented { display: inline-flex; border: 1px solid var(--border); border-radius: 8px; overflow: hidden; background: var(--surface); } +.segmented button { border: none; border-radius: 0; padding: 8px 16px; } +.segmented button.on { background: var(--accent); color: var(--accent-ink); } + +main { display: flex; flex-direction: column; gap: 16px; } +.session { + font-size: 0.85rem; color: var(--text-secondary); padding: 8px 12px; + background: var(--surface); border: 1px solid var(--border); border-radius: 8px; +} +.session code { color: var(--text-primary); } + +.kpis { display: grid; grid-template-columns: 2fr repeat(3, 1fr); gap: 12px; } +.kpi { display: flex; flex-direction: column; gap: 4px; padding: 16px; background: var(--surface); border: 1px solid var(--border); border-radius: var(--radius); } +.kpi-label { font-size: 0.8rem; color: var(--text-secondary); } +.kpi-value { font-size: 1.5rem; font-weight: 600; } +.kpi-hero .kpi-value { font-size: 2.25rem; } + +.card { background: var(--surface); border: 1px solid var(--border); border-radius: var(--radius); padding: 16px; min-width: 0; } +.card-head { display: flex; justify-content: space-between; gap: 12px; margin-bottom: 12px; } +.grid-2 { display: grid; grid-template-columns: 3fr 2fr; gap: 16px; } + +.table-wrap { overflow-x: auto; } +table { width: 100%; border-collapse: collapse; font-size: 0.875rem; } +th { text-align: left; font-weight: 600; color: var(--text-secondary); border-bottom: 1px solid var(--axis); padding: 8px; white-space: nowrap; } +td { padding: 8px; border-bottom: 1px solid var(--grid); white-space: nowrap; } +.num { text-align: right; font-variant-numeric: tabular-nums; } +.tenant-tag { padding: 2px 8px; border-radius: 999px; background: var(--surface-2); font-size: 0.8rem; } +.masked { padding: 2px 6px; border-radius: 4px; background: var(--masked-bg); color: var(--masked-ink); font-family: ui-monospace, Menlo, monospace; font-size: 0.8rem; } +.status { font-size: 0.8rem; } +.status-refunded, .status-cancelled { color: var(--text-muted); } +.pager { display: flex; gap: 8px; align-items: center; justify-content: flex-end; margin-top: 12px; } + +.upload { display: flex; flex-direction: column; gap: 12px; align-items: flex-start; } +.upload label { display: flex; flex-direction: column; gap: 8px; } +.ok { color: var(--good); font-weight: 600; } +.problem { + display: flex; flex-direction: column; gap: 4px; padding: 12px; border-radius: 8px; + background: var(--critical-bg); border: 1px solid var(--border); color: var(--text-primary); font-size: 0.875rem; +} +.problem strong { color: var(--critical); } + +/* Chart */ +.chart-head { display: flex; justify-content: space-between; align-items: center; margin-bottom: 8px; } +.legend { display: flex; gap: 16px; list-style: none; margin: 0; padding: 0; font-size: 0.8rem; color: var(--text-secondary); } +.legend li, .tooltip span { display: inline-flex; align-items: center; gap: 6px; } +.key { display: inline-block; flex: none; } +.key-bar { width: 10px; height: 10px; border-radius: 2px; background: var(--series-1); } +.key-line { width: 14px; height: 2px; border-radius: 1px; background: var(--series-2); } +.plot { position: relative; width: 100%; contain: inline-size; } +.plot svg { display: block; } +.grid { stroke: var(--grid); stroke-width: 1; } +.axis { stroke: var(--axis); stroke-width: 1; } +.tick { fill: var(--text-muted); font-size: 11px; font-variant-numeric: tabular-nums; } +.bar { fill: var(--series-1); } +.avg-line { fill: none; stroke: var(--series-2); stroke-width: 2; stroke-linejoin: round; stroke-linecap: round; } +.avg-dot { fill: var(--series-2); stroke: var(--surface); stroke-width: 2; } +.end-label { fill: var(--text-secondary); font-size: 12px; } +.hover-band { fill: var(--surface-2); } +.hit { fill: transparent; } +.tooltip { + position: absolute; transform: translate(-50%, -100%); pointer-events: none; display: flex; flex-direction: column; gap: 2px; + padding: 8px 10px; border-radius: 8px; background: var(--surface); border: 1px solid var(--border); + box-shadow: 0 4px 16px rgba(0, 0, 0, 0.12); font-size: 0.8rem; white-space: nowrap; +} + +@media (max-width: 860px) { + .kpis { grid-template-columns: 1fr 1fr; } + .kpi-hero { grid-column: 1 / -1; } + .grid-2 { grid-template-columns: 1fr; } + select { min-width: 0; width: 100%; } + .controls, .field { width: 100%; } +} diff --git a/tenantboard/web/tsconfig.json b/tenantboard/web/tsconfig.json new file mode 100644 index 0000000..cdfd561 --- /dev/null +++ b/tenantboard/web/tsconfig.json @@ -0,0 +1,17 @@ +{ + "compilerOptions": { + "target": "ES2022", + "lib": ["ES2022", "DOM", "DOM.Iterable"], + "module": "ESNext", + "moduleResolution": "Bundler", + "jsx": "react-jsx", + "strict": true, + "noUnusedLocals": true, + "noUnusedParameters": true, + "noFallthroughCasesInSwitch": true, + "isolatedModules": true, + "skipLibCheck": true, + "types": ["vite/client"] + }, + "include": ["src"] +} diff --git a/tenantboard/web/vite.config.ts b/tenantboard/web/vite.config.ts new file mode 100644 index 0000000..79e85b8 --- /dev/null +++ b/tenantboard/web/vite.config.ts @@ -0,0 +1,20 @@ +import { defineConfig } from "vite"; +import react from "@vitejs/plugin-react"; + +// The dashboard calls /api/* on its own origin; the dev server forwards those +// requests to the FastAPI service, so the browser never needs CORS. +const apiPort = process.env.API_PORT ?? "8000"; +const webPort = Number(process.env.WEB_PORT ?? "5173"); + +export default defineConfig({ + plugins: [react()], + server: { + host: "127.0.0.1", + port: webPort, + strictPort: true, + proxy: { + "/api": { target: `http://127.0.0.1:${apiPort}`, changeOrigin: false }, + }, + }, + preview: { host: "127.0.0.1", port: webPort, strictPort: true }, +}); From cda4279f64be3828049dd1db85426ca8a8950e83 Mon Sep 17 00:00:00 2001 From: Waldemar Hummer Date: Tue, 27 Oct 2026 11:42:23 +0100 Subject: [PATCH 2/2] update CODEOWNERS --- CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CODEOWNERS b/CODEOWNERS index a94c1a1..4a6fc0a 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -1,3 +1,3 @@ # From: https://docs.github.com/en/repositories/managing-your-repositorys-settings-and-features/customizing-your-repository/about-code-owners#example-of-a-codeowners-file -* @whummer @eruditmorina @hovaesco @sabir-akhadov-localstack +* @hovaesco @sabir-akhadov-localstack