From a50a797f77be770c19429ff815ed70f45df3df8a Mon Sep 17 00:00:00 2001 From: Gautier DI FOLCO Date: Tue, 4 Aug 2026 21:04:49 +0200 Subject: [PATCH] WPB-22972: bootstrap Spar Postgres infrastructure Wire a Hasql Postgres pool, schema-migration run, and shared storage-location config into spar, and add a cassandraSpar client to the background-worker (with full cassandraSpar TLS plumbing mirroring cassandraBrig). This is the shared infrastructure prerequisite for the Spar Cassandra->Postgres store migrations (WPB-22972/22973/22974); no store interpreters, SQL migrations, or migration workers are included yet. --- changelog.d/5-internal/WPB-22972-0 | 1 + charts/wire-server/templates/_helpers.tpl | 8 +++++++ .../background-worker/configmap.yaml | 14 +++++++++++ .../background-worker/deployment.yaml | 9 ++++++++ .../wire-server/templates/spar/configmap.yaml | 7 ++++++ charts/wire-server/values.yaml | 22 ++++++++++++++++++ .../src/developer/reference/config-options.md | 7 +++++- hack/helm_vars/wire-server/values.yaml.gotmpl | 23 +++++++++++++++++++ .../background-worker.integration.yaml | 6 +++++ .../src/Wire/BackgroundWorker/Env.hs | 3 +++ .../src/Wire/BackgroundWorker/Options.hs | 1 + .../Wire/BackendNotificationPusherSpec.hs | 2 ++ .../background-worker/test/Test/Wire/Util.hs | 1 + services/spar/spar.integration.yaml | 19 +++++++++++++++ services/spar/src/Spar/App.hs | 4 +++- .../spar/src/Spar/CanonicalInterpreter.hs | 3 +++ services/spar/src/Spar/Options.hs | 13 ++++++++++- services/spar/src/Spar/Run.hs | 4 ++++ services/spar/test-integration/Util/Core.hs | 1 + services/spar/test/Test/Spar/Saml/IdPSpec.hs | 17 +++++++++++++- 20 files changed, 161 insertions(+), 4 deletions(-) create mode 100644 changelog.d/5-internal/WPB-22972-0 diff --git a/changelog.d/5-internal/WPB-22972-0 b/changelog.d/5-internal/WPB-22972-0 new file mode 100644 index 00000000000..e51a6f26290 --- /dev/null +++ b/changelog.d/5-internal/WPB-22972-0 @@ -0,0 +1 @@ +Bootstrap Spar Postgres infrastructure (connection pool, schema migrations, storage-location config) for the Cassandra->Postgres store migration. diff --git a/charts/wire-server/templates/_helpers.tpl b/charts/wire-server/templates/_helpers.tpl index 94aca197dc7..a6b4cf0ec7e 100644 --- a/charts/wire-server/templates/_helpers.tpl +++ b/charts/wire-server/templates/_helpers.tpl @@ -39,6 +39,14 @@ {{- end -}} {{- end -}} +{{- define "sparTlsSecretRef" -}} +{{- if .cassandraSpar.tlsCaSecretRef -}} +{{ .cassandraSpar.tlsCaSecretRef | toYaml }} +{{- else }} +{{- dict "name" "background-worker-cassandra-spar" "key" "ca.pem" | toYaml -}} +{{- end -}} +{{- end -}} + {{/* BRIG */}} {{- define "brig.tlsSecretRef" -}} {{- if .cassandra.tlsCaSecretRef -}} diff --git a/charts/wire-server/templates/background-worker/configmap.yaml b/charts/wire-server/templates/background-worker/configmap.yaml index 299d0703d3a..99bb7668880 100644 --- a/charts/wire-server/templates/background-worker/configmap.yaml +++ b/charts/wire-server/templates/background-worker/configmap.yaml @@ -64,6 +64,20 @@ data: {{- end }} {{- end }} + {{- if .cassandraSpar }} + cassandraSpar: + endpoint: + host: {{ .cassandraSpar.host }} + port: 9042 + keyspace: spar + {{- if hasKey .cassandraSpar "filterNodesByDatacentre" }} + filterNodesByDatacentre: {{ .cassandraSpar.filterNodesByDatacentre }} + {{- end }} + {{- if eq (include "useCassandraTLS" .cassandraSpar) "true" }} + tlsCa: /etc/wire/background-worker/cassandra-spar/{{- (include "sparTlsSecretRef" . | fromYaml).key }} + {{- end }} + {{- end }} + {{- with .rabbitmq }} rabbitmq: host: {{ .host }} diff --git a/charts/wire-server/templates/background-worker/deployment.yaml b/charts/wire-server/templates/background-worker/deployment.yaml index f91ccb5ee17..c11f35313b0 100644 --- a/charts/wire-server/templates/background-worker/deployment.yaml +++ b/charts/wire-server/templates/background-worker/deployment.yaml @@ -59,6 +59,11 @@ spec: secret: secretName: {{ (include "brigTlsSecretRef" $backgroundWorker.config | fromYaml).name }} {{- end }} + {{- if eq (include "useCassandraTLS" $backgroundWorker.config.cassandraSpar) "true" }} + - name: "background-worker-cassandra-spar" + secret: + secretName: {{ (include "sparTlsSecretRef" $backgroundWorker.config | fromYaml).name }} + {{- end }} {{- if eq (include "useCassandraTLS" .Values.galley.config.cassandra) "true" }} - name: "galley-cassandra" secret: @@ -97,6 +102,10 @@ spec: - name: "background-worker-cassandra-brig" mountPath: "/etc/wire/background-worker/cassandra-brig" {{- end }} + {{- if eq (include "useCassandraTLS" $backgroundWorker.config.cassandraSpar) "true" }} + - name: "background-worker-cassandra-spar" + mountPath: "/etc/wire/background-worker/cassandra-spar" + {{- end }} {{- if eq (include "useCassandraTLS" .Values.galley.config.cassandra) "true" }} - name: "galley-cassandra" mountPath: "/etc/wire/galley/cassandra" diff --git a/charts/wire-server/templates/spar/configmap.yaml b/charts/wire-server/templates/spar/configmap.yaml index 6dfd671442a..f9c42620cf0 100644 --- a/charts/wire-server/templates/spar/configmap.yaml +++ b/charts/wire-server/templates/spar/configmap.yaml @@ -29,6 +29,13 @@ data: tlsCa: /etc/wire/spar/cassandra/{{- (include "spar.tlsSecretRef" . | fromYaml).key }} {{- end }} + postgresql: {{ toYaml .postgresql | nindent 6 }} + postgresqlPool: {{ toYaml .postgresqlPool | nindent 6 }} + {{- if .postgresqlPassword }} + postgresqlPassword: {{ .postgresqlPassword | quote }} + {{- end }} + postgresMigration: {{- toYaml .postgresMigration | nindent 6 }} + maxttlAuthreq: {{ .maxttlAuthreq }} maxttlAuthresp: {{ .maxttlAuthresp }} diff --git a/charts/wire-server/values.yaml b/charts/wire-server/values.yaml index 9a98be5d528..8e53642b532 100644 --- a/charts/wire-server/values.yaml +++ b/charts/wire-server/values.yaml @@ -841,6 +841,26 @@ spar: # tlsCaSecretRef: # name: # key: + + # Postgres connection (libpq key/value map). See: + # https://www.postgresql.org/docs/17/libpq-connect.html#LIBPQ-PARAMKEYWORDS + postgresql: + host: postgresql + port: "5432" + user: wire-server + dbname: wire-server + postgresqlPool: + size: 100 + acquisitionTimeout: 10s + idlenessTimeout: 10m + # Per-store storage-location flags (shared type). Spar-relevant stores are + # added by their migration tasks; defaults stay on Cassandra. + postgresMigration: + conversation: cassandra + conversationCodes: cassandra + teamFeatures: cassandra + domainRegistration: cassandra + user: cassandra richInfoLimit: 5000 maxScimTokens: 0 @@ -981,6 +1001,8 @@ background-worker: host: aws-cassandra cassandraBrig: host: aws-cassandra + cassandraSpar: + host: aws-cassandra postgresqlPool: size: 5 diff --git a/docs/src/developer/reference/config-options.md b/docs/src/developer/reference/config-options.md index 9154c97ea73..655bb3a7766 100644 --- a/docs/src/developer/reference/config-options.md +++ b/docs/src/developer/reference/config-options.md @@ -2015,7 +2015,7 @@ server, verification can be turned off by settings `insecureSkipVerifyTls` to ## Configure PostgreSQL -`brig`, `galley`, `gundeck`, and `background-worker` require a PostgreSQL database. The configured user needs to +`brig`, `galley`, `gundeck`, `spar`, and `background-worker` require a PostgreSQL database. The configured user needs to be able to write data and change the schema (e.g. create and alter tables.) The internal configuration YAML file format and the Helm charts for `brig` and @@ -2100,6 +2100,11 @@ background-worker: migrateDomainRegistration: false ``` +`spar` additionally exposes its own `spar.config.postgresMigration` block, using the +same shared storage-location type. Spar-owned stores (e.g. SAML users, SCIM tokens, +external ids) are added to it by their respective migration tasks; until then the +block mirrors the shared defaults and stays on Cassandra. + #### Migration for existing installations Existing installations should migrate Cassandra-backed data to PostgreSQL over diff --git a/hack/helm_vars/wire-server/values.yaml.gotmpl b/hack/helm_vars/wire-server/values.yaml.gotmpl index 43373b1cf2e..cf25eea38de 100644 --- a/hack/helm_vars/wire-server/values.yaml.gotmpl +++ b/hack/helm_vars/wire-server/values.yaml.gotmpl @@ -582,6 +582,21 @@ spar: name: "cassandra-jks-keystore" key: "ca.crt" {{- end }} + postgresql: + host: postgresql + port: "5432" + user: wire-server + dbname: wire-server + postgresqlPool: + size: 100 + acquisitionTimeout: 10s + idlenessTimeout: 10m + postgresMigration: + conversation: {{ .Values.conversationStore }} + conversationCodes: {{ .Values.conversationCodesStore }} + teamFeatures: {{ .Values.teamFeaturesStore }} + domainRegistration: {{ .Values.domainRegistration }} + user: {{ .Values.userStore }} logLevel: Debug domain: zinfra.io appUri: http://spar:8080/ @@ -688,6 +703,14 @@ background-worker: name: "cassandra-jks-keystore" key: "ca.crt" {{- end }} + cassandraSpar: + host: {{ .Values.cassandraHost }} + replicaCount: 1 + {{- if .Values.useK8ssandraSSL.enabled }} + tlsCaSecretRef: + name: "cassandra-jks-keystore" + key: "ca.crt" + {{- end }} rabbitmq: port: 5671 adminPort: 15671 diff --git a/services/background-worker/background-worker.integration.yaml b/services/background-worker/background-worker.integration.yaml index b0bd0d172e4..657970fc5b9 100644 --- a/services/background-worker/background-worker.integration.yaml +++ b/services/background-worker/background-worker.integration.yaml @@ -30,6 +30,12 @@ cassandraBrig: port: 9042 keyspace: brig_test +cassandraSpar: + endpoint: + host: 127.0.0.1 + port: 9042 + keyspace: spar_test + postgresqlPool: size: 5 acquisitionTimeout: 10s diff --git a/services/background-worker/src/Wire/BackgroundWorker/Env.hs b/services/background-worker/src/Wire/BackgroundWorker/Env.hs index 10886fd8332..307cb411050 100644 --- a/services/background-worker/src/Wire/BackgroundWorker/Env.hs +++ b/services/background-worker/src/Wire/BackgroundWorker/Env.hs @@ -90,6 +90,7 @@ data Env = Env cassandra :: ClientState, cassandraGalley :: ClientState, cassandraBrig :: ClientState, + cassandraSpar :: ClientState, hasqlPool :: Hasql.Pool, -- May contain the PostgreSQL password. Do not unwrap outside the Arbiter boundary. arbiterConnStr :: SecretText, @@ -158,6 +159,8 @@ mkEnv opts galleyOpts = do cassandraGalley <- defInitCassandra galleyOpts._cassandra =<< setLoggerName "cassandra-galley" logger Log.info logger $ Log.msg @Text "Connecting to Cassandra (brig)..." cassandraBrig <- defInitCassandra opts.cassandraBrig =<< setLoggerName "cassandra-brig" logger + Log.info logger $ Log.msg @Text "Connecting to Cassandra (spar)..." + cassandraSpar <- defInitCassandra opts.cassandraSpar =<< setLoggerName "cassandra-spar" logger Log.info logger $ Log.msg @Text "Cassandra connections established" http2Manager <- initHttp2Manager httpManager <- newManager defaultManagerSettings diff --git a/services/background-worker/src/Wire/BackgroundWorker/Options.hs b/services/background-worker/src/Wire/BackgroundWorker/Options.hs index 035460cfc32..814563e3554 100644 --- a/services/background-worker/src/Wire/BackgroundWorker/Options.hs +++ b/services/background-worker/src/Wire/BackgroundWorker/Options.hs @@ -48,6 +48,7 @@ data Opts = Opts backendNotificationPusher :: BackendNotificationsConfig, cassandra :: CassandraOpts, cassandraBrig :: CassandraOpts, + cassandraSpar :: CassandraOpts, postgresqlPool :: !PoolConfig, postgresMigration :: !PostgresMigrationOpts, migrateConversations :: !Bool, diff --git a/services/background-worker/test/Test/Wire/BackendNotificationPusherSpec.hs b/services/background-worker/test/Test/Wire/BackendNotificationPusherSpec.hs index be46a03c648..eb39884a9cc 100644 --- a/services/background-worker/test/Test/Wire/BackendNotificationPusherSpec.hs +++ b/services/background-worker/test/Test/Wire/BackendNotificationPusherSpec.hs @@ -473,6 +473,7 @@ spec = do let cassandra = undefined cassandraGalley = undefined cassandraBrig = undefined + cassandraSpar = undefined federatorInternal = Endpoint "localhost" 8097 http2Manager = undefined statuses = undefined @@ -538,6 +539,7 @@ spec = do httpManager <- newManager defaultManagerSettings let federatorInternal = Endpoint "localhost" 8097 cassandraBrig = undefined + cassandraSpar = undefined http2Manager = undefined statuses = undefined rabbitmqAdminClient = Just $ mockRabbitMqAdminClient mockAdmin diff --git a/services/background-worker/test/Test/Wire/Util.hs b/services/background-worker/test/Test/Wire/Util.hs index 5d89532bfec..bd6c055c848 100644 --- a/services/background-worker/test/Test/Wire/Util.hs +++ b/services/background-worker/test/Test/Wire/Util.hs @@ -44,6 +44,7 @@ testEnv = do let cassandra = undefined cassandraGalley = undefined cassandraBrig = undefined + cassandraSpar = undefined postgresMigration = PostgresMigrationOpts { conversation = CassandraStorage, diff --git a/services/spar/spar.integration.yaml b/services/spar/spar.integration.yaml index a50e065c263..dcd10f6bcf8 100644 --- a/services/spar/spar.integration.yaml +++ b/services/spar/spar.integration.yaml @@ -31,6 +31,25 @@ cassandra: keyspace: spar_test filterNodesByDatacentre: datacenter1 +postgresql: + host: 127.0.0.1 + port: "5432" + user: wire-server + dbname: backendA + password: posty-the-gres + +postgresqlPool: + size: 20 + acquisitionTimeout: 10s + idlenessTimeout: 10m + +postgresMigration: + conversation: cassandra + conversationCodes: cassandra + teamFeatures: cassandra + domainRegistration: cassandra + user: cassandra + # Wire/AWS specific, optional # discoUrl: "https://" diff --git a/services/spar/src/Spar/App.hs b/services/spar/src/Spar/App.hs index a5e959ea23a..1fbe138fc9e 100644 --- a/services/spar/src/Spar/App.hs +++ b/services/spar/src/Spar/App.hs @@ -61,6 +61,7 @@ import qualified Data.Text.Lazy.Encoding as LText import Data.These import qualified Data.X509 as X509 import Data.X509.Extended +import qualified Hasql.Pool.Extended as HasqlPoolExt import Imports hiding (MonadReader, asks, log) import qualified Network.HTTP.Types.Status as Http import qualified Network.Wai.Utilities.Error as Wai @@ -127,7 +128,8 @@ data Env = Env sparCtxHttpBrig :: Bilge.Request, sparCtxHttpGalley :: Bilge.Request, sparCtxRequestId :: RequestId, - sparCtxScimSubsystemConfig :: ScimSubsystemConfig + sparCtxScimSubsystemConfig :: ScimSubsystemConfig, + sparCtxHasqlPool :: HasqlPoolExt.Pool } -- | Get a user by UserRef, no matter what the team. diff --git a/services/spar/src/Spar/CanonicalInterpreter.hs b/services/spar/src/Spar/CanonicalInterpreter.hs index 72155ab880f..069f5c70066 100644 --- a/services/spar/src/Spar/CanonicalInterpreter.hs +++ b/services/spar/src/Spar/CanonicalInterpreter.hs @@ -26,6 +26,7 @@ where import qualified Cassandra as Cas import Control.Monad.Except hiding (mapError) +import qualified Hasql.Pool.Extended as HasqlPoolExt import Imports import Polysemy import Polysemy.Error @@ -120,6 +121,7 @@ type LowerLevelCanonicalEffs = Logger (TinyLog.Msg -> TinyLog.Msg), Input Opts, Input TinyLog.Logger, + Input HasqlPoolExt.Pool, Random, Now, Embed IO, @@ -132,6 +134,7 @@ runSparToIO ctx = . embedToFinal @IO . nowToIO . randomToIO + . runInputConst (sparCtxHasqlPool ctx) . runInputConst (sparCtxLogger ctx) . runInputConst (sparCtxOpts ctx) . loggerToTinyLog (sparCtxLogger ctx) diff --git a/services/spar/src/Spar/Options.hs b/services/spar/src/Spar/Options.hs index 887933a0a9f..73d763584aa 100644 --- a/services/spar/src/Spar/Options.hs +++ b/services/spar/src/Spar/Options.hs @@ -35,6 +35,7 @@ import qualified Data.Set as Set import Data.Time import Data.X509.Extended (Fingerprint, parseFingerprintHex) import qualified Data.Yaml as Yaml +import Hasql.Pool.Extended (PoolConfig) import Imports import Options.Applicative import SAML2.WebSSO @@ -45,6 +46,7 @@ import Util.Options import Wire.API.Routes.Version import Wire.API.User.Orphans () import Wire.API.User.Saml +import Wire.PostgresMigrationOpts (PostgresMigrationOpts) data Opts = Opts { saml :: !SAML.Config, @@ -70,7 +72,16 @@ data Opts = Opts -- disables the check. When set, every cert in 'edCertAuthnResponse' -- (on create/update and on AuthnResponse) must be listed or the request -- is rejected with 'SparIdPCertNotAllowed' (HTTP 403). - idpCertFingerprintAllowlist :: !(Maybe CertFingerprintAllowlist) + idpCertFingerprintAllowlist :: !(Maybe CertFingerprintAllowlist), + -- | Postgres connection parameters (libpq key/value map). + postgresql :: !(Map Text Text), + -- | Postgres connection-pool sizing. + postgresqlPool :: !PoolConfig, + -- | Per-store Cassandra/Postgres storage-location flags (shared type; + -- Spar-relevant stores are added by their migration tasks). + postgresMigration :: !PostgresMigrationOpts, + -- | Optional path to a Postgres password secrets file (prod only). + postgresqlPassword :: !(Maybe FilePathSecrets) } deriving (Show, Generic) diff --git a/services/spar/src/Spar/Run.hs b/services/spar/src/Spar/Run.hs index 36482a10abf..570c5821299 100644 --- a/services/spar/src/Spar/Run.hs +++ b/services/spar/src/Spar/Run.hs @@ -38,6 +38,7 @@ import Data.Id import Data.Metrics.Servant (servantPrometheusMiddleware) import Data.Proxy (Proxy (Proxy)) import Data.Text.Encoding +import Hasql.Pool.Extended (initPostgresPool, rawPool) import Imports import Network.URI import Network.Wai (Application) @@ -59,6 +60,7 @@ import Util.Options import qualified Web.Scim.Schema.Common as Scim import Wire.API.Routes.Version (expandVersionExp) import Wire.API.Routes.Version.Wai +import Wire.PostgresMigrations (runAllMigrations) import Wire.ScimSubsystem.Interpreter ---------------------------------------------------------------------- @@ -82,6 +84,7 @@ runServer sparCtxOpts = do sport :: Int = sparCtxOpts ^. to saml . SAML.cfgSPPort (wrappedApp, ctxOpts) <- mkApp sparCtxOpts let logger = sparCtxLogger ctxOpts + runAllMigrations ctxOpts.sparCtxHasqlPool.rawPool logger let settings = newSettings $ defaultServer shost (fromIntegral sport) logger WU.runSettingsWithShutdown settings wrappedApp Nothing @@ -90,6 +93,7 @@ mkApp sparCtxOpts = do let logLevel = saml sparCtxOpts ^. SAML.cfgLogLevel sparCtxLogger <- Log.mkLogger logLevel (logNetStrings sparCtxOpts) (logFormat sparCtxOpts) sparCtxCas <- initCassandra sparCtxOpts sparCtxLogger + sparCtxHasqlPool <- initPostgresPool (Opt.postgresqlPool sparCtxOpts) (Opt.postgresql sparCtxOpts) (Opt.postgresqlPassword sparCtxOpts) sparCtxHttpManager <- Bilge.newManager Bilge.defaultManagerSettings let sparCtxHttpBrig = Bilge.host (sparCtxOpts ^. to brig . to host . to encodeUtf8) diff --git a/services/spar/test-integration/Util/Core.hs b/services/spar/test-integration/Util/Core.hs index 4fe1e23e92f..991ad7975f6 100644 --- a/services/spar/test-integration/Util/Core.hs +++ b/services/spar/test-integration/Util/Core.hs @@ -277,6 +277,7 @@ mkEnv tstOpts opts = do sparCtxHttpGalley = galley empty sparCtxRequestId = RequestId "" sparCtxScimSubsystemConfig = error "mkEnv: implement sparCtxScimSubsystemConfig when needed" + sparCtxHasqlPool = error "mkEnv: sparCtxHasqlPool is not available in integration tests" pure $ TestEnv mgr diff --git a/services/spar/test/Test/Spar/Saml/IdPSpec.hs b/services/spar/test/Test/Spar/Saml/IdPSpec.hs index 87892f11f77..8ab26e743c6 100644 --- a/services/spar/test/Test/Spar/Saml/IdPSpec.hs +++ b/services/spar/test/Test/Spar/Saml/IdPSpec.hs @@ -25,6 +25,7 @@ import Data.Domain import Data.Id (TeamId, idToText, parseIdFromText) import qualified Data.List.NonEmpty as NonEmptyL import qualified Data.Map as Map +import Data.Misc (unsafeParseDuration) import Data.Range import qualified Data.Set as Set import qualified Data.Text.Lazy as TL @@ -34,6 +35,7 @@ import Data.Time (UTCTime (..), fromGregorian) import Data.X509 (SignedCertificate) import Data.X509.Extended (renderFingerprintHex) import qualified Data.X509.Extended as X509E +import Hasql.Pool.Extended (PoolConfig (..)) import Imports import Polysemy import qualified Polysemy.Error @@ -82,6 +84,7 @@ import Wire.IdPConfigStore import Wire.IdPConfigStore.Mem import Wire.IdPRawMetadataStore import Wire.IdPRawMetadataStore.Mem +import Wire.PostgresMigrationOpts (PostgresMigrationOpts (..), StorageLocation (..)) import Wire.Reporter (Reporter (..)) import Wire.SamlProtocolSettings (SamlProtocolSettings) import Wire.SamlProtocolSettings.Servant (sparRouteToServant) @@ -1033,7 +1036,19 @@ defaultTestOpts = disabledAPIVersions = mempty, scimBaseUri = [uri|http://localhost:8088/scim/v2|], enableIdPByEmailDiscovery = False, - idpCertFingerprintAllowlist = Nothing + idpCertFingerprintAllowlist = Nothing, + -- Postgres fields are placeholders; only the allowlist is read here. + postgresql = mempty, + postgresqlPool = PoolConfig 1 (unsafeParseDuration "10s") (unsafeParseDuration "10m"), + postgresMigration = + PostgresMigrationOpts + { conversation = CassandraStorage, + conversationCodes = CassandraStorage, + teamFeatures = CassandraStorage, + domainRegistration = CassandraStorage, + user = CassandraStorage + }, + postgresqlPassword = Nothing } galleyAccessMock :: Sem (GalleyAPIAccess ': r) a -> Sem r a