From a7cc72d7c9009c6b68844e832a58aedc2bb38bbb Mon Sep 17 00:00:00 2001 From: Francois Ferrand Date: Tue, 18 Aug 2026 11:39:31 +0200 Subject: [PATCH 1/8] Default each log level individually MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The log configuration only defaulted as a whole object: a section setting just one level left the other undefined, and werelogs was then configured with an undefined dump level. The per-extension log section turned that gap into a hard failure by requiring both levels, which leaves no way to set only one — and an override from the environment always produces such a partial section, since a variable names a single field. Each level now carries its own default, shared by the global and the per-extension schemas, so setting one leaves the other at its default instead of unset or rejected. Issue: BB-808 --- lib/config/configItems.joi.js | 30 +++++++++---------- .../IngestionConfigValidator.spec.js | 11 ++----- .../MongoProcessorConfigValidator.spec.js | 11 ++----- 3 files changed, 21 insertions(+), 31 deletions(-) diff --git a/lib/config/configItems.joi.js b/lib/config/configItems.joi.js index 63e229e44..56106bf28 100644 --- a/lib/config/configItems.joi.js +++ b/lib/config/configItems.joi.js @@ -40,21 +40,21 @@ const bootstrapListJoi = joi.array() return a.default === b.default; }); -const LOG_LEVELS = ['error', 'warn', 'info', 'debug', 'trace']; -const logJoi = - joi.object({ - logLevel: joi.alternatives().try(...LOG_LEVELS), - dumpLevel: joi.alternatives().try(...LOG_LEVELS), - }).default({ - logLevel: 'info', - dumpLevel: 'error', - }); - -const logJoiOptional = - joi.object({ - logLevel: joi.alternatives().try(...LOG_LEVELS).required(), - dumpLevel: joi.alternatives().try(...LOG_LEVELS).required(), - }).optional(); +const logLevelJoi = joi.alternatives() + .try('error', 'warn', 'info', 'debug', 'trace'); + +// the levels default individually, so that setting one of them, from the +// configuration or from the environment, leaves the other one alone +const logKeys = { + logLevel: logLevelJoi.default('info'), + dumpLevel: logLevelJoi.default('error'), +}; + +const logJoi = joi.object(logKeys).default(); + +// logJoi with no default : +// Callers fall back to the global log config when this one is not configured +const logJoiOptional = joi.object(logKeys).optional(); const adminCredsJoi = joi.object() .min(1) diff --git a/tests/unit/ingestion/IngestionConfigValidator.spec.js b/tests/unit/ingestion/IngestionConfigValidator.spec.js index ae2ef4f4b..bb034add3 100644 --- a/tests/unit/ingestion/IngestionConfigValidator.spec.js +++ b/tests/unit/ingestion/IngestionConfigValidator.spec.js @@ -28,14 +28,9 @@ describe('IngestionConfigValidator log override', () => { assert.strictEqual(validated.log, undefined); }); - it('should reject a partial log config with missing dumpLevel', () => { - let err; - try { - configValidator({}, { ...baseExtConfig, log: { logLevel: 'debug' } }); - } catch (e) { - err = e; - } - assert(err, 'expected configValidator to throw on partial log config'); + it('should default the level left out of a partial log config', () => { + const validated = configValidator({}, { ...baseExtConfig, log: { logLevel: 'debug' } }); + assert.deepStrictEqual(validated.log, { logLevel: 'debug', dumpLevel: 'error' }); }); }); diff --git a/tests/unit/mongoProcessor/MongoProcessorConfigValidator.spec.js b/tests/unit/mongoProcessor/MongoProcessorConfigValidator.spec.js index dca930934..3778d563c 100644 --- a/tests/unit/mongoProcessor/MongoProcessorConfigValidator.spec.js +++ b/tests/unit/mongoProcessor/MongoProcessorConfigValidator.spec.js @@ -23,13 +23,8 @@ describe('MongoProcessorConfigValidator log override', () => { assert.strictEqual(validated.log, undefined); }); - it('should reject a partial log config with missing dumpLevel', () => { - let err; - try { - configValidator({}, { ...baseConfig, log: { logLevel: 'warn' } }); - } catch (e) { - err = e; - } - assert(err, 'expected configValidator to throw on partial log config'); + it('should default the level left out of a partial log config', () => { + const validated = configValidator({}, { ...baseConfig, log: { logLevel: 'warn' } }); + assert.deepStrictEqual(validated.log, { logLevel: 'warn', dumpLevel: 'error' }); }); }); From 607af20dacd29eb2b948e121fef6c9b6dfd66a18 Mon Sep 17 00:00:00 2001 From: Francois Ferrand Date: Tue, 18 Aug 2026 11:30:22 +0200 Subject: [PATCH 2/8] Apply the env var configuration without rewriting the config file docker-entrypoint.sh applied every environment variable by rewriting conf/config.json in place with jq. That fails outright when the file is mounted read-only, as zenko does: setting any mapped variable crashed the container at startup. The mapping was also a hand-maintained shell table duplicated from the joi schema, and drifting from it. Overrides are now applied in memory before validation, and their names are derived from the schema itself, so a new configuration field becomes settable from the environment as soon as it is declared. The names that do not follow from the config path are annotated next to the field they name, and the few variables setting several fields at once are kept in an explicit table. Values go through the schema like any other configuration: an invalid one fails the startup instead of being silently ignored. Every variable the entrypoint used to apply keeps working, pinned by a test: Zenko and Federation set them, and Federation forwards arbitrary ones from the field. Two exceptions, both already already broken: * the `EXTENSIONS_LIFECYCLE_RULES_*` and `REDIS_LOCALCACHE_*` variables wrote config sections the schema no longer knows, so setting one made validation reject the whole file and the process fail to start - they are now simply ignored; * `MONGODB_HOSTS` no longer selects the "mongo" log source -not supported anymore- and only sets the replica set hosts of the shared MongoDB connection. Issue: BB-808 --- Dockerfile | 4 +- README.md | 1 + docker-entrypoint.sh | 325 --------- docs/configuration.md | 124 ++++ .../gc/GarbageCollectorConfigValidator.js | 8 +- .../ingestion/IngestionConfigValidator.js | 8 +- .../lifecycle/LifecycleConfigValidator.js | 13 +- .../MongoProcessorConfigValidator.js | 8 +- .../NotificationConfigValidator.js | 8 +- .../OplogPopulatorConfigValidator.js | 8 +- .../replication/ReplicationConfigValidator.js | 20 +- lib/Config.js | 24 +- lib/config.joi.js | 13 +- lib/config/configItems.joi.js | 6 +- lib/config/envOverrides.js | 306 +++++++++ lib/config/extensionConfigValidator.js | 21 + lib/config/fields.js | 59 ++ .../IngestionConfigValidator.spec.js | 10 + tests/unit/lib/config/envOverrides.spec.js | 625 ++++++++++++++++++ .../MongoProcessorConfigValidator.spec.js | 10 + 20 files changed, 1200 insertions(+), 401 deletions(-) delete mode 100755 docker-entrypoint.sh create mode 100644 docs/configuration.md create mode 100644 lib/config/envOverrides.js create mode 100644 lib/config/extensionConfigValidator.js create mode 100644 lib/config/fields.js create mode 100644 tests/unit/lib/config/envOverrides.spec.js diff --git a/Dockerfile b/Dockerfile index 051238bbe..7a9d33418 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,6 @@ RUN apt-get update \ bash \ python3 \ git \ - jq \ zlib1g-dev \ libncurses5-dev \ libgdbm-dev \ @@ -45,7 +44,6 @@ FROM node:${NODE_VERSION} RUN apt-get update && \ apt-get install -y --no-install-recommends \ ca-certificates \ - jq \ krb5-user \ libsasl2-2 \ libsasl2-modules-gssapi-mit \ @@ -62,6 +60,6 @@ COPY --from=builder /usr/local/bin/dockerize /usr/local/bin/ ENV AWS_SDK_JS_SUPPRESS_MAINTENANCE_MODE_MESSAGE=1 -ENTRYPOINT ["tini", "-g", "--", "/usr/src/app/docker-entrypoint.sh"] +ENTRYPOINT ["tini", "-g", "--"] EXPOSE 8900 diff --git a/README.md b/README.md index fb0b0a556..6f8a61b76 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,7 @@ updates in a FIFO order. ## DESIGN - [Backbeat core design](/DESIGN.md) +- [Configuration](/docs/configuration.md) - [CRR to AWS S3 workflow](/docs/crr-to-aws-s3.md) - [Object Lifecycle management](/docs/lifecycle.md) - [Metrics](/docs/metrics.md) diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh deleted file mode 100755 index 883146ff6..000000000 --- a/docker-entrypoint.sh +++ /dev/null @@ -1,325 +0,0 @@ -#!/bin/bash - -# set -e stops the execution of a script if a command or pipeline has an error -set -e - -# modifying config.json -JQ_FILTERS_CONFIG="." - -if [[ "$LIVENESS_PROBE_PORT" ]]; then - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .queuePopulator.probeServer.bindAddress=\"0.0.0.0\"" - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .queuePopulator.probeServer.port=\"$LIVENESS_PROBE_PORT\"" - - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .extensions.ingestion.probeServer.bindAddress=\"0.0.0.0\"" - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .extensions.ingestion.probeServer.port=\"$LIVENESS_PROBE_PORT\"" - - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .extensions.mongoProcessor.probeServer.bindAddress=\"0.0.0.0\"" - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .extensions.mongoProcessor.probeServer.port=\"$LIVENESS_PROBE_PORT\"" - - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .extensions.replication.queueProcessor.probeServer.bindAddress=\"0.0.0.0\"" - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .extensions.replication.queueProcessor.probeServer.port=\"$LIVENESS_PROBE_PORT\"" - - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .extensions.replication.replicationStatusProcessor.probeServer.bindAddress=\"0.0.0.0\"" - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .extensions.replication.replicationStatusProcessor.probeServer.port=\"$LIVENESS_PROBE_PORT\"" - - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .extensions.lifecycle.conductor.probeServer.bindAddress=\"0.0.0.0\"" - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .extensions.lifecycle.conductor.probeServer.port=\"$LIVENESS_PROBE_PORT\"" - - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .extensions.lifecycle.bucketProcessor.probeServer.bindAddress=\"0.0.0.0\"" - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .extensions.lifecycle.bucketProcessor.probeServer.port=\"$LIVENESS_PROBE_PORT\"" - - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .extensions.lifecycle.objectProcessor.probeServer.bindAddress=\"0.0.0.0\"" - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .extensions.lifecycle.objectProcessor.probeServer.port=\"$LIVENESS_PROBE_PORT\"" - - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .extensions.gc.probeServer.bindAddress=\"0.0.0.0\"" - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .extensions.gc.probeServer.port=\"$LIVENESS_PROBE_PORT\"" -fi - -if [[ "$LOG_LEVEL" ]]; then - if [[ "$LOG_LEVEL" == "info" || "$LOG_LEVEL" == "debug" || "$LOG_LEVEL" == "trace" ]]; then - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .log.logLevel=\"$LOG_LEVEL\"" - echo "Log level has been modified to $LOG_LEVEL" - else - echo "The log level you provided is incorrect (info/debug/trace)" - fi -fi - -if [[ "$ZOOKEEPER_AUTO_CREATE_NAMESPACE" ]]; then - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .zookeeper.autoCreateNamespace=true" -fi - -if [[ "$ZOOKEEPER_CONNECTION_STRING" ]]; then - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .zookeeper.connectionString=\"$ZOOKEEPER_CONNECTION_STRING\"" -fi - -if [[ "$KAFKA_HOSTS" ]]; then - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .kafka.hosts=\"$KAFKA_HOSTS\"" -fi - -if [[ "$KAFKA_BACKLOG_METRICS_ZKPATH" ]]; then - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .kafka.backlogMetrics.zkPath=\"$KAFKA_BACKLOG_METRICS_ZKPATH\"" -fi - -if [[ "$KAFKA_BACKLOG_METRICS_INTERVALS" ]]; then - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .kafka.backlogMetrics.intervalS=\"$KAFKA_BACKLOG_METRICS_INTERVALS\"" -fi - -if [ -z "$REDIS_HA_NAME" ]; then - REDIS_HA_NAME='mymaster' -fi - -if [[ "$REDIS_SENTINELS" ]]; then - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .redis.name=\"$REDIS_HA_NAME\"" - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .redis.sentinels=\"$REDIS_SENTINELS\"" -elif [[ "$REDIS_HOST" ]]; then - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .redis.host=\"$REDIS_HOST\"" - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .redis.port=6379" -fi - -if [[ "$REDIS_LOCALCACHE_HOST" ]]; then - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .localCache.host=\"$REDIS_LOCALCACHE_HOST\"" - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .localCache.port=6379" -fi - -if [[ "$REDIS_LOCALCACHE_PORT" ]]; then - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .localCache.port=$REDIS_LOCALCACHE_PORT" -fi - -if [[ "$REDIS_PORT" ]] && [[ -z "$REDIS_SENTINELS" ]]; then - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .redis.port=$REDIS_PORT" -fi - -if [[ "$QUEUE_POPULATOR_BATCH_MAX_READ" ]]; then - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .queuePopulator.batchMaxRead=\"$QUEUE_POPULATOR_BATCH_MAX_READ\"" -fi - -if [[ "$QUEUE_POPULATOR_DMD_HOST" ]]; then - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .queuePopulator.dmd.host=\"$QUEUE_POPULATOR_DMD_HOST\"" -fi - -if [[ "$QUEUE_POPULATOR_DMD_PORT" ]]; then - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .queuePopulator.dmd.port=\"$QUEUE_POPULATOR_DMD_PORT\"" -fi - -if [[ "$MONGODB_HOSTS" ]]; then - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .queuePopulator.logSource=\"mongo\"" - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .queuePopulator.mongo.replicaSetHosts=\"$MONGODB_HOSTS\"" -fi - -if [[ "$MONGODB_RS" ]]; then - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .queuePopulator.mongo.replicaSet=\"$MONGODB_RS\"" -fi - -if [[ "$MONGODB_DATABASE" ]]; then - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .queuePopulator.mongo.database=\"$MONGODB_DATABASE\"" -fi - -if [[ "$CLOUDSERVER_HOST" ]]; then - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .s3.host=\"$CLOUDSERVER_HOST\"" -fi - -if [[ "$CLOUDSERVER_PORT" ]]; then - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .s3.port=\"$CLOUDSERVER_PORT\"" -fi - -if [[ "$EXTENSIONS_REPLICATION_SOURCE_S3_HOST" ]]; then - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .extensions.replication.source.s3.host=\"$EXTENSIONS_REPLICATION_SOURCE_S3_HOST\"" -fi - -if [[ "$EXTENSIONS_REPLICATION_SOURCE_S3_PORT" ]]; then - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .extensions.replication.source.s3.port=\"$EXTENSIONS_REPLICATION_SOURCE_S3_PORT\"" -fi - -if [[ "$EXTENSIONS_REPLICATION_SOURCE_AUTH_TYPE" ]]; then - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .extensions.replication.source.auth.type=\"$EXTENSIONS_REPLICATION_SOURCE_AUTH_TYPE\"" -fi - -if [[ "$EXTENSIONS_REPLICATION_SOURCE_AUTH_ACCOUNT" ]]; then - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .extensions.replication.source.auth.account=\"$EXTENSIONS_REPLICATION_SOURCE_AUTH_ACCOUNT\"" -fi - -if [[ "$EXTENSIONS_REPLICATION_DEST_AUTH_TYPE" ]]; then - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .extensions.replication.destination.auth.type=\"$EXTENSIONS_REPLICATION_DEST_AUTH_TYPE\"" -fi - -if [[ "$EXTENSIONS_REPLICATION_DEST_AUTH_ACCOUNT" ]]; then - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .extensions.replication.destination.auth.account=\"$EXTENSIONS_REPLICATION_DEST_AUTH_ACCOUNT\"" -fi - -if [[ "$EXTENSIONS_REPLICATION_DEST_BOOTSTRAPLIST" ]]; then - if [[ "$EXTENSIONS_REPLICATION_DEST_BOOTSTRAPLIST_MORE" ]]; then - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .extensions.replication.destination.bootstrapList=[{\"site\": \"zenko\", \"servers\": [\"$EXTENSIONS_REPLICATION_DEST_BOOTSTRAPLIST\"]}, $EXTENSIONS_REPLICATION_DEST_BOOTSTRAPLIST_MORE]" - else - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .extensions.replication.destination.bootstrapList=[{\"site\": \"zenko\", \"servers\": [\"$EXTENSIONS_REPLICATION_DEST_BOOTSTRAPLIST\"]}]" - fi -fi - -# START Retry config - -# AWS_S3 -if [[ "$EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_AWS_S3_RETRY_TIMEOUT_S" ]]; then - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .extensions.replication.queueProcessor.retry.aws_s3.timeoutS=\"$EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_AWS_S3_RETRY_TIMEOUT_S\"" -fi - -if [[ "$EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_AWS_S3_RETRY_MAX_RETRIES" ]]; then - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .extensions.replication.queueProcessor.retry.aws_s3.maxRetries=\"$EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_AWS_S3_RETRY_MAX_RETRIES\"" -fi - -if [[ "$EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_AWS_S3_RETRY_BACKOFF_MIN" ]]; then - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .extensions.replication.queueProcessor.retry.aws_s3.backoff.min=\"$EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_AWS_S3_RETRY_BACKOFF_MIN\"" -fi - -if [[ "$EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_AWS_S3_RETRY_BACKOFF_MAX" ]]; then - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .extensions.replication.queueProcessor.retry.aws_s3.backoff.max=\"$EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_AWS_S3_RETRY_BACKOFF_MAX\"" -fi - -if [[ "$EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_AWS_S3_RETRY_BACKOFF_JITTER" ]]; then - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .extensions.replication.queueProcessor.retry.aws_s3.backoff.jitter=\"$EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_AWS_S3_RETRY_BACKOFF_JITTER\"" -fi - -if [[ "$EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_AWS_S3_RETRY_BACKOFF_FACTOR" ]]; then - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .extensions.replication.queueProcessor.retry.aws_s3.backoff.factor=\"$EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_AWS_S3_RETRY_BACKOFF_FACTOR\"" -fi - -# AZURE -if [[ "$EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_AZURE_RETRY_TIMEOUT_S" ]]; then - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .extensions.replication.queueProcessor.retry.azure.timeoutS=\"$EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_AZURE_RETRY_TIMEOUT_S\"" -fi - -if [[ "$EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_AZURE_RETRY_MAX_RETRIES" ]]; then - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .extensions.replication.queueProcessor.retry.azure.maxRetries=\"$EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_AZURE_RETRY_MAX_RETRIES\"" -fi - -if [[ "$EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_AZURE_RETRY_BACKOFF_MIN" ]]; then - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .extensions.replication.queueProcessor.retry.azure.backoff.min=\"$EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_AZURE_RETRY_BACKOFF_MIN\"" -fi - -if [[ "$EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_AZURE_RETRY_BACKOFF_MAX" ]]; then - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .extensions.replication.queueProcessor.retry.azure.backoff.max=\"$EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_AZURE_RETRY_BACKOFF_MAX\"" -fi - -if [[ "$EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_AZURE_RETRY_BACKOFF_JITTER" ]]; then - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .extensions.replication.queueProcessor.retry.azure.backoff.jitter=\"$EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_AZURE_RETRY_BACKOFF_JITTER\"" -fi - -if [[ "$EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_AZURE_RETRY_BACKOFF_FACTOR" ]]; then - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .extensions.replication.queueProcessor.retry.azure.backoff.factor=\"$EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_AZURE_RETRY_BACKOFF_FACTOR\"" -fi - -# GCP -if [[ "$EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_GCP_RETRY_TIMEOUT_S" ]]; then - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .extensions.replication.queueProcessor.retry.gcp.timeoutS=\"$EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_GCP_RETRY_TIMEOUT_S\"" -fi - -if [[ "$EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_GCP_RETRY_MAX_RETRIES" ]]; then - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .extensions.replication.queueProcessor.retry.gcp.maxRetries=\"$EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_GCP_RETRY_MAX_RETRIES\"" -fi - -if [[ "$EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_GCP_RETRY_BACKOFF_MIN" ]]; then - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .extensions.replication.queueProcessor.retry.gcp.backoff.min=\"$EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_GCP_RETRY_BACKOFF_MIN\"" -fi - -if [[ "$EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_GCP_RETRY_BACKOFF_MAX" ]]; then - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .extensions.replication.queueProcessor.retry.gcp.backoff.max=\"$EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_GCP_RETRY_BACKOFF_MAX\"" -fi - -if [[ "$EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_GCP_RETRY_BACKOFF_JITTER" ]]; then - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .extensions.replication.queueProcessor.retry.gcp.backoff.jitter=\"$EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_GCP_RETRY_BACKOFF_JITTER\"" -fi - -if [[ "$EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_GCP_RETRY_BACKOFF_FACTOR" ]]; then - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .extensions.replication.queueProcessor.retry.gcp.backoff.factor=\"$EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_GCP_RETRY_BACKOFF_FACTOR\"" -fi - -# END Retry Config - -if [[ "$EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_CONCURRENCY" ]]; then - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .extensions.replication.queueProcessor.concurrency=\"$EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_CONCURRENCY\"" -fi - -if [[ "$EXTENSIONS_REPLICATION_STATUS_PROCESSOR_CONCURRENCY" ]]; then - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .extensions.replication.replicationStatusProcessor.concurrency=\"$EXTENSIONS_REPLICATION_STATUS_PROCESSOR_CONCURRENCY\"" -fi - -if [[ "$EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_MAX_POLL_INTERVAL_MS" ]]; then - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .extensions.replication.queueProcessor.maxPollIntervalMs=\"$EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_MAX_POLL_INTERVAL_MS\"" -fi - -if [[ "$HEALTHCHECKS_ALLOWFROM" ]]; then - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .server.healthChecks.allowFrom=[\"$HEALTHCHECKS_ALLOWFROM\"]" -fi - -if [[ "$EXTENSIONS_LIFECYCLE_ZOOKEEPER_PATH" ]]; then - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .extensions.lifecycle.zookeeperPath=\"$EXTENSIONS_LIFECYCLE_ZOOKEEPER_PATH\"" -fi - -if [[ "$EXTENSIONS_LIFECYCLE_BUCKET_TASK_TOPIC" ]]; then - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .extensions.lifecycle.bucketTasksTopic=\"$EXTENSIONS_LIFECYCLE_BUCKET_TASK_TOPIC\"" -fi - -if [[ "$EXTENSIONS_LIFECYCLE_OBJECT_TASK_TOPIC" ]]; then - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .extensions.lifecycle.objectTasksTopic=\"$EXTENSIONS_LIFECYCLE_OBJECT_TASK_TOPIC\"" -fi - -if [[ "$EXTENSIONS_LIFECYCLE_CONDUCTOR_CRONRULE" ]]; then - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .extensions.lifecycle.conductor.cronRule=\"$EXTENSIONS_LIFECYCLE_CONDUCTOR_CRONRULE\"" -fi - -if [[ "$EXTENSIONS_LIFECYCLE_BUCKET_PROCESSOR_GROUP_ID" ]]; then - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .extensions.lifecycle.bucketProcessor.groupId=\"$EXTENSIONS_LIFECYCLE_BUCKET_PROCESSOR_GROUP_ID\"" -fi - -if [[ "$EXTENSIONS_LIFECYCLE_OBJECT_PROCESSOR_GROUP_ID" ]]; then - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .extensions.lifecycle.objectProcessor.groupId=\"$EXTENSIONS_LIFECYCLE_OBJECT_PROCESSOR_GROUP_ID\"" -fi - -if [[ "$EXTENSIONS_LIFECYCLE_RULES_EXPIRATION_ENABLED" ]]; then - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .extensions.lifecycle.rules.expiration.enabled=\"$EXTENSIONS_LIFECYCLE_RULES_EXPIRATION_ENABLED\"" -fi - -if [[ "$EXTENSIONS_LIFECYCLE_RULES_NC_VERSION_EXPIRATION_ENABLED" ]]; then - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .extensions.lifecycle.rules.noncurrentVersionExpiration.enabled=\"$EXTENSIONS_LIFECYCLE_RULES_NC_VERSION_EXPIRATION_ENABLED\"" -fi - -if [[ "$EXTENSIONS_LIFECYCLE_RULES_TRANSITIONS_ENABLED" ]]; then - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .extensions.lifecycle.rules.transitions.enabled=\"$EXTENSIONS_LIFECYCLE_RULES_TRANSITIONS_ENABLED\"" -fi - -if [[ "$EXTENSIONS_LIFECYCLE_RULES_ABORT_INCOMPLETE_MPU_ENABLED" ]]; then - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .extensions.lifecycle.rules.abortIncompleteMultipartUpload.enabled=\"$EXTENSIONS_LIFECYCLE_RULES_ABORT_INCOMPLETE_MPU_ENABLED\"" -fi - -if [[ "$EXTENSIONS_LIFECYCLE_AUTH_TYPE" ]]; then - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .extensions.lifecycle.auth.type=\"$EXTENSIONS_LIFECYCLE_AUTH_TYPE\"" -fi - -if [[ "$EXTENSIONS_LIFECYCLE_AUTH_ACCOUNT" ]]; then - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .extensions.lifecycle.auth.account=\"$EXTENSIONS_LIFECYCLE_AUTH_ACCOUNT\"" -fi - -if [[ "$EXTENSIONS_GC_TOPIC" ]]; then - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .extensions.gc.topic=\"$EXTENSIONS_GC_TOPIC\"" -fi - -if [[ "$EXTENSIONS_INGESTION_AUTH_TYPE" ]]; then - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .extensions.ingestion.auth.type=\"$EXTENSIONS_INGESTION_AUTH_TYPE\"" -fi - -if [[ "$EXTENSIONS_INGESTION_AUTH_ACCOUNT" ]]; then - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .extensions.ingestion.auth.account=\"$EXTENSIONS_INGESTION_AUTH_ACCOUNT\"" -fi - -if [[ "$EXTENSIONS_INGESTION_MAX_PARALLEL_READERS" ]]; then - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .extensions.ingestion.maxParallelReaders=\"$EXTENSIONS_INGESTION_MAX_PARALLEL_READERS\"" -fi - -if [[ "$REPLICATION_GROUP_ID" ]] ; then - JQ_FILTERS_CONFIG="$JQ_FILTERS_CONFIG | .replicationGroupId=\"$REPLICATION_GROUP_ID\"" -fi - -if [[ $JQ_FILTERS_CONFIG != "." ]]; then - jq "$JQ_FILTERS_CONFIG" conf/config.json > conf/config.json.tmp - mv conf/config.json.tmp conf/config.json -fi - -exec "$@" diff --git a/docs/configuration.md b/docs/configuration.md new file mode 100644 index 000000000..a6e2f7534 --- /dev/null +++ b/docs/configuration.md @@ -0,0 +1,124 @@ +# Configuration + +Backbeat reads its configuration from `conf/config.json`, or from the file named +by the `BACKBEAT_CONFIG_FILE` environment variable. + +Any configuration field can be overridden by an environment variable. Overrides +are applied in memory when the configuration is loaded: the file is never +modified, and can be mounted read-only. + +## Variable names + +The name of a variable is its configuration path, with each segment converted +from camelCase to SNAKE_CASE and joined with `_`: + +- `kafka.hosts` -> `KAFKA_HOSTS` +- `queuePopulator.batchMaxRead` -> `QUEUE_POPULATOR_BATCH_MAX_READ` +- `extensions.gc.topic` -> `EXTENSIONS_GC_TOPIC` +- `extensions.lifecycle.conductor.concurrency` -> + `EXTENSIONS_LIFECYCLE_CONDUCTOR_CONCURRENCY` + +Two annotations, set next to the field they name, tune these names. Given the +schema: + +```js +joi.object({ + foo: joi.object({ + bar: joi.string(), + baz: joi.object({ qux: joi.number() }), + }), +}) +``` + +`foo.bar` is set by `FOO_BAR`, and `foo.baz.qux` by `FOO_BAZ_QUX`. Annotating +`baz` changes the name of the fields it holds: + +- `.meta({ env: 'ZAB' })` renames the segment `baz` contributes, for itself and + its children: `foo.baz.qux` is set by `FOO_ZAB_QUX`, and `FOO_BAZ_QUX` is no + longer a name. +- `.meta({ envVarAlias: 'ZAB' })` adds an extra name to reach the field within + its schema: `foo.baz.qux` is set by `ZAB_QUX`, as well as by `FOO_BAZ_QUX`. + +This is how the historic names keep working, without a hand written mapping: +`s3` carries `env: 'CLOUDSERVER'`, so `s3.host` is set by +`CLOUDSERVER_HOST` (instead of `S3_HOST`); `queuePopulator.mongo` carries +`envVarAlias: 'MONGODB'`, so `queuePopulator.mongo.database` answers to +`MONGODB_DATABASE` as well as to `QUEUE_POPULATOR_MONGO_DATABASE`. In an +extension's schema, an alias keeps the `EXTENSIONS_` prefix, e.g. +`EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_AWS_S3_RETRY_TIMEOUT_S`. + +The fields of an object with unconstrained keys, such as `kafka.producerParams`, +have no derived name, and neither have the fields the schema forbids. + +## Values conversion + +A variable holds a string, which gets converted to the type of the field before +the configuration is validated. The schema has the last word: an invalid value +eventually fails the startup, when joi validates the schema, instead of being +silently ignored. + +- numbers and strings are passed through, and coerced by the schema +- booleans accept `true`/`false`, `1`/`0`, `yes`/`no`, `on`/`off` +- lists are comma separated, e.g. `HEALTHCHECKS_ALLOWFROM=10.0.0.0/8,::1` +- structured values are JSON documents, e.g. + `EXTENSIONS_NOTIFICATION_DESTINATIONS='[{ "resource": "d1", ... }]'` +- an empty value is ignored, and leaves the field as configured: a variable + cannot clear a field + +## Variables setting several fields + +- `LIVENESS_PROBE_PORT`: the port of every probe server configured, bound to + `0.0.0.0`. Per site probe servers are left alone. +- `MONGODB_HOSTS`: `queuePopulator.mongo.replicaSetHosts`. +- `REDIS_SENTINELS`, `REDIS_HA_NAME`: `redis.sentinels` and `redis.name` + (`mymaster` by default). They replace the standalone host and port. The + sentinels are a comma separated list of `host:port`, e.g. + `REDIS_SENTINELS=sentinel1:26379,sentinel2:26379`. +- `REDIS_HOST`, `REDIS_PORT`: standalone `redis.host` and `redis.port` (6379 by + default). Both are ignored when sentinels are configured. +- `EXTENSIONS_REPLICATION_DEST_BOOTSTRAPLIST`: the servers of the `zenko` site + of the replication bootstrap list, comma separated. + `EXTENSIONS_REPLICATION_DEST_BOOTSTRAPLIST_MORE` holds the additional sites, + as raw JSON objects, e.g. `{ "site": "aws", "type": "aws_s3" }`. + +## Other variables + +These are read by the code directly, wherever they are needed, rather than +setting a field of the configuration file. No schema declares them, so no name +is derived for them, and their value is not validated. + +- `BACKBEAT_CONFIG_FILE`: path of the configuration file. +- `BACKBEAT_QUEUEPOPULATOR_EXTENSIONS`: extensions run by this queue populator, + comma separated. +- `BOOTSTRAP_SITE_NAME`: restricts the replication bootstrap list to one site. +- `KAFKA_TOPIC_PREFIX`: prepended to every topic name, to share a cluster. +- `CONF_DIR`: directory holding the notification destination credential files. +- `TYPE`, `SSL`, `PROTOCOL`, `CA`, `CLIENT`, `KEY`, `KEY_PASSWORD`, `KEYTAB`, + `PRINCIPAL`, `SERVICE_NAME`, `BASIC_USERNAME`, `BASIC_PASSWORD`, + `SCRAM_MECHANISM`: auth of the destination a notification processor serves, + passed by the deployment rather than configured in the file. +- `S3AUTH_CONFIG`: path of the account credentials file, for the `account` auth + type. +- `MANAGEMENT_BACKEND`, `REMOTE_MANAGEMENT_DISABLE`: management backend of the + Zenko deployment, and whether to run it. +- `LIFECYCLE_OBJECT_PROCESSOR_TYPE`: the lifecycle object tasks this processor + consumes, `expiration` (the default) or `transition`. +- `LIFECYCLE_MAX_AUTO_INDEX_DOC_COUNT`, + `LIFECYCLE_MAX_AUTO_INDEX_STORAGE_BYTES`: limits above which lifecycle + indexes are not built automatically. +- `BATCH_TIMEOUT_SECONDS`: how long a queue populator batch may run before it + is reported as stuck, 300 by default. +- `CRASH_ON_BATCH_TIMEOUT`, `CRASH_ON_REBALANCE_TIMEOUT`: exit when a batch or a + consumer rebalance times out, rather than waiting for the liveness probe to + report it. Set on S3C, where supervisord only restarts a program on exit. +- `RDKAFKA_DEBUG_LOGS`: librdkafka debug contexts to enable, comma separated. + +The following are meant for testing only: + +- `TIME_PROGRESSION_FACTOR`: decreases the weight of a day, to expedite the + lifecycle of objects. +- `EXPIRE_ONE_DAY_EARLIER`, `TRANSITION_ONE_DAY_EARLIER`: deprecated in favor of + `TIME_PROGRESSION_FACTOR`. +- `BACKBEAT_ECHO_TEST_MODE`, `BACKBEAT_INJECT_REPLICATION_ERROR_RATE`, + `BACKBEAT_INJECT_REPLICATION_ERRORS`, `CI`: fault injection and test + fixtures. diff --git a/extensions/gc/GarbageCollectorConfigValidator.js b/extensions/gc/GarbageCollectorConfigValidator.js index 03ff99226..1de60fc53 100644 --- a/extensions/gc/GarbageCollectorConfigValidator.js +++ b/extensions/gc/GarbageCollectorConfigValidator.js @@ -5,6 +5,7 @@ const { probeServerJoi, hostPortJoi, } = require('../../lib/config/configItems.joi'); +const { extensionConfigValidator } = require('../../lib/config/extensionConfigValidator'); const joiSchema = joi.object({ topic: joi.string().required(), @@ -18,9 +19,4 @@ const joiSchema = joi.object({ vaultAdmin: hostPortJoi, }); -function configValidator(backbeatConfig, extConfig) { - const validatedConfig = joi.attempt(extConfig, joiSchema); - return validatedConfig; -} - -module.exports = configValidator; +module.exports = extensionConfigValidator('gc', joiSchema); diff --git a/extensions/ingestion/IngestionConfigValidator.js b/extensions/ingestion/IngestionConfigValidator.js index 16536ed8e..5860be670 100644 --- a/extensions/ingestion/IngestionConfigValidator.js +++ b/extensions/ingestion/IngestionConfigValidator.js @@ -1,6 +1,7 @@ const joi = require('joi'); const { KAFKA_PRODUCER_PARAMS_SCHEMA } = require('../../lib/config.joi'); const { probeServerJoi, logJoiOptional } = require('../../lib/config/configItems.joi'); +const { extensionConfigValidator } = require('../../lib/config/extensionConfigValidator'); const joiSchema = joi.object({ auth: joi.object({ @@ -22,9 +23,4 @@ const joiSchema = joi.object({ log: logJoiOptional, }); -function configValidator(backbeatConfig, extConfig) { - const validatedConfig = joi.attempt(extConfig, joiSchema); - return validatedConfig; -} - -module.exports = configValidator; +module.exports = extensionConfigValidator('ingestion', joiSchema); diff --git a/extensions/lifecycle/LifecycleConfigValidator.js b/extensions/lifecycle/LifecycleConfigValidator.js index b6487862e..96df288ec 100644 --- a/extensions/lifecycle/LifecycleConfigValidator.js +++ b/extensions/lifecycle/LifecycleConfigValidator.js @@ -7,14 +7,15 @@ const { probeServerJoi, retryParamsJoi, } = require('../../lib/config/configItems.joi'); +const { extensionConfigValidator } = require('../../lib/config/extensionConfigValidator'); const { backbeatConsumer: { MAX_QUEUED_DEFAULT } } = require('../../lib/constants'); const { ValidLifecycleRules: supportedLifecycleRules } = require('arsenal').models; const joiSchema = joi.object({ zookeeperPath: joi.string().required(), - bucketTasksTopic: joi.string().required(), - objectTasksTopic: joi.string().required(), + bucketTasksTopic: joi.string().required().meta({ env: 'BUCKET_TASK_TOPIC' }), + objectTasksTopic: joi.string().required().meta({ env: 'OBJECT_TASK_TOPIC' }), transitionTasksTopic: joi.string().default(parent => parent.objectTasksTopic), coldStorageTopics: joi.array().items(joi.string()).unique().default([]), auth: authJoi.optional(), @@ -28,7 +29,7 @@ const joiSchema = joi.object({ when('bucketSource', { is: 'bucketd', then: joi.required() }), mongodb: mongoJoi. when('bucketSource', { is: 'mongodb', then: joi.required() }), - cronRule: joi.string().required(), + cronRule: joi.string().required().meta({ env: 'CRONRULE' }), concurrency: joi.number().greater(0).default(10), concurrentIndexesBuildLimit: joi.number().greater(0).default(10), backlogControl: joi.object({ @@ -83,8 +84,4 @@ const joiSchema = joi.object({ ).default(supportedLifecycleRules), }); -function configValidator(backbeatConfig, extConfig) { - return joi.attempt(extConfig, joiSchema); -} - -module.exports = configValidator; +module.exports = extensionConfigValidator('lifecycle', joiSchema); diff --git a/extensions/mongoProcessor/MongoProcessorConfigValidator.js b/extensions/mongoProcessor/MongoProcessorConfigValidator.js index 8bb075f9c..b5ae49183 100644 --- a/extensions/mongoProcessor/MongoProcessorConfigValidator.js +++ b/extensions/mongoProcessor/MongoProcessorConfigValidator.js @@ -1,5 +1,6 @@ const joi = require('joi'); const { retryParamsJoi, probeServerJoi, logJoiOptional } = require('../../lib/config/configItems.joi'); +const { extensionConfigValidator } = require('../../lib/config/extensionConfigValidator'); const { MAX_QUEUED_DEFAULT } = require('../../lib/constants').backbeatConsumer; @@ -14,9 +15,4 @@ const joiSchema = joi.object({ log: logJoiOptional, }); -function configValidator(backbeatConfig, extConfig) { - const validatedConfig = joi.attempt(extConfig, joiSchema); - return validatedConfig; -} - -module.exports = configValidator; +module.exports = extensionConfigValidator('mongoProcessor', joiSchema); diff --git a/extensions/notification/NotificationConfigValidator.js b/extensions/notification/NotificationConfigValidator.js index f6854522e..af9e9238a 100644 --- a/extensions/notification/NotificationConfigValidator.js +++ b/extensions/notification/NotificationConfigValidator.js @@ -1,5 +1,6 @@ const joi = require('joi'); const { probeServerJoi } = require('../../lib/config/configItems.joi'); +const { extensionConfigValidator } = require('../../lib/config/extensionConfigValidator'); const { supportedSaslProtocols, supportedScramMechanisms } = require('./constants'); const { MAX_QUEUED_DEFAULT } = require('../../lib/constants').backbeatConsumer; @@ -100,13 +101,8 @@ const joiSchema = joi.object({ zookeeperOpConcurrency: joi.number().default(10), }); -function configValidator(backbeatConfig, extConfig) { - const validatedConfig = joi.attempt(extConfig, joiSchema); - return validatedConfig; -} - module.exports = { - notificationConfigValidator: configValidator, + notificationConfigValidator: extensionConfigValidator('notification', joiSchema), authSchema, credentialsFileSchema, }; diff --git a/extensions/oplogPopulator/OplogPopulatorConfigValidator.js b/extensions/oplogPopulator/OplogPopulatorConfigValidator.js index c65d5dbee..91d94c72d 100644 --- a/extensions/oplogPopulator/OplogPopulatorConfigValidator.js +++ b/extensions/oplogPopulator/OplogPopulatorConfigValidator.js @@ -1,5 +1,6 @@ const joi = require('joi'); const { probeServerJoi } = require('../../lib/config/configItems.joi'); +const { extensionConfigValidator } = require('../../lib/config/extensionConfigValidator'); const joiSchema = joi.object({ topic: joi.string().required(), @@ -13,12 +14,7 @@ const joiSchema = joi.object({ heartbeatIntervalMs: joi.number().default(10000), }); -function configValidator(backbeatConfig, extConfig) { - const validatedConfig = joi.attempt(extConfig, joiSchema); - return validatedConfig; -} - module.exports = { OplogPopulatorConfigJoiSchema: joiSchema, - OplogPopulatorConfigValidator: configValidator + OplogPopulatorConfigValidator: extensionConfigValidator('oplogPopulator', joiSchema) }; diff --git a/extensions/replication/ReplicationConfigValidator.js b/extensions/replication/ReplicationConfigValidator.js index 8f748c807..2a0c8023f 100644 --- a/extensions/replication/ReplicationConfigValidator.js +++ b/extensions/replication/ReplicationConfigValidator.js @@ -4,6 +4,7 @@ const { hostPortJoi, transportJoi, bootstrapListJoi, adminCredsJoi, retryParamsJoi, probeServerJoi, probeServerPerSite, stsConfigJoi } = require('../../lib/config/configItems.joi'); +const { extensionConfigValidator } = require('../../lib/config/extensionConfigValidator'); const { authTypeAccount, authTypeAssumeRole, @@ -13,10 +14,13 @@ const { const { MAX_QUEUED_DEFAULT } = require('../../lib/constants').backbeatConsumer; +// the historic env var names put the backend before `RETRY`, e.g. +// EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_AWS_S3_RETRY_BACKOFF_MIN const qpRetryJoi = joi.object({ - aws_s3: retryParamsJoi, // eslint-disable-line camelcase - azure: retryParamsJoi, - gcp: retryParamsJoi, + // eslint-disable-next-line camelcase + aws_s3: retryParamsJoi.meta({ envVarAlias: 'QUEUE_PROCESSOR_AWS_S3_RETRY' }), + azure: retryParamsJoi.meta({ envVarAlias: 'QUEUE_PROCESSOR_AZURE_RETRY' }), + gcp: retryParamsJoi.meta({ envVarAlias: 'QUEUE_PROCESSOR_GCP_RETRY' }), scality: retryParamsJoi, }); @@ -97,7 +101,7 @@ const joiSchema = joi.object({ otherwise: joi.required(), }), bootstrapList: bootstrapListJoi, - }).required().custom(_validatePerSiteDestinationConfig), + }).required().custom(_validatePerSiteDestinationConfig).meta({ env: 'DEST' }), topic: joi.string().required(), dataMoverTopic: joi.string().optional(), replicationStatusTopic: joi.string().required(), @@ -128,13 +132,13 @@ const joiSchema = joi.object({ circuitBreaker: joi.object().optional(), sourceCheckIfSizeGreaterThanMB: joi.number().positive().default(100), }).required(), - replicationStatusProcessor: { + replicationStatusProcessor: joi.object({ groupId: joi.string().required(), retry: retryParamsJoi, concurrency: joi.number().greater(0).default(10), maxQueued: joi.number().greater(0).default(MAX_QUEUED_DEFAULT), probeServer: probeServerJoi.default(), - }, + }).meta({ env: 'STATUS_PROCESSOR' }), replayProcessor: joi.object({ probeServer: probeServerPerSite, }).optional(), @@ -178,8 +182,10 @@ function _loadAdminCredentialsFromFile(filePath) { return { accessKey, secretKey }; } +const validateConfig = extensionConfigValidator('replication', joiSchema); + function configValidator(backbeatConfig, extConfig) { - const validatedConfig = joi.attempt(extConfig, joiSchema); + const validatedConfig = validateConfig(backbeatConfig, extConfig); const { source, destination } = validatedConfig; if (source.auth.vault) { const { adminCredentialsFile } = source.auth.vault; diff --git a/lib/Config.js b/lib/Config.js index 57a9aaa88..9e32394be 100644 --- a/lib/Config.js +++ b/lib/Config.js @@ -10,6 +10,7 @@ const crypto = require('crypto'); const extensions = require('../extensions'); const { backbeatConfigJoi } = require('./config.joi'); +const { applyCompositeEnvOverrides, applyEnvOverrides } = require('./config/envOverrides'); const locationTypeMatch = { 'location-mem-v1': 'mem', @@ -65,6 +66,9 @@ class Config extends EventEmitter { * @returns {undefined} */ _parseConfig(config) { + applyCompositeEnvOverrides(config); + applyEnvOverrides(config, backbeatConfigJoi); + const parsedConfig = joi.attempt(config, backbeatConfigJoi); if (parsedConfig.extensions) { @@ -143,26 +147,6 @@ class Config extends EventEmitter { parsedConfig.internalCertFilePaths); } - if (process.env.MONGODB_HOSTS) { - parsedConfig.queuePopulator.mongo.replicaSetHosts = - process.env.MONGODB_HOSTS; - } - if (process.env.MONGODB_RS) { - parsedConfig.queuePopulator.mongo.replicatSet = - process.env.MONGODB_RS; - } - if (process.env.MONGODB_DATABASE) { - parsedConfig.queuePopulator.mongo.database = - process.env.MONGODB_DATABASE; - } - if (process.env.MONGODB_AUTH_USERNAME && - process.env.MONGODB_AUTH_PASSWORD) { - parsedConfig.queuePopulator.mongo.authCredentials = { - username: process.env.MONGODB_AUTH_USERNAME, - password: process.env.MONGODB_AUTH_PASSWORD, - }; - } - // Overwrite extension configs if configured // We can specify the list of extensions that should be handled by this // instance of the queuePopulator diff --git a/lib/config.joi.js b/lib/config.joi.js index 5f988066d..8b60784d5 100644 --- a/lib/config.joi.js +++ b/lib/config.joi.js @@ -51,8 +51,10 @@ const joiSchema = joi.object({ kafka: { hosts: joi.string().required(), backlogMetrics: { - zkPath: joi.string().default('/backbeat/run/kafka-backlog-metrics'), - intervalS: joi.number().default(60), + zkPath: joi.string().default('/backbeat/run/kafka-backlog-metrics') + .meta({ env: 'ZKPATH' }), + intervalS: joi.number().default(60) + .meta({ env: 'INTERVALS' }), }, maxRequestSize: joi.number().default(KAFKA_PRODUCER_MESSAGE_MAX_BYTES), site: joi.string(), @@ -62,7 +64,7 @@ const joiSchema = joi.object({ consumerParams: KAFKA_CONSUMER_PARAMS_SCHEMA, }, transport: transportJoi, - s3: hostPortJoi.optional(), + s3: hostPortJoi.meta({ env: 'CLOUDSERVER' }).optional(), vaultAdmin: hostPortJoi, queuePopulator: { auth: authJoi, @@ -79,7 +81,7 @@ const joiSchema = joi.object({ dmd: hostPortJoi.keys({ logName: joi.string().default('s3-recordlog'), }).when('logSource', { is: 'dmd', then: joi.required() }), - mongo: mongoJoi, + mongo: mongoJoi.meta({ envVarAlias: 'MONGODB' }), kafka: qpKafkaJoi.when('logSource', { is: 'kafka', then: joi.required() }), // TODO: BB-625 reset to being required after supporting probeserver in S3C // for bucket notification proceses @@ -98,7 +100,8 @@ const joiSchema = joi.object({ }, server: { healthChecks: joi.object({ - allowFrom: joi.array().items(joi.string()).default([]), + allowFrom: joi.array().items(joi.string()).default([]) + .meta({ envVarAlias: 'HEALTHCHECKS_ALLOWFROM' }), }).required(), host: joi.string().required(), port: joi.number().default(8900), diff --git a/lib/config/configItems.joi.js b/lib/config/configItems.joi.js index 56106bf28..f5803ff62 100644 --- a/lib/config/configItems.joi.js +++ b/lib/config/configItems.joi.js @@ -46,7 +46,7 @@ const logLevelJoi = joi.alternatives() // the levels default individually, so that setting one of them, from the // configuration or from the environment, leaves the other one alone const logKeys = { - logLevel: logLevelJoi.default('info'), + logLevel: logLevelJoi.default('info').meta({ envName: 'LEVEL' }), dumpLevel: logLevelJoi.default('error'), }; @@ -140,13 +140,13 @@ const mongoJoi = joi.object({ then: joi.string().default('rs0'), otherwise: joi.forbidden(), }, - ), + ).meta({ env: 'RS' }), readPreference: joi.string().default('primary'), database: joi.string().default('metadata'), authCredentials: joi.object({ username: joi.string().required(), password: joi.string().required(), - }), + }).meta({ env: 'AUTH' }), }); const kafkaJoi = joi.object({ diff --git a/lib/config/envOverrides.js b/lib/config/envOverrides.js new file mode 100644 index 000000000..6ea31eba3 --- /dev/null +++ b/lib/config/envOverrides.js @@ -0,0 +1,306 @@ +'use strict'; +/** + * Configuration overrides from environment variables, with the names of the + * variables derived from the joi schemas. + */ + +const LIVENESS_PROBE_PORT = 'LIVENESS_PROBE_PORT'; + +/** + * Converts a config key to its env var fragment: `batchMaxRead` to + * `BATCH_MAX_READ`, `minMPUSizeMB` to `MIN_MPU_SIZE_MB`, `aws_s3` to `AWS_S3`. + * + * @param {string} key - config key + * @returns {string} env var fragment + */ +function toSnakeCase(key) { + return key + .replace(/([a-z0-9])([A-Z])/g, '$1_$2') + .replace(/([A-Z]+)([A-Z][a-z])/g, '$1_$2') + .toUpperCase(); +} + +/** + * @param {string[]} path - config path + * @returns {string} env var name derived from the path + */ +function envVarName(path) { + return path.map(toSnakeCase).join('_'); +} + +/** + * @param {string} value - JSON document + * @param {string} name - env var it comes from, for error reporting + * @returns {*} parsed value + */ +function parseJSON(value, name) { + try { + return JSON.parse(value); + } catch (err) { + throw new Error(`invalid JSON value for ${name}: ${err.message}`); + } +} + +/** + * Env vars setting several fields at once, or fields the schema cannot name. + * They are applied before the derived ones, which take precedence for the + * fields they set. + */ +const compositeEnvVars = { + /** Standalone redis: the port has an implicit default, and sentinels win. */ + REDIS_HOST: (config, host, env) => { + if (env.REDIS_SENTINELS) { + return; + } + setField(config, ['redis', 'host'], host); + setField(config, ['redis', 'port'], env.REDIS_PORT || '6379'); + }, + REDIS_PORT: (config, port, env) => { + if (env.REDIS_SENTINELS) { + return; + } + setField(config, ['redis', 'port'], port); + }, + /** + * Sentinels come with their own group name (REDIS_HA_NAME), and replace the + * standalone host and port, which the schema then forbids. + */ + REDIS_SENTINELS: (config, sentinels, env) => { + setField(config, ['redis', 'sentinels'], sentinels); + setField(config, ['redis', 'name'], env.REDIS_HA_NAME || 'mymaster'); + deleteField(config, ['redis', 'host']); + deleteField(config, ['redis', 'port']); + }, + /** + * The historic name of `queuePopulator.mongo.replicaSetHosts`. + */ + MONGODB_HOSTS: (config, hosts) => { + setField(config, ['queuePopulator', 'mongo', 'replicaSetHosts'], hosts); + }, + /** + * Single-site form of the replication bootstrap list: the servers of the + * `zenko` site. The additional sites are cloud backends, which only a + * structured value can describe. + */ + EXTENSIONS_REPLICATION_DEST_BOOTSTRAPLIST: (config, servers, env) => { + if (!config.extensions || !config.extensions.replication) { + return; + } + const bootstrapList = [{ + site: 'zenko', + servers: servers.split(',').map(server => server.trim()), + }]; + const more = env.EXTENSIONS_REPLICATION_DEST_BOOTSTRAPLIST_MORE; + if (more) { + bootstrapList.push(...parseJSON(`[${more}]`, + 'EXTENSIONS_REPLICATION_DEST_BOOTSTRAPLIST_MORE')); + } + setField(config, ['extensions', 'replication', 'destination', 'bootstrapList'], bootstrapList); + }, +}; + +// types joi does not coerce from a string, and which a JSON value can express: +// `null` is an ambiguous type, e.g. a field accepting a string or an array +const JSON_TYPES = ['array', 'object', null]; + +/** + * Values are injected as raw strings and coerced by joi, except for the types a + * string cannot express: booleans accept the usual shell spellings, arrays a + * comma separated list, and structured values a JSON document. + * + * @param {string} value - raw env var value + * @param {string} [type] - joi type of the field, null when ambiguous + * @param {string} name - env var name, for error reporting + * @returns {*} value to inject in the configuration + */ +function coerceValue(value, type, name) { + if (type === 'boolean') { + const spelling = value.toLowerCase(); + if (['1', 'y', 'yes', 'on', 'true'].includes(spelling)) { + return true; + } + if (['0', 'n', 'no', 'off', 'false'].includes(spelling)) { + return false; + } + return value; + } + if (JSON_TYPES.includes(type) && /^\s*[[{]/.test(value)) { + return parseJSON(value, name); + } + if (type === 'array') { + return value.split(',').map(item => item.trim()); + } + return value; +} + +/** + * @param {string[]} names - env var names of the field + * @param {string[]} path - config path, relative to the schema root + * @param {string} type - joi type of the field + * @param {Map} mappings - env var name to { path, type }, updated in place + * @returns {undefined} + */ +function registerMapping(names, path, type, mappings) { + names.forEach(name => { + if (compositeEnvVars[name]) { + // Composite variable are handled separately, as they sets several fields, + // or a field the schema cannot name + return; + } + const existing = mappings.get(name); + if (!existing) { + mappings.set(name, { path, type }); + } else if (existing.path.join('.') !== path.join('.')) { + throw new Error(`env var ${name} maps to both ${existing.path.join('.')} ` + + `and ${path.join('.')}`); + } else if (existing.type !== type) { + // ambiguous type: inject the raw value and let joi coerce it + existing.type = null; + } + }); +} + +/** + * Env var names of a field, from the names of its parent: the `env` annotation + * renames the segment it contributes, and `envVarAlias` adds a name replacing + * the path within the schema. + * + * @param {Object} description - joi description of the node + * @param {string[]} parentNames - env var names of the parent node + * @param {string} root - env var prefix of the schema root + * @param {string} key - config key of the node + * @returns {string[]} env var names of the node + */ +function nodeNames(description, parentNames, root, key) { + const meta = Object.assign({}, ...(description.metas || [])); + const segment = meta.env === undefined ? toSnakeCase(key) : meta.env; + const names = parentNames.map(prefix => (prefix ? `${prefix}_${segment}` : segment)); + if (meta.envVarAlias) { + names.push(root ? `${root}_${meta.envVarAlias}` : meta.envVarAlias); + } + return names; +} + +/** + * Walks a joi schema description, mapping the env var names of each field to + * its config path. + * + * @param {Object} description - joi description of the node + * @param {string[]} names - env var names of the node + * @param {string[]} path - config path of the node, relative to the schema root + * @param {string} root - env var prefix of the schema root + * @param {Map} mappings - env var name to { path, type }, updated in place + * @returns {undefined} + */ +function collectMappings(description, names, path, root, mappings) { + if (description.flags?.presence === 'forbidden') { + // a field the schema rejects has no env var + return; + } + switch (description.type) { + case 'object': + Object.entries(description.keys || {}).forEach(([key, child]) => + collectMappings(child, nodeNames(child, names, root, key), + [...path, key], root, mappings)); + break; + case 'alternatives': + (description.matches || []).forEach(match => + [match.schema, match.then, match.otherwise] + .filter(alternative => alternative) + .forEach(alternative => + collectMappings(alternative, names, path, root, mappings))); + break; + default: + // arrays are leaves: their items have no name to derive from + registerMapping(names, path, description.type, mappings); + } +} + +/** + * Maps the env var names a schema supports to the config path they set. + * + * @param {joi.Schema} schema - configuration schema + * @param {string[]} [prefix] - config path of the schema root + * @returns {Map} env var name to { path, type } + */ +function envVarMappings(schema, prefix = []) { + const mappings = new Map(); + const root = envVarName(prefix); + collectMappings(schema.describe(), [root], [], root, mappings); + return mappings; +} + +/** + * Sets the port of every probe server of the schema, bound to all interfaces. + * Sections missing from the configuration are left alone, so that the port is + * not set on an otherwise unconfigured processor. + * + * @param {Object} config - configuration to update + * @param {Map} mappings - env var name to { path, type } + * @param {string} port - liveness probe port + * @returns {undefined} + */ +function applyLivenessProbePort(config, mappings, port) { + mappings.forEach(({ path }) => { + if (path[path.length - 1] !== 'port' || path[path.length - 2] !== 'probeServer') { + return; + } + const parent = path.slice(0, -2); + const section = getField(config, parent); + // per-site probe servers each have their own port + if (!section || typeof section !== 'object' || Array.isArray(section.probeServer)) { + return; + } + setField(config, [...parent, 'probeServer'], { bindAddress: '0.0.0.0', port }); + }); +} + +/** + * Applies the env vars mapped to the fields of a schema, in place. + * + * @param {Object} config - configuration to update, matching the schema + * @param {joi.Schema} schema - configuration schema + * @param {string[]} [prefix] - config path of the schema root + * @param {Object} [env] - environment to read the overrides from + * @returns {Object} updated configuration. Invalid config returned untouched for joi to report + */ +function applyEnvOverrides(config, schema, prefix = [], env = process.env) { + if (config === null || typeof config !== 'object') { + return config; + } + const mappings = envVarMappings(schema, prefix); + + if (env[LIVENESS_PROBE_PORT]) { + applyLivenessProbePort(config, mappings, env[LIVENESS_PROBE_PORT]); + } + + mappings.forEach(({ path, type }, name) => { + if (env[name]) { + setField(config, path, coerceValue(env[name], type, name)); + } + }); + + return config; +} + +/** + * Applies the env vars setting several config fields at once, in place. + * + * @param {Object} config - backbeat configuration to update + * @param {Object} [env] - environment to read the overrides from + * @returns {Object} updated configuration + */ +function applyCompositeEnvOverrides(config, env = process.env) { + Object.entries(compositeEnvVars).forEach(([name, apply]) => { + if (env[name]) { + apply(config, env[name], env); + } + }); + return config; +} + +module.exports = { + applyCompositeEnvOverrides, + applyEnvOverrides, + envVarMappings, +}; diff --git a/lib/config/extensionConfigValidator.js b/lib/config/extensionConfigValidator.js new file mode 100644 index 000000000..841ca1179 --- /dev/null +++ b/lib/config/extensionConfigValidator.js @@ -0,0 +1,21 @@ +'use strict'; + +const joi = require('joi'); + +const { applyEnvOverrides } = require('./envOverrides'); + +/** + * Builds an extension config validator applying the env var overrides derived + * from the extension schema (e.g. EXTENSIONS_GC_TOPIC for the `topic` field of + * the gc extension) before validating. + * + * @param {string} extName - extension name, as configured in `extensions` + * @param {joi.Schema} schema - extension configuration schema + * @returns {function} extension config validator + */ +function extensionConfigValidator(extName, schema) { + return (backbeatConfig, extConfig) => + joi.attempt(applyEnvOverrides(extConfig, schema, ['extensions', extName]), schema); +} + +module.exports = { extensionConfigValidator }; diff --git a/lib/config/fields.js b/lib/config/fields.js new file mode 100644 index 000000000..4a73bc9bc --- /dev/null +++ b/lib/config/fields.js @@ -0,0 +1,59 @@ +'use strict'; + +/** + * Reading and writing a configuration field from its path, shared by the + * override mechanisms applied to the configuration before validation. + */ + +/** + * @param {Object} config - configuration to read from + * @param {string[]} path - config path + * @returns {*} value of the field, undefined if a node of the path is missing + */ +function getField(config, path) { + return path.reduce((node, key) => node?.[key], config); +} + +/** + * Sets a config value, creating the missing intermediate nodes. A node holding + * anything other than an object is reported rather than replaced. + * + * @param {Object} config - configuration to update + * @param {string[]} path - config path + * @param {*} value - value to set + * @returns {undefined} + */ +function setField(config, path, value) { + const parent = path.slice(0, -1).reduce((node, key, index) => { + if (!node[key]) { + node[key] = {}; // eslint-disable-line no-param-reassign + } else if (typeof node[key] !== 'object' || Array.isArray(node[key])) { + throw new Error(`cannot set ${path.join('.')}: ` + + `${path.slice(0, index + 1).join('.')} is not an object`); + } + return node[key]; + }, config); + parent[path[path.length - 1]] = value; +} + +/** + * Removes a config field, leaving alone a path holding no section to remove it + * from. The field is deleted rather than set to `undefined`, which the schema + * accepts alike but carries the key over into the validated configuration. + * + * @param {Object} config - configuration to update + * @param {string[]} path - config path + * @returns {undefined} + */ +function deleteField(config, path) { + const parent = getField(config, path.slice(0, -1)); + if (parent && typeof parent === 'object') { + delete parent[path[path.length - 1]]; + } +} + +module.exports = { + deleteField, + getField, + setField, +}; diff --git a/tests/unit/ingestion/IngestionConfigValidator.spec.js b/tests/unit/ingestion/IngestionConfigValidator.spec.js index bb034add3..66cfb283e 100644 --- a/tests/unit/ingestion/IngestionConfigValidator.spec.js +++ b/tests/unit/ingestion/IngestionConfigValidator.spec.js @@ -32,6 +32,16 @@ describe('IngestionConfigValidator log override', () => { const validated = configValidator({}, { ...baseExtConfig, log: { logLevel: 'debug' } }); assert.deepStrictEqual(validated.log, { logLevel: 'debug', dumpLevel: 'error' }); }); + + it('should accept the log level from the environment on its own', () => { + process.env.EXTENSIONS_INGESTION_LOG_LEVEL = 'debug'; + try { + const validated = configValidator({}, { ...baseExtConfig }); + assert.deepStrictEqual(validated.log, { logLevel: 'debug', dumpLevel: 'error' }); + } finally { + delete process.env.EXTENSIONS_INGESTION_LOG_LEVEL; + } + }); }); describe('IngestionConfigValidator batchMaxRead fallback', () => { diff --git a/tests/unit/lib/config/envOverrides.spec.js b/tests/unit/lib/config/envOverrides.spec.js new file mode 100644 index 000000000..4ab075447 --- /dev/null +++ b/tests/unit/lib/config/envOverrides.spec.js @@ -0,0 +1,625 @@ +'use strict'; + +const assert = require('assert'); +const joi = require('joi'); + +const { + applyCompositeEnvOverrides, + applyEnvOverrides, + envVarMappings, +} = require('../../../../lib/config/envOverrides'); +const { backbeatConfigJoi } = require('../../../../lib/config.joi'); +const { + logJoi, + logJoiOptional, + probeServerJoi, + probeServerPerSite, +} = require('../../../../lib/config/configItems.joi'); +const { Config } = require('../../../../lib/Config'); +const { getField } = require('../../../../lib/config/fields'); + +describe('config env var mapping', () => { + it('should derive names from the config path', () => { + const schema = joi.object({ + hosts: joi.string(), + queuePopulator: joi.object({ + batchMaxRead: joi.number(), + mongo: joi.object({ replicaSetHosts: joi.string() }), + }), + queueProcessor: joi.object({ + minMPUSizeMB: joi.number(), + retry: joi.object({ aws_s3: joi.object({ timeoutS: joi.number() }) }), // eslint-disable-line camelcase + }), + }); + + assert.deepStrictEqual([...envVarMappings(schema).keys()], [ + 'HOSTS', + 'QUEUE_POPULATOR_BATCH_MAX_READ', + 'QUEUE_POPULATOR_MONGO_REPLICA_SET_HOSTS', + 'QUEUE_PROCESSOR_MIN_MPU_SIZE_MB', + 'QUEUE_PROCESSOR_RETRY_AWS_S3_TIMEOUT_S', + ]); + }); + + it('should prefix the names of an extension schema', () => { + const schema = joi.object({ topic: joi.string(), consumer: joi.object({ groupId: joi.string() }) }); + + assert.deepStrictEqual([...envVarMappings(schema, ['extensions', 'gc']).keys()], + ['EXTENSIONS_GC_TOPIC', 'EXTENSIONS_GC_CONSUMER_GROUP_ID']); + }); + + it('should not derive a name for the fields of an object with unconstrained keys', () => { + const schema = joi.object({ + producerParams: joi.object().unknown(true), + sites: joi.object().pattern(joi.string(), joi.object({ port: joi.number() })), + }); + + assert.deepStrictEqual([...envVarMappings(schema).keys()], []); + }); + + it('should not derive a name for a field the schema forbids', () => { + const schema = joi.object({ + kafka: joi.object({ hosts: joi.forbidden(), site: joi.string() }), + }); + + assert.deepStrictEqual([...envVarMappings(schema).keys()], ['KAFKA_SITE']); + }); + + it('should not derive a name for the params BackbeatProducer sets itself', () => { + const names = [...envVarMappings(backbeatConfigJoi).keys()]; + + assert.ok(!names.some(name => name.startsWith('KAFKA_PRODUCER_PARAMS')), names.join(', ')); + }); + + it('should reject a schema deriving the same name for two fields', () => { + // eslint-disable-next-line camelcase + const schema = joi.object({ logLevel: joi.string(), log_level: joi.string() }); + + assert.throws(() => envVarMappings(schema), /LOG_LEVEL maps to both logLevel and log_level/); + }); + + describe('name annotations', () => { + it('should rename the segment a node contributes, for itself and its children', () => { + const schema = joi.object({ + destination: joi.object({ + transport: joi.string(), + auth: joi.object({ type: joi.string() }), + }).meta({ env: 'DEST' }), + }); + + assert.deepStrictEqual([...envVarMappings(schema, ['extensions', 'replication']).keys()], + ['EXTENSIONS_REPLICATION_DEST_TRANSPORT', + 'EXTENSIONS_REPLICATION_DEST_AUTH_TYPE']); + }); + + it('should add a name replacing the path within the schema', () => { + const schema = joi.object({ + queuePopulator: joi.object({ + mongo: joi.object({ database: joi.string() }).meta({ envVarAlias: 'MONGODB' }), + }), + }); + + assert.deepStrictEqual([...envVarMappings(schema).keys()], + ['QUEUE_POPULATOR_MONGO_DATABASE', 'MONGODB_DATABASE']); + }); + + it('should keep the extension prefix of an alias', () => { + const schema = joi.object({ + bucketTasksTopic: joi.string().meta({ envVarAlias: 'BUCKET_TASK_TOPIC' }), + }); + + assert.deepStrictEqual([...envVarMappings(schema, ['extensions', 'lifecycle']).keys()], + ['EXTENSIONS_LIFECYCLE_BUCKET_TASKS_TOPIC', + 'EXTENSIONS_LIFECYCLE_BUCKET_TASK_TOPIC']); + }); + + it('should only annotate the use site of a shared schema', () => { + const shared = joi.object({ host: joi.string() }); + const schema = joi.object({ + s3: shared.meta({ env: 'CLOUDSERVER' }), + vaultAdmin: shared, + }); + + assert.deepStrictEqual([...envVarMappings(schema).keys()], + ['CLOUDSERVER_HOST', 'VAULT_ADMIN_HOST']); + }); + + it('should apply the annotated names of the backbeat schema', () => { + // renamed by env: the derived LOG_LOG_LEVEL is replaced + assert.strictEqual( + applyEnvOverrides({}, backbeatConfigJoi, [], { LOG_LEVEL: 'warn' }).log.logLevel, + 'warn'); + assert.deepStrictEqual( + applyEnvOverrides({}, backbeatConfigJoi, [], { LOG_LOG_LEVEL: 'warn' }), {}); + + // aliased: both names are honored + ['HEALTHCHECKS_ALLOWFROM', 'SERVER_HEALTH_CHECKS_ALLOW_FROM'].forEach(name => { + const config = applyEnvOverrides({}, backbeatConfigJoi, [], { [name]: '::1' }); + assert.deepStrictEqual(config.server.healthChecks.allowFrom, ['::1'], name); + }); + }); + }); + + describe('value injection', () => { + const schema = joi.object({ + host: joi.string(), + port: joi.number(), + enabled: joi.boolean(), + allowFrom: joi.array().items(joi.string()), + topics: joi.array().items(joi.object({ name: joi.string() })), + sentinels: joi.alternatives([joi.string(), joi.array()]), + }); + const apply = env => applyEnvOverrides({}, schema, [], env); + + it('should leave strings and numbers for joi to coerce', () => { + assert.deepStrictEqual(apply({ HOST: 'h', PORT: '8000' }), { host: 'h', port: '8000' }); + assert.strictEqual(joi.attempt(apply({ PORT: '8000' }), schema).port, 8000); + }); + + it('should accept the usual boolean spellings', () => { + ['true', 'TRUE', '1', 'y', 'yes', 'on'].forEach(value => + assert.strictEqual(apply({ ENABLED: value }).enabled, true, value)); + ['false', 'FALSE', '0', 'n', 'no', 'off'].forEach(value => + assert.strictEqual(apply({ ENABLED: value }).enabled, false, value)); + }); + + it('should leave an unknown boolean spelling for joi to reject', () => { + assert.throws(() => joi.attempt(apply({ ENABLED: 'maybe' }), schema)); + }); + + it('should split a comma separated list into an array', () => { + assert.deepStrictEqual(apply({ ALLOW_FROM: '10.0.0.0/8, ::1' }).allowFrom, + ['10.0.0.0/8', '::1']); + assert.deepStrictEqual(apply({ ALLOW_FROM: '::1' }).allowFrom, ['::1']); + }); + + it('should parse a structured value from JSON', () => { + assert.deepStrictEqual(apply({ TOPICS: '[{ "name": "t1" }]' }).topics, [{ name: 't1' }]); + assert.deepStrictEqual(apply({ SENTINELS: '[{ "host": "h", "port": 26379 }]' }).sentinels, + [{ host: 'h', port: 26379 }]); + }); + + it('should report an invalid JSON value', () => { + assert.throws(() => apply({ TOPICS: '[{ "name" }]' }), + /invalid JSON value for TOPICS/); + }); + + it('should not coerce a value of ambiguous type', () => { + assert.strictEqual(apply({ SENTINELS: 'host1:26379,host2:26379' }).sentinels, + 'host1:26379,host2:26379'); + }); + + it('should ignore an empty value', () => { + assert.deepStrictEqual(apply({ HOST: '' }), {}); + }); + + it('should create the missing intermediate nodes', () => { + const nested = joi.object({ mongo: joi.object({ auth: joi.object({ user: joi.string() }) }) }); + assert.deepStrictEqual(applyEnvOverrides({}, nested, [], { MONGO_AUTH_USER: 'u' }), + { mongo: { auth: { user: 'u' } } }); + }); + + it('should report a node holding something else than a section', () => { + const nested = joi.object({ mongo: joi.object({ auth: joi.object({ user: joi.string() }) }) }); + assert.throws( + () => applyEnvOverrides({ mongo: { auth: 'secret' } }, nested, [], { MONGO_AUTH_USER: 'u' }), + /cannot set mongo.auth.user: mongo.auth is not an object/); + }); + + // one port cannot name the probe server of a specific site + it('should not replace the per site probe servers with a single one', () => { + const perSite = joi.object({ + queueProcessor: joi.object({ + probeServer: joi.alternatives().try(probeServerJoi, probeServerPerSite), + }), + }); + const config = { queueProcessor: { probeServer: [{ port: 4043, site: 'a' }] } }; + + assert.throws( + () => applyEnvOverrides(config, perSite, [], { QUEUE_PROCESSOR_PROBE_SERVER_PORT: '8100' }), + /queueProcessor.probeServer is not an object/); + assert.deepStrictEqual(config.queueProcessor.probeServer, [{ port: 4043, site: 'a' }]); + }); + + // a partially set object is what setting a single field of one yields + it('should leave the schema to complete a partially set object', () => { + const config = applyEnvOverrides({}, backbeatConfigJoi, [], { LOG_LEVEL: 'warn' }); + assert.deepStrictEqual(joi.attempt(config.log, logJoi), + { logLevel: 'warn', dumpLevel: 'error' }); + assert.deepStrictEqual(joi.attempt({ dumpLevel: 'warn' }, logJoiOptional), + { logLevel: 'info', dumpLevel: 'warn' }); + }); + }); + + describe('liveness probe port', () => { + const schema = joi.object({ + queuePopulator: joi.object({ probeServer: probeServerJoi }), + processor: joi.object({ probeServer: probeServerJoi }), + queueProcessor: joi.object({ + probeServer: joi.alternatives().try(probeServerJoi, probeServerPerSite), + }), + }); + const apply = config => + applyEnvOverrides(config, schema, [], { LIVENESS_PROBE_PORT: '8100' }); + + it('should listen on all interfaces, for every probe server of the process', () => { + const config = apply({ queuePopulator: { probeServer: { port: 4042 } }, processor: {} }); + assert.deepStrictEqual(config, { + queuePopulator: { probeServer: { bindAddress: '0.0.0.0', port: '8100' } }, + processor: { probeServer: { bindAddress: '0.0.0.0', port: '8100' } }, + }); + }); + + it('should leave the sections missing from the config alone', () => { + assert.deepStrictEqual(apply({}), {}); + }); + + it('should leave per site probe servers alone', () => { + const perSite = { queueProcessor: { probeServer: [{ port: 4043, site: 'a' }] } }; + assert.deepStrictEqual(apply(perSite), + { queueProcessor: { probeServer: [{ port: 4043, site: 'a' }] } }); + }); + + it('should be overriden by the port of a specific probe server', () => { + const config = applyEnvOverrides({ processor: {} }, schema, [], { + LIVENESS_PROBE_PORT: '8100', + PROCESSOR_PROBE_SERVER_PORT: '8200', + }); + assert.deepStrictEqual(config.processor.probeServer, + { bindAddress: '0.0.0.0', port: '8200' }); + }); + }); +}); + +describe('composite config env vars', () => { + const apply = env => applyCompositeEnvOverrides({ redis: { host: 'localhost', port: 6379 } }, env); + + it('should set the sentinels group name, and drop the standalone host', () => { + assert.deepStrictEqual(apply({ REDIS_SENTINELS: 'host1:26379' }).redis, + { sentinels: 'host1:26379', name: 'mymaster' }); + assert.deepStrictEqual(apply({ REDIS_SENTINELS: 'host1:26379', REDIS_HA_NAME: 'group' }).redis, + { sentinels: 'host1:26379', name: 'group' }); + }); + + it('should default the standalone redis port', () => { + assert.deepStrictEqual(apply({ REDIS_HOST: 'redis' }).redis, { host: 'redis', port: '6379' }); + assert.deepStrictEqual(apply({ REDIS_HOST: 'redis', REDIS_PORT: '6380' }).redis, + { host: 'redis', port: '6380' }); + assert.deepStrictEqual(apply({ REDIS_PORT: '6380' }).redis, + { host: 'localhost', port: '6380' }); + }); + + it('should ignore the standalone redis host and port when sentinels are set', () => { + assert.deepStrictEqual( + apply({ REDIS_SENTINELS: 'host1:26379', REDIS_HOST: 'redis', REDIS_PORT: '6380' }).redis, + { sentinels: 'host1:26379', name: 'mymaster' }); + }); + + it('should set the replica set hosts, and leave the log source alone', () => { + const config = applyCompositeEnvOverrides({}, { MONGODB_HOSTS: 'mongo1:27017,mongo2:27017' }); + assert.deepStrictEqual(config.queuePopulator, { + mongo: { replicaSetHosts: 'mongo1:27017,mongo2:27017' }, + }); + }); + + it('should build the replication bootstrap list of a single site', () => { + const config = applyCompositeEnvOverrides({ extensions: { replication: {} } }, { + EXTENSIONS_REPLICATION_DEST_BOOTSTRAPLIST: 'zenko-1:8000', + }); + assert.deepStrictEqual(config.extensions.replication.destination.bootstrapList, + [{ site: 'zenko', servers: ['zenko-1:8000'] }]); + }); + + it('should split the servers of the replication bootstrap site', () => { + const config = applyCompositeEnvOverrides({ extensions: { replication: {} } }, { + EXTENSIONS_REPLICATION_DEST_BOOTSTRAPLIST: 'zenko-1:8000, zenko-2:8000', + }); + assert.deepStrictEqual(config.extensions.replication.destination.bootstrapList, + [{ site: 'zenko', servers: ['zenko-1:8000', 'zenko-2:8000'] }]); + }); + + it('should append the additional replication bootstrap sites', () => { + const config = applyCompositeEnvOverrides({ extensions: { replication: {} } }, { + EXTENSIONS_REPLICATION_DEST_BOOTSTRAPLIST: 'zenko-1:8000', + EXTENSIONS_REPLICATION_DEST_BOOTSTRAPLIST_MORE: '{ "site": "aws", "type": "aws_s3" }', + }); + assert.deepStrictEqual(config.extensions.replication.destination.bootstrapList, [ + { site: 'zenko', servers: ['zenko-1:8000'] }, + { site: 'aws', type: 'aws_s3' }, + ]); + }); + + it('should leave the bootstrap list alone when replication is not configured', () => { + assert.deepStrictEqual( + applyCompositeEnvOverrides({}, { EXTENSIONS_REPLICATION_DEST_BOOTSTRAPLIST: 'zenko-1:8000' }), + {}); + }); +}); + +/** + * The env var contract is consumed by zenko-operator and CI: every var the + * docker entrypoint used to apply with jq must still set the same fields. + */ +describe('historic config env vars', () => { + /** + * Every name the entrypoint applied with jq, plus the two lib/Config.js + * applied on its own. zenko-operator sets some of them, and Federation + * forwards arbitrary ones through `env_backbeat_extraenv2`, so the whole + * list has to keep working. Each name is checked below to be either + * covered by a contract case, or explicitly removed. + */ + const historicEnvVars = [ + 'CLOUDSERVER_HOST', + 'CLOUDSERVER_PORT', + 'EXTENSIONS_GC_TOPIC', + 'EXTENSIONS_INGESTION_AUTH_ACCOUNT', + 'EXTENSIONS_INGESTION_AUTH_TYPE', + 'EXTENSIONS_INGESTION_MAX_PARALLEL_READERS', + 'EXTENSIONS_LIFECYCLE_AUTH_ACCOUNT', + 'EXTENSIONS_LIFECYCLE_AUTH_TYPE', + 'EXTENSIONS_LIFECYCLE_BUCKET_PROCESSOR_GROUP_ID', + 'EXTENSIONS_LIFECYCLE_BUCKET_TASK_TOPIC', + 'EXTENSIONS_LIFECYCLE_CONDUCTOR_CRONRULE', + 'EXTENSIONS_LIFECYCLE_OBJECT_PROCESSOR_GROUP_ID', + 'EXTENSIONS_LIFECYCLE_OBJECT_TASK_TOPIC', + 'EXTENSIONS_LIFECYCLE_RULES_ABORT_INCOMPLETE_MPU_ENABLED', + 'EXTENSIONS_LIFECYCLE_RULES_EXPIRATION_ENABLED', + 'EXTENSIONS_LIFECYCLE_RULES_NC_VERSION_EXPIRATION_ENABLED', + 'EXTENSIONS_LIFECYCLE_RULES_TRANSITIONS_ENABLED', + 'EXTENSIONS_LIFECYCLE_ZOOKEEPER_PATH', + 'EXTENSIONS_REPLICATION_DEST_AUTH_ACCOUNT', + 'EXTENSIONS_REPLICATION_DEST_AUTH_TYPE', + 'EXTENSIONS_REPLICATION_DEST_BOOTSTRAPLIST', + 'EXTENSIONS_REPLICATION_DEST_BOOTSTRAPLIST_MORE', + 'EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_AWS_S3_RETRY_BACKOFF_FACTOR', + 'EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_AWS_S3_RETRY_BACKOFF_JITTER', + 'EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_AWS_S3_RETRY_BACKOFF_MAX', + 'EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_AWS_S3_RETRY_BACKOFF_MIN', + 'EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_AWS_S3_RETRY_MAX_RETRIES', + 'EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_AWS_S3_RETRY_TIMEOUT_S', + 'EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_AZURE_RETRY_BACKOFF_FACTOR', + 'EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_AZURE_RETRY_BACKOFF_JITTER', + 'EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_AZURE_RETRY_BACKOFF_MAX', + 'EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_AZURE_RETRY_BACKOFF_MIN', + 'EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_AZURE_RETRY_MAX_RETRIES', + 'EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_AZURE_RETRY_TIMEOUT_S', + 'EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_CONCURRENCY', + 'EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_GCP_RETRY_BACKOFF_FACTOR', + 'EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_GCP_RETRY_BACKOFF_JITTER', + 'EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_GCP_RETRY_BACKOFF_MAX', + 'EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_GCP_RETRY_BACKOFF_MIN', + 'EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_GCP_RETRY_MAX_RETRIES', + 'EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_GCP_RETRY_TIMEOUT_S', + 'EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_MAX_POLL_INTERVAL_MS', + 'EXTENSIONS_REPLICATION_SOURCE_AUTH_ACCOUNT', + 'EXTENSIONS_REPLICATION_SOURCE_AUTH_TYPE', + 'EXTENSIONS_REPLICATION_SOURCE_S3_HOST', + 'EXTENSIONS_REPLICATION_SOURCE_S3_PORT', + 'EXTENSIONS_REPLICATION_STATUS_PROCESSOR_CONCURRENCY', + 'HEALTHCHECKS_ALLOWFROM', + 'KAFKA_BACKLOG_METRICS_INTERVALS', + 'KAFKA_BACKLOG_METRICS_ZKPATH', + 'KAFKA_HOSTS', + 'LIVENESS_PROBE_PORT', + 'LOG_LEVEL', + 'MONGODB_AUTH_PASSWORD', + 'MONGODB_AUTH_USERNAME', + 'MONGODB_DATABASE', + 'MONGODB_HOSTS', + 'MONGODB_RS', + 'QUEUE_POPULATOR_BATCH_MAX_READ', + 'QUEUE_POPULATOR_DMD_HOST', + 'QUEUE_POPULATOR_DMD_PORT', + 'REDIS_HA_NAME', + 'REDIS_HOST', + 'REDIS_LOCALCACHE_HOST', + 'REDIS_LOCALCACHE_PORT', + 'REDIS_PORT', + 'REDIS_SENTINELS', + 'REPLICATION_GROUP_ID', + 'ZOOKEEPER_AUTO_CREATE_NAMESPACE', + 'ZOOKEEPER_CONNECTION_STRING', + ]; + + // the lifecycle rules are configured with supportedLifecycleRules, and the + // local cache is not part of the configuration schema + const removedEnvVars = [ + 'EXTENSIONS_LIFECYCLE_RULES_ABORT_INCOMPLETE_MPU_ENABLED', + 'EXTENSIONS_LIFECYCLE_RULES_EXPIRATION_ENABLED', + 'EXTENSIONS_LIFECYCLE_RULES_NC_VERSION_EXPIRATION_ENABLED', + 'EXTENSIONS_LIFECYCLE_RULES_TRANSITIONS_ENABLED', + 'REDIS_LOCALCACHE_HOST', + 'REDIS_LOCALCACHE_PORT', + ]; + + const retryFields = backend => ({ + [`EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_${backend}_RETRY_MAX_RETRIES`]: '1', + [`EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_${backend}_RETRY_TIMEOUT_S`]: '2', + [`EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_${backend}_RETRY_BACKOFF_MIN`]: '3', + [`EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_${backend}_RETRY_BACKOFF_MAX`]: '4', + [`EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_${backend}_RETRY_BACKOFF_JITTER`]: '0.5', + [`EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_${backend}_RETRY_BACKOFF_FACTOR`]: '6', + }); + const retryExpected = backend => ({ + [`extensions.replication.queueProcessor.retry.${backend}.maxRetries`]: 1, + [`extensions.replication.queueProcessor.retry.${backend}.timeoutS`]: 2, + [`extensions.replication.queueProcessor.retry.${backend}.backoff`]: + { min: 3, max: 4, jitter: 0.5, factor: 6 }, + }); + + const probeServer = { bindAddress: '0.0.0.0', port: 8100 }; + const contract = [ + [{ LIVENESS_PROBE_PORT: '8100' }, { + 'queuePopulator.probeServer': probeServer, + 'extensions.ingestion.probeServer': probeServer, + 'extensions.mongoProcessor.probeServer': probeServer, + 'extensions.replication.queueProcessor.probeServer': probeServer, + 'extensions.replication.replicationStatusProcessor.probeServer': probeServer, + 'extensions.lifecycle.conductor.probeServer': probeServer, + 'extensions.lifecycle.bucketProcessor.probeServer': probeServer, + 'extensions.lifecycle.objectProcessor.probeServer': probeServer, + 'extensions.gc.probeServer': probeServer, + }], + [{ LOG_LEVEL: 'debug' }, { 'log.logLevel': 'debug' }], + [{ ZOOKEEPER_AUTO_CREATE_NAMESPACE: 'true' }, { 'zookeeper.autoCreateNamespace': true }], + [{ ZOOKEEPER_CONNECTION_STRING: 'zk:2181/bb' }, { 'zookeeper.connectionString': 'zk:2181/bb' }], + [{ KAFKA_HOSTS: 'kafka:9092' }, { 'kafka.hosts': 'kafka:9092' }], + [{ KAFKA_BACKLOG_METRICS_ZKPATH: '/bb/metrics' }, { 'kafka.backlogMetrics.zkPath': '/bb/metrics' }], + [{ KAFKA_BACKLOG_METRICS_INTERVALS: '30' }, { 'kafka.backlogMetrics.intervalS': 30 }], + [{ REDIS_SENTINELS: 'sentinel1:26379,sentinel2:26379', REDIS_HA_NAME: 'group' }, { + redis: { + name: 'group', + sentinels: [{ host: 'sentinel1', port: 26379 }, { host: 'sentinel2', port: 26379 }], + }, + }], + [{ REDIS_HOST: 'redis' }, { 'redis.host': 'redis', 'redis.port': 6379 }], + [{ REDIS_HOST: 'redis', REDIS_PORT: '6380' }, { 'redis.host': 'redis', 'redis.port': 6380 }], + [{ QUEUE_POPULATOR_BATCH_MAX_READ: '42' }, { 'queuePopulator.batchMaxRead': 42 }], + [{ QUEUE_POPULATOR_DMD_HOST: 'dmd', QUEUE_POPULATOR_DMD_PORT: '9991' }, { + 'queuePopulator.dmd.host': 'dmd', + 'queuePopulator.dmd.port': 9991, + }], + [{ MONGODB_HOSTS: 'mongo1:27017,mongo2:27017' }, { + 'queuePopulator.mongo.replicaSetHosts': 'mongo1:27017,mongo2:27017', + }], + [{ MONGODB_RS: 'rs1' }, { 'queuePopulator.mongo.replicaSet': 'rs1' }], + [{ MONGODB_DATABASE: 'db' }, { 'queuePopulator.mongo.database': 'db' }], + [{ MONGODB_AUTH_USERNAME: 'user', MONGODB_AUTH_PASSWORD: 'pass' }, { + 'queuePopulator.mongo.authCredentials': { username: 'user', password: 'pass' }, + }], + [{ CLOUDSERVER_HOST: 'cloudserver', CLOUDSERVER_PORT: '8001' }, { + 's3.host': 'cloudserver', + 's3.port': 8001, + }], + [{ HEALTHCHECKS_ALLOWFROM: '10.0.0.0/8' }, { + // the loopback addresses are always allowed + 'server.healthChecks.allowFrom': ['10.0.0.0/8', '127.0.0.1/8', '::1'], + }], + [{ REPLICATION_GROUP_ID: 'RG00002' }, { replicationGroupId: 'RG00002' }], + [{ + EXTENSIONS_REPLICATION_SOURCE_S3_HOST: 'cloudserver', + EXTENSIONS_REPLICATION_SOURCE_S3_PORT: '8001', + }, { + 'extensions.replication.source.s3.host': 'cloudserver', + 'extensions.replication.source.s3.port': 8001, + }], + [{ + EXTENSIONS_REPLICATION_SOURCE_AUTH_TYPE: 'account', + EXTENSIONS_REPLICATION_SOURCE_AUTH_ACCOUNT: 'source-account', + }, { + 'extensions.replication.source.auth.type': 'account', + 'extensions.replication.source.auth.account': 'source-account', + }], + [{ + EXTENSIONS_REPLICATION_DEST_AUTH_TYPE: 'account', + EXTENSIONS_REPLICATION_DEST_AUTH_ACCOUNT: 'dest-account', + }, { + 'extensions.replication.destination.auth.type': 'account', + 'extensions.replication.destination.auth.account': 'dest-account', + }], + [{ EXTENSIONS_REPLICATION_DEST_BOOTSTRAPLIST: 'zenko-1:8000' }, { + 'extensions.replication.destination.bootstrapList': + [{ site: 'zenko', servers: ['zenko-1:8000'], echo: false }], + }], + [{ + EXTENSIONS_REPLICATION_DEST_BOOTSTRAPLIST: 'zenko-1:8000', + EXTENSIONS_REPLICATION_DEST_BOOTSTRAPLIST_MORE: '{ "site": "aws", "type": "aws_s3" }', + }, { + 'extensions.replication.destination.bootstrapList': [ + { site: 'zenko', servers: ['zenko-1:8000'], echo: false }, + { site: 'aws', type: 'aws_s3' }, + ], + }], + [retryFields('AWS_S3'), retryExpected('aws_s3')], + [retryFields('AZURE'), retryExpected('azure')], + [retryFields('GCP'), retryExpected('gcp')], + [{ EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_CONCURRENCY: '11' }, + { 'extensions.replication.queueProcessor.concurrency': 11 }], + [{ EXTENSIONS_REPLICATION_STATUS_PROCESSOR_CONCURRENCY: '7' }, + { 'extensions.replication.replicationStatusProcessor.concurrency': 7 }], + [{ EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_MAX_POLL_INTERVAL_MS: '60000' }, + { 'extensions.replication.queueProcessor.maxPollIntervalMs': 60000 }], + [{ EXTENSIONS_LIFECYCLE_ZOOKEEPER_PATH: '/lc' }, { 'extensions.lifecycle.zookeeperPath': '/lc' }], + [{ EXTENSIONS_LIFECYCLE_BUCKET_TASK_TOPIC: 'lc-buckets' }, + { 'extensions.lifecycle.bucketTasksTopic': 'lc-buckets' }], + [{ EXTENSIONS_LIFECYCLE_OBJECT_TASK_TOPIC: 'lc-objects' }, + { 'extensions.lifecycle.objectTasksTopic': 'lc-objects' }], + [{ EXTENSIONS_LIFECYCLE_CONDUCTOR_CRONRULE: '0 0 * * * *' }, + { 'extensions.lifecycle.conductor.cronRule': '0 0 * * * *' }], + [{ EXTENSIONS_LIFECYCLE_BUCKET_PROCESSOR_GROUP_ID: 'lc-bucket-group' }, + { 'extensions.lifecycle.bucketProcessor.groupId': 'lc-bucket-group' }], + [{ EXTENSIONS_LIFECYCLE_OBJECT_PROCESSOR_GROUP_ID: 'lc-object-group' }, + { 'extensions.lifecycle.objectProcessor.groupId': 'lc-object-group' }], + [{ EXTENSIONS_LIFECYCLE_AUTH_TYPE: 'account', EXTENSIONS_LIFECYCLE_AUTH_ACCOUNT: 'lc-account' }, { + 'extensions.lifecycle.auth.type': 'account', + 'extensions.lifecycle.auth.account': 'lc-account', + }], + [{ EXTENSIONS_GC_TOPIC: 'gc-topic' }, { 'extensions.gc.topic': 'gc-topic' }], + [{ EXTENSIONS_INGESTION_AUTH_TYPE: 'service', EXTENSIONS_INGESTION_AUTH_ACCOUNT: 'ingest' }, { + 'extensions.ingestion.auth.type': 'service', + 'extensions.ingestion.auth.account': 'ingest', + }], + [{ EXTENSIONS_INGESTION_MAX_PARALLEL_READERS: '3' }, + { 'extensions.ingestion.maxParallelReaders': 3 }], + ]; + + let ogConfigFile; + + before(() => { + ogConfigFile = process.env.BACKBEAT_CONFIG_FILE; + process.env.BACKBEAT_CONFIG_FILE = `${__dirname}/config.json`; + }); + + after(() => { + if (ogConfigFile === undefined) { + delete process.env.BACKBEAT_CONFIG_FILE; + } else { + process.env.BACKBEAT_CONFIG_FILE = ogConfigFile; + } + }); + + function configWith(env) { + const og = Object.fromEntries(Object.keys(env).map(name => [name, process.env[name]])); + Object.assign(process.env, env); + try { + return new Config(); + } finally { + Object.entries(og).forEach(([name, value]) => { + if (value === undefined) { + delete process.env[name]; + } else { + process.env[name] = value; + } + }); + } + } + + contract.forEach(([env, expected]) => { + it(`should apply ${Object.keys(env).join(', ')}`, () => { + const config = configWith(env); + Object.entries(expected).forEach(([path, value]) => + assert.deepStrictEqual(getField(config, path.split('.')), value, path)); + }); + }); + + it('should account for every historic env var', () => { + const covered = new Set([ + ...contract.flatMap(([env]) => Object.keys(env)), + ...removedEnvVars, + ]); + + assert.deepStrictEqual(historicEnvVars.filter(name => !covered.has(name)), []); + }); + + it('should accept the log levels the entrypoint used to reject', () => { + assert.strictEqual(configWith({ LOG_LEVEL: 'warn' }).log.logLevel, 'warn'); + assert.strictEqual(configWith({ LOG_LEVEL: 'error' }).log.logLevel, 'error'); + }); + + it('should ignore the vars removed with the entrypoint', () => { + const config = configWith(Object.fromEntries(removedEnvVars.map(name => [name, 'redis']))); + + assert.strictEqual(config.extensions.lifecycle.rules, undefined); + assert.strictEqual(config.localCache, undefined); + }); +}); diff --git a/tests/unit/mongoProcessor/MongoProcessorConfigValidator.spec.js b/tests/unit/mongoProcessor/MongoProcessorConfigValidator.spec.js index 3778d563c..af6135620 100644 --- a/tests/unit/mongoProcessor/MongoProcessorConfigValidator.spec.js +++ b/tests/unit/mongoProcessor/MongoProcessorConfigValidator.spec.js @@ -27,4 +27,14 @@ describe('MongoProcessorConfigValidator log override', () => { const validated = configValidator({}, { ...baseConfig, log: { logLevel: 'warn' } }); assert.deepStrictEqual(validated.log, { logLevel: 'warn', dumpLevel: 'error' }); }); + + it('should accept the log level from the environment on its own', () => { + process.env.EXTENSIONS_MONGO_PROCESSOR_LOG_LEVEL = 'warn'; + try { + const validated = configValidator({}, { ...baseConfig }); + assert.deepStrictEqual(validated.log, { logLevel: 'warn', dumpLevel: 'error' }); + } finally { + delete process.env.EXTENSIONS_MONGO_PROCESSOR_LOG_LEVEL; + } + }); }); From 15d63aa169e796229ef80392b0357a146ae725b5 Mon Sep 17 00:00:00 2001 From: Francois Ferrand Date: Tue, 18 Aug 2026 12:39:36 +0200 Subject: [PATCH 3/8] Inherit the log levels an extension does not configure MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Setting a per-extension log level silently reset the dump level to a constant, dropping whatever a deployment had configured globally. An extension now inherits the levels it does not configure from the global log config, so it overrides only what it names — which is also what an override from the environment produces, since a variable names a single field. Extension schemas reach the global configuration through the validation context. The extension validators were already given it, but as the configuration object still being built, so nothing could be read from it. The extension configurations are kept out of it: they are validated one after the other, so referencing one would resolve differently depending on the order of the configuration file, and an extension must not depend on another one. Issue: BB-808 --- lib/Config.js | 14 +++++++++---- lib/config/configItems.joi.js | 17 +++++++-------- lib/config/extensionConfigValidator.js | 11 +++++++++- .../IngestionConfigValidator.spec.js | 20 +++++++++++------- tests/unit/lib/config/envOverrides.spec.js | 3 --- .../MongoProcessorConfigValidator.spec.js | 21 +++++++++++-------- 6 files changed, 52 insertions(+), 34 deletions(-) diff --git a/lib/Config.js b/lib/Config.js index 9e32394be..9b1e4b130 100644 --- a/lib/Config.js +++ b/lib/Config.js @@ -71,17 +71,23 @@ class Config extends EventEmitter { const parsedConfig = joi.attempt(config, backbeatConfigJoi); + // the configuration an extension is validated against: the extension + // configurations are left out, as they are validated one after the + // other, and an extension must not depend on another one + const globalConfig = { ...parsedConfig }; + delete globalConfig.extensions; + if (parsedConfig.extensions) { Object.keys(parsedConfig.extensions).forEach(extName => { - const index = extensions[extName]; - if (!index) { + const extension = extensions[extName]; + if (!extension) { throw new Error(`configured extension ${extName}: ` + 'not found in extensions directory'); } - if (index.configValidator) { + if (extension.configValidator) { const extConfig = parsedConfig.extensions[extName]; const validatedConfig = - index.configValidator(this, extConfig); + extension.configValidator(globalConfig, extConfig); parsedConfig.extensions[extName] = validatedConfig; } }); diff --git a/lib/config/configItems.joi.js b/lib/config/configItems.joi.js index f5803ff62..08eab2b23 100644 --- a/lib/config/configItems.joi.js +++ b/lib/config/configItems.joi.js @@ -43,18 +43,15 @@ const bootstrapListJoi = joi.array() const logLevelJoi = joi.alternatives() .try('error', 'warn', 'info', 'debug', 'trace'); -// the levels default individually, so that setting one of them, from the -// configuration or from the environment, leaves the other one alone -const logKeys = { - logLevel: logLevelJoi.default('info').meta({ envName: 'LEVEL' }), +const logJoi = joi.object({ + logLevel: logLevelJoi.default('info').meta({ env: 'LEVEL' }), dumpLevel: logLevelJoi.default('error'), -}; - -const logJoi = joi.object(logKeys).default(); +}).default(); -// logJoi with no default : -// Callers fall back to the global log config when this one is not configured -const logJoiOptional = joi.object(logKeys).optional(); +const logJoiOptional = joi.object({ + logLevel: logLevelJoi.default(joi.ref('$log.logLevel')).meta({ env: 'LEVEL' }), + dumpLevel: logLevelJoi.default(joi.ref('$log.dumpLevel')), +}).optional(); const adminCredsJoi = joi.object() .min(1) diff --git a/lib/config/extensionConfigValidator.js b/lib/config/extensionConfigValidator.js index 841ca1179..0918c3a98 100644 --- a/lib/config/extensionConfigValidator.js +++ b/lib/config/extensionConfigValidator.js @@ -9,13 +9,22 @@ const { applyEnvOverrides } = require('./envOverrides'); * from the extension schema (e.g. EXTENSIONS_GC_TOPIC for the `topic` field of * the gc extension) before validating. * + * The global backbeat configuration is passed as the validation context, so + * that a field can default to a global one, e.g. `joi.ref('$log.logLevel')`. It + * is already validated when an extension is: its own defaults are set. + * + * It does not carry the extension configurations: those are validated one after + * the other, so referencing one would resolve differently depending on the + * order of the configuration file. An extension must not depend on another one. + * * @param {string} extName - extension name, as configured in `extensions` * @param {joi.Schema} schema - extension configuration schema * @returns {function} extension config validator */ function extensionConfigValidator(extName, schema) { return (backbeatConfig, extConfig) => - joi.attempt(applyEnvOverrides(extConfig, schema, ['extensions', extName]), schema); + joi.attempt(applyEnvOverrides(extConfig, schema, ['extensions', extName]), schema, + { context: backbeatConfig }); } module.exports = { extensionConfigValidator }; diff --git a/tests/unit/ingestion/IngestionConfigValidator.spec.js b/tests/unit/ingestion/IngestionConfigValidator.spec.js index 66cfb283e..7478a3846 100644 --- a/tests/unit/ingestion/IngestionConfigValidator.spec.js +++ b/tests/unit/ingestion/IngestionConfigValidator.spec.js @@ -12,11 +12,14 @@ const baseExtConfig = { probeServer: { port: 4000 }, }; +// the validated backbeat config, passed to every extension validator +const globalConfig = { log: { logLevel: 'info', dumpLevel: 'trace' } }; + const qpBatchMaxRead = config.queuePopulator.batchMaxRead; describe('IngestionConfigValidator log override', () => { it('should pass through log config when set', () => { - const validated = configValidator({}, { + const validated = configValidator(globalConfig, { ...baseExtConfig, log: { logLevel: 'debug', dumpLevel: 'error' }, }); @@ -24,20 +27,23 @@ describe('IngestionConfigValidator log override', () => { }); it('should leave log undefined when not set, deferring to global config.log', () => { - const validated = configValidator({}, baseExtConfig); + const validated = configValidator(globalConfig, baseExtConfig); assert.strictEqual(validated.log, undefined); }); - it('should default the level left out of a partial log config', () => { - const validated = configValidator({}, { ...baseExtConfig, log: { logLevel: 'debug' } }); - assert.deepStrictEqual(validated.log, { logLevel: 'debug', dumpLevel: 'error' }); + it('should inherit the level left out of a partial log config', () => { + const validated = configValidator(globalConfig, { + ...baseExtConfig, + log: { logLevel: 'debug' }, + }); + assert.deepStrictEqual(validated.log, { logLevel: 'debug', dumpLevel: 'trace' }); }); it('should accept the log level from the environment on its own', () => { process.env.EXTENSIONS_INGESTION_LOG_LEVEL = 'debug'; try { - const validated = configValidator({}, { ...baseExtConfig }); - assert.deepStrictEqual(validated.log, { logLevel: 'debug', dumpLevel: 'error' }); + const validated = configValidator(globalConfig, { ...baseExtConfig }); + assert.deepStrictEqual(validated.log, { logLevel: 'debug', dumpLevel: 'trace' }); } finally { delete process.env.EXTENSIONS_INGESTION_LOG_LEVEL; } diff --git a/tests/unit/lib/config/envOverrides.spec.js b/tests/unit/lib/config/envOverrides.spec.js index 4ab075447..cf9f5c407 100644 --- a/tests/unit/lib/config/envOverrides.spec.js +++ b/tests/unit/lib/config/envOverrides.spec.js @@ -11,7 +11,6 @@ const { const { backbeatConfigJoi } = require('../../../../lib/config.joi'); const { logJoi, - logJoiOptional, probeServerJoi, probeServerPerSite, } = require('../../../../lib/config/configItems.joi'); @@ -226,8 +225,6 @@ describe('config env var mapping', () => { const config = applyEnvOverrides({}, backbeatConfigJoi, [], { LOG_LEVEL: 'warn' }); assert.deepStrictEqual(joi.attempt(config.log, logJoi), { logLevel: 'warn', dumpLevel: 'error' }); - assert.deepStrictEqual(joi.attempt({ dumpLevel: 'warn' }, logJoiOptional), - { logLevel: 'info', dumpLevel: 'warn' }); }); }); diff --git a/tests/unit/mongoProcessor/MongoProcessorConfigValidator.spec.js b/tests/unit/mongoProcessor/MongoProcessorConfigValidator.spec.js index af6135620..9723c20a4 100644 --- a/tests/unit/mongoProcessor/MongoProcessorConfigValidator.spec.js +++ b/tests/unit/mongoProcessor/MongoProcessorConfigValidator.spec.js @@ -3,36 +3,39 @@ const assert = require('assert'); const configValidator = require('../../../extensions/mongoProcessor/MongoProcessorConfigValidator'); -const baseConfig = { +const baseExtConfig = { topic: 'backbeat-ingestion', groupId: 'backbeat-ingestion-group', probeServer: { port: 4000 }, }; +// the validated backbeat config, passed to every extension validator +const globalConfig = { log: { logLevel: 'info', dumpLevel: 'trace' } }; + describe('MongoProcessorConfigValidator log override', () => { it('should pass through log config when set', () => { - const validated = configValidator({}, { - ...baseConfig, + const validated = configValidator(globalConfig, { + ...baseExtConfig, log: { logLevel: 'warn', dumpLevel: 'error' }, }); assert.deepStrictEqual(validated.log, { logLevel: 'warn', dumpLevel: 'error' }); }); it('should leave log undefined when not set, deferring to global config.log', () => { - const validated = configValidator({}, baseConfig); + const validated = configValidator(globalConfig, baseExtConfig); assert.strictEqual(validated.log, undefined); }); - it('should default the level left out of a partial log config', () => { - const validated = configValidator({}, { ...baseConfig, log: { logLevel: 'warn' } }); - assert.deepStrictEqual(validated.log, { logLevel: 'warn', dumpLevel: 'error' }); + it('should inherit the level left out of a partial log config', () => { + const validated = configValidator(globalConfig, { ...baseExtConfig, log: { logLevel: 'warn' } }); + assert.deepStrictEqual(validated.log, { logLevel: 'warn', dumpLevel: 'trace' }); }); it('should accept the log level from the environment on its own', () => { process.env.EXTENSIONS_MONGO_PROCESSOR_LOG_LEVEL = 'warn'; try { - const validated = configValidator({}, { ...baseConfig }); - assert.deepStrictEqual(validated.log, { logLevel: 'warn', dumpLevel: 'error' }); + const validated = configValidator(globalConfig, { ...baseExtConfig }); + assert.deepStrictEqual(validated.log, { logLevel: 'warn', dumpLevel: 'trace' }); } finally { delete process.env.EXTENSIONS_MONGO_PROCESSOR_LOG_LEVEL; } From 08f04196c0a56929338f5a54eb01ddf9505a165f Mon Sep 17 00:00:00 2001 From: Francois Ferrand Date: Thu, 20 Aug 2026 20:08:34 +0200 Subject: [PATCH 4/8] Memorize envVarMappings per schema extension validators re-derive the envVarMapping it on every call, memoizing to avoid the redundant cost (~30ms each time). Not critical since it happens only once on startup, but adds up in tests. Issue: BB-808 --- lib/config/envOverrides.js | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/lib/config/envOverrides.js b/lib/config/envOverrides.js index 6ea31eba3..d5941425d 100644 --- a/lib/config/envOverrides.js +++ b/lib/config/envOverrides.js @@ -216,18 +216,30 @@ function collectMappings(description, names, path, root, mappings) { } } +// `describe()` accounts for most of the configuration parsing time, and the +// mappings of a schema never change: they are derived once per process +const mappingsCache = new WeakMap(); + /** - * Maps the env var names a schema supports to the config path they set. + * Maps the env var names a schema supports to the config path they set. The + * returned map is shared between calls, and must not be modified. * * @param {joi.Schema} schema - configuration schema * @param {string[]} [prefix] - config path of the schema root * @returns {Map} env var name to { path, type } */ function envVarMappings(schema, prefix = []) { - const mappings = new Map(); const root = envVarName(prefix); - collectMappings(schema.describe(), [root], [], root, mappings); - return mappings; + if (!mappingsCache.has(schema)) { + mappingsCache.set(schema, new Map()); + } + const cached = mappingsCache.get(schema); + if (!cached.has(root)) { + const mappings = new Map(); + collectMappings(schema.describe(), [root], [], root, mappings); + cached.set(root, mappings); + } + return cached.get(root); } /** From 2acfc939498d0aee60ce9f9ea2414c1548a5e79c Mon Sep 17 00:00:00 2001 From: Francois Ferrand Date: Fri, 21 Aug 2026 18:12:39 +0200 Subject: [PATCH 5/8] Refactor compositeEnvOverrides The complexity was artificial, change mapping to simplify. Issue: BB-808 --- docs/configuration.md | 6 +- .../replication/ReplicationConfigValidator.js | 26 +++- lib/Config.js | 7 +- lib/config.joi.js | 37 +++-- lib/config/configItems.joi.js | 3 +- lib/config/envOverrides.js | 124 +++++------------ lib/config/fields.js | 17 --- tests/config.json | 1 - .../queuePopulator/config/s3c-config.json | 1 - tests/unit/lib/config/config.joi.spec.js | 67 ++++++++++ tests/unit/lib/config/envOverrides.spec.js | 126 ++++++++---------- .../ReplicationConfigValidator.spec.js | 48 +++++++ 12 files changed, 253 insertions(+), 210 deletions(-) create mode 100644 tests/unit/lib/config/config.joi.spec.js diff --git a/docs/configuration.md b/docs/configuration.md index a6e2f7534..322998328 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -69,7 +69,6 @@ silently ignored. - `LIVENESS_PROBE_PORT`: the port of every probe server configured, bound to `0.0.0.0`. Per site probe servers are left alone. -- `MONGODB_HOSTS`: `queuePopulator.mongo.replicaSetHosts`. - `REDIS_SENTINELS`, `REDIS_HA_NAME`: `redis.sentinels` and `redis.name` (`mymaster` by default). They replace the standalone host and port. The sentinels are a comma separated list of `host:port`, e.g. @@ -77,9 +76,10 @@ silently ignored. - `REDIS_HOST`, `REDIS_PORT`: standalone `redis.host` and `redis.port` (6379 by default). Both are ignored when sentinels are configured. - `EXTENSIONS_REPLICATION_DEST_BOOTSTRAPLIST`: the servers of the `zenko` site - of the replication bootstrap list, comma separated. + of the replication bootstrap list, comma separated, for backwards compatibility. `EXTENSIONS_REPLICATION_DEST_BOOTSTRAPLIST_MORE` holds the additional sites, - as raw JSON objects, e.g. `{ "site": "aws", "type": "aws_s3" }`. + as raw JSON objects, e.g. `{ "site": "aws", "type": "aws_s3" }`. It can also + take the full list of sites as a JSON array. ## Other variables diff --git a/extensions/replication/ReplicationConfigValidator.js b/extensions/replication/ReplicationConfigValidator.js index 2a0c8023f..03d2f16d0 100644 --- a/extensions/replication/ReplicationConfigValidator.js +++ b/extensions/replication/ReplicationConfigValidator.js @@ -5,6 +5,7 @@ const { hostPortJoi, transportJoi, bootstrapListJoi, adminCredsJoi, stsConfigJoi } = require('../../lib/config/configItems.joi'); const { extensionConfigValidator } = require('../../lib/config/extensionConfigValidator'); +const { parseJSON } = require('../../lib/config/envOverrides'); const { authTypeAccount, authTypeAssumeRole, @@ -14,6 +15,26 @@ const { const { MAX_QUEUED_DEFAULT } = require('../../lib/constants').backbeatConsumer; +const BOOTSTRAPLIST_MORE = 'EXTENSIONS_REPLICATION_DEST_BOOTSTRAPLIST_MORE'; + +/** + * Decodes the legacy comma separated server list, naming the single `zenko` + * site, with the cloud backends of BOOTSTRAPLIST_MORE as raw JSON objects. + * The JSON form is left to the usual coercion. + * + * @param {string} value - raw env var value + * @param {Object} env - environment the override comes from + * @returns {Array|undefined} bootstrap list, undefined to coerce the value + */ +function decodeBootstrapList(value, env) { + if (value.trimStart().startsWith('[')) { + return undefined; + } + const zenko = { site: 'zenko', servers: value.split(',').map(server => server.trim()) }; + const more = env[BOOTSTRAPLIST_MORE]; + return more ? [zenko, ...parseJSON(`[${more}]`, BOOTSTRAPLIST_MORE)] : [zenko]; +} + // the historic env var names put the backend before `RETRY`, e.g. // EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_AWS_S3_RETRY_BACKOFF_MIN const qpRetryJoi = joi.object({ @@ -100,7 +121,10 @@ const joiSchema = joi.object({ then: joi.optional(), otherwise: joi.required(), }), - bootstrapList: bootstrapListJoi, + bootstrapList: bootstrapListJoi.meta({ + env: 'BOOTSTRAPLIST', + envDecodeHook: decodeBootstrapList, + }), }).required().custom(_validatePerSiteDestinationConfig).meta({ env: 'DEST' }), topic: joi.string().required(), dataMoverTopic: joi.string().optional(), diff --git a/lib/Config.js b/lib/Config.js index 9b1e4b130..58939719f 100644 --- a/lib/Config.js +++ b/lib/Config.js @@ -10,7 +10,7 @@ const crypto = require('crypto'); const extensions = require('../extensions'); const { backbeatConfigJoi } = require('./config.joi'); -const { applyCompositeEnvOverrides, applyEnvOverrides } = require('./config/envOverrides'); +const { applyEnvOverrides } = require('./config/envOverrides'); const locationTypeMatch = { 'location-mem-v1': 'mem', @@ -66,14 +66,11 @@ class Config extends EventEmitter { * @returns {undefined} */ _parseConfig(config) { - applyCompositeEnvOverrides(config); applyEnvOverrides(config, backbeatConfigJoi); const parsedConfig = joi.attempt(config, backbeatConfigJoi); - // the configuration an extension is validated against: the extension - // configurations are left out, as they are validated one after the - // other, and an extension must not depend on another one + // Strip config to ensure extensions cannot use another extension's config const globalConfig = { ...parsedConfig }; delete globalConfig.extensions; diff --git a/lib/config.joi.js b/lib/config.joi.js index 8b60784d5..647b3db86 100644 --- a/lib/config.joi.js +++ b/lib/config.joi.js @@ -106,27 +106,26 @@ const joiSchema = joi.object({ host: joi.string().required(), port: joi.number().default(8900), }, - redis: { - host: joi.string().when('sentinels', { - is: joi.exist(), - then: joi.forbidden(), - otherwise: joi.required(), + redis: joi.alternatives().conditional(joi.ref('.sentinels'), { + is: joi.exist(), + then: joi.object({ + sentinels: joi.alternatives([joi.string(), joi.array().items( + joi.object({ + host: joi.string().required(), + port: joi.number().required(), + }))] + ).required(), + // group name of the master the sentinels watch + name: joi.string().default('mymaster').meta({ env: 'HA_NAME' }), + password: joi.string().default('').allow(''), + sentinelPassword: joi.string().default('').allow(''), }), - port: joi.number().when('sentinels', { - is: joi.exist(), - then: joi.forbidden(), - otherwise: joi.required(), + otherwise: joi.object({ + host: joi.string().required(), + port: joi.number().default(6379), + password: joi.string().default('').allow(''), }), - name: joi.string().default('backbeat'), - password: joi.string().default('').allow(''), - sentinels: joi.alternatives([joi.string(), joi.array().items( - joi.object({ - host: joi.string().required(), - port: joi.number().required(), - }))] - ), - sentinelPassword: joi.string().default('').allow(''), - }, + }), certFilePaths: certFilePathsJoi, internalCertFilePaths: certFilePathsJoi, }); diff --git a/lib/config/configItems.joi.js b/lib/config/configItems.joi.js index 08eab2b23..3ed7cd779 100644 --- a/lib/config/configItems.joi.js +++ b/lib/config/configItems.joi.js @@ -127,7 +127,8 @@ const probeServerPerSite = joi.array().items( ); const mongoJoi = joi.object({ - replicaSetHosts: joi.string().default('localhost:27017'), + // MONGODB_HOSTS, through the alias of the section, is the historic name + replicaSetHosts: joi.string().default('localhost:27017').meta({ env: 'HOSTS' }), logName: joi.string().default('s3-recordlog'), writeConcern: joi.string().default('majority'), shardCollections: joi.boolean().default(false), diff --git a/lib/config/envOverrides.js b/lib/config/envOverrides.js index d5941425d..856ccafa0 100644 --- a/lib/config/envOverrides.js +++ b/lib/config/envOverrides.js @@ -4,6 +4,11 @@ * variables derived from the joi schemas. */ +const { getField, setField } = require('./fields'); + +// A container exposes a single probe endpoint, shared by all the probe servers +// of the process it runs: this variable sets the port of every one of them. No +// derived name can stand for it, as a name maps to a single field. const LIVENESS_PROBE_PORT = 'LIVENESS_PROBE_PORT'; /** @@ -41,64 +46,6 @@ function parseJSON(value, name) { } } -/** - * Env vars setting several fields at once, or fields the schema cannot name. - * They are applied before the derived ones, which take precedence for the - * fields they set. - */ -const compositeEnvVars = { - /** Standalone redis: the port has an implicit default, and sentinels win. */ - REDIS_HOST: (config, host, env) => { - if (env.REDIS_SENTINELS) { - return; - } - setField(config, ['redis', 'host'], host); - setField(config, ['redis', 'port'], env.REDIS_PORT || '6379'); - }, - REDIS_PORT: (config, port, env) => { - if (env.REDIS_SENTINELS) { - return; - } - setField(config, ['redis', 'port'], port); - }, - /** - * Sentinels come with their own group name (REDIS_HA_NAME), and replace the - * standalone host and port, which the schema then forbids. - */ - REDIS_SENTINELS: (config, sentinels, env) => { - setField(config, ['redis', 'sentinels'], sentinels); - setField(config, ['redis', 'name'], env.REDIS_HA_NAME || 'mymaster'); - deleteField(config, ['redis', 'host']); - deleteField(config, ['redis', 'port']); - }, - /** - * The historic name of `queuePopulator.mongo.replicaSetHosts`. - */ - MONGODB_HOSTS: (config, hosts) => { - setField(config, ['queuePopulator', 'mongo', 'replicaSetHosts'], hosts); - }, - /** - * Single-site form of the replication bootstrap list: the servers of the - * `zenko` site. The additional sites are cloud backends, which only a - * structured value can describe. - */ - EXTENSIONS_REPLICATION_DEST_BOOTSTRAPLIST: (config, servers, env) => { - if (!config.extensions || !config.extensions.replication) { - return; - } - const bootstrapList = [{ - site: 'zenko', - servers: servers.split(',').map(server => server.trim()), - }]; - const more = env.EXTENSIONS_REPLICATION_DEST_BOOTSTRAPLIST_MORE; - if (more) { - bootstrapList.push(...parseJSON(`[${more}]`, - 'EXTENSIONS_REPLICATION_DEST_BOOTSTRAPLIST_MORE')); - } - setField(config, ['extensions', 'replication', 'destination', 'bootstrapList'], bootstrapList); - }, -}; - // types joi does not coerce from a string, and which a JSON value can express: // `null` is an ambiguous type, e.g. a field accepting a string or an array const JSON_TYPES = ['array', 'object', null]; @@ -133,27 +80,29 @@ function coerceValue(value, type, name) { return value; } +/** + * @param {Object} description - joi description of a node + * @returns {Object} the annotations of the node, merged + */ +function annotations(description) { + return Object.assign({}, ...(description.metas || [])); +} + /** * @param {string[]} names - env var names of the field - * @param {string[]} path - config path, relative to the schema root - * @param {string} type - joi type of the field - * @param {Map} mappings - env var name to { path, type }, updated in place + * @param {Object} entry - { path, type, decode } of the field + * @param {Map} mappings - env var name to entry, updated in place * @returns {undefined} */ -function registerMapping(names, path, type, mappings) { +function registerMapping(names, entry, mappings) { names.forEach(name => { - if (compositeEnvVars[name]) { - // Composite variable are handled separately, as they sets several fields, - // or a field the schema cannot name - return; - } const existing = mappings.get(name); if (!existing) { - mappings.set(name, { path, type }); - } else if (existing.path.join('.') !== path.join('.')) { + mappings.set(name, entry); + } else if (existing.path.join('.') !== entry.path.join('.')) { throw new Error(`env var ${name} maps to both ${existing.path.join('.')} ` + - `and ${path.join('.')}`); - } else if (existing.type !== type) { + `and ${entry.path.join('.')}`); + } else if (existing.type !== entry.type) { // ambiguous type: inject the raw value and let joi coerce it existing.type = null; } @@ -172,7 +121,7 @@ function registerMapping(names, path, type, mappings) { * @returns {string[]} env var names of the node */ function nodeNames(description, parentNames, root, key) { - const meta = Object.assign({}, ...(description.metas || [])); + const meta = annotations(description); const segment = meta.env === undefined ? toSnakeCase(key) : meta.env; const names = parentNames.map(prefix => (prefix ? `${prefix}_${segment}` : segment)); if (meta.envVarAlias) { @@ -212,7 +161,11 @@ function collectMappings(description, names, path, root, mappings) { break; default: // arrays are leaves: their items have no name to derive from - registerMapping(names, path, description.type, mappings); + registerMapping(names, { + path, + type: description.type, + decode: annotations(description).envDecodeHook, + }, mappings); } } @@ -286,33 +239,20 @@ function applyEnvOverrides(config, schema, prefix = [], env = process.env) { applyLivenessProbePort(config, mappings, env[LIVENESS_PROBE_PORT]); } - mappings.forEach(({ path, type }, name) => { + mappings.forEach(({ path, type, decode }, name) => { if (env[name]) { - setField(config, path, coerceValue(env[name], type, name)); + // a field with a syntax of its own decodes the value itself: it defers + // by returning undefined, and throws to reject a value + const decoded = decode?.(env[name], env); + setField(config, path, decoded ?? coerceValue(env[name], type, name)); } }); return config; } -/** - * Applies the env vars setting several config fields at once, in place. - * - * @param {Object} config - backbeat configuration to update - * @param {Object} [env] - environment to read the overrides from - * @returns {Object} updated configuration - */ -function applyCompositeEnvOverrides(config, env = process.env) { - Object.entries(compositeEnvVars).forEach(([name, apply]) => { - if (env[name]) { - apply(config, env[name], env); - } - }); - return config; -} - module.exports = { - applyCompositeEnvOverrides, applyEnvOverrides, envVarMappings, + parseJSON, }; diff --git a/lib/config/fields.js b/lib/config/fields.js index 4a73bc9bc..f4133f174 100644 --- a/lib/config/fields.js +++ b/lib/config/fields.js @@ -36,24 +36,7 @@ function setField(config, path, value) { parent[path[path.length - 1]] = value; } -/** - * Removes a config field, leaving alone a path holding no section to remove it - * from. The field is deleted rather than set to `undefined`, which the schema - * accepts alike but carries the key over into the validated configuration. - * - * @param {Object} config - configuration to update - * @param {string[]} path - config path - * @returns {undefined} - */ -function deleteField(config, path) { - const parent = getField(config, path.slice(0, -1)); - if (parent && typeof parent === 'object') { - delete parent[path[path.length - 1]]; - } -} - module.exports = { - deleteField, getField, setField, }; diff --git a/tests/config.json b/tests/config.json index 08b95af54..05818ed96 100644 --- a/tests/config.json +++ b/tests/config.json @@ -252,7 +252,6 @@ "port": 8900 }, "redis": { - "name": "backbeat-test", "password": "", "host": "127.0.0.1", "port": 6379 diff --git a/tests/functional/queuePopulator/config/s3c-config.json b/tests/functional/queuePopulator/config/s3c-config.json index 627f79257..ee98d3070 100644 --- a/tests/functional/queuePopulator/config/s3c-config.json +++ b/tests/functional/queuePopulator/config/s3c-config.json @@ -163,7 +163,6 @@ }, "certFilePaths": {}, "redis": { - "name": "scality-s3", "password": "", "host": "127.0.0.1", "port": 6379 diff --git a/tests/unit/lib/config/config.joi.spec.js b/tests/unit/lib/config/config.joi.spec.js new file mode 100644 index 000000000..30ac17dcf --- /dev/null +++ b/tests/unit/lib/config/config.joi.spec.js @@ -0,0 +1,67 @@ +'use strict'; + +const assert = require('assert'); +const joi = require('joi'); + +const { backbeatConfigJoi } = require('../../../../lib/config.joi'); + +describe('backbeat config schema', () => { + describe('redis', () => { + const redisJoi = backbeatConfigJoi.extract('redis'); + const validate = redis => joi.attempt(redis, redisJoi); + + describe('sentinels', () => { + it('should default the group name of the master they watch', () => { + assert.deepStrictEqual(validate({ sentinels: 'host1:26379' }), { + sentinels: 'host1:26379', + name: 'mymaster', + password: '', + sentinelPassword: '', + }); + }); + + it('should keep the configured group name', () => { + assert.strictEqual(validate({ sentinels: 'host1:26379', name: 'group' }).name, 'group'); + }); + + it('should accept a list of host and port', () => { + const sentinels = [{ host: 'host1', port: 26379 }, { host: 'host2', port: 26379 }]; + + assert.deepStrictEqual(validate({ sentinels }).sentinels, sentinels); + }); + + // the two modes are exclusive: a deployment configures one of them + it('should reject the standalone host and port', () => { + assert.throws(() => validate({ sentinels: 'host1:26379', host: 'redis' }), + /"host" is not allowed/); + assert.throws(() => validate({ sentinels: 'host1:26379', port: 6380 }), + /"port" is not allowed/); + }); + }); + + describe('standalone', () => { + it('should default the port', () => { + assert.deepStrictEqual(validate({ host: 'redis' }), + { host: 'redis', port: 6379, password: '' }); + }); + + it('should keep the configured port', () => { + assert.strictEqual(validate({ host: 'redis', port: '6380' }).port, 6380); + }); + + it('should require a host', () => { + assert.throws(() => validate({}), /"host" is required/); + assert.throws(() => validate({ port: 6380 }), /"host" is required/); + }); + + // the group name and the sentinel password only mean something to + // the sentinels, and are not part of this mode + it('should reject the settings of the sentinels', () => { + assert.throws(() => validate({ host: 'redis', name: 'group' }), + /"name" is not allowed/); + assert.throws(() => validate({ host: 'redis', sentinelPassword: 'p' }), + /"sentinelPassword" is not allowed/); + }); + }); + }); +}); diff --git a/tests/unit/lib/config/envOverrides.spec.js b/tests/unit/lib/config/envOverrides.spec.js index cf9f5c407..8cf9df870 100644 --- a/tests/unit/lib/config/envOverrides.spec.js +++ b/tests/unit/lib/config/envOverrides.spec.js @@ -1,10 +1,13 @@ 'use strict'; const assert = require('assert'); +const fs = require('fs'); const joi = require('joi'); +const sinon = require('sinon'); + +const fileConfig = require('./config.json'); const { - applyCompositeEnvOverrides, applyEnvOverrides, envVarMappings, } = require('../../../../lib/config/envOverrides'); @@ -136,6 +139,12 @@ describe('config env var mapping', () => { const config = applyEnvOverrides({}, backbeatConfigJoi, [], { [name]: '::1' }); assert.deepStrictEqual(config.server.healthChecks.allowFrom, ['::1'], name); }); + + // renamed segment, under the alias of the section holding it + assert.strictEqual( + applyEnvOverrides({}, backbeatConfigJoi, [], { MONGODB_HOSTS: 'mongo1:27017' }) + .queuePopulator.mongo.replicaSetHosts, + 'mongo1:27017'); }); }); @@ -228,6 +237,30 @@ describe('config env var mapping', () => { }); }); + describe('field decoder', () => { + // a field with a syntax of its own, spanning a second variable + const schema = joi.object({ + servers: joi.array().items(joi.string()).meta({ + envDecodeHook: (value, env) => (value.startsWith('[') ? undefined + : [value, env.SERVERS_MORE].filter(more => more)), + }), + }); + const apply = env => applyEnvOverrides({}, schema, [], env); + + it('should build the value of the field', () => { + assert.deepStrictEqual(apply({ SERVERS: 'a:8000' }).servers, ['a:8000']); + }); + + it('should read the companion variable from the environment', () => { + assert.deepStrictEqual(apply({ SERVERS: 'a:8000', SERVERS_MORE: 'b:8000' }).servers, + ['a:8000', 'b:8000']); + }); + + it('should coerce the value the decoder defers on', () => { + assert.deepStrictEqual(apply({ SERVERS: '["a:8000"]' }).servers, ['a:8000']); + }); + }); + describe('liveness probe port', () => { const schema = joi.object({ queuePopulator: joi.object({ probeServer: probeServerJoi }), @@ -268,71 +301,6 @@ describe('config env var mapping', () => { }); }); -describe('composite config env vars', () => { - const apply = env => applyCompositeEnvOverrides({ redis: { host: 'localhost', port: 6379 } }, env); - - it('should set the sentinels group name, and drop the standalone host', () => { - assert.deepStrictEqual(apply({ REDIS_SENTINELS: 'host1:26379' }).redis, - { sentinels: 'host1:26379', name: 'mymaster' }); - assert.deepStrictEqual(apply({ REDIS_SENTINELS: 'host1:26379', REDIS_HA_NAME: 'group' }).redis, - { sentinels: 'host1:26379', name: 'group' }); - }); - - it('should default the standalone redis port', () => { - assert.deepStrictEqual(apply({ REDIS_HOST: 'redis' }).redis, { host: 'redis', port: '6379' }); - assert.deepStrictEqual(apply({ REDIS_HOST: 'redis', REDIS_PORT: '6380' }).redis, - { host: 'redis', port: '6380' }); - assert.deepStrictEqual(apply({ REDIS_PORT: '6380' }).redis, - { host: 'localhost', port: '6380' }); - }); - - it('should ignore the standalone redis host and port when sentinels are set', () => { - assert.deepStrictEqual( - apply({ REDIS_SENTINELS: 'host1:26379', REDIS_HOST: 'redis', REDIS_PORT: '6380' }).redis, - { sentinels: 'host1:26379', name: 'mymaster' }); - }); - - it('should set the replica set hosts, and leave the log source alone', () => { - const config = applyCompositeEnvOverrides({}, { MONGODB_HOSTS: 'mongo1:27017,mongo2:27017' }); - assert.deepStrictEqual(config.queuePopulator, { - mongo: { replicaSetHosts: 'mongo1:27017,mongo2:27017' }, - }); - }); - - it('should build the replication bootstrap list of a single site', () => { - const config = applyCompositeEnvOverrides({ extensions: { replication: {} } }, { - EXTENSIONS_REPLICATION_DEST_BOOTSTRAPLIST: 'zenko-1:8000', - }); - assert.deepStrictEqual(config.extensions.replication.destination.bootstrapList, - [{ site: 'zenko', servers: ['zenko-1:8000'] }]); - }); - - it('should split the servers of the replication bootstrap site', () => { - const config = applyCompositeEnvOverrides({ extensions: { replication: {} } }, { - EXTENSIONS_REPLICATION_DEST_BOOTSTRAPLIST: 'zenko-1:8000, zenko-2:8000', - }); - assert.deepStrictEqual(config.extensions.replication.destination.bootstrapList, - [{ site: 'zenko', servers: ['zenko-1:8000', 'zenko-2:8000'] }]); - }); - - it('should append the additional replication bootstrap sites', () => { - const config = applyCompositeEnvOverrides({ extensions: { replication: {} } }, { - EXTENSIONS_REPLICATION_DEST_BOOTSTRAPLIST: 'zenko-1:8000', - EXTENSIONS_REPLICATION_DEST_BOOTSTRAPLIST_MORE: '{ "site": "aws", "type": "aws_s3" }', - }); - assert.deepStrictEqual(config.extensions.replication.destination.bootstrapList, [ - { site: 'zenko', servers: ['zenko-1:8000'] }, - { site: 'aws', type: 'aws_s3' }, - ]); - }); - - it('should leave the bootstrap list alone when replication is not configured', () => { - assert.deepStrictEqual( - applyCompositeEnvOverrides({}, { EXTENSIONS_REPLICATION_DEST_BOOTSTRAPLIST: 'zenko-1:8000' }), - {}); - }); -}); - /** * The env var contract is consumed by zenko-operator and CI: every var the * docker entrypoint used to apply with jq must still set the same fields. @@ -468,7 +436,7 @@ describe('historic config env vars', () => { name: 'group', sentinels: [{ host: 'sentinel1', port: 26379 }, { host: 'sentinel2', port: 26379 }], }, - }], + }, { redis: {} }], [{ REDIS_HOST: 'redis' }, { 'redis.host': 'redis', 'redis.port': 6379 }], [{ REDIS_HOST: 'redis', REDIS_PORT: '6380' }, { 'redis.host': 'redis', 'redis.port': 6380 }], [{ QUEUE_POPULATOR_BATCH_MAX_READ: '42' }, { 'queuePopulator.batchMaxRead': 42 }], @@ -575,12 +543,25 @@ describe('historic config env vars', () => { } }); - function configWith(env) { + /** + * @param {Object} env - env vars of the case + * @param {Object} [sections] - config sections replacing those of the file, + * for a case the fixture cannot host as it stands + * @returns {Config} configuration built from the file and the environment + */ + function configWith(env, sections) { const og = Object.fromEntries(Object.keys(env).map(name => [name, process.env[name]])); Object.assign(process.env, env); + if (sections) { + sinon.stub(fs, 'readFileSync') + .callThrough() + .withArgs(process.env.BACKBEAT_CONFIG_FILE, sinon.match.any) + .returns(JSON.stringify({ ...fileConfig, ...sections })); + } try { return new Config(); } finally { + sinon.restore(); Object.entries(og).forEach(([name, value]) => { if (value === undefined) { delete process.env[name]; @@ -591,14 +572,19 @@ describe('historic config env vars', () => { } } - contract.forEach(([env, expected]) => { + contract.forEach(([env, expected, sections]) => { it(`should apply ${Object.keys(env).join(', ')}`, () => { - const config = configWith(env); + const config = configWith(env, sections); Object.entries(expected).forEach(([path, value]) => assert.deepStrictEqual(getField(config, path.split('.')), value, path)); }); }); + it('should reject the sentinels over a standalone configuration', () => { + assert.throws(() => configWith({ REDIS_SENTINELS: 'sentinel1:26379' }), + /"redis.host" is not allowed/); + }); + it('should account for every historic env var', () => { const covered = new Set([ ...contract.flatMap(([env]) => Object.keys(env)), diff --git a/tests/unit/replication/ReplicationConfigValidator.spec.js b/tests/unit/replication/ReplicationConfigValidator.spec.js index ce8c94f1a..1a6bafe1b 100644 --- a/tests/unit/replication/ReplicationConfigValidator.spec.js +++ b/tests/unit/replication/ReplicationConfigValidator.spec.js @@ -378,3 +378,51 @@ describe('ReplicationConfigValidator maxPollIntervalMs', () => { /less than or equal to 1800000/); }); }); + +describe('ReplicationConfigValidator bootstrapList from the environment', () => { + const bootstrapList = env => { + Object.assign(process.env, env); + try { + // the overrides are applied in place, make sure each case has its own copy + const config = JSON.parse(JSON.stringify(baseConfig)); + return configValidator({}, config).destination.bootstrapList; + } finally { + Object.keys(env).forEach(name => delete process.env[name]); + } + }; + + it('should hold the servers of the zenko site, comma separated', () => { + assert.deepStrictEqual( + bootstrapList({ EXTENSIONS_REPLICATION_DEST_BOOTSTRAPLIST: 'zenko-1:8000, zenko-2:8000' }), + [{ site: 'zenko', servers: ['zenko-1:8000', 'zenko-2:8000'], echo: false }]); + }); + + it('should append the additional sites', () => { + assert.deepStrictEqual( + bootstrapList({ + EXTENSIONS_REPLICATION_DEST_BOOTSTRAPLIST: 'zenko-1:8000', + EXTENSIONS_REPLICATION_DEST_BOOTSTRAPLIST_MORE: '{ "site": "aws", "type": "aws_s3" }', + }), + [{ site: 'zenko', servers: ['zenko-1:8000'], echo: false }, + { site: 'aws', type: 'aws_s3' }]); + }); + + it('should report an invalid additional site', () => { + assert.throws( + () => bootstrapList({ + EXTENSIONS_REPLICATION_DEST_BOOTSTRAPLIST: 'zenko-1:8000', + EXTENSIONS_REPLICATION_DEST_BOOTSTRAPLIST_MORE: '{ site: aws }', + }), + /invalid JSON value for EXTENSIONS_REPLICATION_DEST_BOOTSTRAPLIST_MORE/); + }); + + it('should take the whole list as JSON', () => { + assert.deepStrictEqual( + bootstrapList({ + EXTENSIONS_REPLICATION_DEST_BOOTSTRAPLIST: + '[{ "site": "zenko", "servers": ["zenko-1:8000"] }, { "site": "aws", "type": "aws_s3" }]', + }), + [{ site: 'zenko', servers: ['zenko-1:8000'], echo: false }, + { site: 'aws', type: 'aws_s3' }]); + }); +}); From 8eac56981389d96580e732eb4499f3c96bd6fa72 Mon Sep 17 00:00:00 2001 From: Francois Ferrand Date: Fri, 21 Aug 2026 23:19:44 +0200 Subject: [PATCH 6/8] Use joi to split redis sentinels host & default Issue: BB-808 --- lib/Config.js | 17 -------------- lib/config.joi.js | 19 +++++++++------ tests/unit/lib/config/config.joi.spec.js | 27 +++++++++++++++++++++- tests/unit/lib/config/envOverrides.spec.js | 3 +++ 4 files changed, 41 insertions(+), 25 deletions(-) diff --git a/lib/Config.js b/lib/Config.js index 58939719f..2a67c9398 100644 --- a/lib/Config.js +++ b/lib/Config.js @@ -123,23 +123,6 @@ class Config extends EventEmitter { healthChecks.allowFrom = healthChecks.allowFrom.concat(defaultHealthChecks); - if (parsedConfig.redis && - typeof parsedConfig.redis.sentinels === 'string') { - const redisConf = { sentinels: [], name: parsedConfig.redis.name }; - parsedConfig.redis.sentinels.split(',').forEach(item => { - const [host, port] = item.split(':'); - redisConf.sentinels.push({ host, - port: Number.parseInt(port, 10) }); - }); - parsedConfig.redis = redisConf; - } - - // default to standalone configuration if sentinel not setup - if (!parsedConfig.redis || !parsedConfig.redis.sentinels) { - this.redis = Object.assign({}, parsedConfig.redis, - { host: '127.0.0.1', port: 6379 }); - } - // additional certs checks if (parsedConfig.certFilePaths) { parsedConfig.https = this._parseCertFilePaths( diff --git a/lib/config.joi.js b/lib/config.joi.js index 647b3db86..b67937c2e 100644 --- a/lib/config.joi.js +++ b/lib/config.joi.js @@ -109,14 +109,19 @@ const joiSchema = joi.object({ redis: joi.alternatives().conditional(joi.ref('.sentinels'), { is: joi.exist(), then: joi.object({ - sentinels: joi.alternatives([joi.string(), joi.array().items( - joi.object({ + sentinels: joi.alternatives([ + // the comma separated form is parsed here, so that the validated + // configuration holds the list the redis client expects + joi.string().custom(value => value.split(',').map(sentinel => { + const [host, port] = sentinel.split(':'); + return { host, port: Number.parseInt(port, 10) }; + })), + joi.array().items(joi.object({ host: joi.string().required(), port: joi.number().required(), - }))] - ).required(), - // group name of the master the sentinels watch - name: joi.string().default('mymaster').meta({ env: 'HA_NAME' }), + })), + ]).required(), + name: joi.string().default('mymaster').meta({ env: 'HA_NAME' }), // sentinel master group password: joi.string().default('').allow(''), sentinelPassword: joi.string().default('').allow(''), }), @@ -125,7 +130,7 @@ const joiSchema = joi.object({ port: joi.number().default(6379), password: joi.string().default('').allow(''), }), - }), + }).default({ host: '127.0.0.1', port: 6379 }), certFilePaths: certFilePathsJoi, internalCertFilePaths: certFilePathsJoi, }); diff --git a/tests/unit/lib/config/config.joi.spec.js b/tests/unit/lib/config/config.joi.spec.js index 30ac17dcf..45ce5adc0 100644 --- a/tests/unit/lib/config/config.joi.spec.js +++ b/tests/unit/lib/config/config.joi.spec.js @@ -10,10 +10,17 @@ describe('backbeat config schema', () => { const redisJoi = backbeatConfigJoi.extract('redis'); const validate = redis => joi.attempt(redis, redisJoi); + // a configuration that does not mention redis reaches it locally, as + // the standalone section of the shipped configuration file does + it('should default to a local standalone server', () => { + assert.deepStrictEqual(joi.attempt({}, joi.object({ redis: redisJoi })), + { redis: { host: '127.0.0.1', port: 6379 } }); + }); + describe('sentinels', () => { it('should default the group name of the master they watch', () => { assert.deepStrictEqual(validate({ sentinels: 'host1:26379' }), { - sentinels: 'host1:26379', + sentinels: [{ host: 'host1', port: 26379 }], name: 'mymaster', password: '', sentinelPassword: '', @@ -24,6 +31,24 @@ describe('backbeat config schema', () => { assert.strictEqual(validate({ sentinels: 'host1:26379', name: 'group' }).name, 'group'); }); + // the redis client expects a list: a comma separated one is parsed + it('should parse the comma separated form', () => { + assert.deepStrictEqual( + validate({ sentinels: 'host1:26379,host2:26380' }).sentinels, + [{ host: 'host1', port: 26379 }, { host: 'host2', port: 26380 }]); + }); + + it('should keep the passwords of a comma separated form', () => { + const redis = { sentinels: 'host1:26379', password: 'p', sentinelPassword: 's' }; + + assert.deepStrictEqual(validate(redis), { + sentinels: [{ host: 'host1', port: 26379 }], + name: 'mymaster', + password: 'p', + sentinelPassword: 's', + }); + }); + it('should accept a list of host and port', () => { const sentinels = [{ host: 'host1', port: 26379 }, { host: 'host2', port: 26379 }]; diff --git a/tests/unit/lib/config/envOverrides.spec.js b/tests/unit/lib/config/envOverrides.spec.js index 8cf9df870..f64022e69 100644 --- a/tests/unit/lib/config/envOverrides.spec.js +++ b/tests/unit/lib/config/envOverrides.spec.js @@ -435,6 +435,9 @@ describe('historic config env vars', () => { redis: { name: 'group', sentinels: [{ host: 'sentinel1', port: 26379 }, { host: 'sentinel2', port: 26379 }], + // the passwords of the section survive the parsing of the list + password: '', + sentinelPassword: '', }, }, { redis: {} }], [{ REDIS_HOST: 'redis' }, { 'redis.host': 'redis', 'redis.port': 6379 }], From e4ef4c36d0da9cb2a7f1f47aec39223ebd08807e Mon Sep 17 00:00:00 2001 From: Francois Ferrand Date: Sat, 22 Aug 2026 00:37:29 +0200 Subject: [PATCH 7/8] Refactor config tests Move tests of each source file into its own spec, and ensure each module is tested at its own level. Issue: BB-808 --- tests/unit/conf/Config.js | 190 -------- tests/unit/lib/config/Config.spec.js | 441 ++++++++++++++++++ tests/unit/lib/config/config.joi.spec.js | 46 ++ tests/unit/lib/config/configItems.joi.spec.js | 118 +++++ tests/unit/lib/config/envOverrides.spec.js | 356 +------------- .../config/extensionConfigValidator.spec.js | 118 +++++ tests/unit/lib/config/fields.spec.js | 80 ++++ .../config}/replicationMultiDestConfig.json | 0 .../config}/replicationServersConfig.json | 0 9 files changed, 813 insertions(+), 536 deletions(-) delete mode 100644 tests/unit/conf/Config.js create mode 100644 tests/unit/lib/config/configItems.joi.spec.js create mode 100644 tests/unit/lib/config/extensionConfigValidator.spec.js create mode 100644 tests/unit/lib/config/fields.spec.js rename tests/unit/{conf/configs => lib/config}/replicationMultiDestConfig.json (100%) rename tests/unit/{conf/configs => lib/config}/replicationServersConfig.json (100%) diff --git a/tests/unit/conf/Config.js b/tests/unit/conf/Config.js deleted file mode 100644 index 17a71c28e..000000000 --- a/tests/unit/conf/Config.js +++ /dev/null @@ -1,190 +0,0 @@ -'use strict'; - -const assert = require('assert'); -const joi = require('joi'); -const sinon = require('sinon'); -const config = require('../../../lib/Config'); -const { Config } = require('../../../lib/Config'); -const { authJoi, inheritedAuthJoi } = require('../../../lib/config/configItems.joi'); - -describe('backbeat config parsing and validation', () => { - - it('should parse correctly the default config', () => { - assert.notStrictEqual(config, undefined); - }); - - describe('inherited auth', () => { - const schema = joi.object({ - auth: authJoi.optional(), - child: joi.object({ - auth: inheritedAuthJoi, - }), - }); - - const authObject = { - type: 'service', - account: 'account1', - }; - - it('fail if auth missing in both parent and child', () => { - const obj = { - child: {}, - }; - - assert(schema.validate(obj).error); - }); - - it('allow missing auth in child if defined in parent', () => { - const obj = { - auth: authObject, - child: {}, - }; - - return schema.validateAsync(obj); - }); - - it('allow missing auth in parent if defined in child', () => { - const obj = { - child: { - auth: authObject, - }, - }; - - return schema.validateAsync(obj); - }); - - it('allow auth in both parent and child', () => { - const obj = { - auth: authObject, - child: { - auth: authObject, - }, - }; - - return schema.validateAsync(obj); - }); - }); -}); - -describe('Site name', () => { - let conf; - - beforeEach(() => { - conf = new Config(); - }); - - afterEach(() => { - delete process.env.BOOTSTRAP_SITE_NAME; - }); - - it('should filter bootstrapList based on SITE_NAME', () => { - process.env.BOOTSTRAP_SITE_NAME = 'test-site-2'; - const expectedBootstrapList = conf.bootstrapList.filter(item => item.site === 'test-site-2'); - const newConfig = new Config(); - assert.deepStrictEqual(newConfig.bootstrapList, expectedBootstrapList); - }); - - it('should not filter bootstrapList if SITE_NAME is not set', () => { - const expectedBootstrapList = conf.bootstrapList; - const newConfig = new Config(); - assert.deepStrictEqual(newConfig.bootstrapList, expectedBootstrapList); - }); -}); - - -describe('Config', () => { - describe('getReplicationSiteDestConfig', () => { - let ogConfigFileEnv; - - before(() => { - ogConfigFileEnv = process.env.BACKBEAT_CONFIG_FILE; - }); - - afterEach(() => sinon.restore()); - - after(() => { - if (ogConfigFileEnv) { - process.env.BACKBEAT_CONFIG_FILE = ogConfigFileEnv; - } - }); - - describe('bootstrapList server normalization', () => { - let conf; - - before(() => { - process.env.BACKBEAT_CONFIG_FILE = `${__dirname}/configs/replicationServersConfig.json`; - conf = new Config(); - }); - - it('should normalize server entries with default port 443 for https transport', () => { - const entry = conf.bootstrapList.find(e => e.site === 'https-site'); - assert.deepStrictEqual(entry.servers, ['s3.example.com:443']); - }); - - it('should normalize server entries with default port 80 for http transport', () => { - const entry = conf.bootstrapList.find(e => e.site === 'http-site'); - assert.deepStrictEqual(entry.servers, ['s3.example.com:80']); - }); - - it('should preserve explicit port in server entries', () => { - const entry = conf.bootstrapList.find(e => e.site === 'explicit-port-site'); - assert.deepStrictEqual(entry.servers, ['s3.example.com:8443']); - }); - - it('should not modify endpoint without servers array', () => { - const entry = conf.bootstrapList.find(e => e.site === 'aws-site'); - assert.strictEqual(entry.servers, undefined); - assert.strictEqual(entry.type, 'aws_s3'); - }); - }); - - - describe('getReplicationSiteDestConfig', () => { - it('should return replication site destination config', () => { - process.env.BACKBEAT_CONFIG_FILE = `${__dirname}/configs/replicationMultiDestConfig.json`; - const conf = new Config(); - const destConfig = conf.getReplicationSiteDestConfig('aws3'); - assert.deepStrictEqual(destConfig, { - transport: 'https', - auth: { - type: 'service', - account: 'service-replication-3', - }, - replicationEndpoint: { - site: 'aws3', - type: 'aws_s3', - }, - }); - }); - - it('should return default replication destination config when site one is not available', () => { - process.env.BACKBEAT_CONFIG_FILE = `${__dirname}/configs/replicationMultiDestConfig.json`; - const conf = new Config(); - sinon.stub(conf.extensions.replication, 'destination').value({ - transport: 'https', - auth: { - type: 'service', - account: 'service-replication', - }, - bootstrapList: [ - { site: 'aws1', type: 'aws_s3' }, - { site: 'aws2', type: 'aws_s3' }, - { site: 'aws3', type: 'aws_s3' } - ] - }); - const destConfig = conf.getReplicationSiteDestConfig('aws3'); - assert.deepStrictEqual(destConfig, { - transport: 'https', - auth: { - type: 'service', - account: 'service-replication', - }, - replicationEndpoint: { - site: 'aws3', - type: 'aws_s3', - }, - }); - }); - }); - }); -}); diff --git a/tests/unit/lib/config/Config.spec.js b/tests/unit/lib/config/Config.spec.js index cf73ddade..67d6ba047 100644 --- a/tests/unit/lib/config/Config.spec.js +++ b/tests/unit/lib/config/Config.spec.js @@ -1,8 +1,11 @@ 'use strict'; const assert = require('assert'); +const fs = require('fs'); +const sinon = require('sinon'); const { Config } = require('../../../../lib/Config'); +const { getField } = require('../../../../lib/config/fields'); const backbeatConfig = require('./config.json'); describe('Config', () => { @@ -50,3 +53,441 @@ describe('Config', () => { assert.doesNotThrow(() => config._parseConfig(testConfig)); }); }); + +describe('backbeat config singleton', () => { + it('should parse the configuration file at require time', () => { + assert.notStrictEqual(require('../../../../lib/Config'), undefined); + }); +}); + +describe('Site name', () => { + let conf; + + beforeEach(() => { + conf = new Config(); + }); + + afterEach(() => { + delete process.env.BOOTSTRAP_SITE_NAME; + }); + + it('should filter bootstrapList based on SITE_NAME', () => { + process.env.BOOTSTRAP_SITE_NAME = 'test-site-2'; + const expectedBootstrapList = conf.bootstrapList.filter(item => item.site === 'test-site-2'); + const newConfig = new Config(); + assert.deepStrictEqual(newConfig.bootstrapList, expectedBootstrapList); + }); + + it('should not filter bootstrapList if SITE_NAME is not set', () => { + const expectedBootstrapList = conf.bootstrapList; + const newConfig = new Config(); + assert.deepStrictEqual(newConfig.bootstrapList, expectedBootstrapList); + }); +}); + + +describe('Config', () => { + describe('getReplicationSiteDestConfig', () => { + let ogConfigFileEnv; + + before(() => { + ogConfigFileEnv = process.env.BACKBEAT_CONFIG_FILE; + }); + + afterEach(() => sinon.restore()); + + after(() => { + if (ogConfigFileEnv) { + process.env.BACKBEAT_CONFIG_FILE = ogConfigFileEnv; + } + }); + + describe('bootstrapList server normalization', () => { + let conf; + + before(() => { + process.env.BACKBEAT_CONFIG_FILE = `${__dirname}/replicationServersConfig.json`; + conf = new Config(); + }); + + it('should normalize server entries with default port 443 for https transport', () => { + const entry = conf.bootstrapList.find(e => e.site === 'https-site'); + assert.deepStrictEqual(entry.servers, ['s3.example.com:443']); + }); + + it('should normalize server entries with default port 80 for http transport', () => { + const entry = conf.bootstrapList.find(e => e.site === 'http-site'); + assert.deepStrictEqual(entry.servers, ['s3.example.com:80']); + }); + + it('should preserve explicit port in server entries', () => { + const entry = conf.bootstrapList.find(e => e.site === 'explicit-port-site'); + assert.deepStrictEqual(entry.servers, ['s3.example.com:8443']); + }); + + it('should not modify endpoint without servers array', () => { + const entry = conf.bootstrapList.find(e => e.site === 'aws-site'); + assert.strictEqual(entry.servers, undefined); + assert.strictEqual(entry.type, 'aws_s3'); + }); + }); + + + describe('getReplicationSiteDestConfig', () => { + it('should return replication site destination config', () => { + process.env.BACKBEAT_CONFIG_FILE = `${__dirname}/replicationMultiDestConfig.json`; + const conf = new Config(); + const destConfig = conf.getReplicationSiteDestConfig('aws3'); + assert.deepStrictEqual(destConfig, { + transport: 'https', + auth: { + type: 'service', + account: 'service-replication-3', + }, + replicationEndpoint: { + site: 'aws3', + type: 'aws_s3', + }, + }); + }); + + it('should return default replication destination config when site one is not available', () => { + process.env.BACKBEAT_CONFIG_FILE = `${__dirname}/replicationMultiDestConfig.json`; + const conf = new Config(); + sinon.stub(conf.extensions.replication, 'destination').value({ + transport: 'https', + auth: { + type: 'service', + account: 'service-replication', + }, + bootstrapList: [ + { site: 'aws1', type: 'aws_s3' }, + { site: 'aws2', type: 'aws_s3' }, + { site: 'aws3', type: 'aws_s3' } + ] + }); + const destConfig = conf.getReplicationSiteDestConfig('aws3'); + assert.deepStrictEqual(destConfig, { + transport: 'https', + auth: { + type: 'service', + account: 'service-replication', + }, + replicationEndpoint: { + site: 'aws3', + type: 'aws_s3', + }, + }); + }); + }); + }); +}); + +/** + * The env var contract is consumed by zenko-operator and CI: every var the + * docker entrypoint used to apply with jq must still set the same fields. + */ +describe('historic config env vars', () => { + /** + * Every name the entrypoint applied with jq, plus the two lib/Config.js + * applied on its own. zenko-operator sets some of them, and Federation + * forwards arbitrary ones through `env_backbeat_extraenv2`, so the whole + * list has to keep working. Each name is checked below to be either + * covered by a contract case, or explicitly removed. + */ + const historicEnvVars = [ + 'CLOUDSERVER_HOST', + 'CLOUDSERVER_PORT', + 'EXTENSIONS_GC_TOPIC', + 'EXTENSIONS_INGESTION_AUTH_ACCOUNT', + 'EXTENSIONS_INGESTION_AUTH_TYPE', + 'EXTENSIONS_INGESTION_MAX_PARALLEL_READERS', + 'EXTENSIONS_LIFECYCLE_AUTH_ACCOUNT', + 'EXTENSIONS_LIFECYCLE_AUTH_TYPE', + 'EXTENSIONS_LIFECYCLE_BUCKET_PROCESSOR_GROUP_ID', + 'EXTENSIONS_LIFECYCLE_BUCKET_TASK_TOPIC', + 'EXTENSIONS_LIFECYCLE_CONDUCTOR_CRONRULE', + 'EXTENSIONS_LIFECYCLE_OBJECT_PROCESSOR_GROUP_ID', + 'EXTENSIONS_LIFECYCLE_OBJECT_TASK_TOPIC', + 'EXTENSIONS_LIFECYCLE_RULES_ABORT_INCOMPLETE_MPU_ENABLED', + 'EXTENSIONS_LIFECYCLE_RULES_EXPIRATION_ENABLED', + 'EXTENSIONS_LIFECYCLE_RULES_NC_VERSION_EXPIRATION_ENABLED', + 'EXTENSIONS_LIFECYCLE_RULES_TRANSITIONS_ENABLED', + 'EXTENSIONS_LIFECYCLE_ZOOKEEPER_PATH', + 'EXTENSIONS_REPLICATION_DEST_AUTH_ACCOUNT', + 'EXTENSIONS_REPLICATION_DEST_AUTH_TYPE', + 'EXTENSIONS_REPLICATION_DEST_BOOTSTRAPLIST', + 'EXTENSIONS_REPLICATION_DEST_BOOTSTRAPLIST_MORE', + 'EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_AWS_S3_RETRY_BACKOFF_FACTOR', + 'EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_AWS_S3_RETRY_BACKOFF_JITTER', + 'EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_AWS_S3_RETRY_BACKOFF_MAX', + 'EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_AWS_S3_RETRY_BACKOFF_MIN', + 'EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_AWS_S3_RETRY_MAX_RETRIES', + 'EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_AWS_S3_RETRY_TIMEOUT_S', + 'EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_AZURE_RETRY_BACKOFF_FACTOR', + 'EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_AZURE_RETRY_BACKOFF_JITTER', + 'EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_AZURE_RETRY_BACKOFF_MAX', + 'EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_AZURE_RETRY_BACKOFF_MIN', + 'EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_AZURE_RETRY_MAX_RETRIES', + 'EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_AZURE_RETRY_TIMEOUT_S', + 'EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_CONCURRENCY', + 'EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_GCP_RETRY_BACKOFF_FACTOR', + 'EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_GCP_RETRY_BACKOFF_JITTER', + 'EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_GCP_RETRY_BACKOFF_MAX', + 'EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_GCP_RETRY_BACKOFF_MIN', + 'EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_GCP_RETRY_MAX_RETRIES', + 'EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_GCP_RETRY_TIMEOUT_S', + 'EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_MAX_POLL_INTERVAL_MS', + 'EXTENSIONS_REPLICATION_SOURCE_AUTH_ACCOUNT', + 'EXTENSIONS_REPLICATION_SOURCE_AUTH_TYPE', + 'EXTENSIONS_REPLICATION_SOURCE_S3_HOST', + 'EXTENSIONS_REPLICATION_SOURCE_S3_PORT', + 'EXTENSIONS_REPLICATION_STATUS_PROCESSOR_CONCURRENCY', + 'HEALTHCHECKS_ALLOWFROM', + 'KAFKA_BACKLOG_METRICS_INTERVALS', + 'KAFKA_BACKLOG_METRICS_ZKPATH', + 'KAFKA_HOSTS', + 'LIVENESS_PROBE_PORT', + 'LOG_LEVEL', + 'MONGODB_AUTH_PASSWORD', + 'MONGODB_AUTH_USERNAME', + 'MONGODB_DATABASE', + 'MONGODB_HOSTS', + 'MONGODB_RS', + 'QUEUE_POPULATOR_BATCH_MAX_READ', + 'QUEUE_POPULATOR_DMD_HOST', + 'QUEUE_POPULATOR_DMD_PORT', + 'REDIS_HA_NAME', + 'REDIS_HOST', + 'REDIS_LOCALCACHE_HOST', + 'REDIS_LOCALCACHE_PORT', + 'REDIS_PORT', + 'REDIS_SENTINELS', + 'REPLICATION_GROUP_ID', + 'ZOOKEEPER_AUTO_CREATE_NAMESPACE', + 'ZOOKEEPER_CONNECTION_STRING', + ]; + + // the lifecycle rules are configured with supportedLifecycleRules, and the + // local cache is not part of the configuration schema + const removedEnvVars = [ + 'EXTENSIONS_LIFECYCLE_RULES_ABORT_INCOMPLETE_MPU_ENABLED', + 'EXTENSIONS_LIFECYCLE_RULES_EXPIRATION_ENABLED', + 'EXTENSIONS_LIFECYCLE_RULES_NC_VERSION_EXPIRATION_ENABLED', + 'EXTENSIONS_LIFECYCLE_RULES_TRANSITIONS_ENABLED', + 'REDIS_LOCALCACHE_HOST', + 'REDIS_LOCALCACHE_PORT', + ]; + + const retryFields = backend => ({ + [`EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_${backend}_RETRY_MAX_RETRIES`]: '1', + [`EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_${backend}_RETRY_TIMEOUT_S`]: '2', + [`EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_${backend}_RETRY_BACKOFF_MIN`]: '3', + [`EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_${backend}_RETRY_BACKOFF_MAX`]: '4', + [`EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_${backend}_RETRY_BACKOFF_JITTER`]: '0.5', + [`EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_${backend}_RETRY_BACKOFF_FACTOR`]: '6', + }); + const retryExpected = backend => ({ + [`extensions.replication.queueProcessor.retry.${backend}.maxRetries`]: 1, + [`extensions.replication.queueProcessor.retry.${backend}.timeoutS`]: 2, + [`extensions.replication.queueProcessor.retry.${backend}.backoff`]: + { min: 3, max: 4, jitter: 0.5, factor: 6 }, + }); + + const probeServer = { bindAddress: '0.0.0.0', port: 8100 }; + const contract = [ + [{ LIVENESS_PROBE_PORT: '8100' }, { + 'queuePopulator.probeServer': probeServer, + 'extensions.ingestion.probeServer': probeServer, + 'extensions.mongoProcessor.probeServer': probeServer, + 'extensions.replication.queueProcessor.probeServer': probeServer, + 'extensions.replication.replicationStatusProcessor.probeServer': probeServer, + 'extensions.lifecycle.conductor.probeServer': probeServer, + 'extensions.lifecycle.bucketProcessor.probeServer': probeServer, + 'extensions.lifecycle.objectProcessor.probeServer': probeServer, + 'extensions.gc.probeServer': probeServer, + }], + [{ LOG_LEVEL: 'debug' }, { 'log.logLevel': 'debug' }], + [{ ZOOKEEPER_AUTO_CREATE_NAMESPACE: 'true' }, { 'zookeeper.autoCreateNamespace': true }], + [{ ZOOKEEPER_CONNECTION_STRING: 'zk:2181/bb' }, { 'zookeeper.connectionString': 'zk:2181/bb' }], + [{ KAFKA_HOSTS: 'kafka:9092' }, { 'kafka.hosts': 'kafka:9092' }], + [{ KAFKA_BACKLOG_METRICS_ZKPATH: '/bb/metrics' }, { 'kafka.backlogMetrics.zkPath': '/bb/metrics' }], + [{ KAFKA_BACKLOG_METRICS_INTERVALS: '30' }, { 'kafka.backlogMetrics.intervalS': 30 }], + [{ REDIS_SENTINELS: 'sentinel1:26379,sentinel2:26379', REDIS_HA_NAME: 'group' }, { + redis: { + name: 'group', + sentinels: [{ host: 'sentinel1', port: 26379 }, { host: 'sentinel2', port: 26379 }], + // the passwords of the section survive the parsing of the list + password: '', + sentinelPassword: '', + }, + }, { redis: {} }], + [{ REDIS_HOST: 'redis' }, { 'redis.host': 'redis', 'redis.port': 6379 }], + [{ REDIS_HOST: 'redis', REDIS_PORT: '6380' }, { 'redis.host': 'redis', 'redis.port': 6380 }], + [{ QUEUE_POPULATOR_BATCH_MAX_READ: '42' }, { 'queuePopulator.batchMaxRead': 42 }], + [{ QUEUE_POPULATOR_DMD_HOST: 'dmd', QUEUE_POPULATOR_DMD_PORT: '9991' }, { + 'queuePopulator.dmd.host': 'dmd', + 'queuePopulator.dmd.port': 9991, + }], + [{ MONGODB_HOSTS: 'mongo1:27017,mongo2:27017' }, { + 'queuePopulator.mongo.replicaSetHosts': 'mongo1:27017,mongo2:27017', + }], + [{ MONGODB_RS: 'rs1' }, { 'queuePopulator.mongo.replicaSet': 'rs1' }], + [{ MONGODB_DATABASE: 'db' }, { 'queuePopulator.mongo.database': 'db' }], + [{ MONGODB_AUTH_USERNAME: 'user', MONGODB_AUTH_PASSWORD: 'pass' }, { + 'queuePopulator.mongo.authCredentials': { username: 'user', password: 'pass' }, + }], + [{ CLOUDSERVER_HOST: 'cloudserver', CLOUDSERVER_PORT: '8001' }, { + 's3.host': 'cloudserver', + 's3.port': 8001, + }], + [{ HEALTHCHECKS_ALLOWFROM: '10.0.0.0/8' }, { + // the loopback addresses are always allowed + 'server.healthChecks.allowFrom': ['10.0.0.0/8', '127.0.0.1/8', '::1'], + }], + [{ REPLICATION_GROUP_ID: 'RG00002' }, { replicationGroupId: 'RG00002' }], + [{ + EXTENSIONS_REPLICATION_SOURCE_S3_HOST: 'cloudserver', + EXTENSIONS_REPLICATION_SOURCE_S3_PORT: '8001', + }, { + 'extensions.replication.source.s3.host': 'cloudserver', + 'extensions.replication.source.s3.port': 8001, + }], + [{ + EXTENSIONS_REPLICATION_SOURCE_AUTH_TYPE: 'account', + EXTENSIONS_REPLICATION_SOURCE_AUTH_ACCOUNT: 'source-account', + }, { + 'extensions.replication.source.auth.type': 'account', + 'extensions.replication.source.auth.account': 'source-account', + }], + [{ + EXTENSIONS_REPLICATION_DEST_AUTH_TYPE: 'account', + EXTENSIONS_REPLICATION_DEST_AUTH_ACCOUNT: 'dest-account', + }, { + 'extensions.replication.destination.auth.type': 'account', + 'extensions.replication.destination.auth.account': 'dest-account', + }], + [{ EXTENSIONS_REPLICATION_DEST_BOOTSTRAPLIST: 'zenko-1:8000' }, { + 'extensions.replication.destination.bootstrapList': + [{ site: 'zenko', servers: ['zenko-1:8000'], echo: false }], + }], + [{ + EXTENSIONS_REPLICATION_DEST_BOOTSTRAPLIST: 'zenko-1:8000', + EXTENSIONS_REPLICATION_DEST_BOOTSTRAPLIST_MORE: '{ "site": "aws", "type": "aws_s3" }', + }, { + 'extensions.replication.destination.bootstrapList': [ + { site: 'zenko', servers: ['zenko-1:8000'], echo: false }, + { site: 'aws', type: 'aws_s3' }, + ], + }], + [retryFields('AWS_S3'), retryExpected('aws_s3')], + [retryFields('AZURE'), retryExpected('azure')], + [retryFields('GCP'), retryExpected('gcp')], + [{ EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_CONCURRENCY: '11' }, + { 'extensions.replication.queueProcessor.concurrency': 11 }], + [{ EXTENSIONS_REPLICATION_STATUS_PROCESSOR_CONCURRENCY: '7' }, + { 'extensions.replication.replicationStatusProcessor.concurrency': 7 }], + [{ EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_MAX_POLL_INTERVAL_MS: '60000' }, + { 'extensions.replication.queueProcessor.maxPollIntervalMs': 60000 }], + [{ EXTENSIONS_LIFECYCLE_ZOOKEEPER_PATH: '/lc' }, { 'extensions.lifecycle.zookeeperPath': '/lc' }], + [{ EXTENSIONS_LIFECYCLE_BUCKET_TASK_TOPIC: 'lc-buckets' }, + { 'extensions.lifecycle.bucketTasksTopic': 'lc-buckets' }], + [{ EXTENSIONS_LIFECYCLE_OBJECT_TASK_TOPIC: 'lc-objects' }, + { 'extensions.lifecycle.objectTasksTopic': 'lc-objects' }], + [{ EXTENSIONS_LIFECYCLE_CONDUCTOR_CRONRULE: '0 0 * * * *' }, + { 'extensions.lifecycle.conductor.cronRule': '0 0 * * * *' }], + [{ EXTENSIONS_LIFECYCLE_BUCKET_PROCESSOR_GROUP_ID: 'lc-bucket-group' }, + { 'extensions.lifecycle.bucketProcessor.groupId': 'lc-bucket-group' }], + [{ EXTENSIONS_LIFECYCLE_OBJECT_PROCESSOR_GROUP_ID: 'lc-object-group' }, + { 'extensions.lifecycle.objectProcessor.groupId': 'lc-object-group' }], + [{ EXTENSIONS_LIFECYCLE_AUTH_TYPE: 'account', EXTENSIONS_LIFECYCLE_AUTH_ACCOUNT: 'lc-account' }, { + 'extensions.lifecycle.auth.type': 'account', + 'extensions.lifecycle.auth.account': 'lc-account', + }], + [{ EXTENSIONS_GC_TOPIC: 'gc-topic' }, { 'extensions.gc.topic': 'gc-topic' }], + [{ EXTENSIONS_INGESTION_AUTH_TYPE: 'service', EXTENSIONS_INGESTION_AUTH_ACCOUNT: 'ingest' }, { + 'extensions.ingestion.auth.type': 'service', + 'extensions.ingestion.auth.account': 'ingest', + }], + [{ EXTENSIONS_INGESTION_MAX_PARALLEL_READERS: '3' }, + { 'extensions.ingestion.maxParallelReaders': 3 }], + ]; + + let ogConfigFile; + + before(() => { + ogConfigFile = process.env.BACKBEAT_CONFIG_FILE; + process.env.BACKBEAT_CONFIG_FILE = `${__dirname}/config.json`; + }); + + after(() => { + if (ogConfigFile === undefined) { + delete process.env.BACKBEAT_CONFIG_FILE; + } else { + process.env.BACKBEAT_CONFIG_FILE = ogConfigFile; + } + }); + + /** + * @param {Object} env - env vars of the case + * @param {Object} [sections] - config sections replacing those of the file, + * for a case the fixture cannot host as it stands + * @returns {Config} configuration built from the file and the environment + */ + function configWith(env, sections) { + const og = Object.fromEntries(Object.keys(env).map(name => [name, process.env[name]])); + Object.assign(process.env, env); + if (sections) { + sinon.stub(fs, 'readFileSync') + .callThrough() + .withArgs(process.env.BACKBEAT_CONFIG_FILE, sinon.match.any) + .returns(JSON.stringify({ ...backbeatConfig, ...sections })); + } + try { + return new Config(); + } finally { + sinon.restore(); + Object.entries(og).forEach(([name, value]) => { + if (value === undefined) { + delete process.env[name]; + } else { + process.env[name] = value; + } + }); + } + } + + contract.forEach(([env, expected, sections]) => { + it(`should apply ${Object.keys(env).join(', ')}`, () => { + const config = configWith(env, sections); + Object.entries(expected).forEach(([path, value]) => + assert.deepStrictEqual(getField(config, path.split('.')), value, path)); + }); + }); + + it('should reject the sentinels over a standalone configuration', () => { + assert.throws(() => configWith({ REDIS_SENTINELS: 'sentinel1:26379' }), + /"redis.host" is not allowed/); + }); + + it('should account for every historic env var', () => { + const covered = new Set([ + ...contract.flatMap(([env]) => Object.keys(env)), + ...removedEnvVars, + ]); + + assert.deepStrictEqual(historicEnvVars.filter(name => !covered.has(name)), []); + }); + + it('should accept the log levels the entrypoint used to reject', () => { + assert.strictEqual(configWith({ LOG_LEVEL: 'warn' }).log.logLevel, 'warn'); + assert.strictEqual(configWith({ LOG_LEVEL: 'error' }).log.logLevel, 'error'); + }); + + it('should ignore the vars removed with the entrypoint', () => { + const config = configWith(Object.fromEntries(removedEnvVars.map(name => [name, 'redis']))); + + assert.strictEqual(config.extensions.lifecycle.rules, undefined); + assert.strictEqual(config.localCache, undefined); + }); +}); diff --git a/tests/unit/lib/config/config.joi.spec.js b/tests/unit/lib/config/config.joi.spec.js index 45ce5adc0..61bab3a2d 100644 --- a/tests/unit/lib/config/config.joi.spec.js +++ b/tests/unit/lib/config/config.joi.spec.js @@ -4,8 +4,54 @@ const assert = require('assert'); const joi = require('joi'); const { backbeatConfigJoi } = require('../../../../lib/config.joi'); +const { envVarMappings } = require('../../../../lib/config/envOverrides'); describe('backbeat config schema', () => { + it('should derive no name for the params BackbeatProducer sets itself', () => { + const names = [...envVarMappings(backbeatConfigJoi).keys()]; + + assert.ok(!names.some(name => name.startsWith('KAFKA_PRODUCER_PARAMS')), names.join(', ')); + }); + + describe('queuePopulator', () => { + // the probe server of the populator is conditioned by `...extensions`: + // the section is validated under a parent holding one + const schema = joi.object({ + queuePopulator: backbeatConfigJoi.extract('queuePopulator'), + extensions: joi.object(), + }); + const base = { + auth: { type: 'none', vault: { host: 'vault', port: 8500 } }, + cronRule: '* * * * *', + zookeeperPath: '/backbeat', + probeServer: { port: 8550 }, + }; + const validate = queuePopulator => + schema.validate({ queuePopulator: { ...base, ...queuePopulator }, extensions: { gc: {} } }); + + // the log source names the section the populator reads the oplog from + it('should require the section of its log source', () => { + assert.match(validate({ logSource: 'bucketd' }).error.message, /"queuePopulator.bucketd" is required/); + assert.match(validate({ logSource: 'dmd' }).error.message, /"queuePopulator.dmd" is required/); + assert.match(validate({ logSource: 'kafka' }).error.message, /"queuePopulator.kafka" is required/); + }); + + it('should accept the log source its section configures', () => { + assert.strictEqual(validate({ logSource: 'dmd', dmd: { host: 'dmd', port: 9990 } }).error, + undefined); + }); + + // the ingestion reader is configured by the extension + it('should need no section for the ingestion log source', () => { + assert.strictEqual(validate({ logSource: 'ingestion' }).error, undefined); + }); + + it('should reject a log source it cannot read', () => { + assert.match(validate({ logSource: 'mongo' }).error.message, + /"queuePopulator.logSource" must be one of \[bucketd, ingestion, dmd, kafka\]/); + }); + }); + describe('redis', () => { const redisJoi = backbeatConfigJoi.extract('redis'); const validate = redis => joi.attempt(redis, redisJoi); diff --git a/tests/unit/lib/config/configItems.joi.spec.js b/tests/unit/lib/config/configItems.joi.spec.js new file mode 100644 index 000000000..291539445 --- /dev/null +++ b/tests/unit/lib/config/configItems.joi.spec.js @@ -0,0 +1,118 @@ +'use strict'; + +const assert = require('assert'); +const joi = require('joi'); + +const { + authJoi, + inheritedAuthJoi, + logJoi, + logJoiOptional, + mongoJoi, +} = require('../../../../lib/config/configItems.joi'); +const { envVarMappings } = require('../../../../lib/config/envOverrides'); + +describe('config items schemas', () => { + describe('inherited auth', () => { + const schema = joi.object({ + auth: authJoi.optional(), + child: joi.object({ + auth: inheritedAuthJoi, + }), + }); + + const authObject = { + type: 'service', + account: 'account1', + }; + + it('fail if auth missing in both parent and child', () => { + const obj = { + child: {}, + }; + + assert(schema.validate(obj).error); + }); + + it('allow missing auth in child if defined in parent', () => { + const obj = { + auth: authObject, + child: {}, + }; + + return schema.validateAsync(obj); + }); + + it('allow missing auth in parent if defined in child', () => { + const obj = { + child: { + auth: authObject, + }, + }; + + return schema.validateAsync(obj); + }); + + it('allow auth in both parent and child', () => { + const obj = { + auth: authObject, + child: { + auth: authObject, + }, + }; + + return schema.validateAsync(obj); + }); + }); + + describe('log', () => { + // the levels default individually, so that setting one of them leaves + // the other alone + it('should default each level of the global config', () => { + assert.deepStrictEqual(joi.attempt({}, logJoi), { logLevel: 'info', dumpLevel: 'error' }); + assert.deepStrictEqual(joi.attempt({ logLevel: 'debug' }, logJoi), + { logLevel: 'debug', dumpLevel: 'error' }); + }); + + // an extension inherits the levels it does not configure from the + // global log config, passed as the validation context + it('should inherit the levels an extension does not configure', () => { + const context = { log: { logLevel: 'info', dumpLevel: 'trace' } }; + const validate = log => joi.attempt(log, logJoiOptional, { context }); + + assert.deepStrictEqual(validate({}), { logLevel: 'info', dumpLevel: 'trace' }); + assert.deepStrictEqual(validate({ logLevel: 'debug' }), + { logLevel: 'debug', dumpLevel: 'trace' }); + }); + + it('should name the level LOG_LEVEL rather than LOG_LOG_LEVEL', () => { + assert.deepStrictEqual([...envVarMappings(joi.object({ log: logJoi })).keys()], + ['LOG_LEVEL', 'LOG_DUMP_LEVEL']); + }); + }); + + describe('mongo', () => { + // the historic names of the fields, which their path does not derive + it('should name the replica set fields as the entrypoint did', () => { + const names = [...envVarMappings(joi.object({ + queuePopulator: joi.object({ mongo: mongoJoi.meta({ envVarAlias: 'MONGODB' }) }), + })).keys()]; + + ['MONGODB_HOSTS', 'MONGODB_RS', 'MONGODB_AUTH_USERNAME', 'MONGODB_DATABASE'] + .forEach(name => assert.ok(names.includes(name), `${name} in ${names.join(', ')}`)); + }); + + it('should default the replica set hosts and the database', () => { + const mongo = joi.attempt({}, mongoJoi); + + assert.strictEqual(mongo.replicaSetHosts, 'localhost:27017'); + assert.strictEqual(mongo.database, 'metadata'); + assert.strictEqual(mongo.replicaSet, 'rs0'); + }); + + it('should forbid the replica set of a sharded collection', () => { + assert.throws(() => joi.attempt({ shardCollections: true, replicaSet: 'rs0' }, mongoJoi), + /"replicaSet" is not allowed/); + }); + }); +}); diff --git a/tests/unit/lib/config/envOverrides.spec.js b/tests/unit/lib/config/envOverrides.spec.js index f64022e69..4eaf8a025 100644 --- a/tests/unit/lib/config/envOverrides.spec.js +++ b/tests/unit/lib/config/envOverrides.spec.js @@ -1,24 +1,16 @@ 'use strict'; const assert = require('assert'); -const fs = require('fs'); const joi = require('joi'); -const sinon = require('sinon'); - -const fileConfig = require('./config.json'); const { applyEnvOverrides, envVarMappings, } = require('../../../../lib/config/envOverrides'); -const { backbeatConfigJoi } = require('../../../../lib/config.joi'); const { - logJoi, probeServerJoi, probeServerPerSite, } = require('../../../../lib/config/configItems.joi'); -const { Config } = require('../../../../lib/Config'); -const { getField } = require('../../../../lib/config/fields'); describe('config env var mapping', () => { it('should derive names from the config path', () => { @@ -67,12 +59,6 @@ describe('config env var mapping', () => { assert.deepStrictEqual([...envVarMappings(schema).keys()], ['KAFKA_SITE']); }); - it('should not derive a name for the params BackbeatProducer sets itself', () => { - const names = [...envVarMappings(backbeatConfigJoi).keys()]; - - assert.ok(!names.some(name => name.startsWith('KAFKA_PRODUCER_PARAMS')), names.join(', ')); - }); - it('should reject a schema deriving the same name for two fields', () => { // eslint-disable-next-line camelcase const schema = joi.object({ logLevel: joi.string(), log_level: joi.string() }); @@ -126,26 +112,6 @@ describe('config env var mapping', () => { ['CLOUDSERVER_HOST', 'VAULT_ADMIN_HOST']); }); - it('should apply the annotated names of the backbeat schema', () => { - // renamed by env: the derived LOG_LOG_LEVEL is replaced - assert.strictEqual( - applyEnvOverrides({}, backbeatConfigJoi, [], { LOG_LEVEL: 'warn' }).log.logLevel, - 'warn'); - assert.deepStrictEqual( - applyEnvOverrides({}, backbeatConfigJoi, [], { LOG_LOG_LEVEL: 'warn' }), {}); - - // aliased: both names are honored - ['HEALTHCHECKS_ALLOWFROM', 'SERVER_HEALTH_CHECKS_ALLOW_FROM'].forEach(name => { - const config = applyEnvOverrides({}, backbeatConfigJoi, [], { [name]: '::1' }); - assert.deepStrictEqual(config.server.healthChecks.allowFrom, ['::1'], name); - }); - - // renamed segment, under the alias of the section holding it - assert.strictEqual( - applyEnvOverrides({}, backbeatConfigJoi, [], { MONGODB_HOSTS: 'mongo1:27017' }) - .queuePopulator.mongo.replicaSetHosts, - 'mongo1:27017'); - }); }); describe('value injection', () => { @@ -231,9 +197,16 @@ describe('config env var mapping', () => { // a partially set object is what setting a single field of one yields it('should leave the schema to complete a partially set object', () => { - const config = applyEnvOverrides({}, backbeatConfigJoi, [], { LOG_LEVEL: 'warn' }); - assert.deepStrictEqual(joi.attempt(config.log, logJoi), - { logLevel: 'warn', dumpLevel: 'error' }); + const section = joi.object({ + log: joi.object({ + level: joi.string().default('info'), + dump: joi.string().default('error'), + }), + }); + + assert.deepStrictEqual( + joi.attempt(applyEnvOverrides({}, section, [], { LOG_LEVEL: 'warn' }), section), + { log: { level: 'warn', dump: 'error' } }); }); }); @@ -300,312 +273,3 @@ describe('config env var mapping', () => { }); }); }); - -/** - * The env var contract is consumed by zenko-operator and CI: every var the - * docker entrypoint used to apply with jq must still set the same fields. - */ -describe('historic config env vars', () => { - /** - * Every name the entrypoint applied with jq, plus the two lib/Config.js - * applied on its own. zenko-operator sets some of them, and Federation - * forwards arbitrary ones through `env_backbeat_extraenv2`, so the whole - * list has to keep working. Each name is checked below to be either - * covered by a contract case, or explicitly removed. - */ - const historicEnvVars = [ - 'CLOUDSERVER_HOST', - 'CLOUDSERVER_PORT', - 'EXTENSIONS_GC_TOPIC', - 'EXTENSIONS_INGESTION_AUTH_ACCOUNT', - 'EXTENSIONS_INGESTION_AUTH_TYPE', - 'EXTENSIONS_INGESTION_MAX_PARALLEL_READERS', - 'EXTENSIONS_LIFECYCLE_AUTH_ACCOUNT', - 'EXTENSIONS_LIFECYCLE_AUTH_TYPE', - 'EXTENSIONS_LIFECYCLE_BUCKET_PROCESSOR_GROUP_ID', - 'EXTENSIONS_LIFECYCLE_BUCKET_TASK_TOPIC', - 'EXTENSIONS_LIFECYCLE_CONDUCTOR_CRONRULE', - 'EXTENSIONS_LIFECYCLE_OBJECT_PROCESSOR_GROUP_ID', - 'EXTENSIONS_LIFECYCLE_OBJECT_TASK_TOPIC', - 'EXTENSIONS_LIFECYCLE_RULES_ABORT_INCOMPLETE_MPU_ENABLED', - 'EXTENSIONS_LIFECYCLE_RULES_EXPIRATION_ENABLED', - 'EXTENSIONS_LIFECYCLE_RULES_NC_VERSION_EXPIRATION_ENABLED', - 'EXTENSIONS_LIFECYCLE_RULES_TRANSITIONS_ENABLED', - 'EXTENSIONS_LIFECYCLE_ZOOKEEPER_PATH', - 'EXTENSIONS_REPLICATION_DEST_AUTH_ACCOUNT', - 'EXTENSIONS_REPLICATION_DEST_AUTH_TYPE', - 'EXTENSIONS_REPLICATION_DEST_BOOTSTRAPLIST', - 'EXTENSIONS_REPLICATION_DEST_BOOTSTRAPLIST_MORE', - 'EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_AWS_S3_RETRY_BACKOFF_FACTOR', - 'EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_AWS_S3_RETRY_BACKOFF_JITTER', - 'EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_AWS_S3_RETRY_BACKOFF_MAX', - 'EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_AWS_S3_RETRY_BACKOFF_MIN', - 'EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_AWS_S3_RETRY_MAX_RETRIES', - 'EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_AWS_S3_RETRY_TIMEOUT_S', - 'EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_AZURE_RETRY_BACKOFF_FACTOR', - 'EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_AZURE_RETRY_BACKOFF_JITTER', - 'EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_AZURE_RETRY_BACKOFF_MAX', - 'EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_AZURE_RETRY_BACKOFF_MIN', - 'EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_AZURE_RETRY_MAX_RETRIES', - 'EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_AZURE_RETRY_TIMEOUT_S', - 'EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_CONCURRENCY', - 'EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_GCP_RETRY_BACKOFF_FACTOR', - 'EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_GCP_RETRY_BACKOFF_JITTER', - 'EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_GCP_RETRY_BACKOFF_MAX', - 'EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_GCP_RETRY_BACKOFF_MIN', - 'EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_GCP_RETRY_MAX_RETRIES', - 'EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_GCP_RETRY_TIMEOUT_S', - 'EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_MAX_POLL_INTERVAL_MS', - 'EXTENSIONS_REPLICATION_SOURCE_AUTH_ACCOUNT', - 'EXTENSIONS_REPLICATION_SOURCE_AUTH_TYPE', - 'EXTENSIONS_REPLICATION_SOURCE_S3_HOST', - 'EXTENSIONS_REPLICATION_SOURCE_S3_PORT', - 'EXTENSIONS_REPLICATION_STATUS_PROCESSOR_CONCURRENCY', - 'HEALTHCHECKS_ALLOWFROM', - 'KAFKA_BACKLOG_METRICS_INTERVALS', - 'KAFKA_BACKLOG_METRICS_ZKPATH', - 'KAFKA_HOSTS', - 'LIVENESS_PROBE_PORT', - 'LOG_LEVEL', - 'MONGODB_AUTH_PASSWORD', - 'MONGODB_AUTH_USERNAME', - 'MONGODB_DATABASE', - 'MONGODB_HOSTS', - 'MONGODB_RS', - 'QUEUE_POPULATOR_BATCH_MAX_READ', - 'QUEUE_POPULATOR_DMD_HOST', - 'QUEUE_POPULATOR_DMD_PORT', - 'REDIS_HA_NAME', - 'REDIS_HOST', - 'REDIS_LOCALCACHE_HOST', - 'REDIS_LOCALCACHE_PORT', - 'REDIS_PORT', - 'REDIS_SENTINELS', - 'REPLICATION_GROUP_ID', - 'ZOOKEEPER_AUTO_CREATE_NAMESPACE', - 'ZOOKEEPER_CONNECTION_STRING', - ]; - - // the lifecycle rules are configured with supportedLifecycleRules, and the - // local cache is not part of the configuration schema - const removedEnvVars = [ - 'EXTENSIONS_LIFECYCLE_RULES_ABORT_INCOMPLETE_MPU_ENABLED', - 'EXTENSIONS_LIFECYCLE_RULES_EXPIRATION_ENABLED', - 'EXTENSIONS_LIFECYCLE_RULES_NC_VERSION_EXPIRATION_ENABLED', - 'EXTENSIONS_LIFECYCLE_RULES_TRANSITIONS_ENABLED', - 'REDIS_LOCALCACHE_HOST', - 'REDIS_LOCALCACHE_PORT', - ]; - - const retryFields = backend => ({ - [`EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_${backend}_RETRY_MAX_RETRIES`]: '1', - [`EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_${backend}_RETRY_TIMEOUT_S`]: '2', - [`EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_${backend}_RETRY_BACKOFF_MIN`]: '3', - [`EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_${backend}_RETRY_BACKOFF_MAX`]: '4', - [`EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_${backend}_RETRY_BACKOFF_JITTER`]: '0.5', - [`EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_${backend}_RETRY_BACKOFF_FACTOR`]: '6', - }); - const retryExpected = backend => ({ - [`extensions.replication.queueProcessor.retry.${backend}.maxRetries`]: 1, - [`extensions.replication.queueProcessor.retry.${backend}.timeoutS`]: 2, - [`extensions.replication.queueProcessor.retry.${backend}.backoff`]: - { min: 3, max: 4, jitter: 0.5, factor: 6 }, - }); - - const probeServer = { bindAddress: '0.0.0.0', port: 8100 }; - const contract = [ - [{ LIVENESS_PROBE_PORT: '8100' }, { - 'queuePopulator.probeServer': probeServer, - 'extensions.ingestion.probeServer': probeServer, - 'extensions.mongoProcessor.probeServer': probeServer, - 'extensions.replication.queueProcessor.probeServer': probeServer, - 'extensions.replication.replicationStatusProcessor.probeServer': probeServer, - 'extensions.lifecycle.conductor.probeServer': probeServer, - 'extensions.lifecycle.bucketProcessor.probeServer': probeServer, - 'extensions.lifecycle.objectProcessor.probeServer': probeServer, - 'extensions.gc.probeServer': probeServer, - }], - [{ LOG_LEVEL: 'debug' }, { 'log.logLevel': 'debug' }], - [{ ZOOKEEPER_AUTO_CREATE_NAMESPACE: 'true' }, { 'zookeeper.autoCreateNamespace': true }], - [{ ZOOKEEPER_CONNECTION_STRING: 'zk:2181/bb' }, { 'zookeeper.connectionString': 'zk:2181/bb' }], - [{ KAFKA_HOSTS: 'kafka:9092' }, { 'kafka.hosts': 'kafka:9092' }], - [{ KAFKA_BACKLOG_METRICS_ZKPATH: '/bb/metrics' }, { 'kafka.backlogMetrics.zkPath': '/bb/metrics' }], - [{ KAFKA_BACKLOG_METRICS_INTERVALS: '30' }, { 'kafka.backlogMetrics.intervalS': 30 }], - [{ REDIS_SENTINELS: 'sentinel1:26379,sentinel2:26379', REDIS_HA_NAME: 'group' }, { - redis: { - name: 'group', - sentinels: [{ host: 'sentinel1', port: 26379 }, { host: 'sentinel2', port: 26379 }], - // the passwords of the section survive the parsing of the list - password: '', - sentinelPassword: '', - }, - }, { redis: {} }], - [{ REDIS_HOST: 'redis' }, { 'redis.host': 'redis', 'redis.port': 6379 }], - [{ REDIS_HOST: 'redis', REDIS_PORT: '6380' }, { 'redis.host': 'redis', 'redis.port': 6380 }], - [{ QUEUE_POPULATOR_BATCH_MAX_READ: '42' }, { 'queuePopulator.batchMaxRead': 42 }], - [{ QUEUE_POPULATOR_DMD_HOST: 'dmd', QUEUE_POPULATOR_DMD_PORT: '9991' }, { - 'queuePopulator.dmd.host': 'dmd', - 'queuePopulator.dmd.port': 9991, - }], - [{ MONGODB_HOSTS: 'mongo1:27017,mongo2:27017' }, { - 'queuePopulator.mongo.replicaSetHosts': 'mongo1:27017,mongo2:27017', - }], - [{ MONGODB_RS: 'rs1' }, { 'queuePopulator.mongo.replicaSet': 'rs1' }], - [{ MONGODB_DATABASE: 'db' }, { 'queuePopulator.mongo.database': 'db' }], - [{ MONGODB_AUTH_USERNAME: 'user', MONGODB_AUTH_PASSWORD: 'pass' }, { - 'queuePopulator.mongo.authCredentials': { username: 'user', password: 'pass' }, - }], - [{ CLOUDSERVER_HOST: 'cloudserver', CLOUDSERVER_PORT: '8001' }, { - 's3.host': 'cloudserver', - 's3.port': 8001, - }], - [{ HEALTHCHECKS_ALLOWFROM: '10.0.0.0/8' }, { - // the loopback addresses are always allowed - 'server.healthChecks.allowFrom': ['10.0.0.0/8', '127.0.0.1/8', '::1'], - }], - [{ REPLICATION_GROUP_ID: 'RG00002' }, { replicationGroupId: 'RG00002' }], - [{ - EXTENSIONS_REPLICATION_SOURCE_S3_HOST: 'cloudserver', - EXTENSIONS_REPLICATION_SOURCE_S3_PORT: '8001', - }, { - 'extensions.replication.source.s3.host': 'cloudserver', - 'extensions.replication.source.s3.port': 8001, - }], - [{ - EXTENSIONS_REPLICATION_SOURCE_AUTH_TYPE: 'account', - EXTENSIONS_REPLICATION_SOURCE_AUTH_ACCOUNT: 'source-account', - }, { - 'extensions.replication.source.auth.type': 'account', - 'extensions.replication.source.auth.account': 'source-account', - }], - [{ - EXTENSIONS_REPLICATION_DEST_AUTH_TYPE: 'account', - EXTENSIONS_REPLICATION_DEST_AUTH_ACCOUNT: 'dest-account', - }, { - 'extensions.replication.destination.auth.type': 'account', - 'extensions.replication.destination.auth.account': 'dest-account', - }], - [{ EXTENSIONS_REPLICATION_DEST_BOOTSTRAPLIST: 'zenko-1:8000' }, { - 'extensions.replication.destination.bootstrapList': - [{ site: 'zenko', servers: ['zenko-1:8000'], echo: false }], - }], - [{ - EXTENSIONS_REPLICATION_DEST_BOOTSTRAPLIST: 'zenko-1:8000', - EXTENSIONS_REPLICATION_DEST_BOOTSTRAPLIST_MORE: '{ "site": "aws", "type": "aws_s3" }', - }, { - 'extensions.replication.destination.bootstrapList': [ - { site: 'zenko', servers: ['zenko-1:8000'], echo: false }, - { site: 'aws', type: 'aws_s3' }, - ], - }], - [retryFields('AWS_S3'), retryExpected('aws_s3')], - [retryFields('AZURE'), retryExpected('azure')], - [retryFields('GCP'), retryExpected('gcp')], - [{ EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_CONCURRENCY: '11' }, - { 'extensions.replication.queueProcessor.concurrency': 11 }], - [{ EXTENSIONS_REPLICATION_STATUS_PROCESSOR_CONCURRENCY: '7' }, - { 'extensions.replication.replicationStatusProcessor.concurrency': 7 }], - [{ EXTENSIONS_REPLICATION_QUEUE_PROCESSOR_MAX_POLL_INTERVAL_MS: '60000' }, - { 'extensions.replication.queueProcessor.maxPollIntervalMs': 60000 }], - [{ EXTENSIONS_LIFECYCLE_ZOOKEEPER_PATH: '/lc' }, { 'extensions.lifecycle.zookeeperPath': '/lc' }], - [{ EXTENSIONS_LIFECYCLE_BUCKET_TASK_TOPIC: 'lc-buckets' }, - { 'extensions.lifecycle.bucketTasksTopic': 'lc-buckets' }], - [{ EXTENSIONS_LIFECYCLE_OBJECT_TASK_TOPIC: 'lc-objects' }, - { 'extensions.lifecycle.objectTasksTopic': 'lc-objects' }], - [{ EXTENSIONS_LIFECYCLE_CONDUCTOR_CRONRULE: '0 0 * * * *' }, - { 'extensions.lifecycle.conductor.cronRule': '0 0 * * * *' }], - [{ EXTENSIONS_LIFECYCLE_BUCKET_PROCESSOR_GROUP_ID: 'lc-bucket-group' }, - { 'extensions.lifecycle.bucketProcessor.groupId': 'lc-bucket-group' }], - [{ EXTENSIONS_LIFECYCLE_OBJECT_PROCESSOR_GROUP_ID: 'lc-object-group' }, - { 'extensions.lifecycle.objectProcessor.groupId': 'lc-object-group' }], - [{ EXTENSIONS_LIFECYCLE_AUTH_TYPE: 'account', EXTENSIONS_LIFECYCLE_AUTH_ACCOUNT: 'lc-account' }, { - 'extensions.lifecycle.auth.type': 'account', - 'extensions.lifecycle.auth.account': 'lc-account', - }], - [{ EXTENSIONS_GC_TOPIC: 'gc-topic' }, { 'extensions.gc.topic': 'gc-topic' }], - [{ EXTENSIONS_INGESTION_AUTH_TYPE: 'service', EXTENSIONS_INGESTION_AUTH_ACCOUNT: 'ingest' }, { - 'extensions.ingestion.auth.type': 'service', - 'extensions.ingestion.auth.account': 'ingest', - }], - [{ EXTENSIONS_INGESTION_MAX_PARALLEL_READERS: '3' }, - { 'extensions.ingestion.maxParallelReaders': 3 }], - ]; - - let ogConfigFile; - - before(() => { - ogConfigFile = process.env.BACKBEAT_CONFIG_FILE; - process.env.BACKBEAT_CONFIG_FILE = `${__dirname}/config.json`; - }); - - after(() => { - if (ogConfigFile === undefined) { - delete process.env.BACKBEAT_CONFIG_FILE; - } else { - process.env.BACKBEAT_CONFIG_FILE = ogConfigFile; - } - }); - - /** - * @param {Object} env - env vars of the case - * @param {Object} [sections] - config sections replacing those of the file, - * for a case the fixture cannot host as it stands - * @returns {Config} configuration built from the file and the environment - */ - function configWith(env, sections) { - const og = Object.fromEntries(Object.keys(env).map(name => [name, process.env[name]])); - Object.assign(process.env, env); - if (sections) { - sinon.stub(fs, 'readFileSync') - .callThrough() - .withArgs(process.env.BACKBEAT_CONFIG_FILE, sinon.match.any) - .returns(JSON.stringify({ ...fileConfig, ...sections })); - } - try { - return new Config(); - } finally { - sinon.restore(); - Object.entries(og).forEach(([name, value]) => { - if (value === undefined) { - delete process.env[name]; - } else { - process.env[name] = value; - } - }); - } - } - - contract.forEach(([env, expected, sections]) => { - it(`should apply ${Object.keys(env).join(', ')}`, () => { - const config = configWith(env, sections); - Object.entries(expected).forEach(([path, value]) => - assert.deepStrictEqual(getField(config, path.split('.')), value, path)); - }); - }); - - it('should reject the sentinels over a standalone configuration', () => { - assert.throws(() => configWith({ REDIS_SENTINELS: 'sentinel1:26379' }), - /"redis.host" is not allowed/); - }); - - it('should account for every historic env var', () => { - const covered = new Set([ - ...contract.flatMap(([env]) => Object.keys(env)), - ...removedEnvVars, - ]); - - assert.deepStrictEqual(historicEnvVars.filter(name => !covered.has(name)), []); - }); - - it('should accept the log levels the entrypoint used to reject', () => { - assert.strictEqual(configWith({ LOG_LEVEL: 'warn' }).log.logLevel, 'warn'); - assert.strictEqual(configWith({ LOG_LEVEL: 'error' }).log.logLevel, 'error'); - }); - - it('should ignore the vars removed with the entrypoint', () => { - const config = configWith(Object.fromEntries(removedEnvVars.map(name => [name, 'redis']))); - - assert.strictEqual(config.extensions.lifecycle.rules, undefined); - assert.strictEqual(config.localCache, undefined); - }); -}); diff --git a/tests/unit/lib/config/extensionConfigValidator.spec.js b/tests/unit/lib/config/extensionConfigValidator.spec.js new file mode 100644 index 000000000..8ccecbb77 --- /dev/null +++ b/tests/unit/lib/config/extensionConfigValidator.spec.js @@ -0,0 +1,118 @@ +'use strict'; + +const assert = require('assert'); +const joi = require('joi'); + +const { extensionConfigValidator } = require('../../../../lib/config/extensionConfigValidator'); +const { logJoiOptional } = require('../../../../lib/config/configItems.joi'); +const extensions = require('../../../../extensions'); +const backbeatConfig = require('./config.json'); + +// the validated global configuration, as Config passes it to an extension +const globalConfig = { log: { logLevel: 'info', dumpLevel: 'trace' } }; + +function withEnv(env, validate) { + const og = Object.fromEntries(Object.keys(env).map(name => [name, process.env[name]])); + Object.assign(process.env, env); + try { + return validate(); + } finally { + Object.entries(og).forEach(([name, value]) => { + if (value === undefined) { + delete process.env[name]; + } else { + process.env[name] = value; + } + }); + } +} + +describe('extension config validator', () => { + const schema = joi.object({ + topic: joi.string().required(), + consumer: joi.object({ + groupId: joi.string().required(), + concurrency: joi.number().default(10), + }), + log: logJoiOptional, + }); + const validator = extensionConfigValidator('demo', schema); + // the overrides are applied in place: each case validates its own copy + const extConfig = () => ({ topic: 'demo-topic', consumer: { groupId: 'demo-group' } }); + + it('should apply an env var named after the extension', () => { + const validated = withEnv( + { EXTENSIONS_DEMO_TOPIC: 'from-env' }, + () => validator(globalConfig, extConfig())); + + assert.strictEqual(validated.topic, 'from-env'); + }); + + it('should apply an env var of a nested field', () => { + const validated = withEnv({ EXTENSIONS_DEMO_CONSUMER_CONCURRENCY: '42' }, + () => validator(globalConfig, extConfig())); + + assert.strictEqual(validated.consumer.concurrency, 42); + }); + + it('should apply the defaults of the schema', () => { + assert.strictEqual(validator(globalConfig, extConfig()).consumer.concurrency, 10); + }); + + // the global configuration is the validation context, so that a field can + // default to a global one + it('should inherit a level from the global log config', () => { + const validated = withEnv({ EXTENSIONS_DEMO_LOG_LEVEL: 'debug' }, + () => validator(globalConfig, extConfig())); + + assert.deepStrictEqual(validated.log, { logLevel: 'debug', dumpLevel: 'trace' }); + }); + + it('should reject a value the schema does not accept', () => { + assert.throws(() => withEnv({ EXTENSIONS_DEMO_CONSUMER_CONCURRENCY: 'many' }, + () => validator(globalConfig, extConfig())), + /"consumer.concurrency" must be a number/); + }); + + it('should reject a field the schema does not declare', () => { + assert.throws(() => validator(globalConfig, { ...extConfig(), unknown: 1 }), + /"unknown" is not allowed/); + }); + + /** + * Every extension is validated through the same factory: the env var of one + * of its fields is checked here, so that a schema losing the annotations, + * or an extension added without them, is caught. + */ + describe('extensions', () => { + const cases = [ + ['gc', 'EXTENSIONS_GC_TOPIC', 'topic'], + ['ingestion', 'EXTENSIONS_INGESTION_TOPIC', 'topic'], + ['lifecycle', 'EXTENSIONS_LIFECYCLE_ZOOKEEPER_PATH', 'zookeeperPath'], + ['mongoProcessor', 'EXTENSIONS_MONGO_PROCESSOR_TOPIC', 'topic'], + ['notification', 'EXTENSIONS_NOTIFICATION_TOPIC', 'topic'], + ['oplogPopulator', 'EXTENSIONS_OPLOG_POPULATOR_TOPIC', 'topic'], + ['replication', 'EXTENSIONS_REPLICATION_TOPIC', 'topic'], + ]; + + cases.forEach(([name, envVar, field]) => { + it(`should apply ${envVar}`, () => { + const config = JSON.parse(JSON.stringify(backbeatConfig.extensions[name])); + const validated = withEnv( + { [envVar]: 'from-env' }, + () => extensions[name].configValidator(globalConfig, config)); + + assert.strictEqual(validated[field], 'from-env'); + }); + }); + + it('should account for every extension holding a config validator', () => { + const covered = new Set(cases.map(([name]) => name)); + const validated = Object.entries(extensions) + .filter(([, extension]) => extension.configValidator) + .map(([name]) => name); + + assert.deepStrictEqual(validated.filter(name => !covered.has(name)), []); + }); + }); +}); diff --git a/tests/unit/lib/config/fields.spec.js b/tests/unit/lib/config/fields.spec.js new file mode 100644 index 000000000..eada003b9 --- /dev/null +++ b/tests/unit/lib/config/fields.spec.js @@ -0,0 +1,80 @@ +'use strict'; + +const assert = require('assert'); + +const { getField, setField } = require('../../../../lib/config/fields'); + +describe('config fields', () => { + describe('getField', () => { + const config = { queuePopulator: { mongo: { database: 'metadata' } }, log: null }; + + it('should read the value at the path', () => { + assert.strictEqual(getField(config, ['queuePopulator', 'mongo', 'database']), 'metadata'); + }); + + it('should read a section', () => { + assert.deepStrictEqual(getField(config, ['queuePopulator', 'mongo']), { database: 'metadata' }); + }); + + it('should leave a missing node undefined', () => { + assert.strictEqual(getField(config, ['redis', 'host']), undefined); + assert.strictEqual(getField(config, ['queuePopulator', 'kafka', 'topic']), undefined); + }); + + it('should leave a path through a null node undefined', () => { + assert.strictEqual(getField(config, ['log', 'logLevel']), undefined); + }); + }); + + describe('setField', () => { + it('should set a field of an existing section', () => { + const config = { redis: { host: 'localhost' } }; + + setField(config, ['redis', 'port'], '6380'); + + assert.deepStrictEqual(config, { redis: { host: 'localhost', port: '6380' } }); + }); + + it('should replace the value a field holds', () => { + const config = { redis: { host: 'localhost' } }; + + setField(config, ['redis', 'host'], 'redis'); + + assert.deepStrictEqual(config, { redis: { host: 'redis' } }); + }); + + it('should set a field at the root', () => { + const config = {}; + + setField(config, ['replicationGroupId'], 'RG00002'); + + assert.deepStrictEqual(config, { replicationGroupId: 'RG00002' }); + }); + + it('should create the missing intermediate nodes', () => { + const config = {}; + + setField(config, ['queuePopulator', 'mongo', 'authCredentials', 'username'], 'user'); + + assert.deepStrictEqual(config, + { queuePopulator: { mongo: { authCredentials: { username: 'user' } } } }); + }); + + it('should report a node of the path holding a value', () => { + const config = { queuePopulator: { mongo: 'localhost:27017' } }; + + assert.throws(() => setField(config, ['queuePopulator', 'mongo', 'database'], 'metadata'), + /cannot set queuePopulator.mongo.database: queuePopulator.mongo is not an object/); + }); + + // an array is not a section a field belongs in: the per site probe + // servers are one, and a single port cannot name any of them + it('should report a node of the path holding an array', () => { + const config = { queueProcessor: { probeServer: [{ port: 4043, site: 'a' }] } }; + + assert.throws(() => setField(config, ['queueProcessor', 'probeServer', 'port'], '8100'), + /queueProcessor.probeServer is not an object/); + assert.deepStrictEqual(config.queueProcessor.probeServer, [{ port: 4043, site: 'a' }]); + }); + }); +}); diff --git a/tests/unit/conf/configs/replicationMultiDestConfig.json b/tests/unit/lib/config/replicationMultiDestConfig.json similarity index 100% rename from tests/unit/conf/configs/replicationMultiDestConfig.json rename to tests/unit/lib/config/replicationMultiDestConfig.json diff --git a/tests/unit/conf/configs/replicationServersConfig.json b/tests/unit/lib/config/replicationServersConfig.json similarity index 100% rename from tests/unit/conf/configs/replicationServersConfig.json rename to tests/unit/lib/config/replicationServersConfig.json From 5e1ba6895a216de39937946cbd212e3a3d92bf39 Mon Sep 17 00:00:00 2001 From: Francois Ferrand Date: Thu, 27 Aug 2026 11:57:53 +0200 Subject: [PATCH 8/8] Update AI rules for configuration Issue: BB-808 --- .github/copilot-instructions.md | 4 ++-- CLAUDE.md | 10 ++++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index 1fc06176c..02f5f849f 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -7,7 +7,7 @@ Repo context lives in [CLAUDE.md](../CLAUDE.md) — read it first. When reviewing a PR, analyze the changes against these criteria: | Area | What to check | -|------|---------------| +| ------ | --------------- | | Async error handling | Uncaught promise rejections, missing error callbacks, swallowed errors in streams. Double callbacks in try/catch blocks (callback called in try then again in catch) | | Async/await usage | New or modified code should use async/await instead of callbacks (see [Async/await migration suggestions](#asyncawait-migration-suggestions) below for when to suggest migrating). When code is migrated from callbacks to async/await, verify: no leftover callback or next params, no mixed callback + promise patterns, proper try/catch around awaited calls, errors are re-thrown or handled (not silently swallowed), `return await` rather than returning a bare promise, no `forEach` with async callbacks (use `for...of` or `Promise.all`), callers updated or backward compatibility kept via `util.callbackify`. Watch for the anti-pattern: `try { cb(); } catch(err) { cb(err); }` where an exception after the first `cb()` triggers a second call | | Kafka consumer/producer | Correct topic configuration, proper offset commits, consumer group handling, message serialization. Verify `onEntryCommittable` is always reachable. Check circuit breaker thresholds when adding new downstream topics | @@ -15,7 +15,7 @@ When reviewing a PR, analyze the changes against these criteria: | Dependency pinning | Git-based deps (arsenal, vaultclient, bucketclient, werelogs, breakbeat, httpagent) must pin to a tag, not a branch | | Logging | Proper use of werelogs, no `console.log` in production code, log levels match severity. Include enough context (bucket, object key, version, offset) for production troubleshooting | | Prometheus metrics | New metrics follow existing naming conventions (`s3_backbeat_*`), correct metric types (counter vs gauge vs histogram), bounded label cardinality — avoid per-connector or per-bucket labels that explode with scale | -| Config changes | Backward compatibility, Joi schema updates match new fields, environment variable naming, default values. Env var overrides in `lib/Config.js` must stay consistent with the config file schema | +| Config changes | Backward compatibility, Joi schema updates match new fields, environment variable naming, default values. New setting preferably belong in the joi schema, which derives its env var: flag a raw `process.env` read, or hand-rolled parsing, where a schema field or a meta annotation would do. Any deviation / custom configuration via env variable must be documented in `docs/configuration.md` in the same commit. | | MongoDB / Redis resilience | Reconnection handling, proper timeouts on external calls, no indefinite waits. Network errors to MongoDB must not cause stuck tasks or silent data loss | | Extension architecture | Changes respect the pluggable extension pattern, no cross-extension coupling | | Security | Command injection, prototype pollution, unsafe deserialization, credential exposure in config/env vars, OWASP-relevant issues for Node.js | diff --git a/CLAUDE.md b/CLAUDE.md index 75a6f3431..71bd18b8e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -13,6 +13,16 @@ This is a **Node.js asynchronous queue and job manager** for Scality's S3C and A - CommonJS modules; legacy code is callback-based, migrating to async/await (see below) - Mocha + Sinon test suites (`tests/unit/`, `tests/functional/`, `tests/behavior/`) +## Configuration + +Every configuration field is settable from the environment, with the variable name derived from the joi schema — see [docs/configuration.md](docs/configuration.md): + +- Add a setting by declaring it in the joi schema (`lib/config.joi.js`, `lib/config/configItems.joi.js`, or the extension's own validator). The env var follows from the config path, so nothing else is needed, and the value is validated. +- Do not read `process.env` directly for something the configuration could hold, and do not + hand-roll parsing: prefer a schema field, an `env`/`envVarAlias` annotation to adjust variable + name, or an `envDecodeHook` for custom decoding of the value. +- Anything that escapes that path — a variable read straight from `process.env`, one setting several fields, or a decode hook — is invisible to the schema, so document it in [docs/configuration.md](docs/configuration.md) in the same change, and cover it with a test. + ## Async code style The codebase is migrating from callbacks and the `async` library to async/await, per the [Scality migration guide](https://scality.atlassian.net/wiki/spaces/OS/pages/3523346468/2025-10-30+-+Async+Await+migration):