From 1c4bffed5cd866b5b189dead7ff4b7dee48c5d8a Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Tue, 26 May 2026 18:19:22 +0200 Subject: [PATCH 1/2] test(smoke): Fix assertions --- tests/templates/kuttl/smoke/14-assert.yaml | 284 ---- tests/templates/kuttl/smoke/14-assert.yaml.j2 | 1484 +++++++++++++++++ tests/test-definition.yaml | 6 +- 3 files changed, 1486 insertions(+), 288 deletions(-) delete mode 100644 tests/templates/kuttl/smoke/14-assert.yaml create mode 100644 tests/templates/kuttl/smoke/14-assert.yaml.j2 diff --git a/tests/templates/kuttl/smoke/14-assert.yaml b/tests/templates/kuttl/smoke/14-assert.yaml deleted file mode 100644 index 421f954e..00000000 --- a/tests/templates/kuttl/smoke/14-assert.yaml +++ /dev/null @@ -1,284 +0,0 @@ ---- -# Snapshot the full `.data` of each operator-managed ConfigMap. -# Any code change that alters rendered config values will fail these diffs. -# -# Runs as its own step (after 10/11/12) so kuttl does not re-evaluate the heavy -# heredocs on every 1-second readiness retry of the install step. By this point -# the cluster is in steady state, so each script runs once. -# -# The heredoc is quoted (`<<'YAMLEOF'`) so shell substitution is disabled and -# Java-properties escapes like `${ENV\:VAR}` survive verbatim. Only `__NAMESPACE__` -# is substituted afterwards via `sed`, because kuttl tests run in a randomized -# namespace per invocation. Both sides are normalized to canonical JSON via -# `yq -o=json`; keys are already alphabetical on both sides (operator stores -# BTreeMap; kubectl serializes maps sorted; the heredoc is hand-sorted). -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -timeout: 60 -commands: - # Verify the operator-managed secrets contain the expected data keys. - # Values are random per install and are not asserted. - - script: kubectl -n $NAMESPACE get secret trino-internal-secret -o yaml | yq -e '.data | has("INTERNAL_SECRET")' - - script: kubectl -n $NAMESPACE get secret trino-spooling-secret -o yaml | yq -e '.data | has("SPOOLING_SECRET")' - - script: | - expected=$(cat <<'YAMLEOF' | sed "s|__NAMESPACE__|$NAMESPACE|g" | yq -o=json - access-control.properties: | - access-control.name=opa - opa.allow-permission-management-operations=true - opa.policy.batch-column-masking-uri=http\://opa-server.__NAMESPACE__.svc.cluster.local\:8081/v1/data/trino/batchColumnMasks - opa.policy.batched-uri=http\://opa-server.__NAMESPACE__.svc.cluster.local\:8081/v1/data/trino/batch - opa.policy.row-filters-uri=http\://opa-server.__NAMESPACE__.svc.cluster.local\:8081/v1/data/trino/rowFilters - opa.policy.uri=http\://opa-server.__NAMESPACE__.svc.cluster.local\:8081/v1/data/trino/allow - config.properties: | - coordinator=true - discovery.uri=https\://trino-coordinator-default-0.trino-coordinator-default-headless.__NAMESPACE__.svc.cluster.local\:8443 - http-server.authentication.type=PASSWORD - http-server.https.enabled=true - http-server.https.keystore.key=changeit - http-server.https.keystore.path=/stackable/server_tls/keystore.p12 - http-server.https.port=8443 - http-server.https.truststore.key=changeit - http-server.https.truststore.path=/stackable/server_tls/truststore.p12 - http-server.log.enabled=false - internal-communication.https.keystore.key=changeit - internal-communication.https.keystore.path=/stackable/internal_tls/keystore.p12 - internal-communication.https.truststore.key=changeit - internal-communication.https.truststore.path=/stackable/internal_tls/truststore.p12 - internal-communication.shared-secret=${ENV\:INTERNAL_SECRET} - log.compression=none - log.format=json - log.max-size=5MB - log.max-total-size=10MB - log.path=/stackable/log/trino/server.airlift.json - node-scheduler.include-coordinator=false - node.internal-address-source=FQDN - password-authenticator.config-files=/stackable/rwconfig/trino-users-auth-password-file-auth.properties - query.max-execution-time=5s - query.max-memory=50GB - jvm.config: |- - -server - # Heap settings - -Xms3276m - -Xmx3276m - # Specify security.properties - -Djava.security.properties=/stackable/rwconfig/security.properties - # Prometheus metrics exporter - -javaagent:/stackable/jmx/jmx_prometheus_javaagent.jar=8081:/stackable/jmx/config.yaml - # Truststore settings - -Djavax.net.ssl.trustStore=/stackable/client_tls/truststore.p12 - -Djavax.net.ssl.trustStoreType=pkcs12 - -Djavax.net.ssl.trustStorePassword=changeit - # Recommended JVM arguments from Trino - -XX:InitialRAMPercentage=80 - -XX:MaxRAMPercentage=80 - -XX:G1HeapRegionSize=32M - -XX:+ExplicitGCInvokesConcurrent - -XX:+ExitOnOutOfMemoryError - -XX:+HeapDumpOnOutOfMemoryError - -XX:-OmitStackTraceInFastThrow - -XX:ReservedCodeCacheSize=512M - -XX:PerMethodRecompilationCutoff=10000 - -XX:PerBytecodeRecompilationCutoff=10000 - -Djdk.attach.allowAttachSelf=true - -Djdk.nio.maxCachedBufferSize=2000000 - -Dfile.encoding=UTF-8 - -XX:+EnableDynamicAgentLoading - # Arguments from jvmArgumentOverrides - log.properties: | - =info - node.properties: | - node.environment=trino - security.properties: | - networkaddress.cache.negative.ttl=0 - networkaddress.cache.ttl=30 - trino-users-auth-password-file-auth.properties: | - file.password-file=/stackable/users/trino-users-auth.db - password-authenticator.name=file - YAMLEOF - ) - actual=$(kubectl -n $NAMESPACE get cm trino-coordinator-default -o yaml | yq -o=json '.data') - if [ "$expected" != "$actual" ]; then - echo "ERROR: ConfigMap trino-coordinator-default data drifted from snapshot." - echo "=== expected ===" - printf '%s\n' "$expected" - echo "=== actual ===" - printf '%s\n' "$actual" - exit 1 - fi - - script: | - expected=$(cat <<'YAMLEOF' | sed "s|__NAMESPACE__|$NAMESPACE|g" | yq -o=json - access-control.properties: | - access-control.name=opa - opa.allow-permission-management-operations=true - opa.policy.batch-column-masking-uri=http\://opa-server.__NAMESPACE__.svc.cluster.local\:8081/v1/data/trino/batchColumnMasks - opa.policy.batched-uri=http\://opa-server.__NAMESPACE__.svc.cluster.local\:8081/v1/data/trino/batch - opa.policy.row-filters-uri=http\://opa-server.__NAMESPACE__.svc.cluster.local\:8081/v1/data/trino/rowFilters - opa.policy.uri=http\://opa-server.__NAMESPACE__.svc.cluster.local\:8081/v1/data/trino/allow - config.properties: | - coordinator=false - discovery.uri=https\://trino-coordinator-default-0.trino-coordinator-default-headless.__NAMESPACE__.svc.cluster.local\:8443 - http-server.https.enabled=true - http-server.https.keystore.key=changeit - http-server.https.keystore.path=/stackable/server_tls/keystore.p12 - http-server.https.port=8443 - http-server.https.truststore.key=changeit - http-server.https.truststore.path=/stackable/server_tls/truststore.p12 - http-server.log.enabled=false - internal-communication.https.keystore.key=changeit - internal-communication.https.keystore.path=/stackable/internal_tls/keystore.p12 - internal-communication.https.truststore.key=changeit - internal-communication.https.truststore.path=/stackable/internal_tls/truststore.p12 - internal-communication.shared-secret=${ENV\:INTERNAL_SECRET} - log.compression=none - log.format=json - log.max-size=5MB - log.max-total-size=10MB - log.path=/stackable/log/trino/server.airlift.json - node.internal-address-source=FQDN - query.max-memory=50GB - shutdown.grace-period=30s - jvm.config: |- - -server - # Heap settings - -Xms3276m - -Xmx3276m - # Specify security.properties - -Djava.security.properties=/stackable/rwconfig/security.properties - # Prometheus metrics exporter - -javaagent:/stackable/jmx/jmx_prometheus_javaagent.jar=8081:/stackable/jmx/config.yaml - # Truststore settings - -Djavax.net.ssl.trustStore=/stackable/client_tls/truststore.p12 - -Djavax.net.ssl.trustStoreType=pkcs12 - -Djavax.net.ssl.trustStorePassword=changeit - # Recommended JVM arguments from Trino - -XX:InitialRAMPercentage=80 - -XX:MaxRAMPercentage=80 - -XX:G1HeapRegionSize=32M - -XX:+ExplicitGCInvokesConcurrent - -XX:+ExitOnOutOfMemoryError - -XX:+HeapDumpOnOutOfMemoryError - -XX:-OmitStackTraceInFastThrow - -XX:ReservedCodeCacheSize=512M - -XX:PerMethodRecompilationCutoff=10000 - -XX:PerBytecodeRecompilationCutoff=10000 - -Djdk.attach.allowAttachSelf=true - -Djdk.nio.maxCachedBufferSize=2000000 - -Dfile.encoding=UTF-8 - -XX:+EnableDynamicAgentLoading - # Arguments from jvmArgumentOverrides - log.properties: | - =info - node.properties: | - node.environment=trino - security.properties: | - networkaddress.cache.negative.ttl=0 - networkaddress.cache.ttl=30 - YAMLEOF - ) - actual=$(kubectl -n $NAMESPACE get cm trino-worker-default -o yaml | yq -o=json '.data') - if [ "$expected" != "$actual" ]; then - echo "ERROR: ConfigMap trino-worker-default data drifted from snapshot." - echo "=== expected ===" - printf '%s\n' "$expected" - echo "=== actual ===" - printf '%s\n' "$actual" - exit 1 - fi - - script: | - expected=$(cat <<'YAMLEOF' | sed "s|__NAMESPACE__|$NAMESPACE|g" | yq -o=json - hive.properties: | - connector.name=hive - fs.hadoop.enabled=true - fs.native-s3.enabled=true - hive.config.resources=/stackable/config/catalog/hive/hdfs-config/core-site.xml,/stackable/config/catalog/hive/hdfs-config/hdfs-site.xml - hive.metastore.uri=thrift\://hive-metastore.__NAMESPACE__.svc.cluster.local\:9083 - hive.security=allow-all - s3.aws-access-key=${ENV\:CATALOG_HIVE_S3_AWS_ACCESS_KEY} - s3.aws-secret-key=${ENV\:CATALOG_HIVE_S3_AWS_SECRET_KEY} - s3.endpoint=https\://minio\:9000/ - s3.path-style-access=true - s3.region=us-east-1 - iceberg.properties: | - connector.name=iceberg - fs.hadoop.enabled=true - fs.native-s3.enabled=true - hive.config.resources=/stackable/config/catalog/iceberg/hdfs-config/core-site.xml,/stackable/config/catalog/iceberg/hdfs-config/hdfs-site.xml - hive.metastore.uri=thrift\://hive-metastore.__NAMESPACE__.svc.cluster.local\:9083 - iceberg.security=allow-all - s3.aws-access-key=${ENV\:CATALOG_ICEBERG_S3_AWS_ACCESS_KEY} - s3.aws-secret-key=${ENV\:CATALOG_ICEBERG_S3_AWS_SECRET_KEY} - s3.endpoint=https\://minio\:9000/ - s3.path-style-access=true - s3.region=us-east-1 - postgresql.properties: | - connection-password=${ENV\:POSTGRESQL_POSTGRESQL_DATABASE_PASSWORD} - connection-url=jdbc\:postgresql\://postgresql\:5432/hive - connection-user=${ENV\:POSTGRESQL_POSTGRESQL_DATABASE_USERNAME} - connector.name=postgresql - postgresqlgeneric.properties: | - connection-password=${ENV\:CATALOG_POSTGRESQLGENERIC_CONNECTION_PASSWORD} - connection-url=jdbc\:postgresql\://postgresql\:5432/hive - connection-user=${ENV\:CATALOG_POSTGRESQLGENERIC_CONNECTION_USER} - connector.name=postgresql - tpch.properties: | - connector.name=tpch - YAMLEOF - ) - actual=$(kubectl -n $NAMESPACE get cm trino-coordinator-default-catalog -o yaml | yq -o=json '.data') - if [ "$expected" != "$actual" ]; then - echo "ERROR: ConfigMap trino-coordinator-default-catalog data drifted from snapshot." - echo "=== expected ===" - printf '%s\n' "$expected" - echo "=== actual ===" - printf '%s\n' "$actual" - exit 1 - fi - - script: | - expected=$(cat <<'YAMLEOF' | sed "s|__NAMESPACE__|$NAMESPACE|g" | yq -o=json - hive.properties: | - connector.name=hive - fs.hadoop.enabled=true - fs.native-s3.enabled=true - hive.config.resources=/stackable/config/catalog/hive/hdfs-config/core-site.xml,/stackable/config/catalog/hive/hdfs-config/hdfs-site.xml - hive.metastore.uri=thrift\://hive-metastore.__NAMESPACE__.svc.cluster.local\:9083 - hive.security=allow-all - s3.aws-access-key=${ENV\:CATALOG_HIVE_S3_AWS_ACCESS_KEY} - s3.aws-secret-key=${ENV\:CATALOG_HIVE_S3_AWS_SECRET_KEY} - s3.endpoint=https\://minio\:9000/ - s3.path-style-access=true - s3.region=us-east-1 - iceberg.properties: | - connector.name=iceberg - fs.hadoop.enabled=true - fs.native-s3.enabled=true - hive.config.resources=/stackable/config/catalog/iceberg/hdfs-config/core-site.xml,/stackable/config/catalog/iceberg/hdfs-config/hdfs-site.xml - hive.metastore.uri=thrift\://hive-metastore.__NAMESPACE__.svc.cluster.local\:9083 - iceberg.security=allow-all - s3.aws-access-key=${ENV\:CATALOG_ICEBERG_S3_AWS_ACCESS_KEY} - s3.aws-secret-key=${ENV\:CATALOG_ICEBERG_S3_AWS_SECRET_KEY} - s3.endpoint=https\://minio\:9000/ - s3.path-style-access=true - s3.region=us-east-1 - postgresql.properties: | - connection-password=${ENV\:POSTGRESQL_POSTGRESQL_DATABASE_PASSWORD} - connection-url=jdbc\:postgresql\://postgresql\:5432/hive - connection-user=${ENV\:POSTGRESQL_POSTGRESQL_DATABASE_USERNAME} - connector.name=postgresql - postgresqlgeneric.properties: | - connection-password=${ENV\:CATALOG_POSTGRESQLGENERIC_CONNECTION_PASSWORD} - connection-url=jdbc\:postgresql\://postgresql\:5432/hive - connection-user=${ENV\:CATALOG_POSTGRESQLGENERIC_CONNECTION_USER} - connector.name=postgresql - tpch.properties: | - connector.name=tpch - YAMLEOF - ) - actual=$(kubectl -n $NAMESPACE get cm trino-worker-default-catalog -o yaml | yq -o=json '.data') - if [ "$expected" != "$actual" ]; then - echo "ERROR: ConfigMap trino-worker-default-catalog data drifted from snapshot." - echo "=== expected ===" - printf '%s\n' "$expected" - echo "=== actual ===" - printf '%s\n' "$actual" - exit 1 - fi diff --git a/tests/templates/kuttl/smoke/14-assert.yaml.j2 b/tests/templates/kuttl/smoke/14-assert.yaml.j2 new file mode 100644 index 00000000..299143b7 --- /dev/null +++ b/tests/templates/kuttl/smoke/14-assert.yaml.j2 @@ -0,0 +1,1484 @@ +--- +# Snapshot the full `.data` of each operator-managed ConfigMap. +# Any code change that alters rendered config values will fail these diffs. +# +# Runs as its own step (after 10/11/12) so kuttl does not re-evaluate the heavy +# heredocs on every 1-second readiness retry of the install step. By this point +# the cluster is in steady state, so each script runs once. +# +# The heredoc is quoted (`<<'YAMLEOF'`) so shell substitution is disabled and +# Java-properties escapes like `${ENV\:VAR}` survive verbatim. Only `__NAMESPACE__` +# is substituted afterwards via `sed`, because kuttl tests run in a randomized +# namespace per invocation. Both sides are normalized to canonical JSON via +# `yq -o=json`; keys are already alphabetical on both sides (operator stores +# BTreeMap; kubectl serializes maps sorted; the heredoc is hand-sorted). +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 60 +commands: + # Verify the operator-managed secrets contain the expected data keys. + # Values are random per install and are not asserted. + - script: kubectl -n $NAMESPACE get secret trino-internal-secret -o yaml | yq -e '.data | has("INTERNAL_SECRET")' + - script: kubectl -n $NAMESPACE get secret trino-spooling-secret -o yaml | yq -e '.data | has("SPOOLING_SECRET")' + - script: | + expected=$(cat <<'YAMLEOF' | sed "s|__NAMESPACE__|$NAMESPACE|g" | yq -o=json + access-control.properties: | + access-control.name=opa + opa.allow-permission-management-operations=true + opa.policy.batch-column-masking-uri=http\://opa-server.__NAMESPACE__.svc.cluster.local\:8081/v1/data/trino/batchColumnMasks + opa.policy.batched-uri=http\://opa-server.__NAMESPACE__.svc.cluster.local\:8081/v1/data/trino/batch + opa.policy.row-filters-uri=http\://opa-server.__NAMESPACE__.svc.cluster.local\:8081/v1/data/trino/rowFilters + opa.policy.uri=http\://opa-server.__NAMESPACE__.svc.cluster.local\:8081/v1/data/trino/allow + config.properties: | + coordinator=true + discovery.uri=https\://trino-coordinator-default-0.trino-coordinator-default-headless.__NAMESPACE__.svc.cluster.local\:8443 + http-server.authentication.type=PASSWORD + http-server.https.enabled=true + http-server.https.keystore.key=changeit + http-server.https.keystore.path=/stackable/server_tls/keystore.p12 + http-server.https.port=8443 + http-server.https.truststore.key=changeit + http-server.https.truststore.path=/stackable/server_tls/truststore.p12 + http-server.log.enabled=false + internal-communication.https.keystore.key=changeit + internal-communication.https.keystore.path=/stackable/internal_tls/keystore.p12 + internal-communication.https.truststore.key=changeit + internal-communication.https.truststore.path=/stackable/internal_tls/truststore.p12 + internal-communication.shared-secret=${ENV\:INTERNAL_SECRET} + log.compression=none + log.format=json + log.max-size=5MB + log.max-total-size=10MB + log.path=/stackable/log/trino/server.airlift.json + node-scheduler.include-coordinator=false + node.internal-address-source=FQDN + password-authenticator.config-files=/stackable/rwconfig/trino-users-auth-password-file-auth.properties + query.max-execution-time=5s + query.max-memory=50GB + jvm.config: |- + -server + # Heap settings + -Xms3276m + -Xmx3276m + # Specify security.properties + -Djava.security.properties=/stackable/rwconfig/security.properties + # Prometheus metrics exporter + -javaagent:/stackable/jmx/jmx_prometheus_javaagent.jar=8081:/stackable/jmx/config.yaml + # Truststore settings + -Djavax.net.ssl.trustStore=/stackable/client_tls/truststore.p12 + -Djavax.net.ssl.trustStoreType=pkcs12 + -Djavax.net.ssl.trustStorePassword=changeit + # Recommended JVM arguments from Trino + -XX:InitialRAMPercentage=80 + -XX:MaxRAMPercentage=80 + -XX:G1HeapRegionSize=32M + -XX:+ExplicitGCInvokesConcurrent + -XX:+ExitOnOutOfMemoryError + -XX:+HeapDumpOnOutOfMemoryError + -XX:-OmitStackTraceInFastThrow + -XX:ReservedCodeCacheSize=512M + -XX:PerMethodRecompilationCutoff=10000 + -XX:PerBytecodeRecompilationCutoff=10000 + -Djdk.attach.allowAttachSelf=true + -Djdk.nio.maxCachedBufferSize=2000000 + -Dfile.encoding=UTF-8 + -XX:+EnableDynamicAgentLoading + # Arguments from jvmArgumentOverrides + log.properties: | + =info + node.properties: | + node.environment=trino + security.properties: | + networkaddress.cache.negative.ttl=0 + networkaddress.cache.ttl=30 + trino-users-auth-password-file-auth.properties: | + file.password-file=/stackable/users/trino-users-auth.db + password-authenticator.name=file +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + vector.yaml: | + data_dir: /stackable/log/_vector-state + + log_schema: + host_key: pod + + sources: + vector: + type: internal_logs + + files_stdout: + type: file + include: + - /stackable/log/*/*.stdout.log + + files_stderr: + type: file + include: + - /stackable/log/*/*.stderr.log + + files_log4j: + type: file + include: + - /stackable/log/*/*.log4j.xml + line_delimiter: "\r\n" + multiline: + mode: halt_before + start_pattern: ^" + raw_message + "" + parsed_event, err = parse_xml(wrapped_xml_event) + if err != null { + error = "XML not parsable: " + err + .errors = push(.errors, error) + log(error, level: "warn") + .message = raw_message + } else { + root = object!(parsed_event.root) + if !is_object(root.event) { + error = "Parsed event contains no \"event\" tag." + .errors = push(.errors, error) + log(error, level: "warn") + .message = raw_message + } else { + if keys(root) != ["event"] { + .errors = push(.errors, "Parsed event contains multiple tags: " + join!(keys(root), ", ")) + } + event = object!(root.event) + + epoch_milliseconds, err = to_int(event.@timestamp) + if err == null && epoch_milliseconds != 0 { + converted_timestamp, err = from_unix_timestamp(epoch_milliseconds, "milliseconds") + if err == null { + .timestamp = converted_timestamp + } else { + .errors = push(.errors, "Time not parsable, using current time instead: " + err) + } + } else { + .errors = push(.errors, "Timestamp not found, using current time instead.") + } + + .logger, err = string(event.@logger) + if err != null || is_empty(.logger) { + .errors = push(.errors, "Logger not found.") + } + + level, err = string(event.@level) + if err != null { + .errors = push(.errors, "Level not found, using \"" + .level + "\" instead.") + } else if !includes(["TRACE", "DEBUG", "INFO", "WARN", "ERROR", "FATAL"], level) { + .errors = push(.errors, "Level \"" + level + "\" unknown, using \"" + .level + "\" instead.") + } else { + .level = level + } + + message, err = string(event.message) + if err != null || is_empty(message) { + .errors = push(.errors, "Message not found.") + } + throwable = string(event.throwable) ?? "" + .message = join!(compact([message, throwable]), "\n") + } + } + + processed_files_log4j2: + inputs: + - files_log4j2 + type: remap + source: | + raw_message = string!(.message) + + .timestamp = now() + .logger = "" + .level = "INFO" + .message = "" + .errors = [] + + event = {} + parsed_event, err = parse_xml(raw_message) + if err != null { + error = "XML not parsable: " + err + .errors = push(.errors, error) + log(error, level: "warn") + .message = raw_message + } else { + if !is_object(parsed_event.Event) { + error = "Parsed event contains no \"Event\" tag." + .errors = push(.errors, error) + log(error, level: "warn") + .message = raw_message + } else { + event = object!(parsed_event.Event) + + tag_instant_valid = false + instant, err = object(event.Instant) + if err == null { + epoch_nanoseconds, err = to_int(instant.@epochSecond) * 1_000_000_000 + to_int(instant.@nanoOfSecond) + if err == null && epoch_nanoseconds != 0 { + converted_timestamp, err = from_unix_timestamp(epoch_nanoseconds, "nanoseconds") + if err == null { + .timestamp = converted_timestamp + tag_instant_valid = true + } else { + .errors = push(.errors, "Instant invalid, trying property timeMillis instead: " + err) + } + } else { + .errors = push(.errors, "Instant invalid, trying property timeMillis instead: " + err) + } + } + if !tag_instant_valid { + epoch_milliseconds, err = to_int(event.@timeMillis) + if err == null && epoch_milliseconds != 0 { + converted_timestamp, err = from_unix_timestamp(epoch_milliseconds, "milliseconds") + if err == null { + .timestamp = converted_timestamp + } else { + .errors = push(.errors, "timeMillis not parsable, using current time instead: " + err) + } + } else { + .errors = push(.errors, "timeMillis not parsable, using current time instead: " + err) + } + } + + .logger, err = string(event.@loggerName) + if err != null || is_empty(.logger) { + .errors = push(.errors, "Logger not found.") + } + + level, err = string(event.@level) + if err != null { + .errors = push(.errors, "Level not found, using \"" + .level + "\" instead.") + } else if !includes(["TRACE", "DEBUG", "INFO", "WARN", "ERROR", "FATAL"], level) { + .errors = push(.errors, "Level \"" + level + "\" unknown, using \"" + .level + "\" instead.") + } else { + .level = level + } + + exception = null + thrown = event.Thrown + if is_object(thrown) { + exception = "Exception" + thread, err = string(event.@thread) + if err == null && !is_empty(thread) { + exception = exception + " in thread \"" + thread + "\"" + } + thrown_name, err = string(thrown.@name) + if err == null && !is_empty(exception) { + exception = exception + " " + thrown_name + } + message = string(thrown.@localizedMessage) ?? + string(thrown.@message) ?? + "" + if !is_empty(message) { + exception = exception + ": " + message + } + stacktrace_items = array(thrown.ExtendedStackTrace.ExtendedStackTraceItem) ?? [] + stacktrace = "" + for_each(stacktrace_items) -> |_index, value| { + stacktrace = stacktrace + " " + class = string(value.@class) ?? "" + method = string(value.@method) ?? "" + if !is_empty(class) && !is_empty(method) { + stacktrace = stacktrace + "at " + class + "." + method + } + file = string(value.@file) ?? "" + line = string(value.@line) ?? "" + if !is_empty(file) && !is_empty(line) { + stacktrace = stacktrace + "(" + file + ":" + line + ")" + } + exact = to_bool(value.@exact) ?? false + location = string(value.@location) ?? "" + version = string(value.@version) ?? "" + if !is_empty(location) && !is_empty(version) { + stacktrace = stacktrace + " " + if !exact { + stacktrace = stacktrace + "~" + } + stacktrace = stacktrace + "[" + location + ":" + version + "]" + } + stacktrace = stacktrace + "\n" + } + if stacktrace != "" { + exception = exception + "\n" + stacktrace + } + } + + message, err = string(event.Message) + if err != null || is_empty(message) { + message = null + .errors = push(.errors, "Message not found.") + } + .message = join!(compact([message, exception]), "\n") + } + } + + processed_files_py: + inputs: + - files_py + type: remap + source: | + raw_message = string!(.message) + + .timestamp = now() + .logger = "" + .level = "INFO" + .message = "" + .errors = [] + + parsed_event, err = parse_json(raw_message) + if err != null { + error = "JSON not parsable: " + err + .errors = push(.errors, error) + log(error, level: "warn") + .message = raw_message + } else if !is_object(parsed_event) { + error = "Parsed event is not a JSON object." + .errors = push(.errors, error) + log(error, level: "warn") + .message = raw_message + } else { + event = object!(parsed_event) + + asctime, err = string(event.asctime) + if err == null { + parsed_timestamp, err = parse_timestamp(asctime, "%F %T,%3f") + if err == null { + .timestamp = parsed_timestamp + } else { + .errors = push(.errors, "Timestamp not parsable, using current time instead: "+ err) + } + } else { + .errors = push(.errors, "Timestamp not found, using current time instead.") + } + + .logger, err = string(event.name) + if err != null || is_empty(.logger) { + .errors = push(.errors, "Logger not found.") + } + + level, err = string(event.levelname) + if err != null { + .errors = push(.errors, "Level not found, using \"" + .level + "\" instead.") + } else if level == "DEBUG" { + .level = "DEBUG" + } else if level == "INFO" { + .level = "INFO" + } else if level == "WARNING" { + .level = "WARN" + } else if level == "ERROR" { + .level = "ERROR" + } else if level == "CRITICAL" { + .level = "FATAL" + } else { + .errors = push(.errors, "Level \"" + level + "\" unknown, using \"" + .level + "\" instead.") + } + + .message, err = string(event.message) + if err != null || is_empty(.message) { + .errors = push(.errors, "Message not found.") + } + } + + processed_files_airlift: + inputs: + - files_airlift + type: remap + source: | + raw_message = string!(.message) + + .timestamp = now() + .logger = "" + .level = "INFO" + .message = "" + .errors = [] + + parsed_event, err = parse_json(raw_message) + if err != null { + error = "JSON not parsable: " + err + .errors = push(.errors, error) + log(error, level: "warn") + .message = raw_message + } else if !is_object(parsed_event) { + error = "Parsed event is not a JSON object." + .errors = push(.errors, error) + log(error, level: "warn") + .message = raw_message + } else { + event = object!(parsed_event) + + timestamp_string, err = string(event.timestamp) + if err == null { + parsed_timestamp, err = parse_timestamp(timestamp_string, "%Y-%m-%dT%H:%M:%S.%fZ") + if err == null { + .timestamp = parsed_timestamp + } else { + .errors = push(.errors, "Timestamp not parsable, using current time instead: " + err) + } + } else { + .errors = push(.errors, "Timestamp not found, using current time instead.") + } + + .logger, err = string(event.logger) + if err != null || is_empty(.logger) { + .errors = push(.errors, "Logger not found.") + } + + level, err = string(event.level) + if err != null { + .errors = push(.errors, "Level not found, using \"" + .level + "\" instead.") + } else if !includes(["TRACE", "DEBUG", "INFO", "WARN", "ERROR", "FATAL"], level) { + .errors = push(.errors, "Level \"" + level + "\" unknown, using \"" + .level + "\" instead.") + } else { + .level = level + } + + .thread = string(parsed_event.thread) ?? null + + .message, err = string(event.message) + if err != null || is_empty(.message) { + .errors = push(.errors, "Message not found.") + } + stacktrace = string(event.stackTrace) ?? "" + .message = join!(compact([.message, stacktrace]), "\n\n") + } + + extended_logs_files: + inputs: + - processed_files_* + type: remap + source: | + del(.source_type) + if .errors == [] { + del(.errors) + } + . |= parse_regex!(.file, r'^/stackable/log/(?P.*?)/(?P.*?)$') + + filtered_logs_vector: + inputs: + - vector + type: filter + condition: '!includes(["TRACE", "DEBUG"], .metadata.level)' + + extended_logs_vector: + inputs: + - filtered_logs_vector + type: remap + source: | + .container = "vector" + .level = .metadata.level + .logger = .metadata.module_path + if exists(.file) { .processed_file = del(.file) } + del(.metadata) + del(.pid) + del(.source_type) + + extended_logs: + inputs: + - extended_logs_* + type: remap + source: | + .namespace = "__NAMESPACE__" + .cluster = "trino" + .role = "coordinator" + .roleGroup = "default" + + sinks: + aggregator: + inputs: + - extended_logs + type: vector + address: $VECTOR_AGGREGATOR_ADDRESS +{% endif %} + YAMLEOF + ) + actual=$(kubectl -n $NAMESPACE get cm trino-coordinator-default -o yaml | yq -o=json '.data') + if [ "$expected" != "$actual" ]; then + echo "ERROR: ConfigMap trino-coordinator-default data drifted from snapshot." + echo "=== expected ===" + printf '%s\n' "$expected" + echo "=== actual ===" + printf '%s\n' "$actual" + exit 1 + fi + - script: | + expected=$(cat <<'YAMLEOF' | sed "s|__NAMESPACE__|$NAMESPACE|g" | yq -o=json + access-control.properties: | + access-control.name=opa + opa.allow-permission-management-operations=true + opa.policy.batch-column-masking-uri=http\://opa-server.__NAMESPACE__.svc.cluster.local\:8081/v1/data/trino/batchColumnMasks + opa.policy.batched-uri=http\://opa-server.__NAMESPACE__.svc.cluster.local\:8081/v1/data/trino/batch + opa.policy.row-filters-uri=http\://opa-server.__NAMESPACE__.svc.cluster.local\:8081/v1/data/trino/rowFilters + opa.policy.uri=http\://opa-server.__NAMESPACE__.svc.cluster.local\:8081/v1/data/trino/allow + config.properties: | + coordinator=false + discovery.uri=https\://trino-coordinator-default-0.trino-coordinator-default-headless.__NAMESPACE__.svc.cluster.local\:8443 + http-server.https.enabled=true + http-server.https.keystore.key=changeit + http-server.https.keystore.path=/stackable/server_tls/keystore.p12 + http-server.https.port=8443 + http-server.https.truststore.key=changeit + http-server.https.truststore.path=/stackable/server_tls/truststore.p12 + http-server.log.enabled=false + internal-communication.https.keystore.key=changeit + internal-communication.https.keystore.path=/stackable/internal_tls/keystore.p12 + internal-communication.https.truststore.key=changeit + internal-communication.https.truststore.path=/stackable/internal_tls/truststore.p12 + internal-communication.shared-secret=${ENV\:INTERNAL_SECRET} + log.compression=none + log.format=json + log.max-size=5MB + log.max-total-size=10MB + log.path=/stackable/log/trino/server.airlift.json + node.internal-address-source=FQDN + query.max-memory=50GB + shutdown.grace-period=30s + jvm.config: |- + -server + # Heap settings + -Xms3276m + -Xmx3276m + # Specify security.properties + -Djava.security.properties=/stackable/rwconfig/security.properties + # Prometheus metrics exporter + -javaagent:/stackable/jmx/jmx_prometheus_javaagent.jar=8081:/stackable/jmx/config.yaml + # Truststore settings + -Djavax.net.ssl.trustStore=/stackable/client_tls/truststore.p12 + -Djavax.net.ssl.trustStoreType=pkcs12 + -Djavax.net.ssl.trustStorePassword=changeit + # Recommended JVM arguments from Trino + -XX:InitialRAMPercentage=80 + -XX:MaxRAMPercentage=80 + -XX:G1HeapRegionSize=32M + -XX:+ExplicitGCInvokesConcurrent + -XX:+ExitOnOutOfMemoryError + -XX:+HeapDumpOnOutOfMemoryError + -XX:-OmitStackTraceInFastThrow + -XX:ReservedCodeCacheSize=512M + -XX:PerMethodRecompilationCutoff=10000 + -XX:PerBytecodeRecompilationCutoff=10000 + -Djdk.attach.allowAttachSelf=true + -Djdk.nio.maxCachedBufferSize=2000000 + -Dfile.encoding=UTF-8 + -XX:+EnableDynamicAgentLoading + # Arguments from jvmArgumentOverrides + log.properties: | + =info + node.properties: | + node.environment=trino + security.properties: | + networkaddress.cache.negative.ttl=0 + networkaddress.cache.ttl=30 +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + vector.yaml: | + data_dir: /stackable/log/_vector-state + + log_schema: + host_key: pod + + sources: + vector: + type: internal_logs + + files_stdout: + type: file + include: + - /stackable/log/*/*.stdout.log + + files_stderr: + type: file + include: + - /stackable/log/*/*.stderr.log + + files_log4j: + type: file + include: + - /stackable/log/*/*.log4j.xml + line_delimiter: "\r\n" + multiline: + mode: halt_before + start_pattern: ^" + raw_message + "" + parsed_event, err = parse_xml(wrapped_xml_event) + if err != null { + error = "XML not parsable: " + err + .errors = push(.errors, error) + log(error, level: "warn") + .message = raw_message + } else { + root = object!(parsed_event.root) + if !is_object(root.event) { + error = "Parsed event contains no \"event\" tag." + .errors = push(.errors, error) + log(error, level: "warn") + .message = raw_message + } else { + if keys(root) != ["event"] { + .errors = push(.errors, "Parsed event contains multiple tags: " + join!(keys(root), ", ")) + } + event = object!(root.event) + + epoch_milliseconds, err = to_int(event.@timestamp) + if err == null && epoch_milliseconds != 0 { + converted_timestamp, err = from_unix_timestamp(epoch_milliseconds, "milliseconds") + if err == null { + .timestamp = converted_timestamp + } else { + .errors = push(.errors, "Time not parsable, using current time instead: " + err) + } + } else { + .errors = push(.errors, "Timestamp not found, using current time instead.") + } + + .logger, err = string(event.@logger) + if err != null || is_empty(.logger) { + .errors = push(.errors, "Logger not found.") + } + + level, err = string(event.@level) + if err != null { + .errors = push(.errors, "Level not found, using \"" + .level + "\" instead.") + } else if !includes(["TRACE", "DEBUG", "INFO", "WARN", "ERROR", "FATAL"], level) { + .errors = push(.errors, "Level \"" + level + "\" unknown, using \"" + .level + "\" instead.") + } else { + .level = level + } + + message, err = string(event.message) + if err != null || is_empty(message) { + .errors = push(.errors, "Message not found.") + } + throwable = string(event.throwable) ?? "" + .message = join!(compact([message, throwable]), "\n") + } + } + + processed_files_log4j2: + inputs: + - files_log4j2 + type: remap + source: | + raw_message = string!(.message) + + .timestamp = now() + .logger = "" + .level = "INFO" + .message = "" + .errors = [] + + event = {} + parsed_event, err = parse_xml(raw_message) + if err != null { + error = "XML not parsable: " + err + .errors = push(.errors, error) + log(error, level: "warn") + .message = raw_message + } else { + if !is_object(parsed_event.Event) { + error = "Parsed event contains no \"Event\" tag." + .errors = push(.errors, error) + log(error, level: "warn") + .message = raw_message + } else { + event = object!(parsed_event.Event) + + tag_instant_valid = false + instant, err = object(event.Instant) + if err == null { + epoch_nanoseconds, err = to_int(instant.@epochSecond) * 1_000_000_000 + to_int(instant.@nanoOfSecond) + if err == null && epoch_nanoseconds != 0 { + converted_timestamp, err = from_unix_timestamp(epoch_nanoseconds, "nanoseconds") + if err == null { + .timestamp = converted_timestamp + tag_instant_valid = true + } else { + .errors = push(.errors, "Instant invalid, trying property timeMillis instead: " + err) + } + } else { + .errors = push(.errors, "Instant invalid, trying property timeMillis instead: " + err) + } + } + if !tag_instant_valid { + epoch_milliseconds, err = to_int(event.@timeMillis) + if err == null && epoch_milliseconds != 0 { + converted_timestamp, err = from_unix_timestamp(epoch_milliseconds, "milliseconds") + if err == null { + .timestamp = converted_timestamp + } else { + .errors = push(.errors, "timeMillis not parsable, using current time instead: " + err) + } + } else { + .errors = push(.errors, "timeMillis not parsable, using current time instead: " + err) + } + } + + .logger, err = string(event.@loggerName) + if err != null || is_empty(.logger) { + .errors = push(.errors, "Logger not found.") + } + + level, err = string(event.@level) + if err != null { + .errors = push(.errors, "Level not found, using \"" + .level + "\" instead.") + } else if !includes(["TRACE", "DEBUG", "INFO", "WARN", "ERROR", "FATAL"], level) { + .errors = push(.errors, "Level \"" + level + "\" unknown, using \"" + .level + "\" instead.") + } else { + .level = level + } + + exception = null + thrown = event.Thrown + if is_object(thrown) { + exception = "Exception" + thread, err = string(event.@thread) + if err == null && !is_empty(thread) { + exception = exception + " in thread \"" + thread + "\"" + } + thrown_name, err = string(thrown.@name) + if err == null && !is_empty(exception) { + exception = exception + " " + thrown_name + } + message = string(thrown.@localizedMessage) ?? + string(thrown.@message) ?? + "" + if !is_empty(message) { + exception = exception + ": " + message + } + stacktrace_items = array(thrown.ExtendedStackTrace.ExtendedStackTraceItem) ?? [] + stacktrace = "" + for_each(stacktrace_items) -> |_index, value| { + stacktrace = stacktrace + " " + class = string(value.@class) ?? "" + method = string(value.@method) ?? "" + if !is_empty(class) && !is_empty(method) { + stacktrace = stacktrace + "at " + class + "." + method + } + file = string(value.@file) ?? "" + line = string(value.@line) ?? "" + if !is_empty(file) && !is_empty(line) { + stacktrace = stacktrace + "(" + file + ":" + line + ")" + } + exact = to_bool(value.@exact) ?? false + location = string(value.@location) ?? "" + version = string(value.@version) ?? "" + if !is_empty(location) && !is_empty(version) { + stacktrace = stacktrace + " " + if !exact { + stacktrace = stacktrace + "~" + } + stacktrace = stacktrace + "[" + location + ":" + version + "]" + } + stacktrace = stacktrace + "\n" + } + if stacktrace != "" { + exception = exception + "\n" + stacktrace + } + } + + message, err = string(event.Message) + if err != null || is_empty(message) { + message = null + .errors = push(.errors, "Message not found.") + } + .message = join!(compact([message, exception]), "\n") + } + } + + processed_files_py: + inputs: + - files_py + type: remap + source: | + raw_message = string!(.message) + + .timestamp = now() + .logger = "" + .level = "INFO" + .message = "" + .errors = [] + + parsed_event, err = parse_json(raw_message) + if err != null { + error = "JSON not parsable: " + err + .errors = push(.errors, error) + log(error, level: "warn") + .message = raw_message + } else if !is_object(parsed_event) { + error = "Parsed event is not a JSON object." + .errors = push(.errors, error) + log(error, level: "warn") + .message = raw_message + } else { + event = object!(parsed_event) + + asctime, err = string(event.asctime) + if err == null { + parsed_timestamp, err = parse_timestamp(asctime, "%F %T,%3f") + if err == null { + .timestamp = parsed_timestamp + } else { + .errors = push(.errors, "Timestamp not parsable, using current time instead: "+ err) + } + } else { + .errors = push(.errors, "Timestamp not found, using current time instead.") + } + + .logger, err = string(event.name) + if err != null || is_empty(.logger) { + .errors = push(.errors, "Logger not found.") + } + + level, err = string(event.levelname) + if err != null { + .errors = push(.errors, "Level not found, using \"" + .level + "\" instead.") + } else if level == "DEBUG" { + .level = "DEBUG" + } else if level == "INFO" { + .level = "INFO" + } else if level == "WARNING" { + .level = "WARN" + } else if level == "ERROR" { + .level = "ERROR" + } else if level == "CRITICAL" { + .level = "FATAL" + } else { + .errors = push(.errors, "Level \"" + level + "\" unknown, using \"" + .level + "\" instead.") + } + + .message, err = string(event.message) + if err != null || is_empty(.message) { + .errors = push(.errors, "Message not found.") + } + } + + processed_files_airlift: + inputs: + - files_airlift + type: remap + source: | + raw_message = string!(.message) + + .timestamp = now() + .logger = "" + .level = "INFO" + .message = "" + .errors = [] + + parsed_event, err = parse_json(raw_message) + if err != null { + error = "JSON not parsable: " + err + .errors = push(.errors, error) + log(error, level: "warn") + .message = raw_message + } else if !is_object(parsed_event) { + error = "Parsed event is not a JSON object." + .errors = push(.errors, error) + log(error, level: "warn") + .message = raw_message + } else { + event = object!(parsed_event) + + timestamp_string, err = string(event.timestamp) + if err == null { + parsed_timestamp, err = parse_timestamp(timestamp_string, "%Y-%m-%dT%H:%M:%S.%fZ") + if err == null { + .timestamp = parsed_timestamp + } else { + .errors = push(.errors, "Timestamp not parsable, using current time instead: " + err) + } + } else { + .errors = push(.errors, "Timestamp not found, using current time instead.") + } + + .logger, err = string(event.logger) + if err != null || is_empty(.logger) { + .errors = push(.errors, "Logger not found.") + } + + level, err = string(event.level) + if err != null { + .errors = push(.errors, "Level not found, using \"" + .level + "\" instead.") + } else if !includes(["TRACE", "DEBUG", "INFO", "WARN", "ERROR", "FATAL"], level) { + .errors = push(.errors, "Level \"" + level + "\" unknown, using \"" + .level + "\" instead.") + } else { + .level = level + } + + .thread = string(parsed_event.thread) ?? null + + .message, err = string(event.message) + if err != null || is_empty(.message) { + .errors = push(.errors, "Message not found.") + } + stacktrace = string(event.stackTrace) ?? "" + .message = join!(compact([.message, stacktrace]), "\n\n") + } + + extended_logs_files: + inputs: + - processed_files_* + type: remap + source: | + del(.source_type) + if .errors == [] { + del(.errors) + } + . |= parse_regex!(.file, r'^/stackable/log/(?P.*?)/(?P.*?)$') + + filtered_logs_vector: + inputs: + - vector + type: filter + condition: '!includes(["TRACE", "DEBUG"], .metadata.level)' + + extended_logs_vector: + inputs: + - filtered_logs_vector + type: remap + source: | + .container = "vector" + .level = .metadata.level + .logger = .metadata.module_path + if exists(.file) { .processed_file = del(.file) } + del(.metadata) + del(.pid) + del(.source_type) + + extended_logs: + inputs: + - extended_logs_* + type: remap + source: | + .namespace = "__NAMESPACE__" + .cluster = "trino" + .role = "worker" + .roleGroup = "default" + + sinks: + aggregator: + inputs: + - extended_logs + type: vector + address: $VECTOR_AGGREGATOR_ADDRESS +{% endif %} + YAMLEOF + ) + actual=$(kubectl -n $NAMESPACE get cm trino-worker-default -o yaml | yq -o=json '.data') + if [ "$expected" != "$actual" ]; then + echo "ERROR: ConfigMap trino-worker-default data drifted from snapshot." + echo "=== expected ===" + printf '%s\n' "$expected" + echo "=== actual ===" + printf '%s\n' "$actual" + exit 1 + fi + - script: | + expected=$(cat <<'YAMLEOF' | sed "s|__NAMESPACE__|$NAMESPACE|g" | yq -o=json + hive.properties: | + connector.name=hive + fs.hadoop.enabled=true + fs.native-s3.enabled=true + hive.config.resources=/stackable/config/catalog/hive/hdfs-config/core-site.xml,/stackable/config/catalog/hive/hdfs-config/hdfs-site.xml + hive.metastore.uri=thrift\://hive-metastore.__NAMESPACE__.svc.cluster.local\:9083 + hive.security=allow-all + s3.aws-access-key=${ENV\:CATALOG_HIVE_S3_AWS_ACCESS_KEY} + s3.aws-secret-key=${ENV\:CATALOG_HIVE_S3_AWS_SECRET_KEY} + s3.endpoint=https\://minio\:9000/ + s3.path-style-access=true + s3.region=us-east-1 + iceberg.properties: | + connector.name=iceberg + fs.hadoop.enabled=true + fs.native-s3.enabled=true + hive.config.resources=/stackable/config/catalog/iceberg/hdfs-config/core-site.xml,/stackable/config/catalog/iceberg/hdfs-config/hdfs-site.xml + hive.metastore.uri=thrift\://hive-metastore.__NAMESPACE__.svc.cluster.local\:9083 + iceberg.security=allow-all + s3.aws-access-key=${ENV\:CATALOG_ICEBERG_S3_AWS_ACCESS_KEY} + s3.aws-secret-key=${ENV\:CATALOG_ICEBERG_S3_AWS_SECRET_KEY} + s3.endpoint=https\://minio\:9000/ + s3.path-style-access=true + s3.region=us-east-1 + postgresql.properties: | + connection-password=${ENV\:POSTGRESQL_POSTGRESQL_DATABASE_PASSWORD} + connection-url=jdbc\:postgresql\://postgresql\:5432/hive + connection-user=${ENV\:POSTGRESQL_POSTGRESQL_DATABASE_USERNAME} + connector.name=postgresql + postgresqlgeneric.properties: | + connection-password=${ENV\:CATALOG_POSTGRESQLGENERIC_CONNECTION_PASSWORD} + connection-url=jdbc\:postgresql\://postgresql\:5432/hive + connection-user=${ENV\:CATALOG_POSTGRESQLGENERIC_CONNECTION_USER} + connector.name=postgresql + tpch.properties: | + connector.name=tpch + YAMLEOF + ) + actual=$(kubectl -n $NAMESPACE get cm trino-coordinator-default-catalog -o yaml | yq -o=json '.data') + if [ "$expected" != "$actual" ]; then + echo "ERROR: ConfigMap trino-coordinator-default-catalog data drifted from snapshot." + echo "=== expected ===" + printf '%s\n' "$expected" + echo "=== actual ===" + printf '%s\n' "$actual" + exit 1 + fi + - script: | + expected=$(cat <<'YAMLEOF' | sed "s|__NAMESPACE__|$NAMESPACE|g" | yq -o=json + hive.properties: | + connector.name=hive + fs.hadoop.enabled=true + fs.native-s3.enabled=true + hive.config.resources=/stackable/config/catalog/hive/hdfs-config/core-site.xml,/stackable/config/catalog/hive/hdfs-config/hdfs-site.xml + hive.metastore.uri=thrift\://hive-metastore.__NAMESPACE__.svc.cluster.local\:9083 + hive.security=allow-all + s3.aws-access-key=${ENV\:CATALOG_HIVE_S3_AWS_ACCESS_KEY} + s3.aws-secret-key=${ENV\:CATALOG_HIVE_S3_AWS_SECRET_KEY} + s3.endpoint=https\://minio\:9000/ + s3.path-style-access=true + s3.region=us-east-1 + iceberg.properties: | + connector.name=iceberg + fs.hadoop.enabled=true + fs.native-s3.enabled=true + hive.config.resources=/stackable/config/catalog/iceberg/hdfs-config/core-site.xml,/stackable/config/catalog/iceberg/hdfs-config/hdfs-site.xml + hive.metastore.uri=thrift\://hive-metastore.__NAMESPACE__.svc.cluster.local\:9083 + iceberg.security=allow-all + s3.aws-access-key=${ENV\:CATALOG_ICEBERG_S3_AWS_ACCESS_KEY} + s3.aws-secret-key=${ENV\:CATALOG_ICEBERG_S3_AWS_SECRET_KEY} + s3.endpoint=https\://minio\:9000/ + s3.path-style-access=true + s3.region=us-east-1 + postgresql.properties: | + connection-password=${ENV\:POSTGRESQL_POSTGRESQL_DATABASE_PASSWORD} + connection-url=jdbc\:postgresql\://postgresql\:5432/hive + connection-user=${ENV\:POSTGRESQL_POSTGRESQL_DATABASE_USERNAME} + connector.name=postgresql + postgresqlgeneric.properties: | + connection-password=${ENV\:CATALOG_POSTGRESQLGENERIC_CONNECTION_PASSWORD} + connection-url=jdbc\:postgresql\://postgresql\:5432/hive + connection-user=${ENV\:CATALOG_POSTGRESQLGENERIC_CONNECTION_USER} + connector.name=postgresql + tpch.properties: | + connector.name=tpch + YAMLEOF + ) + actual=$(kubectl -n $NAMESPACE get cm trino-worker-default-catalog -o yaml | yq -o=json '.data') + if [ "$expected" != "$actual" ]; then + echo "ERROR: ConfigMap trino-worker-default-catalog data drifted from snapshot." + echo "=== expected ===" + printf '%s\n' "$expected" + echo "=== actual ===" + printf '%s\n' "$actual" + exit 1 + fi diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index 8e38057f..1ec704b8 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -74,7 +74,7 @@ tests: dimensions: - trino - hive - - opa + - opa-latest - hdfs - zookeeper - openshift @@ -110,7 +110,7 @@ tests: dimensions: - trino - hive-latest - - opa + - opa-latest - keycloak - openshift - name: fault-tolerant-execution @@ -143,8 +143,6 @@ suites: expr: last - name: hive expr: last - - name: opa - expr: last - name: zookeeper expr: last - name: smoke-latest From 2e91e3a899ac9c568676377c2fddac6341893009 Mon Sep 17 00:00:00 2001 From: Siegfried Weber Date: Wed, 27 May 2026 15:28:56 +0200 Subject: [PATCH 2/2] test(smoke): Improve the diff of actual and expected ConfigMap content --- tests/templates/kuttl/smoke/14-assert.yaml.j2 | 48 +++++++++++-------- 1 file changed, 28 insertions(+), 20 deletions(-) diff --git a/tests/templates/kuttl/smoke/14-assert.yaml.j2 b/tests/templates/kuttl/smoke/14-assert.yaml.j2 index 299143b7..39b7ed1e 100644 --- a/tests/templates/kuttl/smoke/14-assert.yaml.j2 +++ b/tests/templates/kuttl/smoke/14-assert.yaml.j2 @@ -697,14 +697,16 @@ commands: YAMLEOF ) actual=$(kubectl -n $NAMESPACE get cm trino-coordinator-default -o yaml | yq -o=json '.data') - if [ "$expected" != "$actual" ]; then + expected_file=$(mktemp) && actual_file=$(mktemp) + printf '%s\n' "$expected" > "$expected_file" + printf '%s\n' "$actual" > "$actual_file" + if ! diff_out=$(diff -u "$expected_file" "$actual_file"); then echo "ERROR: ConfigMap trino-coordinator-default data drifted from snapshot." - echo "=== expected ===" - printf '%s\n' "$expected" - echo "=== actual ===" - printf '%s\n' "$actual" + printf '%s\n' "$diff_out" + rm -f "$expected_file" "$actual_file" exit 1 fi + rm -f "$expected_file" "$actual_file" - script: | expected=$(cat <<'YAMLEOF' | sed "s|__NAMESPACE__|$NAMESPACE|g" | yq -o=json access-control.properties: | @@ -1376,14 +1378,16 @@ commands: YAMLEOF ) actual=$(kubectl -n $NAMESPACE get cm trino-worker-default -o yaml | yq -o=json '.data') - if [ "$expected" != "$actual" ]; then + expected_file=$(mktemp) && actual_file=$(mktemp) + printf '%s\n' "$expected" > "$expected_file" + printf '%s\n' "$actual" > "$actual_file" + if ! diff_out=$(diff -u "$expected_file" "$actual_file"); then echo "ERROR: ConfigMap trino-worker-default data drifted from snapshot." - echo "=== expected ===" - printf '%s\n' "$expected" - echo "=== actual ===" - printf '%s\n' "$actual" + printf '%s\n' "$diff_out" + rm -f "$expected_file" "$actual_file" exit 1 fi + rm -f "$expected_file" "$actual_file" - script: | expected=$(cat <<'YAMLEOF' | sed "s|__NAMESPACE__|$NAMESPACE|g" | yq -o=json hive.properties: | @@ -1425,14 +1429,16 @@ commands: YAMLEOF ) actual=$(kubectl -n $NAMESPACE get cm trino-coordinator-default-catalog -o yaml | yq -o=json '.data') - if [ "$expected" != "$actual" ]; then + expected_file=$(mktemp) && actual_file=$(mktemp) + printf '%s\n' "$expected" > "$expected_file" + printf '%s\n' "$actual" > "$actual_file" + if ! diff_out=$(diff -u "$expected_file" "$actual_file"); then echo "ERROR: ConfigMap trino-coordinator-default-catalog data drifted from snapshot." - echo "=== expected ===" - printf '%s\n' "$expected" - echo "=== actual ===" - printf '%s\n' "$actual" + printf '%s\n' "$diff_out" + rm -f "$expected_file" "$actual_file" exit 1 fi + rm -f "$expected_file" "$actual_file" - script: | expected=$(cat <<'YAMLEOF' | sed "s|__NAMESPACE__|$NAMESPACE|g" | yq -o=json hive.properties: | @@ -1474,11 +1480,13 @@ commands: YAMLEOF ) actual=$(kubectl -n $NAMESPACE get cm trino-worker-default-catalog -o yaml | yq -o=json '.data') - if [ "$expected" != "$actual" ]; then + expected_file=$(mktemp) && actual_file=$(mktemp) + printf '%s\n' "$expected" > "$expected_file" + printf '%s\n' "$actual" > "$actual_file" + if ! diff_out=$(diff -u "$expected_file" "$actual_file"); then echo "ERROR: ConfigMap trino-worker-default-catalog data drifted from snapshot." - echo "=== expected ===" - printf '%s\n' "$expected" - echo "=== actual ===" - printf '%s\n' "$actual" + printf '%s\n' "$diff_out" + rm -f "$expected_file" "$actual_file" exit 1 fi + rm -f "$expected_file" "$actual_file"