From a422a63f2e5f05d85caa7595b85facb23268e4c1 Mon Sep 17 00:00:00 2001 From: Dylan Date: Fri, 23 Oct 2026 00:12:18 +0800 Subject: [PATCH] [fix](hudi) Keep storage credentials out of scanner logs ### What problem does this PR solve? Issue Number: None Related PR: None Problem Summary: Hudi JNI scanners logged every input parameter value at DEBUG, including explicit S3A access keys, secret keys and session tokens. Log parameter names and the query ID separately while preserving the Hadoop configuration used to read data. Add a regression test that captures DEBUG output, checks that query IDs remain available, rejects bucket-specific and custom credential values, and verifies that the scanner retains the original Hadoop properties. ### Release note Hudi JNI scanner debug logs no longer record storage credential values and continue to include query IDs for diagnostics. ### Check List (For Author) - Test: Unit Test - the targeted Hudi parameter-log test passes. Maven validate / Checkstyle and git diff --check pass. - Behavior changed: Yes - scanner debug logs show parameter names and the query ID without storage credentials. - Does this need documentation: No --- .../doris/hudi/HadoopHudiJniScanner.java | 3 +- .../hudi/HadoopHudiJniScannerLogTest.java | 137 ++++++++++++++++++ 2 files changed, 139 insertions(+), 1 deletion(-) create mode 100644 fe/be-java-extensions/hadoop-hudi-scanner/src/test/java/org/apache/doris/hudi/HadoopHudiJniScannerLogTest.java diff --git a/fe/be-java-extensions/hadoop-hudi-scanner/src/main/java/org/apache/doris/hudi/HadoopHudiJniScanner.java b/fe/be-java-extensions/hadoop-hudi-scanner/src/main/java/org/apache/doris/hudi/HadoopHudiJniScanner.java index e4627c56f57b2e..351d9b11c982ab 100644 --- a/fe/be-java-extensions/hadoop-hudi-scanner/src/main/java/org/apache/doris/hudi/HadoopHudiJniScanner.java +++ b/fe/be-java-extensions/hadoop-hudi-scanner/src/main/java/org/apache/doris/hudi/HadoopHudiJniScanner.java @@ -121,12 +121,13 @@ public HadoopHudiJniScanner(int fetchSize, Map params) { this.fieldInspectors = new ObjectInspector[requiredFields.length]; this.structFields = new StructField[requiredFields.length]; this.fsOptionsProps = Maps.newHashMap(); + LOG.debug("initialize hudi scanner for query {}", params.get("query_id")); for (Map.Entry entry : params.entrySet()) { if (entry.getKey().startsWith(HADOOP_CONF_PREFIX)) { fsOptionsProps.put(entry.getKey().substring(HADOOP_CONF_PREFIX.length()), entry.getValue()); } if (LOG.isDebugEnabled()) { - LOG.debug("get hudi params {}: {}", entry.getKey(), entry.getValue()); + LOG.debug("get hudi parameter: {}", entry.getKey()); } } this.preExecutionAuthenticator = PreExecutionAuthenticatorCache.getAuthenticator(fsOptionsProps); diff --git a/fe/be-java-extensions/hadoop-hudi-scanner/src/test/java/org/apache/doris/hudi/HadoopHudiJniScannerLogTest.java b/fe/be-java-extensions/hadoop-hudi-scanner/src/test/java/org/apache/doris/hudi/HadoopHudiJniScannerLogTest.java new file mode 100644 index 00000000000000..656c8928e61d28 --- /dev/null +++ b/fe/be-java-extensions/hadoop-hudi-scanner/src/test/java/org/apache/doris/hudi/HadoopHudiJniScannerLogTest.java @@ -0,0 +1,137 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.hudi; + +import org.apache.logging.log4j.Level; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.core.LogEvent; +import org.apache.logging.log4j.core.Logger; +import org.apache.logging.log4j.core.appender.AbstractAppender; +import org.apache.logging.log4j.core.config.Property; +import org.junit.Assert; +import org.junit.Test; +import org.slf4j.LoggerFactory; + +import java.lang.reflect.Field; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.CopyOnWriteArrayList; + +public class HadoopHudiJniScannerLogTest { + @Test + public void logsQueryIdAndParameterNamesWithoutCredentialValues() throws Exception { + Map hadoopProperties = new HashMap<>(); + hadoopProperties.put("fs.s3a.access.key", "synthetic-access-key"); + hadoopProperties.put("fs.s3a.secret.key", "synthetic-secret-key"); + hadoopProperties.put("fs.s3a.session.token", "synthetic-session-token"); + hadoopProperties.put("fs.s3a.bucket.test-bucket.access.key", "synthetic-bucket-access-key"); + hadoopProperties.put("fs.s3a.bucket.test-bucket.secret.key", "synthetic-bucket-secret-key"); + hadoopProperties.put("fs.s3a.bucket.test-bucket.session.token", "synthetic-bucket-session-token"); + hadoopProperties.put("custom.setting", "synthetic-custom-value"); + + Map params = new HashMap<>(); + params.put("query_id", "0123456789abcdef-0123456789abcdef"); + params.put("base_path", "file:///unused"); + params.put("data_file_path", "file:///unused/data.parquet"); + params.put("data_file_length", "0"); + params.put("hudi_column_names", "id"); + params.put("hudi_column_types", "int"); + params.put("columns_types", "int"); + params.put("required_fields", "id"); + params.put("time_zone", "UTC"); + params.put("hoodie.custom.token", "synthetic-hudi-token"); + hadoopProperties.forEach((key, value) -> params.put("hadoop_conf." + key, value)); + + try (CapturedLog capture = new CapturedLog()) { + Assert.assertTrue("The test must exercise DEBUG parameter logging", + LoggerFactory.getLogger(HadoopHudiJniScanner.class).isDebugEnabled()); + HadoopHudiJniScanner scanner = new HadoopHudiJniScanner(16, params); + String text = String.join("\n", capture.messages); + Assert.assertTrue("Keep the query ID for diagnostics", text.contains(params.get("query_id"))); + for (String key : params.keySet()) { + Assert.assertTrue("Missing parameter name: " + key, text.contains(key)); + } + for (String value : hadoopProperties.values()) { + Assert.assertFalse("Hadoop parameter values must not be logged", text.contains(value)); + } + Assert.assertFalse("Non-Hadoop credentials must not be logged either", + text.contains(params.get("hoodie.custom.token"))); + + Field properties = HadoopHudiJniScanner.class.getDeclaredField("fsOptionsProps"); + properties.setAccessible(true); + Assert.assertEquals("Removing values from logs must preserve the actual Hadoop configuration", + hadoopProperties, properties.get(scanner)); + } + } + + private static class CapturedLog implements AutoCloseable { + private final List messages = new CopyOnWriteArrayList<>(); + private final Logger logger = (Logger) LogManager.getLogger(HadoopHudiJniScanner.class); + private final org.apache.log4j.Logger reload4j = + org.apache.log4j.Logger.getLogger(HadoopHudiJniScanner.class); + private final org.apache.log4j.Level previousReload4jLevel = reload4j.getLevel(); + private final boolean previousReload4jAdditive = reload4j.getAdditivity(); + private final org.apache.log4j.AppenderSkeleton reload4jAppender = new org.apache.log4j.AppenderSkeleton() { + @Override + protected void append(org.apache.log4j.spi.LoggingEvent event) { + messages.add(event.getRenderedMessage()); + } + + @Override + public void close() { + } + + @Override + public boolean requiresLayout() { + return false; + } + }; + private final Level previousLevel = logger.getLevel(); + private final boolean previousAdditive = logger.isAdditive(); + private final AbstractAppender appender = new AbstractAppender("hudi-parameter-test", null, + null, true, Property.EMPTY_ARRAY) { + @Override + public void append(LogEvent event) { + messages.add(event.getMessage().getFormattedMessage()); + } + }; + + CapturedLog() { + reload4j.setLevel(org.apache.log4j.Level.DEBUG); + reload4j.setAdditivity(false); + reload4j.addAppender(reload4jAppender); + appender.start(); + logger.setLevel(Level.DEBUG); + logger.setAdditive(false); + logger.addAppender(appender); + } + + @Override + public void close() { + logger.removeAppender(appender); + logger.setLevel(previousLevel); + logger.setAdditive(previousAdditive); + appender.stop(); + reload4j.removeAppender(reload4jAppender); + reload4j.setLevel(previousReload4jLevel); + reload4j.setAdditivity(previousReload4jAdditive); + reload4jAppender.close(); + } + } +}