From 0df5cd4adca192d4a438f27cac423886b8dcf2bf Mon Sep 17 00:00:00 2001 From: Dylan Date: Fri, 23 Oct 2026 17:27:53 +0800 Subject: [PATCH 01/13] [fix](fe) Disable LOCAL TVF by default ### What problem does this PR solve? Issue Number: N/A Problem Summary: Add the startup-only enable_local_tvf setting, defaulting to false. Reject LOCAL TVF and FILE TVF with local storage before property parsing or backend access. Explicit opt-in restores each branch's existing behavior. Dedicated regression deployments opt in to retain LOCAL coverage. ### Release note LOCAL TVF and FILE TVF with local storage are disabled by default. Set enable_local_tvf=true in fe.conf on every query-serving FE and restart those FEs to restore the existing behavior. ### Check List (For Author) - Test: 7 focused FE unit tests passed with zero failures or skips; focused FE Checkstyle passed. SQL regression coverage added; runtime validation recorded separately. - Behavior changed: Yes. Local TVF access requires explicit startup opt-in. - Does this need documentation: Yes. Release note documents the default and opt-in. --- .../java/org/apache/doris/common/Config.java | 4 + .../LocalTableValuedFunction.java | 6 + .../LocalTableValuedFunctionEnableTest.java | 169 ++++++++++++++++++ .../pipeline/cloud_p0/conf/fe_custom.conf | 3 + .../pipeline/cloud_p1/conf/fe_custom.conf | 3 + .../pipeline/external/conf/fe.conf | 3 + .../pipeline/nonConcurrent/conf/fe.conf | 3 + regression-test/pipeline/p0/conf/fe.conf | 3 + regression-test/pipeline/p1/conf/fe.conf | 3 + .../pipeline/vault_p0/conf/fe_custom.conf | 3 + .../account_p0/test_local_tvf_enable.groovy | 33 ++++ 11 files changed, 233 insertions(+) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/tablefunction/LocalTableValuedFunctionEnableTest.java create mode 100644 regression-test/suites/account_p0/test_local_tvf_enable.groovy diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index e1fc70a673362b..366e9af2ac4afd 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -3101,6 +3101,10 @@ public class Config extends ConfigBase { }) public static boolean enable_java_udf = true; + @ConfField(description = "Whether to enable LOCAL TVF and FILE TVF with local storage. Disabled by default. " + + "Set this in fe.conf and restart every FE to enable it. Existing access checks still apply.") + public static boolean enable_local_tvf = false; + @ConfField(mutable = true, masterOnly = true, description = { "开启后，可以在导入时，利用创建的全局 java_udf 函数处理数据，默认为 false。", "When enabled, data can be processed using the globally created java_udf function during import." diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/LocalTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/LocalTableValuedFunction.java index 4e4bf163145cf9..0b04465517572b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/LocalTableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/LocalTableValuedFunction.java @@ -21,6 +21,7 @@ import org.apache.doris.analysis.StorageBackend.StorageType; import org.apache.doris.catalog.Env; import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.Config; import org.apache.doris.common.UserException; import org.apache.doris.datasource.FederationBackendPolicy; import org.apache.doris.datasource.property.storage.StorageProperties; @@ -61,6 +62,11 @@ public class LocalTableValuedFunction extends ExternalFileTableValuedFunction { private boolean sharedStorage = false; public LocalTableValuedFunction(Map properties) throws AnalysisException { + if (!Config.enable_local_tvf) { + throw new org.apache.doris.nereids.exceptions.AnalysisException( + "LOCAL TVF is disabled. Set enable_local_tvf=true in fe.conf and restart every FE to enable it."); + } + // 1. analyze common properties Map props = super.parseCommonProperties(properties); diff --git a/fe/fe-core/src/test/java/org/apache/doris/tablefunction/LocalTableValuedFunctionEnableTest.java b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/LocalTableValuedFunctionEnableTest.java new file mode 100644 index 00000000000000..47c8a28bd1e4ee --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/LocalTableValuedFunctionEnableTest.java @@ -0,0 +1,169 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.tablefunction; + +import org.apache.doris.catalog.Env; +import org.apache.doris.common.Config; +import org.apache.doris.mysql.privilege.AccessControllerManager; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.proto.InternalService.PGlobResponse; +import org.apache.doris.proto.Types.PStatus; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.SessionVariable; +import org.apache.doris.resource.computegroup.ComputeGroup; +import org.apache.doris.rpc.BackendServiceProxy; +import org.apache.doris.system.Backend; +import org.apache.doris.system.SystemInfoService; +import org.apache.doris.thrift.TNetworkAddress; + +import com.google.common.collect.ImmutableMap; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.MockedConstruction; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +import java.util.Collections; +import java.util.Map; +import java.util.concurrent.CompletableFuture; + +public class LocalTableValuedFunctionEnableTest { + private static final Map PROPERTIES = ImmutableMap.of( + "file_path", "local_privilege_test.csv", "backend_id", "1", "format", "csv"); + + private final Env env = Mockito.mock(Env.class); + private final ConnectContext context = Mockito.mock(ConnectContext.class); + private final AccessControllerManager accessManager = Mockito.mock(AccessControllerManager.class); + private final SystemInfoService systemInfo = Mockito.mock(SystemInfoService.class); + private final BackendServiceProxy proxy = Mockito.mock(BackendServiceProxy.class); + private boolean originalEnableLocalTvf; + private MockedStatic mockedEnv; + private MockedStatic mockedContext; + private MockedStatic mockedProxy; + + @BeforeEach + public void setUp() throws Exception { + originalEnableLocalTvf = Config.enable_local_tvf; + Config.enable_local_tvf = false; + mockedEnv = Mockito.mockStatic(Env.class); + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + mockedEnv.when(Env::getCurrentSystemInfo).thenReturn(systemInfo); + mockedContext = Mockito.mockStatic(ConnectContext.class); + mockedContext.when(ConnectContext::get).thenReturn(context); + mockedProxy = Mockito.mockStatic(BackendServiceProxy.class); + mockedProxy.when(BackendServiceProxy::getInstance).thenReturn(proxy); + Mockito.when(env.getAccessManager()).thenReturn(accessManager); + Mockito.when(context.getSessionVariable()).thenReturn(new SessionVariable()); + Mockito.when(context.getQueryTimeoutS()).thenReturn(60); + + Backend backend = Mockito.mock(Backend.class); + Mockito.when(backend.getId()).thenReturn(1L); + Mockito.when(backend.getHost()).thenReturn("127.0.0.1"); + Mockito.when(backend.isAlive()).thenReturn(true); + Mockito.when(backend.isQueryAvailable()).thenReturn(true); + Mockito.when(backend.isLoadAvailable()).thenReturn(true); + ComputeGroup computeGroup = Mockito.mock(ComputeGroup.class); + Mockito.when(context.getComputeGroup()).thenReturn(computeGroup); + Mockito.when(computeGroup.getBackendList()).thenReturn(Collections.singletonList(backend)); + Mockito.when(systemInfo.getBackend(1L)).thenReturn(backend); + Mockito.when(backend.getBrpcAddress()).thenReturn(new TNetworkAddress("127.0.0.1", 8060)); + Mockito.when(proxy.glob(Mockito.any(), Mockito.any())).thenReturn(CompletableFuture.completedFuture( + PGlobResponse.newBuilder().setStatus(PStatus.newBuilder().setStatusCode(0)).build())); + } + + @AfterEach + public void tearDown() { + Config.enable_local_tvf = originalEnableLocalTvf; + mockedProxy.close(); + mockedContext.close(); + mockedEnv.close(); + } + + @Test + public void testDisabledRejectsLocalBeforePropertyParsing() { + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> new LocalTableValuedFunction(ImmutableMap.of())); + Assertions.assertTrue(exception.getMessage().contains("enable_local_tvf=true")); + Mockito.verifyNoInteractions(accessManager, systemInfo, proxy); + } + + @Test + public void testDisabledRejectsCsvAndLanceBeforeFileAccess() { + for (String format : new String[] {"csv", "lance"}) { + Map properties = ImmutableMap.of( + "file_path", "local_enable_test.csv", "backend_id", "1", "format", format); + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> new LocalTableValuedFunction(properties)); + Assertions.assertTrue(exception.getMessage().contains("LOCAL TVF is disabled")); + } + Mockito.verifyNoInteractions(accessManager, systemInfo, proxy); + } + + @Test + public void testEnabledLocalReachesExistingFileAccess() throws Exception { + Config.enable_local_tvf = true; + Mockito.when(accessManager.checkGlobalPriv(context, PrivPredicate.ADMIN)).thenReturn(true); + new LocalTableValuedFunction(PROPERTIES); + Mockito.verify(proxy).glob(Mockito.any(), Mockito.any()); + } + + @Test + public void testDisabledRejectsFileLocalBeforeFileAccess() { + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> new FileTableValuedFunction(PROPERTIES)); + Assertions.assertTrue(exception.getMessage().contains("LOCAL TVF is disabled")); + Mockito.verifyNoInteractions(accessManager, systemInfo, proxy); + } + + @Test + public void testEnabledFileLocalReachesExistingFileAccess() throws Exception { + Config.enable_local_tvf = true; + Mockito.when(accessManager.checkGlobalPriv(context, PrivPredicate.ADMIN)).thenReturn(true); + new FileTableValuedFunction(PROPERTIES); + Mockito.verify(proxy).glob(Mockito.any(), Mockito.any()); + } + + @Test + public void testDisabledLocalDoesNotBlockFileS3() { + Map properties = ImmutableMap.builder() + .put("uri", "s3://test-bucket/local_enable_test.csv") + .put("s3.endpoint", "s3.us-east-1.amazonaws.com") + .put("s3.region", "us-east-1") + .put("s3.access_key", "test-access-key") + .put("s3.secret_key", "test-secret-key") + .put("format", "csv") + .build(); + try (MockedConstruction mockedS3 = + Mockito.mockConstruction(S3TableValuedFunction.class)) { + Assertions.assertDoesNotThrow(() -> new FileTableValuedFunction(properties)); + Assertions.assertEquals(1, mockedS3.constructed().size()); + } + Mockito.verifyNoInteractions(accessManager, systemInfo, proxy); + } + + @Test + public void testEnabledLancePreservesExistingBackendSelection() throws Exception { + Config.enable_local_tvf = true; + new LocalTableValuedFunction(ImmutableMap.of( + "file_path", "local_enable_test.lance", "backend_id", "1", "format", "lance")); + Mockito.verifyNoInteractions(proxy); + } +} diff --git a/regression-test/pipeline/cloud_p0/conf/fe_custom.conf b/regression-test/pipeline/cloud_p0/conf/fe_custom.conf index d6c2070aa939e2..7139bfa6e02aa8 100644 --- a/regression-test/pipeline/cloud_p0/conf/fe_custom.conf +++ b/regression-test/pipeline/cloud_p0/conf/fe_custom.conf @@ -1,3 +1,6 @@ +# Exercise LOCAL TVF explicitly; production defaults to disabled. +enable_local_tvf=true + stream_load_default_timeout_second = 72000 replication_num_forced_in_cloud_mode = true ignore_unsupported_properties_in_cloud_mode = true diff --git a/regression-test/pipeline/cloud_p1/conf/fe_custom.conf b/regression-test/pipeline/cloud_p1/conf/fe_custom.conf index 157f3a07a09b47..e671d0a7dd15b2 100644 --- a/regression-test/pipeline/cloud_p1/conf/fe_custom.conf +++ b/regression-test/pipeline/cloud_p1/conf/fe_custom.conf @@ -1,3 +1,6 @@ +# Exercise LOCAL TVF explicitly; production defaults to disabled. +enable_local_tvf=true + stream_load_default_timeout_second = 72000 replication_num_forced_in_cloud_mode = true ignore_unsupported_properties_in_cloud_mode = true diff --git a/regression-test/pipeline/external/conf/fe.conf b/regression-test/pipeline/external/conf/fe.conf index 4466998ae67163..c0160fa91f07b5 100644 --- a/regression-test/pipeline/external/conf/fe.conf +++ b/regression-test/pipeline/external/conf/fe.conf @@ -1,3 +1,6 @@ +# Exercise LOCAL TVF explicitly; production defaults to disabled. +enable_local_tvf=true + # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/regression-test/pipeline/nonConcurrent/conf/fe.conf b/regression-test/pipeline/nonConcurrent/conf/fe.conf index e89a00d209ccf7..737e305a1bacf4 100644 --- a/regression-test/pipeline/nonConcurrent/conf/fe.conf +++ b/regression-test/pipeline/nonConcurrent/conf/fe.conf @@ -1,3 +1,6 @@ +# Exercise LOCAL TVF explicitly; production defaults to disabled. +enable_local_tvf=true + # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/regression-test/pipeline/p0/conf/fe.conf b/regression-test/pipeline/p0/conf/fe.conf index 3d86c5c95e6332..494bc15cfb9bd1 100644 --- a/regression-test/pipeline/p0/conf/fe.conf +++ b/regression-test/pipeline/p0/conf/fe.conf @@ -1,3 +1,6 @@ +# Exercise LOCAL TVF explicitly; production defaults to disabled. +enable_local_tvf=true + # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/regression-test/pipeline/p1/conf/fe.conf b/regression-test/pipeline/p1/conf/fe.conf index f20d51b62e6575..04037acd3644b9 100644 --- a/regression-test/pipeline/p1/conf/fe.conf +++ b/regression-test/pipeline/p1/conf/fe.conf @@ -1,3 +1,6 @@ +# Exercise LOCAL TVF explicitly; production defaults to disabled. +enable_local_tvf=true + # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/regression-test/pipeline/vault_p0/conf/fe_custom.conf b/regression-test/pipeline/vault_p0/conf/fe_custom.conf index f62ffa19fc7676..f4c14df5d447b0 100644 --- a/regression-test/pipeline/vault_p0/conf/fe_custom.conf +++ b/regression-test/pipeline/vault_p0/conf/fe_custom.conf @@ -1,3 +1,6 @@ +# Exercise LOCAL TVF explicitly; production defaults to disabled. +enable_local_tvf=true + stream_load_default_timeout_second = 72000 replication_num_forced_in_cloud_mode = true ignore_unsupported_properties_in_cloud_mode = true diff --git a/regression-test/suites/account_p0/test_local_tvf_enable.groovy b/regression-test/suites/account_p0/test_local_tvf_enable.groovy new file mode 100644 index 00000000000000..b6853b51ebd5cb --- /dev/null +++ b/regression-test/suites/account_p0/test_local_tvf_enable.groovy @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_local_tvf_enable", "p0") { + def enabled = (sql "ADMIN SHOW FRONTEND CONFIG LIKE 'enable_local_tvf'")[0][1].toBoolean() + for (def functionName : ["LOCAL", "FILE"]) { + for (def prefix : ["", "EXPLAIN "]) { + test { + sql """${prefix}SELECT * FROM ${functionName}( + "file_path" = "local_enable_test.csv", + "backend_id" = "-1", + "format" = "csv" + )""" + // With opt-in enabled, the existing backend validation must be reached. + exception (enabled ? "'backend_id' is required" : "LOCAL TVF is disabled") + } + } + } +} From bd44070f296c1a02893f5a0f33594847985c68fe Mon Sep 17 00:00:00 2001 From: Dylan Date: Fri, 23 Oct 2026 18:54:13 +0800 Subject: [PATCH 02/13] [fix](tvf) Enforce secure paths for enabled LOCAL reads ### What problem does this PR solve? Issue Number: N/A Related PR: #69526 Problem Summary: Canonical glob results were not checked against the secure root, and LOCAL Lance bypassed local-path validation. Validate canonical roots and resolved paths, check ADMIN/OPERATOR before file access, validate LOCAL Lance schema/scan paths, and preserve LOCAL/S3 range types. Limit Lance validation to the existing LOCAL TVF version-zero contract so fixed-version local catalogs retain their warehouse paths. Reject LocalProperties selected through S3 Lance before dataset opening. ### Release note Enabled LOCAL reads remain restricted to user_files_secure_path and require ADMIN/OPERATOR. LOCAL Lance accepts only local paths inside that directory. The default-disabled startup gate is retained. ### Check List (For Author) - Test: 26 focused FE tests, Java Checkstyle and changed-line clang-format 16 passed. BE execution is blocked by the installed Lance-C SDK missing the existing lance_scanner_set_distance_range API; no BE runtime pass is claimed. - Behavior changed: Yes, enforce secure local paths and authorization after explicit opt-in. - Does this need documentation: Covered by the PR release note. --- be/src/format_v2/table/lance_reader.cpp | 36 ++++++-- be/src/io/fs/local_file_system.cpp | 33 +++++-- be/src/io/fs/local_file_system.h | 2 + be/test/format_v2/table/lance_reader_test.cpp | 84 +++++++++++++++++ be/test/io/fs/local_file_system_test.cpp | 90 +++++++++++++++++++ .../datasource/tvf/source/TVFScanNode.java | 3 + .../LocalTableValuedFunction.java | 15 ++++ .../tablefunction/S3TableValuedFunction.java | 6 ++ .../tvf/source/TVFScanNodeTest.java | 23 +++++ .../LocalTableValuedFunctionEnableTest.java | 36 ++++++++ .../S3TableValuedFunctionTest.java | 9 ++ 11 files changed, 325 insertions(+), 12 deletions(-) diff --git a/be/src/format_v2/table/lance_reader.cpp b/be/src/format_v2/table/lance_reader.cpp index e0e2884efe7a4a..cb47679fb8bbfa 100644 --- a/be/src/format_v2/table/lance_reader.cpp +++ b/be/src/format_v2/table/lance_reader.cpp @@ -35,11 +35,36 @@ #include "format_v2/lance/lance_reader_helper.h" #include "format_v2/lance/lance_runtime_filter_helper.h" #include "format_v2/lance/lance_session_manager.h" +#include "io/fs/local_file_system.h" #include "runtime/file_scan_profile.h" #include "runtime/runtime_state.h" namespace doris::format::lance { +namespace { +Status resolve_dataset_uri(const TFileRangeDesc& range, const TFileScanRangeParams& scan_params, + std::string* uri) { + const auto& requested_uri = range.table_format_params.lance_params.dataset_uri; + // Modern plans tag each range; older plans put the type in the scan parameters. + // Untagged catalog scans retain their original storage-provider selection. + const bool is_local = range.__isset.file_type + ? range.file_type == TFileType::FILE_LOCAL + : scan_params.__isset.file_type && + scan_params.file_type == TFileType::FILE_LOCAL; + // LOCAL TVF uses version zero for both schema discovery and whole-dataset scans. + // Catalog scans pin a positive snapshot version, including file:// catalogs whose + // operator-configured warehouse is independent of the LOCAL TVF secure directory. + if (is_local && range.table_format_params.lance_params.version == 0) { + // LOCAL Lance bypasses glob because it opens a dataset directory. Apply the same + // canonical secure-root check before either schema discovery or a cached dataset open. + return io::global_local_filesystem()->canonicalize_local_file( + config::user_files_secure_path, requested_uri, uri); + } + *uri = requested_uri; + return Status::OK(); +} +} // namespace + LanceTableReader::~LanceTableReader() { static_cast(close()); } @@ -52,6 +77,8 @@ Status LanceTableReader::fetch_schema(const TFileRangeDesc& range, return Status::InvalidArgument("Lance schema output must not be null"); } const auto& params = range.table_format_params.lance_params; + std::string dataset_uri; + RETURN_IF_ERROR(resolve_dataset_uri(range, scan_params, &dataset_uri)); std::vector storage_options; RETURN_IF_ERROR(build_lance_storage_options(&scan_params, &storage_options)); std::vector storage_option_ptrs; @@ -61,10 +88,9 @@ Status LanceTableReader::fetch_schema(const TFileRangeDesc& range, } storage_option_ptrs.emplace_back(nullptr); - std::unique_ptr dataset( - lance_dataset_open(params.dataset_uri.c_str(), - storage_options.empty() ? nullptr : storage_option_ptrs.data(), - static_cast(params.version))); + std::unique_ptr dataset(lance_dataset_open( + dataset_uri.c_str(), storage_options.empty() ? nullptr : storage_option_ptrs.data(), + static_cast(params.version))); if (dataset == nullptr) { return lance_error("open Lance dataset for schema"); } @@ -1329,7 +1355,7 @@ Status LanceTableReader::_fill_block_from_lance_batch(LanceBatch* batch, Block* Status LanceTableReader::_dataset_key(const TFileRangeDesc& range, DatasetKey* key) const { const auto& params = range.table_format_params.lance_params; - key->uri = params.dataset_uri; + RETURN_IF_ERROR(resolve_dataset_uri(range, *_scan_params, &key->uri)); key->version = params.version; return build_lance_storage_options(_scan_params, &key->storage_options); } diff --git a/be/src/io/fs/local_file_system.cpp b/be/src/io/fs/local_file_system.cpp index 4f9b6baf2a75cf..ff4cd3eaedaed2 100644 --- a/be/src/io/fs/local_file_system.cpp +++ b/be/src/io/fs/local_file_system.cpp @@ -455,14 +455,26 @@ const std::shared_ptr& global_local_filesystem() { Status LocalFileSystem::canonicalize_local_file(const std::string& dir, const std::string& file_path, std::string* full_path) { - const std::string absolute_path = dir + "/" + file_path; + // An empty root would turn a relative request into an unrestricted absolute path. + if (dir.empty()) { + return Status::InvalidArgument("user_files_secure_path must not be empty"); + } + // Reject embedded NULs before passing paths to filesystem APIs that use C strings. + if (file_path.empty() || file_path.find('\0') != std::string::npos) { + return Status::InvalidArgument("local file path must not be empty or contain NUL"); + } + Path local_path; + // Convert file: URIs to filesystem paths and reject remote schemes before any reader opens them. + RETURN_IF_ERROR(convert_to_abs_path(file_path, local_path)); + std::string secure_path; + RETURN_IF_ERROR(canonicalize(dir, &secure_path)); std::string canonical_path; - RETURN_IF_ERROR(canonicalize(absolute_path, &canonical_path)); - if (!contain_path(dir, canonical_path)) { + RETURN_IF_ERROR(canonicalize(Path(secure_path) / local_path, &canonical_path)); + // Canonicalize both sides: a symlink inside the root may point to a sibling directory. + if (!contain_path(secure_path, canonical_path)) { return Status::InvalidArgument("file path is not allowed: {}", canonical_path); } - - *full_path = canonical_path; + *full_path = std::move(canonical_path); return Status::OK(); } @@ -470,13 +482,20 @@ Status LocalFileSystem::safe_glob(const std::string& path, std::vector if (path.find("..") != std::string::npos) { return Status::InvalidArgument("can not contain '..' in path"); } - std::string full_path = config::user_files_secure_path + "/" + path; + // An empty root would turn the prefix below into "/" and allow any local file. + if (config::user_files_secure_path.empty()) { + return Status::InvalidArgument("user_files_secure_path must not be empty"); + } + std::string secure_path; + RETURN_IF_ERROR(canonicalize(config::user_files_secure_path, &secure_path)); + std::string full_path = secure_path + "/" + path; std::vector files; RETURN_IF_ERROR(_glob(full_path, &files)); for (auto& file : files) { FileInfo fi; fi.is_file = true; - RETURN_IF_ERROR(canonicalize_local_file("", file, &(fi.file_name))); + // Glob classes and symlinks can escape without a literal ".." in the pattern. + RETURN_IF_ERROR(canonicalize_local_file(secure_path, file, &(fi.file_name))); RETURN_IF_ERROR(file_size_impl(fi.file_name, &(fi.file_size))); res->push_back(std::move(fi)); } diff --git a/be/src/io/fs/local_file_system.h b/be/src/io/fs/local_file_system.h index b1546bc473426f..1f7ffc8da51964 100644 --- a/be/src/io/fs/local_file_system.h +++ b/be/src/io/fs/local_file_system.h @@ -70,6 +70,8 @@ class LocalFileSystem final : public FileSystem { static std::filesystem::perms PERMS_OWNER_RW; + // Resolve a local path relative to a nonempty secure root, then check canonical containment. + // Absolute paths and file: URIs must resolve inside the same root; remote schemes are rejected. Status canonicalize_local_file(const std::string& dir, const std::string& file_path, std::string* full_path); diff --git a/be/test/format_v2/table/lance_reader_test.cpp b/be/test/format_v2/table/lance_reader_test.cpp index 6531fc21de72e8..3ebc50e11c7189 100644 --- a/be/test/format_v2/table/lance_reader_test.cpp +++ b/be/test/format_v2/table/lance_reader_test.cpp @@ -49,6 +49,7 @@ #include #include +#include "common/config.h" #include "common/consts.h" #include "core/assert_cast.h" #include "core/block/block.h" @@ -4313,3 +4314,86 @@ TEST(LanceTableReaderTypeTest, ReadsTemporalTypesFromAllTypesFixture) { } } // namespace doris::format::lance + +namespace doris::format::lance { +TEST(LanceTableReaderLocalPathTest, SchemaAndScanUseTheSameSecurePath) { + const auto original_secure_path = config::user_files_secure_path; + DEFER(config::user_files_secure_path = original_secure_path); + const auto root = std::filesystem::canonical("./be/test/format_v2/table/lance/data"); + config::user_files_secure_path = root.string(); + TFileScanRangeParams scan_params; + LanceTableReader reader; + reader._scan_params = &scan_params; + auto range = make_latest_lance_range("all_types.lance"); + range.__set_file_type(TFileType::FILE_LOCAL); + std::vector names; + std::vector types; + ASSERT_TRUE(reader.fetch_schema(range, scan_params, &names, &types).ok()); + EXPECT_FALSE(names.empty()); + LanceTableReader::DatasetKey key; + ASSERT_TRUE(reader._dataset_key(range, &key).ok()); + EXPECT_EQ((root / "all_types.lance").string(), key.uri); + + // Legacy scan-level tags must enforce the same restriction as per-range tags. + range.__isset.file_type = false; + scan_params.__set_file_type(TFileType::FILE_LOCAL); + ASSERT_TRUE(reader._dataset_key(range, &key).ok()); + EXPECT_EQ((root / "all_types.lance").string(), key.uri); + + for (const auto* uri : {"https://invalid/dataset", "http://invalid/dataset", + "s3://bucket/dataset", "../data-other/dataset"}) { + range.table_format_params.lance_params.__set_dataset_uri(uri); + const auto schema_status = reader.fetch_schema(range, scan_params, &names, &types); + const auto scan_status = reader._dataset_key(range, &key); + EXPECT_FALSE(schema_status.ok()) << uri; + EXPECT_EQ(schema_status.to_string(), scan_status.to_string()); + } +} + +TEST(LanceTableReaderLocalPathTest, ScansRelativeDatasetInsideSecureRoot) { + const auto original_secure_path = config::user_files_secure_path; + DEFER(config::user_files_secure_path = original_secure_path); + config::user_files_secure_path = + std::filesystem::canonical("./be/test/format_v2/table/lance/data").string(); + TQueryGlobals query_globals; + RuntimeState state(query_globals); + RuntimeProfile profile("lance_secure_local_scan"); + TFileScanRangeParams scan_params; + const Columns columns {projected_column("row_id", TYPE_BIGINT, false)}; + LanceTableReader reader; + ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, &scan_params).ok()); + auto range = make_latest_lance_range("all_types.lance"); + range.__set_file_type(TFileType::FILE_LOCAL); + ASSERT_TRUE(prepare_range(&reader, range).ok()); + ASSERT_TRUE(reader.close().ok()); +} + +TEST(LanceTableReaderLocalPathTest, FixedVersionLocalCatalogKeepsItsUri) { + TFileScanRangeParams scan_params; + LanceTableReader reader; + reader._scan_params = &scan_params; + // Catalog warehouses use their own location, not the LOCAL TVF secure directory. + auto range = make_lance_range("file:///catalog-owned/dataset.lance", 7, {0}); + range.__set_file_type(TFileType::FILE_LOCAL); + LanceTableReader::DatasetKey key; + ASSERT_TRUE(reader._dataset_key(range, &key).ok()); + EXPECT_EQ("file:///catalog-owned/dataset.lance", key.uri); + EXPECT_EQ(7, key.version); +} + +TEST(LanceTableReaderLocalPathTest, RemoteRangeKeepsItsUriAndOptions) { + TFileScanRangeParams scan_params; + scan_params.__set_file_type(TFileType::FILE_LOCAL); + TLanceScanParams lance_params; + lance_params.__set_lance_storage_options({{"aws_region", "us-east-1"}}); + scan_params.__set_lance_scan_params(lance_params); + LanceTableReader reader; + reader._scan_params = &scan_params; + auto range = make_latest_lance_range("s3://bucket/dataset.lance"); + range.__set_file_type(TFileType::FILE_S3); + LanceTableReader::DatasetKey key; + ASSERT_TRUE(reader._dataset_key(range, &key).ok()); + EXPECT_EQ("s3://bucket/dataset.lance", key.uri); + EXPECT_EQ((std::vector {"aws_region", "us-east-1"}), key.storage_options); +} +} // namespace doris::format::lance diff --git a/be/test/io/fs/local_file_system_test.cpp b/be/test/io/fs/local_file_system_test.cpp index 4321a2ae56290f..314cb8ac2438ac 100644 --- a/be/test/io/fs/local_file_system_test.cpp +++ b/be/test/io/fs/local_file_system_test.cpp @@ -30,12 +30,14 @@ #include #include +#include "common/config.h" #include "common/status.h" #include "cpp/sync_point.h" #include "gtest/gtest_pred_impl.h" #include "io/fs/file_reader.h" #include "io/fs/file_writer.h" #include "io/fs/local_file_writer.h" +#include "util/defer_op.h" #include "util/slice.h" namespace doris { @@ -547,6 +549,10 @@ TEST_F(LocalFileSystemTest, AbnormalWriteRead) { } TEST_F(LocalFileSystemTest, TestGlob) { + const std::string original_secure_path = config::user_files_secure_path; + DEFER(config::user_files_secure_path = original_secure_path); + // Bind this fixture to its own files, independent of the runner's build-profile directory. + config::user_files_secure_path = std::filesystem::absolute("./be/ut_build_ASAN/test").string(); std::string path = "./be/ut_build_ASAN/test/file_path/"; EXPECT_TRUE(io::global_local_filesystem()->delete_directory(path).ok()); EXPECT_TRUE(io::global_local_filesystem() @@ -578,6 +584,90 @@ TEST_F(LocalFileSystemTest, TestGlob) { EXPECT_TRUE(io::global_local_filesystem()->delete_directory(path).ok()); } +TEST_F(LocalFileSystemTest, TestSafeGlobPathContainment) { + auto fs = io::global_local_filesystem(); + const std::string original_secure_path = config::user_files_secure_path; + DEFER(config::user_files_secure_path = original_secure_path); + const auto root = std::filesystem::canonical(test_dir); + const auto secure = root / "secure"; + const auto outside = root / "secure-other"; + std::filesystem::create_directories(secure / "nested"); + std::filesystem::create_directory(outside); + ASSERT_TRUE(save_string_file((secure / "nested/inside.csv").string(), "inside").ok()); + ASSERT_TRUE(save_string_file((outside / "outside.csv").string(), "outside").ok()); + std::filesystem::create_directory_symlink(outside, secure / "escape"); + std::filesystem::create_symlink(outside / "outside.csv", secure / "outside.csv"); + std::filesystem::create_directory_symlink(secure / "nested", secure / "inside"); + config::user_files_secure_path = secure.string(); + + for (const auto* pattern : {".[.]/secure-other/outside.csv", ".?/secure-other/outside.csv", + ".[.-.]/secure-other/outside.csv", "escape/*.csv", "outside.csv"}) { + std::vector files; + const auto status = fs->safe_glob(pattern, &files); + EXPECT_FALSE(status.ok()) << pattern; + EXPECT_NE(status.to_string().find("file path is not allowed"), std::string::npos) << status; + } + + for (const auto* pattern : {"nested/*.csv", "inside/*.csv"}) { + std::vector files; + ASSERT_TRUE(fs->safe_glob(pattern, &files).ok()) << pattern; + ASSERT_EQ(1, files.size()); + EXPECT_EQ((secure / "nested/inside.csv").string(), files[0].file_name); + EXPECT_EQ(6, files[0].file_size); + } +} + +TEST_F(LocalFileSystemTest, TestSafeGlobSecureRoot) { + auto fs = io::global_local_filesystem(); + const std::string original_secure_path = config::user_files_secure_path; + DEFER(config::user_files_secure_path = original_secure_path); + const auto root = std::filesystem::canonical(test_dir); + std::filesystem::create_directory(root / "secure"); + std::filesystem::create_directory_symlink(root / "secure", root / "secure-link"); + ASSERT_TRUE(save_string_file((root / "secure/inside.csv").string(), "inside").ok()); + + for (const auto& secure : {std::string(test_dir) + "/secure", (root / "secure/").string(), + (root / "secure-link").string()}) { + config::user_files_secure_path = secure; + std::vector files; + ASSERT_TRUE(fs->safe_glob("*.csv", &files).ok()) << secure; + ASSERT_EQ(1, files.size()); + EXPECT_EQ((root / "secure/inside.csv").string(), files[0].file_name); + } + config::user_files_secure_path = ""; + std::vector files; + EXPECT_FALSE(fs->safe_glob("*.csv", &files).ok()); + config::user_files_secure_path = (root / "missing").string(); + EXPECT_FALSE(fs->safe_glob("*.csv", &files).ok()); +} + +TEST_F(LocalFileSystemTest, TestCanonicalizeSecureLocalPath) { + auto fs = io::global_local_filesystem(); + const auto root = std::filesystem::canonical(test_dir); + const auto secure = root / "secure"; + std::filesystem::create_directories(secure / "nested"); + std::filesystem::create_directory(root / "secure-other"); + std::filesystem::create_directory_symlink(root / "secure-other", secure / "escape"); + std::filesystem::create_directory_symlink(secure / "nested", secure / "inside"); + for (const auto& path : + {std::string("nested"), std::string("nested/../inside"), (secure / "nested").string(), + "file://" + (secure / "nested").string()}) { + std::string resolved; + ASSERT_TRUE(fs->canonicalize_local_file(secure.string(), path, &resolved).ok()) << path; + EXPECT_EQ((secure / "nested").string(), resolved); + } + for (const auto& path : + {std::string("../secure-other"), std::string("escape"), (root / "secure-other").string(), + std::string("https://invalid/dataset"), std::string("s3://bucket/dataset"), + std::string("http://invalid/dataset"), std::string(""), + std::string("nested\0ignored", 14)}) { + std::string resolved; + EXPECT_FALSE(fs->canonicalize_local_file(secure.string(), path, &resolved).ok()) << path; + } + std::string resolved; + EXPECT_FALSE(fs->canonicalize_local_file("", secure.string(), &resolved).ok()); +} + TEST_F(LocalFileSystemTest, TestConvertToAbsPath) { io::Path abs_path; Status st; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/tvf/source/TVFScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/tvf/source/TVFScanNode.java index 297272cb582250..40959143298cdd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/tvf/source/TVFScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/tvf/source/TVFScanNode.java @@ -241,6 +241,9 @@ protected void setScanParams(TFileRangeDesc rangeDesc, Split split) { if (!(split instanceof LanceSplit)) { throw new IllegalArgumentException("Expected LanceSplit but got " + split.getClass().getName()); } + // A bare local path has no URI scheme, so split inference cannot supply its type. + // Preserve the TVF's LOCAL/S3 identity for the BE's pre-open path validation. + rangeDesc.setFileType(tableValuedFunction.getTFileType()); LanceSplit lanceSplit = (LanceSplit) split; TLanceFileDesc lanceParams = new TLanceFileDesc(); lanceParams.setDatasetUri(lanceSplit.getDatasetUri()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/LocalTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/LocalTableValuedFunction.java index 0b04465517572b..8d2d09e5de06ee 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/LocalTableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/LocalTableValuedFunction.java @@ -22,9 +22,11 @@ import org.apache.doris.catalog.Env; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Config; +import org.apache.doris.common.ErrorCode; import org.apache.doris.common.UserException; import org.apache.doris.datasource.FederationBackendPolicy; import org.apache.doris.datasource.property.storage.StorageProperties; +import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.proto.InternalService; import org.apache.doris.proto.InternalService.PGlobResponse; import org.apache.doris.proto.InternalService.PGlobResponse.PFileInfo; @@ -67,6 +69,9 @@ public LocalTableValuedFunction(Map properties) throws AnalysisE "LOCAL TVF is disabled. Set enable_local_tvf=true in fe.conf and restart every FE to enable it."); } + // Check privileges before listing files or discovering their schema. + checkAuth(ConnectContext.get()); + // 1. analyze common properties Map props = super.parseCommonProperties(properties); @@ -141,6 +146,16 @@ private void getFileListFromBackend() throws AnalysisException { } } + @Override + public void checkAuth(ConnectContext ctx) { + if (!Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ctx, PrivPredicate.ADMIN) + && !Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ctx, PrivPredicate.OPERATOR)) { + String message = ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR.formatErrorMsg("ADMIN/OPERATOR"); + throw new org.apache.doris.nereids.exceptions.AnalysisException(message); + } + super.checkAuth(ctx); + } + @Override public TFileType getTFileType() { return TFileType.FILE_LOCAL; diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java index 91e79dda10e641..03a2799678ae2f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java @@ -24,6 +24,7 @@ import org.apache.doris.common.UserException; import org.apache.doris.datasource.lance.metadata.LanceMetadataLoader; import org.apache.doris.datasource.lance.metadata.LanceTableMetadata; +import org.apache.doris.datasource.property.storage.LocalProperties; import org.apache.doris.datasource.property.storage.StorageProperties; import org.apache.doris.thrift.TBrokerFileStatus; import org.apache.doris.thrift.TFileType; @@ -63,6 +64,11 @@ public S3TableValuedFunction(Map properties, List new LocalTableValuedFunction(PROPERTIES)); + Assertions.assertTrue(exception.getMessage().contains("ADMIN/OPERATOR")); + Mockito.verifyNoInteractions(systemInfo, proxy); + } + + @Test + public void testOperatorCanAccessLocalFiles() throws Exception { + Config.enable_local_tvf = true; + Mockito.when(accessManager.checkGlobalPriv(context, PrivPredicate.OPERATOR)).thenReturn(true); + new LocalTableValuedFunction(PROPERTIES); + Mockito.verify(proxy).glob(Mockito.any(), Mockito.any()); + } + + @Test + public void testCheckAuthRejectsRevokedPrivileges() throws Exception { + Config.enable_local_tvf = true; + Mockito.when(accessManager.checkGlobalPriv(context, PrivPredicate.ADMIN)).thenReturn(true); + LocalTableValuedFunction local = new LocalTableValuedFunction(PROPERTIES); + Mockito.when(accessManager.checkGlobalPriv(context, PrivPredicate.ADMIN)).thenReturn(false); + Assertions.assertThrows(AnalysisException.class, () -> local.checkAuth(context)); + } + + @Test + public void testEnabledFileRejectsUnprivilegedUserBeforeFileAccess() { + Config.enable_local_tvf = true; + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> new FileTableValuedFunction(PROPERTIES)); + Assertions.assertTrue(exception.getMessage().contains("ADMIN/OPERATOR")); + Mockito.verifyNoInteractions(systemInfo, proxy); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/tablefunction/S3TableValuedFunctionTest.java b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/S3TableValuedFunctionTest.java index 2d79b5a776a0de..12885a42e62198 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/tablefunction/S3TableValuedFunctionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/S3TableValuedFunctionTest.java @@ -71,6 +71,15 @@ public void tearDown() { } } + @Test + public void testLanceRejectsLocalStorageBeforeOpeningDataset() { + org.apache.doris.common.AnalysisException exception = Assertions.assertThrows( + org.apache.doris.common.AnalysisException.class, + () -> new S3TableValuedFunction(ImmutableMap.of( + "file_path", "dataset.lance", "format", "lance"))); + Assertions.assertTrue(exception.getMessage().contains("requires remote storage")); + } + @Test public void testExactFilesKeepLiteralPathsAndSkipListing() throws Exception { FeConstants.runningUnitTest = false; From 2de7b316d80bdab19e2557f31f43597ecfc1383e Mon Sep 17 00:00:00 2001 From: Dylan Date: Fri, 23 Oct 2026 19:18:51 +0800 Subject: [PATCH 03/13] [fix](tvf) Redact resolved names from local path errors ### What problem does this PR solve? Issue Number: N/A Related PR: #69526 Problem Summary: A rejected glob result could disclose its expanded filename through a canonical-path error or a failed resolution of a dangling symlink. Return one generic target-validation error for both cases before the status reaches SQL clients. Extend temporary-directory tests to assert that neither real target names nor paths appear in errors. ### Release note LOCAL path validation errors no longer reveal expanded or resolved target filenames. ### Check List (For Author) - Test: Targeted compilation diagnostics passed for the two modified C++ files. Full BE execution remains blocked by the previously recorded Lance-C SDK/GCS dependency gaps. Modified lines pass clang-format 16; independent Standards and Spec review found no remaining issue in this supplement. - Behavior changed: Yes, target-validation errors use a generic message. - Does this need documentation: Covered by the PR release note. --- be/src/io/fs/local_file_system.cpp | 9 +++++---- be/test/io/fs/local_file_system_test.cpp | 15 +++++++++++++-- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/be/src/io/fs/local_file_system.cpp b/be/src/io/fs/local_file_system.cpp index ff4cd3eaedaed2..da9624fe544b0f 100644 --- a/be/src/io/fs/local_file_system.cpp +++ b/be/src/io/fs/local_file_system.cpp @@ -469,10 +469,11 @@ Status LocalFileSystem::canonicalize_local_file(const std::string& dir, std::string secure_path; RETURN_IF_ERROR(canonicalize(dir, &secure_path)); std::string canonical_path; - RETURN_IF_ERROR(canonicalize(Path(secure_path) / local_path, &canonical_path)); - // Canonicalize both sides: a symlink inside the root may point to a sibling directory. - if (!contain_path(secure_path, canonical_path)) { - return Status::InvalidArgument("file path is not allowed: {}", canonical_path); + const auto status = canonicalize(Path(secure_path) / local_path, &canonical_path); + // A glob result can reveal external names even when resolution fails on a dangling symlink. + // Reject both failed resolution and out-of-root targets without exposing expanded names. + if (!status.ok() || !contain_path(secure_path, canonical_path)) { + return Status::InvalidArgument("file path is not allowed"); } *full_path = std::move(canonical_path); return Status::OK(); diff --git a/be/test/io/fs/local_file_system_test.cpp b/be/test/io/fs/local_file_system_test.cpp index 314cb8ac2438ac..0b679a7e0c908c 100644 --- a/be/test/io/fs/local_file_system_test.cpp +++ b/be/test/io/fs/local_file_system_test.cpp @@ -597,15 +597,24 @@ TEST_F(LocalFileSystemTest, TestSafeGlobPathContainment) { ASSERT_TRUE(save_string_file((outside / "outside.csv").string(), "outside").ok()); std::filesystem::create_directory_symlink(outside, secure / "escape"); std::filesystem::create_symlink(outside / "outside.csv", secure / "outside.csv"); + std::filesystem::create_symlink(outside / "missing.csv", outside / "broken.csv"); std::filesystem::create_directory_symlink(secure / "nested", secure / "inside"); config::user_files_secure_path = secure.string(); for (const auto* pattern : {".[.]/secure-other/outside.csv", ".?/secure-other/outside.csv", - ".[.-.]/secure-other/outside.csv", "escape/*.csv", "outside.csv"}) { + ".[.-.]/secure-other/outside.csv", "escape/*.csv", "outside.csv", + ".[.]/secure-other/*.csv", "escape/broken*.csv"}) { std::vector files; const auto status = fs->safe_glob(pattern, &files); EXPECT_FALSE(status.ok()) << pattern; EXPECT_NE(status.to_string().find("file path is not allowed"), std::string::npos) << status; + // The pattern need not name the matched file, so errors must not reveal that name. + const auto message = status.to_string(); + EXPECT_EQ(std::string::npos, message.find(root.string())) << message; + for (const auto* name : {"outside.csv", "broken.csv", "missing.csv"}) { + EXPECT_EQ(std::string::npos, message.find(name)) << message; + } + EXPECT_TRUE(files.empty()); } for (const auto* pattern : {"nested/*.csv", "inside/*.csv"}) { @@ -662,7 +671,9 @@ TEST_F(LocalFileSystemTest, TestCanonicalizeSecureLocalPath) { std::string("http://invalid/dataset"), std::string(""), std::string("nested\0ignored", 14)}) { std::string resolved; - EXPECT_FALSE(fs->canonicalize_local_file(secure.string(), path, &resolved).ok()) << path; + const auto status = fs->canonicalize_local_file(secure.string(), path, &resolved); + EXPECT_FALSE(status.ok()) << path; + EXPECT_EQ(std::string::npos, status.to_string().find(root.string())) << status; } std::string resolved; EXPECT_FALSE(fs->canonicalize_local_file("", secure.string(), &resolved).ok()); From fdff474e8ddb427f77e24ccb3b6ab690f486166a Mon Sep 17 00:00:00 2001 From: Dylan Date: Fri, 23 Oct 2026 21:33:46 +0800 Subject: [PATCH 04/13] [test](lance) Read rows through secure LOCAL paths --- be/test/format_v2/table/lance_reader_test.cpp | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/be/test/format_v2/table/lance_reader_test.cpp b/be/test/format_v2/table/lance_reader_test.cpp index 3ebc50e11c7189..3fcb445291fcca 100644 --- a/be/test/format_v2/table/lance_reader_test.cpp +++ b/be/test/format_v2/table/lance_reader_test.cpp @@ -4365,6 +4365,25 @@ TEST(LanceTableReaderLocalPathTest, ScansRelativeDatasetInsideSecureRoot) { auto range = make_latest_lance_range("all_types.lance"); range.__set_file_type(TFileType::FILE_LOCAL); ASSERT_TRUE(prepare_range(&reader, range).ok()); + + // Exercise data reads through the validated relative LOCAL range. + Block block; + add_output_columns(&block, columns); + std::vector row_ids; + bool eos = false; + while (!eos) { + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + if (eos) { + continue; + } + const auto& values = + assert_cast(*block.get_by_position(0).column).get_data(); + for (size_t row = 0; row < block.rows(); ++row) { + row_ids.push_back(values[row]); + } + } + std::sort(row_ids.begin(), row_ids.end()); + EXPECT_EQ((std::vector {1, 2, 3, 4}), row_ids); ASSERT_TRUE(reader.close().ok()); } From c4103b6ec3f366ce9096ea20424e998585edbe60 Mon Sep 17 00:00:00 2001 From: Dylan Date: Fri, 23 Oct 2026 23:03:31 +0800 Subject: [PATCH 05/13] [fix](tvf) Validate remote URIs before opening S3 Lance ### What problem does this PR solve? Issue Number: N/A Related PR: #69526 Problem Summary: S3 Lance could open a local dataset through BrokerProperties because the entry point rejected only LocalProperties. Validate the final dataset URI before loading metadata. Allow the pinned SDK's remote providers and reject local aliases and malformed authorities that could fall back to local paths. Keep valid ports, underscore buckets, hostless GooseFS and normalized S3/OSS HTTP locations working without changing catalog or non-Lance paths. ### Release note S3 Lance rejects local dataset URIs and malformed remote URI authorities before opening a dataset, regardless of storage-property selection. ### Check List (For Author) - Test: 31 focused FE unit tests passed on this branch; changed Java files pass Checkstyle and git diff --check passes. - Behavior changed: Yes. S3 Lance requires a valid supported remote URI. - Does this need documentation: No additional documentation for this security validation; the parent PR's configuration documentation is pending. --- .../tablefunction/S3TableValuedFunction.java | 47 +++++++++- .../S3TableValuedFunctionTest.java | 91 +++++++++++++++++++ 2 files changed, 133 insertions(+), 5 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java index 03a2799678ae2f..6e0aec7f0031cc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java @@ -24,14 +24,20 @@ import org.apache.doris.common.UserException; import org.apache.doris.datasource.lance.metadata.LanceMetadataLoader; import org.apache.doris.datasource.lance.metadata.LanceTableMetadata; -import org.apache.doris.datasource.property.storage.LocalProperties; import org.apache.doris.datasource.property.storage.StorageProperties; import org.apache.doris.thrift.TBrokerFileStatus; import org.apache.doris.thrift.TFileType; +import com.google.common.collect.ImmutableSet; +import com.google.common.net.HostAndPort; + +import java.net.URI; +import java.net.URISyntaxException; import java.util.Collections; import java.util.List; +import java.util.Locale; import java.util.Map; +import java.util.Set; /** * The Implement of table valued function @@ -47,6 +53,11 @@ public class S3TableValuedFunction extends ExternalFileTableValuedFunction { public static final String NAME = "s3"; + // Remote providers registered by the pinned Lance SDK. File providers also have aliases, + // so checking only for "file" (or a particular StorageProperties type) is insufficient. + private static final Set LANCE_REMOTE_SCHEMES = ImmutableSet.of( + "s3", "s3+ddb", "az", "abfss", "gs", "goosefs", "oss", "cos", "hf", "tos"); + public S3TableValuedFunction(Map properties) throws AnalysisException { this(properties, null); } @@ -64,10 +75,10 @@ public S3TableValuedFunction(Map properties, List properties, List loader = Mockito.mockStatic(LanceMetadataLoader.class)) { + for (String uri : Arrays.asList("dataset.lance", "/tmp/dataset.lance", "../dataset.lance", + "//localhost/tmp/dataset.lance", "file:///tmp/dataset.lance", "file:/tmp/dataset.lance", + "file:dataset.lance", "FILE:///tmp/dataset.lance", "file://localhost/tmp/dataset.lance", + "file-object-store://ignored/tmp/dataset.lance", "file+uring://ignored/tmp/dataset.lance", + "FILE-OBJECT-STORE://ignored/tmp/dataset.lance", "FILE+URING://ignored/tmp/dataset.lance", + "c://ignored/tmp/dataset.lance", "memory://dataset.lance", "shared-memory://dataset.lance", + "s3://bucket:bad/../../dataset.lance", "s3://bucket:65536/../../dataset.lance", + "s3://:9000/dataset.lance", "s3://user@/dataset.lance", + "s3://[::1%25eth0]/dataset.lance", "https://bucket:bad/../../dataset.lance", + "", "s3:/bucket/dataset.lance", "s3://", "s3://bucket/bad path")) { + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> new S3TableValuedFunction(ImmutableMap.of( + "format", "lance", "uri", uri, "fs.broker.support", "true")), uri); + Assertions.assertTrue(exception.getMessage().contains("requires remote storage"), uri); + } + loader.verifyNoInteractions(); + } + } + + @Test + public void testLanceRejectsGuessedBrokerFileUrisBeforeOpeningDataset() { + FeConstants.runningUnitTest = false; + try (MockedStatic loader = Mockito.mockStatic(LanceMetadataLoader.class)) { + for (String uri : Arrays.asList("file:///tmp/dataset.lance", "file://localhost/tmp/dataset.lance")) { + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> new S3TableValuedFunction(ImmutableMap.of( + "format", "lance", "uri", uri, "broker.name", "test"))); + Assertions.assertTrue(exception.getMessage().contains("requires remote storage")); + } + loader.verifyNoInteractions(); + } + } + + @Test + public void testLanceRejectsBrokerWithoutUriBeforeOpeningDataset() { + FeConstants.runningUnitTest = false; + try (MockedStatic loader = Mockito.mockStatic(LanceMetadataLoader.class)) { + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> new S3TableValuedFunction(ImmutableMap.of("format", "lance", "fs.broker.support", "true"))); + Assertions.assertTrue(exception.getMessage().contains("requires remote storage")); + loader.verifyNoInteractions(); + } + } + + @Test + public void testLanceRemoteUrisReachMetadataLoader() throws Exception { + FeConstants.runningUnitTest = false; + for (String uri : Arrays.asList("s3://bucket/dataset.lance", "s3+ddb://bucket/dataset.lance", + "oss://bucket/dataset.lance", "gs://bucket/dataset.lance", "az://container/dataset.lance", + "abfss://container@account.dfs.core.windows.net/dataset.lance", + "gs://bucket_with_underscore/dataset.lance", "s3://bucket:9000/dataset.lance", + "goosefs://bucket/dataset.lance", "goosefs:///dataset.lance", "goosefs:/dataset.lance", + "cos://bucket/dataset.lance", + "hf://datasets/repo/data.lance", + "tos://bucket/dataset.lance", "S3://bucket/dataset.lance")) { + assertLanceMetadataLoad(ImmutableMap.of( + "format", "lance", "uri", uri, "fs.broker.support", "true"), uri); + } + } + + @Test + public void testLanceHttpStorageUrisAreNormalizedBeforeValidation() throws Exception { + FeConstants.runningUnitTest = false; + Map s3Properties = new HashMap<>(PROPERTIES); + s3Properties.put("format", "lance"); + s3Properties.put("uri", "https://bucket.s3.us-west-2.amazonaws.com/dataset.lance"); + assertLanceMetadataLoad(s3Properties, "s3://bucket/dataset.lance"); + assertLanceMetadataLoad(ImmutableMap.of( + "format", "lance", "uri", "https://bucket.oss-cn-hangzhou.aliyuncs.com/dataset.lance", + "oss.endpoint", "oss-cn-hangzhou.aliyuncs.com", "oss.access_key", "ak", "oss.secret_key", "sk"), + "s3://bucket/dataset.lance"); + } + + private void assertLanceMetadataLoad(Map properties, String uri) throws Exception { + try (MockedStatic loader = Mockito.mockStatic(LanceMetadataLoader.class)) { + IllegalStateException reachedLoader = new IllegalStateException("remote metadata loader reached"); + loader.when(() -> LanceMetadataLoader.loadLatestForTvf(Mockito.eq(uri), Mockito.anyList())) + .thenThrow(reachedLoader); + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> new S3TableValuedFunction(properties)); + Assertions.assertSame(reachedLoader, exception.getCause(), uri); + loader.verify(() -> LanceMetadataLoader.loadLatestForTvf(Mockito.eq(uri), Mockito.anyList())); + } + } + @Test public void testExactFilesKeepLiteralPathsAndSkipListing() throws Exception { FeConstants.runningUnitTest = false; From 398acc158bbb9e2a9525a0960a9186f351d73fc6 Mon Sep 17 00:00:00 2001 From: Dylan Date: Sat, 24 Oct 2026 11:53:10 +0800 Subject: [PATCH 06/13] [fix](tvf) Close LOCAL admission and path review gaps ### What problem does this PR solve? Issue Number: N/A Related PR: #69528 Problem Summary: Apply the startup-only LOCAL gate to INSERT and EXPLAIN INSERT. Escape the canonical secure root as a literal glob prefix. Validate raw S3 Lance ports and IPv6 before SDK access; decode file URI paths once after removing query and fragment while preserving bare paths. ### Release note LOCAL TVF INSERT and EXPLAIN INSERT honor enable_local_tvf. Secure directories containing glob metacharacters remain usable. LOCAL Lance preserves percent-encoded file URI identity; malformed remote Lance authorities are rejected before SDK access. ### Check List (For Author) - Test: Unit Test: 35 FE and 11 BE selected tests passed; relevant Java Checkstyle, clang-format 16, and git diff --check passed. Regression assertions added, full Groovy not run. - Behavior changed: Yes, enforce the existing LOCAL startup setting at the write entry and preserve valid local path identity. - Does this need documentation: Yes, the original PR release note describes the startup opt-in requirement; documentation PR remains pending. --- be/src/io/fs/local_file_system.cpp | 43 ++++++++++++- be/test/format_v2/table/lance_reader_test.cpp | 63 +++++++++++++++++++ be/test/io/fs/local_file_system_test.cpp | 39 ++++++++++++ .../nereids/rules/analysis/BindSink.java | 6 ++ .../LocalTableValuedFunction.java | 13 ++-- .../tablefunction/S3TableValuedFunction.java | 20 ++++-- .../rules/analysis/BindTVFTableSinkTest.java | 58 +++++++++++++++++ .../S3TableValuedFunctionTest.java | 5 ++ .../account_p0/test_local_tvf_enable.groovy | 13 ++++ 9 files changed, 251 insertions(+), 9 deletions(-) diff --git a/be/src/io/fs/local_file_system.cpp b/be/src/io/fs/local_file_system.cpp index da9624fe544b0f..944ad55ab56b35 100644 --- a/be/src/io/fs/local_file_system.cpp +++ b/be/src/io/fs/local_file_system.cpp @@ -466,6 +466,38 @@ Status LocalFileSystem::canonicalize_local_file(const std::string& dir, Path local_path; // Convert file: URIs to filesystem paths and reject remote schemes before any reader opens them. RETURN_IF_ERROR(convert_to_abs_path(file_path, local_path)); + if (file_path.rfind("file:", 0) == 0) { + // file:a%20b names "a b", but a bare path containing "%20" must stay literal. + // Decode once, preserve '+' and invalid escapes, matching Lance's file provider. + auto encoded = local_path.string(); + // Query and fragment are URI components, not part of the filesystem path. + encoded = encoded.substr(0, encoded.find_first_of("?#")); + const auto hex_digit = [](char c) -> int { + if (c >= '0' && c <= '9') return c - '0'; + if (c >= 'a' && c <= 'f') return c - 'a' + 10; + if (c >= 'A' && c <= 'F') return c - 'A' + 10; + return -1; + }; + std::string decoded; + decoded.reserve(encoded.size()); + for (size_t i = 0; i < encoded.size(); ++i) { + if (encoded[i] == '%' && i + 2 < encoded.size()) { + const int high = hex_digit(encoded[i + 1]); + const int low = hex_digit(encoded[i + 2]); + if (high >= 0 && low >= 0) { + decoded += static_cast((high << 4) | low); + i += 2; + continue; + } + } + decoded += encoded[i]; + } + // %00 must not truncate the path when canonicalization reaches C filesystem APIs. + if (decoded.find('\0') != std::string::npos) { + return Status::InvalidArgument("local file path must not contain NUL"); + } + local_path = std::move(decoded); + } std::string secure_path; RETURN_IF_ERROR(canonicalize(dir, &secure_path)); std::string canonical_path; @@ -489,7 +521,16 @@ Status LocalFileSystem::safe_glob(const std::string& path, std::vector } std::string secure_path; RETURN_IF_ERROR(canonicalize(config::user_files_secure_path, &secure_path)); - std::string full_path = secure_path + "/" + path; + // A symlink root may canonicalize to a directory such as "user_files[prod]". + // The root is literal; only the user-supplied suffix may contain glob operators. + std::string full_path; + for (char c : secure_path) { + if (c == '\\' || c == '*' || c == '?' || c == '[' || c == ']') { + full_path += '\\'; + } + full_path += c; + } + full_path += "/" + path; std::vector files; RETURN_IF_ERROR(_glob(full_path, &files)); for (auto& file : files) { diff --git a/be/test/format_v2/table/lance_reader_test.cpp b/be/test/format_v2/table/lance_reader_test.cpp index 3fcb445291fcca..96b9c9e34b56cb 100644 --- a/be/test/format_v2/table/lance_reader_test.cpp +++ b/be/test/format_v2/table/lance_reader_test.cpp @@ -4387,6 +4387,69 @@ TEST(LanceTableReaderLocalPathTest, ScansRelativeDatasetInsideSecureRoot) { ASSERT_TRUE(reader.close().ok()); } +TEST(LanceTableReaderLocalPathTest, FileUrisPreserveEscapedDatasetIdentity) { + const auto original_secure_path = config::user_files_secure_path; + DEFER(config::user_files_secure_path = original_secure_path); + const auto root = std::filesystem::temp_directory_path() / + ("lance_escaped_uri_" + + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count())); + DEFER({ + std::error_code error; + std::filesystem::remove_all(root, error); + }); + const auto fixture = + std::filesystem::canonical("./be/test/format_v2/table/lance/data/all_types.lance"); + for (const auto* name : {"a b.lance", "a%20b.lance", "plus+percent%.lance"}) { + std::filesystem::create_directories(root / name); + std::filesystem::copy(fixture, root / name, std::filesystem::copy_options::recursive); + } + config::user_files_secure_path = root.string(); + for (const auto& input : + {std::make_pair("file://" + (root / "a%20b.lance").string(), root / "a b.lance"), + std::make_pair(std::string("a%20b.lance"), root / "a%20b.lance"), + std::make_pair("file://" + (root / "a%20b.lance").string() + "?q=%00#tag", + root / "a b.lance"), + std::make_pair("file://" + (root / "a%2520b.lance").string(), root / "a%20b.lance"), + std::make_pair("file://" + (root / "plus+percent%25.lance").string(), + root / "plus+percent%.lance")}) { + TQueryGlobals query_globals; + RuntimeState state(query_globals); + RuntimeProfile profile("lance_escaped_file_uri"); + TFileScanRangeParams scan_params; + LanceTableReader reader; + const Columns columns = {projected_column("row_id", TYPE_BIGINT, false)}; + ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, &scan_params).ok()); + auto range = make_latest_lance_range(input.first); + range.__set_file_type(TFileType::FILE_LOCAL); + std::vector names; + std::vector types; + ASSERT_TRUE(reader.fetch_schema(range, scan_params, &names, &types).ok()) << input.first; + EXPECT_FALSE(names.empty()); + LanceTableReader::DatasetKey key; + ASSERT_TRUE(reader._dataset_key(range, &key).ok()); + EXPECT_EQ(input.second.string(), key.uri); + ASSERT_TRUE(prepare_range(&reader, range).ok()); + Block block; + add_output_columns(&block, columns); + std::vector row_ids; + bool eos = false; + while (!eos) { + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + if (!eos) { + const auto& values = + assert_cast(*block.get_by_position(0).column) + .get_data(); + for (size_t row = 0; row < block.rows(); ++row) { + row_ids.push_back(values[row]); + } + } + } + std::sort(row_ids.begin(), row_ids.end()); + EXPECT_EQ((std::vector {1, 2, 3, 4}), row_ids); + ASSERT_TRUE(reader.close().ok()); + } +} + TEST(LanceTableReaderLocalPathTest, FixedVersionLocalCatalogKeepsItsUri) { TFileScanRangeParams scan_params; LanceTableReader reader; diff --git a/be/test/io/fs/local_file_system_test.cpp b/be/test/io/fs/local_file_system_test.cpp index 0b679a7e0c908c..a052c024e38d82 100644 --- a/be/test/io/fs/local_file_system_test.cpp +++ b/be/test/io/fs/local_file_system_test.cpp @@ -643,6 +643,17 @@ TEST_F(LocalFileSystemTest, TestSafeGlobSecureRoot) { ASSERT_EQ(1, files.size()); EXPECT_EQ((root / "secure/inside.csv").string(), files[0].file_name); } + const auto special_root = root / "secure[prod]*?\\root"; + std::filesystem::create_directory(special_root); + std::filesystem::create_directory_symlink(special_root, root / "special-root-link"); + ASSERT_TRUE(save_string_file((special_root / "inside.csv").string(), "inside").ok()); + for (const auto& secure : {special_root, root / "special-root-link"}) { + config::user_files_secure_path = secure.string(); + std::vector matches; + ASSERT_TRUE(fs->safe_glob("*.csv", &matches).ok()) << secure; + ASSERT_EQ(1, matches.size()); + EXPECT_EQ((special_root / "inside.csv").string(), matches[0].file_name); + } config::user_files_secure_path = ""; std::vector files; EXPECT_FALSE(fs->safe_glob("*.csv", &files).ok()); @@ -675,6 +686,34 @@ TEST_F(LocalFileSystemTest, TestCanonicalizeSecureLocalPath) { EXPECT_FALSE(status.ok()) << path; EXPECT_EQ(std::string::npos, status.to_string().find(root.string())) << status; } + // URI escapes and bare '%' paths may name two different, simultaneously existing datasets. + for (const auto* name : {"a b.lance", "a%20b.lance", "plus+percent%.lance", "bad%2G.lance"}) { + std::filesystem::create_directory(secure / name); + } + for (const auto& input : + {std::make_pair("file://" + (secure / "a%20b.lance").string(), secure / "a b.lance"), + std::make_pair(std::string("a%20b.lance"), secure / "a%20b.lance"), + std::make_pair("file://" + (secure / "a%20b.lance").string() + "#tag", + secure / "a b.lance"), + std::make_pair("file://" + (secure / "a%20b.lance").string() + "?q=%00#tag", + secure / "a b.lance"), + std::make_pair("file://" + (secure / "a%2520b.lance").string(), secure / "a%20b.lance"), + std::make_pair("file://" + (secure / "plus+percent%25.lance").string(), + secure / "plus+percent%.lance"), + std::make_pair("file://" + (secure / "bad%2G.lance").string(), + secure / "bad%2G.lance")}) { + std::string actual; + ASSERT_TRUE(fs->canonicalize_local_file(secure.string(), input.first, &actual).ok()) + << input.first; + EXPECT_EQ(input.second.string(), actual); + } + for (const auto& suffix : + {"/%2e%2e/secure-other", "/nested%2f..%2f..%2fsecure-other", "/a%00b.lance"}) { + std::string actual; + EXPECT_FALSE(fs->canonicalize_local_file(secure.string(), + "file://" + secure.string() + suffix, &actual) + .ok()); + } std::string resolved; EXPECT_FALSE(fs->canonicalize_local_file("", secure.string(), &resolved).ok()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java index 1d2580e52fc848..0fac5ba4344f39 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java @@ -116,6 +116,7 @@ import org.apache.doris.qe.AutoCloseSessionVariable; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.SessionVariable; +import org.apache.doris.tablefunction.LocalTableValuedFunction; import org.apache.doris.thrift.TPartialUpdateNewRowPolicy; import com.google.common.base.Preconditions; @@ -659,6 +660,11 @@ private Plan bindTVFTableSink(MatchingContext> ctx) { "INSERT INTO TVF only supports local/s3/hdfs, but got: " + tvfName); } + // INSERT/EXPLAIN INSERT bind a sink without constructing the LOCAL read TVF. + if (tvfName.equals("local")) { + LocalTableValuedFunction.checkEnabled(); + } + // Validate required properties if (!properties.containsKey("file_path")) { throw new AnalysisException("TVF sink requires 'file_path' property"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/LocalTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/LocalTableValuedFunction.java index 8d2d09e5de06ee..335ae199cc137a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/LocalTableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/LocalTableValuedFunction.java @@ -64,10 +64,7 @@ public class LocalTableValuedFunction extends ExternalFileTableValuedFunction { private boolean sharedStorage = false; public LocalTableValuedFunction(Map properties) throws AnalysisException { - if (!Config.enable_local_tvf) { - throw new org.apache.doris.nereids.exceptions.AnalysisException( - "LOCAL TVF is disabled. Set enable_local_tvf=true in fe.conf and restart every FE to enable it."); - } + checkEnabled(); // Check privileges before listing files or discovering their schema. checkAuth(ConnectContext.get()); @@ -146,6 +143,14 @@ private void getFileListFromBackend() throws AnalysisException { } } + /** Check read and write admission before properties or backend access. */ + public static void checkEnabled() { + if (!Config.enable_local_tvf) { + throw new org.apache.doris.nereids.exceptions.AnalysisException( + "LOCAL TVF is disabled. Set enable_local_tvf=true in fe.conf and restart every FE to enable it."); + } + } + @Override public void checkAuth(ConnectContext ctx) { if (!Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ctx, PrivPredicate.ADMIN) diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java index 6e0aec7f0031cc..183e7304b6add2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java @@ -30,6 +30,7 @@ import com.google.common.collect.ImmutableSet; import com.google.common.net.HostAndPort; +import com.google.common.net.InetAddresses; import java.net.URI; import java.net.URISyntaxException; @@ -123,10 +124,21 @@ private static boolean isRemoteLanceUri(String path) { // Lance falls back to a local path if Rust's URL parser rejects the URI. Java's URI // accepts registry authorities with invalid ports, which Rust rejects. Validate the // host/port separately without disallowing legitimate bucket names with underscores. - String host = HostAndPort.fromString(authority.substring(authority.lastIndexOf('@') + 1)) - .requireBracketsForIPv6().getHost(); - // Java accepts IPv6 zone identifiers, while Rust's URL parser rejects them. - return !host.isEmpty() && !(host.contains(":") && host.contains("%")); + String hostPort = authority.substring(authority.lastIndexOf('@') + 1); + HostAndPort parsed = HostAndPort.fromString(hostPort).requireBracketsForIPv6(); + if (parsed.hasPort()) { + // Guava parses "-0" as zero, but Rust rejects signs and falls back to local. + String port = hostPort.substring(hostPort.lastIndexOf(':') + 1); + for (int i = 0; i < port.length(); i++) { + if (port.charAt(i) < '0' || port.charAt(i) > '9') { + return false; + } + } + } + String host = parsed.getHost(); + // Rust rejects zones and IPv4 tails with leading zeros, such as ::ffff:127.0.0.01. + return !host.isEmpty() && (!host.contains(":") + || (!host.contains("%") && InetAddresses.isInetAddress(host))); } catch (URISyntaxException | IllegalArgumentException e) { return false; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindTVFTableSinkTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindTVFTableSinkTest.java index b76ba57ddb5303..b789c4c391fd22 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindTVFTableSinkTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindTVFTableSinkTest.java @@ -17,25 +17,83 @@ package org.apache.doris.nereids.rules.analysis; +import org.apache.doris.common.Config; +import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.parser.NereidsParser; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.commands.ExplainCommand; import org.apache.doris.nereids.trees.plans.commands.insert.InsertIntoTVFCommand; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.trees.plans.logical.LogicalTVFTableSink; import org.apache.doris.nereids.util.MemoTestUtils; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.qe.ConnectContext; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import java.util.Arrays; import java.util.List; class BindTVFTableSinkTest { + private boolean originalEnableLocalTvf; + + @BeforeEach + void setUp() { + originalEnableLocalTvf = Config.enable_local_tvf; + Config.enable_local_tvf = false; + } + + @AfterEach + void tearDown() { + Config.enable_local_tvf = originalEnableLocalTvf; + } + + @Test + void disabledLocalSinkRejectsInsertAndExplainBeforeProperties() { + for (String prefix : Arrays.asList("", "EXPLAIN ")) { + for (String properties : Arrays.asList("'format'='csv'", + "'file_path'='/tmp/local_gate_', 'backend_id'='1', 'format'='csv'")) { + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> analyzeInsert(prefix + "INSERT INTO local(" + properties + ") SELECT 1")); + Assertions.assertTrue(exception.getMessage().contains("LOCAL TVF is disabled")); + } + } + } + + @Test + void enabledLocalSinkKeepsPropertyValidation() { + Config.enable_local_tvf = true; + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> analyzeInsert("INSERT INTO local('format'='csv') SELECT 1")); + Assertions.assertTrue(exception.getMessage().contains("requires 'file_path'")); + } + + @Test + void disabledLocalDoesNotBlockRemoteSinks() { + for (String name : Arrays.asList("s3", "hdfs")) { + Assertions.assertInstanceOf(LogicalTVFTableSink.class, + analyzeInsert("INSERT INTO " + name + "('file_path'='remote/data', 'format'='csv') SELECT 1")); + } + } + + private Plan analyzeInsert(String sql) { + ConnectContext context = MemoTestUtils.createConnectContext(); + LogicalPlan parsed = new NereidsParser().parseSingle(sql); + if (parsed instanceof ExplainCommand) { + parsed = ((ExplainCommand) parsed).getLogicalPlan(); + } + InsertIntoTVFCommand command = (InsertIntoTVFCommand) parsed; + return PlanChecker.from(context).analyze(command.getExplainPlan(context)).getPlan(); + } @Test void duplicateOutputNamesKeepDistinctPositions() { + Config.enable_local_tvf = true; ConnectContext context = MemoTestUtils.createConnectContext(); InsertIntoTVFCommand command = (InsertIntoTVFCommand) new NereidsParser().parseSingle( "INSERT INTO local(" diff --git a/fe/fe-core/src/test/java/org/apache/doris/tablefunction/S3TableValuedFunctionTest.java b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/S3TableValuedFunctionTest.java index c6b0813c605b18..6050a4209aa8b6 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/tablefunction/S3TableValuedFunctionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/S3TableValuedFunctionTest.java @@ -92,6 +92,8 @@ public void testLanceRejectsBrokerLocalUrisBeforeOpeningDataset() { "file-object-store://ignored/tmp/dataset.lance", "file+uring://ignored/tmp/dataset.lance", "FILE-OBJECT-STORE://ignored/tmp/dataset.lance", "FILE+URING://ignored/tmp/dataset.lance", "c://ignored/tmp/dataset.lance", "memory://dataset.lance", "shared-memory://dataset.lance", + "s3://bucket:-0/../../dataset.lance", "s3://bucket:-00/../../dataset.lance", + "s3://bucket:+0/../../dataset.lance", "s3://[::ffff:127.0.0.01]/../../dataset.lance", "s3://bucket:bad/../../dataset.lance", "s3://bucket:65536/../../dataset.lance", "s3://:9000/dataset.lance", "s3://user@/dataset.lance", "s3://[::1%25eth0]/dataset.lance", "https://bucket:bad/../../dataset.lance", @@ -136,6 +138,9 @@ public void testLanceRemoteUrisReachMetadataLoader() throws Exception { for (String uri : Arrays.asList("s3://bucket/dataset.lance", "s3+ddb://bucket/dataset.lance", "oss://bucket/dataset.lance", "gs://bucket/dataset.lance", "az://container/dataset.lance", "abfss://container@account.dfs.core.windows.net/dataset.lance", + "s3://bucket:0/dataset.lance", "s3://bucket:00080/dataset.lance", + "s3://bucket:65535/dataset.lance", "s3://bucket:/dataset.lance", + "s3://[::1]/dataset.lance", "s3://[::ffff:127.0.0.1]/dataset.lance", "gs://bucket_with_underscore/dataset.lance", "s3://bucket:9000/dataset.lance", "goosefs://bucket/dataset.lance", "goosefs:///dataset.lance", "goosefs:/dataset.lance", "cos://bucket/dataset.lance", diff --git a/regression-test/suites/account_p0/test_local_tvf_enable.groovy b/regression-test/suites/account_p0/test_local_tvf_enable.groovy index b6853b51ebd5cb..3cc8bf71ce3d92 100644 --- a/regression-test/suites/account_p0/test_local_tvf_enable.groovy +++ b/regression-test/suites/account_p0/test_local_tvf_enable.groovy @@ -30,4 +30,17 @@ suite("test_local_tvf_enable", "p0") { } } } + if (!enabled) { + for (def prefix : ["", "EXPLAIN "]) { + test { + sql """${prefix}INSERT INTO LOCAL( + "file_path" = "/tmp/local_enable_test_", + "backend_id" = "-1", + "format" = "csv" + ) SELECT 1""" + exception "LOCAL TVF is disabled" + } + } + } + } From 71be77f48222334cf3311ed71ba9f4b6142043e4 Mon Sep 17 00:00:00 2001 From: Dylan Date: Sat, 24 Oct 2026 12:50:27 +0800 Subject: [PATCH 07/13] [fix](tvf) Preserve LOCAL file URI dataset identity ### What problem does this PR solve? Issue Number: N/A Related PR: #69528 Problem Summary: LOCAL Lance canonicalization followed symlinks before URL dot-segment normalization, so a valid file URI could open a different in-root dataset. Uppercase file schemes were rejected. Normalize the actual file URI scheme, separators and raw dot segments before one-pass decoding and filesystem containment; preserve bare path behavior. ### Release note LOCAL Lance preserves file URI dataset identity for dot segments, case-insensitive schemes and raw URI separators while retaining secure-root containment. ### Check List (For Author) - Test: 13 selected BE filesystem/Lance tests passed with fresh XML, zero failures/skips and unchanged sources. New regressions fail before the fix, including actual schema and row-value mismatch. Changed-line clang-format16 and git diff --check passed. FE source is unchanged. - Behavior changed: Yes, restore valid file URI identity without changing bare paths, remote TVFs or fixed-version catalogs. - Does this need documentation: The original PR documents the LOCAL startup opt-in; no additional public configuration or syntax is introduced. --- be/src/io/fs/local_file_system.cpp | 57 ++++++++++++++- be/test/format_v2/table/lance_reader_test.cpp | 72 +++++++++++++++++++ be/test/io/fs/local_file_system_test.cpp | 55 ++++++++++++++ 3 files changed, 182 insertions(+), 2 deletions(-) diff --git a/be/src/io/fs/local_file_system.cpp b/be/src/io/fs/local_file_system.cpp index 944ad55ab56b35..dd83bcf7eae327 100644 --- a/be/src/io/fs/local_file_system.cpp +++ b/be/src/io/fs/local_file_system.cpp @@ -463,15 +463,68 @@ Status LocalFileSystem::canonicalize_local_file(const std::string& dir, if (file_path.empty() || file_path.find('\0') != std::string::npos) { return Status::InvalidArgument("local file path must not be empty or contain NUL"); } + // URI schemes are ASCII case-insensitive; do not alter spelling in bare paths. + const bool is_file_uri = file_path.size() >= 5 && + (file_path[0] == 'f' || file_path[0] == 'F') && + (file_path[1] == 'i' || file_path[1] == 'I') && + (file_path[2] == 'l' || file_path[2] == 'L') && + (file_path[3] == 'e' || file_path[3] == 'E') && file_path[4] == ':'; + std::string local_input = file_path; + if (is_file_uri) { + local_input.replace(0, 4, "file"); + // File URLs treat raw backslashes as separators, including before the authority. + // Do this before conversion; percent-encoded backslashes must remain literal. + for (size_t i = 5; i < local_input.size(); ++i) { + if (local_input[i] == '\\') { + local_input[i] = '/'; + } + } + } Path local_path; // Convert file: URIs to filesystem paths and reject remote schemes before any reader opens them. - RETURN_IF_ERROR(convert_to_abs_path(file_path, local_path)); - if (file_path.rfind("file:", 0) == 0) { + RETURN_IF_ERROR(convert_to_abs_path(local_input, local_path)); + if (is_file_uri) { // file:a%20b names "a b", but a bare path containing "%20" must stay literal. // Decode once, preserve '+' and invalid escapes, matching Lance's file provider. auto encoded = local_path.string(); // Query and fragment are URI components, not part of the filesystem path. encoded = encoded.substr(0, encoded.find_first_of("?#")); + // URL dot segments are removed before symlinks are followed and before %2F + // becomes a path separator. Keep empty segments: "a//../b" denotes "a/b". + const bool absolute = !encoded.empty() && encoded[0] == '/'; + std::vector segments; + for (size_t start = absolute ? 1 : 0;;) { + const size_t end = encoded.find('/', start); + const auto segment = + encoded.substr(start, end == std::string::npos ? end : end - start); + const bool dot = segment == "." || segment == "%2e" || segment == "%2E"; + const bool parent = segment == ".." || segment == ".%2e" || segment == ".%2E" || + segment == "%2e." || segment == "%2E." || segment == "%2e%2e" || + segment == "%2e%2E" || segment == "%2E%2e" || segment == "%2E%2E"; + if (parent) { + if (!segments.empty() && segments.back() != "..") { + segments.pop_back(); + } else if (!absolute) { + segments.emplace_back(".."); + } + } else if (!dot) { + segments.push_back(segment); + } + if (end == std::string::npos) { + if (dot || parent) { + segments.emplace_back(""); + } + break; + } + start = end + 1; + } + encoded = absolute ? "/" : ""; + for (size_t i = 0; i < segments.size(); ++i) { + if (i > 0) { + encoded += '/'; + } + encoded += segments[i]; + } const auto hex_digit = [](char c) -> int { if (c >= '0' && c <= '9') return c - '0'; if (c >= 'a' && c <= 'f') return c - 'a' + 10; diff --git a/be/test/format_v2/table/lance_reader_test.cpp b/be/test/format_v2/table/lance_reader_test.cpp index 96b9c9e34b56cb..cbfa3789cd9f03 100644 --- a/be/test/format_v2/table/lance_reader_test.cpp +++ b/be/test/format_v2/table/lance_reader_test.cpp @@ -4450,6 +4450,78 @@ TEST(LanceTableReaderLocalPathTest, FileUrisPreserveEscapedDatasetIdentity) { } } +TEST(LanceTableReaderLocalPathTest, FileUrisNormalizeBeforeFilesystemResolution) { + const auto original_secure_path = config::user_files_secure_path; + DEFER(config::user_files_secure_path = original_secure_path); + const auto root = std::filesystem::temp_directory_path() / + ("lance_uri_normalization_" + + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count())); + DEFER({ + std::error_code error; + std::filesystem::remove_all(root, error); + }); + std::filesystem::create_directories(root / "nested/child"); + std::filesystem::create_directory_symlink(root / "nested/child", root / "jump"); + const auto backslash_uri = std::string(R"(fIlE:\\localhost)") + root.string() + "/data.lance"; + for (const bool nested : {false, true}) { + arrow::Int64Builder values_builder; + ASSERT_TRUE(values_builder.Append(nested ? 42 : 7).ok()); + std::shared_ptr values; + ASSERT_TRUE(values_builder.Finish(&values).ok()); + const auto schema = arrow::schema( + {arrow::field("row_id", arrow::int64(), false), + arrow::field(nested ? "nested_only" : "root_only", arrow::int64(), false)}); + ASSERT_TRUE(write_lance_record_batch(root / (nested ? "nested/data.lance" : "data.lance"), + arrow::RecordBatch::Make(schema, 1, {values, values})) + .ok()); + } + config::user_files_secure_path = root.string(); + for (const auto& input : + {std::make_pair("file://" + root.string() + "/jump/../data.lance", false), + std::make_pair("file://" + root.string() + "/missing/../data.lance", false), + std::make_pair("file://" + root.string() + "/jump/%2e%2E/data.lance", false), + std::make_pair("FILE://" + root.string() + "/data.lance", false), + std::make_pair(backslash_uri, false), + std::make_pair("fIlE://" + root.string() + "/jump/.%2e/data.lance", false), + std::make_pair("file://" + root.string() + "/jump%2f../data.lance", true), + std::make_pair(std::string("jump/../data.lance"), true)}) { + SCOPED_TRACE(input.first); + TQueryGlobals globals; + RuntimeState state(globals); + RuntimeProfile profile("lance_uri_normalization"); + TFileScanRangeParams scan_params; + LanceTableReader reader; + const Columns columns {projected_column("row_id", TYPE_BIGINT, false)}; + ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, &scan_params).ok()); + auto range = make_latest_lance_range(input.first); + range.__set_file_type(TFileType::FILE_LOCAL); + std::vector names; + std::vector types; + ASSERT_TRUE(reader.fetch_schema(range, scan_params, &names, &types).ok()); + EXPECT_EQ((std::vector {"row_id", input.second ? "nested_only" : "root_only"}), + names); + LanceTableReader::DatasetKey key; + ASSERT_TRUE(reader._dataset_key(range, &key).ok()); + EXPECT_EQ((root / (input.second ? "nested/data.lance" : "data.lance")).string(), key.uri); + ASSERT_TRUE(prepare_range(&reader, range).ok()); + Block block; + add_output_columns(&block, columns); + std::vector row_ids; + bool eos = false; + while (!eos) { + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + if (!eos) { + const auto& values = + assert_cast(*block.get_by_position(0).column) + .get_data(); + row_ids.insert(row_ids.end(), values.begin(), values.end()); + } + } + EXPECT_EQ((std::vector {input.second ? 42 : 7}), row_ids); + ASSERT_TRUE(reader.close().ok()); + } +} + TEST(LanceTableReaderLocalPathTest, FixedVersionLocalCatalogKeepsItsUri) { TFileScanRangeParams scan_params; LanceTableReader reader; diff --git a/be/test/io/fs/local_file_system_test.cpp b/be/test/io/fs/local_file_system_test.cpp index a052c024e38d82..48f0a901a673e1 100644 --- a/be/test/io/fs/local_file_system_test.cpp +++ b/be/test/io/fs/local_file_system_test.cpp @@ -718,6 +718,61 @@ TEST_F(LocalFileSystemTest, TestCanonicalizeSecureLocalPath) { EXPECT_FALSE(fs->canonicalize_local_file("", secure.string(), &resolved).ok()); } +TEST_F(LocalFileSystemTest, TestCanonicalizeFileUriIdentity) { + auto fs = io::global_local_filesystem(); + const auto root = std::filesystem::canonical(test_dir); + const auto secure = root / "secure"; + std::filesystem::create_directories(secure / "nested/child/data.lance"); + std::filesystem::create_directory(secure / "nested/data.lance"); + std::filesystem::create_directory(secure / "data.lance"); + std::filesystem::create_directory(secure / "a b.lance"); + std::filesystem::create_directory(root / "outside"); + std::filesystem::create_directory_symlink(secure / "nested/child", secure / "jump"); + std::filesystem::create_directory(secure / "literal\\name.lance"); + const auto backslash_uri = std::string(R"(fIlE:\\localhost)") + secure.string() + "/data.lance"; + for (const auto& input : + {std::make_pair("file://" + secure.string() + "/jump/../data.lance", + secure / "data.lance"), + std::make_pair("file://" + secure.string() + "/missing/../data.lance", + secure / "data.lance"), + std::make_pair("file://" + secure.string() + "/jump/%2e%2E/data.lance", + secure / "data.lance"), + std::make_pair("file://" + secure.string() + "/jump/.%2e/data.lance", + secure / "data.lance"), + std::make_pair("file://" + secure.string() + "/jump/%2E./data.lance", + secure / "data.lance"), + std::make_pair("file://" + secure.string() + "/missing/.././data.lance", + secure / "data.lance"), + std::make_pair("file://" + secure.string() + "/jump//../data.lance", + secure / "nested/child/data.lance"), + std::make_pair("file://" + secure.string() + "/jump%2f../data.lance", + secure / "nested/data.lance"), + std::make_pair("file://" + secure.string() + "/jump\\..\\data.lance", + secure / "data.lance"), + std::make_pair("FILE://" + secure.string() + "/data.lance", secure / "data.lance"), + std::make_pair(backslash_uri, secure / "data.lance"), + std::make_pair("file://" + secure.string() + "/literal%5cname.lance", + secure / "literal\\name.lance"), + std::make_pair(std::string("literal\\name.lance"), secure / "literal\\name.lance"), + std::make_pair("fIlE://" + secure.string() + "/a%20b.lance?q=%00#tag", + secure / "a b.lance"), + std::make_pair(std::string("jump/../data.lance"), secure / "nested/data.lance")}) { + std::string actual; + const auto status = fs->canonicalize_local_file(secure.string(), input.first, &actual); + ASSERT_TRUE(status.ok()) << input.first << " " << status; + EXPECT_EQ(input.second.string(), actual) << input.first; + } + for (const auto& input : + {"file://" + secure.string() + "/jump/../../outside", + "FILE://" + secure.string() + "/%2e%2e/outside", + "fIlE://" + secure.string() + "/a%00b.lance", std::string("HTTP://invalid/dataset")}) { + std::string actual; + const auto status = fs->canonicalize_local_file(secure.string(), input, &actual); + EXPECT_FALSE(status.ok()) << input; + EXPECT_EQ(std::string::npos, status.to_string().find(root.string())); + } +} + TEST_F(LocalFileSystemTest, TestConvertToAbsPath) { io::Path abs_path; Status st; From 0c4fa140e43bd3be71e097713f0cbd3f25d38c8c Mon Sep 17 00:00:00 2001 From: Dylan Date: Sat, 24 Oct 2026 13:42:16 +0800 Subject: [PATCH 08/13] [fix](tvf) Align LOCAL Lance path normalization with the SDK ### What problem does this PR solve? Issue Number: N/A Related PR: #69528 Problem Summary: LOCAL Lance file URLs without a slash were resolved relative to the secure root, URI whitespace was treated as literal data, and bare dot segments followed symlinks before SDK lexical normalization. Match file URI preprocessing and absolute-root semantics, and normalize bare paths only at the LOCAL Lance entry before canonical containment. Preserve literal bare whitespace, encoded spaces, glob behavior and existing remote/catalog routing. ### Release note LOCAL Lance preserves dataset identity for rootless file URLs, raw URI whitespace and bare dot segments while retaining secure-root containment. This does not add FE execution support for rootless Hadoop paths. ### Check List (For Author) - Test: 13 selected BE filesystem/Lance tests passed with fresh XML, zero failures/skips and unchanged sources. New regressions fail before the fix, including actual schema and row-value mismatch. Changed-line clang-format16 and git diff --check passed. FE source is unchanged. - Behavior changed: Yes, restore valid file URI identity without changing bare paths, remote TVFs or fixed-version catalogs. - Does this need documentation: The original PR documents the LOCAL startup opt-in; no additional public configuration or syntax is introduced. --- be/src/format_v2/table/lance_reader.cpp | 16 ++++- be/src/io/fs/local_file_system.cpp | 42 ++++++++---- be/test/format_v2/table/lance_reader_test.cpp | 66 ++++++++++++------- be/test/io/fs/local_file_system_test.cpp | 10 +++ 4 files changed, 98 insertions(+), 36 deletions(-) diff --git a/be/src/format_v2/table/lance_reader.cpp b/be/src/format_v2/table/lance_reader.cpp index cb47679fb8bbfa..c1db3b5f98b322 100644 --- a/be/src/format_v2/table/lance_reader.cpp +++ b/be/src/format_v2/table/lance_reader.cpp @@ -57,8 +57,22 @@ Status resolve_dataset_uri(const TFileRangeDesc& range, const TFileScanRangePara if (is_local && range.table_format_params.lance_params.version == 0) { // LOCAL Lance bypasses glob because it opens a dataset directory. Apply the same // canonical secure-root check before either schema discovery or a cached dataset open. + std::string local_input = requested_uri; + const auto colon = requested_uri.find(':'); + const auto slash = requested_uri.find('/'); + const bool bare_path = + colon == std::string::npos || (slash != std::string::npos && slash < colon); + if (bare_path && !requested_uri.empty() && requested_uri.find('\0') == std::string::npos && + !config::user_files_secure_path.empty()) { + // Lance's PathAbs fallback resolves bare dot segments before following symlinks. + // Anchor relative TVF paths to the canonical secure root, not the process cwd. + std::string secure_root; + RETURN_IF_ERROR(io::global_local_filesystem()->canonicalize( + config::user_files_secure_path, &secure_root)); + local_input = (io::Path(secure_root) / requested_uri).lexically_normal().string(); + } return io::global_local_filesystem()->canonicalize_local_file( - config::user_files_secure_path, requested_uri, uri); + config::user_files_secure_path, local_input, uri); } *uri = requested_uri; return Status::OK(); diff --git a/be/src/io/fs/local_file_system.cpp b/be/src/io/fs/local_file_system.cpp index dd83bcf7eae327..d51ecb6decf4d1 100644 --- a/be/src/io/fs/local_file_system.cpp +++ b/be/src/io/fs/local_file_system.cpp @@ -463,13 +463,28 @@ Status LocalFileSystem::canonicalize_local_file(const std::string& dir, if (file_path.empty() || file_path.find('\0') != std::string::npos) { return Status::InvalidArgument("local file path must not be empty or contain NUL"); } - // URI schemes are ASCII case-insensitive; do not alter spelling in bare paths. - const bool is_file_uri = file_path.size() >= 5 && - (file_path[0] == 'f' || file_path[0] == 'F') && - (file_path[1] == 'i' || file_path[1] == 'I') && - (file_path[2] == 'l' || file_path[2] == 'L') && - (file_path[3] == 'e' || file_path[3] == 'E') && file_path[4] == ':'; - std::string local_input = file_path; + // Match URL preprocessing only for actual file URIs; bare paths keep literal whitespace. + const auto is_c0_or_space = [](unsigned char c) { return c <= 0x20; }; + size_t begin = 0; + size_t end = file_path.size(); + while (begin < end && is_c0_or_space(file_path[begin])) { + ++begin; + } + while (end > begin && is_c0_or_space(file_path[end - 1])) { + --end; + } + std::string uri_input; + for (size_t i = begin; i < end; ++i) { + if (file_path[i] != '\t' && file_path[i] != '\r' && file_path[i] != '\n') { + uri_input += file_path[i]; + } + } + const bool is_file_uri = uri_input.size() >= 5 && + (uri_input[0] == 'f' || uri_input[0] == 'F') && + (uri_input[1] == 'i' || uri_input[1] == 'I') && + (uri_input[2] == 'l' || uri_input[2] == 'L') && + (uri_input[3] == 'e' || uri_input[3] == 'E') && uri_input[4] == ':'; + std::string local_input = is_file_uri ? std::move(uri_input) : file_path; if (is_file_uri) { local_input.replace(0, 4, "file"); // File URLs treat raw backslashes as separators, including before the authority. @@ -489,11 +504,14 @@ Status LocalFileSystem::canonicalize_local_file(const std::string& dir, auto encoded = local_path.string(); // Query and fragment are URI components, not part of the filesystem path. encoded = encoded.substr(0, encoded.find_first_of("?#")); + // A file URL without a leading slash still names a filesystem-rooted path. + if (encoded.empty() || encoded[0] != '/') { + encoded.insert(0, "/"); + } // URL dot segments are removed before symlinks are followed and before %2F // becomes a path separator. Keep empty segments: "a//../b" denotes "a/b". - const bool absolute = !encoded.empty() && encoded[0] == '/'; std::vector segments; - for (size_t start = absolute ? 1 : 0;;) { + for (size_t start = 1;;) { const size_t end = encoded.find('/', start); const auto segment = encoded.substr(start, end == std::string::npos ? end : end - start); @@ -502,10 +520,8 @@ Status LocalFileSystem::canonicalize_local_file(const std::string& dir, segment == "%2e." || segment == "%2E." || segment == "%2e%2e" || segment == "%2e%2E" || segment == "%2E%2e" || segment == "%2E%2E"; if (parent) { - if (!segments.empty() && segments.back() != "..") { + if (!segments.empty()) { segments.pop_back(); - } else if (!absolute) { - segments.emplace_back(".."); } } else if (!dot) { segments.push_back(segment); @@ -518,7 +534,7 @@ Status LocalFileSystem::canonicalize_local_file(const std::string& dir, } start = end + 1; } - encoded = absolute ? "/" : ""; + encoded = "/"; for (size_t i = 0; i < segments.size(); ++i) { if (i > 0) { encoded += '/'; diff --git a/be/test/format_v2/table/lance_reader_test.cpp b/be/test/format_v2/table/lance_reader_test.cpp index cbfa3789cd9f03..6aa3b32a8bb3ba 100644 --- a/be/test/format_v2/table/lance_reader_test.cpp +++ b/be/test/format_v2/table/lance_reader_test.cpp @@ -4340,8 +4340,9 @@ TEST(LanceTableReaderLocalPathTest, SchemaAndScanUseTheSameSecurePath) { ASSERT_TRUE(reader._dataset_key(range, &key).ok()); EXPECT_EQ((root / "all_types.lance").string(), key.uri); - for (const auto* uri : {"https://invalid/dataset", "http://invalid/dataset", - "s3://bucket/dataset", "../data-other/dataset"}) { + for (const auto& uri : std::vector { + "https://invalid/dataset", "http://invalid/dataset", "s3://bucket/dataset", + "../data-other/dataset", "", std::string("a\0b", 3) + "/../all_types.lance"}) { range.table_format_params.lance_params.__set_dataset_uri(uri); const auto schema_status = reader.fetch_schema(range, scan_params, &names, &types); const auto scan_status = reader._dataset_key(range, &key); @@ -4462,30 +4463,52 @@ TEST(LanceTableReaderLocalPathTest, FileUrisNormalizeBeforeFilesystemResolution) }); std::filesystem::create_directories(root / "nested/child"); std::filesystem::create_directory_symlink(root / "nested/child", root / "jump"); + std::filesystem::create_directory_symlink(root, root / "secure-link"); const auto backslash_uri = std::string(R"(fIlE:\\localhost)") + root.string() + "/data.lance"; - for (const bool nested : {false, true}) { + struct DatasetFixture { + const char* column_name; + int64_t row_id; + std::filesystem::path path; + }; + const DatasetFixture datasets[] = { + {"root_only", 7, root / "data.lance"}, + {"nested_only", 42, root / "nested/data.lance"}, + {"spaced_only", 99, root / "data.lance "}, + {"prefixed_only", 123, root / (root / "data.lance").relative_path()}}; + for (const auto& dataset : datasets) { + std::filesystem::create_directories(dataset.path.parent_path()); arrow::Int64Builder values_builder; - ASSERT_TRUE(values_builder.Append(nested ? 42 : 7).ok()); + ASSERT_TRUE(values_builder.Append(dataset.row_id).ok()); std::shared_ptr values; ASSERT_TRUE(values_builder.Finish(&values).ok()); - const auto schema = arrow::schema( - {arrow::field("row_id", arrow::int64(), false), - arrow::field(nested ? "nested_only" : "root_only", arrow::int64(), false)}); - ASSERT_TRUE(write_lance_record_batch(root / (nested ? "nested/data.lance" : "data.lance"), + const auto schema = + arrow::schema({arrow::field("row_id", arrow::int64(), false), + arrow::field(dataset.column_name, arrow::int64(), false)}); + ASSERT_TRUE(write_lance_record_batch(dataset.path, arrow::RecordBatch::Make(schema, 1, {values, values})) .ok()); } - config::user_files_secure_path = root.string(); + config::user_files_secure_path = (root / "secure-link").string(); for (const auto& input : - {std::make_pair("file://" + root.string() + "/jump/../data.lance", false), - std::make_pair("file://" + root.string() + "/missing/../data.lance", false), - std::make_pair("file://" + root.string() + "/jump/%2e%2E/data.lance", false), - std::make_pair("FILE://" + root.string() + "/data.lance", false), - std::make_pair(backslash_uri, false), - std::make_pair("fIlE://" + root.string() + "/jump/.%2e/data.lance", false), - std::make_pair("file://" + root.string() + "/jump%2f../data.lance", true), - std::make_pair(std::string("jump/../data.lance"), true)}) { - SCOPED_TRACE(input.first); + {std::make_pair("file://" + root.string() + "/jump/../data.lance", &datasets[0]), + std::make_pair("file://" + root.string() + "/missing/../data.lance", &datasets[0]), + std::make_pair("file://" + root.string() + "/jump/%2e%2E/data.lance", &datasets[0]), + std::make_pair("FILE://" + root.string() + "/data.lance", &datasets[0]), + std::make_pair(backslash_uri, &datasets[0]), + std::make_pair("fIlE://" + root.string() + "/jump/.%2e/data.lance", &datasets[0]), + std::make_pair("file://" + root.string() + "/jump%2f../data.lance", &datasets[1]), + std::make_pair(std::string("jump/../data.lance"), &datasets[0]), + std::make_pair((root / "jump/../data.lance").string(), &datasets[0]), + std::make_pair((root / "missing/../data.lance").string(), &datasets[0]), + std::make_pair("file:" + (root / "data.lance").relative_path().string(), &datasets[0]), + std::make_pair("file://" + root.string() + "/data.lance ", &datasets[0]), + std::make_pair(" \tFi\nLe://" + root.string() + "/data.la\tnce\r\n ", &datasets[0]), + std::make_pair("file://" + root.string() + "/data.lance%20", &datasets[2]), + std::make_pair("file://" + root.string() + "/data.lance%20 ", &datasets[2]), + std::make_pair(std::string("data.lance "), &datasets[2]), + std::make_pair((root / "data.lance ").string(), &datasets[2])}) { + SCOPED_TRACE("[" + input.first + "]"); + const auto& expected = *input.second; TQueryGlobals globals; RuntimeState state(globals); RuntimeProfile profile("lance_uri_normalization"); @@ -4498,11 +4521,10 @@ TEST(LanceTableReaderLocalPathTest, FileUrisNormalizeBeforeFilesystemResolution) std::vector names; std::vector types; ASSERT_TRUE(reader.fetch_schema(range, scan_params, &names, &types).ok()); - EXPECT_EQ((std::vector {"row_id", input.second ? "nested_only" : "root_only"}), - names); + EXPECT_EQ((std::vector {"row_id", expected.column_name}), names); LanceTableReader::DatasetKey key; ASSERT_TRUE(reader._dataset_key(range, &key).ok()); - EXPECT_EQ((root / (input.second ? "nested/data.lance" : "data.lance")).string(), key.uri); + EXPECT_EQ(expected.path.string(), key.uri); ASSERT_TRUE(prepare_range(&reader, range).ok()); Block block; add_output_columns(&block, columns); @@ -4517,7 +4539,7 @@ TEST(LanceTableReaderLocalPathTest, FileUrisNormalizeBeforeFilesystemResolution) row_ids.insert(row_ids.end(), values.begin(), values.end()); } } - EXPECT_EQ((std::vector {input.second ? 42 : 7}), row_ids); + EXPECT_EQ((std::vector {expected.row_id}), row_ids); ASSERT_TRUE(reader.close().ok()); } } diff --git a/be/test/io/fs/local_file_system_test.cpp b/be/test/io/fs/local_file_system_test.cpp index 48f0a901a673e1..09660e2d84741c 100644 --- a/be/test/io/fs/local_file_system_test.cpp +++ b/be/test/io/fs/local_file_system_test.cpp @@ -726,6 +726,8 @@ TEST_F(LocalFileSystemTest, TestCanonicalizeFileUriIdentity) { std::filesystem::create_directory(secure / "nested/data.lance"); std::filesystem::create_directory(secure / "data.lance"); std::filesystem::create_directory(secure / "a b.lance"); + std::filesystem::create_directory(secure / "data.lance "); + std::filesystem::create_directories(secure / (secure / "data.lance").relative_path()); std::filesystem::create_directory(root / "outside"); std::filesystem::create_directory_symlink(secure / "nested/child", secure / "jump"); std::filesystem::create_directory(secure / "literal\\name.lance"); @@ -756,6 +758,14 @@ TEST_F(LocalFileSystemTest, TestCanonicalizeFileUriIdentity) { std::make_pair(std::string("literal\\name.lance"), secure / "literal\\name.lance"), std::make_pair("fIlE://" + secure.string() + "/a%20b.lance?q=%00#tag", secure / "a b.lance"), + std::make_pair("file:" + (secure / "data.lance").relative_path().string(), + secure / "data.lance"), + std::make_pair("file://" + secure.string() + "/data.lance ", secure / "data.lance"), + std::make_pair(" \tFi\nLe://" + secure.string() + "/data.la\tnce\r\n ", + secure / "data.lance"), + std::make_pair("file://" + secure.string() + "/data.lance%20", secure / "data.lance "), + std::make_pair("file://" + secure.string() + "/data.lance%20 ", secure / "data.lance "), + std::make_pair(std::string("data.lance "), secure / "data.lance "), std::make_pair(std::string("jump/../data.lance"), secure / "nested/data.lance")}) { std::string actual; const auto status = fs->canonicalize_local_file(secure.string(), input.first, &actual); From 66d2b918a4cbcc4a5dca2271396700da91bbe365 Mon Sep 17 00:00:00 2001 From: Dylan Date: Sat, 24 Oct 2026 15:23:12 +0800 Subject: [PATCH 09/13] [fix](tvf) Enforce LOCAL writer admission and preserve local path identity ### What problem does this PR solve? Issue Number: N/A Related PR: #69528 Problem Summary: Remote-named TVF sinks could select the built-in local JNI writer while LOCAL was disabled. Apply the shared LOCAL admission check before sink analysis and final sink binding, matching the loader's class-name representation. LOCAL Lance also needs to preserve the pinned SDK's home aliases and file-URL drive components before canonical containment; add distinguishable schema and scan fixtures without changing ordinary glob or remote/catalog routing. ### Release note The LOCAL startup setting covers sinks using the built-in local JNI writer under S3/HDFS names. LOCAL Lance preserves supported home aliases and file URL drive identity inside the secure directory. ### Check List (For Author) - Test: 39 selected FE tests and 15 selected BE filesystem/Lance tests passed through standard wrappers with fresh XML, zero failures/errors/skips and unchanged sources. Java Checkstyle, changed-line clang-format16 where applicable and git diff --check passed. - Behavior changed: Yes, enforce the existing default-disable policy and retain valid local dataset identity. - Does this need documentation: The original PR documents the startup opt-in; this supplement adds no new configuration or syntax. --- be/src/format_v2/table/lance_reader.cpp | 35 +++++- be/src/io/fs/local_file_system.cpp | 111 ++++++++++++------ be/src/io/fs/local_file_system.h | 3 + be/test/format_v2/table/lance_reader_test.cpp | 103 +++++++++++++++- be/test/io/fs/local_file_system_test.cpp | 29 +++++ .../nereids/rules/analysis/BindSink.java | 4 +- .../apache/doris/planner/TVFTableSink.java | 2 + .../LocalTableValuedFunction.java | 14 +++ .../rules/analysis/BindTVFTableSinkTest.java | 93 +++++++++++++++ 9 files changed, 356 insertions(+), 38 deletions(-) diff --git a/be/src/format_v2/table/lance_reader.cpp b/be/src/format_v2/table/lance_reader.cpp index c1db3b5f98b322..808d23ce2cc911 100644 --- a/be/src/format_v2/table/lance_reader.cpp +++ b/be/src/format_v2/table/lance_reader.cpp @@ -22,9 +22,13 @@ #include #include #include +#include +#include #include #include +#include +#include #include #include #include @@ -42,6 +46,33 @@ namespace doris::format::lance { namespace { +std::string expand_home_path(const std::string& path) { + if (path != "~" && !path.starts_with("~/")) { + return path; + } + // Match the SDK's std::env::home_dir: HOME first, then the current user's passwd entry. + std::string home; + if (const char* value = std::getenv("HOME"); value != nullptr && value[0] != '\0') { + home = value; + } else { + std::vector buffer(1024); + struct passwd entry {}; + struct passwd* result = nullptr; + int error; + while ((error = getpwuid_r(getuid(), &entry, buffer.data(), buffer.size(), &result)) == + ERANGE) { + buffer.resize(buffer.size() * 2); + } + if (error == 0 && result != nullptr && entry.pw_dir != nullptr) { + home = entry.pw_dir; + } + } + if (home.empty()) { + return path; + } + return path == "~" ? home : (io::Path(home) / path.substr(2)).string(); +} + Status resolve_dataset_uri(const TFileRangeDesc& range, const TFileScanRangeParams& scan_params, std::string* uri) { const auto& requested_uri = range.table_format_params.lance_params.dataset_uri; @@ -69,7 +100,9 @@ Status resolve_dataset_uri(const TFileRangeDesc& range, const TFileScanRangePara std::string secure_root; RETURN_IF_ERROR(io::global_local_filesystem()->canonicalize( config::user_files_secure_path, &secure_root)); - local_input = (io::Path(secure_root) / requested_uri).lexically_normal().string(); + local_input = (io::Path(secure_root) / expand_home_path(requested_uri)) + .lexically_normal() + .string(); } return io::global_local_filesystem()->canonicalize_local_file( config::user_files_secure_path, local_input, uri); diff --git a/be/src/io/fs/local_file_system.cpp b/be/src/io/fs/local_file_system.cpp index d51ecb6decf4d1..769ca84995c3c9 100644 --- a/be/src/io/fs/local_file_system.cpp +++ b/be/src/io/fs/local_file_system.cpp @@ -452,13 +452,7 @@ const std::shared_ptr& global_local_filesystem() { return local_fs; } -Status LocalFileSystem::canonicalize_local_file(const std::string& dir, - const std::string& file_path, - std::string* full_path) { - // An empty root would turn a relative request into an unrestricted absolute path. - if (dir.empty()) { - return Status::InvalidArgument("user_files_secure_path must not be empty"); - } +Status LocalFileSystem::parse_local_path(const std::string& file_path, Path& local_path) { // Reject embedded NULs before passing paths to filesystem APIs that use C strings. if (file_path.empty() || file_path.find('\0') != std::string::npos) { return Status::InvalidArgument("local file path must not be empty or contain NUL"); @@ -495,7 +489,50 @@ Status LocalFileSystem::canonicalize_local_file(const std::string& dir, } } } - Path local_path; + const auto hex_digit = [](char c) -> int { + if (c >= '0' && c <= '9') return c - '0'; + if (c >= 'a' && c <= 'f') return c - 'a' + 10; + if (c >= 'A' && c <= 'F') return c - 'A' + 10; + return -1; + }; + const auto percent_decode = [&](const std::string& encoded) { + std::string decoded; + decoded.reserve(encoded.size()); + for (size_t i = 0; i < encoded.size(); ++i) { + if (encoded[i] == '%' && i + 2 < encoded.size()) { + const int high = hex_digit(encoded[i + 1]); + const int low = hex_digit(encoded[i + 2]); + if (high >= 0 && low >= 0) { + decoded += static_cast((high << 4) | low); + i += 2; + continue; + } + } + decoded += encoded[i]; + } + return decoded; + }; + const auto is_drive = [](const std::string& segment) { + return segment.size() == 2 && + ((segment[0] >= 'a' && segment[0] <= 'z') || + (segment[0] >= 'A' && segment[0] <= 'Z')) && + (segment[1] == ':' || segment[1] == '|'); + }; + bool empty_file_host = false; + if (is_file_uri && local_input.compare(5, 2, "//") == 0) { + const auto host_end = local_input.find_first_of("/?#", 7); + auto host = local_input.substr(7, host_end == std::string::npos ? host_end : host_end - 7); + if (is_drive(host)) { + // The URL parser treats a drive in the host position as the first path segment. + local_input.erase(5, 2); + } else { + host = percent_decode(host); + std::transform(host.begin(), host.end(), host.begin(), [](unsigned char c) { + return c >= 'A' && c <= 'Z' ? c + ('a' - 'A') : c; + }); + empty_file_host = host.empty() || host == "localhost"; + } + } // Convert file: URIs to filesystem paths and reject remote schemes before any reader opens them. RETURN_IF_ERROR(convert_to_abs_path(local_input, local_path)); if (is_file_uri) { @@ -510,20 +547,27 @@ Status LocalFileSystem::canonicalize_local_file(const std::string& dir, } // URL dot segments are removed before symlinks are followed and before %2F // becomes a path separator. Keep empty segments: "a//../b" denotes "a/b". + // url 2.5.8 retains an initial empty segment for empty/localhost authorities + // until after drive detection and dot reduction (file:///C| differs from file:/C|). + if (empty_file_host) { + encoded.insert(0, "/"); + } std::vector segments; for (size_t start = 1;;) { const size_t end = encoded.find('/', start); - const auto segment = - encoded.substr(start, end == std::string::npos ? end : end - start); + auto segment = encoded.substr(start, end == std::string::npos ? end : end - start); const bool dot = segment == "." || segment == "%2e" || segment == "%2E"; const bool parent = segment == ".." || segment == ".%2e" || segment == ".%2E" || segment == "%2e." || segment == "%2E." || segment == "%2e%2e" || segment == "%2e%2E" || segment == "%2E%2e" || segment == "%2E%2E"; if (parent) { - if (!segments.empty()) { + if (!segments.empty() && !is_drive(segments.back())) { segments.pop_back(); } } else if (!dot) { + if (segments.empty() && is_drive(segment)) { + segment[1] = ':'; + } segments.push_back(segment); } if (end == std::string::npos) { @@ -534,39 +578,40 @@ Status LocalFileSystem::canonicalize_local_file(const std::string& dir, } start = end + 1; } + size_t first = 0; + while (first < segments.size() && segments[first].empty()) { + ++first; + } encoded = "/"; - for (size_t i = 0; i < segments.size(); ++i) { - if (i > 0) { + for (size_t i = first; i < segments.size(); ++i) { + if (i > first) { encoded += '/'; } encoded += segments[i]; } - const auto hex_digit = [](char c) -> int { - if (c >= '0' && c <= '9') return c - '0'; - if (c >= 'a' && c <= 'f') return c - 'a' + 10; - if (c >= 'A' && c <= 'F') return c - 'A' + 10; - return -1; - }; - std::string decoded; - decoded.reserve(encoded.size()); - for (size_t i = 0; i < encoded.size(); ++i) { - if (encoded[i] == '%' && i + 2 < encoded.size()) { - const int high = hex_digit(encoded[i + 1]); - const int low = hex_digit(encoded[i + 2]); - if (high >= 0 && low >= 0) { - decoded += static_cast((high << 4) | low); - i += 2; - continue; - } - } - decoded += encoded[i]; - } + auto decoded = percent_decode(encoded); // %00 must not truncate the path when canonicalization reaches C filesystem APIs. if (decoded.find('\0') != std::string::npos) { return Status::InvalidArgument("local file path must not contain NUL"); } + // The SDK's Unix file-path conversion appends a slash after a trailing drive. + if (decoded.size() > 2 && is_drive(decoded.substr(decoded.size() - 2))) { + decoded += '/'; + } local_path = std::move(decoded); } + return Status::OK(); +} + +Status LocalFileSystem::canonicalize_local_file(const std::string& dir, + const std::string& file_path, + std::string* full_path) { + // An empty root would turn a relative request into an unrestricted absolute path. + if (dir.empty()) { + return Status::InvalidArgument("user_files_secure_path must not be empty"); + } + Path local_path; + RETURN_IF_ERROR(parse_local_path(file_path, local_path)); std::string secure_path; RETURN_IF_ERROR(canonicalize(dir, &secure_path)); std::string canonical_path; diff --git a/be/src/io/fs/local_file_system.h b/be/src/io/fs/local_file_system.h index 1f7ffc8da51964..2b691d519a0912 100644 --- a/be/src/io/fs/local_file_system.h +++ b/be/src/io/fs/local_file_system.h @@ -70,6 +70,9 @@ class LocalFileSystem final : public FileSystem { static std::filesystem::perms PERMS_OWNER_RW; + // Parse a bare path or file URI without filesystem access, home expansion or containment checks. + static Status parse_local_path(const std::string& file_path, Path& local_path); + // Resolve a local path relative to a nonempty secure root, then check canonical containment. // Absolute paths and file: URIs must resolve inside the same root; remote schemes are rejected. Status canonicalize_local_file(const std::string& dir, const std::string& file_path, diff --git a/be/test/format_v2/table/lance_reader_test.cpp b/be/test/format_v2/table/lance_reader_test.cpp index 6aa3b32a8bb3ba..5cad7e3202f697 100644 --- a/be/test/format_v2/table/lance_reader_test.cpp +++ b/be/test/format_v2/table/lance_reader_test.cpp @@ -36,6 +36,7 @@ #include #include #include +#include #include #include #include @@ -4474,7 +4475,9 @@ TEST(LanceTableReaderLocalPathTest, FileUrisNormalizeBeforeFilesystemResolution) {"root_only", 7, root / "data.lance"}, {"nested_only", 42, root / "nested/data.lance"}, {"spaced_only", 99, root / "data.lance "}, - {"prefixed_only", 123, root / (root / "data.lance").relative_path()}}; + {"prefixed_only", 123, root / (root / "data.lance").relative_path()}, + {"pipe_drive_only", 151, root / "C|/data.lance"}, + {"colon_drive_only", 163, root / "C:/data.lance"}}; for (const auto& dataset : datasets) { std::filesystem::create_directories(dataset.path.parent_path()); arrow::Int64Builder values_builder; @@ -4498,6 +4501,10 @@ TEST(LanceTableReaderLocalPathTest, FileUrisNormalizeBeforeFilesystemResolution) std::make_pair("fIlE://" + root.string() + "/jump/.%2e/data.lance", &datasets[0]), std::make_pair("file://" + root.string() + "/jump%2f../data.lance", &datasets[1]), std::make_pair(std::string("jump/../data.lance"), &datasets[0]), + std::make_pair("file://" + root.string() + "/C|/../data.lance", &datasets[4]), + std::make_pair("file://" + root.string() + "/C:/../data.lance", &datasets[5]), + std::make_pair("file://" + root.string() + "/C%7C/../data.lance", &datasets[0]), + std::make_pair(std::string("C|/../data.lance"), &datasets[0]), std::make_pair((root / "jump/../data.lance").string(), &datasets[0]), std::make_pair((root / "missing/../data.lance").string(), &datasets[0]), std::make_pair("file:" + (root / "data.lance").relative_path().string(), &datasets[0]), @@ -4544,6 +4551,100 @@ TEST(LanceTableReaderLocalPathTest, FileUrisNormalizeBeforeFilesystemResolution) } } +TEST(LanceTableReaderLocalPathTest, HomeAliasesPreserveDatasetIdentityAndContainment) { + const char* home = std::getenv("HOME"); + ASSERT_NE(nullptr, home); + ASSERT_NE('\0', home[0]); + const auto home_path = std::filesystem::canonical(home); + const auto name = "lance_home_identity_" + + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count()); + const auto root = home_path / name; + ASSERT_TRUE(std::filesystem::create_directory(root)); + DEFER({ + std::error_code error; + std::filesystem::remove_all(root, error); + }); + const auto original_secure_path = config::user_files_secure_path; + DEFER(config::user_files_secure_path = original_secure_path); + struct DatasetFixture { + const char* column_name; + int64_t row_id; + std::filesystem::path path; + }; + const DatasetFixture datasets[] = {{"home_only", 17, root / "data.lance"}, + {"literal_only", 29, root / "~" / name / "data.lance"}, + {"named_only", 41, root / "~other/data.lance"}}; + for (const auto& dataset : datasets) { + std::filesystem::create_directories(dataset.path.parent_path()); + arrow::Int64Builder values_builder; + ASSERT_TRUE(values_builder.Append(dataset.row_id).ok()); + std::shared_ptr values; + ASSERT_TRUE(values_builder.Finish(&values).ok()); + const auto schema = + arrow::schema({arrow::field("row_id", arrow::int64(), false), + arrow::field(dataset.column_name, arrow::int64(), false)}); + ASSERT_TRUE(write_lance_record_batch(dataset.path, + arrow::RecordBatch::Make(schema, 1, {values, values})) + .ok()); + } + config::user_files_secure_path = root.string(); + for (const auto& input : {std::make_pair("~/" + name + "/data.lance", &datasets[0]), + std::make_pair("file://" + datasets[1].path.string(), &datasets[1]), + std::make_pair(std::string("~other/data.lance"), &datasets[2])}) { + SCOPED_TRACE(input.first); + TQueryGlobals globals; + RuntimeState state(globals); + RuntimeProfile profile("lance_home_identity"); + TFileScanRangeParams scan_params; + LanceTableReader reader; + const Columns columns {projected_column("row_id", TYPE_BIGINT, false)}; + ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, &scan_params).ok()); + auto range = make_latest_lance_range(input.first); + range.__set_file_type(TFileType::FILE_LOCAL); + std::vector names; + std::vector types; + ASSERT_TRUE(reader.fetch_schema(range, scan_params, &names, &types).ok()); + EXPECT_EQ((std::vector {"row_id", input.second->column_name}), names); + LanceTableReader::DatasetKey key; + ASSERT_TRUE(reader._dataset_key(range, &key).ok()); + EXPECT_EQ(input.second->path.string(), key.uri); + ASSERT_TRUE(prepare_range(&reader, range).ok()); + Block block; + add_output_columns(&block, columns); + std::vector row_ids; + bool eos = false; + while (!eos) { + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + if (!eos) { + const auto& values = + assert_cast(*block.get_by_position(0).column) + .get_data(); + row_ids.insert(row_ids.end(), values.begin(), values.end()); + } + } + EXPECT_EQ((std::vector {input.second->row_id}), row_ids); + ASSERT_TRUE(reader.close().ok()); + } + TFileScanRangeParams scan_params; + LanceTableReader reader; + reader._scan_params = &scan_params; + auto range = make_latest_lance_range("~"); + range.__set_file_type(TFileType::FILE_LOCAL); + LanceTableReader::DatasetKey key; + EXPECT_FALSE(reader._dataset_key(range, &key).ok()); + std::vector names; + std::vector types; + EXPECT_FALSE(reader.fetch_schema(range, scan_params, &names, &types).ok()); + config::user_files_secure_path = home_path.string(); + ASSERT_TRUE(reader._dataset_key(range, &key).ok()); + EXPECT_EQ(home_path.string(), key.uri); + // A real home dataset must still be rejected when expansion leaves the configured root. + config::user_files_secure_path = (root / "~").string(); + range.table_format_params.lance_params.__set_dataset_uri("~/" + name + "/data.lance"); + EXPECT_FALSE(reader._dataset_key(range, &key).ok()); + EXPECT_FALSE(reader.fetch_schema(range, scan_params, &names, &types).ok()); +} + TEST(LanceTableReaderLocalPathTest, FixedVersionLocalCatalogKeepsItsUri) { TFileScanRangeParams scan_params; LanceTableReader reader; diff --git a/be/test/io/fs/local_file_system_test.cpp b/be/test/io/fs/local_file_system_test.cpp index 09660e2d84741c..ffa4653b77ad4b 100644 --- a/be/test/io/fs/local_file_system_test.cpp +++ b/be/test/io/fs/local_file_system_test.cpp @@ -783,6 +783,35 @@ TEST_F(LocalFileSystemTest, TestCanonicalizeFileUriIdentity) { } } +TEST_F(LocalFileSystemTest, TestParseFileUriDriveIdentity) { + // Expected Unix paths from the pinned Lance SDK's url 2.5.8, without root-level fixtures. + for (const auto& input : std::vector> { + {"file://C:/data.lance", "/C:/data.lance"}, + {"file://C|/data.lance", "/C:/data.lance"}, + {"file:C|/data.lance", "/C:/data.lance"}, + {"file:/C|/data.lance", "/C:/data.lance"}, + {"file://localhost/C|/data.lance", "/C|/data.lance"}, + {"file://LoCaL%68oSt/C|/data.lance", "/C|/data.lance"}, + {"file://host/C|/data.lance", "/C:/data.lance"}, + {"file:///C|/data.lance", "/C|/data.lance"}, + {"file:////C|/data.lance", "/C|/data.lance"}, + {"file:///../C|/data.lance", "/C:/data.lance"}, + {"file:/x/../C|/data.lance", "/C:/data.lance"}, + {"file:///C:/../../data.lance", "/C:/data.lance"}, + {"file:///C|/../data.lance", "/C|/data.lance"}, + {"file:///x/C:/../data.lance", "/x/C:/data.lance"}, + {"file:///C%7C/data.lance", "/C|/data.lance"}, + {"file:/C%7C/../data.lance", "/data.lance"}, + {"file://C:", "/C:/"}, + {"file:///C|", "/C|/"}, + {"/C|/data.lance", "/C|/data.lance"}}) { + SCOPED_TRACE(input.first); + io::Path actual; + ASSERT_TRUE(io::LocalFileSystem::parse_local_path(input.first, actual).ok()); + EXPECT_EQ(input.second, actual.string()); + } +} + TEST_F(LocalFileSystemTest, TestConvertToAbsPath) { io::Path abs_path; Status st; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java index 0fac5ba4344f39..dbb01c6089985f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java @@ -661,9 +661,7 @@ private Plan bindTVFTableSink(MatchingContext> ctx) { } // INSERT/EXPLAIN INSERT bind a sink without constructing the LOCAL read TVF. - if (tvfName.equals("local")) { - LocalTableValuedFunction.checkEnabled(); - } + LocalTableValuedFunction.checkSinkEnabled(tvfName, properties); // Validate required properties if (!properties.containsKey("file_path")) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/TVFTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/planner/TVFTableSink.java index 102da9af2e466f..ed0d9018ab486a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/TVFTableSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/TVFTableSink.java @@ -24,6 +24,7 @@ import org.apache.doris.datasource.property.fileformat.FileFormatProperties; import org.apache.doris.datasource.property.fileformat.OrcFileFormatProperties; import org.apache.doris.datasource.property.storage.StorageProperties; +import org.apache.doris.tablefunction.LocalTableValuedFunction; import org.apache.doris.thrift.TColumn; import org.apache.doris.thrift.TDataSink; import org.apache.doris.thrift.TDataSinkType; @@ -82,6 +83,7 @@ public long getBackendId() { } public void bindDataSink() throws AnalysisException { + LocalTableValuedFunction.checkSinkEnabled(tvfName, properties); TTVFTableSink tSink = new TTVFTableSink(); tSink.setTvfName(tvfName); diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/LocalTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/LocalTableValuedFunction.java index 335ae199cc137a..d65311b309b62a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/LocalTableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/LocalTableValuedFunction.java @@ -143,6 +143,20 @@ private void getFileListFromBackend() throws AnalysisException { } } + /** Apply LOCAL admission to sinks that select the built-in local JNI writer as well. */ + public static void checkSinkEnabled(String tvfName, Map properties) { + String writerClass = properties.getOrDefault("writer_class", ""); + // JNI consumes a C string, and ScannerLoader accepts slash-separated class names. + int nul = writerClass.indexOf('\0'); + if (nul >= 0) { + writerClass = writerClass.substring(0, nul); + } + if ("local".equalsIgnoreCase(tvfName) + || "org.apache.doris.writer.LocalFileJniWriter".equals(writerClass.replace('/', '.'))) { + checkEnabled(); + } + } + /** Check read and write admission before properties or backend access. */ public static void checkEnabled() { if (!Config.enable_local_tvf) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindTVFTableSinkTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindTVFTableSinkTest.java index b789c4c391fd22..1197c27ee6bb92 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindTVFTableSinkTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindTVFTableSinkTest.java @@ -17,7 +17,10 @@ package org.apache.doris.nereids.rules.analysis; +import org.apache.doris.catalog.Env; import org.apache.doris.common.Config; +import org.apache.doris.mysql.privilege.AccessControllerManager; +import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.parser.NereidsParser; import org.apache.doris.nereids.trees.expressions.NamedExpression; @@ -29,15 +32,24 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalTVFTableSink; import org.apache.doris.nereids.util.MemoTestUtils; import org.apache.doris.nereids.util.PlanChecker; +import org.apache.doris.planner.PlanNodeId; +import org.apache.doris.planner.TVFTableSink; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.StmtExecutor; +import org.apache.doris.tablefunction.LocalTableValuedFunction; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; class BindTVFTableSinkTest { private boolean originalEnableLocalTvf; @@ -81,6 +93,87 @@ void disabledLocalDoesNotBlockRemoteSinks() { } } + @Test + void disabledLocalWriterRejectsRemoteInsertAndExplain() { + for (String prefix : Arrays.asList("", "EXPLAIN ")) { + for (String name : Arrays.asList("s3", "hdfs")) { + for (String writer : Arrays.asList("org.apache.doris.writer.LocalFileJniWriter", + "org/apache/doris/writer/LocalFileJniWriter")) { + for (String props : Arrays.asList("", "'file_path'='/tmp/local_gate_', 'format'='csv',")) { + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> analyzeInsert(prefix + "INSERT INTO " + name + "(" + props + + "'writer_class'='" + writer + "') SELECT 1")); + Assertions.assertTrue(exception.getMessage().contains("LOCAL TVF is disabled")); + } + } + } + } + } + + @Test + void directSinkBindingChecksEffectiveLocalWriter() throws Exception { + Map properties = new HashMap<>(); + properties.put("file_path", "s3://bucket/data"); + properties.put("format", "csv"); + properties.put("s3.endpoint", "http://127.0.0.1:9000"); + properties.put("s3.region", "us-east-1"); + properties.put("s3.access_key", "test-key"); + properties.put("s3.secret_key", "test-secret"); + for (String name : Arrays.asList("s3", "hdfs", "local")) { + for (String writer : Arrays.asList("org.apache.doris.writer.LocalFileJniWriter", + "org/apache/doris/writer/LocalFileJniWriter", + "org/apache/doris/writer/LocalFileJniWriter\0ignored")) { + properties.put("writer_class", writer); + TVFTableSink sink = new TVFTableSink(new PlanNodeId(0), name, properties, + Collections.emptyList()); + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, sink::bindDataSink); + Assertions.assertTrue(exception.getMessage().contains("LOCAL TVF is disabled")); + } + } + Config.enable_local_tvf = true; + properties.put("writer_class", "org.apache.doris.writer.LocalFileJniWriter"); + new TVFTableSink(new PlanNodeId(0), "s3", properties, Collections.emptyList()).bindDataSink(); + Config.enable_local_tvf = false; + properties.remove("writer_class"); + new TVFTableSink(new PlanNodeId(0), "s3", properties, Collections.emptyList()).bindDataSink(); + properties.put("writer_class", "org.apache.doris.writer.OtherWriter"); + Assertions.assertDoesNotThrow(() -> LocalTableValuedFunction.checkSinkEnabled("s3", properties)); + } + + @Test + void enabledLocalWriterKeepsRemotePropertyValidation() { + Config.enable_local_tvf = true; + for (String name : Arrays.asList("s3", "hdfs")) { + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> analyzeInsert("INSERT INTO " + name + + "('writer_class'='org.apache.doris.writer.LocalFileJniWriter') SELECT 1")); + Assertions.assertTrue(exception.getMessage().contains("requires 'file_path'")); + } + } + + @Test + void loadUserCannotExecuteLocalWriterWhileDisabled() { + ConnectContext context = MemoTestUtils.createConnectContext(); + String sql = "INSERT INTO s3('file_path'='/tmp/local_gate_', 'format'='csv'," + + "'writer_class'='org.apache.doris.writer.LocalFileJniWriter') SELECT 1"; + MemoTestUtils.createStatementContext(context, sql); + InsertIntoTVFCommand command = (InsertIntoTVFCommand) new NereidsParser().parseSingle(sql); + Env env = Mockito.spy(Env.getCurrentEnv()); + AccessControllerManager accessManager = Mockito.mock(AccessControllerManager.class); + Mockito.doReturn(accessManager).when(env).getAccessManager(); + Mockito.when(accessManager.checkGlobalPriv(context, PrivPredicate.LOAD)).thenReturn(true); + StmtExecutor executor = Mockito.mock(StmtExecutor.class); + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class, Mockito.CALLS_REAL_METHODS)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> command.run(context, executor)); + Assertions.assertTrue(exception.getMessage().contains("LOCAL TVF is disabled")); + Mockito.verify(accessManager).checkGlobalPriv(context, PrivPredicate.ADMIN); + Mockito.verify(accessManager).checkGlobalPriv(context, PrivPredicate.LOAD); + Mockito.verifyNoInteractions(executor); + } + } + private Plan analyzeInsert(String sql) { ConnectContext context = MemoTestUtils.createConnectContext(); LogicalPlan parsed = new NereidsParser().parseSingle(sql); From de692c0306f780a7a327f53d89700aa77182c753 Mon Sep 17 00:00:00 2001 From: Dylan Date: Sat, 24 Oct 2026 16:43:54 +0800 Subject: [PATCH 10/13] [fix](tvf) Gate effective local Hadoop and JNI writers ### What problem does this PR solve? Issue Number: N/A Related PR: #69528 Problem Summary: Native HDFS sinks could select Hadoop local storage and perform BE writes or FE cleanup while LOCAL was disabled. Check the selected filesystem before analysis, after backend-property normalization and before cleanup, including built-in local provider overrides, Hadoop key normalization and default-FS aliases. Indirect filesystem selectors require the existing startup opt-in because the FE cannot prove their runtime storage is remote. ### Release note The LOCAL startup opt-in also covers effective local Hadoop sinks and FE cleanup. Indirect Hadoop filesystem selectors require enable_local_tvf=true and an FE restart. Explicit remote configurations and unselected local providers remain available. ### Check List (For Author) - Test: 45 selected FE tests passed via the standard JDK17 wrapper with fresh XML, zero failures/errors/skips and unchanged sources; Java Checkstyle and git diff --check passed. Tests cover INSERT/EXPLAIN, LOAD, normalized config resources, provider/default aliases, opt-in, remote controls and rejection before filesystem creation. BE/Lance source is unchanged. - Behavior changed: Yes, complete the existing default-off LOCAL write policy. - Does this need documentation: Document the effective/indirect filesystem selector boundary alongside the original startup opt-in. --- .../commands/insert/InsertIntoTVFCommand.java | 7 + .../apache/doris/planner/TVFTableSink.java | 4 + .../LocalTableValuedFunction.java | 117 ++++++++- .../rules/analysis/BindTVFTableSinkTest.java | 230 ++++++++++++++++++ 4 files changed, 349 insertions(+), 9 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTVFCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTVFCommand.java index fae7747b5d42f3..fa7593427d9c9a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTVFCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTVFCommand.java @@ -24,6 +24,7 @@ import org.apache.doris.common.Status; import org.apache.doris.common.UserException; import org.apache.doris.common.util.BrokerUtil; +import org.apache.doris.datasource.property.storage.HdfsCompatibleProperties; import org.apache.doris.datasource.property.storage.StorageProperties; import org.apache.doris.fs.FileSystemFactory; import org.apache.doris.fs.remote.RemoteFileSystem; @@ -45,6 +46,7 @@ import org.apache.doris.qe.QeProcessorImpl; import org.apache.doris.qe.QeProcessorImpl.QueryInfo; import org.apache.doris.qe.StmtExecutor; +import org.apache.doris.tablefunction.LocalTableValuedFunction; import org.apache.doris.task.LoadEtlTask; import org.apache.doris.thrift.TStatusCode; import org.apache.doris.thrift.TUniqueId; @@ -175,6 +177,7 @@ public Plan getExplainPlan(ConnectContext ctx) { private void deleteExistingFilesInFE(String tvfName, Map props) throws Exception { + LocalTableValuedFunction.checkSinkEnabled(tvfName, props); String filePath = props.get("file_path"); // Extract parent directory from prefix path: s3://bucket/path/to/prefix_ -> s3://bucket/path/to/ String parentDir = BrokerUtil.extractParentDirectory(filePath); @@ -192,6 +195,10 @@ private void deleteExistingFilesInFE(String tvfName, Map props) fsCopyProps.remove("compress_type"); StorageProperties storageProps = StorageProperties.createPrimary(fsCopyProps); + if (storageProps instanceof HdfsCompatibleProperties) { + LocalTableValuedFunction.checkHadoopFileSystemEnabled( + filePath, storageProps.getBackendConfigProperties()); + } RemoteFileSystem fs = FileSystemFactory.get(storageProps); org.apache.doris.backup.Status deleteStatus = fs.deleteDirectory(parentDir); if (!deleteStatus.ok()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/TVFTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/planner/TVFTableSink.java index ed0d9018ab486a..579910cb10139b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/TVFTableSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/TVFTableSink.java @@ -130,6 +130,10 @@ public void bindDataSink() throws AnalysisException { } } + if (fileType == TFileType.FILE_HDFS && !properties.containsKey("writer_class")) { + LocalTableValuedFunction.checkHdfsSinkEnabled(backendConnectProps); + } + String filePath = properties.get("file_path"); tSink.setFilePath(filePath); diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/LocalTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/LocalTableValuedFunction.java index d65311b309b62a..c399a2c3d9096c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/LocalTableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/LocalTableValuedFunction.java @@ -143,18 +143,117 @@ private void getFileListFromBackend() throws AnalysisException { } } - /** Apply LOCAL admission to sinks that select the built-in local JNI writer as well. */ + /** Apply LOCAL admission to the effective sink before planning or filesystem side effects. */ public static void checkSinkEnabled(String tvfName, Map properties) { - String writerClass = properties.getOrDefault("writer_class", ""); - // JNI consumes a C string, and ScannerLoader accepts slash-separated class names. - int nul = writerClass.indexOf('\0'); - if (nul >= 0) { - writerClass = writerClass.substring(0, nul); - } - if ("local".equalsIgnoreCase(tvfName) - || "org.apache.doris.writer.LocalFileJniWriter".equals(writerClass.replace('/', '.'))) { + if ("local".equalsIgnoreCase(tvfName) || isLocalWriter(properties.getOrDefault("writer_class", ""))) { checkEnabled(); } + if ("hdfs".equalsIgnoreCase(tvfName)) { + if (!properties.containsKey("writer_class")) { + checkHdfsSinkEnabled(properties); + } + if (Boolean.parseBoolean(properties.getOrDefault("delete_existing_files", "false"))) { + checkHadoopFileSystemEnabled(properties.getOrDefault("file_path", ""), properties); + } + } + } + + private static boolean isLocalWriter(String writer) { + // ScannerLoader accepts slashes, and JNI passes the class name as a C string. + return "org.apache.doris.writer.LocalFileJniWriter".equals(nativeStringPrefix(writer).replace('/', '.')); + } + + /** Check the NameNode and provider used by the native FILE_HDFS writer. */ + public static void checkHdfsSinkEnabled(Map properties) { + String nameNode = nativeStringPrefix(properties.getOrDefault("fs.defaultFS", "")).trim(); + if ("default".equals(nameNode)) { + // libhdfs delegates this selector to the BE's defaults, which may select local storage. + checkEnabled(); + } + // libhdfs prefixes bare NameNodes with hdfs:// rather than using the output path's scheme. + checkHadoopSchemeEnabled(nameNode.contains("://") ? uriScheme(nameNode) : "hdfs", properties); + } + + /** Check Hadoop's path/default-filesystem selection before FE-side cleanup. */ + public static void checkHadoopFileSystemEnabled(String path, Map properties) { + String scheme = uriScheme(path); + if (!scheme.isEmpty()) { + checkHadoopSchemeEnabled(scheme, properties); + return; + } + String defaultFs = hadoopSelectorProperty("fs.defaultFS", properties); + String legacyDefaultFs = hadoopSelectorProperty("fs.default.name", properties); + // Configuration.set synchronizes aliases; either value can win when the map is applied. + // Do not assume that the canonical key overrides a local legacy selector. + if (defaultFs != null) { + checkHadoopDefaultEnabled(defaultFs, properties); + } + if (legacyDefaultFs != null) { + checkHadoopDefaultEnabled(legacyDefaultFs, properties); + } + if (defaultFs == null && legacyDefaultFs == null) { + checkHadoopSchemeEnabled("file", properties); + } + } + + private static void checkHadoopDefaultEnabled(String value, Map properties) { + value = hadoopSelectorValue(value).trim(); + // Hadoop retains the old "local" default-FS spelling. + String scheme = "local".equals(value) ? "file" : uriScheme(value); + checkHadoopSchemeEnabled(scheme.isEmpty() ? "hdfs" : scheme, properties); + } + + private static void checkHadoopSchemeEnabled(String scheme, Map properties) { + if ("file".equalsIgnoreCase(scheme)) { + checkEnabled(); + } + String implementationKey = "fs." + scheme + ".impl"; + for (Map.Entry entry : properties.entrySet()) { + // The native Hadoop configuration boundary consumes C strings for keys and values. + if (implementationKey.equals(nativeStringPrefix(entry.getKey()).trim())) { + String implementation = hadoopSelectorValue(nativeStringPrefix(entry.getValue())).trim(); + if ("org.apache.hadoop.fs.LocalFileSystem".equals(implementation) + || "org.apache.hadoop.fs.RawLocalFileSystem".equals(implementation)) { + checkEnabled(); + } + } + } + } + + private static String hadoopSelectorProperty(String key, Map properties) { + String value = null; + for (Map.Entry entry : properties.entrySet()) { + // Configuration.set trims keys; native configuration strings also stop at NUL. + if (key.equals(nativeStringPrefix(entry.getKey()).trim())) { + String candidate = nativeStringPrefix(entry.getValue()); + if (value != null && !value.equals(candidate)) { + // FE and BE map iteration can choose different conflicting spellings. + checkEnabled(); + } + value = candidate; + } + } + return value; + } + + private static String hadoopSelectorValue(String value) { + if (value.contains("${")) { + // Hadoop aliases and BE runtime properties can change the referenced value. + // An indirect filesystem selector needs opt-in; the FE cannot prove it is remote. + checkEnabled(); + } + return value; + } + + private static String uriScheme(String path) { + int colon = path.indexOf(':'); + int slash = path.indexOf('/'); + return colon > 0 && (slash < 0 || colon < slash) ? path.substring(0, colon) : ""; + } + + private static String nativeStringPrefix(String value) { + int nul = value.indexOf('\0'); + return nul < 0 ? value : value.substring(0, nul); } /** Check read and write admission before properties or backend access. */ diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindTVFTableSinkTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindTVFTableSinkTest.java index 1197c27ee6bb92..bf2af2d875e8e8 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindTVFTableSinkTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindTVFTableSinkTest.java @@ -19,6 +19,7 @@ import org.apache.doris.catalog.Env; import org.apache.doris.common.Config; +import org.apache.doris.fs.FileSystemFactory; import org.apache.doris.mysql.privilege.AccessControllerManager; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.nereids.exceptions.AnalysisException; @@ -45,6 +46,11 @@ import org.mockito.MockedStatic; import org.mockito.Mockito; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -174,6 +180,230 @@ void loadUserCannotExecuteLocalWriterWhileDisabled() { } } + @Test + void disabledNativeHadoopLocalSinksRejectInsertAndExplain() { + for (String prefix : Arrays.asList("", "EXPLAIN ")) { + for (String storage : Arrays.asList("'fs.defaultFS'='file:///'", + "'fs.defaultFS'='hdfs://namenode:8020'," + + "'fs.hdfs.impl'='org.apache.hadoop.fs.LocalFileSystem'", + "'fs.defaultFS'='hdfs://namenode:8020'," + + "'fs.hdfs.impl'='org.apache.hadoop.fs.RawLocalFileSystem'", + "'fs.defaultFS'='hdfs://namenode:8020','fs.hdfs.impl'='${fs.file.impl}'," + + "'fs.file.impl'='org.apache.hadoop.fs.LocalFileSystem'", + "'fs.defaultFS'='hdfs://namenode:8020'," + + "'fs.hdfs.impl '='org.apache.hadoop.fs.LocalFileSystem'")) { + for (String cleanup : Arrays.asList("false", "true")) { + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> analyzeInsert(prefix + "INSERT INTO hdfs('file_path'='/tmp/local_gate_'," + + "'format'='csv','delete_existing_files'='" + cleanup + "'," + + storage + ") SELECT 1")); + Assertions.assertTrue(exception.getMessage().contains("LOCAL TVF is disabled")); + } + } + } + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> analyzeInsert("INSERT INTO hdfs('fs.defaultFS'='file:///') SELECT 1")); + Assertions.assertTrue(exception.getMessage().contains("LOCAL TVF is disabled")); + } + + @Test + void directNativeHadoopBindingHonorsLocalOptIn() throws Exception { + Map properties = new HashMap<>(); + properties.put("file_path", "/tmp/local_gate_"); + properties.put("format", "csv"); + for (String defaultFs : Arrays.asList("file:///", "default")) { + properties.put("fs.defaultFS", defaultFs); + TVFTableSink sink = new TVFTableSink(new PlanNodeId(0), "hdfs", properties, + Collections.emptyList()); + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, sink::bindDataSink); + Assertions.assertTrue(exception.getMessage().contains("LOCAL TVF is disabled")); + } + properties.put("fs.defaultFS", "hdfs://namenode:8020"); + properties.put("fs.hdfs.impl", "org.apache.hadoop.fs.RawLocalFileSystem"); + TVFTableSink sink = new TVFTableSink(new PlanNodeId(0), "hdfs", properties, Collections.emptyList()); + Assertions.assertThrows(AnalysisException.class, sink::bindDataSink); + Config.enable_local_tvf = true; + sink.bindDataSink(); + properties.put("fs.defaultFS", "file:///"); + sink.bindDataSink(); + Config.enable_local_tvf = false; + properties.put("fs.defaultFS", "hdfs://namenode:8020"); + properties.remove("fs.hdfs.impl"); + // Hadoop normally provides a local implementation for file; it is not selected by hdfs. + properties.put("fs.file.impl", "org.apache.hadoop.fs.LocalFileSystem"); + sink.bindDataSink(); + Assertions.assertInstanceOf(LogicalTVFTableSink.class, + analyzeInsert("INSERT INTO hdfs('file_path'='/tmp/remote_', 'format'='csv'," + + "'fs.defaultFS'='hdfs://namenode:8020'," + + "'fs.file.impl'='org.apache.hadoop.fs.LocalFileSystem') SELECT 1")); + } + + @Test + void normalizedHadoopConfigIsCheckedBeforeNativeBinding() throws Exception { + String originalConfigDir = Config.hadoop_config_dir; + String originalConfigProperty = System.getProperty("doris.hadoop.config.dir"); + Path directory = Files.createTempDirectory("tvf-hadoop-admission-"); + Path resource = directory.resolve("local-filesystem.xml"); + try { + Files.write(resource, ("fs.defaultFS" + + "file:///").getBytes(StandardCharsets.UTF_8)); + Config.hadoop_config_dir = directory.toString() + "/"; + Map properties = new HashMap<>(); + properties.put("file_path", "/tmp/local_gate_"); + properties.put("format", "csv"); + properties.put("fs.hdfs.support", "true"); + properties.put("hadoop.config.resources", resource.getFileName().toString()); + TVFTableSink sink = new TVFTableSink(new PlanNodeId(0), "hdfs", properties, + Collections.emptyList()); + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, sink::bindDataSink); + Assertions.assertTrue(exception.getMessage().contains("LOCAL TVF is disabled")); + Config.enable_local_tvf = true; + sink.bindDataSink(); + } finally { + Config.hadoop_config_dir = originalConfigDir; + if (originalConfigProperty == null) { + System.clearProperty("doris.hadoop.config.dir"); + } else { + System.setProperty("doris.hadoop.config.dir", originalConfigProperty); + } + Files.deleteIfExists(resource); + Files.deleteIfExists(directory); + } + } + + @Test + void hadoopCleanupChecksSelectedFilesystemWithoutOpeningIt() { + Map properties = new HashMap<>(); + properties.put("fs.defaultFS", "hdfs://namenode:8020"); + properties.put("fs.file.impl", "org.apache.hadoop.fs.LocalFileSystem"); + Assertions.assertDoesNotThrow(() -> LocalTableValuedFunction.checkHadoopFileSystemEnabled( + "/remote/path", properties)); + Assertions.assertThrows(AnalysisException.class, + () -> LocalTableValuedFunction.checkHadoopFileSystemEnabled("file:///tmp/prefix_", properties)); + properties.remove("fs.defaultFS"); + for (String defaultFs : Arrays.asList("local", "file:///")) { + properties.put("fs.default.name", defaultFs); + Assertions.assertThrows(AnalysisException.class, + () -> LocalTableValuedFunction.checkHadoopFileSystemEnabled("/tmp/prefix_", properties)); + } + properties.clear(); + Assertions.assertThrows(AnalysisException.class, + () -> LocalTableValuedFunction.checkHadoopFileSystemEnabled("/tmp/prefix_", properties)); + properties.put("fs.defaultFS", "hdfs://namenode:8020"); + properties.put("fs.default.name", "file:///"); + Assertions.assertThrows(AnalysisException.class, + () -> LocalTableValuedFunction.checkHadoopFileSystemEnabled("/tmp/prefix_", properties)); + properties.put("fs.default.name", "hdfs://other-namenode:8020"); + Assertions.assertDoesNotThrow(() -> LocalTableValuedFunction.checkHadoopFileSystemEnabled( + "/remote/path", properties)); + properties.remove("fs.defaultFS"); + Assertions.assertDoesNotThrow(() -> LocalTableValuedFunction.checkHadoopFileSystemEnabled( + "/remote/path", properties)); + properties.clear(); + properties.put("fs.defaultFS", "viewfs://cluster"); + properties.put("fs.viewfs.impl", "org.apache.hadoop.fs.RawLocalFileSystem"); + Assertions.assertThrows(AnalysisException.class, + () -> LocalTableValuedFunction.checkHadoopFileSystemEnabled("viewfs://cluster/tmp", properties)); + properties.clear(); + properties.put("fs.defaultFS", "${fs.selected.uri}"); + properties.put("fs.selected.uri", "file:///"); + Assertions.assertThrows(AnalysisException.class, + () -> LocalTableValuedFunction.checkHadoopFileSystemEnabled("/tmp/prefix_", properties)); + properties.put("fs.selected.uri", "hdfs://namenode:8020"); + Assertions.assertThrows(AnalysisException.class, + () -> LocalTableValuedFunction.checkHadoopFileSystemEnabled("/remote/path", properties)); + Config.enable_local_tvf = true; + Assertions.assertDoesNotThrow(() -> LocalTableValuedFunction.checkHadoopFileSystemEnabled( + "/remote/path", properties)); + Config.enable_local_tvf = false; + properties.put("fs.defaultFS", "hdfs://namenode:8020"); + properties.put("fs.hdfs.impl", "${fs.selected.impl}"); + properties.put("fs.selected.impl", "org.apache.hadoop.fs.RawLocalFileSystem"); + Assertions.assertThrows(AnalysisException.class, + () -> LocalTableValuedFunction.checkHdfsSinkEnabled(properties)); + properties.put("fs.selected.impl", "org.apache.hadoop.hdfs.DistributedFileSystem"); + Assertions.assertThrows(AnalysisException.class, + () -> LocalTableValuedFunction.checkHdfsSinkEnabled(properties)); + Config.enable_local_tvf = true; + Assertions.assertDoesNotThrow(() -> LocalTableValuedFunction.checkHdfsSinkEnabled(properties)); + Config.enable_local_tvf = false; + properties.put("fs.hdfs.impl", "${fs.selected.impl}"); + properties.remove("fs.selected.impl"); + properties.put("fs.selected.impl ", "org.apache.hadoop.fs.LocalFileSystem"); + Assertions.assertThrows(AnalysisException.class, + () -> LocalTableValuedFunction.checkHdfsSinkEnabled(properties)); + properties.put("fs.selected.impl ", "org.apache.hadoop.hdfs.DistributedFileSystem"); + Assertions.assertThrows(AnalysisException.class, + () -> LocalTableValuedFunction.checkHdfsSinkEnabled(properties)); + Config.enable_local_tvf = true; + Assertions.assertDoesNotThrow(() -> LocalTableValuedFunction.checkHdfsSinkEnabled(properties)); + Config.enable_local_tvf = false; + // BE consumes canonical defaultFS as its NameNode; the old alias can redefine + // that key inside Hadoop. Do not infer this indirect provider from the FE's raw map. + properties.put("fs.default.name", "org.apache.hadoop.fs.LocalFileSystem"); + properties.put("fs.hdfs.impl", "${fs.defaultFS}"); + Assertions.assertThrows(AnalysisException.class, + () -> LocalTableValuedFunction.checkHdfsSinkEnabled(properties)); + properties.put("fs.hdfs.impl", "${env:TVF_TEST_UNRESOLVED_IMPL}"); + Assertions.assertThrows(AnalysisException.class, + () -> LocalTableValuedFunction.checkHdfsSinkEnabled(properties)); + Config.enable_local_tvf = true; + Assertions.assertDoesNotThrow(() -> LocalTableValuedFunction.checkHdfsSinkEnabled(properties)); + Config.enable_local_tvf = false; + properties.clear(); + properties.put("fs.defaultFS", "hdfs://namenode:8020"); + properties.put("fs.hdfs.impl\0ignored", "org.apache.hadoop.fs.LocalFileSystem\0ignored"); + Assertions.assertThrows(AnalysisException.class, + () -> LocalTableValuedFunction.checkHdfsSinkEnabled(properties)); + } + + @Test + void cleanupEntryRejectsHadoopLocalStorageBeforeFilesystemCreation() throws Exception { + InsertIntoTVFCommand command = (InsertIntoTVFCommand) new NereidsParser().parseSingle( + "INSERT INTO s3('file_path'='/tmp/local_gate_', 'format'='csv') SELECT 1"); + Method cleanup = InsertIntoTVFCommand.class.getDeclaredMethod( + "deleteExistingFilesInFE", String.class, Map.class); + cleanup.setAccessible(true); + Map properties = new HashMap<>(); + properties.put("file_path", "/tmp/local_gate_"); + properties.put("format", "csv"); + properties.put("delete_existing_files", "true"); + properties.put("fs.defaultFS", "file:///"); + try (MockedStatic factory = Mockito.mockStatic(FileSystemFactory.class)) { + // The nominal S3 name must not hide Hadoop storage selected for FE-side cleanup. + InvocationTargetException exception = Assertions.assertThrows(InvocationTargetException.class, + () -> cleanup.invoke(command, "s3", properties)); + Assertions.assertInstanceOf(AnalysisException.class, exception.getCause()); + Assertions.assertTrue(exception.getCause().getMessage().contains("LOCAL TVF is disabled")); + factory.verifyNoInteractions(); + } + } + + @Test + void loadUserCannotReachHadoopLocalCleanupOrExecution() { + ConnectContext context = MemoTestUtils.createConnectContext(); + String sql = "INSERT INTO hdfs('file_path'='/tmp/local_gate_', 'format'='csv'," + + "'delete_existing_files'='true', 'fs.defaultFS'='file:///') SELECT 1"; + MemoTestUtils.createStatementContext(context, sql); + InsertIntoTVFCommand command = (InsertIntoTVFCommand) new NereidsParser().parseSingle(sql); + Env env = Mockito.spy(Env.getCurrentEnv()); + AccessControllerManager accessManager = Mockito.mock(AccessControllerManager.class); + Mockito.doReturn(accessManager).when(env).getAccessManager(); + Mockito.when(accessManager.checkGlobalPriv(context, PrivPredicate.LOAD)).thenReturn(true); + StmtExecutor executor = Mockito.mock(StmtExecutor.class); + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class, Mockito.CALLS_REAL_METHODS); + MockedStatic factory = Mockito.mockStatic(FileSystemFactory.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> command.run(context, executor)); + Assertions.assertTrue(exception.getMessage().contains("LOCAL TVF is disabled")); + Mockito.verify(accessManager).checkGlobalPriv(context, PrivPredicate.ADMIN); + Mockito.verify(accessManager).checkGlobalPriv(context, PrivPredicate.LOAD); + Mockito.verifyNoInteractions(executor); + factory.verifyNoInteractions(); + } + } + private Plan analyzeInsert(String sql) { ConnectContext context = MemoTestUtils.createConnectContext(); LogicalPlan parsed = new NereidsParser().parseSingle(sql); From 7d221b69466d9b6496c4b52e17401d73def22508 Mon Sep 17 00:00:00 2001 From: Dylan Date: Sat, 24 Oct 2026 17:51:36 +0800 Subject: [PATCH 11/13] [fix](tvf) Confine Lance children and delegated local storage ### What problem does this PR solve? Issue Number: N/A Related PR: #69528 Problem Summary: ViewFS can delegate a remote-named native sink or FE cleanup to local storage. Check selected mount targets at the actual generated-file/cleanup directory; require the existing LOCAL opt-in for runtime or ambiguous delegation. Preserve unambiguous remote mounts and unused local mounts. A real in-root Lance directory can have outward manifest or fragment symlinks. Validate reachable LOCAL/version-zero descendants before schema/cache/scan opens. Preserve in-root links and SDK-supported remote URI path spaces while retaining strict remote authority checks. ### Release note LOCAL admission covers delegated ViewFS writes and cleanup. LOCAL Lance descendant symlinks must stay inside the secure root; valid remote Lance path spaces are preserved. ### Check List (For Author) - Test: 51 selected FE tests and 17 BE filesystem/Lance tests passed through standard wrappers with fresh XML, no failures/errors/skips and unchanged sources. Java Checkstyle, changed-line clang-format16 where applicable and git diff --check passed. ViewFS/cleanup tests use mocks; Lance tests use only owned fixtures. - Behavior changed: Yes, complete the original LOCAL admission/containment requirements while preserving valid remote routing and in-root paths. - Does this need documentation: The PR describes the existing startup opt-in and the effective/indirect filesystem boundary. --- be/src/format_v2/table/lance_reader.cpp | 58 ++++- be/test/format_v2/table/lance_reader_test.cpp | 138 +++++++++++- .../commands/insert/InsertIntoTVFCommand.java | 2 +- .../apache/doris/planner/TVFTableSink.java | 2 +- .../LocalTableValuedFunction.java | 206 +++++++++++++++++- .../tablefunction/S3TableValuedFunction.java | 30 ++- .../tvf/source/TVFScanNodeTest.java | 26 +++ .../rules/analysis/BindTVFTableSinkTest.java | 184 ++++++++++++++++ .../S3TableValuedFunctionTest.java | 27 ++- 9 files changed, 654 insertions(+), 19 deletions(-) diff --git a/be/src/format_v2/table/lance_reader.cpp b/be/src/format_v2/table/lance_reader.cpp index 808d23ce2cc911..a9aedd6c8edeaf 100644 --- a/be/src/format_v2/table/lance_reader.cpp +++ b/be/src/format_v2/table/lance_reader.cpp @@ -30,8 +30,10 @@ #include #include #include +#include #include #include +#include #include "common/config.h" #include "common/logging.h" @@ -73,6 +75,57 @@ std::string expand_home_path(const std::string& path) { return path == "~" ? home : (io::Path(home) / path.substr(2)).string(); } +Status validate_local_dataset_files(const std::string& dataset_uri) { + // Lance opens manifests and fragments itself, so a confined dataset directory is not + // enough: its descendants can redirect those opens through symlinks. + std::error_code error; + const auto secure_root = std::filesystem::canonical(config::user_files_secure_path, error); + if (error) { + return Status::InvalidArgument("local Lance dataset path is not allowed"); + } + std::vector pending {dataset_uri}; + std::unordered_set visited; + while (!pending.empty()) { + auto directory = std::move(pending.back()); + pending.pop_back(); + // Canonical directory identity bounds cycles and repeated in-root aliases without + // imposing a file-count or nesting limit on otherwise valid datasets. + if (!visited.insert(directory.string()).second) { + continue; + } + std::filesystem::directory_iterator entries(directory, error); + if (error) { + return Status::InvalidArgument("local Lance dataset path is not allowed"); + } + const std::filesystem::directory_iterator end; + while (entries != end) { + auto target = entries->path(); + auto status = entries->symlink_status(error); + if (error) { + return Status::InvalidArgument("local Lance dataset path is not allowed"); + } + if (std::filesystem::is_symlink(status)) { + target = std::filesystem::canonical(target, error); + if (error || !io::LocalFileSystem::contain_path(secure_root, target)) { + return Status::InvalidArgument("local Lance dataset path is not allowed"); + } + status = std::filesystem::status(target, error); + if (error) { + return Status::InvalidArgument("local Lance dataset path is not allowed"); + } + } + if (std::filesystem::is_directory(status)) { + pending.push_back(std::move(target)); + } + entries.increment(error); + if (error) { + return Status::InvalidArgument("local Lance dataset path is not allowed"); + } + } + } + return Status::OK(); +} + Status resolve_dataset_uri(const TFileRangeDesc& range, const TFileScanRangeParams& scan_params, std::string* uri) { const auto& requested_uri = range.table_format_params.lance_params.dataset_uri; @@ -104,8 +157,9 @@ Status resolve_dataset_uri(const TFileRangeDesc& range, const TFileScanRangePara .lexically_normal() .string(); } - return io::global_local_filesystem()->canonicalize_local_file( - config::user_files_secure_path, local_input, uri); + RETURN_IF_ERROR(io::global_local_filesystem()->canonicalize_local_file( + config::user_files_secure_path, local_input, uri)); + return validate_local_dataset_files(*uri); } *uri = requested_uri; return Status::OK(); diff --git a/be/test/format_v2/table/lance_reader_test.cpp b/be/test/format_v2/table/lance_reader_test.cpp index 5cad7e3202f697..6460df54bda052 100644 --- a/be/test/format_v2/table/lance_reader_test.cpp +++ b/be/test/format_v2/table/lance_reader_test.cpp @@ -4635,9 +4635,6 @@ TEST(LanceTableReaderLocalPathTest, HomeAliasesPreserveDatasetIdentityAndContain std::vector names; std::vector types; EXPECT_FALSE(reader.fetch_schema(range, scan_params, &names, &types).ok()); - config::user_files_secure_path = home_path.string(); - ASSERT_TRUE(reader._dataset_key(range, &key).ok()); - EXPECT_EQ(home_path.string(), key.uri); // A real home dataset must still be rejected when expansion leaves the configured root. config::user_files_secure_path = (root / "~").string(); range.table_format_params.lance_params.__set_dataset_uri("~/" + name + "/data.lance"); @@ -4645,6 +4642,141 @@ TEST(LanceTableReaderLocalPathTest, HomeAliasesPreserveDatasetIdentityAndContain EXPECT_FALSE(reader.fetch_schema(range, scan_params, &names, &types).ok()); } +TEST(LanceTableReaderLocalPathTest, RejectsOutwardDatasetChildrenBeforeSchemaAndScan) { + const auto original_secure_path = config::user_files_secure_path; + DEFER(config::user_files_secure_path = original_secure_path); + const auto base = std::filesystem::temp_directory_path() / + ("lance_child_containment_" + + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count())); + ASSERT_TRUE(std::filesystem::create_directory(base)); + DEFER({ + std::error_code error; + std::filesystem::remove_all(base, error); + }); + const auto secure = base / "allowed"; + const auto outside = base / "private.lance"; + const auto fixture = + std::filesystem::canonical("./be/test/format_v2/table/lance/data/all_types.lance"); + std::filesystem::create_directory(secure); + std::filesystem::copy(fixture, outside, std::filesystem::copy_options::recursive); + config::user_files_secure_path = secure.string(); + for (const auto& mode : + {"directories", "data_directory", "manifest_file", "fragment_file", "deep", "dangling"}) { + SCOPED_TRACE(mode); + const auto dataset = secure / (std::string(mode) + ".lance"); + std::filesystem::copy(fixture, dataset, std::filesystem::copy_options::recursive); + if (std::string(mode) == "directories" || std::string(mode) == "data_directory") { + std::filesystem::remove_all(dataset / "data"); + std::filesystem::create_directory_symlink(outside / "data", dataset / "data"); + if (std::string(mode) == "directories") { + std::filesystem::remove_all(dataset / "_versions"); + std::filesystem::create_directory_symlink(outside / "_versions", + dataset / "_versions"); + } + } else if (std::string(mode) == "manifest_file" || std::string(mode) == "fragment_file") { + const auto directory = std::string(mode) == "manifest_file" ? "_versions" : "data"; + std::vector files; + for (const auto& entry : std::filesystem::directory_iterator(dataset / directory)) { + const auto extension = + std::string(mode) == "manifest_file" ? ".manifest" : ".lance"; + if (entry.is_regular_file() && entry.path().extension() == extension) { + files.push_back(entry.path()); + } + } + ASSERT_FALSE(files.empty()); + // This fixture uses v2 reversed manifest names; the first is the latest snapshot. + std::sort(files.begin(), files.end()); + const auto file = files.front(); + std::filesystem::remove(file); + std::filesystem::create_symlink(outside / directory / file.filename(), file); + } else if (std::string(mode) == "deep") { + std::filesystem::create_directories(dataset / "data/nested"); + std::filesystem::create_directory_symlink(outside / "data", + dataset / "data/nested/external"); + } else { + std::filesystem::remove_all(dataset / "_versions"); + std::filesystem::create_directory_symlink(base / "missing", dataset / "_versions"); + } + TQueryGlobals globals; + RuntimeState state(globals); + RuntimeProfile profile("lance_child_containment"); + TFileScanRangeParams scan_params; + LanceTableReader reader; + const Columns columns {projected_column("row_id", TYPE_BIGINT, false)}; + ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, &scan_params).ok()); + auto range = make_latest_lance_range(dataset); + range.__set_file_type(TFileType::FILE_LOCAL); + std::vector names; + std::vector types; + const auto schema = reader.fetch_schema(range, scan_params, &names, &types); + EXPECT_FALSE(schema.ok()); + EXPECT_EQ(std::string::npos, schema.to_string().find(outside.string())); + LanceTableReader::DatasetKey key; + const auto scan = reader._dataset_key(range, &key); + EXPECT_FALSE(scan.ok()); + EXPECT_EQ(schema.to_string(), scan.to_string()); + EXPECT_FALSE(prepare_range(&reader, range).ok()); + ASSERT_TRUE(reader.close().ok()); + } +} + +TEST(LanceTableReaderLocalPathTest, ReadsInRootDatasetChildrenAndTerminatesAliasCycles) { + const auto original_secure_path = config::user_files_secure_path; + DEFER(config::user_files_secure_path = original_secure_path); + const auto secure = + std::filesystem::temp_directory_path() / + ("lance_internal_children_" + + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count())); + ASSERT_TRUE(std::filesystem::create_directory(secure)); + DEFER({ + std::error_code error; + std::filesystem::remove_all(secure, error); + }); + const auto fixture = + std::filesystem::canonical("./be/test/format_v2/table/lance/data/all_types.lance"); + const auto real = secure / "real.lance"; + const auto alias = secure / "alias.lance"; + std::filesystem::copy(fixture, real, std::filesystem::copy_options::recursive); + std::filesystem::create_directory(alias); + std::filesystem::create_directory_symlink(real / "_versions", alias / "_versions"); + std::filesystem::create_directory(alias / "data"); + for (const auto& entry : std::filesystem::directory_iterator(real / "data")) { + std::filesystem::create_symlink(entry.path(), alias / "data" / entry.path().filename()); + } + // A visited canonical directory is checked once even when an in-root alias points back to it. + std::filesystem::create_directory_symlink(alias, alias / "cycle"); + config::user_files_secure_path = secure.string(); + TQueryGlobals globals; + RuntimeState state(globals); + RuntimeProfile profile("lance_internal_children"); + TFileScanRangeParams scan_params; + LanceTableReader reader; + const Columns columns {projected_column("row_id", TYPE_BIGINT, false)}; + ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, &scan_params).ok()); + auto range = make_latest_lance_range(alias); + range.__set_file_type(TFileType::FILE_LOCAL); + std::vector names; + std::vector types; + ASSERT_TRUE(reader.fetch_schema(range, scan_params, &names, &types).ok()); + EXPECT_NE(names.end(), std::find(names.begin(), names.end(), "row_id")); + ASSERT_TRUE(prepare_range(&reader, range).ok()); + Block block; + add_output_columns(&block, columns); + std::vector row_ids; + bool eos = false; + while (!eos) { + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + if (!eos) { + const auto& values = + assert_cast(*block.get_by_position(0).column).get_data(); + row_ids.insert(row_ids.end(), values.begin(), values.end()); + } + } + std::sort(row_ids.begin(), row_ids.end()); + EXPECT_EQ((std::vector {1, 2, 3, 4}), row_ids); + ASSERT_TRUE(reader.close().ok()); +} + TEST(LanceTableReaderLocalPathTest, FixedVersionLocalCatalogKeepsItsUri) { TFileScanRangeParams scan_params; LanceTableReader reader; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTVFCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTVFCommand.java index fa7593427d9c9a..a1114e0e814ac2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTVFCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTVFCommand.java @@ -197,7 +197,7 @@ private void deleteExistingFilesInFE(String tvfName, Map props) StorageProperties storageProps = StorageProperties.createPrimary(fsCopyProps); if (storageProps instanceof HdfsCompatibleProperties) { LocalTableValuedFunction.checkHadoopFileSystemEnabled( - filePath, storageProps.getBackendConfigProperties()); + parentDir, storageProps.getBackendConfigProperties()); } RemoteFileSystem fs = FileSystemFactory.get(storageProps); org.apache.doris.backup.Status deleteStatus = fs.deleteDirectory(parentDir); diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/TVFTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/planner/TVFTableSink.java index 579910cb10139b..306d954f84d302 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/TVFTableSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/TVFTableSink.java @@ -131,7 +131,7 @@ public void bindDataSink() throws AnalysisException { } if (fileType == TFileType.FILE_HDFS && !properties.containsKey("writer_class")) { - LocalTableValuedFunction.checkHdfsSinkEnabled(backendConnectProps); + LocalTableValuedFunction.checkHdfsSinkEnabled(properties.get("file_path"), backendConnectProps); } String filePath = properties.get("file_path"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/LocalTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/LocalTableValuedFunction.java index c399a2c3d9096c..920720ac0b7a6a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/LocalTableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/LocalTableValuedFunction.java @@ -41,7 +41,13 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.net.URI; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; @@ -165,6 +171,10 @@ private static boolean isLocalWriter(String writer) { /** Check the NameNode and provider used by the native FILE_HDFS writer. */ public static void checkHdfsSinkEnabled(Map properties) { + checkHdfsSinkEnabled(properties.getOrDefault("file_path", ""), properties); + } + + public static void checkHdfsSinkEnabled(String outputPrefix, Map properties) { String nameNode = nativeStringPrefix(properties.getOrDefault("fs.defaultFS", "")).trim(); if ("default".equals(nameNode)) { // libhdfs delegates this selector to the BE's defaults, which may select local storage. @@ -172,6 +182,29 @@ public static void checkHdfsSinkEnabled(Map properties) { } // libhdfs prefixes bare NameNodes with hdfs:// rather than using the output path's scheme. checkHadoopSchemeEnabled(nameNode.contains("://") ? uriScheme(nameNode) : "hdfs", properties); + if (!Config.enable_local_tvf) { + try { + URI filesystem = new URI(nameNode.contains("://") ? nameNode : "hdfs://" + nameNode); + // BE appends a generated filename directly to this prefix, then convert_path + // removes the NameNode. Resolve its parent, not an exact mount-point filename. + String generated = outputPrefix + "doris-tvf-file"; + String rawNameNode = properties.getOrDefault("fs.defaultFS", ""); + if (!rawNameNode.isEmpty() && generated.contains(rawNameNode)) { + generated = generated.substring(rawNameNode.length()); + } + if (!generated.startsWith("/")) { + generated = "/" + generated; + } + String path = hadoopPathUri(nativeStringPrefix(generated)).getPath(); + path = path.substring(0, path.lastIndexOf('/') + 1); + if (path.length() > 1) { + path = path.substring(0, path.length() - 1); + } + checkViewFsEnabled(filesystem, path, properties, new HashSet<>()); + } catch (URISyntaxException e) { + // Invalid NameNodes remain subject to the existing storage validation. + } + } } /** Check Hadoop's path/default-filesystem selection before FE-side cleanup. */ @@ -179,6 +212,7 @@ public static void checkHadoopFileSystemEnabled(String path, Map String scheme = uriScheme(path); if (!scheme.isEmpty()) { checkHadoopSchemeEnabled(scheme, properties); + checkViewFsPathEnabled(path, path, properties); return; } String defaultFs = hadoopSelectorProperty("fs.defaultFS", properties); @@ -186,21 +220,33 @@ public static void checkHadoopFileSystemEnabled(String path, Map // Configuration.set synchronizes aliases; either value can win when the map is applied. // Do not assume that the canonical key overrides a local legacy selector. if (defaultFs != null) { - checkHadoopDefaultEnabled(defaultFs, properties); + checkHadoopDefaultEnabled(defaultFs, path, properties); } if (legacyDefaultFs != null) { - checkHadoopDefaultEnabled(legacyDefaultFs, properties); + checkHadoopDefaultEnabled(legacyDefaultFs, path, properties); } if (defaultFs == null && legacyDefaultFs == null) { checkHadoopSchemeEnabled("file", properties); } } - private static void checkHadoopDefaultEnabled(String value, Map properties) { + private static void checkHadoopDefaultEnabled(String value, String path, Map properties) { value = hadoopSelectorValue(value).trim(); // Hadoop retains the old "local" default-FS spelling. String scheme = "local".equals(value) ? "file" : uriScheme(value); checkHadoopSchemeEnabled(scheme.isEmpty() ? "hdfs" : scheme, properties); + checkViewFsPathEnabled(scheme.isEmpty() ? "hdfs://" + value : value, path, properties); + } + + private static void checkViewFsPathEnabled(String location, String path, Map properties) { + if (!Config.enable_local_tvf) { + try { + URI filesystem = uriScheme(path).isEmpty() ? new URI(location) : hadoopPathUri(path); + checkViewFsEnabled(filesystem, hadoopPathUri(path).getPath(), properties, new HashSet<>()); + } catch (URISyntaxException e) { + checkEnabled(); + } + } } private static void checkHadoopSchemeEnabled(String scheme, Map properties) { @@ -220,6 +266,160 @@ private static void checkHadoopSchemeEnabled(String scheme, Map } } + private static void checkViewFsEnabled(URI filesystem, String path, Map properties, + Set activeTables) { + if (Config.enable_local_tvf) { + return; + } + String scheme = filesystem.getScheme(); + checkHadoopSchemeEnabled(scheme, properties); + String implementation = hadoopSelectorProperty("fs." + scheme + ".impl", properties); + if (implementation != null + && "org.apache.hadoop.fs.viewfs.ViewFileSystemOverloadScheme".equals(implementation.trim())) { + // OverloadScheme can reload runtime mount tables and overrides provider selection. + checkEnabled(); + } + boolean viewFs = isViewFs(scheme, implementation); + if (!viewFs) { + return; + } + String table = filesystem.getAuthority(); + String ignorePort = hadoopSelectorProperty("fs.viewfs.ignore.port.in.mount.table.name", properties); + if (ignorePort != null && Boolean.parseBoolean(hadoopSelectorValue(ignorePort).trim()) + && table != null && filesystem.getPort() != -1) { + table = filesystem.getHost(); + if (table == null) { + checkEnabled(); + } + } + if (table == null || table.isEmpty()) { + table = hadoopSelectorProperty("fs.viewfs.mounttable.default.name.key", properties); + table = table == null ? "default" : hadoopSelectorValue(table); + } + // Nested ViewFS cycles or arbitrarily deep delegation cannot establish remote storage. + if (activeTables.size() >= 32 || !activeTables.add(table)) { + checkEnabled(); + } + try { + String prefix = "fs.viewfs.mounttable." + table + "."; + String source = ""; + List selectedKeys = new ArrayList<>(); + List mountSources = new ArrayList<>(); + String fallback = null; + String rootMerge = null; + for (String rawKey : properties.keySet()) { + String key = nativeStringPrefix(rawKey).trim(); + if (!key.startsWith(prefix)) { + continue; + } + String entry = key.substring(prefix.length()); + if (entry.equals("linkFallback")) { + fallback = key; + continue; + } + if (entry.equals("linkMergeSlash")) { + rootMerge = key; + continue; + } + if (entry.startsWith("linkRegex")) { + // Regex interception rewrites runtime paths before ordinary mount resolution. + checkEnabled(); + } + int slash = entry.indexOf('/'); + if (slash < 0 || !(entry.startsWith("link.") || entry.startsWith("linkMerge.") + || entry.startsWith("linkNfly."))) { + continue; + } + String candidate = entry.substring(slash); + // InodeTree splits source components and discards trailing empty components. + while (candidate.length() > 1 && candidate.endsWith("/")) { + candidate = candidate.substring(0, candidate.length() - 1); + } + mountSources.add(candidate); + if (!(path.equals(candidate) || path.startsWith(candidate + "/"))) { + continue; + } + if (candidate.length() > source.length()) { + source = candidate; + selectedKeys.clear(); + } + if (candidate.equals(source)) { + selectedKeys.add(key); + } + } + if (rootMerge != null) { + selectedKeys.add(rootMerge); + source = ""; + } else if (selectedKeys.isEmpty() && fallback != null) { + selectedKeys.add(fallback); + } + if (selectedKeys.isEmpty()) { + // No explicit target can prove what a BE-inherited mount table will select. + checkEnabled(); + } + for (String key : selectedKeys) { + String targets = hadoopSelectorValue(hadoopSelectorProperty(key, properties)); + for (String target : targets.split(",")) { + if (target.isEmpty()) { + continue; + } + URI uri = new URI(target); + if (uri.getScheme() == null) { + checkEnabled(); + } + String targetImplementation = hadoopSelectorProperty("fs." + uri.getScheme() + ".impl", + properties); + if (isViewFs(uri.getScheme(), targetImplementation)) { + for (String nested : mountSources) { + if (!nested.startsWith(source + "/")) { + continue; + } + int branchEnd = nested.indexOf('/', source.length() + 1); + String branch = branchEnd < 0 ? nested : nested.substring(0, branchEnd); + if (path.equals(branch) || path.startsWith(branch + "/")) { + // A missing child below an internal nested-mount directory uses + // a different suffix in InodeTree. Do not guess its delegation. + checkEnabled(); + } + } + } + String base = uri.getPath() == null ? "/" : uri.getPath(); + String remaining = path.substring(source.length()); + String nextPath = new URI(null, null, + (base.endsWith("/") ? base : base + "/") + + (remaining.startsWith("/") ? remaining.substring(1) : remaining), null) + .normalize().getPath(); + checkViewFsEnabled(uri, nextPath, properties, activeTables); + } + } + } catch (URISyntaxException e) { + checkEnabled(); + } finally { + activeTables.remove(table); + } + } + + private static boolean isViewFs(String scheme, String implementation) { + return implementation == null ? "viewfs".equals(scheme) + : "org.apache.hadoop.fs.viewfs.ViewFileSystem".equals(implementation.trim()); + } + + private static URI hadoopPathUri(String path) throws URISyntaxException { + String scheme = uriScheme(path); + String authority = null; + String relative = scheme.isEmpty() ? path : path.substring(scheme.length() + 1); + if (relative.startsWith("//")) { + int slash = relative.indexOf('/', 2); + authority = slash < 0 ? relative.substring(2) : relative.substring(2, slash); + relative = slash < 0 ? "/" : relative.substring(slash); + } + if (!relative.startsWith("/")) { + relative = "/" + relative; + } + // Hadoop Path(String) quotes path bytes before URI normalization; percent text is literal. + return new URI(scheme.isEmpty() ? null : scheme, authority, relative, null, null).normalize(); + } + private static String hadoopSelectorProperty(String key, Map properties) { String value = null; for (Map.Entry entry : properties.entrySet()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java index 183e7304b6add2..44517f0fb4c2b7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java @@ -109,17 +109,31 @@ public S3TableValuedFunction(Map properties, List= 0) { + authorityEnd = Math.min(authorityEnd, end); + } + } + // Validate only scheme/authority against SDK-local fallback. Rust's URL parser + // accepts and encodes raw spaces in paths; preserve that spelling for the SDK. + URI uri = new URI(path.substring(0, authorityEnd) + "/"); + String authority = uri.getRawAuthority(); + if (authority == null || authority.isEmpty()) { + return "goosefs".equalsIgnoreCase(scheme); } // Lance falls back to a local path if Rust's URL parser rejects the URI. Java's URI // accepts registry authorities with invalid ports, which Rust rejects. Validate the diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/tvf/source/TVFScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/tvf/source/TVFScanNodeTest.java index aa3f1e726123ab..430882e7ce72b1 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/tvf/source/TVFScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/tvf/source/TVFScanNodeTest.java @@ -192,6 +192,32 @@ public void testLanceSplitsPinVersionAndFragments() throws Exception { range.getTableFormatParams().getLanceParams().getFragmentIds()); } + @Test + public void testRemoteLanceSpacePathsPreserveScanRangeIdentity() throws Exception { + for (String uri : Arrays.asList("s3://bucket/sample data.lance", "s3://bucket/sample%20data.lance")) { + SessionVariable sv = new SessionVariable(); + TupleDescriptor desc = new TupleDescriptor(new TupleId(0)); + FunctionGenTable table = Mockito.mock(FunctionGenTable.class); + ExternalFileTableValuedFunction tvf = Mockito.mock(ExternalFileTableValuedFunction.class); + Mockito.when(table.getTvf()).thenReturn(tvf); + Mockito.when(tvf.isLanceFormat()).thenReturn(true); + Mockito.when(tvf.getTFileType()).thenReturn(TFileType.FILE_S3); + Mockito.when(tvf.getFilePath()).thenReturn(uri); + Mockito.when(tvf.getLanceDatasetVersion()).thenReturn(42L); + Mockito.when(tvf.getLanceFragments()).thenReturn( + Collections.singletonList(new LanceFragmentInfo(7, 1000, 1000))); + desc.setTable(table); + TVFScanNode node = new TVFScanNode(new PlanNodeId(0), desc, false, sv, ScanContext.EMPTY); + LanceSplit split = (LanceSplit) node.getSplits(1).get(0); + Assert.assertEquals(uri, split.getDatasetUri()); + TFileRangeDesc range = new TFileRangeDesc(); + node.setScanParams(range, split); + Assert.assertEquals(TFileType.FILE_S3, range.getFileType()); + Assert.assertEquals(uri, range.getTableFormatParams().getLanceParams().getDatasetUri()); + Assert.assertEquals(42L, range.getTableFormatParams().getLanceParams().getVersion()); + } + } + @Test public void testLocalLanceUsesOneLatestWholeDatasetSplit() throws Exception { SessionVariable sv = new SessionVariable(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindTVFTableSinkTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindTVFTableSinkTest.java index bf2af2d875e8e8..a47833c1364290 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindTVFTableSinkTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindTVFTableSinkTest.java @@ -404,6 +404,190 @@ void loadUserCannotReachHadoopLocalCleanupOrExecution() { } } + @Test + void disabledViewFsLocalMountsRejectInsertAndExplain() { + for (String prefix : Arrays.asList("", "EXPLAIN ")) { + for (String cleanup : Arrays.asList("false", "true")) { + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> analyzeInsert(prefix + "INSERT INTO hdfs('file_path'='/local/sub/prefix_'," + + "'format'='csv','delete_existing_files'='" + cleanup + "'," + + "'fs.defaultFS'='viewfs://cluster'," + + "'fs.viewfs.mounttable.cluster.link./local'='file:///tmp') SELECT 1")); + Assertions.assertTrue(exception.getMessage().contains("LOCAL TVF is disabled")); + } + } + } + + @Test + void viewFsChecksSelectedTargetsAndPreservesRemoteMounts() throws Exception { + Map properties = new HashMap<>(); + properties.put("file_path", "/remote/prefix_"); + properties.put("format", "csv"); + properties.put("fs.defaultFS", "viewfs://cluster"); + properties.put("fs.viewfs.mounttable.cluster.link./remote", "hdfs://namenode:8020/base"); + properties.put("fs.viewfs.mounttable.cluster.link./local", "file:///tmp"); + properties.put("fs.viewfs.mounttable.cluster.linkFallback", "hdfs://namenode:8020/fallback"); + new TVFTableSink(new PlanNodeId(0), "hdfs", properties, Collections.emptyList()).bindDataSink(); + Assertions.assertDoesNotThrow(() -> LocalTableValuedFunction.checkHdfsSinkEnabled( + "/locality/prefix_", properties)); + // A prefix naming a mount point generates a sibling filename; a trailing slash writes inside it. + Assertions.assertDoesNotThrow(() -> LocalTableValuedFunction.checkHdfsSinkEnabled("/local", properties)); + Assertions.assertThrows(AnalysisException.class, + () -> LocalTableValuedFunction.checkHdfsSinkEnabled("/local/", properties)); + properties.put("file_path", "/local/sub/prefix_"); + TVFTableSink sink = new TVFTableSink(new PlanNodeId(0), "hdfs", properties, Collections.emptyList()); + Assertions.assertThrows(AnalysisException.class, sink::bindDataSink); + Config.enable_local_tvf = true; + sink.bindDataSink(); + Config.enable_local_tvf = false; + properties.remove("fs.viewfs.mounttable.cluster.link./local"); + properties.put("fs.viewfs.mounttable.cluster.link./local/", "file:///tmp"); + Assertions.assertThrows(AnalysisException.class, + () -> LocalTableValuedFunction.checkHdfsSinkEnabled("/local/sub/prefix_", properties)); + properties.remove("fs.viewfs.mounttable.cluster.link./local/"); + properties.put("fs.viewfs.mounttable.cluster.link./local", "file:///tmp"); + properties.put("fs.viewfs.mounttable.cluster.link./local/remote", "hdfs://namenode:8020/base"); + Assertions.assertDoesNotThrow(() -> LocalTableValuedFunction.checkHdfsSinkEnabled( + "/local/remote/prefix_", properties)); + properties.put("fs.defaultFS", "viewfs://cl%75ster"); + Assertions.assertThrows(AnalysisException.class, + () -> LocalTableValuedFunction.checkHdfsSinkEnabled("/local/sub/prefix_", properties)); + properties.put("fs.defaultFS", "viewfs://cluster_name"); + properties.put("fs.viewfs.ignore.port.in.mount.table.name", "true"); + properties.put("fs.viewfs.mounttable.cluster_name.link./remote", "hdfs://namenode:8020/base"); + Assertions.assertDoesNotThrow(() -> LocalTableValuedFunction.checkHdfsSinkEnabled( + "/remote/prefix_", properties)); + properties.put("fs.defaultFS", "viewfs://cluster:8020"); + properties.put("fs.viewfs.ignore.port.in.mount.table.name", "true"); + Assertions.assertThrows(AnalysisException.class, + () -> LocalTableValuedFunction.checkHdfsSinkEnabled("/local/sub/prefix_", properties)); + properties.clear(); + properties.put("fs.defaultFS", "viewfs:///"); + properties.put("fs.viewfs.mounttable.default.name.key", "named"); + properties.put("fs.viewfs.mounttable.named.link./local", "file:///tmp"); + Assertions.assertThrows(AnalysisException.class, + () -> LocalTableValuedFunction.checkHdfsSinkEnabled("/local/sub/prefix_", properties)); + properties.clear(); + properties.put("fs.defaultFS", "viewfs://cluster"); + properties.put("fs.viewfs.mounttable.cluster.linkFallback", "file:///tmp"); + Assertions.assertThrows(AnalysisException.class, + () -> LocalTableValuedFunction.checkHdfsSinkEnabled("/unmapped/prefix_", properties)); + properties.remove("fs.viewfs.mounttable.cluster.linkFallback"); + for (String key : Arrays.asList("linkMerge./local", + "linkNfly.minReplication=2,repairOnRead=true./local", "linkMergeSlash")) { + String mountKey = "fs.viewfs.mounttable.cluster." + key; + properties.put(mountKey, "hdfs://namenode:8020/base,file:///tmp"); + Assertions.assertThrows(AnalysisException.class, + () -> LocalTableValuedFunction.checkHdfsSinkEnabled("/local/sub/prefix_", properties)); + properties.put(mountKey, "hdfs://namenode:8020/base,hdfs://other:8020/base"); + Assertions.assertDoesNotThrow(() -> LocalTableValuedFunction.checkHdfsSinkEnabled( + "/local/sub/prefix_", properties)); + properties.remove(mountKey); + } + properties.put("fs.viewfs.mounttable.cluster.link./local", "viewfs://inner/root"); + properties.put("fs.viewfs.mounttable.inner.link./root", "file:///tmp"); + Assertions.assertThrows(AnalysisException.class, + () -> LocalTableValuedFunction.checkHdfsSinkEnabled("/local/sub/prefix_", properties)); + properties.put("fs.viewfs.mounttable.inner.link./root", "hdfs://namenode:8020/base"); + Assertions.assertDoesNotThrow(() -> LocalTableValuedFunction.checkHdfsSinkEnabled( + "/local/sub/prefix_", properties)); + properties.put("fs.viewfs.mounttable.inner.link./root", "viewfs://cluster/local"); + Assertions.assertThrows(AnalysisException.class, + () -> LocalTableValuedFunction.checkHdfsSinkEnabled("/local/sub/prefix_", properties)); + properties.clear(); + properties.put("fs.defaultFS", "viewfs://A"); + properties.put("fs.viewfs.mounttable.A.link./a", "viewfs://B/base"); + properties.put("fs.viewfs.mounttable.A.link./a/b/c", "hdfs://namenode:8020/remote"); + properties.put("fs.viewfs.mounttable.B.link./base/d", "file:///tmp"); + properties.put("fs.viewfs.mounttable.B.link./base/b/d", "hdfs://namenode:8020/remote"); + properties.put("fs.viewfs.mounttable.B.link./base/z", "hdfs://namenode:8020/remote"); + Assertions.assertDoesNotThrow(() -> LocalTableValuedFunction.checkHdfsSinkEnabled( + "/a/z/sub/prefix_", properties)); + Assertions.assertThrows(AnalysisException.class, + () -> LocalTableValuedFunction.checkHdfsSinkEnabled("/a/b/d/sub/prefix_", properties)); + properties.clear(); + properties.put("fs.defaultFS", "hdfs://cluster:8020"); + properties.put("fs.hdfs.impl", "org.apache.hadoop.fs.viewfs.ViewFileSystemOverloadScheme"); + properties.put("fs.viewfs.mounttable.cluster:8020.link./local", "hdfs://namenode:8020/base"); + properties.put("fs.viewfs.mounttable.cluster.link./local", "file:///tmp"); + Assertions.assertThrows(AnalysisException.class, + () -> LocalTableValuedFunction.checkHdfsSinkEnabled("/local/sub/prefix_", properties)); + properties.clear(); + properties.put("fs.defaultFS", "viewfs://inherited"); + Assertions.assertThrows(AnalysisException.class, + () -> LocalTableValuedFunction.checkHdfsSinkEnabled("/unknown/prefix_", properties)); + } + + @Test + void viewFsResourcesAndCleanupRejectBeforeFilesystemCreation() throws Exception { + String originalConfigDir = Config.hadoop_config_dir; + String originalConfigProperty = System.getProperty("doris.hadoop.config.dir"); + Path directory = Files.createTempDirectory("tvf-viewfs-admission-"); + Path resource = directory.resolve("viewfs.xml"); + try { + Files.write(resource, ("fs.defaultFS" + + "viewfs://cluster" + + "fs.viewfs.mounttable.cluster.link./local" + + "file:///tmp").getBytes(StandardCharsets.UTF_8)); + Config.hadoop_config_dir = directory.toString() + "/"; + Map properties = new HashMap<>(); + properties.put("file_path", "/local/sub/prefix_"); + properties.put("format", "csv"); + properties.put("fs.hdfs.support", "true"); + properties.put("hadoop.config.resources", resource.getFileName().toString()); + TVFTableSink sink = new TVFTableSink(new PlanNodeId(0), "hdfs", properties, + Collections.emptyList()); + Assertions.assertThrows(AnalysisException.class, sink::bindDataSink); + InsertIntoTVFCommand command = (InsertIntoTVFCommand) new NereidsParser().parseSingle( + "INSERT INTO s3('file_path'='/local/sub/prefix_', 'format'='csv') SELECT 1"); + Method cleanup = InsertIntoTVFCommand.class.getDeclaredMethod( + "deleteExistingFilesInFE", String.class, Map.class); + cleanup.setAccessible(true); + properties.put("delete_existing_files", "true"); + try (MockedStatic factory = Mockito.mockStatic(FileSystemFactory.class)) { + InvocationTargetException exception = Assertions.assertThrows(InvocationTargetException.class, + () -> cleanup.invoke(command, "s3", properties)); + Assertions.assertInstanceOf(AnalysisException.class, exception.getCause()); + factory.verifyNoInteractions(); + } + } finally { + Config.hadoop_config_dir = originalConfigDir; + if (originalConfigProperty == null) { + System.clearProperty("doris.hadoop.config.dir"); + } else { + System.setProperty("doris.hadoop.config.dir", originalConfigProperty); + } + Files.deleteIfExists(resource); + Files.deleteIfExists(directory); + } + } + + @Test + void loadUserCannotExecuteViewFsLocalWritesOrCleanup() { + ConnectContext context = MemoTestUtils.createConnectContext(); + String sql = "INSERT INTO hdfs('file_path'='/local/sub/prefix_', 'format'='csv'," + + "'delete_existing_files'='true','fs.defaultFS'='viewfs://cluster'," + + "'fs.viewfs.mounttable.cluster.link./local'='file:///tmp') SELECT 1"; + MemoTestUtils.createStatementContext(context, sql); + InsertIntoTVFCommand command = (InsertIntoTVFCommand) new NereidsParser().parseSingle(sql); + Env env = Mockito.spy(Env.getCurrentEnv()); + AccessControllerManager accessManager = Mockito.mock(AccessControllerManager.class); + Mockito.doReturn(accessManager).when(env).getAccessManager(); + Mockito.when(accessManager.checkGlobalPriv(context, PrivPredicate.LOAD)).thenReturn(true); + StmtExecutor executor = Mockito.mock(StmtExecutor.class); + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class, Mockito.CALLS_REAL_METHODS); + MockedStatic factory = Mockito.mockStatic(FileSystemFactory.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> command.run(context, executor)); + Assertions.assertTrue(exception.getMessage().contains("LOCAL TVF is disabled")); + Mockito.verify(accessManager).checkGlobalPriv(context, PrivPredicate.ADMIN); + Mockito.verify(accessManager).checkGlobalPriv(context, PrivPredicate.LOAD); + Mockito.verifyNoInteractions(executor); + factory.verifyNoInteractions(); + } + } + private Plan analyzeInsert(String sql) { ConnectContext context = MemoTestUtils.createConnectContext(); LogicalPlan parsed = new NereidsParser().parseSingle(sql); diff --git a/fe/fe-core/src/test/java/org/apache/doris/tablefunction/S3TableValuedFunctionTest.java b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/S3TableValuedFunctionTest.java index 6050a4209aa8b6..8cb2249479cd88 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/tablefunction/S3TableValuedFunctionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/S3TableValuedFunctionTest.java @@ -97,7 +97,7 @@ public void testLanceRejectsBrokerLocalUrisBeforeOpeningDataset() { "s3://bucket:bad/../../dataset.lance", "s3://bucket:65536/../../dataset.lance", "s3://:9000/dataset.lance", "s3://user@/dataset.lance", "s3://[::1%25eth0]/dataset.lance", "https://bucket:bad/../../dataset.lance", - "", "s3:/bucket/dataset.lance", "s3://", "s3://bucket/bad path")) { + "", "s3:/bucket/dataset.lance", "s3://")) { AnalysisException exception = Assertions.assertThrows(AnalysisException.class, () -> new S3TableValuedFunction(ImmutableMap.of( "format", "lance", "uri", uri, "fs.broker.support", "true")), uri); @@ -164,6 +164,31 @@ public void testLanceHttpStorageUrisAreNormalizedBeforeValidation() throws Excep "s3://bucket/dataset.lance"); } + @Test + public void testLanceSpacePathsReachMetadataThroughS3AndFile() throws Exception { + FeConstants.runningUnitTest = false; + for (String uri : Arrays.asList("s3://bucket/sample data.lance", "s3://bucket/sample%20data.lance")) { + Map properties = new HashMap<>(PROPERTIES); + properties.put("format", "lance"); + properties.remove("file_path"); + properties.put("uri", uri); + assertLanceMetadataLoad(properties, uri); + try (MockedStatic loader = Mockito.mockStatic(LanceMetadataLoader.class)) { + IllegalStateException reachedLoader = new IllegalStateException("file remote metadata reached"); + loader.when(() -> LanceMetadataLoader.loadLatestForTvf(Mockito.eq(uri), Mockito.anyList())) + .thenThrow(reachedLoader); + RuntimeException exception = Assertions.assertThrows(RuntimeException.class, + () -> new FileTableValuedFunction(properties)); + Throwable cause = exception; + while (cause.getCause() != null) { + cause = cause.getCause(); + } + Assertions.assertSame(reachedLoader, cause); + loader.verify(() -> LanceMetadataLoader.loadLatestForTvf(Mockito.eq(uri), Mockito.anyList())); + } + } + } + private void assertLanceMetadataLoad(Map properties, String uri) throws Exception { try (MockedStatic loader = Mockito.mockStatic(LanceMetadataLoader.class)) { IllegalStateException reachedLoader = new IllegalStateException("remote metadata loader reached"); From 9df0508783e6b1da6be6ef9909de9ebc0db45db8 Mon Sep 17 00:00:00 2001 From: Dylan Date: Sat, 24 Oct 2026 18:14:54 +0800 Subject: [PATCH 12/13] [fix](tvf) Check the actual cleanup parent for ViewFS admission ### What problem does this PR solve? Issue Number: N/A Related PR: #69528 Problem Summary: A file prefix equal to a local ViewFS mount name creates a sibling output file and cleans its parent, which can select a remote fallback. Use the existing parent-directory helper in early cleanup admission to preserve that valid route. ### Release note Preserve remote ViewFS cleanup when only the filename prefix matches a local mount. ### Check List (For Author) - Test: 51 selected FE tests passed through the standard JDK17 wrapper with fresh XML, zero failures/errors/skips and unchanged source. Added a remote-fallback cleanup binding control. Targeted Checkstyle and git diff --check passed. - Behavior changed: Yes. Cleanup admission follows the actual parent directory while retaining the LOCAL startup opt-in. - Does this need documentation: Covered by the PR's existing effective-storage admission description. --- .../apache/doris/tablefunction/LocalTableValuedFunction.java | 4 +++- .../doris/nereids/rules/analysis/BindTVFTableSinkTest.java | 5 +++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/LocalTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/LocalTableValuedFunction.java index 920720ac0b7a6a..da83d4f7c3089b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/LocalTableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/LocalTableValuedFunction.java @@ -24,6 +24,7 @@ import org.apache.doris.common.Config; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.UserException; +import org.apache.doris.common.util.BrokerUtil; import org.apache.doris.datasource.FederationBackendPolicy; import org.apache.doris.datasource.property.storage.StorageProperties; import org.apache.doris.mysql.privilege.PrivPredicate; @@ -159,7 +160,8 @@ public static void checkSinkEnabled(String tvfName, Map properti checkHdfsSinkEnabled(properties); } if (Boolean.parseBoolean(properties.getOrDefault("delete_existing_files", "false"))) { - checkHadoopFileSystemEnabled(properties.getOrDefault("file_path", ""), properties); + checkHadoopFileSystemEnabled( + BrokerUtil.extractParentDirectory(properties.getOrDefault("file_path", "")), properties); } } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindTVFTableSinkTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindTVFTableSinkTest.java index a47833c1364290..61326cf60ec1e7 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindTVFTableSinkTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindTVFTableSinkTest.java @@ -432,6 +432,11 @@ void viewFsChecksSelectedTargetsAndPreservesRemoteMounts() throws Exception { "/locality/prefix_", properties)); // A prefix naming a mount point generates a sibling filename; a trailing slash writes inside it. Assertions.assertDoesNotThrow(() -> LocalTableValuedFunction.checkHdfsSinkEnabled("/local", properties)); + Assertions.assertInstanceOf(LogicalTVFTableSink.class, + analyzeInsert("INSERT INTO hdfs('file_path'='/local', 'format'='csv'," + + "'delete_existing_files'='true', 'fs.defaultFS'='viewfs://cluster'," + + "'fs.viewfs.mounttable.cluster.link./local'='file:///tmp'," + + "'fs.viewfs.mounttable.cluster.linkFallback'='hdfs://namenode:8020/base') SELECT 1")); Assertions.assertThrows(AnalysisException.class, () -> LocalTableValuedFunction.checkHdfsSinkEnabled("/local/", properties)); properties.put("file_path", "/local/sub/prefix_"); From 2d7318284d162ad1dc9b1912a8e1150e1f74c674 Mon Sep 17 00:00:00 2001 From: Dylan Date: Sat, 24 Oct 2026 22:19:10 +0800 Subject: [PATCH 13/13] [fix](tvf) Validate effective storage before TVF consumers ### What problem does this PR solve? Issue Number: N/A Related PR: #69528 Problem Summary: Share normalized Hadoop TVF admission across native writes, Java cleanup and reads, matching each consumer's path and property interpretation. Preserve exact canonical LOCAL discovery paths and supported remote providers. Validate primary and manifest-selected Lance storage before schema or delegated reads; isolate TVF policy from catalog/cache reuse. Add a metadata-only Lance C ABI with matching header/archive build compatibility gates. ### Release note Effective local TVF storage honors startup opt-in and secure LOCAL read containment. ### Check List (For Author) - Test: 86 FE and 22 BE tests passed through standard wrappers, with fresh XML and zero failures/errors/skips. Relevant style and contract checks passed. Full Groovy and complete community CI remain unverified locally. - Behavior changed: Yes, reject effective local/unknown routes before TVF side effects. - Does this need documentation: The PR describes startup opt-in and upgrade requirements. --- be/cmake/thirdparty.cmake | 6 + be/src/format_v2/table/lance_reader.cpp | 97 +++- be/src/format_v2/table/lance_reader.h | 6 + .../table/lance/data/tvf-storage-fixtures.md | 13 + .../_versions/18446744073709551612.manifest | Bin 0 -> 2184 bytes .../_versions/18446744073709551614.manifest | Bin 0 -> 4108 bytes .../lance/generate_storage_policy_fixtures.rs | 172 +++++++ be/test/format_v2/table/lance_reader_test.cpp | 186 +++++++ be/test/io/fs/local_file_system_test.cpp | 26 + build.sh | 43 ++ .../lance/metadata/LanceMetadataLoader.java | 1 + .../lance/metadata/LanceTvfStoragePolicy.java | 103 ++++ .../datasource/tvf/source/TVFScanNode.java | 1 + .../nereids/rules/analysis/BindSink.java | 16 + .../commands/insert/InsertIntoTVFCommand.java | 7 +- .../apache/doris/planner/TVFTableSink.java | 4 +- .../ExternalFileTableValuedFunction.java | 1 + .../tablefunction/HadoopTvfAdmission.java | 477 ++++++++++++++++++ .../HdfsTableValuedFunction.java | 6 + .../LocalTableValuedFunction.java | 299 +---------- .../tablefunction/S3TableValuedFunction.java | 68 +-- .../metadata/LanceTvfStoragePolicyTest.java | 118 +++++ .../tvf/source/TVFScanNodeTest.java | 2 + .../rules/analysis/BindTVFTableSinkTest.java | 189 +++++-- .../tablefunction/HadoopTvfAdmissionTest.java | 301 +++++++++++ .../HadoopTvfReadAdmissionTest.java | 145 ++++++ .../LocalTableValuedFunctionEnableTest.java | 21 + gensrc/thrift/PlanNodes.thrift | 4 + thirdparty/build-thirdparty.sh | 31 +- thirdparty/download-thirdparty.sh | 6 +- .../patches/lance-c-storage-bases.patch | 362 +++++++++++++ thirdparty/test/lance-build-test.py | 306 +++++++++++ thirdparty/test/lance-prefilter-patch-test.sh | 12 +- 33 files changed, 2604 insertions(+), 425 deletions(-) create mode 100644 be/test/format_v2/table/lance/data/tvf-storage-fixtures.md create mode 100644 be/test/format_v2/table/lance/data/tvf_storage_clone.lance/_versions/18446744073709551612.manifest create mode 100644 be/test/format_v2/table/lance/data/tvf_storage_direct_base.lance/_versions/18446744073709551614.manifest create mode 100644 be/test/format_v2/table/lance/generate_storage_policy_fixtures.rs create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/lance/metadata/LanceTvfStoragePolicy.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/tablefunction/HadoopTvfAdmission.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/lance/metadata/LanceTvfStoragePolicyTest.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/tablefunction/HadoopTvfAdmissionTest.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/tablefunction/HadoopTvfReadAdmissionTest.java create mode 100644 thirdparty/patches/lance-c-storage-bases.patch create mode 100644 thirdparty/test/lance-build-test.py diff --git a/be/cmake/thirdparty.cmake b/be/cmake/thirdparty.cmake index b67c8e9d0aa3c9..fd8c555d35ef9d 100644 --- a/be/cmake/thirdparty.cmake +++ b/be/cmake/thirdparty.cmake @@ -111,6 +111,12 @@ add_thirdparty(arrow_flight_sql LIB64) add_thirdparty(arrow_dataset LIB64) add_thirdparty(arrow_acero LIB64) add_thirdparty(parquet LIB64) +# Never consume a header/archive pair left by an interrupted Lance installation. +if (EXISTS "${THIRDPARTY_DIR}/lib64/.lance-installing") + message(FATAL_ERROR + "Lance SDK installation is incomplete. Retry thirdparty/build-thirdparty.sh lance_c " + "or refresh include/lance and lib64/liblance_c.a as a matching pair.") +endif () add_thirdparty(lance_c LIB64 NOTADD) # liblance_c.a contains compiler_builtins cbrt symbols. Place libm before it # so the final linker resolves C math symbols from the system library first. diff --git a/be/src/format_v2/table/lance_reader.cpp b/be/src/format_v2/table/lance_reader.cpp index a9aedd6c8edeaf..e1f95fcd647651 100644 --- a/be/src/format_v2/table/lance_reader.cpp +++ b/be/src/format_v2/table/lance_reader.cpp @@ -23,6 +23,7 @@ #include #include #include +#include #include #include @@ -45,6 +46,10 @@ #include "runtime/file_scan_profile.h" #include "runtime/runtime_state.h" +#ifndef LANCE_STORAGE_BASES_JSON_VERSION +#error "Rebuild lance-c with TVF storage-base metadata support before building Doris" +#endif + namespace doris::format::lance { namespace { @@ -126,19 +131,21 @@ Status validate_local_dataset_files(const std::string& dataset_uri) { return Status::OK(); } +bool is_local_lance_tvf(const TFileRangeDesc& range, const TFileScanRangeParams& scan_params) { + const bool local = range.__isset.file_type + ? range.file_type == TFileType::FILE_LOCAL + : scan_params.__isset.file_type && + scan_params.file_type == TFileType::FILE_LOCAL; + const auto& params = range.table_format_params.lance_params; + return local && (params.version == 0 || (params.__isset.is_tvf && params.is_tvf)); +} + Status resolve_dataset_uri(const TFileRangeDesc& range, const TFileScanRangeParams& scan_params, std::string* uri) { const auto& requested_uri = range.table_format_params.lance_params.dataset_uri; - // Modern plans tag each range; older plans put the type in the scan parameters. - // Untagged catalog scans retain their original storage-provider selection. - const bool is_local = range.__isset.file_type - ? range.file_type == TFileType::FILE_LOCAL - : scan_params.__isset.file_type && - scan_params.file_type == TFileType::FILE_LOCAL; - // LOCAL TVF uses version zero for both schema discovery and whole-dataset scans. - // Catalog scans pin a positive snapshot version, including file:// catalogs whose - // operator-configured warehouse is independent of the LOCAL TVF secure directory. - if (is_local && range.table_format_params.lance_params.version == 0) { + // Explicit TVF identity applies regardless of snapshot selection. Older LOCAL TVFs + // used version zero; configured fixed-version catalogs do not send the TVF marker. + if (is_local_lance_tvf(range, scan_params)) { // LOCAL Lance bypasses glob because it opens a dataset directory. Apply the same // canonical secure-root check before either schema discovery or a cached dataset open. std::string local_input = requested_uri; @@ -166,6 +173,67 @@ Status resolve_dataset_uri(const TFileRangeDesc& range, const TFileScanRangePara } } // namespace +LanceTableReader::TvfStoragePolicy LanceTableReader::_tvf_storage_policy( + const TFileRangeDesc& range, const TFileScanRangeParams& scan_params) { + // Positive versions are also used by S3 TVFs, so only the marker can distinguish + // those requests from configured catalogs. Legacy LOCAL TVFs keep their old restriction. + if (is_local_lance_tvf(range, scan_params)) { + return TvfStoragePolicy::LOCAL; + } + const auto& params = range.table_format_params.lance_params; + if (!params.__isset.is_tvf || !params.is_tvf) { + return TvfStoragePolicy::NONE; + } + return TvfStoragePolicy::REMOTE; +} + +Status LanceTableReader::_validate_dataset_storage(LanceDataset* dataset, TvfStoragePolicy policy) { + if (policy == TvfStoragePolicy::NONE) { + return Status::OK(); + } + // The SDK inspects the opened snapshot without constructing delegated providers. Its + // local paths already have the SDK's decoding/normalization; never URI-decode them again. + std::unique_ptr json( + lance_dataset_storage_bases_json(dataset), &lance_free_string); + if (!json) { + return Status::InvalidArgument("Lance TVF storage bases could not be validated"); + } + rapidjson::Document metadata; + metadata.Parse(json.get()); + if (metadata.HasParseError() || !metadata.IsObject() || !metadata.HasMember("format_version") || + !metadata["format_version"].IsInt() || metadata["format_version"].GetInt() != 1 || + !metadata.HasMember("bases") || !metadata["bases"].IsArray() || metadata["bases"].Empty()) { + return Status::InvalidArgument("Invalid Lance TVF storage-base metadata"); + } + for (const auto& base : metadata["bases"].GetArray()) { + if (!base.IsObject() || !base.HasMember("is_local") || !base["is_local"].IsBool() || + !base.HasMember("primary") || !base["primary"].IsBool()) { + return Status::InvalidArgument("Invalid Lance TVF storage-base metadata"); + } + const bool is_local = base["is_local"].GetBool(); + if (policy == TvfStoragePolicy::REMOTE) { + if (is_local) { + return Status::InvalidArgument("Remote Lance TVF requires remote storage bases"); + } + continue; + } + if (!is_local || !base.HasMember("local_path") || !base["local_path"].IsString()) { + return Status::InvalidArgument("LOCAL Lance TVF requires local storage bases"); + } + std::string canonical; + const std::string path(base["local_path"].GetString(), + base["local_path"].GetStringLength()); + RETURN_IF_ERROR(io::global_local_filesystem()->canonicalize_local_file( + config::user_files_secure_path, path, &canonical)); + // The primary tree was inspected before opening its manifest. Registered bases can + // live elsewhere in the secure root, so inspect their descendants before SDK reads. + if (!base["primary"].GetBool()) { + RETURN_IF_ERROR(validate_local_dataset_files(canonical)); + } + } + return Status::OK(); +} + LanceTableReader::~LanceTableReader() { static_cast(close()); } @@ -196,6 +264,8 @@ Status LanceTableReader::fetch_schema(const TFileRangeDesc& range, return lance_error("open Lance dataset for schema"); } + RETURN_IF_ERROR( + _validate_dataset_storage(dataset.get(), _tvf_storage_policy(range, scan_params))); std::shared_ptr schema; RETURN_IF_ERROR(import_lance_dataset_schema(dataset.get(), &schema)); return convert_arrow_schema_to_doris(schema, column_names, column_types); @@ -755,6 +825,10 @@ Status LanceTableReader::_ensure_dataset_open(const TFileRangeDesc& range, } else if (!_opened_dataset_key.has_value() || *_opened_dataset_key != key) { return Status::InvalidArgument( "Lance reader cannot mix dataset snapshots or storage options"); + } else { + // Revalidate reused handles before scanners, indexes or row-id reads. Fresh handles + // are checked inside _open_dataset before they become reader state. + RETURN_IF_ERROR(_validate_dataset_storage(_dataset, key.storage_policy)); } if (_search_kind == SearchKind::FULL_TEXT && prepare_fts_context && _fts_query_context == nullptr) { @@ -785,6 +859,8 @@ Status LanceTableReader::_open_dataset(const DatasetKey& key) { dataset.reset(raw_dataset); } + RETURN_IF_ERROR(_validate_dataset_storage(dataset.get(), key.storage_policy)); + // EXPLAIN plans a moving selector separately, so only the profile shows what this execution // read. The version is the one opened, which version 0 (the latest) resolves. _opened_version = static_cast(lance_dataset_version(dataset.get())); @@ -1457,6 +1533,7 @@ Status LanceTableReader::_fill_block_from_lance_batch(LanceBatch* batch, Block* Status LanceTableReader::_dataset_key(const TFileRangeDesc& range, DatasetKey* key) const { const auto& params = range.table_format_params.lance_params; RETURN_IF_ERROR(resolve_dataset_uri(range, *_scan_params, &key->uri)); + key->storage_policy = _tvf_storage_policy(range, *_scan_params); key->version = params.version; return build_lance_storage_options(_scan_params, &key->storage_options); } diff --git a/be/src/format_v2/table/lance_reader.h b/be/src/format_v2/table/lance_reader.h index 205c082d12052d..61b12865c0926c 100644 --- a/be/src/format_v2/table/lance_reader.h +++ b/be/src/format_v2/table/lance_reader.h @@ -65,13 +65,19 @@ class LanceTableReader final : public TableReader { Status close() override; private: + enum class TvfStoragePolicy { NONE, LOCAL, REMOTE }; + struct DatasetKey { std::string uri; int64_t version = 0; std::vector storage_options; + TvfStoragePolicy storage_policy = TvfStoragePolicy::NONE; bool operator==(const DatasetKey&) const = default; }; + static TvfStoragePolicy _tvf_storage_policy(const TFileRangeDesc& range, + const TFileScanRangeParams& scan_params); + static Status _validate_dataset_storage(LanceDataset* dataset, TvfStoragePolicy policy); Status _resolve_search_kind(); Status _validate_external_search_request() const; Status _ensure_dataset_open(const TFileRangeDesc& range, bool prepare_fts_context = true); diff --git a/be/test/format_v2/table/lance/data/tvf-storage-fixtures.md b/be/test/format_v2/table/lance/data/tvf-storage-fixtures.md new file mode 100644 index 00000000000000..966d1bc3297189 --- /dev/null +++ b/be/test/format_v2/table/lance/data/tvf-storage-fixtures.md @@ -0,0 +1,13 @@ +The metadata-only storage-policy fixtures were generated with lance-c 4fafcd64525b274b113208390bbcca86bbbc093b and its pinned Lance SDK 68c12dfd7efe02f90ad2d7f3a239a7eb64884e57. Their fixed relative base references the existing all_types.lance fixture. No data files were copied and no manifest bytes were hand-written. Both snapshots reopen and scan row_id 1, 2, 3, 4. + +tvf_storage_clone.lance uses a dataset-root base. tvf_storage_direct_base.lance uses a data-directory base. The tests vary user_files_secure_path to admit or reject the same valid references. Run BE tests from the repository root, as required by run-be-ut.sh. + +To regenerate, copy ../generate_storage_policy_fixtures.rs into the pinned lance-c examples directory as generate_tvf_storage_clone.rs. From that SDK checkout run: + +```sh +cargo run --release --locked --example generate_tvf_storage_clone -- /absolute/doris/repository /absolute/new-output-directory +``` + +The output directory must not exist. The generator uses official shallow-clone/commit APIs, reopens both manifests, and verifies four actual rows before reporting success. Copy the resulting two dataset directories into this directory. + +The optional remote-storage test uses an explicitly selected isolated S3 fixture. Upload all_types.lance as source.lance and the metadata-only clone as clone.lance. Set DORIS_LANCE_TEST_REMOTE_SOURCE_URI, DORIS_LANCE_TEST_REMOTE_CLONE_URI, DORIS_LANCE_TEST_S3_ENDPOINT, DORIS_LANCE_TEST_S3_ACCESS_KEY and DORIS_LANCE_TEST_S3_SECRET_KEY only for that test run. The clone's relative base deliberately resolves to the local repository fixture, while the source control reads remote objects. diff --git a/be/test/format_v2/table/lance/data/tvf_storage_clone.lance/_versions/18446744073709551612.manifest b/be/test/format_v2/table/lance/data/tvf_storage_clone.lance/_versions/18446744073709551612.manifest new file mode 100644 index 0000000000000000000000000000000000000000..73800825370503978fd767d3b29d472409790d64 GIT binary patch literal 2184 zcmb_eOK)C;o3`onHGRA9QYndFacu9d?uG?IEG`l|Yiy4zgA-S=Q<}R+LRYN# z1uXdi{1z67-@t+mERf*y*ls7~Rzm1bBxf?;IdjhU%}mB#rBZ3v(7CK>ZV(K$o~oE; zPp5{}GiAf<1w+{*foqMFq57_6wgp@}Rb4trX_5|tcseGjbFU6k;zo23k%>nKK1s=d zM3Ixu?`YCzgDdSyMLa;wY5cVl`qJNJ6K)#eBsFwX5YLe4#&P7_lW0buJKo$UW ziB{5ZG6zn9t1jSR++?_Gw3>{;!xON^12a*@V8}DH^aQFu2We^{+i}C&CtTw>&d?d| z0xbtooRx$n(OiJB2!marjvvpUVg;_nuLNvGlD7FMGw!aBR*giPnfI22e`|~%i8S6CPrt4b46}vmVM@n%OWI_ove^5lNF*ub z8-k0{u`9Ph`7}0wg;R6V8GhU+ReVBx4f)Du#m$Quf zxkcuFvq=0%IxMZ{YIo$8;$!J(LG1)}{n?bHVLWjX>cta3!4hCN;cL`Qym(3z$DKP# zJeztHpYRsmSGZH@q*N~pcP5>d(vap>S-5i;{`az7Nawr<2{j_(LQyVxVSaZG(GnfI z)c0Xs@D-2R3Bm{JJ4yJKI$5!7_Sa==S+&qR0evHgCY-peWnW76WF6?4G8rkl?vH#| z4MyI`bOTNIJk<;Gn*beGgletcXtvr*;&NwYb*;OOHn#BY{=u8eN{AwVP&>bA zrJZV79afXf6tk5*;iFm#(Oo^TMwTY)o*PiOb#;x%)(5ro%PX)`#WlOr{`S}3e{OdF z`st59s~>ki>wZynZ?xWjx$XPu!{p0V5q1$=>ZoD%<-Yv3Q~s{0ef4|6z{3A|8~9PE Q+`@ZD-l~;PUVZxdAC*eDkpKVy literal 0 HcmV?d00001 diff --git a/be/test/format_v2/table/lance/data/tvf_storage_direct_base.lance/_versions/18446744073709551614.manifest b/be/test/format_v2/table/lance/data/tvf_storage_direct_base.lance/_versions/18446744073709551614.manifest new file mode 100644 index 0000000000000000000000000000000000000000..ba1cb6d0f91a9e7eeabda04b0df58836708f10a9 GIT binary patch literal 4108 zcmd5(03NvCO>PG9cZeV0m2{7Pb*)z#gwK!^oM?5wdpt_)6G#ZGDN9tj~< z`~o&etXS|9_zOq~q5B(Huw(kSe@XC(e?$rX(nQjeRMqJ@ zmhATIo@TjrdiclL#&lhd=UseskB z^$mV=YkOyRPbi<4#cz9BP2kYiTs^lV!E?j;6~BwYnWjh$OyM zJHIQropN5Cq$aVc#8&(bA6b`(YN@W?(-l#*EtguZt13j)->RM8UVxo4-oe#j@VV(Z z!e9BrUCeresHbsNA_t&TybyVVF>o?mc>)LRT7o;ri(#Mm z{ylJp1;(OMH$fiZxqDFMAxM!k*@op^-Q#8-;$V};_D=A;>jxw^SDk<*Jqhd#H=JN( z`81nEjm1^dt_0Y|^A5E=n29NwB41~5OIeR0$(kV<;zNRN%KY4;1l>qyCV`b)K?DiJ zGMW<-Vy3}TG*jj0AE#(WS{Ih0gG%FZik_(`cp*2YXr?6c3oK`ub*8}l)9EuhiVoHm zSpzt$RSa_9WV<^IuzM($T(SH6)#0zpN0|XUuVOvu=wU(uQ3XZk3V{u zunyB*PKHA==2uyK#rMJ}!K~qm7tubyCaflJWw!SZF(MJn$r?p8AOeZ@DSIerl{9Rb&_hk@()l;2`SvcW_S zm=)WL#{9Ohm5&GN$%sC^p+3I@9n6TQxO(G-UKH}X?11y}j9aP2#QkbE@jYQTx1Oq9 zlPk*a3wsl42e|2sh9vTWff-Ue7&sv>0EQ7h$IZ|VhBP#-u^9%Vp-uS%*21*H9SR4z zdS19A;V_qmX|CmkJBH?eF58K4%z6;;tWPW`$|)<%?#>RLqkW4y4y+4*#-cV{@0L1d z=)Iz5Tr49wx(p?**2TN^+F{S4ZP*P_+YN^O__&kX5rOlpon&eo_A+V1>1#&gYZ|6e zk~HqrOC|KLF@TE104f**sBjE`?9(v-vV~#*6^j8>GzQRrjRBTWC = std::result::Result>; +const SOURCE_URI: &str = "be/test/format_v2/table/lance/data/all_types.lance"; + +fn main() -> Result<()> { + let args = std::env::args().skip(1).collect::>(); + if args.len() != 2 { + return Err("usage: generate_tvf_storage_clone ".into()); + } + let repository = std::fs::canonicalize(&args[0])?; + let output = PathBuf::from(&args[1]); + if !output.is_absolute() || output.exists() { + return Err( + "output must be a new absolute directory (existing content is never removed)".into(), + ); + } + std::fs::create_dir_all(&output)?; + // The committed relative base URI intentionally resolves under the test repository cwd. + std::env::set_current_dir(repository)?; + lance_c::runtime::block_on(generate(&output)) +} + +async fn row_ids(dataset: &Dataset) -> Result> { + let mut scanner = dataset.scan(); + scanner.project(&["row_id"])?; + let batch = scanner.try_into_batch().await?; + let mut values = Vec::with_capacity(batch.num_rows()); + for row in 0..batch.num_rows() { + values.push( + arrow::util::display::array_value_to_string(batch.column(0).as_ref(), row)?.parse()?, + ); + } + values.sort_unstable(); + if values != vec![1, 2, 3, 4] { + return Err(format!("expected row_id [1,2,3,4], got {values:?}").into()); + } + Ok(values) +} + +async fn publish_manifest_only(dataset: &Dataset, destination: &Path) -> Result<()> { + // The SDK reports the actual latest manifest; do not guess v1/v2 filename ordering. + let location = dataset.manifest_location(); + let manifest = PathBuf::from(lance_io::local::to_local_path(&location.path)); + let filename = manifest.file_name().ok_or("manifest has no filename")?; + let versions = destination.join("_versions"); + std::fs::create_dir_all(&versions)?; + std::fs::copy(&manifest, versions.join(filename))?; + // Deliberately copy neither data nor transactions. Reopen/scan proves that only the + // primary manifest plus the unchanged existing source fixture is sufficient. + let reopened = DatasetBuilder::from_uri(destination.to_str().ok_or("non-UTF8 destination")?) + .load() + .await?; + let values = row_ids(&reopened).await?; + let registrations = reopened + .manifest() + .base_paths + .values() + .map(|base| { + serde_json::json!({ + "id": base.id, + "uri": base.path, + "is_dataset_root": base.is_dataset_root, + }) + }) + .collect::>(); + println!( + "{}", + serde_json::json!({ + "fixture": destination, + "manifest": filename.to_string_lossy(), + "row_ids": values, + "bases": registrations, + }) + ); + Ok(()) +} + +async fn generate(output: &Path) -> Result<()> { + let mut source = DatasetBuilder::from_uri(SOURCE_URI).load().await?; + row_ids(&source).await?; + if source.uri() != SOURCE_URI { + return Err(format!("SDK changed source URI identity: {:?}", source.uri()).into()); + } + let scratch = output.join("sdk-clone-root"); + let version = source.version_id(); + let clone = source + .shallow_clone(scratch.to_str().ok_or("non-UTF8 scratch")?, version, None) + .await?; + if !clone + .manifest() + .base_paths + .values() + .any(|base| base.path == SOURCE_URI && base.is_dataset_root) + { + return Err( + "official shallow_clone did not retain the required relative source base URI".into(), + ); + } + row_ids(&clone).await?; + publish_manifest_only(&clone, &output.join("tvf_storage_clone.lance")).await?; + + // Optional direct-directory base: reuse the exact source fragments without writing files. + // Official Overwrite rejects inherited deletion files, so skip this optional fixture for + // a source that has deletions rather than changing its row semantics. + if source + .manifest() + .fragments + .iter() + .all(|fragment| fragment.deletion_file.is_none()) + { + let base_id = 7; + let mut fragments = source.manifest().fragments.as_ref().clone(); + for fragment in &mut fragments { + for file in fragment.referenced_lance_files_mut() { + file.base_id = Some(base_id); + } + } + let transaction = Transaction::new( + 0, + Operation::Overwrite { + fragments, + schema: source.schema().clone(), + config_upsert_values: None, + initial_bases: Some(vec![BasePath::new( + base_id, + format!("{SOURCE_URI}/data"), + None, + false, + )]), + }, + None, + ); + let direct_scratch = output.join("sdk-clone-direct-directory"); + let direct = CommitBuilder::new(direct_scratch.to_str().ok_or("non-UTF8 scratch")?) + .execute(transaction) + .await?; + row_ids(&direct).await?; + publish_manifest_only(&direct, &output.join("tvf_storage_direct_base.lance")).await?; + } else { + eprintln!("optional direct-directory fixture skipped: source contains deletion files"); + } + Ok(()) +} diff --git a/be/test/format_v2/table/lance_reader_test.cpp b/be/test/format_v2/table/lance_reader_test.cpp index 6460df54bda052..15cb01f96f93ae 100644 --- a/be/test/format_v2/table/lance_reader_test.cpp +++ b/be/test/format_v2/table/lance_reader_test.cpp @@ -4806,3 +4806,189 @@ TEST(LanceTableReaderLocalPathTest, RemoteRangeKeepsItsUriAndOptions) { EXPECT_EQ((std::vector {"aws_region", "us-east-1"}), key.storage_options); } } // namespace doris::format::lance + +namespace doris::format::lance { +namespace { +Status collect_storage_policy_rows(LanceTableReader* reader, const Columns& columns, + std::vector* rows) { + Block block; + add_output_columns(&block, columns); + bool eos = false; + while (!eos) { + RETURN_IF_ERROR(reader->get_block(&block, &eos)); + if (!eos) { + const auto& values = + assert_cast(*block.get_by_position(0).column).get_data(); + rows->insert(rows->end(), values.begin(), values.end()); + } + } + std::sort(rows->begin(), rows->end()); + return Status::OK(); +} +} // namespace + +TEST(LanceTableReaderLocalPathTest, ManifestBasesAreCheckedBeforeSchemaAndScan) { + const auto original_secure_path = config::user_files_secure_path; + DEFER(config::user_files_secure_path = original_secure_path); + for (const auto* name : {"tvf_storage_clone.lance", "tvf_storage_direct_base.lance"}) { + SCOPED_TRACE(name); + const auto clone = std::filesystem::canonical( + std::filesystem::path("be/test/format_v2/table/lance/data") / name); + // The clone contains only regular metadata; its real fragments live in a sibling dataset. + config::user_files_secure_path = clone.string(); + TQueryGlobals globals; + RuntimeState state(globals); + RuntimeProfile profile("lance_manifest_containment"); + TFileScanRangeParams scan_params; + LanceTableReader reader; + const Columns columns {projected_column("row_id", TYPE_BIGINT, false)}; + ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, &scan_params).ok()); + auto range = make_latest_lance_range(clone); + range.__set_file_type(TFileType::FILE_LOCAL); + range.table_format_params.lance_params.__set_is_tvf(true); + std::vector names; + std::vector types; + const auto schema = reader.fetch_schema(range, scan_params, &names, &types); + EXPECT_FALSE(schema.ok()); + EXPECT_EQ(std::string::npos, schema.to_string().find("all_types.lance")); + EXPECT_FALSE(prepare_range(&reader, range).ok()); + EXPECT_EQ(nullptr, reader._dataset); + EXPECT_EQ(nullptr, reader._scanner); + ASSERT_TRUE(reader.close().ok()); + } +} + +TEST(LanceTableReaderLocalPathTest, InRootManifestBasesReadActualRows) { + const auto original_secure_path = config::user_files_secure_path; + DEFER(config::user_files_secure_path = original_secure_path); + const auto root = std::filesystem::canonical("be/test/format_v2/table/lance/data"); + config::user_files_secure_path = root.string(); + for (const auto* name : {"tvf_storage_clone.lance", "tvf_storage_direct_base.lance"}) { + SCOPED_TRACE(name); + TQueryGlobals globals; + RuntimeState state(globals); + RuntimeProfile profile("lance_manifest_in_root"); + TFileScanRangeParams scan_params; + LanceTableReader reader; + const Columns columns {projected_column("row_id", TYPE_BIGINT, false)}; + ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, &scan_params).ok()); + auto range = make_latest_lance_range(root / name); + range.__set_file_type(TFileType::FILE_LOCAL); + range.table_format_params.lance_params.__set_is_tvf(true); + std::vector names; + std::vector types; + ASSERT_TRUE(reader.fetch_schema(range, scan_params, &names, &types).ok()); + EXPECT_NE(names.end(), std::find(names.begin(), names.end(), "row_id")); + ASSERT_TRUE(prepare_range(&reader, range).ok()); + std::vector rows; + ASSERT_TRUE(collect_storage_policy_rows(&reader, columns, &rows).ok()); + EXPECT_EQ((std::vector {1, 2, 3, 4}), rows); + ASSERT_TRUE(reader.close().ok()); + } +} + +TEST(LanceTableReaderLocalPathTest, TvfPolicyDoesNotInheritCatalogAdmission) { + const auto original_secure_path = config::user_files_secure_path; + DEFER(config::user_files_secure_path = original_secure_path); + const auto clone = std::filesystem::canonical( + "be/test/format_v2/table/lance/data/tvf_storage_clone.lance"); + config::user_files_secure_path = clone.string(); + std::unique_ptr snapshot( + lance_dataset_open(clone.c_str(), nullptr, 0)); + ASSERT_NE(nullptr, snapshot); + const auto version = lance_dataset_version(snapshot.get()); + TQueryGlobals globals; + RuntimeState state(globals); + RuntimeProfile profile("lance_manifest_catalog_policy"); + TFileScanRangeParams scan_params; + LanceTableReader reader; + const Columns columns {projected_column("row_id", TYPE_BIGINT, false)}; + ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, &scan_params).ok()); + auto range = make_latest_lance_range("file://" + clone.string()); + range.__set_file_type(TFileType::FILE_LOCAL); + range.table_format_params.lance_params.__set_version(version); + // Unmarked fixed-version catalogs retain their warehouse policy. + ASSERT_TRUE(prepare_range(&reader, range).ok()); + LanceTableReader::DatasetKey catalog_key; + ASSERT_TRUE(reader._dataset_key(range, &catalog_key).ok()); + range.table_format_params.lance_params.__set_is_tvf(true); + LanceTableReader::DatasetKey tvf_key; + ASSERT_TRUE(reader._dataset_key(range, &tvf_key).ok()); + EXPECT_NE(catalog_key, tvf_key); + EXPECT_FALSE(reader._ensure_dataset_open(range).ok()); + ASSERT_TRUE(reader.close().ok()); + + // A fresh handle can reuse the process Session/manifest cache, but must still enforce TVF bases. + RuntimeProfile tvf_profile("lance_manifest_reused_session"); + LanceTableReader tvf_reader; + ASSERT_TRUE(init_reader(&tvf_reader, columns, &state, &tvf_profile, &scan_params).ok()); + std::vector names; + std::vector types; + EXPECT_FALSE(tvf_reader.fetch_schema(range, scan_params, &names, &types).ok()); + EXPECT_FALSE(prepare_range(&tvf_reader, range).ok()); + EXPECT_EQ(nullptr, tvf_reader._dataset); + ASSERT_TRUE(tvf_reader.close().ok()); +} + +TEST(LanceTableReaderRemoteStorageTest, RemoteManifestCannotReadLocalBase) { + const auto* clone_uri = std::getenv("DORIS_LANCE_TEST_REMOTE_CLONE_URI"); + const auto* source_uri = std::getenv("DORIS_LANCE_TEST_REMOTE_SOURCE_URI"); + if (clone_uri == nullptr || source_uri == nullptr) { + GTEST_SKIP() << "Requires an explicitly selected isolated S3 storage-policy fixture"; + } + const auto* endpoint = std::getenv("DORIS_LANCE_TEST_S3_ENDPOINT"); + const auto* access_key = std::getenv("DORIS_LANCE_TEST_S3_ACCESS_KEY"); + const auto* secret_key = std::getenv("DORIS_LANCE_TEST_S3_SECRET_KEY"); + ASSERT_NE(nullptr, endpoint); + ASSERT_NE(nullptr, access_key); + ASSERT_NE(nullptr, secret_key); + std::map options {{"aws_endpoint", endpoint}, + {"aws_access_key_id", access_key}, + {"aws_secret_access_key", secret_key}, + {"aws_region", "us-east-1"}, + {"aws_allow_http", "true"}, + {"aws_virtual_hosted_style_request", "false"}}; + std::vector native_options; + for (const auto& [key, value] : options) { + native_options.push_back(key.c_str()); + native_options.push_back(value.c_str()); + } + native_options.push_back(nullptr); + std::unique_ptr snapshot( + lance_dataset_open(clone_uri, native_options.data(), 0)); + ASSERT_NE(nullptr, snapshot) << lance_error("open remote clone").to_string(); + TQueryGlobals globals; + RuntimeState state(globals); + RuntimeProfile profile("lance_remote_manifest_policy"); + TFileScanRangeParams scan_params; + TLanceScanParams params; + params.__set_lance_storage_options(options); + scan_params.__set_lance_scan_params(params); + LanceTableReader reader; + const Columns columns {projected_column("row_id", TYPE_BIGINT, false)}; + ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, &scan_params).ok()); + auto range = make_latest_lance_range(clone_uri); + range.__set_file_type(TFileType::FILE_S3); + range.table_format_params.lance_params.__set_is_tvf(true); + range.table_format_params.lance_params.__set_version(lance_dataset_version(snapshot.get())); + const auto status = prepare_range(&reader, range); + EXPECT_FALSE(status.ok()); + EXPECT_NE(std::string::npos, status.to_string().find("remote storage bases")); + EXPECT_EQ(nullptr, reader._dataset); + EXPECT_EQ(nullptr, reader._scanner); + ASSERT_TRUE(reader.close().ok()); + + // An ordinary remote dataset must still return real rows with the identical storage options. + RuntimeProfile remote_profile("lance_remote_storage_control"); + LanceTableReader remote; + ASSERT_TRUE(init_reader(&remote, columns, &state, &remote_profile, &scan_params).ok()); + range = make_latest_lance_range(source_uri); + range.__set_file_type(TFileType::FILE_S3); + range.table_format_params.lance_params.__set_is_tvf(true); + ASSERT_TRUE(prepare_range(&remote, range).ok()); + std::vector rows; + ASSERT_TRUE(collect_storage_policy_rows(&remote, columns, &rows).ok()); + EXPECT_EQ((std::vector {1, 2, 3, 4}), rows); + ASSERT_TRUE(remote.close().ok()); +} +} // namespace doris::format::lance diff --git a/be/test/io/fs/local_file_system_test.cpp b/be/test/io/fs/local_file_system_test.cpp index ffa4653b77ad4b..f7233487407037 100644 --- a/be/test/io/fs/local_file_system_test.cpp +++ b/be/test/io/fs/local_file_system_test.cpp @@ -871,4 +871,30 @@ TEST_F(LocalFileSystemTest, TestEqualOrSubPath) { EXPECT_FALSE(io::LocalFileSystem::equal_or_sub_path("/data/store1", "/data/store11/snapshot")); EXPECT_FALSE(io::LocalFileSystem::equal_or_sub_path("/data/store/snapshot", "/data/store")); } + +TEST_F(LocalFileSystemTest, TestSafeGlobPreservesWhitespaceIdentity) { + const auto original_secure_path = config::user_files_secure_path; + DEFER(config::user_files_secure_path = original_secure_path); + const auto root = std::filesystem::canonical(test_dir); + const auto secure = root / "whitespace-secure"; + const auto outside = root / "whitespace-outside.csv"; + ASSERT_TRUE(std::filesystem::create_directory(secure)); + ASSERT_TRUE(save_string_file(outside.string(), "outside").ok()); + ASSERT_TRUE(save_string_file((secure / "name.csv ").string(), "safe").ok()); + std::filesystem::create_symlink(outside, secure / "name.csv"); + config::user_files_secure_path = secure.string(); + std::vector files; + auto fs = io::global_local_filesystem(); + ASSERT_TRUE(fs->safe_glob("name.csv?", &files).ok()); + ASSERT_EQ(1, files.size()); + EXPECT_EQ((secure / "name.csv ").string(), files[0].file_name); + io::FileReaderSPtr reader; + ASSERT_TRUE(fs->open_file(files[0].file_name, &reader).ok()); + char buffer[4]; + size_t bytes_read = 0; + ASSERT_TRUE(reader->read_at(0, {buffer, sizeof(buffer)}, &bytes_read).ok()); + EXPECT_EQ("safe", std::string(buffer, bytes_read)); + EXPECT_FALSE(fs->safe_glob("name.csv", &files).ok()); +} + } // namespace doris diff --git a/build.sh b/build.sh index 75179dfac57393..444c69bbadf67c 100755 --- a/build.sh +++ b/build.sh @@ -473,6 +473,49 @@ if [[ ! -f "${DORIS_THIRDPARTY}/installed/lib/${LAST_THIRDPARTY_LIB}" || fi fi +# Lance TVF storage policy needs both sides of the metadata-only SDK ABI. +# Check the selected third-party prefix only; a system Lance install cannot satisfy it. +lance_c_is_compatible() { + local install_dir="$1" + [[ ! -e "${install_dir}/lib64/.lance-installing" && + -s "${install_dir}/include/lance/lance.h" && + -s "${install_dir}/lib64/liblance_c.a" ]] || return 1 + grep -Eq '^#[[:space:]]*define[[:space:]]+LANCE_STORAGE_BASES_JSON_VERSION[[:space:]]+1([[:space:]]|$)' \ + "${install_dir}/include/lance/lance.h" || return 1 + # GNU nm and Darwin nm spell C exports differently. Read the whole stream: + # early awk exit would turn a large archive's producer SIGPIPE into failure. + "${NM:-nm}" -g "${install_dir}/lib64/liblance_c.a" 2>/dev/null | awk ' + NF >= 2 && $(NF - 1) ~ /^[TW]$/ && $NF ~ /^_?lance_dataset_storage_bases_json$/ { found = 1 } + END { exit !found } + ' +} + +lance_c_source_matches() { + local source_dir="$1" + [[ -f "${source_dir}/build-thirdparty.sh" && -f "${source_dir}/download-thirdparty.sh" && + -f "${source_dir}/vars.sh" && -f "${source_dir}/patches/lance-c-storage-bases.patch" ]] || return 1 + grep -Fq 'install_lance_c() {' "${source_dir}/build-thirdparty.sh" || return 1 + grep -Fq 'lance-c-storage-bases.patch' "${source_dir}/download-thirdparty.sh" || return 1 + cmp -s "${DORIS_HOME}/thirdparty/patches/lance-c-storage-bases.patch" \ + "${source_dir}/patches/lance-c-storage-bases.patch" || return 1 + cmp -s <(grep -E '^LANCE_C_(DOWNLOAD|NAME|SOURCE)=' "${DORIS_HOME}/thirdparty/vars.sh") \ + <(grep -E '^LANCE_C_(DOWNLOAD|NAME|SOURCE)=' "${source_dir}/vars.sh") +} + +if [[ "${BUILD_BE}" -eq 1 ]] && ! lance_c_is_compatible "${DORIS_THIRDPARTY}/installed"; then + if ! lance_c_source_matches "${DORIS_THIRDPARTY}"; then + echo "Lance SDK header/archive pair lacks LANCE_STORAGE_BASES_JSON_VERSION=1 or its exported API, or is incomplete." >&2 + echo "Refresh the compilation image's include/lance and lib64/liblance_c.a together, or set DORIS_THIRDPARTY to matching complete third-party sources." >&2 + exit 1 + fi + echo "Rebuilding only lance_c for the TVF storage-base metadata ABI ..." + bash "${DORIS_THIRDPARTY}/build-thirdparty.sh" -j "${PARALLEL}" lance_c + if ! lance_c_is_compatible "${DORIS_THIRDPARTY}/installed"; then + echo "Rebuilt Lance SDK header/archive pair is incompatible or incomplete; refresh the matching pair before building BE." >&2 + exit 1 + fi +fi + update_submodule() { local submodule_path=$1 local submodule_name=$2 diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/metadata/LanceMetadataLoader.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/metadata/LanceMetadataLoader.java index 69bc50c2e35b5d..e167d758f188ef 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/metadata/LanceMetadataLoader.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/metadata/LanceMetadataLoader.java @@ -69,6 +69,7 @@ public static LanceTableMetadata loadLatestForTvf( Dataset dataset = Dataset.open().allocator(allocator).uri(datasetUri) .readOptions(LanceReadOptions.forIndependentRead( access.getStorageOptions(), OptionalLong.empty())).build()) { + LanceTvfStoragePolicy.checkRemoteBases(dataset); return read(dataset, access, MetadataScope.BASIC); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/metadata/LanceTvfStoragePolicy.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/metadata/LanceTvfStoragePolicy.java new file mode 100644 index 00000000000000..e419c6511decc7 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/metadata/LanceTvfStoragePolicy.java @@ -0,0 +1,103 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.lance.metadata; + +import org.apache.doris.common.AnalysisException; + +import com.google.common.collect.ImmutableSet; +import com.google.common.net.HostAndPort; +import com.google.common.net.InetAddresses; +import org.lance.BasePath; +import org.lance.Dataset; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Locale; +import java.util.Set; + +/** Storage admission for independent Lance TVFs; catalog reads retain their configured storage. */ +public final class LanceTvfStoragePolicy { + // Remote providers registered by the pinned Lance SDK. File providers also have aliases, + // so checking only for "file" (or a particular StorageProperties type) is insufficient. + private static final Set LANCE_REMOTE_SCHEMES = ImmutableSet.of( + "s3", "s3+ddb", "az", "abfss", "gs", "goosefs", "oss", "cos", "hf", "tos"); + + private LanceTvfStoragePolicy() { + } + + /** Inspect manifest registrations before BASIC metadata can open any referenced fragments. */ + public static void checkRemoteBases(Dataset dataset) throws AnalysisException { + for (BasePath base : dataset.getBasePaths()) { + if (!isRemoteUri(base.getPath())) { + throw new AnalysisException("Remote Lance TVF requires remote manifest storage bases"); + } + } + } + + public static boolean isRemoteUri(String path) { + try { + if (path == null) { + return false; + } + int colon = path.indexOf(':'); + if (colon < 0 || !LANCE_REMOTE_SCHEMES.contains(path.substring(0, colon).toLowerCase(Locale.ROOT))) { + return false; + } + String scheme = path.substring(0, colon); + if (!path.startsWith("//", colon + 1)) { + // GooseFS can resolve its remote master from storage options or configuration. + return "goosefs".equalsIgnoreCase(scheme) && path.startsWith("/", colon + 1); + } + int authorityEnd = path.length(); + for (char separator : new char[] {'/', '?', '#'}) { + int end = path.indexOf(separator, colon + 3); + if (end >= 0) { + authorityEnd = Math.min(authorityEnd, end); + } + } + // Validate only scheme/authority against SDK-local fallback. Rust's URL parser + // accepts and encodes raw spaces in paths; preserve that spelling for the SDK. + URI uri = new URI(path.substring(0, authorityEnd) + "/"); + String authority = uri.getRawAuthority(); + if (authority == null || authority.isEmpty()) { + return "goosefs".equalsIgnoreCase(scheme); + } + // Lance falls back to a local path if Rust's URL parser rejects the URI. Java's URI + // accepts registry authorities with invalid ports, which Rust rejects. Validate the + // host/port separately without disallowing legitimate bucket names with underscores. + String hostPort = authority.substring(authority.lastIndexOf('@') + 1); + HostAndPort parsed = HostAndPort.fromString(hostPort).requireBracketsForIPv6(); + if (parsed.hasPort()) { + // Guava parses "-0" as zero, but Rust rejects signs and falls back to local. + String port = hostPort.substring(hostPort.lastIndexOf(':') + 1); + for (int i = 0; i < port.length(); i++) { + if (port.charAt(i) < '0' || port.charAt(i) > '9') { + return false; + } + } + } + String host = parsed.getHost(); + // Rust rejects zones and IPv4 tails with leading zeros, such as ::ffff:127.0.0.01. + return !host.isEmpty() && (!host.contains(":") + || (!host.contains("%") && InetAddresses.isInetAddress(host))); + } catch (URISyntaxException | IllegalArgumentException e) { + return false; + } + } + +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/tvf/source/TVFScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/tvf/source/TVFScanNode.java index 40959143298cdd..d0dc8af243f0d4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/tvf/source/TVFScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/tvf/source/TVFScanNode.java @@ -247,6 +247,7 @@ protected void setScanParams(TFileRangeDesc rangeDesc, Split split) { LanceSplit lanceSplit = (LanceSplit) split; TLanceFileDesc lanceParams = new TLanceFileDesc(); lanceParams.setDatasetUri(lanceSplit.getDatasetUri()); + lanceParams.setIsTvf(true); lanceParams.setVersion(lanceSplit.getVersion()); if (lanceSplit.hasFragmentIds()) { lanceParams.setFragmentIds(lanceSplit.getFragmentIds()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java index dbb01c6089985f..a741cc775862da 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java @@ -32,6 +32,7 @@ import org.apache.doris.common.Config; import org.apache.doris.common.IdGenerator; import org.apache.doris.common.Pair; +import org.apache.doris.common.util.BrokerUtil; import org.apache.doris.datasource.VariantWritePlanValidator; import org.apache.doris.datasource.hive.HMSExternalDatabase; import org.apache.doris.datasource.hive.HMSExternalTable; @@ -51,6 +52,7 @@ import org.apache.doris.datasource.paimon.PaimonExternalTable; import org.apache.doris.datasource.paimon.PaimonVariantWriteAnalyzer; import org.apache.doris.datasource.paimon.PaimonWriteTarget; +import org.apache.doris.datasource.property.storage.StorageProperties; import org.apache.doris.dictionary.Dictionary; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.StatementContext; @@ -116,6 +118,7 @@ import org.apache.doris.qe.AutoCloseSessionVariable; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.SessionVariable; +import org.apache.doris.tablefunction.HadoopTvfAdmission; import org.apache.doris.tablefunction.LocalTableValuedFunction; import org.apache.doris.thrift.TPartialUpdateNewRowPolicy; @@ -689,6 +692,19 @@ private Plan bindTVFTableSink(MatchingContext> ctx) { "delete_existing_files=true is not supported for local TVF"); } + if ("hdfs".equals(tvfName)) { + // Storage normalization loads operator-controlled XML before route classification, + // while leaving filesystem construction to execution. + Map normalized = StorageProperties.createPrimary(properties).getBackendConfigProperties(); + if (!properties.containsKey("writer_class")) { + HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, filePath, normalized); + } + if (deleteExisting) { + HadoopTvfAdmission.check(HadoopTvfAdmission.Access.JAVA_DELETE, + BrokerUtil.extractParentDirectory(filePath), normalized); + } + } + LogicalPlan child = ((LogicalPlan) sink.child()); // Always derive schema from child query output diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTVFCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTVFCommand.java index a1114e0e814ac2..bae564fd9fe21b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTVFCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTVFCommand.java @@ -40,12 +40,13 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; import org.apache.doris.nereids.trees.plans.physical.PhysicalTVFTableSink; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; -import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.ConnectContext.ConnectType; +import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.Coordinator; -import org.apache.doris.qe.QeProcessorImpl; import org.apache.doris.qe.QeProcessorImpl.QueryInfo; +import org.apache.doris.qe.QeProcessorImpl; import org.apache.doris.qe.StmtExecutor; +import org.apache.doris.tablefunction.HadoopTvfAdmission; import org.apache.doris.tablefunction.LocalTableValuedFunction; import org.apache.doris.task.LoadEtlTask; import org.apache.doris.thrift.TStatusCode; @@ -196,7 +197,7 @@ private void deleteExistingFilesInFE(String tvfName, Map props) StorageProperties storageProps = StorageProperties.createPrimary(fsCopyProps); if (storageProps instanceof HdfsCompatibleProperties) { - LocalTableValuedFunction.checkHadoopFileSystemEnabled( + HadoopTvfAdmission.check(HadoopTvfAdmission.Access.JAVA_DELETE, parentDir, storageProps.getBackendConfigProperties()); } RemoteFileSystem fs = FileSystemFactory.get(storageProps); diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/TVFTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/planner/TVFTableSink.java index 306d954f84d302..36ce1426c8658b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/TVFTableSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/TVFTableSink.java @@ -24,6 +24,7 @@ import org.apache.doris.datasource.property.fileformat.FileFormatProperties; import org.apache.doris.datasource.property.fileformat.OrcFileFormatProperties; import org.apache.doris.datasource.property.storage.StorageProperties; +import org.apache.doris.tablefunction.HadoopTvfAdmission; import org.apache.doris.tablefunction.LocalTableValuedFunction; import org.apache.doris.thrift.TColumn; import org.apache.doris.thrift.TDataSink; @@ -131,7 +132,8 @@ public void bindDataSink() throws AnalysisException { } if (fileType == TFileType.FILE_HDFS && !properties.containsKey("writer_class")) { - LocalTableValuedFunction.checkHdfsSinkEnabled(properties.get("file_path"), backendConnectProps); + HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, + properties.get("file_path"), backendConnectProps); } String filePath = properties.get("file_path"); diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java index 41466e0648a718..3d863f276ccb19 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java @@ -648,6 +648,7 @@ private PFetchTableSchemaRequest getFetchTableStructureRequest() throws TExcepti if (isLanceFormat()) { TLanceFileDesc lanceParams = new TLanceFileDesc(); lanceParams.setDatasetUri(getFilePath()); + lanceParams.setIsTvf(true); // Local TVF schema discovery and execution independently open the latest snapshot. // S3 TVF obtains its schema and fixed snapshot metadata directly in FE instead. lanceParams.setVersion(0); diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/HadoopTvfAdmission.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/HadoopTvfAdmission.java new file mode 100644 index 00000000000000..d27e1e940b065e --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/HadoopTvfAdmission.java @@ -0,0 +1,477 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.tablefunction; + +import org.apache.doris.common.Config; +import org.apache.doris.common.util.LocationPath; +import org.apache.doris.nereids.exceptions.AnalysisException; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * TVF-only admission for Hadoop consumers. Input properties have already been normalized by storage. + * This module neither loads Hadoop resources nor constructs a filesystem. Java strings and native + * C strings are projected separately before filesystem selection and mount-table interpretation. + */ +public final class HadoopTvfAdmission { + public enum Access { + NATIVE_WRITE, + JAVA_DELETE, + HADOOP_READ + } + + private enum Provider { + REMOTE, + LOCAL, + MOUNT_TABLE, + DYNAMIC, + UNKNOWN + } + + private static final Set REMOTE_SCHEMES = new HashSet<>(Arrays.asList( + "hdfs", "s3a", "s3n", "oss", "ofs", "jfs", "abfs", "abfss", "wasb", "wasbs", "gs", "cosn")); + private static final Set REMOTE_IMPLEMENTATIONS = new HashSet<>(Arrays.asList( + "org.apache.hadoop.hdfs.DistributedFileSystem", + "org.apache.hadoop.fs.s3a.S3AFileSystem", + "org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem", + "org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem", + "org.apache.hadoop.fs.azure.NativeAzureFileSystem", + "org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem", + "com.aliyun.jindodata.oss.JindoOssFileSystem", + "org.apache.hadoop.fs.obs.OBSFileSystem", + "com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem")); + + private final Access access; + private final Map properties = new HashMap<>(); + + private HadoopTvfAdmission(Access access, Map normalized, boolean nativeConsumer) { + this.access = access; + for (Map.Entry entry : normalized.entrySet()) { + // parse_properties / generateHdfsParam consumes the canonical NameNode before JNI. + if (nativeConsumer && "fs.defaultFS".equalsIgnoreCase(entry.getKey())) { + continue; + } + String key = (nativeConsumer ? nativeStringPrefix(entry.getKey()) : entry.getKey()).trim(); + String value = nativeConsumer ? nativeStringPrefix(entry.getValue()) : entry.getValue(); + String previous = properties.put(key, value); + if (previous != null && !previous.equals(value) && isRoutingKey(key)) { + reject(); + } + } + } + + /** Reject local/unknown Hadoop TVF routes before listing, deletion, schema discovery or data IO. */ + public static void check(Access access, String path, Map normalized) { + if (access != Access.HADOOP_READ && Config.enable_local_tvf) { + return; + } + try { + if (access == Access.NATIVE_WRITE) { + String nameNode = nativeNameNode(access, normalized); + URI output = nativeFileUri(access, path + "doris-tvf-file", nameNode); + String parent = output.getPath(); + parent = parent.substring(0, parent.lastIndexOf('/') + 1); + if (parent.length() > 1) { + parent = parent.substring(0, parent.length() - 1); + } + new HadoopTvfAdmission(access, normalized, true).checkNative(nameNode, parent); + } else { + HadoopTvfAdmission javaConsumer = new HadoopTvfAdmission(access, normalized, false); + // Project the Java consumer independently from native convert_path/JNI. + URI javaPath = hadoopPathUri(path.trim()); + javaConsumer.checkRoute(javaPath, javaPath.getPath(), new HashSet<>()); + if (access == Access.HADOOP_READ) { + // FileQueryScanNode sends this filesystem identifier in range.fs_name. BE + // falls back to hdfs_params.fs_name only when that range field is empty. + URI fileUri = LocationPath.of(path).getPath().toUri(); + String nameNode = fileUri.getScheme() + "://" + fileUri.getAuthority(); + if (nameNode.isEmpty()) { + nameNode = nativeNameNode(access, normalized); + } + HadoopTvfAdmission nativeConsumer = new HadoopTvfAdmission(access, normalized, true); + nativeConsumer.checkNativeRead(nameNode, path); + // Schema RPCs have no range.fs_name; they use THdfsParams.fs_name instead. + nativeConsumer.checkNativeRead(nativeNameNode(access, normalized), path); + } + } + } catch (URISyntaxException e) { + new HadoopTvfAdmission(access, normalized, false).reject(); + } + } + + private static String nativeNameNode(Access access, Map normalized) { + String result = null; + for (Map.Entry entry : normalized.entrySet()) { + if ("fs.defaultFS".equalsIgnoreCase(entry.getKey())) { + if (result != null && !result.equals(entry.getValue())) { + reject(access); + } + result = entry.getValue(); + } + } + return result == null ? "" : result; + } + + private void checkNative(String nameNode, String path) throws URISyntaxException { + nameNode = selector(nativeStringPrefix(nameNode)).trim(); + if (nameNode.isEmpty() && access == Access.NATIVE_WRITE) { + // BE passes fs_name.c_str(), never NULL. libhdfs prefixes this empty string + // with hdfs://, whose URI construction fails before any filesystem is selected. + // Preserve existing analysis; execution reports the invalid NameNode as before. + return; + } + if ("default".equals(nameNode) || nameNode.isEmpty()) { + reject(); + } + checkRoute(new URI(nameNode.contains("://") ? nameNode : "hdfs://" + nameNode), + path, new HashSet<>()); + } + + private void checkNativeRead(String nameNode, String path) throws URISyntaxException { + checkNative(nameNode, nativeFileUri(access, path, nameNode).getPath()); + } + + private List defaults() throws URISyntaxException { + List defaults = new ArrayList<>(); + for (String key : Arrays.asList("fs.defaultFS", "fs.default.name")) { + String value = property(key); + if (value != null) { + value = selector(value).trim(); + defaults.add(new URI("local".equals(value) ? "file:///" + : (uriScheme(value).isEmpty() ? "hdfs://" + value : value))); + } + } + if (defaults.isEmpty()) { + defaults.add(new URI("file:///")); + } + return defaults; + } + + private void checkRoute(URI filesystem, String path, Set activeTables) throws URISyntaxException { + if (filesystem.getScheme() == null) { + for (URI defaultFs : defaults()) { + checkViewFsEnabled(defaultFs, path, activeTables); + } + return; + } + if (filesystem.getAuthority() == null) { + boolean completed = false; + for (URI defaultFs : defaults()) { + if (filesystem.getScheme().equals(defaultFs.getScheme()) && defaultFs.getAuthority() != null) { + checkViewFsEnabled(defaultFs, path, activeTables); + completed = true; + } + } + if (completed) { + return; + } + } + checkViewFsEnabled(filesystem, path, activeTables); + } + + private Provider provider(String scheme) { + if ("file".equalsIgnoreCase(scheme)) { + return Provider.LOCAL; + } + String implementation = property("fs." + scheme + ".impl"); + if (implementation != null) { + implementation = selector(implementation).trim(); + if ("org.apache.hadoop.fs.LocalFileSystem".equals(implementation) + || "org.apache.hadoop.fs.RawLocalFileSystem".equals(implementation)) { + return Provider.LOCAL; + } + if ("org.apache.hadoop.fs.viewfs.ViewFileSystem".equals(implementation)) { + return Provider.MOUNT_TABLE; + } + if ("org.apache.hadoop.fs.viewfs.ViewFileSystemOverloadScheme".equals(implementation) + || "org.apache.hadoop.hdfs.ViewDistributedFileSystem".equals(implementation)) { + return Provider.DYNAMIC; + } + return REMOTE_IMPLEMENTATIONS.contains(implementation) ? Provider.REMOTE : Provider.UNKNOWN; + } + if ("viewfs".equals(scheme)) { + return Provider.MOUNT_TABLE; + } + return REMOTE_SCHEMES.contains(scheme) ? Provider.REMOTE : Provider.UNKNOWN; + } + + private void checkGlobTargets(String key, Set activeTables) throws URISyntaxException { + for (String target : selector(property(key)).split(",")) { + URI uri = new URI(target); + // Listing can touch any descendant of a selected mount. Reuse the same route + // interpreter for nested tables instead of assuming their root is representative. + String path = uri.getPath() == null ? "/" : uri.getPath(); + checkRoute(uri, (path.endsWith("/") ? path : path + "/") + "*", activeTables); + } + } + + private void checkViewFsEnabled(URI filesystem, String path, Set activeTables) { + String scheme = filesystem.getScheme(); + Provider provider = provider(scheme); + if (provider == Provider.LOCAL || provider == Provider.UNKNOWN || provider == Provider.DYNAMIC) { + reject(); + } + if (provider != Provider.MOUNT_TABLE) { + return; + } + if (!path.startsWith("/")) { + // Java resolves relative paths against the selected filesystem's working directory. + // Its user/home configuration is not a proven absolute mount-table path here. + reject(); + } + String table = filesystem.getAuthority(); + String ignorePort = property("fs.viewfs.ignore.port.in.mount.table.name"); + if (ignorePort != null && Boolean.parseBoolean(selector(ignorePort).trim()) + && table != null && filesystem.getPort() != -1) { + table = filesystem.getHost(); + if (table == null) { + reject(); + } + } + if (table == null || table.isEmpty()) { + table = property("fs.viewfs.mounttable.default.name.key"); + table = table == null ? "default" : selector(table); + } + // Nested ViewFS cycles or arbitrarily deep delegation cannot establish remote storage. + if (activeTables.size() >= 32 || !activeTables.add(table)) { + reject(); + } + try { + String prefix = "fs.viewfs.mounttable." + table + "."; + String source = ""; + List selectedKeys = new ArrayList<>(); + List mountSources = new ArrayList<>(); + String fallback = null; + String rootMerge = null; + for (String key : properties.keySet()) { + if (!key.startsWith(prefix)) { + continue; + } + String entry = key.substring(prefix.length()); + if (entry.equals("linkFallback")) { + fallback = key; + continue; + } + if (entry.equals("linkMergeSlash")) { + rootMerge = key; + continue; + } + if (entry.startsWith("linkRegex")) { + // Regex interception rewrites runtime paths before ordinary mount resolution. + reject(); + } + int slash = entry.indexOf('/'); + if (slash < 0 || !(entry.startsWith("link.") || entry.startsWith("linkMerge.") + || entry.startsWith("linkNfly."))) { + continue; + } + String candidate = entry.substring(slash); + // InodeTree splits source components and discards trailing empty components. + while (candidate.length() > 1 && candidate.endsWith("/")) { + candidate = candidate.substring(0, candidate.length() - 1); + } + mountSources.add(candidate); + if (!(path.equals(candidate) || path.startsWith(candidate + "/"))) { + continue; + } + if (candidate.length() > source.length()) { + source = candidate; + selectedKeys.clear(); + } + if (candidate.equals(source)) { + selectedKeys.add(key); + } + } + if (access == Access.HADOOP_READ && hasGlob(path)) { + String literal = path.substring(0, firstGlob(path)); + int slash = literal.lastIndexOf('/'); + literal = slash < 0 ? "/" : literal.substring(0, slash + 1); + for (String key : properties.keySet()) { + if (!key.startsWith(prefix)) { + continue; + } + String entry = key.substring(prefix.length()); + int sourceStart = entry.indexOf('/'); + if (sourceStart >= 0 && (entry.startsWith("link.") || entry.startsWith("linkMerge.") + || entry.startsWith("linkNfly."))) { + String mount = entry.substring(sourceStart); + if (mount.startsWith(literal) || literal.startsWith(mount + "/")) { + checkGlobTargets(key, activeTables); + } + } + } + } + if (access == Access.HADOOP_READ && !hasGlob(path)) { + String subtree = path.endsWith("/") ? path : path + "/"; + for (String key : properties.keySet()) { + if (key.startsWith(prefix)) { + String entry = key.substring(prefix.length()); + int sourceStart = entry.indexOf('/'); + if (sourceStart >= 0 && entry.substring(sourceStart).startsWith(subtree)) { + checkGlobTargets(key, activeTables); + } + } + } + } + if (rootMerge != null) { + selectedKeys.add(rootMerge); + source = ""; + } else if (selectedKeys.isEmpty() && fallback != null) { + selectedKeys.add(fallback); + } + if (selectedKeys.isEmpty()) { + // No explicit target can prove what a BE-inherited mount table will select. + reject(); + } + for (String key : selectedKeys) { + String targets = selector(property(key)); + for (String target : targets.split(",")) { + if (target.isEmpty()) { + continue; + } + URI uri = new URI(target); + if (uri.getScheme() == null) { + reject(); + } + if (provider(uri.getScheme()) == Provider.MOUNT_TABLE) { + for (String nested : mountSources) { + if (!nested.startsWith(source + "/")) { + continue; + } + int branchEnd = nested.indexOf('/', source.length() + 1); + String branch = branchEnd < 0 ? nested : nested.substring(0, branchEnd); + if (path.equals(branch) || path.startsWith(branch + "/")) { + // A missing child below an internal nested-mount directory uses + // a different suffix in InodeTree. Do not guess its delegation. + reject(); + } + } + } + String base = uri.getPath() == null ? "/" : uri.getPath(); + String remaining = path.substring(source.length()); + String nextPath = new URI(null, null, + (base.endsWith("/") ? base : base + "/") + + (remaining.startsWith("/") ? remaining.substring(1) : remaining), null) + .normalize().getPath(); + checkRoute(uri, nextPath, activeTables); + } + } + } catch (URISyntaxException e) { + reject(); + } finally { + activeTables.remove(table); + } + } + + private static URI nativeFileUri(Access access, String path, String nameNode) + throws URISyntaxException { + // Match BE convert_path before the JNI C-string conversion. A different URI prefix + // becomes a literal rooted Hadoop path; it must not be parsed as that URI's path. + if (!nameNode.isEmpty() && path.contains(nameNode)) { + if (!path.startsWith(nameNode)) { + // convert_path truncates bytes at NameNode.length even for a non-prefix match. + // Java character offsets cannot establish that malformed path's mount identity. + reject(access); + } + path = path.substring(nameNode.length()); + } + if (!path.startsWith("/")) { + path = "/" + path; + } + return hadoopPathUri(nativeStringPrefix(path)); + } + + private static URI hadoopPathUri(String path) throws URISyntaxException { + String scheme = uriScheme(path); + String authority = null; + String relative = scheme.isEmpty() ? path : path.substring(scheme.length() + 1); + if (relative.startsWith("//")) { + int slash = relative.indexOf('/', 2); + authority = slash < 0 ? relative.substring(2) : relative.substring(2, slash); + relative = slash < 0 ? "/" : relative.substring(slash); + } + if (!scheme.isEmpty() && !relative.startsWith("/")) { + relative = "/" + relative; + } + // Hadoop Path(String) quotes path bytes before URI normalization; percent text is literal. + return new URI(scheme.isEmpty() ? null : scheme, authority, relative, null, null).normalize(); + } + + private static String uriScheme(String path) { + int colon = path.indexOf(':'); + int slash = path.indexOf('/'); + return colon > 0 && (slash < 0 || colon < slash) ? path.substring(0, colon) : ""; + } + + private String property(String key) { + return properties.get(key); + } + + private String selector(String value) { + if (value.contains("${")) { + // Runtime environment and Hadoop substitutions are not available to this FE projection. + reject(); + } + return value; + } + + private static boolean isRoutingKey(String key) { + return "fs.defaultFS".equals(key) || "fs.default.name".equals(key) + || key.startsWith("fs.viewfs.") || (key.startsWith("fs.") && key.endsWith(".impl")); + } + + private static int firstGlob(String path) { + int first = path.length(); + for (char marker : new char[] {'*', '?', '[', '{'}) { + int index = path.indexOf(marker); + if (index >= 0) { + first = Math.min(first, index); + } + } + return first; + } + + private static boolean hasGlob(String path) { + return firstGlob(path) < path.length(); + } + + private static String nativeStringPrefix(String value) { + int nul = value.indexOf('\0'); + return nul < 0 ? value : value.substring(0, nul); + } + + private void reject() { + reject(access); + } + + private static void reject(Access access) { + if (access == Access.HADOOP_READ) { + throw new AnalysisException("Hadoop TVFs require a proven remote filesystem route. " + + "Use local() for local files with LOCAL TVF privileges and secure-path restrictions."); + } + LocalTableValuedFunction.checkEnabled(); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/HdfsTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/HdfsTableValuedFunction.java index e99340eb105bec..9edc57b6e18791 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/HdfsTableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/HdfsTableValuedFunction.java @@ -23,6 +23,7 @@ import org.apache.doris.common.FeConstants; import org.apache.doris.common.UserException; import org.apache.doris.datasource.property.storage.StorageProperties; +import org.apache.doris.thrift.TBrokerFileStatus; import org.apache.doris.thrift.TFileType; import org.apache.log4j.LogManager; @@ -55,9 +56,14 @@ private void init(Map properties) throws AnalysisException { throw new AnalysisException("Failed check storage props, " + e.getMessage(), e); } + HadoopTvfAdmission.check(HadoopTvfAdmission.Access.HADOOP_READ, filePath, backendConnectProperties); if (!FeConstants.runningUnitTest) { // 4. parse file parseFile(); + for (TBrokerFileStatus file : fileStatuses) { + HadoopTvfAdmission.check(HadoopTvfAdmission.Access.HADOOP_READ, + file.getPath(), backendConnectProperties); + } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/LocalTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/LocalTableValuedFunction.java index da83d4f7c3089b..3eb16008e814d7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/LocalTableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/LocalTableValuedFunction.java @@ -24,7 +24,6 @@ import org.apache.doris.common.Config; import org.apache.doris.common.ErrorCode; import org.apache.doris.common.UserException; -import org.apache.doris.common.util.BrokerUtil; import org.apache.doris.datasource.FederationBackendPolicy; import org.apache.doris.datasource.property.storage.StorageProperties; import org.apache.doris.mysql.privilege.PrivPredicate; @@ -42,13 +41,8 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import java.net.URI; -import java.net.URISyntaxException; -import java.util.ArrayList; -import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; @@ -142,7 +136,7 @@ private void getFileListFromBackend() throws AnalysisException { + ", error msg: " + globResponse.getStatus().getErrorMsgsList()); } for (PFileInfo file : globResponse.getFilesList()) { - fileStatuses.add(new TBrokerFileStatus(file.getFile().trim(), false, file.getSize(), true)); + fileStatuses.add(new TBrokerFileStatus(file.getFile(), false, file.getSize(), true)); LOG.info("get file from backend success. file: {}, size: {}", file.getFile(), file.getSize()); } } catch (Exception e) { @@ -155,15 +149,6 @@ public static void checkSinkEnabled(String tvfName, Map properti if ("local".equalsIgnoreCase(tvfName) || isLocalWriter(properties.getOrDefault("writer_class", ""))) { checkEnabled(); } - if ("hdfs".equalsIgnoreCase(tvfName)) { - if (!properties.containsKey("writer_class")) { - checkHdfsSinkEnabled(properties); - } - if (Boolean.parseBoolean(properties.getOrDefault("delete_existing_files", "false"))) { - checkHadoopFileSystemEnabled( - BrokerUtil.extractParentDirectory(properties.getOrDefault("file_path", "")), properties); - } - } } private static boolean isLocalWriter(String writer) { @@ -171,288 +156,6 @@ private static boolean isLocalWriter(String writer) { return "org.apache.doris.writer.LocalFileJniWriter".equals(nativeStringPrefix(writer).replace('/', '.')); } - /** Check the NameNode and provider used by the native FILE_HDFS writer. */ - public static void checkHdfsSinkEnabled(Map properties) { - checkHdfsSinkEnabled(properties.getOrDefault("file_path", ""), properties); - } - - public static void checkHdfsSinkEnabled(String outputPrefix, Map properties) { - String nameNode = nativeStringPrefix(properties.getOrDefault("fs.defaultFS", "")).trim(); - if ("default".equals(nameNode)) { - // libhdfs delegates this selector to the BE's defaults, which may select local storage. - checkEnabled(); - } - // libhdfs prefixes bare NameNodes with hdfs:// rather than using the output path's scheme. - checkHadoopSchemeEnabled(nameNode.contains("://") ? uriScheme(nameNode) : "hdfs", properties); - if (!Config.enable_local_tvf) { - try { - URI filesystem = new URI(nameNode.contains("://") ? nameNode : "hdfs://" + nameNode); - // BE appends a generated filename directly to this prefix, then convert_path - // removes the NameNode. Resolve its parent, not an exact mount-point filename. - String generated = outputPrefix + "doris-tvf-file"; - String rawNameNode = properties.getOrDefault("fs.defaultFS", ""); - if (!rawNameNode.isEmpty() && generated.contains(rawNameNode)) { - generated = generated.substring(rawNameNode.length()); - } - if (!generated.startsWith("/")) { - generated = "/" + generated; - } - String path = hadoopPathUri(nativeStringPrefix(generated)).getPath(); - path = path.substring(0, path.lastIndexOf('/') + 1); - if (path.length() > 1) { - path = path.substring(0, path.length() - 1); - } - checkViewFsEnabled(filesystem, path, properties, new HashSet<>()); - } catch (URISyntaxException e) { - // Invalid NameNodes remain subject to the existing storage validation. - } - } - } - - /** Check Hadoop's path/default-filesystem selection before FE-side cleanup. */ - public static void checkHadoopFileSystemEnabled(String path, Map properties) { - String scheme = uriScheme(path); - if (!scheme.isEmpty()) { - checkHadoopSchemeEnabled(scheme, properties); - checkViewFsPathEnabled(path, path, properties); - return; - } - String defaultFs = hadoopSelectorProperty("fs.defaultFS", properties); - String legacyDefaultFs = hadoopSelectorProperty("fs.default.name", properties); - // Configuration.set synchronizes aliases; either value can win when the map is applied. - // Do not assume that the canonical key overrides a local legacy selector. - if (defaultFs != null) { - checkHadoopDefaultEnabled(defaultFs, path, properties); - } - if (legacyDefaultFs != null) { - checkHadoopDefaultEnabled(legacyDefaultFs, path, properties); - } - if (defaultFs == null && legacyDefaultFs == null) { - checkHadoopSchemeEnabled("file", properties); - } - } - - private static void checkHadoopDefaultEnabled(String value, String path, Map properties) { - value = hadoopSelectorValue(value).trim(); - // Hadoop retains the old "local" default-FS spelling. - String scheme = "local".equals(value) ? "file" : uriScheme(value); - checkHadoopSchemeEnabled(scheme.isEmpty() ? "hdfs" : scheme, properties); - checkViewFsPathEnabled(scheme.isEmpty() ? "hdfs://" + value : value, path, properties); - } - - private static void checkViewFsPathEnabled(String location, String path, Map properties) { - if (!Config.enable_local_tvf) { - try { - URI filesystem = uriScheme(path).isEmpty() ? new URI(location) : hadoopPathUri(path); - checkViewFsEnabled(filesystem, hadoopPathUri(path).getPath(), properties, new HashSet<>()); - } catch (URISyntaxException e) { - checkEnabled(); - } - } - } - - private static void checkHadoopSchemeEnabled(String scheme, Map properties) { - if ("file".equalsIgnoreCase(scheme)) { - checkEnabled(); - } - String implementationKey = "fs." + scheme + ".impl"; - for (Map.Entry entry : properties.entrySet()) { - // The native Hadoop configuration boundary consumes C strings for keys and values. - if (implementationKey.equals(nativeStringPrefix(entry.getKey()).trim())) { - String implementation = hadoopSelectorValue(nativeStringPrefix(entry.getValue())).trim(); - if ("org.apache.hadoop.fs.LocalFileSystem".equals(implementation) - || "org.apache.hadoop.fs.RawLocalFileSystem".equals(implementation)) { - checkEnabled(); - } - } - } - } - - private static void checkViewFsEnabled(URI filesystem, String path, Map properties, - Set activeTables) { - if (Config.enable_local_tvf) { - return; - } - String scheme = filesystem.getScheme(); - checkHadoopSchemeEnabled(scheme, properties); - String implementation = hadoopSelectorProperty("fs." + scheme + ".impl", properties); - if (implementation != null - && "org.apache.hadoop.fs.viewfs.ViewFileSystemOverloadScheme".equals(implementation.trim())) { - // OverloadScheme can reload runtime mount tables and overrides provider selection. - checkEnabled(); - } - boolean viewFs = isViewFs(scheme, implementation); - if (!viewFs) { - return; - } - String table = filesystem.getAuthority(); - String ignorePort = hadoopSelectorProperty("fs.viewfs.ignore.port.in.mount.table.name", properties); - if (ignorePort != null && Boolean.parseBoolean(hadoopSelectorValue(ignorePort).trim()) - && table != null && filesystem.getPort() != -1) { - table = filesystem.getHost(); - if (table == null) { - checkEnabled(); - } - } - if (table == null || table.isEmpty()) { - table = hadoopSelectorProperty("fs.viewfs.mounttable.default.name.key", properties); - table = table == null ? "default" : hadoopSelectorValue(table); - } - // Nested ViewFS cycles or arbitrarily deep delegation cannot establish remote storage. - if (activeTables.size() >= 32 || !activeTables.add(table)) { - checkEnabled(); - } - try { - String prefix = "fs.viewfs.mounttable." + table + "."; - String source = ""; - List selectedKeys = new ArrayList<>(); - List mountSources = new ArrayList<>(); - String fallback = null; - String rootMerge = null; - for (String rawKey : properties.keySet()) { - String key = nativeStringPrefix(rawKey).trim(); - if (!key.startsWith(prefix)) { - continue; - } - String entry = key.substring(prefix.length()); - if (entry.equals("linkFallback")) { - fallback = key; - continue; - } - if (entry.equals("linkMergeSlash")) { - rootMerge = key; - continue; - } - if (entry.startsWith("linkRegex")) { - // Regex interception rewrites runtime paths before ordinary mount resolution. - checkEnabled(); - } - int slash = entry.indexOf('/'); - if (slash < 0 || !(entry.startsWith("link.") || entry.startsWith("linkMerge.") - || entry.startsWith("linkNfly."))) { - continue; - } - String candidate = entry.substring(slash); - // InodeTree splits source components and discards trailing empty components. - while (candidate.length() > 1 && candidate.endsWith("/")) { - candidate = candidate.substring(0, candidate.length() - 1); - } - mountSources.add(candidate); - if (!(path.equals(candidate) || path.startsWith(candidate + "/"))) { - continue; - } - if (candidate.length() > source.length()) { - source = candidate; - selectedKeys.clear(); - } - if (candidate.equals(source)) { - selectedKeys.add(key); - } - } - if (rootMerge != null) { - selectedKeys.add(rootMerge); - source = ""; - } else if (selectedKeys.isEmpty() && fallback != null) { - selectedKeys.add(fallback); - } - if (selectedKeys.isEmpty()) { - // No explicit target can prove what a BE-inherited mount table will select. - checkEnabled(); - } - for (String key : selectedKeys) { - String targets = hadoopSelectorValue(hadoopSelectorProperty(key, properties)); - for (String target : targets.split(",")) { - if (target.isEmpty()) { - continue; - } - URI uri = new URI(target); - if (uri.getScheme() == null) { - checkEnabled(); - } - String targetImplementation = hadoopSelectorProperty("fs." + uri.getScheme() + ".impl", - properties); - if (isViewFs(uri.getScheme(), targetImplementation)) { - for (String nested : mountSources) { - if (!nested.startsWith(source + "/")) { - continue; - } - int branchEnd = nested.indexOf('/', source.length() + 1); - String branch = branchEnd < 0 ? nested : nested.substring(0, branchEnd); - if (path.equals(branch) || path.startsWith(branch + "/")) { - // A missing child below an internal nested-mount directory uses - // a different suffix in InodeTree. Do not guess its delegation. - checkEnabled(); - } - } - } - String base = uri.getPath() == null ? "/" : uri.getPath(); - String remaining = path.substring(source.length()); - String nextPath = new URI(null, null, - (base.endsWith("/") ? base : base + "/") - + (remaining.startsWith("/") ? remaining.substring(1) : remaining), null) - .normalize().getPath(); - checkViewFsEnabled(uri, nextPath, properties, activeTables); - } - } - } catch (URISyntaxException e) { - checkEnabled(); - } finally { - activeTables.remove(table); - } - } - - private static boolean isViewFs(String scheme, String implementation) { - return implementation == null ? "viewfs".equals(scheme) - : "org.apache.hadoop.fs.viewfs.ViewFileSystem".equals(implementation.trim()); - } - - private static URI hadoopPathUri(String path) throws URISyntaxException { - String scheme = uriScheme(path); - String authority = null; - String relative = scheme.isEmpty() ? path : path.substring(scheme.length() + 1); - if (relative.startsWith("//")) { - int slash = relative.indexOf('/', 2); - authority = slash < 0 ? relative.substring(2) : relative.substring(2, slash); - relative = slash < 0 ? "/" : relative.substring(slash); - } - if (!relative.startsWith("/")) { - relative = "/" + relative; - } - // Hadoop Path(String) quotes path bytes before URI normalization; percent text is literal. - return new URI(scheme.isEmpty() ? null : scheme, authority, relative, null, null).normalize(); - } - - private static String hadoopSelectorProperty(String key, Map properties) { - String value = null; - for (Map.Entry entry : properties.entrySet()) { - // Configuration.set trims keys; native configuration strings also stop at NUL. - if (key.equals(nativeStringPrefix(entry.getKey()).trim())) { - String candidate = nativeStringPrefix(entry.getValue()); - if (value != null && !value.equals(candidate)) { - // FE and BE map iteration can choose different conflicting spellings. - checkEnabled(); - } - value = candidate; - } - } - return value; - } - - private static String hadoopSelectorValue(String value) { - if (value.contains("${")) { - // Hadoop aliases and BE runtime properties can change the referenced value. - // An indirect filesystem selector needs opt-in; the FE cannot prove it is remote. - checkEnabled(); - } - return value; - } - - private static String uriScheme(String path) { - int colon = path.indexOf(':'); - int slash = path.indexOf('/'); - return colon > 0 && (slash < 0 || colon < slash) ? path.substring(0, colon) : ""; - } - private static String nativeStringPrefix(String value) { int nul = value.indexOf('\0'); return nul < 0 ? value : value.substring(0, nul); diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java index 44517f0fb4c2b7..225f998419dfee 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java @@ -24,21 +24,14 @@ import org.apache.doris.common.UserException; import org.apache.doris.datasource.lance.metadata.LanceMetadataLoader; import org.apache.doris.datasource.lance.metadata.LanceTableMetadata; +import org.apache.doris.datasource.lance.metadata.LanceTvfStoragePolicy; import org.apache.doris.datasource.property.storage.StorageProperties; import org.apache.doris.thrift.TBrokerFileStatus; import org.apache.doris.thrift.TFileType; -import com.google.common.collect.ImmutableSet; -import com.google.common.net.HostAndPort; -import com.google.common.net.InetAddresses; - -import java.net.URI; -import java.net.URISyntaxException; import java.util.Collections; import java.util.List; -import java.util.Locale; import java.util.Map; -import java.util.Set; /** * The Implement of table valued function @@ -54,11 +47,6 @@ public class S3TableValuedFunction extends ExternalFileTableValuedFunction { public static final String NAME = "s3"; - // Remote providers registered by the pinned Lance SDK. File providers also have aliases, - // so checking only for "file" (or a particular StorageProperties type) is insufficient. - private static final Set LANCE_REMOTE_SCHEMES = ImmutableSet.of( - "s3", "s3+ddb", "az", "abfss", "gs", "goosefs", "oss", "cos", "hf", "tos"); - public S3TableValuedFunction(Map properties) throws AnalysisException { this(properties, null); } @@ -78,7 +66,7 @@ public S3TableValuedFunction(Map properties, List properties, List= 0) { - authorityEnd = Math.min(authorityEnd, end); - } - } - // Validate only scheme/authority against SDK-local fallback. Rust's URL parser - // accepts and encodes raw spaces in paths; preserve that spelling for the SDK. - URI uri = new URI(path.substring(0, authorityEnd) + "/"); - String authority = uri.getRawAuthority(); - if (authority == null || authority.isEmpty()) { - return "goosefs".equalsIgnoreCase(scheme); - } - // Lance falls back to a local path if Rust's URL parser rejects the URI. Java's URI - // accepts registry authorities with invalid ports, which Rust rejects. Validate the - // host/port separately without disallowing legitimate bucket names with underscores. - String hostPort = authority.substring(authority.lastIndexOf('@') + 1); - HostAndPort parsed = HostAndPort.fromString(hostPort).requireBracketsForIPv6(); - if (parsed.hasPort()) { - // Guava parses "-0" as zero, but Rust rejects signs and falls back to local. - String port = hostPort.substring(hostPort.lastIndexOf(':') + 1); - for (int i = 0; i < port.length(); i++) { - if (port.charAt(i) < '0' || port.charAt(i) > '9') { - return false; - } - } - } - String host = parsed.getHost(); - // Rust rejects zones and IPv4 tails with leading zeros, such as ::ffff:127.0.0.01. - return !host.isEmpty() && (!host.contains(":") - || (!host.contains("%") && InetAddresses.isInetAddress(host))); - } catch (URISyntaxException | IllegalArgumentException e) { - return false; - } - } - // =========== implement abstract methods of ExternalFileTableValuedFunction ================= @Override public TFileType getTFileType() { diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/metadata/LanceTvfStoragePolicyTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/metadata/LanceTvfStoragePolicyTest.java new file mode 100644 index 00000000000000..356e07680838d8 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/lance/metadata/LanceTvfStoragePolicyTest.java @@ -0,0 +1,118 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.lance.metadata; + +import org.apache.doris.common.AnalysisException; +import org.apache.doris.datasource.property.storage.StorageProperties; + +import org.apache.arrow.vector.types.pojo.Schema; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.Test; +import org.lance.BasePath; +import org.lance.Dataset; +import org.lance.OpenDatasetBuilder; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; + +class LanceTvfStoragePolicyTest { + @Test + void remoteTvfRejectsLocalManifestBasesBeforeReadingMetadata() throws Exception { + for (String base : Arrays.asList("file:///private/source.lance", "/private/source.lance", + "file:/private/source.lance", "file-object-store:///private/source.lance", + "s3://bucket:-0/source.lance")) { + Dataset dataset = Mockito.mock(Dataset.class); + Mockito.when(dataset.getBasePaths()).thenReturn(Collections.singletonList( + new BasePath(1, Optional.empty(), base, true))); + OpenDatasetBuilder builder = Mockito.mock(OpenDatasetBuilder.class, Mockito.RETURNS_SELF); + Mockito.when(builder.build()).thenReturn(dataset); + try (MockedStatic datasets = Mockito.mockStatic(Dataset.class)) { + datasets.when(Dataset::open).thenReturn(builder); + AnalysisException error = Assertions.assertThrows(AnalysisException.class, + () -> LanceMetadataLoader.loadLatestForTvf( + "s3://bucket/clone.lance", Collections.emptyList())); + Assertions.assertFalse(error.getMessage().contains("private")); + Mockito.verify(dataset, Mockito.never()).getFragments(); + Mockito.verify(dataset, Mockito.never()).getSchema(); + Mockito.verify(dataset).close(); + } + } + } + + @Test + void remoteTvfPreservesValidRemoteManifestBases() throws Exception { + Dataset dataset = Mockito.mock(Dataset.class); + Mockito.when(dataset.getBasePaths()).thenReturn(Arrays.asList( + new BasePath(1, Optional.empty(), "s3://bucket/source data.lance", true), + new BasePath(2, Optional.empty(), "s3://bucket/other%20data", false))); + Mockito.when(dataset.version()).thenReturn(7L); + Mockito.when(dataset.getFragments()).thenReturn(Collections.emptyList()); + Mockito.when(dataset.getSchema()).thenReturn(new Schema(Collections.emptyList())); + OpenDatasetBuilder builder = Mockito.mock(OpenDatasetBuilder.class, Mockito.RETURNS_SELF); + Mockito.when(builder.build()).thenReturn(dataset); + try (MockedStatic datasets = Mockito.mockStatic(Dataset.class)) { + datasets.when(Dataset::open).thenReturn(builder); + LanceTableMetadata metadata = LanceMetadataLoader.loadLatestForTvf( + "s3://bucket/clone.lance", Collections.emptyList()); + Assertions.assertEquals(7L, metadata.getVersion()); + Mockito.verify(dataset).getFragments(); + Mockito.verify(dataset).getSchema(); + Mockito.verify(dataset).close(); + } + } + + @Test + void isolatedRemoteManifestRejectsLocalBaseAndKeepsRemoteReads() throws Exception { + String clone = System.getenv("DORIS_LANCE_TEST_REMOTE_CLONE_URI"); + String source = System.getenv("DORIS_LANCE_TEST_REMOTE_SOURCE_URI"); + Assumptions.assumeTrue(clone != null && source != null, + "Requires an explicitly selected isolated S3 storage-policy fixture"); + Map properties = new HashMap<>(); + properties.put("s3.endpoint", System.getenv("DORIS_LANCE_TEST_S3_ENDPOINT")); + properties.put("s3.access_key", System.getenv("DORIS_LANCE_TEST_S3_ACCESS_KEY")); + properties.put("s3.secret_key", System.getenv("DORIS_LANCE_TEST_S3_SECRET_KEY")); + properties.put("s3.region", "us-east-1"); + properties.put("use_path_style", "true"); + StorageProperties storage = StorageProperties.createPrimary(properties); + AnalysisException error = Assertions.assertThrows(AnalysisException.class, + () -> LanceMetadataLoader.loadLatestForTvf(clone, Collections.singletonList(storage))); + Assertions.assertTrue(error.getMessage().contains("remote manifest storage bases")); + LanceTableMetadata metadata = LanceMetadataLoader.loadLatestForTvf( + source, Collections.singletonList(storage)); + Assertions.assertTrue(metadata.getVersion() > 0); + Assertions.assertTrue(metadata.getSchema().getFields().stream() + .anyMatch(field -> "row_id".equals(field.getName()))); + } + + @Test + void catalogSnapshotReadKeepsItsConfiguredManifestBases() { + Dataset dataset = Mockito.mock(Dataset.class); + Mockito.when(dataset.version()).thenReturn(7L); + Mockito.when(dataset.getFragments()).thenReturn(Collections.emptyList()); + Mockito.when(dataset.getSchema()).thenReturn(new Schema(Collections.emptyList())); + LanceMetadataLoader.read(dataset, new LanceTableAccess("file:///warehouse/table.lance", + Collections.emptyMap()), LanceMetadataLoader.MetadataScope.BASIC); + Mockito.verify(dataset, Mockito.never()).getBasePaths(); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/tvf/source/TVFScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/tvf/source/TVFScanNodeTest.java index 430882e7ce72b1..3db6d1ed02f1fe 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/tvf/source/TVFScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/tvf/source/TVFScanNodeTest.java @@ -188,6 +188,7 @@ public void testLanceSplitsPinVersionAndFragments() throws Exception { Assert.assertEquals("s3://bucket/table.lance", range.getTableFormatParams().getLanceParams().getDatasetUri()); Assert.assertEquals(42L, range.getTableFormatParams().getLanceParams().getVersion()); + Assert.assertTrue(range.getTableFormatParams().getLanceParams().isIsTvf()); Assert.assertEquals(Collections.singletonList(7L), range.getTableFormatParams().getLanceParams().getFragmentIds()); } @@ -242,6 +243,7 @@ public void testLocalLanceUsesOneLatestWholeDatasetSplit() throws Exception { TFileRangeDesc range = new TFileRangeDesc(); node.setScanParams(range, split); Assert.assertEquals(0L, range.getTableFormatParams().getLanceParams().getVersion()); + Assert.assertTrue(range.getTableFormatParams().getLanceParams().isIsTvf()); Assert.assertFalse(range.getTableFormatParams().getLanceParams().isSetFragmentIds()); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindTVFTableSinkTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindTVFTableSinkTest.java index 61326cf60ec1e7..18e57f68782d48 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindTVFTableSinkTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindTVFTableSinkTest.java @@ -37,6 +37,7 @@ import org.apache.doris.planner.TVFTableSink; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.StmtExecutor; +import org.apache.doris.tablefunction.HadoopTvfAdmission; import org.apache.doris.tablefunction.LocalTableValuedFunction; import org.junit.jupiter.api.AfterEach; @@ -203,7 +204,7 @@ void disabledNativeHadoopLocalSinksRejectInsertAndExplain() { } AnalysisException exception = Assertions.assertThrows(AnalysisException.class, () -> analyzeInsert("INSERT INTO hdfs('fs.defaultFS'='file:///') SELECT 1")); - Assertions.assertTrue(exception.getMessage().contains("LOCAL TVF is disabled")); + Assertions.assertTrue(exception.getMessage().contains("requires 'file_path'")); } @Test @@ -276,85 +277,90 @@ void hadoopCleanupChecksSelectedFilesystemWithoutOpeningIt() { Map properties = new HashMap<>(); properties.put("fs.defaultFS", "hdfs://namenode:8020"); properties.put("fs.file.impl", "org.apache.hadoop.fs.LocalFileSystem"); - Assertions.assertDoesNotThrow(() -> LocalTableValuedFunction.checkHadoopFileSystemEnabled( + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.JAVA_DELETE, "/remote/path", properties)); Assertions.assertThrows(AnalysisException.class, - () -> LocalTableValuedFunction.checkHadoopFileSystemEnabled("file:///tmp/prefix_", properties)); + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.JAVA_DELETE, + "file:///tmp/prefix_", properties)); properties.remove("fs.defaultFS"); for (String defaultFs : Arrays.asList("local", "file:///")) { properties.put("fs.default.name", defaultFs); Assertions.assertThrows(AnalysisException.class, - () -> LocalTableValuedFunction.checkHadoopFileSystemEnabled("/tmp/prefix_", properties)); + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.JAVA_DELETE, "/tmp/prefix_", properties)); } properties.clear(); Assertions.assertThrows(AnalysisException.class, - () -> LocalTableValuedFunction.checkHadoopFileSystemEnabled("/tmp/prefix_", properties)); + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.JAVA_DELETE, "/tmp/prefix_", properties)); properties.put("fs.defaultFS", "hdfs://namenode:8020"); properties.put("fs.default.name", "file:///"); Assertions.assertThrows(AnalysisException.class, - () -> LocalTableValuedFunction.checkHadoopFileSystemEnabled("/tmp/prefix_", properties)); + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.JAVA_DELETE, "/tmp/prefix_", properties)); properties.put("fs.default.name", "hdfs://other-namenode:8020"); - Assertions.assertDoesNotThrow(() -> LocalTableValuedFunction.checkHadoopFileSystemEnabled( + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.JAVA_DELETE, "/remote/path", properties)); properties.remove("fs.defaultFS"); - Assertions.assertDoesNotThrow(() -> LocalTableValuedFunction.checkHadoopFileSystemEnabled( + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.JAVA_DELETE, "/remote/path", properties)); properties.clear(); properties.put("fs.defaultFS", "viewfs://cluster"); properties.put("fs.viewfs.impl", "org.apache.hadoop.fs.RawLocalFileSystem"); Assertions.assertThrows(AnalysisException.class, - () -> LocalTableValuedFunction.checkHadoopFileSystemEnabled("viewfs://cluster/tmp", properties)); + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.JAVA_DELETE, + "viewfs://cluster/tmp", properties)); properties.clear(); properties.put("fs.defaultFS", "${fs.selected.uri}"); properties.put("fs.selected.uri", "file:///"); Assertions.assertThrows(AnalysisException.class, - () -> LocalTableValuedFunction.checkHadoopFileSystemEnabled("/tmp/prefix_", properties)); + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.JAVA_DELETE, "/tmp/prefix_", properties)); properties.put("fs.selected.uri", "hdfs://namenode:8020"); Assertions.assertThrows(AnalysisException.class, - () -> LocalTableValuedFunction.checkHadoopFileSystemEnabled("/remote/path", properties)); + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.JAVA_DELETE, "/remote/path", properties)); Config.enable_local_tvf = true; - Assertions.assertDoesNotThrow(() -> LocalTableValuedFunction.checkHadoopFileSystemEnabled( + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.JAVA_DELETE, "/remote/path", properties)); Config.enable_local_tvf = false; properties.put("fs.defaultFS", "hdfs://namenode:8020"); properties.put("fs.hdfs.impl", "${fs.selected.impl}"); properties.put("fs.selected.impl", "org.apache.hadoop.fs.RawLocalFileSystem"); Assertions.assertThrows(AnalysisException.class, - () -> LocalTableValuedFunction.checkHdfsSinkEnabled(properties)); + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, "", properties)); properties.put("fs.selected.impl", "org.apache.hadoop.hdfs.DistributedFileSystem"); Assertions.assertThrows(AnalysisException.class, - () -> LocalTableValuedFunction.checkHdfsSinkEnabled(properties)); + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, "", properties)); Config.enable_local_tvf = true; - Assertions.assertDoesNotThrow(() -> LocalTableValuedFunction.checkHdfsSinkEnabled(properties)); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, + "", properties)); Config.enable_local_tvf = false; properties.put("fs.hdfs.impl", "${fs.selected.impl}"); properties.remove("fs.selected.impl"); properties.put("fs.selected.impl ", "org.apache.hadoop.fs.LocalFileSystem"); Assertions.assertThrows(AnalysisException.class, - () -> LocalTableValuedFunction.checkHdfsSinkEnabled(properties)); + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, "", properties)); properties.put("fs.selected.impl ", "org.apache.hadoop.hdfs.DistributedFileSystem"); Assertions.assertThrows(AnalysisException.class, - () -> LocalTableValuedFunction.checkHdfsSinkEnabled(properties)); + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, "", properties)); Config.enable_local_tvf = true; - Assertions.assertDoesNotThrow(() -> LocalTableValuedFunction.checkHdfsSinkEnabled(properties)); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, + "", properties)); Config.enable_local_tvf = false; // BE consumes canonical defaultFS as its NameNode; the old alias can redefine // that key inside Hadoop. Do not infer this indirect provider from the FE's raw map. properties.put("fs.default.name", "org.apache.hadoop.fs.LocalFileSystem"); properties.put("fs.hdfs.impl", "${fs.defaultFS}"); Assertions.assertThrows(AnalysisException.class, - () -> LocalTableValuedFunction.checkHdfsSinkEnabled(properties)); + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, "", properties)); properties.put("fs.hdfs.impl", "${env:TVF_TEST_UNRESOLVED_IMPL}"); Assertions.assertThrows(AnalysisException.class, - () -> LocalTableValuedFunction.checkHdfsSinkEnabled(properties)); + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, "", properties)); Config.enable_local_tvf = true; - Assertions.assertDoesNotThrow(() -> LocalTableValuedFunction.checkHdfsSinkEnabled(properties)); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, + "", properties)); Config.enable_local_tvf = false; properties.clear(); properties.put("fs.defaultFS", "hdfs://namenode:8020"); properties.put("fs.hdfs.impl\0ignored", "org.apache.hadoop.fs.LocalFileSystem\0ignored"); Assertions.assertThrows(AnalysisException.class, - () -> LocalTableValuedFunction.checkHdfsSinkEnabled(properties)); + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, "", properties)); } @Test @@ -428,17 +434,18 @@ void viewFsChecksSelectedTargetsAndPreservesRemoteMounts() throws Exception { properties.put("fs.viewfs.mounttable.cluster.link./local", "file:///tmp"); properties.put("fs.viewfs.mounttable.cluster.linkFallback", "hdfs://namenode:8020/fallback"); new TVFTableSink(new PlanNodeId(0), "hdfs", properties, Collections.emptyList()).bindDataSink(); - Assertions.assertDoesNotThrow(() -> LocalTableValuedFunction.checkHdfsSinkEnabled( + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, "/locality/prefix_", properties)); // A prefix naming a mount point generates a sibling filename; a trailing slash writes inside it. - Assertions.assertDoesNotThrow(() -> LocalTableValuedFunction.checkHdfsSinkEnabled("/local", properties)); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, + "/local", properties)); Assertions.assertInstanceOf(LogicalTVFTableSink.class, analyzeInsert("INSERT INTO hdfs('file_path'='/local', 'format'='csv'," + "'delete_existing_files'='true', 'fs.defaultFS'='viewfs://cluster'," + "'fs.viewfs.mounttable.cluster.link./local'='file:///tmp'," + "'fs.viewfs.mounttable.cluster.linkFallback'='hdfs://namenode:8020/base') SELECT 1")); Assertions.assertThrows(AnalysisException.class, - () -> LocalTableValuedFunction.checkHdfsSinkEnabled("/local/", properties)); + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, "/local/", properties)); properties.put("file_path", "/local/sub/prefix_"); TVFTableSink sink = new TVFTableSink(new PlanNodeId(0), "hdfs", properties, Collections.emptyList()); Assertions.assertThrows(AnalysisException.class, sink::bindDataSink); @@ -448,57 +455,65 @@ void viewFsChecksSelectedTargetsAndPreservesRemoteMounts() throws Exception { properties.remove("fs.viewfs.mounttable.cluster.link./local"); properties.put("fs.viewfs.mounttable.cluster.link./local/", "file:///tmp"); Assertions.assertThrows(AnalysisException.class, - () -> LocalTableValuedFunction.checkHdfsSinkEnabled("/local/sub/prefix_", properties)); + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, + "/local/sub/prefix_", properties)); properties.remove("fs.viewfs.mounttable.cluster.link./local/"); properties.put("fs.viewfs.mounttable.cluster.link./local", "file:///tmp"); properties.put("fs.viewfs.mounttable.cluster.link./local/remote", "hdfs://namenode:8020/base"); - Assertions.assertDoesNotThrow(() -> LocalTableValuedFunction.checkHdfsSinkEnabled( + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, "/local/remote/prefix_", properties)); properties.put("fs.defaultFS", "viewfs://cl%75ster"); Assertions.assertThrows(AnalysisException.class, - () -> LocalTableValuedFunction.checkHdfsSinkEnabled("/local/sub/prefix_", properties)); + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, + "/local/sub/prefix_", properties)); properties.put("fs.defaultFS", "viewfs://cluster_name"); properties.put("fs.viewfs.ignore.port.in.mount.table.name", "true"); properties.put("fs.viewfs.mounttable.cluster_name.link./remote", "hdfs://namenode:8020/base"); - Assertions.assertDoesNotThrow(() -> LocalTableValuedFunction.checkHdfsSinkEnabled( + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, "/remote/prefix_", properties)); properties.put("fs.defaultFS", "viewfs://cluster:8020"); properties.put("fs.viewfs.ignore.port.in.mount.table.name", "true"); Assertions.assertThrows(AnalysisException.class, - () -> LocalTableValuedFunction.checkHdfsSinkEnabled("/local/sub/prefix_", properties)); + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, + "/local/sub/prefix_", properties)); properties.clear(); properties.put("fs.defaultFS", "viewfs:///"); properties.put("fs.viewfs.mounttable.default.name.key", "named"); properties.put("fs.viewfs.mounttable.named.link./local", "file:///tmp"); Assertions.assertThrows(AnalysisException.class, - () -> LocalTableValuedFunction.checkHdfsSinkEnabled("/local/sub/prefix_", properties)); + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, + "/local/sub/prefix_", properties)); properties.clear(); properties.put("fs.defaultFS", "viewfs://cluster"); properties.put("fs.viewfs.mounttable.cluster.linkFallback", "file:///tmp"); Assertions.assertThrows(AnalysisException.class, - () -> LocalTableValuedFunction.checkHdfsSinkEnabled("/unmapped/prefix_", properties)); + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, + "/unmapped/prefix_", properties)); properties.remove("fs.viewfs.mounttable.cluster.linkFallback"); for (String key : Arrays.asList("linkMerge./local", "linkNfly.minReplication=2,repairOnRead=true./local", "linkMergeSlash")) { String mountKey = "fs.viewfs.mounttable.cluster." + key; properties.put(mountKey, "hdfs://namenode:8020/base,file:///tmp"); Assertions.assertThrows(AnalysisException.class, - () -> LocalTableValuedFunction.checkHdfsSinkEnabled("/local/sub/prefix_", properties)); + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, + "/local/sub/prefix_", properties)); properties.put(mountKey, "hdfs://namenode:8020/base,hdfs://other:8020/base"); - Assertions.assertDoesNotThrow(() -> LocalTableValuedFunction.checkHdfsSinkEnabled( + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, "/local/sub/prefix_", properties)); properties.remove(mountKey); } properties.put("fs.viewfs.mounttable.cluster.link./local", "viewfs://inner/root"); properties.put("fs.viewfs.mounttable.inner.link./root", "file:///tmp"); Assertions.assertThrows(AnalysisException.class, - () -> LocalTableValuedFunction.checkHdfsSinkEnabled("/local/sub/prefix_", properties)); + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, + "/local/sub/prefix_", properties)); properties.put("fs.viewfs.mounttable.inner.link./root", "hdfs://namenode:8020/base"); - Assertions.assertDoesNotThrow(() -> LocalTableValuedFunction.checkHdfsSinkEnabled( + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, "/local/sub/prefix_", properties)); properties.put("fs.viewfs.mounttable.inner.link./root", "viewfs://cluster/local"); Assertions.assertThrows(AnalysisException.class, - () -> LocalTableValuedFunction.checkHdfsSinkEnabled("/local/sub/prefix_", properties)); + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, + "/local/sub/prefix_", properties)); properties.clear(); properties.put("fs.defaultFS", "viewfs://A"); properties.put("fs.viewfs.mounttable.A.link./a", "viewfs://B/base"); @@ -506,21 +521,63 @@ void viewFsChecksSelectedTargetsAndPreservesRemoteMounts() throws Exception { properties.put("fs.viewfs.mounttable.B.link./base/d", "file:///tmp"); properties.put("fs.viewfs.mounttable.B.link./base/b/d", "hdfs://namenode:8020/remote"); properties.put("fs.viewfs.mounttable.B.link./base/z", "hdfs://namenode:8020/remote"); - Assertions.assertDoesNotThrow(() -> LocalTableValuedFunction.checkHdfsSinkEnabled( + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, "/a/z/sub/prefix_", properties)); Assertions.assertThrows(AnalysisException.class, - () -> LocalTableValuedFunction.checkHdfsSinkEnabled("/a/b/d/sub/prefix_", properties)); + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, + "/a/b/d/sub/prefix_", properties)); properties.clear(); properties.put("fs.defaultFS", "hdfs://cluster:8020"); properties.put("fs.hdfs.impl", "org.apache.hadoop.fs.viewfs.ViewFileSystemOverloadScheme"); properties.put("fs.viewfs.mounttable.cluster:8020.link./local", "hdfs://namenode:8020/base"); properties.put("fs.viewfs.mounttable.cluster.link./local", "file:///tmp"); Assertions.assertThrows(AnalysisException.class, - () -> LocalTableValuedFunction.checkHdfsSinkEnabled("/local/sub/prefix_", properties)); + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, + "/local/sub/prefix_", properties)); properties.clear(); properties.put("fs.defaultFS", "viewfs://inherited"); Assertions.assertThrows(AnalysisException.class, - () -> LocalTableValuedFunction.checkHdfsSinkEnabled("/unknown/prefix_", properties)); + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, "/unknown/prefix_", properties)); + } + + @Test + void explicitRemoteViewFsResourceIsNormalizedBeforeAdmission() throws Exception { + String originalConfigDir = Config.hadoop_config_dir; + String originalConfigProperty = System.getProperty("doris.hadoop.config.dir"); + Path directory = Files.createTempDirectory("tvf-remote-viewfs-"); + Path resource = directory.resolve("remote.xml"); + try { + Files.write(resource, ("" + + "fs.viewfs.mounttable.cluster.link./data" + + "hdfs://remote/base") + .getBytes(StandardCharsets.UTF_8)); + Config.hadoop_config_dir = directory.toString() + "/"; + String statement = "INSERT INTO hdfs('file_path'='/data/sub/prefix_', 'format'='csv'," + + "'fs.defaultFS'='viewfs://cluster', 'hadoop.config.resources'='remote.xml') SELECT 1"; + try (MockedStatic factory = Mockito.mockStatic(FileSystemFactory.class)) { + for (String prefix : Arrays.asList("", "EXPLAIN ")) { + Assertions.assertInstanceOf(LogicalTVFTableSink.class, analyzeInsert(prefix + statement)); + } + Map properties = new HashMap<>(); + properties.put("file_path", "/data/sub/prefix_"); + properties.put("format", "csv"); + properties.put("fs.defaultFS", "viewfs://cluster"); + properties.put("hadoop.config.resources", "remote.xml"); + TVFTableSink sink = new TVFTableSink(new PlanNodeId(0), "hdfs", properties, + Collections.emptyList()); + Assertions.assertDoesNotThrow(sink::bindDataSink); + factory.verifyNoInteractions(); + } + } finally { + Config.hadoop_config_dir = originalConfigDir; + if (originalConfigProperty == null) { + System.clearProperty("doris.hadoop.config.dir"); + } else { + System.setProperty("doris.hadoop.config.dir", originalConfigProperty); + } + Files.deleteIfExists(resource); + Files.deleteIfExists(directory); + } } @Test @@ -593,6 +650,56 @@ void loadUserCannotExecuteViewFsLocalWritesOrCleanup() { } } + @Test + void nativeUriPrefixesCannotSelectAnUncheckedLocalMount() throws Exception { + String properties = "'file_path'='hdfs://unrelated/remote/prefix_', 'format'='csv'," + + "'fs.defaultFS'='viewfs://cluster'," + + "'fs.viewfs.mounttable.cluster.link./remote'='hdfs://remote/base'," + + "'fs.viewfs.mounttable.cluster.link./hdfs:'='file:///tmp'"; + for (String prefix : Arrays.asList("", "EXPLAIN ")) { + Assertions.assertThrows(AnalysisException.class, + () -> analyzeInsert(prefix + "INSERT INTO hdfs(" + properties + ") SELECT 1")); + } + Map props = new HashMap<>(); + props.put("file_path", "hdfs://unrelated/remote/prefix_"); + props.put("format", "csv"); + props.put("fs.defaultFS", "viewfs://cluster"); + props.put("fs.viewfs.mounttable.cluster.link./remote", "hdfs://remote/base"); + props.put("fs.viewfs.mounttable.cluster.link./hdfs:", "file:///tmp"); + TVFTableSink sink = new TVFTableSink(new PlanNodeId(0), "hdfs", props, Collections.emptyList()); + Assertions.assertThrows(AnalysisException.class, sink::bindDataSink); + } + + @Test + void relativeViewFsCleanupRejectsBeforeFilesystemCreation() throws Exception { + Map properties = new HashMap<>(); + properties.put("file_path", "data/sub/prefix_"); + properties.put("format", "csv"); + properties.put("fs.hdfs.support", "true"); + properties.put("fs.defaultFS", "viewfs://cluster"); + properties.put("hadoop.username", "tvf"); + properties.put("fs.viewfs.mounttable.cluster.link./data", "hdfs://remote/base"); + properties.put("fs.viewfs.mounttable.cluster.link./user/tvf/data", "file:///tmp"); + InsertIntoTVFCommand command = (InsertIntoTVFCommand) new NereidsParser().parseSingle( + "INSERT INTO hdfs('file_path'='data/sub/prefix_', 'format'='csv') SELECT 1"); + Method cleanup = InsertIntoTVFCommand.class.getDeclaredMethod( + "deleteExistingFilesInFE", String.class, Map.class); + cleanup.setAccessible(true); + try (MockedStatic factory = Mockito.mockStatic(FileSystemFactory.class)) { + InvocationTargetException exception = Assertions.assertThrows(InvocationTargetException.class, + () -> cleanup.invoke(command, "hdfs", properties)); + Assertions.assertInstanceOf(AnalysisException.class, exception.getCause()); + factory.verifyNoInteractions(); + properties.put("file_path", " /local/prefix_"); + properties.put("fs.viewfs.mounttable.cluster.link./local", "file:///tmp"); + properties.put("fs.viewfs.mounttable.cluster.linkFallback", "hdfs://remote/base"); + exception = Assertions.assertThrows(InvocationTargetException.class, + () -> cleanup.invoke(command, "hdfs", properties)); + Assertions.assertInstanceOf(AnalysisException.class, exception.getCause()); + factory.verifyNoInteractions(); + } + } + private Plan analyzeInsert(String sql) { ConnectContext context = MemoTestUtils.createConnectContext(); LogicalPlan parsed = new NereidsParser().parseSingle(sql); diff --git a/fe/fe-core/src/test/java/org/apache/doris/tablefunction/HadoopTvfAdmissionTest.java b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/HadoopTvfAdmissionTest.java new file mode 100644 index 00000000000000..9205ec4a26d46a --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/HadoopTvfAdmissionTest.java @@ -0,0 +1,301 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.tablefunction; + +import org.apache.doris.common.Config; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.tablefunction.HadoopTvfAdmission.Access; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.HashMap; +import java.util.Map; + +class HadoopTvfAdmissionTest { + private boolean originalEnable; + + @BeforeEach + void setUp() { + originalEnable = Config.enable_local_tvf; + Config.enable_local_tvf = false; + } + + @AfterEach + void tearDown() { + Config.enable_local_tvf = originalEnable; + } + + @Test + void javaMountKeysPreserveNulWhileNativeKeysAreCStringPrefixes() { + Map properties = viewFs(); + properties.put("fs.viewfs.mounttable.cluster.link./data\0ignored", "hdfs://remote/base"); + properties.put("fs.viewfs.mounttable.cluster.linkFallback", "file:///tmp"); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, "/data/sub/prefix_", properties)); + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + Access.JAVA_DELETE, "/data/sub/", properties)); + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + Access.HADOOP_READ, "viewfs://cluster/data/sub/file.csv", properties)); + } + + @Test + void javaImplementationKeysAreNotTruncatedAtNul() { + Map properties = new HashMap<>(); + properties.put("fs.defaultFS", "hdfs://remote"); + properties.put("fs.hdfs.impl\0ignored", "org.apache.hadoop.fs.LocalFileSystem\0ignored"); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.JAVA_DELETE, "/data/sub/", properties)); + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, "/data/sub/prefix_", properties)); + } + + @Test + void nativeAuthorityCompletionUsesTheRemainingLegacyConfiguration() { + Map properties = new HashMap<>(); + properties.put("fs.defaultFS", "viewfs:///"); + properties.put("fs.default.name", "viewfs://local"); + properties.put("fs.viewfs.mounttable.default.link./data", "hdfs://remote/base"); + properties.put("fs.viewfs.mounttable.local.link./data", "file:///tmp"); + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, "/data/sub/prefix_", properties)); + properties.put("fs.viewfs.mounttable.local.link./data", "hdfs://remote/base"); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, "/data/sub/prefix_", properties)); + } + + @Test + void distributedViewWrappersCannotBeMistakenForDirectRemoteProviders() { + Map properties = new HashMap<>(); + properties.put("fs.defaultFS", "hdfs://cluster"); + properties.put("fs.hdfs.impl", "org.apache.hadoop.hdfs.ViewDistributedFileSystem"); + properties.put("fs.viewfs.mounttable.cluster.link./data", "file:///tmp"); + for (Access access : Access.values()) { + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + access, "hdfs://cluster/data/sub/file.csv", properties)); + } + Config.enable_local_tvf = true; + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, "/data/sub/prefix_", properties)); + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + Access.HADOOP_READ, "hdfs://cluster/data/sub/file.csv", properties)); + } + + @Test + void hadoopReadsRejectLocalDelegationEvenWithLocalOptIn() { + Map properties = viewFs(); + properties.put("fs.viewfs.mounttable.cluster.link./data", "file:///tmp"); + for (boolean enabled : new boolean[] {false, true}) { + Config.enable_local_tvf = enabled; + AnalysisException error = Assertions.assertThrows(AnalysisException.class, + () -> HadoopTvfAdmission.check(Access.HADOOP_READ, + "viewfs://cluster/data/sub/file.csv", properties)); + Assertions.assertTrue(error.getMessage().contains("Use local()")); + } + } + + @Test + void remoteReadsDoNotRejectAnUnrelatedLocalSiblingMount() { + Map properties = viewFs(); + properties.put("fs.viewfs.mounttable.cluster.link./data", "hdfs://remote/base"); + properties.put("fs.viewfs.mounttable.cluster.link./local", "file:///tmp"); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.HADOOP_READ, "viewfs://cluster/data/file.csv", properties)); + } + + @Test + void readPatternsCheckEveryIntersectingMountBeforeListing() { + Map properties = viewFs(); + properties.put("fs.viewfs.mounttable.cluster.link./data", "hdfs://remote/base"); + properties.put("fs.viewfs.mounttable.cluster.link./data/local", "file:///tmp"); + for (String pattern : new String[] {"/data/*", "/data/?/file.csv", "/data/{a,b}/file.csv", "/data"}) { + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + Access.HADOOP_READ, "viewfs://cluster" + pattern, properties)); + } + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.HADOOP_READ, "viewfs://cluster/data/remote/file.csv", properties)); + properties.put("fs.viewfs.mounttable.cluster.link./data/local", "hdfs://remote/other"); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.HADOOP_READ, "viewfs://cluster/data/*", properties)); + } + + @Test + void schemaAndDataReadNameNodesAreCheckedIndependently() { + Map properties = new HashMap<>(); + properties.put("fs.defaultFS", "viewfs://schema"); + properties.put("fs.viewfs.mounttable.schema.link./viewfs:", "file:///tmp"); + properties.put("fs.viewfs.mounttable.data.link./data", "hdfs://remote/base"); + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + Access.HADOOP_READ, "viewfs://data/data/file.csv", properties)); + properties.put("fs.viewfs.mounttable.schema.link./viewfs:", "hdfs://remote/schema"); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.HADOOP_READ, "viewfs://data/data/file.csv", properties)); + } + + @Test + void unknownImplementationsAndRuntimeMountsNeedExplicitWriteOptIn() { + Map properties = new HashMap<>(); + properties.put("fs.defaultFS", "hdfs://remote"); + properties.put("fs.hdfs.impl", "example.DynamicFileSystem"); + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, "/data/prefix_", properties)); + Config.enable_local_tvf = true; + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, "/data/prefix_", properties)); + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + Access.HADOOP_READ, "hdfs://remote/data/file.csv", properties)); + } + + @Test + void nativeProjectionRejectsConflictingCStringKeys() { + Map properties = viewFs(); + properties.put("fs.viewfs.mounttable.cluster.link./data", "hdfs://remote/base"); + properties.put("fs.viewfs.mounttable.cluster.link./data\0ignored", "file:///tmp"); + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, "/data/sub/prefix_", properties)); + } + + @Test + void nativeNameNodeCaseCollisionsCannotChooseAnUncheckedProvider() { + Map properties = new HashMap<>(); + properties.put("fs.defaultFS", "hdfs://remote"); + properties.put("fs.DEFAULTFS", "file:///"); + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, "/data/prefix_", properties)); + } + + @Test + void nestedRemoteReadPatternsReuseTheSameMountInterpreter() { + Map properties = viewFs(); + properties.put("fs.viewfs.mounttable.cluster.link./data", "viewfs://inner/base"); + properties.put("fs.viewfs.mounttable.inner.link./base", "hdfs://remote/base"); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.HADOOP_READ, "viewfs://cluster/data/*", properties)); + properties.put("fs.viewfs.mounttable.inner.link./base/private", "file:///tmp"); + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + Access.HADOOP_READ, "viewfs://cluster/data/*", properties)); + } + + @Test + void emptyNativeWriteNameNodeKeepsExistingAnalysisWithoutSelectingLocal() { + Map properties = new HashMap<>(); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, "remote/data", properties)); + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + Access.HADOOP_READ, "hdfs://remote/data", properties)); + properties.put("fs.defaultFS", "default"); + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, "remote/data", properties)); + properties.put("fs.defaultFS", "\0ignored"); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, "remote/data", properties)); + } + + @Test + void builtInRemoteProviderImplementationsRemainAvailable() { + for (String[] provider : new String[][] { + {"oss", "com.aliyun.jindodata.oss.JindoOssFileSystem"}, + {"obs", "org.apache.hadoop.fs.obs.OBSFileSystem"}}) { + Map properties = new HashMap<>(); + properties.put("fs.defaultFS", provider[0] + "://bucket"); + properties.put("fs." + provider[0] + ".impl", provider[1]); + for (Access access : Access.values()) { + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + access, provider[0] + "://bucket/data/file.csv", properties)); + } + } + } + + @Test + void nativePathsAreRootedBeforeHadoopUriParsing() { + Map properties = viewFs(); + properties.put("fs.viewfs.mounttable.cluster.link./remote", "hdfs://remote/base"); + properties.put("fs.viewfs.mounttable.cluster.link./hdfs:", "file:///tmp"); + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, "hdfs://unrelated/remote/prefix_", properties)); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, "remote/prefix_", properties)); + + properties.put("fs.defaultFS", "viewfs://schema"); + properties.put("fs.viewfs.mounttable.data.link./remote", "hdfs://remote/base"); + properties.put("fs.viewfs.mounttable.schema.link./remote", "hdfs://remote/base"); + properties.put("fs.viewfs.mounttable.schema.link./viewfs:", "file:///tmp"); + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + Access.HADOOP_READ, "viewfs://data/remote/file.csv", properties)); + properties.put("fs.viewfs.mounttable.schema.link./viewfs:", "hdfs://remote/schema"); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.HADOOP_READ, "viewfs://data/remote/file.csv", properties)); + } + + @Test + void nonPrefixNameNodeMatchesRequireAnEstablishedNativePath() { + String nameNode = "viewfs://clüster"; + Map properties = new HashMap<>(); + properties.put("fs.defaultFS", nameNode); + properties.put("fs.viewfs.mounttable.clüster.link./x", "hdfs://remote/base"); + properties.put("fs.viewfs.mounttable.clüster.link./local", "file:///tmp"); + String path = "abcdefghijklmnopx/local/prefix_" + nameNode + "/a"; + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, path, properties)); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, nameNode + "/x/prefix_", properties)); + Config.enable_local_tvf = true; + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, path, properties)); + } + + @Test + void relativeJavaViewFsPathsCannotAssumeTheFilesystemRoot() { + Map properties = viewFs(); + properties.put("fs.viewfs.mounttable.cluster.link./data", "hdfs://remote/base"); + properties.put("fs.viewfs.mounttable.cluster.link./user/tvf/data", "file:///tmp"); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, "data/sub/prefix_", properties)); + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + Access.JAVA_DELETE, "data/sub/", properties)); + properties.put("fs.defaultFS", "hdfs://remote"); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.JAVA_DELETE, "data/sub/", properties)); + properties.put("fs.defaultFS", "viewfs://cluster"); + Config.enable_local_tvf = true; + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.JAVA_DELETE, "data/sub/", properties)); + } + + @Test + void javaListingChecksTheTrimmedConsumerPath() { + Map properties = viewFs(); + properties.put("fs.viewfs.mounttable.cluster.link./data", "file:///tmp"); + properties.put("fs.viewfs.mounttable.cluster.link./data/file.csv /", "hdfs://remote/base"); + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + Access.HADOOP_READ, "viewfs://cluster/data/file.csv ", properties)); + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + Access.JAVA_DELETE, " viewfs://cluster/data/sub/", properties)); + properties.put("fs.viewfs.mounttable.cluster.link./data", "hdfs://remote/base"); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.HADOOP_READ, "viewfs://cluster/data/file.csv ", properties)); + } + + private static Map viewFs() { + Map properties = new HashMap<>(); + properties.put("fs.defaultFS", "viewfs://cluster"); + return properties; + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/tablefunction/HadoopTvfReadAdmissionTest.java b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/HadoopTvfReadAdmissionTest.java new file mode 100644 index 00000000000000..ee4ecff9a9a306 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/HadoopTvfReadAdmissionTest.java @@ -0,0 +1,145 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.tablefunction; + +import org.apache.doris.common.Config; +import org.apache.doris.common.FeConstants; +import org.apache.doris.fs.FileSystemFactory; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.thrift.TBrokerFileStatus; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +import java.util.HashMap; +import java.util.Map; + +class HadoopTvfReadAdmissionTest { + private boolean originalEnable; + private boolean originalUnitTest; + + @BeforeEach + void setUp() { + originalEnable = Config.enable_local_tvf; + originalUnitTest = FeConstants.runningUnitTest; + FeConstants.runningUnitTest = false; + } + + @AfterEach + void tearDown() { + Config.enable_local_tvf = originalEnable; + FeConstants.runningUnitTest = originalUnitTest; + } + + @Test + void hdfsAndFileRejectLocalRoutesBeforeFilesystemCreation() { + Map properties = properties("local"); + for (boolean enabled : new boolean[] {false, true}) { + Config.enable_local_tvf = enabled; + try (MockedStatic factory = Mockito.mockStatic(FileSystemFactory.class)) { + Assertions.assertThrows(AnalysisException.class, + () -> new HdfsTableValuedFunction(properties)); + Assertions.assertThrows(AnalysisException.class, + () -> new FileTableValuedFunction(properties)); + factory.verifyNoInteractions(); + } + } + } + + @Test + void remoteReadAdmissionAllowsTheExistingListingPath() throws Exception { + Config.enable_local_tvf = false; + HdfsTableValuedFunction tvf = new ListedHdfsTvf(properties("remote"), false); + Assertions.assertEquals(1, tvf.getFileStatuses().size()); + Assertions.assertEquals("viewfs://review/remote/listed.csv", tvf.getFileStatuses().get(0).getPath()); + } + + @Test + void returnedFilePathsAreCheckedBeforeSchemaOrDataAccess() { + Config.enable_local_tvf = true; + Assertions.assertThrows(AnalysisException.class, + () -> new ListedHdfsTvf(properties("remote"), true)); + } + + @Test + void fileKeepsTheBuiltInOssHdfsStorageSelection() throws Exception { + FeConstants.runningUnitTest = true; + Config.enable_local_tvf = false; + Map properties = new HashMap<>(); + properties.put("uri", "oss://bucket/file.csv"); + properties.put("fs.defaultFS", "oss://bucket"); + properties.put("fs.oss-hdfs.support", "true"); + properties.put("oss.endpoint", "cn-beijing.oss-dls.aliyuncs.com"); + properties.put("oss.access_key", "test-access-key"); + properties.put("oss.secret_key", "test-secret-key"); + properties.put("format", "csv"); + properties.put("csv_schema", "id:int"); + FileTableValuedFunction file = new FileTableValuedFunction(properties); + Assertions.assertEquals(org.apache.doris.thrift.TFileType.FILE_HDFS, file.getTFileType()); + Assertions.assertEquals("com.aliyun.jindodata.oss.JindoOssFileSystem", + file.getBackendConnectProperties().get("fs.oss.impl")); + } + + @Test + void listingPathTrimmingCannotSelectAnUncheckedLocalMount() { + Config.enable_local_tvf = false; + Map properties = properties("local"); + properties.put("uri", "viewfs://review/local/file.csv "); + properties.put("fs.viewfs.mounttable.review.link./local/file.csv /", "hdfs://remote/base"); + try (MockedStatic factory = Mockito.mockStatic(FileSystemFactory.class)) { + Assertions.assertThrows(AnalysisException.class, () -> new HdfsTableValuedFunction(properties)); + Assertions.assertThrows(AnalysisException.class, () -> new FileTableValuedFunction(properties)); + factory.verifyNoInteractions(); + } + } + + private static Map properties(String mount) { + Map properties = new HashMap<>(); + properties.put("uri", "viewfs://review/" + mount + "/file.csv"); + properties.put("fs.defaultFS", "viewfs://review"); + properties.put("fs.viewfs.mounttable.review.link./remote", "hdfs://remote/base"); + properties.put("fs.viewfs.mounttable.review.link./local", "file:///tmp"); + properties.put("format", "csv"); + properties.put("csv_schema", "id:int"); + return properties; + } + + private static class ListedHdfsTvf extends HdfsTableValuedFunction { + // Constructor-time listing is the existing TVF lifecycle. Select the fake listing through + // statement properties, which are available before super() calls parseFile(). + ListedHdfsTvf(Map properties, boolean localListing) + throws org.apache.doris.common.AnalysisException { + super(withListing(properties, localListing)); + } + + private static Map withListing(Map properties, boolean localListing) { + properties.put("fs.test.listing", localListing ? "local" : "remote"); + return properties; + } + + @Override + protected void parseFile() { + String mount = backendConnectProperties.get("fs.test.listing"); + fileStatuses.add(new TBrokerFileStatus("viewfs://review/" + mount + "/listed.csv", false, 10, true)); + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/tablefunction/LocalTableValuedFunctionEnableTest.java b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/LocalTableValuedFunctionEnableTest.java index aaa96239628bf9..556ae26fed314e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/tablefunction/LocalTableValuedFunctionEnableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/LocalTableValuedFunctionEnableTest.java @@ -23,6 +23,7 @@ import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.proto.InternalService.PGlobResponse; +import org.apache.doris.proto.InternalService.PGlobResponse.PFileInfo; import org.apache.doris.proto.Types.PStatus; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.SessionVariable; @@ -117,6 +118,26 @@ public void testDisabledRejectsCsvAndLanceBeforeFileAccess() { Mockito.verifyNoInteractions(accessManager, systemInfo, proxy); } + @Test + public void testCanonicalGlobPathsKeepTheirExactWhitespace() throws Exception { + Config.enable_local_tvf = true; + Mockito.when(accessManager.checkGlobalPriv(context, PrivPredicate.ADMIN)).thenReturn(true); + String[] paths = {"/secure/name.csv ", "/secure/ name.csv", "/secure/\tname.csv\t"}; + PGlobResponse.Builder response = PGlobResponse.newBuilder() + .setStatus(PStatus.newBuilder().setStatusCode(0)); + for (String path : paths) { + response.addFiles(PFileInfo.newBuilder().setFile(path).setSize(4)); + } + Mockito.when(proxy.glob(Mockito.any(), Mockito.any())) + .thenReturn(CompletableFuture.completedFuture(response.build())); + LocalTableValuedFunction local = new LocalTableValuedFunction(PROPERTIES); + FileTableValuedFunction file = new FileTableValuedFunction(PROPERTIES); + for (int i = 0; i < paths.length; i++) { + Assertions.assertEquals(paths[i], local.getFileStatuses().get(i).getPath()); + Assertions.assertEquals(paths[i], file.getFileStatuses().get(i).getPath()); + } + } + @Test public void testEnabledLocalReachesExistingFileAccess() throws Exception { Config.enable_local_tvf = true; diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift index 1488f75db78787..1fa34d29e5a9ba 100644 --- a/gensrc/thrift/PlanNodes.thrift +++ b/gensrc/thrift/PlanNodes.thrift @@ -598,6 +598,10 @@ struct TLanceFileDesc { // so these tasks filter their rows without repeating global scalar-index evaluation. // Unset preserves Lance's default; an explicit scalar segment must not be combined with false. 6: optional bool use_scalar_index + // Independent LOCAL/S3 TVF requests enforce storage admission for manifest-selected bases. + // Unset preserves legacy/catalog routing; old LOCAL TVFs are recognized by version zero. + 7: optional bool is_tvf + } struct TLanceScanParams { diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh index a5e209a0aedb32..5b8224dca4e910 100755 --- a/thirdparty/build-thirdparty.sh +++ b/thirdparty/build-thirdparty.sh @@ -2156,6 +2156,32 @@ install_paimon_rust() { ) } +# Publish the Lance headers and archive as one recoverable installation. +install_lance_c() { + ( + set -e + local archive="$1" + local headers="$2" + local destination="${TP_INSTALL_DIR}/include/lance" + local incomplete="${TP_INSTALL_DIR}/lib64/.lance-installing" + mkdir -p "${TP_INSTALL_DIR}/include" "${TP_INSTALL_DIR}/lib64" + local staged + staged="$(mktemp -d "${TP_INSTALL_DIR}/include/.lance.tmp.XXXXXX")" + trap 'rm -rf "${staged}"' EXIT + cp -pR "${headers}/." "${staged}/" + test -s "${staged}/lance.h" + test -s "${staged}/lance.hpp" + test -s "${archive}" + # The archive and header-directory renames cannot be atomic together. + # Keep this marker after any interrupted publication until a retry repairs both. + touch "${incomplete}" + install_rust_archive "${archive}" + rm -rf "${destination}" + mv "${staged}" "${destination}" + rm -f "${incomplete}" + ) +} + build_lance_c() { check_if_source_exist "${LANCE_C_SOURCE}" cd "${TP_SOURCE_DIR}/${LANCE_C_SOURCE}" @@ -2227,10 +2253,7 @@ build_lance_c() { fi env "${cargo_env[@]}" "${cargo_bin}" "${cargo_args[@]}" - mkdir -p "${TP_INSTALL_DIR}/include" "${TP_INSTALL_DIR}/lib64" - rm -rf "${TP_INSTALL_DIR}/include/lance" - cp -av include/lance "${TP_INSTALL_DIR}/include/" - install_rust_archive "${BUILD_DIR}/release/liblance_c.a" + install_lance_c "${BUILD_DIR}/release/liblance_c.a" "${PWD}/include/lance" } # paimon-rust diff --git a/thirdparty/download-thirdparty.sh b/thirdparty/download-thirdparty.sh index 431d5a5d013443..e905249c621de6 100755 --- a/thirdparty/download-thirdparty.sh +++ b/thirdparty/download-thirdparty.sh @@ -719,9 +719,11 @@ if [[ " ${TP_ARCHIVES[*]} " =~ " AZURE " ]]; then fi # Foyer remains a local patch until its cache interface is accepted upstream. +# TVF storage-base inspection is an additive C ABI patch on the pinned SDK. # All search fixes are supplied by the immutable lance-c dependency revision. if [[ " ${TP_ARCHIVES[*]} " =~ " LANCE_C " ]]; then - foyer_patch_checksum="$(cksum < "${TP_PATCH_DIR}/lance-c-foyer.patch")" + foyer_patch_checksum="$(cat "${TP_PATCH_DIR}/lance-c-foyer.patch" \ + "${TP_PATCH_DIR}/lance-c-storage-bases.patch" | cksum)" foyer_patch_marker="${TP_SOURCE_DIR}/${LANCE_C_SOURCE}/${PATCHED_MARK}_foyer" # A new local patch must also replace previously patched cached sources. # Empty markers from older builds cannot identify the applied patch version. @@ -734,6 +736,8 @@ if [[ " ${TP_ARCHIVES[*]} " =~ " LANCE_C " ]]; then if [[ ! -f "${PATCHED_MARK}_foyer" ]]; then patch --batch --forward --reject-file=- --fuzz=0 --no-backup-if-mismatch -s \ -p1 <"${TP_PATCH_DIR}/lance-c-foyer.patch" + patch --batch --forward --reject-file=- --fuzz=0 --no-backup-if-mismatch -s \ + -p1 <"${TP_PATCH_DIR}/lance-c-storage-bases.patch" printf '%s\n' "${foyer_patch_checksum}" > "${PATCHED_MARK}_foyer" fi cd - diff --git a/thirdparty/patches/lance-c-storage-bases.patch b/thirdparty/patches/lance-c-storage-bases.patch new file mode 100644 index 00000000000000..f5747e89af46b8 --- /dev/null +++ b/thirdparty/patches/lance-c-storage-bases.patch @@ -0,0 +1,362 @@ +--- a/Cargo.toml ++++ b/Cargo.toml +@@ -51,6 +51,7 @@ + object_store = "0.14.2" + pin-project = "1.0" + prost = "0.14" ++serde_json = "1" + snafu = "0.9" + uuid = { version = "1", features = ["v4"] } + +--- a/Cargo.lock ++++ b/Cargo.lock +@@ -4442,6 +4442,7 @@ + "opendal", + "pin-project", + "prost", ++ "serde_json", + "snafu", + "tempfile", + "tokio", +--- a/include/lance/lance.h ++++ b/include/lance/lance.h +@@ -291,6 +291,31 @@ + ); + + /* ─── Dataset lifecycle ─── */ ++ ++/** Metadata-only base inspection contract; header/archive must both provide this API. */ ++#define LANCE_STORAGE_BASES_JSON_VERSION 1 ++ ++/** ++ * Return the primary and all registered bases of the opened snapshot as JSON. ++ * ++ * This inspects SDK-decoded metadata and extracts paths using the SDK provider; ++ * it never constructs a base object store or reads any base object. It includes ++ * registrations regardless of the current projection or whether they are unused. ++ * ++ * JSON: {"format_version":1,"bases":[{"id":0,"primary":false,"uri":"...","scheme":"file", ++ * "is_local":true,"local_path":"/absolute/path","is_dataset_root":true}]}. ++ * Nonlocal entries have local_path:null. local_path is already decoded exactly as ++ * the SDK reader uses it; callers MUST NOT URI-decode it again. The entries are ++ * primary is first with id:null and primary:true; registrations follow sorted ++ * by id. This does not enumerate absolute blob URIs encoded inside data ++ * values, and is not a policy hook for blob payload materialization. ++ * ++ * @return Owned UTF-8 string, to be freed with lance_free_string(), or NULL on ++ * error (use lance_last_error_code/message). Safe with shared sessions. ++ */ ++const char* lance_dataset_storage_bases_json(const LanceDataset* dataset); ++ ++ + + /** + * Open a Lance dataset. +--- a/include/lance/lance.hpp ++++ b/include/lance/lance.hpp +@@ -820,6 +820,15 @@ + handle_.get(), stream, batch_size) != 0) { + check_error(); + } ++ } ++ ++ /// Owned JSON describing the SDK-decoded primary and registered storage bases. ++ /// Metadata-only: this does not instantiate delegated object stores. ++ std::string storage_bases_json() const { ++ const char* json = lance_dataset_storage_bases_json(handle_.get()); ++ if (!json) check_error(); ++ std::unique_ptr owned(json, &lance_free_string); ++ return std::string(owned.get()); + } + + /// Export the schema as an Arrow C Data Interface struct. +--- a/src/dataset.rs ++++ b/src/dataset.rs +@@ -20,6 +20,9 @@ + use crate::runtime::block_on; + use crate::session::LanceSession; + use crate::stream_guard::guarded_ffi_stream_from_reader; ++ ++/// Version of the metadata-only storage-base inspection JSON contract. ++pub const LANCE_STORAGE_BASES_JSON_VERSION: u32 = 1; + + /// Opaque handle representing an opened Lance dataset. + pub struct LanceDataset { +@@ -219,6 +222,82 @@ + // Metadata (in-memory, sync only) + // --------------------------------------------------------------------------- + ++/// Inspect registered base metadata before any scanner/index/delegated reader runs. ++#[unsafe(no_mangle)] ++pub unsafe extern "C" fn lance_dataset_storage_bases_json( ++ dataset: *const LanceDataset, ++) -> *const c_char { ++ ffi_try!(unsafe { storage_bases_json_inner(dataset) }, null) ++} ++ ++fn storage_base_description( ++ base: &lance_table::format::BasePath, ++ registry: Arc, ++) -> Result { ++ // These are the SDK's own parse/extract operations. Do not instantiate a ++ // provider merely to discover whether the manifest selected local storage. ++ let url = lance_io::object_store::uri_to_url(&base.path)?; ++ let path = lance_io::object_store::ObjectStore::extract_path_from_uri(registry, &base.path)?; ++ // The C session uses the pinned SDK's default registry, whose local aliases ++ // all resolve to FileStoreProvider. There is no C provider-registration API. ++ let is_local = matches!(url.scheme(), "file" | "file-object-store" | "file+uring"); ++ let local_path = is_local.then(|| lance_io::local::to_local_path(&path)); ++ Ok(serde_json::json!({ ++ "id": base.id, ++ "primary": false, ++ "uri": base.path, ++ "scheme": url.scheme(), ++ "is_local": is_local, ++ "local_path": local_path, ++ "is_dataset_root": base.is_dataset_root, ++ })) ++} ++ ++unsafe fn storage_bases_json_inner(dataset: *const LanceDataset) -> Result<*const c_char> { ++ if dataset.is_null() { ++ return Err(lance_core::Error::invalid_input_source( ++ "dataset is NULL".into(), ++ )); ++ } ++ let snapshot = unsafe { &*dataset }.snapshot(); ++ let registry = snapshot.session().store_registry(); ++ let mut bases = snapshot.manifest().base_paths.values().collect::>(); ++ bases.sort_unstable_by_key(|base| base.id); ++ let mut entries = bases ++ .into_iter() ++ .map(|base| storage_base_description(base, registry.clone())) ++ .collect::>>()?; ++ // object_store(None) returns the already-opened primary Arc; it never opens ++ // a delegated base. branch_location carries the actual parsed SDK base path. ++ let primary_store = block_on(snapshot.object_store(None))?; ++ let primary_path = snapshot.branch_location().path; ++ let primary_is_local = matches!( ++ primary_store.scheme(), ++ "file" | "file-object-store" | "file+uring" ++ ); ++ entries.insert( ++ 0, ++ serde_json::json!({ ++ "id": serde_json::Value::Null, ++ "primary": true, ++ "uri": snapshot.uri(), ++ "scheme": primary_store.scheme(), ++ "is_local": primary_is_local, ++ "local_path": primary_is_local.then(|| lance_io::local::to_local_path(&primary_path)), ++ "is_dataset_root": true, ++ }), ++ ); ++ let json = serde_json::json!({ ++ "format_version": LANCE_STORAGE_BASES_JSON_VERSION, ++ "bases": entries, ++ }) ++ .to_string(); ++ let owned = std::ffi::CString::new(json).map_err(|_| { ++ lance_core::Error::invalid_input_source("storage base metadata contains NUL".into()) ++ })?; ++ Ok(owned.into_raw() as *const c_char) ++} ++ + /// Return the version number of this dataset snapshot. + /// Returns 0 on error — check `lance_last_error_code()`. + #[unsafe(no_mangle)] +@@ -588,3 +667,121 @@ + assert_eq!(block_on(snap.count_rows(None)).unwrap(), 2); + } + } ++ ++#[cfg(test)] ++mod storage_base_metadata_tests { ++ use super::*; ++ use lance_io::object_store::ObjectStoreRegistry; ++ use lance_table::format::BasePath; ++ ++ #[test] ++ fn base_inspection_uses_sdk_local_identity_without_opening_files() { ++ let registry = Arc::new(ObjectStoreRegistry::default()); ++ for (uri, expected) in [ ++ ( ++ "file:///nonexistent/sdbsec/a%20b", ++ "/nonexistent/sdbsec/a b", ++ ), ++ ( ++ "file:///nonexistent/sdbsec/a%2520b", ++ "/nonexistent/sdbsec/a%20b", ++ ), ++ ( ++ "file-object-store:///nonexistent/sdbsec/a%20b", ++ "/nonexistent/sdbsec/a b", ++ ), ++ ] { ++ let base = BasePath::new(7, uri.into(), None, false); ++ let value = storage_base_description(&base, registry.clone()).unwrap(); ++ assert_eq!(value["id"], 7); ++ assert_eq!(value["is_local"], true); ++ assert_eq!(value["local_path"], expected); ++ assert_eq!(value["is_dataset_root"], false); ++ } ++ } ++ ++ #[test] ++ fn base_inspection_preserves_remote_base_without_opening_provider() { ++ let registry = Arc::new(ObjectStoreRegistry::default()); ++ let base = BasePath::new(3, "s3://unreachable.invalid/clone-base".into(), None, true); ++ let value = storage_base_description(&base, registry).unwrap(); ++ assert_eq!(value["is_local"], false); ++ assert_eq!(value["local_path"], serde_json::Value::Null); ++ assert_eq!(value["uri"], base.path); ++ assert_eq!(value["is_dataset_root"], true); ++ } ++ ++ #[cfg(target_os = "linux")] ++ #[test] ++ fn io_uring_local_alias_is_identified_without_opening_provider() { ++ let base = BasePath::new( ++ 8, ++ "file+uring:///nonexistent/sdbsec/a%20b".into(), ++ None, ++ false, ++ ); ++ let value = ++ storage_base_description(&base, Arc::new(ObjectStoreRegistry::default())).unwrap(); ++ assert_eq!(value["is_local"], true); ++ assert_eq!(value["local_path"], "/nonexistent/sdbsec/a b"); ++ } ++ ++ #[test] ++ fn metadata_snapshot_contains_primary_and_every_registered_base() { ++ use arrow_array::{Int64Array, RecordBatch}; ++ use arrow_schema::{DataType, Field, Schema}; ++ let temp = tempfile::tempdir().unwrap(); ++ let uri = temp.path().join("primary with spaces.lance"); ++ let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)])); ++ let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(Int64Array::from(vec![1]))]) ++ .unwrap(); ++ let mut dataset = block_on(Dataset::write( ++ arrow::record_batch::RecordBatchIterator::new(vec![Ok(batch)], schema), ++ uri.to_str().unwrap(), ++ None, ++ )) ++ .unwrap(); ++ // Add unreferenced registrations in memory: listing every registered base ++ // must not instantiate the unreachable remote provider or open local data. ++ let manifest = Arc::make_mut(&mut dataset.manifest); ++ manifest.base_paths.insert( ++ 9, ++ BasePath::new(9, "s3://unreachable.invalid/base".into(), None, true), ++ ); ++ manifest.base_paths.insert( ++ 4, ++ BasePath::new( ++ 4, ++ "file:///nonexistent/sdbsec/base%20path".into(), ++ None, ++ false, ++ ), ++ ); ++ let handle = LanceDataset { ++ inner: RwLock::new(Arc::new(dataset)), ++ data_cache: None, ++ }; ++ let raw = unsafe { lance_dataset_storage_bases_json(&handle) }; ++ assert!(!raw.is_null()); ++ let value: serde_json::Value = ++ serde_json::from_str(unsafe { std::ffi::CStr::from_ptr(raw) }.to_str().unwrap()) ++ .unwrap(); ++ unsafe { crate::error::lance_free_string(raw) }; ++ assert_eq!(value["format_version"], 1); ++ let entries = value["bases"].as_array().unwrap(); ++ assert_eq!(entries.len(), 3); ++ assert_eq!(entries[0]["primary"], true); ++ assert_eq!(entries[0]["id"], serde_json::Value::Null); ++ assert_eq!(entries[0]["is_local"], true); ++ assert_eq!(entries[0]["local_path"], uri.to_str().unwrap()); ++ assert_eq!(entries[1]["id"], 4); ++ assert_eq!(entries[1]["local_path"], "/nonexistent/sdbsec/base path"); ++ assert_eq!(entries[2]["id"], 9); ++ assert_eq!(entries[2]["is_local"], false); ++ } ++ ++ #[test] ++ fn null_dataset_base_inspection_fails() { ++ assert!(unsafe { lance_dataset_storage_bases_json(std::ptr::null()) }.is_null()); ++ } ++} +--- a/tests/c_api_test.rs ++++ b/tests/c_api_test.rs +@@ -436,6 +436,70 @@ + + // Closing NULL is safe. + unsafe { lance_dataset_close(ptr::null_mut()) }; ++} ++ ++#[test] ++fn test_storage_bases_json_null_dataset_reports_error() { ++ let result = unsafe { lance_dataset_storage_bases_json(ptr::null()) }; ++ assert!(result.is_null()); ++ assert_eq!(lance_last_error_code(), LanceErrorCode::InvalidArgument); ++ let message = lance_last_error_message(); ++ assert!(!message.is_null()); ++ unsafe { lance_free_string(message) }; ++} ++ ++#[test] ++fn test_storage_bases_json_primary_snapshot_with_multiple_fragments() { ++ let tmp = tempfile::tempdir().unwrap(); ++ let uri = tmp ++ .path() ++ .join("base metadata with spaces.lance") ++ .to_str() ++ .unwrap() ++ .to_string(); ++ let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)])); ++ for fragment in 0..2 { ++ let batch = RecordBatch::try_new( ++ schema.clone(), ++ vec![Arc::new(Int32Array::from(vec![fragment]))], ++ ) ++ .unwrap(); ++ let params = lance::dataset::WriteParams { ++ mode: if fragment == 0 { ++ lance::dataset::WriteMode::Create ++ } else { ++ lance::dataset::WriteMode::Append ++ }, ++ ..Default::default() ++ }; ++ lance_c::runtime::block_on(Dataset::write( ++ arrow::record_batch::RecordBatchIterator::new(vec![Ok(batch)], schema.clone()), ++ &uri, ++ Some(params), ++ )) ++ .unwrap(); ++ } ++ let c_uri = c_str(&uri); ++ let dataset = unsafe { lance_dataset_open(c_uri.as_ptr(), ptr::null(), 0) }; ++ assert!(!dataset.is_null()); ++ assert_eq!(unsafe { lance_dataset_fragment_count(dataset) }, 2); ++ let raw = unsafe { lance_dataset_storage_bases_json(dataset) }; ++ assert!(!raw.is_null()); ++ assert_eq!(lance_last_error_code(), LanceErrorCode::Ok); ++ let value: serde_json::Value = ++ serde_json::from_str(unsafe { std::ffi::CStr::from_ptr(raw) }.to_str().unwrap()).unwrap(); ++ unsafe { lance_free_string(raw) }; ++ assert_eq!(value["format_version"], LANCE_STORAGE_BASES_JSON_VERSION); ++ let bases = value["bases"].as_array().unwrap(); ++ assert_eq!(bases.len(), 1); ++ assert_eq!(bases[0]["primary"], true); ++ assert_eq!(bases[0]["id"], serde_json::Value::Null); ++ assert_eq!(bases[0]["is_local"], true); ++ assert_eq!(bases[0]["local_path"], uri); ++ assert_eq!(bases[0]["is_dataset_root"], true); ++ // Inspection does not consume or mutate the Dataset handle. ++ assert_eq!(unsafe { lance_dataset_fragment_count(dataset) }, 2); ++ unsafe { lance_dataset_close(dataset) }; + } + + #[test] diff --git a/thirdparty/test/lance-build-test.py b/thirdparty/test/lance-build-test.py new file mode 100644 index 00000000000000..1a56c6b27b3029 --- /dev/null +++ b/thirdparty/test/lance-build-test.py @@ -0,0 +1,306 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Run with python3 thirdparty/test/lance-build-test.py; no Rust build or downloads.""" + +import os +from pathlib import Path +import re +import shutil +import subprocess +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[2] +SYMBOL = "lance_dataset_storage_bases_json" +HEADER = "#define LANCE_STORAGE_BASES_JSON_VERSION 1\n" + + +class LanceBuildTest(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.work = Path(self.temp.name) + self.prefix = self.work / "thirdparty" + self.env = dict(os.environ) + self.env.pop("NM", None) + self.env.update(DORIS_HOME=str(ROOT), DORIS_THIRDPARTY=str(self.prefix), + BUILD_BE="1", CLEAN="0", PARALLEL="2") + self.write("thirdparty/installed/lib64/unrelated.a", "keep unrelated dependency") + + def write(self, name, text): + path = self.work / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text) + return path + + def executable(self, name, text): + path = self.write(name, "#!/usr/bin/env bash\nset -e\n" + text) + path.chmod(0o755) + return path + + def run_bash(self, script): + return subprocess.run(["bash", "-eo", "pipefail", "-c", script], env=self.env, + cwd=self.work, text=True, capture_output=True) + + def archive(self, name, symbol=SYMBOL, undefined=False): + source = self.write(name + ".c", (f"extern void {symbol}(void);\n" + f"void caller(void) {{ {symbol}(); }}\n") if undefined + else f"void {symbol}(void) {{}}\n") + obj = source.with_suffix(".o") + target = source.with_suffix(".a") + subprocess.run(["cc", "-c", str(source), "-o", str(obj)], check=True, + capture_output=True) + subprocess.run(["ar", "rcs", str(target), str(obj)], check=True, capture_output=True) + return target + + def installed(self, header=HEADER, symbol=SYMBOL, undefined=False): + self.write("thirdparty/installed/include/lance/lance.h", header) + archive = self.archive("fixtures/installed", symbol, undefined) + shutil.copy2(archive, self.prefix / "installed/lib64/liblance_c.a") + + def gate(self): + script = (ROOT / "build.sh").read_text() + return script[script.index("# Lance TVF storage policy needs"): + script.index("update_submodule() {")] + + def compatible(self): + functions = self.gate().split('if [[ "${BUILD_BE}" -eq 1 ]]')[0] + return self.run_bash(functions + '\nlance_c_is_compatible "$DORIS_THIRDPARTY/installed"') + + def matching_source(self, repair=True): + self.prefix.mkdir(parents=True, exist_ok=True) + shutil.copy2(ROOT / "thirdparty/vars.sh", self.prefix / "vars.sh") + self.write("thirdparty/download-thirdparty.sh", "# lance-c-storage-bases.patch\n") + patch = self.prefix / "patches/lance-c-storage-bases.patch" + patch.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(ROOT / "thirdparty/patches/lance-c-storage-bases.patch", patch) + new_header = self.write("fixtures/new.h", HEADER) + new_archive = self.archive("fixtures/new") + self.env.update(BUILD_LOG=str(self.work / "build.log"), NEW_HEADER=str(new_header), + NEW_ARCHIVE=str(new_archive)) + script = '''install_lance_c() { :; } +printf '%s\\n' "$*" > "$BUILD_LOG" +''' + if repair: + script += '''cp "$NEW_HEADER" "$DORIS_THIRDPARTY/installed/include/lance/lance.h" +cp "$NEW_ARCHIVE" "$DORIS_THIRDPARTY/installed/lib64/liblance_c.a" +rm -f "$DORIS_THIRDPARTY/installed/lib64/.lance-installing" +''' + self.executable("thirdparty/build-thirdparty.sh", script) + + def assert_unrelated_preserved(self): + self.assertEqual((self.prefix / "installed/lib64/unrelated.a").read_text(), + "keep unrelated dependency") + + def test_complete_installed_only_pair_is_reused(self): + self.installed() + result = self.run_bash(self.gate()) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertFalse((self.work / "build.log").exists()) + self.assert_unrelated_preserved() + + def test_old_header_new_archive_and_old_archive_new_header_fail_closed(self): + for header, symbol, undefined in [("/* old header */\n", SYMBOL, False), + (HEADER, "old_lance_api", False), + (HEADER, SYMBOL, True), + (HEADER.replace("VERSION 1", "VERSION 2"), SYMBOL, False)]: + with self.subTest(header=header, symbol=symbol, undefined=undefined): + self.installed(header, symbol, undefined) + archive = self.prefix / "installed/lib64/liblance_c.a" + before = archive.read_bytes() + result = self.run_bash(self.gate()) + self.assertNotEqual(result.returncode, 0) + self.assertIn("Refresh the compilation image", result.stderr) + self.assertEqual(before, archive.read_bytes()) + self.assert_unrelated_preserved() + + def test_incomplete_installed_pair_is_not_reused(self): + for defect in ("marker", "missing_header", "empty_header", "empty_archive"): + with self.subTest(defect=defect): + self.installed() + marker = self.prefix / "installed/lib64/.lance-installing" + marker.unlink(missing_ok=True) + header = self.prefix / "installed/include/lance/lance.h" + archive = self.prefix / "installed/lib64/liblance_c.a" + if defect == "marker": + marker.touch() + elif defect == "missing_header": + header.unlink() + elif defect == "empty_header": + header.write_text("") + else: + archive.write_text("") + result = self.run_bash(self.gate()) + self.assertNotEqual(result.returncode, 0) + self.assertIn("header/archive pair", result.stderr) + self.assertEqual(marker.exists(), defect == "marker") + self.assert_unrelated_preserved() + + def test_fe_only_does_not_check_the_new_lance_abi(self): + self.installed("/* old header */\n", "old_lance_api") + self.env.update(BUILD_BE="0", NM=str(self.executable("bin/nm", "exit 77\n"))) + result = self.run_bash(self.gate()) + self.assertEqual(result.returncode, 0, result.stderr) + self.assert_unrelated_preserved() + + def test_complete_matching_source_rebuilds_only_lance_and_repairs_marker(self): + for clean in ("0", "1"): + with self.subTest(clean=clean): + self.installed("/* old header */\n", "old_lance_api") + self.write("thirdparty/installed/lib64/.lance-installing", "") + self.matching_source() + self.env["CLEAN"] = clean + result = self.run_bash(self.gate()) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual((self.work / "build.log").read_text().strip(), "-j 2 lance_c") + self.assertFalse((self.prefix / "installed/lib64/.lance-installing").exists()) + self.assertEqual(self.compatible().returncode, 0) + self.assert_unrelated_preserved() + + def test_old_or_incomplete_source_is_not_used_for_rebuild(self): + for defect in ("patch", "pin", "download", "installer"): + with self.subTest(defect=defect): + self.installed("/* old header */\n", "old_lance_api") + self.matching_source() + (self.work / "build.log").unlink(missing_ok=True) + if defect == "patch": + self.write("thirdparty/patches/lance-c-storage-bases.patch", "old patch\n") + elif defect == "pin": + with (self.prefix / "vars.sh").open("a") as handle: + handle.write('\nLANCE_C_SOURCE="another-source"\n') + elif defect == "download": + self.write("thirdparty/download-thirdparty.sh", "# old download recipe\n") + else: + self.write("thirdparty/build-thirdparty.sh", "# old installer\n") + result = self.run_bash(self.gate()) + self.assertNotEqual(result.returncode, 0) + self.assertIn("matching complete third-party sources", result.stderr) + self.assertFalse((self.work / "build.log").exists()) + self.assert_unrelated_preserved() + + def test_rebuild_failure_or_incompatible_output_does_not_continue(self): + for mode in ("failure", "old_output"): + with self.subTest(mode=mode): + self.installed("/* old header */\n", "old_lance_api") + self.matching_source(repair=False) + if mode == "failure": + with (self.prefix / "build-thirdparty.sh").open("a") as handle: + handle.write("exit 23\n") + result = self.run_bash(self.gate()) + self.assertNotEqual(result.returncode, 0) + if mode == "old_output": + self.assertIn("Rebuilt Lance SDK header/archive pair", result.stderr) + self.assert_unrelated_preserved() + + def test_nm_symbol_spelling_definition_and_stream_completion(self): + self.installed() + cases = [ + ("gnu", f"printf '00000000 T {SYMBOL}\\n'\n", True), + ("darwin", f"printf '00000000 T _{SYMBOL}\\n'\n", True), + ("weak", f"printf '00000000 W {SYMBOL}\\n'\n", True), + ("undefined", f"printf ' U {SYMBOL}\\n'\n", False), + ("darwin-undefined", f"printf ' U _{SYMBOL}\\n'\n", False), + ("nm-failure", f"printf '00000000 T {SYMBOL}\\n'; exit 2\n", False), + ("long", f"printf '\\nmember.o:\\n00000000 T {SYMBOL}\\n'\n" + "for ((i=0; i<20000; i++)); do printf '00000000 T other_symbol_%s\\n' \"$i\"; done\n", True), + ] + for name, script, expected in cases: + with self.subTest(name=name): + self.env["NM"] = str(self.executable("bin/nm", script)) + result = self.compatible() + self.assertEqual(result.returncode == 0, expected, result.stderr) + + def installer(self): + script = (ROOT / "thirdparty/build-thirdparty.sh").read_text() + return "\n".join(re.search(r"^" + name + r"\(\) \{\n.*?^\}", script, re.M | re.S)[0] + for name in ("install_rust_archive", "install_lance_c")) + + def test_header_archive_pair_publication_and_retry(self): + archive = self.write("build/liblance_c.a", "new archive") + headers = self.work / "build/include/lance" + self.write("build/include/lance/lance.h", "new C header") + self.write("build/include/lance/lance.hpp", "new C++ header") + self.env.update(TP_INSTALL_DIR=str(self.prefix / "installed"), STRIP_TP_LIB="ON", + KERNEL="Linux", ARCHIVE=str(archive), HEADERS=str(headers)) + target_archive = self.prefix / "installed/lib64/liblance_c.a" + target_headers = self.prefix / "installed/include/lance" + marker = self.prefix / "installed/lib64/.lance-installing" + failures = { + "header-copy": 'cp() { if [[ "$1" = -pR ]]; then ' + 'printf partial > "${@: -1}/lance.h"; return 1; fi; command cp "$@"; }', + "archive-copy": 'cp() { if [[ "$1" = -p && "$2" = "$ARCHIVE" ]]; then ' + 'printf partial > "${@: -1}"; return 1; fi; command cp "$@"; }', + "archive-strip": 'strip() { printf damaged > "${@: -1}"; return 1; }', + "archive-publish": 'mv() { if [[ "${@: -1}" = */liblance_c.a ]]; then return 1; fi; ' + 'command mv "$@"; }', + "header-publish": 'mv() { if [[ "${@: -1}" = */include/lance ]]; then return 1; fi; ' + 'command mv "$@"; }', + "success": "", + } + for replacement in (False, True): + for stage, injection in failures.items(): + with self.subTest(replacement=replacement, stage=stage): + for path in (target_archive, marker): + path.unlink(missing_ok=True) + if target_headers.exists(): + shutil.rmtree(target_headers) + if replacement: + self.write("thirdparty/installed/include/lance/lance.h", "old C header") + self.write("thirdparty/installed/include/lance/lance.hpp", "old C++ header") + target_archive.write_text("old archive") + result = self.run_bash(self.installer() + '\nstrip() { :; }\n' + injection + + '\ninstall_lance_c "$ARCHIVE" "$HEADERS"') + if stage == "success": + self.assertEqual(result.returncode, 0, result.stderr) + self.assertFalse(marker.exists()) + else: + self.assertNotEqual(result.returncode, 0) + self.assertEqual(marker.exists(), stage != "header-copy") + if replacement and stage != "header-publish": + self.assertEqual((target_headers / "lance.h").read_text(), "old C header") + retry = self.run_bash(self.installer() + '\nstrip() { :; }\n' + + 'install_lance_c "$ARCHIVE" "$HEADERS"') + self.assertEqual(retry.returncode, 0, retry.stderr) + self.assertFalse(marker.exists()) + self.assertEqual((target_headers / "lance.h").read_text(), "new C header") + self.assertEqual((target_headers / "lance.hpp").read_text(), "new C++ header") + self.assertEqual(target_archive.read_text(), "new archive") + self.assertEqual(list(target_headers.parent.glob(".lance.tmp.*")), []) + self.assertEqual(list(target_archive.parent.glob("liblance_c.a.tmp.*")), []) + self.assert_unrelated_preserved() + + def test_unstaged_headers_do_not_begin_publication(self): + archive = self.write("build/liblance_c.a", "new archive") + headers = self.work / "build/include/lance" + self.write("build/include/lance/lance.h", "new C header") + self.env.update(TP_INSTALL_DIR=str(self.prefix / "installed"), STRIP_TP_LIB="OFF", + KERNEL="Linux", ARCHIVE=str(archive), HEADERS=str(headers)) + self.write("thirdparty/installed/include/lance/lance.h", "old header") + self.write("thirdparty/installed/lib64/liblance_c.a", "old archive") + result = self.run_bash(self.installer() + '\ninstall_lance_c "$ARCHIVE" "$HEADERS"') + self.assertNotEqual(result.returncode, 0) + self.assertEqual((self.prefix / "installed/include/lance/lance.h").read_text(), "old header") + self.assertEqual((self.prefix / "installed/lib64/liblance_c.a").read_text(), "old archive") + self.assertFalse((self.prefix / "installed/lib64/.lance-installing").exists()) + self.assert_unrelated_preserved() + + +if __name__ == "__main__": + unittest.main() diff --git a/thirdparty/test/lance-prefilter-patch-test.sh b/thirdparty/test/lance-prefilter-patch-test.sh index ab3587c56d1fc0..21b6e2bba6c5cd 100755 --- a/thirdparty/test/lance-prefilter-patch-test.sh +++ b/thirdparty/test/lance-prefilter-patch-test.sh @@ -75,7 +75,17 @@ check_sources() { || fail "missing upstream C++ prewarm API" grep -q 'fn test_prewarm_index_vector_segments_reuse_shared_session' "${source}/tests/c_api_test.rs" \ || fail "missing upstream shared-session prewarm regression" - [[ -f "${source}/patched_mark_foyer" ]] || fail "missing Foyer patch marker" + grep -q '^#define LANCE_STORAGE_BASES_JSON_VERSION 1$' "${source}/include/lance/lance.h" \ + || fail "missing storage-base metadata ABI version" + grep -q 'lance_dataset_storage_bases_json' "${source}/include/lance/lance.h" \ + || fail "missing storage-base C API" + grep -q 'std::string storage_bases_json() const' "${source}/include/lance/lance.hpp" \ + || fail "missing storage-base C++ API" + grep -q 'pub unsafe extern "C" fn lance_dataset_storage_bases_json' "${source}/src/dataset.rs" \ + || fail "missing storage-base implementation" + grep -q 'fn test_storage_bases_json_primary_snapshot_with_multiple_fragments' "${source}/tests/c_api_test.rs" \ + || fail "missing storage-base snapshot regression" + [[ -f "${source}/patched_mark_foyer" ]] || fail "missing Foyer/storage-base patch marker" } prepare "${tmpdir}/fresh"