diff --git a/be/cmake/thirdparty.cmake b/be/cmake/thirdparty.cmake index 19b5c1ecc1dbd6..b1be1a32fbd360 100644 --- a/be/cmake/thirdparty.cmake +++ b/be/cmake/thirdparty.cmake @@ -114,6 +114,12 @@ add_thirdparty(arrow_flight_sql LIB64) add_thirdparty(arrow_dataset LIB64) add_thirdparty(arrow_acero LIB64) add_thirdparty(parquet LIB64) +# Never consume a header/archive pair left by an interrupted Lance installation. +if (EXISTS "${THIRDPARTY_DIR}/lib64/.lance-installing") + message(FATAL_ERROR + "Lance SDK installation is incomplete. Retry thirdparty/build-thirdparty.sh lance_c " + "or refresh include/lance and lib64/liblance_c.a as a matching pair.") +endif () add_thirdparty(lance_c LIB64 NOTADD) # liblance_c.a contains compiler_builtins cbrt symbols. Place libm before it # so the final linker resolves C math symbols from the system library first. diff --git a/be/src/format_v2/table/lance_reader.cpp b/be/src/format_v2/table/lance_reader.cpp index e0e2884efe7a4a..e1f95fcd647651 100644 --- a/be/src/format_v2/table/lance_reader.cpp +++ b/be/src/format_v2/table/lance_reader.cpp @@ -22,12 +22,19 @@ #include #include #include +#include +#include +#include #include #include +#include +#include #include +#include #include #include +#include #include "common/config.h" #include "common/logging.h" @@ -35,11 +42,198 @@ #include "format_v2/lance/lance_reader_helper.h" #include "format_v2/lance/lance_runtime_filter_helper.h" #include "format_v2/lance/lance_session_manager.h" +#include "io/fs/local_file_system.h" #include "runtime/file_scan_profile.h" #include "runtime/runtime_state.h" +#ifndef LANCE_STORAGE_BASES_JSON_VERSION +#error "Rebuild lance-c with TVF storage-base metadata support before building Doris" +#endif + namespace doris::format::lance { +namespace { +std::string expand_home_path(const std::string& path) { + if (path != "~" && !path.starts_with("~/")) { + return path; + } + // Match the SDK's std::env::home_dir: HOME first, then the current user's passwd entry. + std::string home; + if (const char* value = std::getenv("HOME"); value != nullptr && value[0] != '\0') { + home = value; + } else { + std::vector buffer(1024); + struct passwd entry {}; + struct passwd* result = nullptr; + int error; + while ((error = getpwuid_r(getuid(), &entry, buffer.data(), buffer.size(), &result)) == + ERANGE) { + buffer.resize(buffer.size() * 2); + } + if (error == 0 && result != nullptr && entry.pw_dir != nullptr) { + home = entry.pw_dir; + } + } + if (home.empty()) { + return path; + } + return path == "~" ? home : (io::Path(home) / path.substr(2)).string(); +} + +Status validate_local_dataset_files(const std::string& dataset_uri) { + // Lance opens manifests and fragments itself, so a confined dataset directory is not + // enough: its descendants can redirect those opens through symlinks. + std::error_code error; + const auto secure_root = std::filesystem::canonical(config::user_files_secure_path, error); + if (error) { + return Status::InvalidArgument("local Lance dataset path is not allowed"); + } + std::vector pending {dataset_uri}; + std::unordered_set visited; + while (!pending.empty()) { + auto directory = std::move(pending.back()); + pending.pop_back(); + // Canonical directory identity bounds cycles and repeated in-root aliases without + // imposing a file-count or nesting limit on otherwise valid datasets. + if (!visited.insert(directory.string()).second) { + continue; + } + std::filesystem::directory_iterator entries(directory, error); + if (error) { + return Status::InvalidArgument("local Lance dataset path is not allowed"); + } + const std::filesystem::directory_iterator end; + while (entries != end) { + auto target = entries->path(); + auto status = entries->symlink_status(error); + if (error) { + return Status::InvalidArgument("local Lance dataset path is not allowed"); + } + if (std::filesystem::is_symlink(status)) { + target = std::filesystem::canonical(target, error); + if (error || !io::LocalFileSystem::contain_path(secure_root, target)) { + return Status::InvalidArgument("local Lance dataset path is not allowed"); + } + status = std::filesystem::status(target, error); + if (error) { + return Status::InvalidArgument("local Lance dataset path is not allowed"); + } + } + if (std::filesystem::is_directory(status)) { + pending.push_back(std::move(target)); + } + entries.increment(error); + if (error) { + return Status::InvalidArgument("local Lance dataset path is not allowed"); + } + } + } + return Status::OK(); +} + +bool is_local_lance_tvf(const TFileRangeDesc& range, const TFileScanRangeParams& scan_params) { + const bool local = range.__isset.file_type + ? range.file_type == TFileType::FILE_LOCAL + : scan_params.__isset.file_type && + scan_params.file_type == TFileType::FILE_LOCAL; + const auto& params = range.table_format_params.lance_params; + return local && (params.version == 0 || (params.__isset.is_tvf && params.is_tvf)); +} + +Status resolve_dataset_uri(const TFileRangeDesc& range, const TFileScanRangeParams& scan_params, + std::string* uri) { + const auto& requested_uri = range.table_format_params.lance_params.dataset_uri; + // Explicit TVF identity applies regardless of snapshot selection. Older LOCAL TVFs + // used version zero; configured fixed-version catalogs do not send the TVF marker. + if (is_local_lance_tvf(range, scan_params)) { + // LOCAL Lance bypasses glob because it opens a dataset directory. Apply the same + // canonical secure-root check before either schema discovery or a cached dataset open. + std::string local_input = requested_uri; + const auto colon = requested_uri.find(':'); + const auto slash = requested_uri.find('/'); + const bool bare_path = + colon == std::string::npos || (slash != std::string::npos && slash < colon); + if (bare_path && !requested_uri.empty() && requested_uri.find('\0') == std::string::npos && + !config::user_files_secure_path.empty()) { + // Lance's PathAbs fallback resolves bare dot segments before following symlinks. + // Anchor relative TVF paths to the canonical secure root, not the process cwd. + std::string secure_root; + RETURN_IF_ERROR(io::global_local_filesystem()->canonicalize( + config::user_files_secure_path, &secure_root)); + local_input = (io::Path(secure_root) / expand_home_path(requested_uri)) + .lexically_normal() + .string(); + } + RETURN_IF_ERROR(io::global_local_filesystem()->canonicalize_local_file( + config::user_files_secure_path, local_input, uri)); + return validate_local_dataset_files(*uri); + } + *uri = requested_uri; + return Status::OK(); +} +} // namespace + +LanceTableReader::TvfStoragePolicy LanceTableReader::_tvf_storage_policy( + const TFileRangeDesc& range, const TFileScanRangeParams& scan_params) { + // Positive versions are also used by S3 TVFs, so only the marker can distinguish + // those requests from configured catalogs. Legacy LOCAL TVFs keep their old restriction. + if (is_local_lance_tvf(range, scan_params)) { + return TvfStoragePolicy::LOCAL; + } + const auto& params = range.table_format_params.lance_params; + if (!params.__isset.is_tvf || !params.is_tvf) { + return TvfStoragePolicy::NONE; + } + return TvfStoragePolicy::REMOTE; +} + +Status LanceTableReader::_validate_dataset_storage(LanceDataset* dataset, TvfStoragePolicy policy) { + if (policy == TvfStoragePolicy::NONE) { + return Status::OK(); + } + // The SDK inspects the opened snapshot without constructing delegated providers. Its + // local paths already have the SDK's decoding/normalization; never URI-decode them again. + std::unique_ptr json( + lance_dataset_storage_bases_json(dataset), &lance_free_string); + if (!json) { + return Status::InvalidArgument("Lance TVF storage bases could not be validated"); + } + rapidjson::Document metadata; + metadata.Parse(json.get()); + if (metadata.HasParseError() || !metadata.IsObject() || !metadata.HasMember("format_version") || + !metadata["format_version"].IsInt() || metadata["format_version"].GetInt() != 1 || + !metadata.HasMember("bases") || !metadata["bases"].IsArray() || metadata["bases"].Empty()) { + return Status::InvalidArgument("Invalid Lance TVF storage-base metadata"); + } + for (const auto& base : metadata["bases"].GetArray()) { + if (!base.IsObject() || !base.HasMember("is_local") || !base["is_local"].IsBool() || + !base.HasMember("primary") || !base["primary"].IsBool()) { + return Status::InvalidArgument("Invalid Lance TVF storage-base metadata"); + } + const bool is_local = base["is_local"].GetBool(); + if (policy == TvfStoragePolicy::REMOTE) { + if (is_local) { + return Status::InvalidArgument("Remote Lance TVF requires remote storage bases"); + } + continue; + } + if (!is_local || !base.HasMember("local_path") || !base["local_path"].IsString()) { + return Status::InvalidArgument("LOCAL Lance TVF requires local storage bases"); + } + std::string canonical; + const std::string path(base["local_path"].GetString(), + base["local_path"].GetStringLength()); + RETURN_IF_ERROR(io::global_local_filesystem()->canonicalize_local_file( + config::user_files_secure_path, path, &canonical)); + // The primary tree was inspected before opening its manifest. Registered bases can + // live elsewhere in the secure root, so inspect their descendants before SDK reads. + if (!base["primary"].GetBool()) { + RETURN_IF_ERROR(validate_local_dataset_files(canonical)); + } + } + return Status::OK(); +} + LanceTableReader::~LanceTableReader() { static_cast(close()); } @@ -52,6 +246,8 @@ Status LanceTableReader::fetch_schema(const TFileRangeDesc& range, return Status::InvalidArgument("Lance schema output must not be null"); } const auto& params = range.table_format_params.lance_params; + std::string dataset_uri; + RETURN_IF_ERROR(resolve_dataset_uri(range, scan_params, &dataset_uri)); std::vector storage_options; RETURN_IF_ERROR(build_lance_storage_options(&scan_params, &storage_options)); std::vector storage_option_ptrs; @@ -61,14 +257,15 @@ Status LanceTableReader::fetch_schema(const TFileRangeDesc& range, } storage_option_ptrs.emplace_back(nullptr); - std::unique_ptr dataset( - lance_dataset_open(params.dataset_uri.c_str(), - storage_options.empty() ? nullptr : storage_option_ptrs.data(), - static_cast(params.version))); + std::unique_ptr dataset(lance_dataset_open( + dataset_uri.c_str(), storage_options.empty() ? nullptr : storage_option_ptrs.data(), + static_cast(params.version))); if (dataset == nullptr) { return lance_error("open Lance dataset for schema"); } + RETURN_IF_ERROR( + _validate_dataset_storage(dataset.get(), _tvf_storage_policy(range, scan_params))); std::shared_ptr schema; RETURN_IF_ERROR(import_lance_dataset_schema(dataset.get(), &schema)); return convert_arrow_schema_to_doris(schema, column_names, column_types); @@ -628,6 +825,10 @@ Status LanceTableReader::_ensure_dataset_open(const TFileRangeDesc& range, } else if (!_opened_dataset_key.has_value() || *_opened_dataset_key != key) { return Status::InvalidArgument( "Lance reader cannot mix dataset snapshots or storage options"); + } else { + // Revalidate reused handles before scanners, indexes or row-id reads. Fresh handles + // are checked inside _open_dataset before they become reader state. + RETURN_IF_ERROR(_validate_dataset_storage(_dataset, key.storage_policy)); } if (_search_kind == SearchKind::FULL_TEXT && prepare_fts_context && _fts_query_context == nullptr) { @@ -658,6 +859,8 @@ Status LanceTableReader::_open_dataset(const DatasetKey& key) { dataset.reset(raw_dataset); } + RETURN_IF_ERROR(_validate_dataset_storage(dataset.get(), key.storage_policy)); + // EXPLAIN plans a moving selector separately, so only the profile shows what this execution // read. The version is the one opened, which version 0 (the latest) resolves. _opened_version = static_cast(lance_dataset_version(dataset.get())); @@ -1329,7 +1532,8 @@ Status LanceTableReader::_fill_block_from_lance_batch(LanceBatch* batch, Block* Status LanceTableReader::_dataset_key(const TFileRangeDesc& range, DatasetKey* key) const { const auto& params = range.table_format_params.lance_params; - key->uri = params.dataset_uri; + RETURN_IF_ERROR(resolve_dataset_uri(range, *_scan_params, &key->uri)); + key->storage_policy = _tvf_storage_policy(range, *_scan_params); key->version = params.version; return build_lance_storage_options(_scan_params, &key->storage_options); } diff --git a/be/src/format_v2/table/lance_reader.h b/be/src/format_v2/table/lance_reader.h index 205c082d12052d..61b12865c0926c 100644 --- a/be/src/format_v2/table/lance_reader.h +++ b/be/src/format_v2/table/lance_reader.h @@ -65,13 +65,19 @@ class LanceTableReader final : public TableReader { Status close() override; private: + enum class TvfStoragePolicy { NONE, LOCAL, REMOTE }; + struct DatasetKey { std::string uri; int64_t version = 0; std::vector storage_options; + TvfStoragePolicy storage_policy = TvfStoragePolicy::NONE; bool operator==(const DatasetKey&) const = default; }; + static TvfStoragePolicy _tvf_storage_policy(const TFileRangeDesc& range, + const TFileScanRangeParams& scan_params); + static Status _validate_dataset_storage(LanceDataset* dataset, TvfStoragePolicy policy); Status _resolve_search_kind(); Status _validate_external_search_request() const; Status _ensure_dataset_open(const TFileRangeDesc& range, bool prepare_fts_context = true); diff --git a/be/src/io/fs/local_file_system.cpp b/be/src/io/fs/local_file_system.cpp index 4f9b6baf2a75cf..769ca84995c3c9 100644 --- a/be/src/io/fs/local_file_system.cpp +++ b/be/src/io/fs/local_file_system.cpp @@ -452,17 +452,176 @@ const std::shared_ptr& global_local_filesystem() { return local_fs; } +Status LocalFileSystem::parse_local_path(const std::string& file_path, Path& local_path) { + // Reject embedded NULs before passing paths to filesystem APIs that use C strings. + if (file_path.empty() || file_path.find('\0') != std::string::npos) { + return Status::InvalidArgument("local file path must not be empty or contain NUL"); + } + // Match URL preprocessing only for actual file URIs; bare paths keep literal whitespace. + const auto is_c0_or_space = [](unsigned char c) { return c <= 0x20; }; + size_t begin = 0; + size_t end = file_path.size(); + while (begin < end && is_c0_or_space(file_path[begin])) { + ++begin; + } + while (end > begin && is_c0_or_space(file_path[end - 1])) { + --end; + } + std::string uri_input; + for (size_t i = begin; i < end; ++i) { + if (file_path[i] != '\t' && file_path[i] != '\r' && file_path[i] != '\n') { + uri_input += file_path[i]; + } + } + const bool is_file_uri = uri_input.size() >= 5 && + (uri_input[0] == 'f' || uri_input[0] == 'F') && + (uri_input[1] == 'i' || uri_input[1] == 'I') && + (uri_input[2] == 'l' || uri_input[2] == 'L') && + (uri_input[3] == 'e' || uri_input[3] == 'E') && uri_input[4] == ':'; + std::string local_input = is_file_uri ? std::move(uri_input) : file_path; + if (is_file_uri) { + local_input.replace(0, 4, "file"); + // File URLs treat raw backslashes as separators, including before the authority. + // Do this before conversion; percent-encoded backslashes must remain literal. + for (size_t i = 5; i < local_input.size(); ++i) { + if (local_input[i] == '\\') { + local_input[i] = '/'; + } + } + } + const auto hex_digit = [](char c) -> int { + if (c >= '0' && c <= '9') return c - '0'; + if (c >= 'a' && c <= 'f') return c - 'a' + 10; + if (c >= 'A' && c <= 'F') return c - 'A' + 10; + return -1; + }; + const auto percent_decode = [&](const std::string& encoded) { + std::string decoded; + decoded.reserve(encoded.size()); + for (size_t i = 0; i < encoded.size(); ++i) { + if (encoded[i] == '%' && i + 2 < encoded.size()) { + const int high = hex_digit(encoded[i + 1]); + const int low = hex_digit(encoded[i + 2]); + if (high >= 0 && low >= 0) { + decoded += static_cast((high << 4) | low); + i += 2; + continue; + } + } + decoded += encoded[i]; + } + return decoded; + }; + const auto is_drive = [](const std::string& segment) { + return segment.size() == 2 && + ((segment[0] >= 'a' && segment[0] <= 'z') || + (segment[0] >= 'A' && segment[0] <= 'Z')) && + (segment[1] == ':' || segment[1] == '|'); + }; + bool empty_file_host = false; + if (is_file_uri && local_input.compare(5, 2, "//") == 0) { + const auto host_end = local_input.find_first_of("/?#", 7); + auto host = local_input.substr(7, host_end == std::string::npos ? host_end : host_end - 7); + if (is_drive(host)) { + // The URL parser treats a drive in the host position as the first path segment. + local_input.erase(5, 2); + } else { + host = percent_decode(host); + std::transform(host.begin(), host.end(), host.begin(), [](unsigned char c) { + return c >= 'A' && c <= 'Z' ? c + ('a' - 'A') : c; + }); + empty_file_host = host.empty() || host == "localhost"; + } + } + // Convert file: URIs to filesystem paths and reject remote schemes before any reader opens them. + RETURN_IF_ERROR(convert_to_abs_path(local_input, local_path)); + if (is_file_uri) { + // file:a%20b names "a b", but a bare path containing "%20" must stay literal. + // Decode once, preserve '+' and invalid escapes, matching Lance's file provider. + auto encoded = local_path.string(); + // Query and fragment are URI components, not part of the filesystem path. + encoded = encoded.substr(0, encoded.find_first_of("?#")); + // A file URL without a leading slash still names a filesystem-rooted path. + if (encoded.empty() || encoded[0] != '/') { + encoded.insert(0, "/"); + } + // URL dot segments are removed before symlinks are followed and before %2F + // becomes a path separator. Keep empty segments: "a//../b" denotes "a/b". + // url 2.5.8 retains an initial empty segment for empty/localhost authorities + // until after drive detection and dot reduction (file:///C| differs from file:/C|). + if (empty_file_host) { + encoded.insert(0, "/"); + } + std::vector segments; + for (size_t start = 1;;) { + const size_t end = encoded.find('/', start); + auto segment = encoded.substr(start, end == std::string::npos ? end : end - start); + const bool dot = segment == "." || segment == "%2e" || segment == "%2E"; + const bool parent = segment == ".." || segment == ".%2e" || segment == ".%2E" || + segment == "%2e." || segment == "%2E." || segment == "%2e%2e" || + segment == "%2e%2E" || segment == "%2E%2e" || segment == "%2E%2E"; + if (parent) { + if (!segments.empty() && !is_drive(segments.back())) { + segments.pop_back(); + } + } else if (!dot) { + if (segments.empty() && is_drive(segment)) { + segment[1] = ':'; + } + segments.push_back(segment); + } + if (end == std::string::npos) { + if (dot || parent) { + segments.emplace_back(""); + } + break; + } + start = end + 1; + } + size_t first = 0; + while (first < segments.size() && segments[first].empty()) { + ++first; + } + encoded = "/"; + for (size_t i = first; i < segments.size(); ++i) { + if (i > first) { + encoded += '/'; + } + encoded += segments[i]; + } + auto decoded = percent_decode(encoded); + // %00 must not truncate the path when canonicalization reaches C filesystem APIs. + if (decoded.find('\0') != std::string::npos) { + return Status::InvalidArgument("local file path must not contain NUL"); + } + // The SDK's Unix file-path conversion appends a slash after a trailing drive. + if (decoded.size() > 2 && is_drive(decoded.substr(decoded.size() - 2))) { + decoded += '/'; + } + local_path = std::move(decoded); + } + return Status::OK(); +} + Status LocalFileSystem::canonicalize_local_file(const std::string& dir, const std::string& file_path, std::string* full_path) { - const std::string absolute_path = dir + "/" + file_path; + // An empty root would turn a relative request into an unrestricted absolute path. + if (dir.empty()) { + return Status::InvalidArgument("user_files_secure_path must not be empty"); + } + Path local_path; + RETURN_IF_ERROR(parse_local_path(file_path, local_path)); + std::string secure_path; + RETURN_IF_ERROR(canonicalize(dir, &secure_path)); std::string canonical_path; - RETURN_IF_ERROR(canonicalize(absolute_path, &canonical_path)); - if (!contain_path(dir, canonical_path)) { - return Status::InvalidArgument("file path is not allowed: {}", canonical_path); + const auto status = canonicalize(Path(secure_path) / local_path, &canonical_path); + // A glob result can reveal external names even when resolution fails on a dangling symlink. + // Reject both failed resolution and out-of-root targets without exposing expanded names. + if (!status.ok() || !contain_path(secure_path, canonical_path)) { + return Status::InvalidArgument("file path is not allowed"); } - - *full_path = canonical_path; + *full_path = std::move(canonical_path); return Status::OK(); } @@ -470,13 +629,29 @@ Status LocalFileSystem::safe_glob(const std::string& path, std::vector if (path.find("..") != std::string::npos) { return Status::InvalidArgument("can not contain '..' in path"); } - std::string full_path = config::user_files_secure_path + "/" + path; + // An empty root would turn the prefix below into "/" and allow any local file. + if (config::user_files_secure_path.empty()) { + return Status::InvalidArgument("user_files_secure_path must not be empty"); + } + std::string secure_path; + RETURN_IF_ERROR(canonicalize(config::user_files_secure_path, &secure_path)); + // A symlink root may canonicalize to a directory such as "user_files[prod]". + // The root is literal; only the user-supplied suffix may contain glob operators. + std::string full_path; + for (char c : secure_path) { + if (c == '\\' || c == '*' || c == '?' || c == '[' || c == ']') { + full_path += '\\'; + } + full_path += c; + } + full_path += "/" + path; std::vector files; RETURN_IF_ERROR(_glob(full_path, &files)); for (auto& file : files) { FileInfo fi; fi.is_file = true; - RETURN_IF_ERROR(canonicalize_local_file("", file, &(fi.file_name))); + // Glob classes and symlinks can escape without a literal ".." in the pattern. + RETURN_IF_ERROR(canonicalize_local_file(secure_path, file, &(fi.file_name))); RETURN_IF_ERROR(file_size_impl(fi.file_name, &(fi.file_size))); res->push_back(std::move(fi)); } diff --git a/be/src/io/fs/local_file_system.h b/be/src/io/fs/local_file_system.h index b1546bc473426f..2b691d519a0912 100644 --- a/be/src/io/fs/local_file_system.h +++ b/be/src/io/fs/local_file_system.h @@ -70,6 +70,11 @@ class LocalFileSystem final : public FileSystem { static std::filesystem::perms PERMS_OWNER_RW; + // Parse a bare path or file URI without filesystem access, home expansion or containment checks. + static Status parse_local_path(const std::string& file_path, Path& local_path); + + // Resolve a local path relative to a nonempty secure root, then check canonical containment. + // Absolute paths and file: URIs must resolve inside the same root; remote schemes are rejected. Status canonicalize_local_file(const std::string& dir, const std::string& file_path, std::string* full_path); diff --git a/be/test/format_v2/table/lance/data/tvf-storage-fixtures.md b/be/test/format_v2/table/lance/data/tvf-storage-fixtures.md new file mode 100644 index 00000000000000..966d1bc3297189 --- /dev/null +++ b/be/test/format_v2/table/lance/data/tvf-storage-fixtures.md @@ -0,0 +1,13 @@ +The metadata-only storage-policy fixtures were generated with lance-c 4fafcd64525b274b113208390bbcca86bbbc093b and its pinned Lance SDK 68c12dfd7efe02f90ad2d7f3a239a7eb64884e57. Their fixed relative base references the existing all_types.lance fixture. No data files were copied and no manifest bytes were hand-written. Both snapshots reopen and scan row_id 1, 2, 3, 4. + +tvf_storage_clone.lance uses a dataset-root base. tvf_storage_direct_base.lance uses a data-directory base. The tests vary user_files_secure_path to admit or reject the same valid references. Run BE tests from the repository root, as required by run-be-ut.sh. + +To regenerate, copy ../generate_storage_policy_fixtures.rs into the pinned lance-c examples directory as generate_tvf_storage_clone.rs. From that SDK checkout run: + +```sh +cargo run --release --locked --example generate_tvf_storage_clone -- /absolute/doris/repository /absolute/new-output-directory +``` + +The output directory must not exist. The generator uses official shallow-clone/commit APIs, reopens both manifests, and verifies four actual rows before reporting success. Copy the resulting two dataset directories into this directory. + +The optional remote-storage test uses an explicitly selected isolated S3 fixture. Upload all_types.lance as source.lance and the metadata-only clone as clone.lance. Set DORIS_LANCE_TEST_REMOTE_SOURCE_URI, DORIS_LANCE_TEST_REMOTE_CLONE_URI, DORIS_LANCE_TEST_S3_ENDPOINT, DORIS_LANCE_TEST_S3_ACCESS_KEY and DORIS_LANCE_TEST_S3_SECRET_KEY only for that test run. The clone's relative base deliberately resolves to the local repository fixture, while the source control reads remote objects. diff --git a/be/test/format_v2/table/lance/data/tvf_storage_clone.lance/_versions/18446744073709551612.manifest b/be/test/format_v2/table/lance/data/tvf_storage_clone.lance/_versions/18446744073709551612.manifest new file mode 100644 index 00000000000000..73800825370503 Binary files /dev/null and b/be/test/format_v2/table/lance/data/tvf_storage_clone.lance/_versions/18446744073709551612.manifest differ diff --git a/be/test/format_v2/table/lance/data/tvf_storage_direct_base.lance/_versions/18446744073709551614.manifest b/be/test/format_v2/table/lance/data/tvf_storage_direct_base.lance/_versions/18446744073709551614.manifest new file mode 100644 index 00000000000000..ba1cb6d0f91a9e Binary files /dev/null and b/be/test/format_v2/table/lance/data/tvf_storage_direct_base.lance/_versions/18446744073709551614.manifest differ diff --git a/be/test/format_v2/table/lance/generate_storage_policy_fixtures.rs b/be/test/format_v2/table/lance/generate_storage_policy_fixtures.rs new file mode 100644 index 00000000000000..60cb64fecfb5d9 --- /dev/null +++ b/be/test/format_v2/table/lance/generate_storage_policy_fixtures.rs @@ -0,0 +1,172 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// SPDX-License-Identifier: Apache-2.0 +// Draft example for the pinned lance-c crate; run only after its standard build finishes. +// Creates metadata-only fixtures with official Lance APIs. Never writes source data. + +use std::error::Error; +use std::path::{Path, PathBuf}; + +use lance::Dataset; +use lance::dataset::builder::DatasetBuilder; +use lance::dataset::transaction::{Operation, Transaction}; +use lance::dataset::write::CommitBuilder; +use lance_table::format::BasePath; + +type Result = std::result::Result>; +const SOURCE_URI: &str = "be/test/format_v2/table/lance/data/all_types.lance"; + +fn main() -> Result<()> { + let args = std::env::args().skip(1).collect::>(); + if args.len() != 2 { + return Err("usage: generate_tvf_storage_clone ".into()); + } + let repository = std::fs::canonicalize(&args[0])?; + let output = PathBuf::from(&args[1]); + if !output.is_absolute() || output.exists() { + return Err( + "output must be a new absolute directory (existing content is never removed)".into(), + ); + } + std::fs::create_dir_all(&output)?; + // The committed relative base URI intentionally resolves under the test repository cwd. + std::env::set_current_dir(repository)?; + lance_c::runtime::block_on(generate(&output)) +} + +async fn row_ids(dataset: &Dataset) -> Result> { + let mut scanner = dataset.scan(); + scanner.project(&["row_id"])?; + let batch = scanner.try_into_batch().await?; + let mut values = Vec::with_capacity(batch.num_rows()); + for row in 0..batch.num_rows() { + values.push( + arrow::util::display::array_value_to_string(batch.column(0).as_ref(), row)?.parse()?, + ); + } + values.sort_unstable(); + if values != vec![1, 2, 3, 4] { + return Err(format!("expected row_id [1,2,3,4], got {values:?}").into()); + } + Ok(values) +} + +async fn publish_manifest_only(dataset: &Dataset, destination: &Path) -> Result<()> { + // The SDK reports the actual latest manifest; do not guess v1/v2 filename ordering. + let location = dataset.manifest_location(); + let manifest = PathBuf::from(lance_io::local::to_local_path(&location.path)); + let filename = manifest.file_name().ok_or("manifest has no filename")?; + let versions = destination.join("_versions"); + std::fs::create_dir_all(&versions)?; + std::fs::copy(&manifest, versions.join(filename))?; + // Deliberately copy neither data nor transactions. Reopen/scan proves that only the + // primary manifest plus the unchanged existing source fixture is sufficient. + let reopened = DatasetBuilder::from_uri(destination.to_str().ok_or("non-UTF8 destination")?) + .load() + .await?; + let values = row_ids(&reopened).await?; + let registrations = reopened + .manifest() + .base_paths + .values() + .map(|base| { + serde_json::json!({ + "id": base.id, + "uri": base.path, + "is_dataset_root": base.is_dataset_root, + }) + }) + .collect::>(); + println!( + "{}", + serde_json::json!({ + "fixture": destination, + "manifest": filename.to_string_lossy(), + "row_ids": values, + "bases": registrations, + }) + ); + Ok(()) +} + +async fn generate(output: &Path) -> Result<()> { + let mut source = DatasetBuilder::from_uri(SOURCE_URI).load().await?; + row_ids(&source).await?; + if source.uri() != SOURCE_URI { + return Err(format!("SDK changed source URI identity: {:?}", source.uri()).into()); + } + let scratch = output.join("sdk-clone-root"); + let version = source.version_id(); + let clone = source + .shallow_clone(scratch.to_str().ok_or("non-UTF8 scratch")?, version, None) + .await?; + if !clone + .manifest() + .base_paths + .values() + .any(|base| base.path == SOURCE_URI && base.is_dataset_root) + { + return Err( + "official shallow_clone did not retain the required relative source base URI".into(), + ); + } + row_ids(&clone).await?; + publish_manifest_only(&clone, &output.join("tvf_storage_clone.lance")).await?; + + // Optional direct-directory base: reuse the exact source fragments without writing files. + // Official Overwrite rejects inherited deletion files, so skip this optional fixture for + // a source that has deletions rather than changing its row semantics. + if source + .manifest() + .fragments + .iter() + .all(|fragment| fragment.deletion_file.is_none()) + { + let base_id = 7; + let mut fragments = source.manifest().fragments.as_ref().clone(); + for fragment in &mut fragments { + for file in fragment.referenced_lance_files_mut() { + file.base_id = Some(base_id); + } + } + let transaction = Transaction::new( + 0, + Operation::Overwrite { + fragments, + schema: source.schema().clone(), + config_upsert_values: None, + initial_bases: Some(vec![BasePath::new( + base_id, + format!("{SOURCE_URI}/data"), + None, + false, + )]), + }, + None, + ); + let direct_scratch = output.join("sdk-clone-direct-directory"); + let direct = CommitBuilder::new(direct_scratch.to_str().ok_or("non-UTF8 scratch")?) + .execute(transaction) + .await?; + row_ids(&direct).await?; + publish_manifest_only(&direct, &output.join("tvf_storage_direct_base.lance")).await?; + } else { + eprintln!("optional direct-directory fixture skipped: source contains deletion files"); + } + Ok(()) +} diff --git a/be/test/format_v2/table/lance_reader_test.cpp b/be/test/format_v2/table/lance_reader_test.cpp index 6531fc21de72e8..15cb01f96f93ae 100644 --- a/be/test/format_v2/table/lance_reader_test.cpp +++ b/be/test/format_v2/table/lance_reader_test.cpp @@ -36,6 +36,7 @@ #include #include #include +#include #include #include #include @@ -49,6 +50,7 @@ #include #include +#include "common/config.h" #include "common/consts.h" #include "core/assert_cast.h" #include "core/block/block.h" @@ -4313,3 +4315,680 @@ TEST(LanceTableReaderTypeTest, ReadsTemporalTypesFromAllTypesFixture) { } } // namespace doris::format::lance + +namespace doris::format::lance { +TEST(LanceTableReaderLocalPathTest, SchemaAndScanUseTheSameSecurePath) { + const auto original_secure_path = config::user_files_secure_path; + DEFER(config::user_files_secure_path = original_secure_path); + const auto root = std::filesystem::canonical("./be/test/format_v2/table/lance/data"); + config::user_files_secure_path = root.string(); + TFileScanRangeParams scan_params; + LanceTableReader reader; + reader._scan_params = &scan_params; + auto range = make_latest_lance_range("all_types.lance"); + range.__set_file_type(TFileType::FILE_LOCAL); + std::vector names; + std::vector types; + ASSERT_TRUE(reader.fetch_schema(range, scan_params, &names, &types).ok()); + EXPECT_FALSE(names.empty()); + LanceTableReader::DatasetKey key; + ASSERT_TRUE(reader._dataset_key(range, &key).ok()); + EXPECT_EQ((root / "all_types.lance").string(), key.uri); + + // Legacy scan-level tags must enforce the same restriction as per-range tags. + range.__isset.file_type = false; + scan_params.__set_file_type(TFileType::FILE_LOCAL); + ASSERT_TRUE(reader._dataset_key(range, &key).ok()); + EXPECT_EQ((root / "all_types.lance").string(), key.uri); + + for (const auto& uri : std::vector { + "https://invalid/dataset", "http://invalid/dataset", "s3://bucket/dataset", + "../data-other/dataset", "", std::string("a\0b", 3) + "/../all_types.lance"}) { + range.table_format_params.lance_params.__set_dataset_uri(uri); + const auto schema_status = reader.fetch_schema(range, scan_params, &names, &types); + const auto scan_status = reader._dataset_key(range, &key); + EXPECT_FALSE(schema_status.ok()) << uri; + EXPECT_EQ(schema_status.to_string(), scan_status.to_string()); + } +} + +TEST(LanceTableReaderLocalPathTest, ScansRelativeDatasetInsideSecureRoot) { + const auto original_secure_path = config::user_files_secure_path; + DEFER(config::user_files_secure_path = original_secure_path); + config::user_files_secure_path = + std::filesystem::canonical("./be/test/format_v2/table/lance/data").string(); + TQueryGlobals query_globals; + RuntimeState state(query_globals); + RuntimeProfile profile("lance_secure_local_scan"); + TFileScanRangeParams scan_params; + const Columns columns {projected_column("row_id", TYPE_BIGINT, false)}; + LanceTableReader reader; + ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, &scan_params).ok()); + auto range = make_latest_lance_range("all_types.lance"); + range.__set_file_type(TFileType::FILE_LOCAL); + ASSERT_TRUE(prepare_range(&reader, range).ok()); + + // Exercise data reads through the validated relative LOCAL range. + Block block; + add_output_columns(&block, columns); + std::vector row_ids; + bool eos = false; + while (!eos) { + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + if (eos) { + continue; + } + const auto& values = + assert_cast(*block.get_by_position(0).column).get_data(); + for (size_t row = 0; row < block.rows(); ++row) { + row_ids.push_back(values[row]); + } + } + std::sort(row_ids.begin(), row_ids.end()); + EXPECT_EQ((std::vector {1, 2, 3, 4}), row_ids); + ASSERT_TRUE(reader.close().ok()); +} + +TEST(LanceTableReaderLocalPathTest, FileUrisPreserveEscapedDatasetIdentity) { + const auto original_secure_path = config::user_files_secure_path; + DEFER(config::user_files_secure_path = original_secure_path); + const auto root = std::filesystem::temp_directory_path() / + ("lance_escaped_uri_" + + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count())); + DEFER({ + std::error_code error; + std::filesystem::remove_all(root, error); + }); + const auto fixture = + std::filesystem::canonical("./be/test/format_v2/table/lance/data/all_types.lance"); + for (const auto* name : {"a b.lance", "a%20b.lance", "plus+percent%.lance"}) { + std::filesystem::create_directories(root / name); + std::filesystem::copy(fixture, root / name, std::filesystem::copy_options::recursive); + } + config::user_files_secure_path = root.string(); + for (const auto& input : + {std::make_pair("file://" + (root / "a%20b.lance").string(), root / "a b.lance"), + std::make_pair(std::string("a%20b.lance"), root / "a%20b.lance"), + std::make_pair("file://" + (root / "a%20b.lance").string() + "?q=%00#tag", + root / "a b.lance"), + std::make_pair("file://" + (root / "a%2520b.lance").string(), root / "a%20b.lance"), + std::make_pair("file://" + (root / "plus+percent%25.lance").string(), + root / "plus+percent%.lance")}) { + TQueryGlobals query_globals; + RuntimeState state(query_globals); + RuntimeProfile profile("lance_escaped_file_uri"); + TFileScanRangeParams scan_params; + LanceTableReader reader; + const Columns columns = {projected_column("row_id", TYPE_BIGINT, false)}; + ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, &scan_params).ok()); + auto range = make_latest_lance_range(input.first); + range.__set_file_type(TFileType::FILE_LOCAL); + std::vector names; + std::vector types; + ASSERT_TRUE(reader.fetch_schema(range, scan_params, &names, &types).ok()) << input.first; + EXPECT_FALSE(names.empty()); + LanceTableReader::DatasetKey key; + ASSERT_TRUE(reader._dataset_key(range, &key).ok()); + EXPECT_EQ(input.second.string(), key.uri); + ASSERT_TRUE(prepare_range(&reader, range).ok()); + Block block; + add_output_columns(&block, columns); + std::vector row_ids; + bool eos = false; + while (!eos) { + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + if (!eos) { + const auto& values = + assert_cast(*block.get_by_position(0).column) + .get_data(); + for (size_t row = 0; row < block.rows(); ++row) { + row_ids.push_back(values[row]); + } + } + } + std::sort(row_ids.begin(), row_ids.end()); + EXPECT_EQ((std::vector {1, 2, 3, 4}), row_ids); + ASSERT_TRUE(reader.close().ok()); + } +} + +TEST(LanceTableReaderLocalPathTest, FileUrisNormalizeBeforeFilesystemResolution) { + const auto original_secure_path = config::user_files_secure_path; + DEFER(config::user_files_secure_path = original_secure_path); + const auto root = std::filesystem::temp_directory_path() / + ("lance_uri_normalization_" + + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count())); + DEFER({ + std::error_code error; + std::filesystem::remove_all(root, error); + }); + std::filesystem::create_directories(root / "nested/child"); + std::filesystem::create_directory_symlink(root / "nested/child", root / "jump"); + std::filesystem::create_directory_symlink(root, root / "secure-link"); + const auto backslash_uri = std::string(R"(fIlE:\\localhost)") + root.string() + "/data.lance"; + struct DatasetFixture { + const char* column_name; + int64_t row_id; + std::filesystem::path path; + }; + const DatasetFixture datasets[] = { + {"root_only", 7, root / "data.lance"}, + {"nested_only", 42, root / "nested/data.lance"}, + {"spaced_only", 99, root / "data.lance "}, + {"prefixed_only", 123, root / (root / "data.lance").relative_path()}, + {"pipe_drive_only", 151, root / "C|/data.lance"}, + {"colon_drive_only", 163, root / "C:/data.lance"}}; + for (const auto& dataset : datasets) { + std::filesystem::create_directories(dataset.path.parent_path()); + arrow::Int64Builder values_builder; + ASSERT_TRUE(values_builder.Append(dataset.row_id).ok()); + std::shared_ptr values; + ASSERT_TRUE(values_builder.Finish(&values).ok()); + const auto schema = + arrow::schema({arrow::field("row_id", arrow::int64(), false), + arrow::field(dataset.column_name, arrow::int64(), false)}); + ASSERT_TRUE(write_lance_record_batch(dataset.path, + arrow::RecordBatch::Make(schema, 1, {values, values})) + .ok()); + } + config::user_files_secure_path = (root / "secure-link").string(); + for (const auto& input : + {std::make_pair("file://" + root.string() + "/jump/../data.lance", &datasets[0]), + std::make_pair("file://" + root.string() + "/missing/../data.lance", &datasets[0]), + std::make_pair("file://" + root.string() + "/jump/%2e%2E/data.lance", &datasets[0]), + std::make_pair("FILE://" + root.string() + "/data.lance", &datasets[0]), + std::make_pair(backslash_uri, &datasets[0]), + std::make_pair("fIlE://" + root.string() + "/jump/.%2e/data.lance", &datasets[0]), + std::make_pair("file://" + root.string() + "/jump%2f../data.lance", &datasets[1]), + std::make_pair(std::string("jump/../data.lance"), &datasets[0]), + std::make_pair("file://" + root.string() + "/C|/../data.lance", &datasets[4]), + std::make_pair("file://" + root.string() + "/C:/../data.lance", &datasets[5]), + std::make_pair("file://" + root.string() + "/C%7C/../data.lance", &datasets[0]), + std::make_pair(std::string("C|/../data.lance"), &datasets[0]), + std::make_pair((root / "jump/../data.lance").string(), &datasets[0]), + std::make_pair((root / "missing/../data.lance").string(), &datasets[0]), + std::make_pair("file:" + (root / "data.lance").relative_path().string(), &datasets[0]), + std::make_pair("file://" + root.string() + "/data.lance ", &datasets[0]), + std::make_pair(" \tFi\nLe://" + root.string() + "/data.la\tnce\r\n ", &datasets[0]), + std::make_pair("file://" + root.string() + "/data.lance%20", &datasets[2]), + std::make_pair("file://" + root.string() + "/data.lance%20 ", &datasets[2]), + std::make_pair(std::string("data.lance "), &datasets[2]), + std::make_pair((root / "data.lance ").string(), &datasets[2])}) { + SCOPED_TRACE("[" + input.first + "]"); + const auto& expected = *input.second; + TQueryGlobals globals; + RuntimeState state(globals); + RuntimeProfile profile("lance_uri_normalization"); + TFileScanRangeParams scan_params; + LanceTableReader reader; + const Columns columns {projected_column("row_id", TYPE_BIGINT, false)}; + ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, &scan_params).ok()); + auto range = make_latest_lance_range(input.first); + range.__set_file_type(TFileType::FILE_LOCAL); + std::vector names; + std::vector types; + ASSERT_TRUE(reader.fetch_schema(range, scan_params, &names, &types).ok()); + EXPECT_EQ((std::vector {"row_id", expected.column_name}), names); + LanceTableReader::DatasetKey key; + ASSERT_TRUE(reader._dataset_key(range, &key).ok()); + EXPECT_EQ(expected.path.string(), key.uri); + ASSERT_TRUE(prepare_range(&reader, range).ok()); + Block block; + add_output_columns(&block, columns); + std::vector row_ids; + bool eos = false; + while (!eos) { + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + if (!eos) { + const auto& values = + assert_cast(*block.get_by_position(0).column) + .get_data(); + row_ids.insert(row_ids.end(), values.begin(), values.end()); + } + } + EXPECT_EQ((std::vector {expected.row_id}), row_ids); + ASSERT_TRUE(reader.close().ok()); + } +} + +TEST(LanceTableReaderLocalPathTest, HomeAliasesPreserveDatasetIdentityAndContainment) { + const char* home = std::getenv("HOME"); + ASSERT_NE(nullptr, home); + ASSERT_NE('\0', home[0]); + const auto home_path = std::filesystem::canonical(home); + const auto name = "lance_home_identity_" + + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count()); + const auto root = home_path / name; + ASSERT_TRUE(std::filesystem::create_directory(root)); + DEFER({ + std::error_code error; + std::filesystem::remove_all(root, error); + }); + const auto original_secure_path = config::user_files_secure_path; + DEFER(config::user_files_secure_path = original_secure_path); + struct DatasetFixture { + const char* column_name; + int64_t row_id; + std::filesystem::path path; + }; + const DatasetFixture datasets[] = {{"home_only", 17, root / "data.lance"}, + {"literal_only", 29, root / "~" / name / "data.lance"}, + {"named_only", 41, root / "~other/data.lance"}}; + for (const auto& dataset : datasets) { + std::filesystem::create_directories(dataset.path.parent_path()); + arrow::Int64Builder values_builder; + ASSERT_TRUE(values_builder.Append(dataset.row_id).ok()); + std::shared_ptr values; + ASSERT_TRUE(values_builder.Finish(&values).ok()); + const auto schema = + arrow::schema({arrow::field("row_id", arrow::int64(), false), + arrow::field(dataset.column_name, arrow::int64(), false)}); + ASSERT_TRUE(write_lance_record_batch(dataset.path, + arrow::RecordBatch::Make(schema, 1, {values, values})) + .ok()); + } + config::user_files_secure_path = root.string(); + for (const auto& input : {std::make_pair("~/" + name + "/data.lance", &datasets[0]), + std::make_pair("file://" + datasets[1].path.string(), &datasets[1]), + std::make_pair(std::string("~other/data.lance"), &datasets[2])}) { + SCOPED_TRACE(input.first); + TQueryGlobals globals; + RuntimeState state(globals); + RuntimeProfile profile("lance_home_identity"); + TFileScanRangeParams scan_params; + LanceTableReader reader; + const Columns columns {projected_column("row_id", TYPE_BIGINT, false)}; + ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, &scan_params).ok()); + auto range = make_latest_lance_range(input.first); + range.__set_file_type(TFileType::FILE_LOCAL); + std::vector names; + std::vector types; + ASSERT_TRUE(reader.fetch_schema(range, scan_params, &names, &types).ok()); + EXPECT_EQ((std::vector {"row_id", input.second->column_name}), names); + LanceTableReader::DatasetKey key; + ASSERT_TRUE(reader._dataset_key(range, &key).ok()); + EXPECT_EQ(input.second->path.string(), key.uri); + ASSERT_TRUE(prepare_range(&reader, range).ok()); + Block block; + add_output_columns(&block, columns); + std::vector row_ids; + bool eos = false; + while (!eos) { + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + if (!eos) { + const auto& values = + assert_cast(*block.get_by_position(0).column) + .get_data(); + row_ids.insert(row_ids.end(), values.begin(), values.end()); + } + } + EXPECT_EQ((std::vector {input.second->row_id}), row_ids); + ASSERT_TRUE(reader.close().ok()); + } + TFileScanRangeParams scan_params; + LanceTableReader reader; + reader._scan_params = &scan_params; + auto range = make_latest_lance_range("~"); + range.__set_file_type(TFileType::FILE_LOCAL); + LanceTableReader::DatasetKey key; + EXPECT_FALSE(reader._dataset_key(range, &key).ok()); + std::vector names; + std::vector types; + EXPECT_FALSE(reader.fetch_schema(range, scan_params, &names, &types).ok()); + // A real home dataset must still be rejected when expansion leaves the configured root. + config::user_files_secure_path = (root / "~").string(); + range.table_format_params.lance_params.__set_dataset_uri("~/" + name + "/data.lance"); + EXPECT_FALSE(reader._dataset_key(range, &key).ok()); + EXPECT_FALSE(reader.fetch_schema(range, scan_params, &names, &types).ok()); +} + +TEST(LanceTableReaderLocalPathTest, RejectsOutwardDatasetChildrenBeforeSchemaAndScan) { + const auto original_secure_path = config::user_files_secure_path; + DEFER(config::user_files_secure_path = original_secure_path); + const auto base = std::filesystem::temp_directory_path() / + ("lance_child_containment_" + + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count())); + ASSERT_TRUE(std::filesystem::create_directory(base)); + DEFER({ + std::error_code error; + std::filesystem::remove_all(base, error); + }); + const auto secure = base / "allowed"; + const auto outside = base / "private.lance"; + const auto fixture = + std::filesystem::canonical("./be/test/format_v2/table/lance/data/all_types.lance"); + std::filesystem::create_directory(secure); + std::filesystem::copy(fixture, outside, std::filesystem::copy_options::recursive); + config::user_files_secure_path = secure.string(); + for (const auto& mode : + {"directories", "data_directory", "manifest_file", "fragment_file", "deep", "dangling"}) { + SCOPED_TRACE(mode); + const auto dataset = secure / (std::string(mode) + ".lance"); + std::filesystem::copy(fixture, dataset, std::filesystem::copy_options::recursive); + if (std::string(mode) == "directories" || std::string(mode) == "data_directory") { + std::filesystem::remove_all(dataset / "data"); + std::filesystem::create_directory_symlink(outside / "data", dataset / "data"); + if (std::string(mode) == "directories") { + std::filesystem::remove_all(dataset / "_versions"); + std::filesystem::create_directory_symlink(outside / "_versions", + dataset / "_versions"); + } + } else if (std::string(mode) == "manifest_file" || std::string(mode) == "fragment_file") { + const auto directory = std::string(mode) == "manifest_file" ? "_versions" : "data"; + std::vector files; + for (const auto& entry : std::filesystem::directory_iterator(dataset / directory)) { + const auto extension = + std::string(mode) == "manifest_file" ? ".manifest" : ".lance"; + if (entry.is_regular_file() && entry.path().extension() == extension) { + files.push_back(entry.path()); + } + } + ASSERT_FALSE(files.empty()); + // This fixture uses v2 reversed manifest names; the first is the latest snapshot. + std::sort(files.begin(), files.end()); + const auto file = files.front(); + std::filesystem::remove(file); + std::filesystem::create_symlink(outside / directory / file.filename(), file); + } else if (std::string(mode) == "deep") { + std::filesystem::create_directories(dataset / "data/nested"); + std::filesystem::create_directory_symlink(outside / "data", + dataset / "data/nested/external"); + } else { + std::filesystem::remove_all(dataset / "_versions"); + std::filesystem::create_directory_symlink(base / "missing", dataset / "_versions"); + } + TQueryGlobals globals; + RuntimeState state(globals); + RuntimeProfile profile("lance_child_containment"); + TFileScanRangeParams scan_params; + LanceTableReader reader; + const Columns columns {projected_column("row_id", TYPE_BIGINT, false)}; + ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, &scan_params).ok()); + auto range = make_latest_lance_range(dataset); + range.__set_file_type(TFileType::FILE_LOCAL); + std::vector names; + std::vector types; + const auto schema = reader.fetch_schema(range, scan_params, &names, &types); + EXPECT_FALSE(schema.ok()); + EXPECT_EQ(std::string::npos, schema.to_string().find(outside.string())); + LanceTableReader::DatasetKey key; + const auto scan = reader._dataset_key(range, &key); + EXPECT_FALSE(scan.ok()); + EXPECT_EQ(schema.to_string(), scan.to_string()); + EXPECT_FALSE(prepare_range(&reader, range).ok()); + ASSERT_TRUE(reader.close().ok()); + } +} + +TEST(LanceTableReaderLocalPathTest, ReadsInRootDatasetChildrenAndTerminatesAliasCycles) { + const auto original_secure_path = config::user_files_secure_path; + DEFER(config::user_files_secure_path = original_secure_path); + const auto secure = + std::filesystem::temp_directory_path() / + ("lance_internal_children_" + + std::to_string(std::chrono::steady_clock::now().time_since_epoch().count())); + ASSERT_TRUE(std::filesystem::create_directory(secure)); + DEFER({ + std::error_code error; + std::filesystem::remove_all(secure, error); + }); + const auto fixture = + std::filesystem::canonical("./be/test/format_v2/table/lance/data/all_types.lance"); + const auto real = secure / "real.lance"; + const auto alias = secure / "alias.lance"; + std::filesystem::copy(fixture, real, std::filesystem::copy_options::recursive); + std::filesystem::create_directory(alias); + std::filesystem::create_directory_symlink(real / "_versions", alias / "_versions"); + std::filesystem::create_directory(alias / "data"); + for (const auto& entry : std::filesystem::directory_iterator(real / "data")) { + std::filesystem::create_symlink(entry.path(), alias / "data" / entry.path().filename()); + } + // A visited canonical directory is checked once even when an in-root alias points back to it. + std::filesystem::create_directory_symlink(alias, alias / "cycle"); + config::user_files_secure_path = secure.string(); + TQueryGlobals globals; + RuntimeState state(globals); + RuntimeProfile profile("lance_internal_children"); + TFileScanRangeParams scan_params; + LanceTableReader reader; + const Columns columns {projected_column("row_id", TYPE_BIGINT, false)}; + ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, &scan_params).ok()); + auto range = make_latest_lance_range(alias); + range.__set_file_type(TFileType::FILE_LOCAL); + std::vector names; + std::vector types; + ASSERT_TRUE(reader.fetch_schema(range, scan_params, &names, &types).ok()); + EXPECT_NE(names.end(), std::find(names.begin(), names.end(), "row_id")); + ASSERT_TRUE(prepare_range(&reader, range).ok()); + Block block; + add_output_columns(&block, columns); + std::vector row_ids; + bool eos = false; + while (!eos) { + ASSERT_TRUE(reader.get_block(&block, &eos).ok()); + if (!eos) { + const auto& values = + assert_cast(*block.get_by_position(0).column).get_data(); + row_ids.insert(row_ids.end(), values.begin(), values.end()); + } + } + std::sort(row_ids.begin(), row_ids.end()); + EXPECT_EQ((std::vector {1, 2, 3, 4}), row_ids); + ASSERT_TRUE(reader.close().ok()); +} + +TEST(LanceTableReaderLocalPathTest, FixedVersionLocalCatalogKeepsItsUri) { + TFileScanRangeParams scan_params; + LanceTableReader reader; + reader._scan_params = &scan_params; + // Catalog warehouses use their own location, not the LOCAL TVF secure directory. + auto range = make_lance_range("file:///catalog-owned/dataset.lance", 7, {0}); + range.__set_file_type(TFileType::FILE_LOCAL); + LanceTableReader::DatasetKey key; + ASSERT_TRUE(reader._dataset_key(range, &key).ok()); + EXPECT_EQ("file:///catalog-owned/dataset.lance", key.uri); + EXPECT_EQ(7, key.version); +} + +TEST(LanceTableReaderLocalPathTest, RemoteRangeKeepsItsUriAndOptions) { + TFileScanRangeParams scan_params; + scan_params.__set_file_type(TFileType::FILE_LOCAL); + TLanceScanParams lance_params; + lance_params.__set_lance_storage_options({{"aws_region", "us-east-1"}}); + scan_params.__set_lance_scan_params(lance_params); + LanceTableReader reader; + reader._scan_params = &scan_params; + auto range = make_latest_lance_range("s3://bucket/dataset.lance"); + range.__set_file_type(TFileType::FILE_S3); + LanceTableReader::DatasetKey key; + ASSERT_TRUE(reader._dataset_key(range, &key).ok()); + EXPECT_EQ("s3://bucket/dataset.lance", key.uri); + EXPECT_EQ((std::vector {"aws_region", "us-east-1"}), key.storage_options); +} +} // namespace doris::format::lance + +namespace doris::format::lance { +namespace { +Status collect_storage_policy_rows(LanceTableReader* reader, const Columns& columns, + std::vector* rows) { + Block block; + add_output_columns(&block, columns); + bool eos = false; + while (!eos) { + RETURN_IF_ERROR(reader->get_block(&block, &eos)); + if (!eos) { + const auto& values = + assert_cast(*block.get_by_position(0).column).get_data(); + rows->insert(rows->end(), values.begin(), values.end()); + } + } + std::sort(rows->begin(), rows->end()); + return Status::OK(); +} +} // namespace + +TEST(LanceTableReaderLocalPathTest, ManifestBasesAreCheckedBeforeSchemaAndScan) { + const auto original_secure_path = config::user_files_secure_path; + DEFER(config::user_files_secure_path = original_secure_path); + for (const auto* name : {"tvf_storage_clone.lance", "tvf_storage_direct_base.lance"}) { + SCOPED_TRACE(name); + const auto clone = std::filesystem::canonical( + std::filesystem::path("be/test/format_v2/table/lance/data") / name); + // The clone contains only regular metadata; its real fragments live in a sibling dataset. + config::user_files_secure_path = clone.string(); + TQueryGlobals globals; + RuntimeState state(globals); + RuntimeProfile profile("lance_manifest_containment"); + TFileScanRangeParams scan_params; + LanceTableReader reader; + const Columns columns {projected_column("row_id", TYPE_BIGINT, false)}; + ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, &scan_params).ok()); + auto range = make_latest_lance_range(clone); + range.__set_file_type(TFileType::FILE_LOCAL); + range.table_format_params.lance_params.__set_is_tvf(true); + std::vector names; + std::vector types; + const auto schema = reader.fetch_schema(range, scan_params, &names, &types); + EXPECT_FALSE(schema.ok()); + EXPECT_EQ(std::string::npos, schema.to_string().find("all_types.lance")); + EXPECT_FALSE(prepare_range(&reader, range).ok()); + EXPECT_EQ(nullptr, reader._dataset); + EXPECT_EQ(nullptr, reader._scanner); + ASSERT_TRUE(reader.close().ok()); + } +} + +TEST(LanceTableReaderLocalPathTest, InRootManifestBasesReadActualRows) { + const auto original_secure_path = config::user_files_secure_path; + DEFER(config::user_files_secure_path = original_secure_path); + const auto root = std::filesystem::canonical("be/test/format_v2/table/lance/data"); + config::user_files_secure_path = root.string(); + for (const auto* name : {"tvf_storage_clone.lance", "tvf_storage_direct_base.lance"}) { + SCOPED_TRACE(name); + TQueryGlobals globals; + RuntimeState state(globals); + RuntimeProfile profile("lance_manifest_in_root"); + TFileScanRangeParams scan_params; + LanceTableReader reader; + const Columns columns {projected_column("row_id", TYPE_BIGINT, false)}; + ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, &scan_params).ok()); + auto range = make_latest_lance_range(root / name); + range.__set_file_type(TFileType::FILE_LOCAL); + range.table_format_params.lance_params.__set_is_tvf(true); + std::vector names; + std::vector types; + ASSERT_TRUE(reader.fetch_schema(range, scan_params, &names, &types).ok()); + EXPECT_NE(names.end(), std::find(names.begin(), names.end(), "row_id")); + ASSERT_TRUE(prepare_range(&reader, range).ok()); + std::vector rows; + ASSERT_TRUE(collect_storage_policy_rows(&reader, columns, &rows).ok()); + EXPECT_EQ((std::vector {1, 2, 3, 4}), rows); + ASSERT_TRUE(reader.close().ok()); + } +} + +TEST(LanceTableReaderLocalPathTest, TvfPolicyDoesNotInheritCatalogAdmission) { + const auto original_secure_path = config::user_files_secure_path; + DEFER(config::user_files_secure_path = original_secure_path); + const auto clone = std::filesystem::canonical( + "be/test/format_v2/table/lance/data/tvf_storage_clone.lance"); + config::user_files_secure_path = clone.string(); + std::unique_ptr snapshot( + lance_dataset_open(clone.c_str(), nullptr, 0)); + ASSERT_NE(nullptr, snapshot); + const auto version = lance_dataset_version(snapshot.get()); + TQueryGlobals globals; + RuntimeState state(globals); + RuntimeProfile profile("lance_manifest_catalog_policy"); + TFileScanRangeParams scan_params; + LanceTableReader reader; + const Columns columns {projected_column("row_id", TYPE_BIGINT, false)}; + ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, &scan_params).ok()); + auto range = make_latest_lance_range("file://" + clone.string()); + range.__set_file_type(TFileType::FILE_LOCAL); + range.table_format_params.lance_params.__set_version(version); + // Unmarked fixed-version catalogs retain their warehouse policy. + ASSERT_TRUE(prepare_range(&reader, range).ok()); + LanceTableReader::DatasetKey catalog_key; + ASSERT_TRUE(reader._dataset_key(range, &catalog_key).ok()); + range.table_format_params.lance_params.__set_is_tvf(true); + LanceTableReader::DatasetKey tvf_key; + ASSERT_TRUE(reader._dataset_key(range, &tvf_key).ok()); + EXPECT_NE(catalog_key, tvf_key); + EXPECT_FALSE(reader._ensure_dataset_open(range).ok()); + ASSERT_TRUE(reader.close().ok()); + + // A fresh handle can reuse the process Session/manifest cache, but must still enforce TVF bases. + RuntimeProfile tvf_profile("lance_manifest_reused_session"); + LanceTableReader tvf_reader; + ASSERT_TRUE(init_reader(&tvf_reader, columns, &state, &tvf_profile, &scan_params).ok()); + std::vector names; + std::vector types; + EXPECT_FALSE(tvf_reader.fetch_schema(range, scan_params, &names, &types).ok()); + EXPECT_FALSE(prepare_range(&tvf_reader, range).ok()); + EXPECT_EQ(nullptr, tvf_reader._dataset); + ASSERT_TRUE(tvf_reader.close().ok()); +} + +TEST(LanceTableReaderRemoteStorageTest, RemoteManifestCannotReadLocalBase) { + const auto* clone_uri = std::getenv("DORIS_LANCE_TEST_REMOTE_CLONE_URI"); + const auto* source_uri = std::getenv("DORIS_LANCE_TEST_REMOTE_SOURCE_URI"); + if (clone_uri == nullptr || source_uri == nullptr) { + GTEST_SKIP() << "Requires an explicitly selected isolated S3 storage-policy fixture"; + } + const auto* endpoint = std::getenv("DORIS_LANCE_TEST_S3_ENDPOINT"); + const auto* access_key = std::getenv("DORIS_LANCE_TEST_S3_ACCESS_KEY"); + const auto* secret_key = std::getenv("DORIS_LANCE_TEST_S3_SECRET_KEY"); + ASSERT_NE(nullptr, endpoint); + ASSERT_NE(nullptr, access_key); + ASSERT_NE(nullptr, secret_key); + std::map options {{"aws_endpoint", endpoint}, + {"aws_access_key_id", access_key}, + {"aws_secret_access_key", secret_key}, + {"aws_region", "us-east-1"}, + {"aws_allow_http", "true"}, + {"aws_virtual_hosted_style_request", "false"}}; + std::vector native_options; + for (const auto& [key, value] : options) { + native_options.push_back(key.c_str()); + native_options.push_back(value.c_str()); + } + native_options.push_back(nullptr); + std::unique_ptr snapshot( + lance_dataset_open(clone_uri, native_options.data(), 0)); + ASSERT_NE(nullptr, snapshot) << lance_error("open remote clone").to_string(); + TQueryGlobals globals; + RuntimeState state(globals); + RuntimeProfile profile("lance_remote_manifest_policy"); + TFileScanRangeParams scan_params; + TLanceScanParams params; + params.__set_lance_storage_options(options); + scan_params.__set_lance_scan_params(params); + LanceTableReader reader; + const Columns columns {projected_column("row_id", TYPE_BIGINT, false)}; + ASSERT_TRUE(init_reader(&reader, columns, &state, &profile, &scan_params).ok()); + auto range = make_latest_lance_range(clone_uri); + range.__set_file_type(TFileType::FILE_S3); + range.table_format_params.lance_params.__set_is_tvf(true); + range.table_format_params.lance_params.__set_version(lance_dataset_version(snapshot.get())); + const auto status = prepare_range(&reader, range); + EXPECT_FALSE(status.ok()); + EXPECT_NE(std::string::npos, status.to_string().find("remote storage bases")); + EXPECT_EQ(nullptr, reader._dataset); + EXPECT_EQ(nullptr, reader._scanner); + ASSERT_TRUE(reader.close().ok()); + + // An ordinary remote dataset must still return real rows with the identical storage options. + RuntimeProfile remote_profile("lance_remote_storage_control"); + LanceTableReader remote; + ASSERT_TRUE(init_reader(&remote, columns, &state, &remote_profile, &scan_params).ok()); + range = make_latest_lance_range(source_uri); + range.__set_file_type(TFileType::FILE_S3); + range.table_format_params.lance_params.__set_is_tvf(true); + ASSERT_TRUE(prepare_range(&remote, range).ok()); + std::vector rows; + ASSERT_TRUE(collect_storage_policy_rows(&remote, columns, &rows).ok()); + EXPECT_EQ((std::vector {1, 2, 3, 4}), rows); + ASSERT_TRUE(remote.close().ok()); +} +} // namespace doris::format::lance diff --git a/be/test/io/fs/local_file_system_test.cpp b/be/test/io/fs/local_file_system_test.cpp index 4321a2ae56290f..f7233487407037 100644 --- a/be/test/io/fs/local_file_system_test.cpp +++ b/be/test/io/fs/local_file_system_test.cpp @@ -30,12 +30,14 @@ #include #include +#include "common/config.h" #include "common/status.h" #include "cpp/sync_point.h" #include "gtest/gtest_pred_impl.h" #include "io/fs/file_reader.h" #include "io/fs/file_writer.h" #include "io/fs/local_file_writer.h" +#include "util/defer_op.h" #include "util/slice.h" namespace doris { @@ -547,6 +549,10 @@ TEST_F(LocalFileSystemTest, AbnormalWriteRead) { } TEST_F(LocalFileSystemTest, TestGlob) { + const std::string original_secure_path = config::user_files_secure_path; + DEFER(config::user_files_secure_path = original_secure_path); + // Bind this fixture to its own files, independent of the runner's build-profile directory. + config::user_files_secure_path = std::filesystem::absolute("./be/ut_build_ASAN/test").string(); std::string path = "./be/ut_build_ASAN/test/file_path/"; EXPECT_TRUE(io::global_local_filesystem()->delete_directory(path).ok()); EXPECT_TRUE(io::global_local_filesystem() @@ -578,6 +584,234 @@ TEST_F(LocalFileSystemTest, TestGlob) { EXPECT_TRUE(io::global_local_filesystem()->delete_directory(path).ok()); } +TEST_F(LocalFileSystemTest, TestSafeGlobPathContainment) { + auto fs = io::global_local_filesystem(); + const std::string original_secure_path = config::user_files_secure_path; + DEFER(config::user_files_secure_path = original_secure_path); + const auto root = std::filesystem::canonical(test_dir); + const auto secure = root / "secure"; + const auto outside = root / "secure-other"; + std::filesystem::create_directories(secure / "nested"); + std::filesystem::create_directory(outside); + ASSERT_TRUE(save_string_file((secure / "nested/inside.csv").string(), "inside").ok()); + ASSERT_TRUE(save_string_file((outside / "outside.csv").string(), "outside").ok()); + std::filesystem::create_directory_symlink(outside, secure / "escape"); + std::filesystem::create_symlink(outside / "outside.csv", secure / "outside.csv"); + std::filesystem::create_symlink(outside / "missing.csv", outside / "broken.csv"); + std::filesystem::create_directory_symlink(secure / "nested", secure / "inside"); + config::user_files_secure_path = secure.string(); + + for (const auto* pattern : {".[.]/secure-other/outside.csv", ".?/secure-other/outside.csv", + ".[.-.]/secure-other/outside.csv", "escape/*.csv", "outside.csv", + ".[.]/secure-other/*.csv", "escape/broken*.csv"}) { + std::vector files; + const auto status = fs->safe_glob(pattern, &files); + EXPECT_FALSE(status.ok()) << pattern; + EXPECT_NE(status.to_string().find("file path is not allowed"), std::string::npos) << status; + // The pattern need not name the matched file, so errors must not reveal that name. + const auto message = status.to_string(); + EXPECT_EQ(std::string::npos, message.find(root.string())) << message; + for (const auto* name : {"outside.csv", "broken.csv", "missing.csv"}) { + EXPECT_EQ(std::string::npos, message.find(name)) << message; + } + EXPECT_TRUE(files.empty()); + } + + for (const auto* pattern : {"nested/*.csv", "inside/*.csv"}) { + std::vector files; + ASSERT_TRUE(fs->safe_glob(pattern, &files).ok()) << pattern; + ASSERT_EQ(1, files.size()); + EXPECT_EQ((secure / "nested/inside.csv").string(), files[0].file_name); + EXPECT_EQ(6, files[0].file_size); + } +} + +TEST_F(LocalFileSystemTest, TestSafeGlobSecureRoot) { + auto fs = io::global_local_filesystem(); + const std::string original_secure_path = config::user_files_secure_path; + DEFER(config::user_files_secure_path = original_secure_path); + const auto root = std::filesystem::canonical(test_dir); + std::filesystem::create_directory(root / "secure"); + std::filesystem::create_directory_symlink(root / "secure", root / "secure-link"); + ASSERT_TRUE(save_string_file((root / "secure/inside.csv").string(), "inside").ok()); + + for (const auto& secure : {std::string(test_dir) + "/secure", (root / "secure/").string(), + (root / "secure-link").string()}) { + config::user_files_secure_path = secure; + std::vector files; + ASSERT_TRUE(fs->safe_glob("*.csv", &files).ok()) << secure; + ASSERT_EQ(1, files.size()); + EXPECT_EQ((root / "secure/inside.csv").string(), files[0].file_name); + } + const auto special_root = root / "secure[prod]*?\\root"; + std::filesystem::create_directory(special_root); + std::filesystem::create_directory_symlink(special_root, root / "special-root-link"); + ASSERT_TRUE(save_string_file((special_root / "inside.csv").string(), "inside").ok()); + for (const auto& secure : {special_root, root / "special-root-link"}) { + config::user_files_secure_path = secure.string(); + std::vector matches; + ASSERT_TRUE(fs->safe_glob("*.csv", &matches).ok()) << secure; + ASSERT_EQ(1, matches.size()); + EXPECT_EQ((special_root / "inside.csv").string(), matches[0].file_name); + } + config::user_files_secure_path = ""; + std::vector files; + EXPECT_FALSE(fs->safe_glob("*.csv", &files).ok()); + config::user_files_secure_path = (root / "missing").string(); + EXPECT_FALSE(fs->safe_glob("*.csv", &files).ok()); +} + +TEST_F(LocalFileSystemTest, TestCanonicalizeSecureLocalPath) { + auto fs = io::global_local_filesystem(); + const auto root = std::filesystem::canonical(test_dir); + const auto secure = root / "secure"; + std::filesystem::create_directories(secure / "nested"); + std::filesystem::create_directory(root / "secure-other"); + std::filesystem::create_directory_symlink(root / "secure-other", secure / "escape"); + std::filesystem::create_directory_symlink(secure / "nested", secure / "inside"); + for (const auto& path : + {std::string("nested"), std::string("nested/../inside"), (secure / "nested").string(), + "file://" + (secure / "nested").string()}) { + std::string resolved; + ASSERT_TRUE(fs->canonicalize_local_file(secure.string(), path, &resolved).ok()) << path; + EXPECT_EQ((secure / "nested").string(), resolved); + } + for (const auto& path : + {std::string("../secure-other"), std::string("escape"), (root / "secure-other").string(), + std::string("https://invalid/dataset"), std::string("s3://bucket/dataset"), + std::string("http://invalid/dataset"), std::string(""), + std::string("nested\0ignored", 14)}) { + std::string resolved; + const auto status = fs->canonicalize_local_file(secure.string(), path, &resolved); + EXPECT_FALSE(status.ok()) << path; + EXPECT_EQ(std::string::npos, status.to_string().find(root.string())) << status; + } + // URI escapes and bare '%' paths may name two different, simultaneously existing datasets. + for (const auto* name : {"a b.lance", "a%20b.lance", "plus+percent%.lance", "bad%2G.lance"}) { + std::filesystem::create_directory(secure / name); + } + for (const auto& input : + {std::make_pair("file://" + (secure / "a%20b.lance").string(), secure / "a b.lance"), + std::make_pair(std::string("a%20b.lance"), secure / "a%20b.lance"), + std::make_pair("file://" + (secure / "a%20b.lance").string() + "#tag", + secure / "a b.lance"), + std::make_pair("file://" + (secure / "a%20b.lance").string() + "?q=%00#tag", + secure / "a b.lance"), + std::make_pair("file://" + (secure / "a%2520b.lance").string(), secure / "a%20b.lance"), + std::make_pair("file://" + (secure / "plus+percent%25.lance").string(), + secure / "plus+percent%.lance"), + std::make_pair("file://" + (secure / "bad%2G.lance").string(), + secure / "bad%2G.lance")}) { + std::string actual; + ASSERT_TRUE(fs->canonicalize_local_file(secure.string(), input.first, &actual).ok()) + << input.first; + EXPECT_EQ(input.second.string(), actual); + } + for (const auto& suffix : + {"/%2e%2e/secure-other", "/nested%2f..%2f..%2fsecure-other", "/a%00b.lance"}) { + std::string actual; + EXPECT_FALSE(fs->canonicalize_local_file(secure.string(), + "file://" + secure.string() + suffix, &actual) + .ok()); + } + std::string resolved; + EXPECT_FALSE(fs->canonicalize_local_file("", secure.string(), &resolved).ok()); +} + +TEST_F(LocalFileSystemTest, TestCanonicalizeFileUriIdentity) { + auto fs = io::global_local_filesystem(); + const auto root = std::filesystem::canonical(test_dir); + const auto secure = root / "secure"; + std::filesystem::create_directories(secure / "nested/child/data.lance"); + std::filesystem::create_directory(secure / "nested/data.lance"); + std::filesystem::create_directory(secure / "data.lance"); + std::filesystem::create_directory(secure / "a b.lance"); + std::filesystem::create_directory(secure / "data.lance "); + std::filesystem::create_directories(secure / (secure / "data.lance").relative_path()); + std::filesystem::create_directory(root / "outside"); + std::filesystem::create_directory_symlink(secure / "nested/child", secure / "jump"); + std::filesystem::create_directory(secure / "literal\\name.lance"); + const auto backslash_uri = std::string(R"(fIlE:\\localhost)") + secure.string() + "/data.lance"; + for (const auto& input : + {std::make_pair("file://" + secure.string() + "/jump/../data.lance", + secure / "data.lance"), + std::make_pair("file://" + secure.string() + "/missing/../data.lance", + secure / "data.lance"), + std::make_pair("file://" + secure.string() + "/jump/%2e%2E/data.lance", + secure / "data.lance"), + std::make_pair("file://" + secure.string() + "/jump/.%2e/data.lance", + secure / "data.lance"), + std::make_pair("file://" + secure.string() + "/jump/%2E./data.lance", + secure / "data.lance"), + std::make_pair("file://" + secure.string() + "/missing/.././data.lance", + secure / "data.lance"), + std::make_pair("file://" + secure.string() + "/jump//../data.lance", + secure / "nested/child/data.lance"), + std::make_pair("file://" + secure.string() + "/jump%2f../data.lance", + secure / "nested/data.lance"), + std::make_pair("file://" + secure.string() + "/jump\\..\\data.lance", + secure / "data.lance"), + std::make_pair("FILE://" + secure.string() + "/data.lance", secure / "data.lance"), + std::make_pair(backslash_uri, secure / "data.lance"), + std::make_pair("file://" + secure.string() + "/literal%5cname.lance", + secure / "literal\\name.lance"), + std::make_pair(std::string("literal\\name.lance"), secure / "literal\\name.lance"), + std::make_pair("fIlE://" + secure.string() + "/a%20b.lance?q=%00#tag", + secure / "a b.lance"), + std::make_pair("file:" + (secure / "data.lance").relative_path().string(), + secure / "data.lance"), + std::make_pair("file://" + secure.string() + "/data.lance ", secure / "data.lance"), + std::make_pair(" \tFi\nLe://" + secure.string() + "/data.la\tnce\r\n ", + secure / "data.lance"), + std::make_pair("file://" + secure.string() + "/data.lance%20", secure / "data.lance "), + std::make_pair("file://" + secure.string() + "/data.lance%20 ", secure / "data.lance "), + std::make_pair(std::string("data.lance "), secure / "data.lance "), + std::make_pair(std::string("jump/../data.lance"), secure / "nested/data.lance")}) { + std::string actual; + const auto status = fs->canonicalize_local_file(secure.string(), input.first, &actual); + ASSERT_TRUE(status.ok()) << input.first << " " << status; + EXPECT_EQ(input.second.string(), actual) << input.first; + } + for (const auto& input : + {"file://" + secure.string() + "/jump/../../outside", + "FILE://" + secure.string() + "/%2e%2e/outside", + "fIlE://" + secure.string() + "/a%00b.lance", std::string("HTTP://invalid/dataset")}) { + std::string actual; + const auto status = fs->canonicalize_local_file(secure.string(), input, &actual); + EXPECT_FALSE(status.ok()) << input; + EXPECT_EQ(std::string::npos, status.to_string().find(root.string())); + } +} + +TEST_F(LocalFileSystemTest, TestParseFileUriDriveIdentity) { + // Expected Unix paths from the pinned Lance SDK's url 2.5.8, without root-level fixtures. + for (const auto& input : std::vector> { + {"file://C:/data.lance", "/C:/data.lance"}, + {"file://C|/data.lance", "/C:/data.lance"}, + {"file:C|/data.lance", "/C:/data.lance"}, + {"file:/C|/data.lance", "/C:/data.lance"}, + {"file://localhost/C|/data.lance", "/C|/data.lance"}, + {"file://LoCaL%68oSt/C|/data.lance", "/C|/data.lance"}, + {"file://host/C|/data.lance", "/C:/data.lance"}, + {"file:///C|/data.lance", "/C|/data.lance"}, + {"file:////C|/data.lance", "/C|/data.lance"}, + {"file:///../C|/data.lance", "/C:/data.lance"}, + {"file:/x/../C|/data.lance", "/C:/data.lance"}, + {"file:///C:/../../data.lance", "/C:/data.lance"}, + {"file:///C|/../data.lance", "/C|/data.lance"}, + {"file:///x/C:/../data.lance", "/x/C:/data.lance"}, + {"file:///C%7C/data.lance", "/C|/data.lance"}, + {"file:/C%7C/../data.lance", "/data.lance"}, + {"file://C:", "/C:/"}, + {"file:///C|", "/C|/"}, + {"/C|/data.lance", "/C|/data.lance"}}) { + SCOPED_TRACE(input.first); + io::Path actual; + ASSERT_TRUE(io::LocalFileSystem::parse_local_path(input.first, actual).ok()); + EXPECT_EQ(input.second, actual.string()); + } +} + TEST_F(LocalFileSystemTest, TestConvertToAbsPath) { io::Path abs_path; Status st; @@ -637,4 +871,30 @@ TEST_F(LocalFileSystemTest, TestEqualOrSubPath) { EXPECT_FALSE(io::LocalFileSystem::equal_or_sub_path("/data/store1", "/data/store11/snapshot")); EXPECT_FALSE(io::LocalFileSystem::equal_or_sub_path("/data/store/snapshot", "/data/store")); } + +TEST_F(LocalFileSystemTest, TestSafeGlobPreservesWhitespaceIdentity) { + const auto original_secure_path = config::user_files_secure_path; + DEFER(config::user_files_secure_path = original_secure_path); + const auto root = std::filesystem::canonical(test_dir); + const auto secure = root / "whitespace-secure"; + const auto outside = root / "whitespace-outside.csv"; + ASSERT_TRUE(std::filesystem::create_directory(secure)); + ASSERT_TRUE(save_string_file(outside.string(), "outside").ok()); + ASSERT_TRUE(save_string_file((secure / "name.csv ").string(), "safe").ok()); + std::filesystem::create_symlink(outside, secure / "name.csv"); + config::user_files_secure_path = secure.string(); + std::vector files; + auto fs = io::global_local_filesystem(); + ASSERT_TRUE(fs->safe_glob("name.csv?", &files).ok()); + ASSERT_EQ(1, files.size()); + EXPECT_EQ((secure / "name.csv ").string(), files[0].file_name); + io::FileReaderSPtr reader; + ASSERT_TRUE(fs->open_file(files[0].file_name, &reader).ok()); + char buffer[4]; + size_t bytes_read = 0; + ASSERT_TRUE(reader->read_at(0, {buffer, sizeof(buffer)}, &bytes_read).ok()); + EXPECT_EQ("safe", std::string(buffer, bytes_read)); + EXPECT_FALSE(fs->safe_glob("name.csv", &files).ok()); +} + } // namespace doris diff --git a/build.sh b/build.sh index 8449eab271a884..7a585b3723a55f 100755 --- a/build.sh +++ b/build.sh @@ -474,6 +474,59 @@ if [[ ! -f "${DORIS_THIRDPARTY}/installed/lib/${LAST_THIRDPARTY_LIB}" || fi fi +# Lance TVF storage policy needs both sides of the metadata-only SDK ABI. +# Check the selected third-party prefix only; a system Lance install cannot satisfy it. +lance_c_is_compatible() { + local install_dir="$1" + [[ ! -e "${install_dir}/lib64/.lance-installing" && + -s "${install_dir}/include/lance/lance.h" && + -s "${install_dir}/lib64/liblance_c.a" ]] || return 1 + grep -Eq '^#[[:space:]]*define[[:space:]]+LANCE_STORAGE_BASES_JSON_VERSION[[:space:]]+1([[:space:]]|$)' \ + "${install_dir}/include/lance/lance.h" || return 1 + # GNU nm and Darwin nm spell C exports differently. Read the whole stream: + # early awk exit would turn a large archive's producer SIGPIPE into failure. + "${NM:-nm}" -g "${install_dir}/lib64/liblance_c.a" 2>/dev/null | awk ' + NF >= 2 && $(NF - 1) ~ /^[TW]$/ && $NF ~ /^_?lance_dataset_storage_bases_json$/ { found = 1 } + END { exit !found } + ' +} + +lance_c_source_matches() { + local source_dir="$1" + [[ -f "${source_dir}/build-thirdparty.sh" && -f "${source_dir}/download-thirdparty.sh" && + -f "${source_dir}/vars.sh" && -f "${source_dir}/patches/lance-c-storage-bases.patch" ]] || return 1 + grep -Fq 'install_lance_c() {' "${source_dir}/build-thirdparty.sh" || return 1 + grep -Fq 'build_paimon_rust() {' "${source_dir}/build-thirdparty.sh" || return 1 + grep -Fq 'install_paimon_rust() {' "${source_dir}/build-thirdparty.sh" || return 1 + grep -Fq 'lance-c-storage-bases.patch' "${source_dir}/download-thirdparty.sh" || return 1 + cmp -s "${DORIS_HOME}/thirdparty/patches/lance-c-storage-bases.patch" \ + "${source_dir}/patches/lance-c-storage-bases.patch" || return 1 + cmp -s <(grep -E '^(LANCE_C|PAIMON_RUST)_(DOWNLOAD|NAME|SOURCE)=' "${DORIS_HOME}/thirdparty/vars.sh") \ + <(grep -E '^(LANCE_C|PAIMON_RUST)_(DOWNLOAD|NAME|SOURCE)=' "${source_dir}/vars.sh") +} + +if [[ "${BUILD_BE}" -eq 1 ]] && ! lance_c_is_compatible "${DORIS_THIRDPARTY}/installed"; then + if ! lance_c_source_matches "${DORIS_THIRDPARTY}"; then + echo "Lance SDK header/archive pair lacks LANCE_STORAGE_BASES_JSON_VERSION=1 or its exported API, or is incomplete." >&2 + echo "Refresh the compilation image's include/lance and lib64/liblance_c.a together, or set DORIS_THIRDPARTY to matching complete third-party sources." >&2 + exit 1 + fi + # Both Rust staticlibs enter BE's link; repairing only Lance can mix std runtimes. + if [[ -n "${LANCE_C_CARGO:-}" && -n "${PAIMON_RUST_CARGO:-}" && + "${LANCE_C_CARGO}" != "${PAIMON_RUST_CARGO}" ]]; then + echo "Lance/Paimon ABI repair requires the same cargo command; align LANCE_C_CARGO and PAIMON_RUST_CARGO." >&2 + exit 1 + fi + paired_cargo="${LANCE_C_CARGO:-${PAIMON_RUST_CARGO:-${CARGO:-cargo}}}" + echo "Rebuilding lance_c and paimon_rust together for the TVF storage-base metadata ABI ..." + LANCE_C_CARGO="${paired_cargo}" PAIMON_RUST_CARGO="${paired_cargo}" \ + bash "${DORIS_THIRDPARTY}/build-thirdparty.sh" -j "${PARALLEL}" lance_c paimon_rust + if ! lance_c_is_compatible "${DORIS_THIRDPARTY}/installed"; then + echo "Rebuilt Lance SDK header/archive pair is incompatible or incomplete; refresh the matching pair before building BE." >&2 + exit 1 + fi +fi + update_submodule() { local submodule_path=$1 local submodule_name=$2 diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index c311f4c55ec483..a8cd486e8840af 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -3103,6 +3103,10 @@ public class Config extends ConfigBase { }) public static boolean enable_java_udf = true; + @ConfField(description = "Whether to enable LOCAL TVF and FILE TVF with local storage. Disabled by default. " + + "Set this in fe.conf and restart every FE to enable it. Existing access checks still apply.") + public static boolean enable_local_tvf = false; + @ConfField(mutable = true, masterOnly = true, description = { "开启后，可以在导入时，利用创建的全局 java_udf 函数处理数据，默认为 false。", "When enabled, data can be processed using the globally created java_udf function during import." diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/metadata/LanceMetadataLoader.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/metadata/LanceMetadataLoader.java index 69bc50c2e35b5d..e167d758f188ef 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/metadata/LanceMetadataLoader.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/metadata/LanceMetadataLoader.java @@ -69,6 +69,7 @@ public static LanceTableMetadata loadLatestForTvf( Dataset dataset = Dataset.open().allocator(allocator).uri(datasetUri) .readOptions(LanceReadOptions.forIndependentRead( access.getStorageOptions(), OptionalLong.empty())).build()) { + LanceTvfStoragePolicy.checkRemoteBases(dataset); return read(dataset, access, MetadataScope.BASIC); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/metadata/LanceTvfStoragePolicy.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/metadata/LanceTvfStoragePolicy.java new file mode 100644 index 00000000000000..e419c6511decc7 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/lance/metadata/LanceTvfStoragePolicy.java @@ -0,0 +1,103 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.lance.metadata; + +import org.apache.doris.common.AnalysisException; + +import com.google.common.collect.ImmutableSet; +import com.google.common.net.HostAndPort; +import com.google.common.net.InetAddresses; +import org.lance.BasePath; +import org.lance.Dataset; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Locale; +import java.util.Set; + +/** Storage admission for independent Lance TVFs; catalog reads retain their configured storage. */ +public final class LanceTvfStoragePolicy { + // Remote providers registered by the pinned Lance SDK. File providers also have aliases, + // so checking only for "file" (or a particular StorageProperties type) is insufficient. + private static final Set LANCE_REMOTE_SCHEMES = ImmutableSet.of( + "s3", "s3+ddb", "az", "abfss", "gs", "goosefs", "oss", "cos", "hf", "tos"); + + private LanceTvfStoragePolicy() { + } + + /** Inspect manifest registrations before BASIC metadata can open any referenced fragments. */ + public static void checkRemoteBases(Dataset dataset) throws AnalysisException { + for (BasePath base : dataset.getBasePaths()) { + if (!isRemoteUri(base.getPath())) { + throw new AnalysisException("Remote Lance TVF requires remote manifest storage bases"); + } + } + } + + public static boolean isRemoteUri(String path) { + try { + if (path == null) { + return false; + } + int colon = path.indexOf(':'); + if (colon < 0 || !LANCE_REMOTE_SCHEMES.contains(path.substring(0, colon).toLowerCase(Locale.ROOT))) { + return false; + } + String scheme = path.substring(0, colon); + if (!path.startsWith("//", colon + 1)) { + // GooseFS can resolve its remote master from storage options or configuration. + return "goosefs".equalsIgnoreCase(scheme) && path.startsWith("/", colon + 1); + } + int authorityEnd = path.length(); + for (char separator : new char[] {'/', '?', '#'}) { + int end = path.indexOf(separator, colon + 3); + if (end >= 0) { + authorityEnd = Math.min(authorityEnd, end); + } + } + // Validate only scheme/authority against SDK-local fallback. Rust's URL parser + // accepts and encodes raw spaces in paths; preserve that spelling for the SDK. + URI uri = new URI(path.substring(0, authorityEnd) + "/"); + String authority = uri.getRawAuthority(); + if (authority == null || authority.isEmpty()) { + return "goosefs".equalsIgnoreCase(scheme); + } + // Lance falls back to a local path if Rust's URL parser rejects the URI. Java's URI + // accepts registry authorities with invalid ports, which Rust rejects. Validate the + // host/port separately without disallowing legitimate bucket names with underscores. + String hostPort = authority.substring(authority.lastIndexOf('@') + 1); + HostAndPort parsed = HostAndPort.fromString(hostPort).requireBracketsForIPv6(); + if (parsed.hasPort()) { + // Guava parses "-0" as zero, but Rust rejects signs and falls back to local. + String port = hostPort.substring(hostPort.lastIndexOf(':') + 1); + for (int i = 0; i < port.length(); i++) { + if (port.charAt(i) < '0' || port.charAt(i) > '9') { + return false; + } + } + } + String host = parsed.getHost(); + // Rust rejects zones and IPv4 tails with leading zeros, such as ::ffff:127.0.0.01. + return !host.isEmpty() && (!host.contains(":") + || (!host.contains("%") && InetAddresses.isInetAddress(host))); + } catch (URISyntaxException | IllegalArgumentException e) { + return false; + } + } + +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/tvf/source/TVFScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/tvf/source/TVFScanNode.java index 297272cb582250..d0dc8af243f0d4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/tvf/source/TVFScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/tvf/source/TVFScanNode.java @@ -241,9 +241,13 @@ protected void setScanParams(TFileRangeDesc rangeDesc, Split split) { if (!(split instanceof LanceSplit)) { throw new IllegalArgumentException("Expected LanceSplit but got " + split.getClass().getName()); } + // A bare local path has no URI scheme, so split inference cannot supply its type. + // Preserve the TVF's LOCAL/S3 identity for the BE's pre-open path validation. + rangeDesc.setFileType(tableValuedFunction.getTFileType()); LanceSplit lanceSplit = (LanceSplit) split; TLanceFileDesc lanceParams = new TLanceFileDesc(); lanceParams.setDatasetUri(lanceSplit.getDatasetUri()); + lanceParams.setIsTvf(true); lanceParams.setVersion(lanceSplit.getVersion()); if (lanceSplit.hasFragmentIds()) { lanceParams.setFragmentIds(lanceSplit.getFragmentIds()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java index f4669ec001bb6c..2669785f7aa24c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java @@ -32,6 +32,7 @@ import org.apache.doris.common.Config; import org.apache.doris.common.IdGenerator; import org.apache.doris.common.Pair; +import org.apache.doris.common.util.BrokerUtil; import org.apache.doris.datasource.VariantWritePlanValidator; import org.apache.doris.datasource.hive.HMSExternalDatabase; import org.apache.doris.datasource.hive.HMSExternalTable; @@ -52,6 +53,7 @@ import org.apache.doris.datasource.paimon.PaimonExternalTable; import org.apache.doris.datasource.paimon.PaimonVariantWriteAnalyzer; import org.apache.doris.datasource.paimon.PaimonWriteTarget; +import org.apache.doris.datasource.property.storage.StorageProperties; import org.apache.doris.dictionary.Dictionary; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.StatementContext; @@ -117,6 +119,8 @@ import org.apache.doris.qe.AutoCloseSessionVariable; import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.SessionVariable; +import org.apache.doris.tablefunction.HadoopTvfAdmission; +import org.apache.doris.tablefunction.LocalTableValuedFunction; import org.apache.doris.thrift.TPartialUpdateNewRowPolicy; import com.google.common.base.Preconditions; @@ -662,6 +666,9 @@ private Plan bindTVFTableSink(MatchingContext> ctx) { "INSERT INTO TVF only supports local/s3/hdfs, but got: " + tvfName); } + // INSERT/EXPLAIN INSERT bind a sink without constructing the LOCAL read TVF. + LocalTableValuedFunction.checkSinkEnabled(tvfName, properties); + // Validate required properties if (!properties.containsKey("file_path")) { throw new AnalysisException("TVF sink requires 'file_path' property"); @@ -688,6 +695,19 @@ private Plan bindTVFTableSink(MatchingContext> ctx) { "delete_existing_files=true is not supported for local TVF"); } + if ("hdfs".equals(tvfName)) { + // Storage normalization loads operator-controlled XML before route classification, + // while leaving filesystem construction to execution. + Map normalized = StorageProperties.createPrimary(properties).getBackendConfigProperties(); + if (!properties.containsKey("writer_class")) { + HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, filePath, normalized); + } + if (deleteExisting) { + HadoopTvfAdmission.check(HadoopTvfAdmission.Access.JAVA_DELETE, + BrokerUtil.extractParentDirectory(filePath), normalized); + } + } + LogicalPlan child = ((LogicalPlan) sink.child()); // Always derive schema from child query output diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTVFCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTVFCommand.java index fae7747b5d42f3..bae564fd9fe21b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTVFCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTVFCommand.java @@ -24,6 +24,7 @@ import org.apache.doris.common.Status; import org.apache.doris.common.UserException; import org.apache.doris.common.util.BrokerUtil; +import org.apache.doris.datasource.property.storage.HdfsCompatibleProperties; import org.apache.doris.datasource.property.storage.StorageProperties; import org.apache.doris.fs.FileSystemFactory; import org.apache.doris.fs.remote.RemoteFileSystem; @@ -39,12 +40,14 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; import org.apache.doris.nereids.trees.plans.physical.PhysicalTVFTableSink; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; -import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.ConnectContext.ConnectType; +import org.apache.doris.qe.ConnectContext; import org.apache.doris.qe.Coordinator; -import org.apache.doris.qe.QeProcessorImpl; import org.apache.doris.qe.QeProcessorImpl.QueryInfo; +import org.apache.doris.qe.QeProcessorImpl; import org.apache.doris.qe.StmtExecutor; +import org.apache.doris.tablefunction.HadoopTvfAdmission; +import org.apache.doris.tablefunction.LocalTableValuedFunction; import org.apache.doris.task.LoadEtlTask; import org.apache.doris.thrift.TStatusCode; import org.apache.doris.thrift.TUniqueId; @@ -175,6 +178,7 @@ public Plan getExplainPlan(ConnectContext ctx) { private void deleteExistingFilesInFE(String tvfName, Map props) throws Exception { + LocalTableValuedFunction.checkSinkEnabled(tvfName, props); String filePath = props.get("file_path"); // Extract parent directory from prefix path: s3://bucket/path/to/prefix_ -> s3://bucket/path/to/ String parentDir = BrokerUtil.extractParentDirectory(filePath); @@ -192,6 +196,10 @@ private void deleteExistingFilesInFE(String tvfName, Map props) fsCopyProps.remove("compress_type"); StorageProperties storageProps = StorageProperties.createPrimary(fsCopyProps); + if (storageProps instanceof HdfsCompatibleProperties) { + HadoopTvfAdmission.check(HadoopTvfAdmission.Access.JAVA_DELETE, + parentDir, storageProps.getBackendConfigProperties()); + } RemoteFileSystem fs = FileSystemFactory.get(storageProps); org.apache.doris.backup.Status deleteStatus = fs.deleteDirectory(parentDir); if (!deleteStatus.ok()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/TVFTableSink.java b/fe/fe-core/src/main/java/org/apache/doris/planner/TVFTableSink.java index 102da9af2e466f..36ce1426c8658b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/planner/TVFTableSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/planner/TVFTableSink.java @@ -24,6 +24,8 @@ import org.apache.doris.datasource.property.fileformat.FileFormatProperties; import org.apache.doris.datasource.property.fileformat.OrcFileFormatProperties; import org.apache.doris.datasource.property.storage.StorageProperties; +import org.apache.doris.tablefunction.HadoopTvfAdmission; +import org.apache.doris.tablefunction.LocalTableValuedFunction; import org.apache.doris.thrift.TColumn; import org.apache.doris.thrift.TDataSink; import org.apache.doris.thrift.TDataSinkType; @@ -82,6 +84,7 @@ public long getBackendId() { } public void bindDataSink() throws AnalysisException { + LocalTableValuedFunction.checkSinkEnabled(tvfName, properties); TTVFTableSink tSink = new TTVFTableSink(); tSink.setTvfName(tvfName); @@ -128,6 +131,11 @@ public void bindDataSink() throws AnalysisException { } } + if (fileType == TFileType.FILE_HDFS && !properties.containsKey("writer_class")) { + HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, + properties.get("file_path"), backendConnectProps); + } + String filePath = properties.get("file_path"); tSink.setFilePath(filePath); diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java index c26bc2f472fe3c..8b8824f417dc67 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunction.java @@ -317,6 +317,10 @@ public ScanNode getScanNode(PlanNodeId id, TupleDescriptor desc, SessionVariable ScanContext.builder().clusterName(sv.resolveCloudClusterName()).build()); } + protected boolean needsSchemaDiscovery() { + return csvSchema.isEmpty(); + } + @Override public List getTableColumns() throws AnalysisException { if (!csvSchema.isEmpty()) { @@ -647,6 +651,7 @@ private PFetchTableSchemaRequest getFetchTableStructureRequest() throws TExcepti if (isLanceFormat()) { TLanceFileDesc lanceParams = new TLanceFileDesc(); lanceParams.setDatasetUri(getFilePath()); + lanceParams.setIsTvf(true); // Local TVF schema discovery and execution independently open the latest snapshot. // S3 TVF obtains its schema and fixed snapshot metadata directly in FE instead. lanceParams.setVersion(0); diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/HadoopTvfAdmission.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/HadoopTvfAdmission.java new file mode 100644 index 00000000000000..4f7eb452929a78 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/HadoopTvfAdmission.java @@ -0,0 +1,534 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.tablefunction; + +import org.apache.doris.common.Config; +import org.apache.doris.common.util.LocationPath; +import org.apache.doris.nereids.exceptions.AnalysisException; + +import java.net.URI; +import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * TVF-only admission for Hadoop consumers. Input properties have already been normalized by storage. + * This module neither loads Hadoop resources nor constructs a filesystem. Java strings and native + * C strings are projected separately before filesystem selection and mount-table interpretation. + */ +public final class HadoopTvfAdmission { + public enum Access { + NATIVE_WRITE, + JAVA_DELETE, + HADOOP_READ + } + + private enum Provider { + REMOTE, + LOCAL, + MOUNT_TABLE, + DYNAMIC, + UNKNOWN + } + + private static final Set REMOTE_SCHEMES = new HashSet<>(Arrays.asList( + "hdfs", "s3a", "s3n", "oss", "ofs", "jfs", "abfs", "abfss", "wasb", "wasbs", "gs", "cosn")); + private static final Set REMOTE_IMPLEMENTATIONS = new HashSet<>(Arrays.asList( + "org.apache.hadoop.hdfs.DistributedFileSystem", + "org.apache.hadoop.fs.s3a.S3AFileSystem", + "org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem", + "org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem", + "org.apache.hadoop.fs.azure.NativeAzureFileSystem", + "org.apache.hadoop.fs.aliyun.oss.AliyunOSSFileSystem", + "com.aliyun.jindodata.oss.JindoOssFileSystem", + "org.apache.hadoop.fs.obs.OBSFileSystem", + "com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem")); + + private final Access access; + private final Map properties = new HashMap<>(); + + private HadoopTvfAdmission(Access access, Map normalized, boolean nativeConsumer) { + this.access = access; + Set nativeRoutingKeys = new HashSet<>(); + for (Map.Entry entry : normalized.entrySet()) { + // parse_properties / generateHdfsParam consumes the canonical NameNode before JNI. + if (nativeConsumer && "fs.defaultFS".equalsIgnoreCase(entry.getKey())) { + continue; + } + if (nativeConsumer && isRoutingKey(asciiCaseFold(nativeStringPrefix(entry.getKey()).trim()))) { + checkNativeCharacters(access, entry.getKey()); + checkNativeCharacters(access, entry.getValue()); + } + String key = (nativeConsumer ? nativeStringPrefix(entry.getKey()) : entry.getKey()).trim(); + // Native writes first pass through StringCaseMap, before JNI truncates strings. + // Its comparator uses strncasecmp followed by the full UTF-8 byte length. + if (nativeConsumer && access == Access.NATIVE_WRITE && isRoutingKey(asciiCaseFold(key))) { + String identity = asciiCaseFold(nativeStringPrefix(entry.getKey())) + "\0" + + entry.getKey().getBytes(StandardCharsets.UTF_8).length; + if (!nativeRoutingKeys.add(identity)) { + reject(); + } + } + String value = nativeConsumer ? nativeStringPrefix(entry.getValue()) : entry.getValue(); + String previous = properties.put(key, value); + if (previous != null && !previous.equals(value) && isRoutingKey(key)) { + reject(); + } + } + } + + /** Reject local/unknown Hadoop TVF routes before listing, deletion, schema discovery or data IO. */ + public static void check(Access access, String path, Map normalized) { + check(access, path, normalized, true); + } + + /** A declared CSV schema skips the schema RPC; listing and data admission still apply. */ + public static void check(Access access, String path, Map normalized, + boolean needsSchemaDiscovery) { + if (access != Access.HADOOP_READ && Config.enable_local_tvf) { + return; + } + try { + if (access == Access.NATIVE_WRITE) { + String nameNode = nativeNameNode(access, normalized); + URI output = nativeFileUri(access, path + "doris-tvf-file", nameNode); + String parent = output.getPath(); + parent = parent.substring(0, parent.lastIndexOf('/') + 1); + if (parent.length() > 1) { + parent = parent.substring(0, parent.length() - 1); + } + new HadoopTvfAdmission(access, normalized, true).checkNative(nameNode, parent); + } else { + HadoopTvfAdmission javaConsumer = new HadoopTvfAdmission(access, normalized, false); + // Project the Java consumer independently from native convert_path/JNI. + URI javaPath = hadoopPathUri(path.trim()); + javaConsumer.checkRoute(javaPath, javaPath.getPath(), new HashSet<>()); + if (access == Access.HADOOP_READ) { + // FileQueryScanNode sends this filesystem identifier in range.fs_name. BE + // falls back to hdfs_params.fs_name only when that range field is empty. + URI fileUri = LocationPath.of(path).getPath().toUri(); + String nameNode = fileUri.getScheme() + "://" + fileUri.getAuthority(); + if (nameNode.isEmpty()) { + nameNode = nativeNameNode(access, normalized); + } + HadoopTvfAdmission nativeConsumer = new HadoopTvfAdmission(access, normalized, true); + nativeConsumer.checkNativeRead(nameNode, path); + // Schema RPCs have no range.fs_name; they use THdfsParams.fs_name instead. + if (needsSchemaDiscovery) { + nativeConsumer.checkNativeRead(nativeNameNode(access, normalized), path); + } + } + } + } catch (URISyntaxException e) { + new HadoopTvfAdmission(access, normalized, false).reject(); + } + } + + private static String nativeNameNode(Access access, Map normalized) { + String result = null; + for (Map.Entry entry : normalized.entrySet()) { + if ("fs.defaultFS".equalsIgnoreCase(entry.getKey())) { + if (result != null && !result.equals(entry.getValue())) { + reject(access); + } + result = entry.getValue(); + } + } + return result == null ? "" : result; + } + + private void checkNative(String nameNode, String path) throws URISyntaxException { + checkNativeCharacters(access, nameNode); + nameNode = selector(nativeStringPrefix(nameNode)).trim(); + if (nameNode.isEmpty() && access == Access.NATIVE_WRITE) { + // BE passes fs_name.c_str(), never NULL. libhdfs prefixes this empty string + // with hdfs://, whose URI construction fails before any filesystem is selected. + // Preserve existing analysis; execution reports the invalid NameNode as before. + return; + } + if ("default".equals(nameNode) || nameNode.isEmpty()) { + reject(); + } + checkRoute(new URI(nameNode.contains("://") ? nameNode : "hdfs://" + nameNode), + path, new HashSet<>()); + } + + private void checkNativeRead(String nameNode, String path) throws URISyntaxException { + checkNative(nameNode, nativeFileUri(access, path, nameNode).getPath()); + } + + private List defaults() throws URISyntaxException { + List defaults = new ArrayList<>(); + for (String key : Arrays.asList("fs.defaultFS", "fs.default.name")) { + String value = property(key); + if (value != null) { + value = selector(value).trim(); + // FileSystem.fixName treats slash-free host[:port] strings as HDFS defaults. + defaults.add(new URI("local".equals(value) ? "file:///" + : (value.indexOf('/') < 0 ? "hdfs://" + value : value))); + } + } + if (defaults.isEmpty()) { + defaults.add(new URI("file:///")); + } + return defaults; + } + + private void checkRoute(URI filesystem, String path, Set activeTables) throws URISyntaxException { + if (filesystem.getScheme() == null) { + for (URI defaultFs : defaults()) { + checkViewFsEnabled(defaultFs, path, activeTables); + } + return; + } + if (filesystem.getAuthority() == null) { + boolean completed = false; + for (URI defaultFs : defaults()) { + if (filesystem.getScheme().equals(defaultFs.getScheme()) && defaultFs.getAuthority() != null) { + checkViewFsEnabled(defaultFs, path, activeTables); + completed = true; + } + } + if (completed) { + return; + } + } + checkViewFsEnabled(filesystem, path, activeTables); + } + + private Provider provider(String scheme) { + if ("file".equalsIgnoreCase(scheme)) { + return Provider.LOCAL; + } + String implementation = property("fs." + scheme + ".impl"); + if (implementation != null) { + implementation = selector(implementation).trim(); + if ("org.apache.hadoop.fs.LocalFileSystem".equals(implementation) + || "org.apache.hadoop.fs.RawLocalFileSystem".equals(implementation)) { + return Provider.LOCAL; + } + if ("org.apache.hadoop.fs.viewfs.ViewFileSystem".equals(implementation)) { + return Provider.MOUNT_TABLE; + } + if ("org.apache.hadoop.fs.viewfs.ViewFileSystemOverloadScheme".equals(implementation) + || "org.apache.hadoop.hdfs.ViewDistributedFileSystem".equals(implementation)) { + return Provider.DYNAMIC; + } + return REMOTE_IMPLEMENTATIONS.contains(implementation) ? Provider.REMOTE : Provider.UNKNOWN; + } + if ("viewfs".equals(scheme)) { + return Provider.MOUNT_TABLE; + } + return REMOTE_SCHEMES.contains(scheme) ? Provider.REMOTE : Provider.UNKNOWN; + } + + private String[] targets(String key, String prefix) { + String value = selector(property(key)); + // InodeTree treats SINGLE, SINGLE_FALLBACK and MERGE_SLASH as one URI. + // Commas delimit targets only for MERGE and NFLY links. + String entry = key.substring(prefix.length()); + boolean multiple = entry.startsWith("linkMerge.") || entry.startsWith("linkNfly."); + return multiple ? value.split(",", -1) : new String[] {value}; + } + + private void checkGlobTargets(String key, String prefix, Set activeTables) throws URISyntaxException { + for (String target : targets(key, prefix)) { + if (target.isEmpty()) { + reject(); + } + URI uri = new URI(target); + // Listing can touch any descendant of a selected mount. Reuse the same route + // interpreter for nested tables instead of assuming their root is representative. + String path = uri.getPath() == null ? "/" : uri.getPath(); + checkRoute(uri, (path.endsWith("/") ? path : path + "/") + "*", activeTables); + } + } + + private void checkViewFsEnabled(URI filesystem, String path, Set activeTables) { + String scheme = filesystem.getScheme(); + Provider provider = provider(scheme); + if (provider == Provider.LOCAL || provider == Provider.UNKNOWN || provider == Provider.DYNAMIC) { + reject(); + } + if (provider != Provider.MOUNT_TABLE) { + return; + } + if (!path.startsWith("/")) { + // Java resolves relative paths against the selected filesystem's working directory. + // Its user/home configuration is not a proven absolute mount-table path here. + reject(); + } + String table = filesystem.getAuthority(); + String ignorePort = property("fs.viewfs.ignore.port.in.mount.table.name"); + if (ignorePort != null && Boolean.parseBoolean(selector(ignorePort).trim()) + && table != null && filesystem.getPort() != -1) { + table = filesystem.getHost(); + if (table == null) { + reject(); + } + } + if (table == null || table.isEmpty()) { + table = property("fs.viewfs.mounttable.default.name.key"); + table = table == null ? "default" : selector(table); + } + // Nested ViewFS cycles or arbitrarily deep delegation cannot establish remote storage. + if (activeTables.size() >= 32 || !activeTables.add(table)) { + reject(); + } + try { + String prefix = "fs.viewfs.mounttable." + table + "."; + String source = ""; + List selectedKeys = new ArrayList<>(); + List mountSources = new ArrayList<>(); + String fallback = null; + String rootMerge = null; + for (String key : properties.keySet()) { + if (!key.startsWith(prefix)) { + continue; + } + String entry = key.substring(prefix.length()); + if (entry.equals("linkFallback")) { + fallback = key; + continue; + } + if (entry.equals("linkMergeSlash")) { + rootMerge = key; + continue; + } + if (entry.startsWith("linkRegex")) { + // Regex interception rewrites runtime paths before ordinary mount resolution. + reject(); + } + int slash = entry.indexOf('/'); + if (slash < 0 || !(entry.startsWith("link.") || entry.startsWith("linkMerge.") + || entry.startsWith("linkNfly."))) { + continue; + } + String candidate = entry.substring(slash); + // InodeTree splits source components and discards trailing empty components. + while (candidate.length() > 1 && candidate.endsWith("/")) { + candidate = candidate.substring(0, candidate.length() - 1); + } + mountSources.add(candidate); + if (!(path.equals(candidate) || path.startsWith(candidate + "/"))) { + continue; + } + if (candidate.length() > source.length()) { + source = candidate; + selectedKeys.clear(); + } + if (candidate.equals(source)) { + selectedKeys.add(key); + } + } + if (access == Access.HADOOP_READ && hasGlob(path)) { + String literal = path.substring(0, firstGlob(path)); + int slash = literal.lastIndexOf('/'); + literal = slash < 0 ? "/" : literal.substring(0, slash + 1); + for (String key : properties.keySet()) { + if (!key.startsWith(prefix)) { + continue; + } + String entry = key.substring(prefix.length()); + int sourceStart = entry.indexOf('/'); + if (sourceStart >= 0 && (entry.startsWith("link.") || entry.startsWith("linkMerge.") + || entry.startsWith("linkNfly."))) { + String mount = entry.substring(sourceStart); + if (mount.startsWith(literal) || literal.startsWith(mount + "/")) { + checkGlobTargets(key, prefix, activeTables); + } + } + } + } + if (access == Access.HADOOP_READ && !hasGlob(path)) { + String subtree = path.endsWith("/") ? path : path + "/"; + for (String key : properties.keySet()) { + if (key.startsWith(prefix)) { + String entry = key.substring(prefix.length()); + int sourceStart = entry.indexOf('/'); + if (sourceStart >= 0 && entry.substring(sourceStart).startsWith(subtree)) { + checkGlobTargets(key, prefix, activeTables); + } + } + } + } + if (rootMerge != null) { + selectedKeys.add(rootMerge); + source = ""; + } else if (selectedKeys.isEmpty() && fallback != null) { + selectedKeys.add(fallback); + } + if (selectedKeys.isEmpty()) { + // Missing inline fallback does not prove absence of one in default resources. + // Keep rejecting unproven routes, including globs that cross mount names. + reject(); + } + for (String key : selectedKeys) { + for (String target : targets(key, prefix)) { + if (target.isEmpty()) { + reject(); + } + URI uri = new URI(target); + if (uri.getScheme() == null) { + reject(); + } + if (provider(uri.getScheme()) == Provider.MOUNT_TABLE) { + for (String nested : mountSources) { + if (!nested.startsWith(source + "/")) { + continue; + } + int branchEnd = nested.indexOf('/', source.length() + 1); + String branch = branchEnd < 0 ? nested : nested.substring(0, branchEnd); + if (path.equals(branch) || path.startsWith(branch + "/")) { + // A missing child below an internal nested-mount directory uses + // a different suffix in InodeTree. Do not guess its delegation. + reject(); + } + } + } + String base = uri.getPath() == null ? "/" : uri.getPath(); + String remaining = path.substring(source.length()); + String nextPath = new URI(null, null, + (base.endsWith("/") ? base : base + "/") + + (remaining.startsWith("/") ? remaining.substring(1) : remaining), null) + .normalize().getPath(); + checkRoute(uri, nextPath, activeTables); + } + } + } catch (URISyntaxException e) { + reject(); + } finally { + activeTables.remove(table); + } + } + + private static URI nativeFileUri(Access access, String path, String nameNode) + throws URISyntaxException { + checkNativeCharacters(access, path); + // Match BE convert_path before the JNI C-string conversion. A different URI prefix + // becomes a literal rooted Hadoop path; it must not be parsed as that URI's path. + if (!nameNode.isEmpty() && path.contains(nameNode)) { + if (!path.startsWith(nameNode)) { + // convert_path truncates bytes at NameNode.length even for a non-prefix match. + // Java character offsets cannot establish that malformed path's mount identity. + reject(access); + } + path = path.substring(nameNode.length()); + } + if (!path.startsWith("/")) { + path = "/" + path; + } + return hadoopPathUri(nativeStringPrefix(path)); + } + + private static URI hadoopPathUri(String path) throws URISyntaxException { + String scheme = uriScheme(path); + String authority = null; + String relative = scheme.isEmpty() ? path : path.substring(scheme.length() + 1); + if (relative.startsWith("//")) { + int slash = relative.indexOf('/', 2); + authority = slash < 0 ? relative.substring(2) : relative.substring(2, slash); + relative = slash < 0 ? "/" : relative.substring(slash); + } + if (!scheme.isEmpty() && !relative.startsWith("/")) { + relative = "/" + relative; + } + // Hadoop Path(String) quotes path bytes before URI normalization; percent text is literal. + return new URI(scheme.isEmpty() ? null : scheme, authority, relative, null, null).normalize(); + } + + private static String uriScheme(String path) { + int colon = path.indexOf(':'); + int slash = path.indexOf('/'); + return colon > 0 && (slash < 0 || colon < slash) ? path.substring(0, colon) : ""; + } + + private String property(String key) { + return properties.get(key); + } + + private String selector(String value) { + if (value.contains("${")) { + // Runtime environment and Hadoop substitutions are not available to this FE projection. + reject(); + } + return value; + } + + private static boolean isRoutingKey(String key) { + return "fs.defaultFS".equalsIgnoreCase(key) || "fs.default.name".equals(key) + || key.startsWith("fs.viewfs.") || (key.startsWith("fs.") && key.endsWith(".impl")); + } + + private static int firstGlob(String path) { + int first = path.length(); + for (char marker : new char[] {'*', '?', '[', '{'}) { + int index = path.indexOf(marker); + if (index >= 0) { + first = Math.min(first, index); + } + } + return first; + } + + private static boolean hasGlob(String path) { + return firstGlob(path) < path.length(); + } + + private static String asciiCaseFold(String value) { + StringBuilder folded = new StringBuilder(value.length()); + for (int i = 0; i < value.length(); i++) { + char c = value.charAt(i); + folded.append(c >= 'A' && c <= 'Z' ? (char) (c + ('a' - 'A')) : c); + } + return folded.toString(); + } + + private static void checkNativeCharacters(Access access, String value) { + // libhdfs uses NewStringUTF on standard UTF-8 bytes. Surrogates encode differently + // in modified UTF-8, so supplementary/unpaired input cannot prove the same route. + for (int i = 0; i < value.length() && value.charAt(i) != '\0'; i++) { + if (Character.isSurrogate(value.charAt(i))) { + reject(access); + } + } + } + + private static String nativeStringPrefix(String value) { + int nul = value.indexOf('\0'); + return nul < 0 ? value : value.substring(0, nul); + } + + private void reject() { + reject(access); + } + + private static void reject(Access access) { + if (access == Access.HADOOP_READ) { + throw new AnalysisException("Hadoop TVFs require a proven remote filesystem route. " + + "Use local() for local files with LOCAL TVF privileges and secure-path restrictions."); + } + LocalTableValuedFunction.checkEnabled(); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/HdfsTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/HdfsTableValuedFunction.java index e99340eb105bec..74bec05b4284c2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/HdfsTableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/HdfsTableValuedFunction.java @@ -23,6 +23,7 @@ import org.apache.doris.common.FeConstants; import org.apache.doris.common.UserException; import org.apache.doris.datasource.property.storage.StorageProperties; +import org.apache.doris.thrift.TBrokerFileStatus; import org.apache.doris.thrift.TFileType; import org.apache.log4j.LogManager; @@ -55,9 +56,15 @@ private void init(Map properties) throws AnalysisException { throw new AnalysisException("Failed check storage props, " + e.getMessage(), e); } + HadoopTvfAdmission.check(HadoopTvfAdmission.Access.HADOOP_READ, + filePath, backendConnectProperties, needsSchemaDiscovery()); if (!FeConstants.runningUnitTest) { // 4. parse file parseFile(); + for (TBrokerFileStatus file : fileStatuses) { + HadoopTvfAdmission.check(HadoopTvfAdmission.Access.HADOOP_READ, + file.getPath(), backendConnectProperties, needsSchemaDiscovery()); + } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/LocalTableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/LocalTableValuedFunction.java index 4e4bf163145cf9..3eb16008e814d7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/LocalTableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/LocalTableValuedFunction.java @@ -21,9 +21,12 @@ import org.apache.doris.analysis.StorageBackend.StorageType; import org.apache.doris.catalog.Env; import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.Config; +import org.apache.doris.common.ErrorCode; import org.apache.doris.common.UserException; import org.apache.doris.datasource.FederationBackendPolicy; import org.apache.doris.datasource.property.storage.StorageProperties; +import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.proto.InternalService; import org.apache.doris.proto.InternalService.PGlobResponse; import org.apache.doris.proto.InternalService.PGlobResponse.PFileInfo; @@ -38,6 +41,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.util.List; import java.util.Map; import java.util.concurrent.Future; import java.util.concurrent.TimeUnit; @@ -61,6 +65,11 @@ public class LocalTableValuedFunction extends ExternalFileTableValuedFunction { private boolean sharedStorage = false; public LocalTableValuedFunction(Map properties) throws AnalysisException { + checkEnabled(); + + // Check privileges before listing files or discovering their schema. + checkAuth(ConnectContext.get()); + // 1. analyze common properties Map props = super.parseCommonProperties(properties); @@ -127,7 +136,7 @@ private void getFileListFromBackend() throws AnalysisException { + ", error msg: " + globResponse.getStatus().getErrorMsgsList()); } for (PFileInfo file : globResponse.getFilesList()) { - fileStatuses.add(new TBrokerFileStatus(file.getFile().trim(), false, file.getSize(), true)); + fileStatuses.add(new TBrokerFileStatus(file.getFile(), false, file.getSize(), true)); LOG.info("get file from backend success. file: {}, size: {}", file.getFile(), file.getSize()); } } catch (Exception e) { @@ -135,6 +144,41 @@ private void getFileListFromBackend() throws AnalysisException { } } + /** Apply LOCAL admission to the effective sink before planning or filesystem side effects. */ + public static void checkSinkEnabled(String tvfName, Map properties) { + if ("local".equalsIgnoreCase(tvfName) || isLocalWriter(properties.getOrDefault("writer_class", ""))) { + checkEnabled(); + } + } + + private static boolean isLocalWriter(String writer) { + // ScannerLoader accepts slashes, and JNI passes the class name as a C string. + return "org.apache.doris.writer.LocalFileJniWriter".equals(nativeStringPrefix(writer).replace('/', '.')); + } + + private static String nativeStringPrefix(String value) { + int nul = value.indexOf('\0'); + return nul < 0 ? value : value.substring(0, nul); + } + + /** Check read and write admission before properties or backend access. */ + public static void checkEnabled() { + if (!Config.enable_local_tvf) { + throw new org.apache.doris.nereids.exceptions.AnalysisException( + "LOCAL TVF is disabled. Set enable_local_tvf=true in fe.conf and restart every FE to enable it."); + } + } + + @Override + public void checkAuth(ConnectContext ctx) { + if (!Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ctx, PrivPredicate.ADMIN) + && !Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ctx, PrivPredicate.OPERATOR)) { + String message = ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR.formatErrorMsg("ADMIN/OPERATOR"); + throw new org.apache.doris.nereids.exceptions.AnalysisException(message); + } + super.checkAuth(ctx); + } + @Override public TFileType getTFileType() { return TFileType.FILE_LOCAL; diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java index 91e79dda10e641..2f79e4fedf711f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/S3TableValuedFunction.java @@ -21,9 +21,12 @@ import org.apache.doris.catalog.Column; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.FeConstants; +import org.apache.doris.common.util.LocationPath; import org.apache.doris.common.UserException; import org.apache.doris.datasource.lance.metadata.LanceMetadataLoader; import org.apache.doris.datasource.lance.metadata.LanceTableMetadata; +import org.apache.doris.datasource.lance.metadata.LanceTvfStoragePolicy; +import org.apache.doris.datasource.property.storage.HdfsCompatibleProperties; import org.apache.doris.datasource.property.storage.StorageProperties; import org.apache.doris.thrift.TBrokerFileStatus; import org.apache.doris.thrift.TFileType; @@ -63,6 +66,21 @@ public S3TableValuedFunction(Map properties, List properties, List properties, List datasets = Mockito.mockStatic(Dataset.class)) { + datasets.when(Dataset::open).thenReturn(builder); + AnalysisException error = Assertions.assertThrows(AnalysisException.class, + () -> LanceMetadataLoader.loadLatestForTvf( + "s3://bucket/clone.lance", Collections.emptyList())); + Assertions.assertFalse(error.getMessage().contains("private")); + Mockito.verify(dataset, Mockito.never()).getFragments(); + Mockito.verify(dataset, Mockito.never()).getSchema(); + Mockito.verify(dataset).close(); + } + } + } + + @Test + void remoteTvfPreservesValidRemoteManifestBases() throws Exception { + Dataset dataset = Mockito.mock(Dataset.class); + Mockito.when(dataset.getBasePaths()).thenReturn(Arrays.asList( + new BasePath(1, Optional.empty(), "s3://bucket/source data.lance", true), + new BasePath(2, Optional.empty(), "s3://bucket/other%20data", false))); + Mockito.when(dataset.version()).thenReturn(7L); + Mockito.when(dataset.getFragments()).thenReturn(Collections.emptyList()); + Mockito.when(dataset.getSchema()).thenReturn(new Schema(Collections.emptyList())); + OpenDatasetBuilder builder = Mockito.mock(OpenDatasetBuilder.class, Mockito.RETURNS_SELF); + Mockito.when(builder.build()).thenReturn(dataset); + try (MockedStatic datasets = Mockito.mockStatic(Dataset.class)) { + datasets.when(Dataset::open).thenReturn(builder); + LanceTableMetadata metadata = LanceMetadataLoader.loadLatestForTvf( + "s3://bucket/clone.lance", Collections.emptyList()); + Assertions.assertEquals(7L, metadata.getVersion()); + Mockito.verify(dataset).getFragments(); + Mockito.verify(dataset).getSchema(); + Mockito.verify(dataset).close(); + } + } + + @Test + void isolatedRemoteManifestRejectsLocalBaseAndKeepsRemoteReads() throws Exception { + String clone = System.getenv("DORIS_LANCE_TEST_REMOTE_CLONE_URI"); + String source = System.getenv("DORIS_LANCE_TEST_REMOTE_SOURCE_URI"); + Assumptions.assumeTrue(clone != null && source != null, + "Requires an explicitly selected isolated S3 storage-policy fixture"); + Map properties = new HashMap<>(); + properties.put("s3.endpoint", System.getenv("DORIS_LANCE_TEST_S3_ENDPOINT")); + properties.put("s3.access_key", System.getenv("DORIS_LANCE_TEST_S3_ACCESS_KEY")); + properties.put("s3.secret_key", System.getenv("DORIS_LANCE_TEST_S3_SECRET_KEY")); + properties.put("s3.region", "us-east-1"); + properties.put("use_path_style", "true"); + StorageProperties storage = StorageProperties.createPrimary(properties); + AnalysisException error = Assertions.assertThrows(AnalysisException.class, + () -> LanceMetadataLoader.loadLatestForTvf(clone, Collections.singletonList(storage))); + Assertions.assertTrue(error.getMessage().contains("remote manifest storage bases")); + LanceTableMetadata metadata = LanceMetadataLoader.loadLatestForTvf( + source, Collections.singletonList(storage)); + Assertions.assertTrue(metadata.getVersion() > 0); + Assertions.assertTrue(metadata.getSchema().getFields().stream() + .anyMatch(field -> "row_id".equals(field.getName()))); + } + + @Test + void catalogSnapshotReadKeepsItsConfiguredManifestBases() { + Dataset dataset = Mockito.mock(Dataset.class); + Mockito.when(dataset.version()).thenReturn(7L); + Mockito.when(dataset.getFragments()).thenReturn(Collections.emptyList()); + Mockito.when(dataset.getSchema()).thenReturn(new Schema(Collections.emptyList())); + LanceMetadataLoader.read(dataset, new LanceTableAccess("file:///warehouse/table.lance", + Collections.emptyMap()), LanceMetadataLoader.MetadataScope.BASIC); + Mockito.verify(dataset, Mockito.never()).getBasePaths(); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/tvf/source/TVFScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/tvf/source/TVFScanNodeTest.java index f6db0c87ea05e6..3db6d1ed02f1fe 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/tvf/source/TVFScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/tvf/source/TVFScanNodeTest.java @@ -160,6 +160,7 @@ public void testLanceSplitsPinVersionAndFragments() throws Exception { ExternalFileTableValuedFunction tvf = Mockito.mock(ExternalFileTableValuedFunction.class); Mockito.when(table.getTvf()).thenReturn(tvf); Mockito.when(tvf.isLanceFormat()).thenReturn(true); + Mockito.when(tvf.getTFileType()).thenReturn(TFileType.FILE_S3); Mockito.when(tvf.getFilePath()).thenReturn("s3://bucket/table.lance"); Mockito.when(tvf.getLanceDatasetVersion()).thenReturn(42L); Mockito.when(tvf.getLanceFragments()).thenReturn(Arrays.asList( @@ -181,15 +182,43 @@ public void testLanceSplitsPinVersionAndFragments() throws Exception { TFileRangeDesc range = new TFileRangeDesc(); node.setScanParams(range, first); + Assert.assertEquals(TFileType.FILE_S3, range.getFileType()); Assert.assertTrue(range.isSetTableFormatParams()); Assert.assertEquals("lance", range.getTableFormatParams().getTableFormatType()); Assert.assertEquals("s3://bucket/table.lance", range.getTableFormatParams().getLanceParams().getDatasetUri()); Assert.assertEquals(42L, range.getTableFormatParams().getLanceParams().getVersion()); + Assert.assertTrue(range.getTableFormatParams().getLanceParams().isIsTvf()); Assert.assertEquals(Collections.singletonList(7L), range.getTableFormatParams().getLanceParams().getFragmentIds()); } + @Test + public void testRemoteLanceSpacePathsPreserveScanRangeIdentity() throws Exception { + for (String uri : Arrays.asList("s3://bucket/sample data.lance", "s3://bucket/sample%20data.lance")) { + SessionVariable sv = new SessionVariable(); + TupleDescriptor desc = new TupleDescriptor(new TupleId(0)); + FunctionGenTable table = Mockito.mock(FunctionGenTable.class); + ExternalFileTableValuedFunction tvf = Mockito.mock(ExternalFileTableValuedFunction.class); + Mockito.when(table.getTvf()).thenReturn(tvf); + Mockito.when(tvf.isLanceFormat()).thenReturn(true); + Mockito.when(tvf.getTFileType()).thenReturn(TFileType.FILE_S3); + Mockito.when(tvf.getFilePath()).thenReturn(uri); + Mockito.when(tvf.getLanceDatasetVersion()).thenReturn(42L); + Mockito.when(tvf.getLanceFragments()).thenReturn( + Collections.singletonList(new LanceFragmentInfo(7, 1000, 1000))); + desc.setTable(table); + TVFScanNode node = new TVFScanNode(new PlanNodeId(0), desc, false, sv, ScanContext.EMPTY); + LanceSplit split = (LanceSplit) node.getSplits(1).get(0); + Assert.assertEquals(uri, split.getDatasetUri()); + TFileRangeDesc range = new TFileRangeDesc(); + node.setScanParams(range, split); + Assert.assertEquals(TFileType.FILE_S3, range.getFileType()); + Assert.assertEquals(uri, range.getTableFormatParams().getLanceParams().getDatasetUri()); + Assert.assertEquals(42L, range.getTableFormatParams().getLanceParams().getVersion()); + } + } + @Test public void testLocalLanceUsesOneLatestWholeDatasetSplit() throws Exception { SessionVariable sv = new SessionVariable(); @@ -214,9 +243,31 @@ public void testLocalLanceUsesOneLatestWholeDatasetSplit() throws Exception { TFileRangeDesc range = new TFileRangeDesc(); node.setScanParams(range, split); Assert.assertEquals(0L, range.getTableFormatParams().getLanceParams().getVersion()); + Assert.assertTrue(range.getTableFormatParams().getLanceParams().isIsTvf()); Assert.assertFalse(range.getTableFormatParams().getLanceParams().isSetFragmentIds()); } + @Test + public void testLocalLanceRangeRetainsFileTypeWithoutSchemeInference() throws Exception { + for (String path : Arrays.asList("data/table.lance", "/data/table.lance", "file:///data/table.lance")) { + SessionVariable sv = new SessionVariable(); + TupleDescriptor desc = new TupleDescriptor(new TupleId(0)); + FunctionGenTable table = Mockito.mock(FunctionGenTable.class); + ExternalFileTableValuedFunction tvf = Mockito.mock(ExternalFileTableValuedFunction.class); + Mockito.when(table.getTvf()).thenReturn(tvf); + Mockito.when(tvf.isLanceFormat()).thenReturn(true); + Mockito.when(tvf.getTFileType()).thenReturn(TFileType.FILE_LOCAL); + Mockito.when(tvf.getFilePath()).thenReturn(path); + desc.setTable(table); + TVFScanNode node = new TVFScanNode(new PlanNodeId(0), desc, false, sv, ScanContext.EMPTY); + TFileRangeDesc range = new TFileRangeDesc(); + node.setScanParams(range, node.getSplits(1).get(0)); + Assert.assertTrue(range.isSetFileType()); + Assert.assertEquals(TFileType.FILE_LOCAL, range.getFileType()); + Assert.assertEquals(path, range.getTableFormatParams().getLanceParams().getDatasetUri()); + } + } + // Verifies local Lance execution is restricted to the backend that provided its schema. @Test public void testLocalLancePinsExecutionToSchemaBackend() throws Exception { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindTVFTableSinkTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindTVFTableSinkTest.java index b76ba57ddb5303..5bf1c24ec32011 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindTVFTableSinkTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindTVFTableSinkTest.java @@ -17,25 +17,754 @@ package org.apache.doris.nereids.rules.analysis; +import org.apache.doris.catalog.Env; +import org.apache.doris.common.Config; +import org.apache.doris.fs.FileSystemFactory; +import org.apache.doris.mysql.privilege.AccessControllerManager; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.parser.NereidsParser; import org.apache.doris.nereids.trees.expressions.NamedExpression; import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.commands.ExplainCommand; import org.apache.doris.nereids.trees.plans.commands.insert.InsertIntoTVFCommand; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.trees.plans.logical.LogicalTVFTableSink; import org.apache.doris.nereids.util.MemoTestUtils; import org.apache.doris.nereids.util.PlanChecker; +import org.apache.doris.planner.PlanNodeId; +import org.apache.doris.planner.TVFTableSink; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.StmtExecutor; +import org.apache.doris.tablefunction.HadoopTvfAdmission; +import org.apache.doris.tablefunction.LocalTableValuedFunction; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; import java.util.List; +import java.util.Map; class BindTVFTableSinkTest { + private boolean originalEnableLocalTvf; + + @BeforeEach + void setUp() { + originalEnableLocalTvf = Config.enable_local_tvf; + Config.enable_local_tvf = false; + } + + @AfterEach + void tearDown() { + Config.enable_local_tvf = originalEnableLocalTvf; + } + + @Test + void disabledLocalSinkRejectsInsertAndExplainBeforeProperties() { + for (String prefix : Arrays.asList("", "EXPLAIN ")) { + for (String properties : Arrays.asList("'format'='csv'", + "'file_path'='/tmp/local_gate_', 'backend_id'='1', 'format'='csv'")) { + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> analyzeInsert(prefix + "INSERT INTO local(" + properties + ") SELECT 1")); + Assertions.assertTrue(exception.getMessage().contains("LOCAL TVF is disabled")); + } + } + } + + @Test + void enabledLocalSinkKeepsPropertyValidation() { + Config.enable_local_tvf = true; + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> analyzeInsert("INSERT INTO local('format'='csv') SELECT 1")); + Assertions.assertTrue(exception.getMessage().contains("requires 'file_path'")); + } + + @Test + void disabledLocalDoesNotBlockRemoteSinks() { + for (String name : Arrays.asList("s3", "hdfs")) { + Assertions.assertInstanceOf(LogicalTVFTableSink.class, + analyzeInsert("INSERT INTO " + name + "('file_path'='remote/data', 'format'='csv') SELECT 1")); + } + } + + @Test + void disabledLocalWriterRejectsRemoteInsertAndExplain() { + for (String prefix : Arrays.asList("", "EXPLAIN ")) { + for (String name : Arrays.asList("s3", "hdfs")) { + for (String writer : Arrays.asList("org.apache.doris.writer.LocalFileJniWriter", + "org/apache/doris/writer/LocalFileJniWriter")) { + for (String props : Arrays.asList("", "'file_path'='/tmp/local_gate_', 'format'='csv',")) { + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> analyzeInsert(prefix + "INSERT INTO " + name + "(" + props + + "'writer_class'='" + writer + "') SELECT 1")); + Assertions.assertTrue(exception.getMessage().contains("LOCAL TVF is disabled")); + } + } + } + } + } + + @Test + void directSinkBindingChecksEffectiveLocalWriter() throws Exception { + Map properties = new HashMap<>(); + properties.put("file_path", "s3://bucket/data"); + properties.put("format", "csv"); + properties.put("s3.endpoint", "http://127.0.0.1:9000"); + properties.put("s3.region", "us-east-1"); + properties.put("s3.access_key", "test-key"); + properties.put("s3.secret_key", "test-secret"); + for (String name : Arrays.asList("s3", "hdfs", "local")) { + for (String writer : Arrays.asList("org.apache.doris.writer.LocalFileJniWriter", + "org/apache/doris/writer/LocalFileJniWriter", + "org/apache/doris/writer/LocalFileJniWriter\0ignored")) { + properties.put("writer_class", writer); + TVFTableSink sink = new TVFTableSink(new PlanNodeId(0), name, properties, + Collections.emptyList()); + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, sink::bindDataSink); + Assertions.assertTrue(exception.getMessage().contains("LOCAL TVF is disabled")); + } + } + Config.enable_local_tvf = true; + properties.put("writer_class", "org.apache.doris.writer.LocalFileJniWriter"); + new TVFTableSink(new PlanNodeId(0), "s3", properties, Collections.emptyList()).bindDataSink(); + Config.enable_local_tvf = false; + properties.remove("writer_class"); + new TVFTableSink(new PlanNodeId(0), "s3", properties, Collections.emptyList()).bindDataSink(); + properties.put("writer_class", "org.apache.doris.writer.OtherWriter"); + Assertions.assertDoesNotThrow(() -> LocalTableValuedFunction.checkSinkEnabled("s3", properties)); + } + + @Test + void enabledLocalWriterKeepsRemotePropertyValidation() { + Config.enable_local_tvf = true; + for (String name : Arrays.asList("s3", "hdfs")) { + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> analyzeInsert("INSERT INTO " + name + + "('writer_class'='org.apache.doris.writer.LocalFileJniWriter') SELECT 1")); + Assertions.assertTrue(exception.getMessage().contains("requires 'file_path'")); + } + } + + @Test + void loadUserCannotExecuteLocalWriterWhileDisabled() { + ConnectContext context = MemoTestUtils.createConnectContext(); + String sql = "INSERT INTO s3('file_path'='/tmp/local_gate_', 'format'='csv'," + + "'writer_class'='org.apache.doris.writer.LocalFileJniWriter') SELECT 1"; + MemoTestUtils.createStatementContext(context, sql); + InsertIntoTVFCommand command = (InsertIntoTVFCommand) new NereidsParser().parseSingle(sql); + Env env = Mockito.spy(Env.getCurrentEnv()); + AccessControllerManager accessManager = Mockito.mock(AccessControllerManager.class); + Mockito.doReturn(accessManager).when(env).getAccessManager(); + Mockito.when(accessManager.checkGlobalPriv(context, PrivPredicate.LOAD)).thenReturn(true); + StmtExecutor executor = Mockito.mock(StmtExecutor.class); + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class, Mockito.CALLS_REAL_METHODS)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> command.run(context, executor)); + Assertions.assertTrue(exception.getMessage().contains("LOCAL TVF is disabled")); + Mockito.verify(accessManager).checkGlobalPriv(context, PrivPredicate.ADMIN); + Mockito.verify(accessManager).checkGlobalPriv(context, PrivPredicate.LOAD); + Mockito.verifyNoInteractions(executor); + } + } + + @Test + void disabledNativeHadoopLocalSinksRejectInsertAndExplain() { + for (String prefix : Arrays.asList("", "EXPLAIN ")) { + for (String storage : Arrays.asList("'fs.defaultFS'='file:///'", + "'fs.defaultFS'='hdfs://namenode:8020'," + + "'fs.hdfs.impl'='org.apache.hadoop.fs.LocalFileSystem'", + "'fs.defaultFS'='hdfs://namenode:8020'," + + "'fs.hdfs.impl'='org.apache.hadoop.fs.RawLocalFileSystem'", + "'fs.defaultFS'='hdfs://namenode:8020','fs.hdfs.impl'='${fs.file.impl}'," + + "'fs.file.impl'='org.apache.hadoop.fs.LocalFileSystem'", + "'fs.defaultFS'='hdfs://namenode:8020'," + + "'fs.hdfs.impl '='org.apache.hadoop.fs.LocalFileSystem'")) { + for (String cleanup : Arrays.asList("false", "true")) { + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> analyzeInsert(prefix + "INSERT INTO hdfs('file_path'='/tmp/local_gate_'," + + "'format'='csv','delete_existing_files'='" + cleanup + "'," + + storage + ") SELECT 1")); + Assertions.assertTrue(exception.getMessage().contains("LOCAL TVF is disabled")); + } + } + } + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> analyzeInsert("INSERT INTO hdfs('fs.defaultFS'='file:///') SELECT 1")); + Assertions.assertTrue(exception.getMessage().contains("requires 'file_path'")); + } + + @Test + void directNativeHadoopBindingHonorsLocalOptIn() throws Exception { + Map properties = new HashMap<>(); + properties.put("file_path", "/tmp/local_gate_"); + properties.put("format", "csv"); + for (String defaultFs : Arrays.asList("file:///", "default")) { + properties.put("fs.defaultFS", defaultFs); + TVFTableSink sink = new TVFTableSink(new PlanNodeId(0), "hdfs", properties, + Collections.emptyList()); + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, sink::bindDataSink); + Assertions.assertTrue(exception.getMessage().contains("LOCAL TVF is disabled")); + } + properties.put("fs.defaultFS", "hdfs://namenode:8020"); + properties.put("fs.hdfs.impl", "org.apache.hadoop.fs.RawLocalFileSystem"); + TVFTableSink sink = new TVFTableSink(new PlanNodeId(0), "hdfs", properties, Collections.emptyList()); + Assertions.assertThrows(AnalysisException.class, sink::bindDataSink); + Config.enable_local_tvf = true; + sink.bindDataSink(); + properties.put("fs.defaultFS", "file:///"); + sink.bindDataSink(); + Config.enable_local_tvf = false; + properties.put("fs.defaultFS", "hdfs://namenode:8020"); + properties.remove("fs.hdfs.impl"); + // Hadoop normally provides a local implementation for file; it is not selected by hdfs. + properties.put("fs.file.impl", "org.apache.hadoop.fs.LocalFileSystem"); + sink.bindDataSink(); + Assertions.assertInstanceOf(LogicalTVFTableSink.class, + analyzeInsert("INSERT INTO hdfs('file_path'='/tmp/remote_', 'format'='csv'," + + "'fs.defaultFS'='hdfs://namenode:8020'," + + "'fs.file.impl'='org.apache.hadoop.fs.LocalFileSystem') SELECT 1")); + } + + @Test + void normalizedHadoopConfigIsCheckedBeforeNativeBinding() throws Exception { + String originalConfigDir = Config.hadoop_config_dir; + String originalConfigProperty = System.getProperty("doris.hadoop.config.dir"); + Path directory = Files.createTempDirectory("tvf-hadoop-admission-"); + Path resource = directory.resolve("local-filesystem.xml"); + try { + Files.write(resource, ("fs.defaultFS" + + "file:///").getBytes(StandardCharsets.UTF_8)); + Config.hadoop_config_dir = directory.toString() + "/"; + Map properties = new HashMap<>(); + properties.put("file_path", "/tmp/local_gate_"); + properties.put("format", "csv"); + properties.put("fs.hdfs.support", "true"); + properties.put("hadoop.config.resources", resource.getFileName().toString()); + TVFTableSink sink = new TVFTableSink(new PlanNodeId(0), "hdfs", properties, + Collections.emptyList()); + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, sink::bindDataSink); + Assertions.assertTrue(exception.getMessage().contains("LOCAL TVF is disabled")); + Config.enable_local_tvf = true; + sink.bindDataSink(); + } finally { + Config.hadoop_config_dir = originalConfigDir; + if (originalConfigProperty == null) { + System.clearProperty("doris.hadoop.config.dir"); + } else { + System.setProperty("doris.hadoop.config.dir", originalConfigProperty); + } + Files.deleteIfExists(resource); + Files.deleteIfExists(directory); + } + } + + @Test + void hadoopCleanupChecksSelectedFilesystemWithoutOpeningIt() { + Map properties = new HashMap<>(); + properties.put("fs.defaultFS", "hdfs://namenode:8020"); + properties.put("fs.file.impl", "org.apache.hadoop.fs.LocalFileSystem"); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.JAVA_DELETE, + "/remote/path", properties)); + Assertions.assertThrows(AnalysisException.class, + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.JAVA_DELETE, + "file:///tmp/prefix_", properties)); + properties.remove("fs.defaultFS"); + for (String defaultFs : Arrays.asList("local", "file:///")) { + properties.put("fs.default.name", defaultFs); + Assertions.assertThrows(AnalysisException.class, + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.JAVA_DELETE, "/tmp/prefix_", properties)); + } + properties.clear(); + Assertions.assertThrows(AnalysisException.class, + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.JAVA_DELETE, "/tmp/prefix_", properties)); + properties.put("fs.defaultFS", "hdfs://namenode:8020"); + properties.put("fs.default.name", "file:///"); + Assertions.assertThrows(AnalysisException.class, + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.JAVA_DELETE, "/tmp/prefix_", properties)); + properties.put("fs.default.name", "hdfs://other-namenode:8020"); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.JAVA_DELETE, + "/remote/path", properties)); + properties.remove("fs.defaultFS"); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.JAVA_DELETE, + "/remote/path", properties)); + properties.clear(); + properties.put("fs.defaultFS", "viewfs://cluster"); + properties.put("fs.viewfs.impl", "org.apache.hadoop.fs.RawLocalFileSystem"); + Assertions.assertThrows(AnalysisException.class, + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.JAVA_DELETE, + "viewfs://cluster/tmp", properties)); + properties.clear(); + properties.put("fs.defaultFS", "${fs.selected.uri}"); + properties.put("fs.selected.uri", "file:///"); + Assertions.assertThrows(AnalysisException.class, + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.JAVA_DELETE, "/tmp/prefix_", properties)); + properties.put("fs.selected.uri", "hdfs://namenode:8020"); + Assertions.assertThrows(AnalysisException.class, + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.JAVA_DELETE, "/remote/path", properties)); + Config.enable_local_tvf = true; + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.JAVA_DELETE, + "/remote/path", properties)); + Config.enable_local_tvf = false; + properties.put("fs.defaultFS", "hdfs://namenode:8020"); + properties.put("fs.hdfs.impl", "${fs.selected.impl}"); + properties.put("fs.selected.impl", "org.apache.hadoop.fs.RawLocalFileSystem"); + Assertions.assertThrows(AnalysisException.class, + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, "", properties)); + properties.put("fs.selected.impl", "org.apache.hadoop.hdfs.DistributedFileSystem"); + Assertions.assertThrows(AnalysisException.class, + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, "", properties)); + Config.enable_local_tvf = true; + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, + "", properties)); + Config.enable_local_tvf = false; + properties.put("fs.hdfs.impl", "${fs.selected.impl}"); + properties.remove("fs.selected.impl"); + properties.put("fs.selected.impl ", "org.apache.hadoop.fs.LocalFileSystem"); + Assertions.assertThrows(AnalysisException.class, + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, "", properties)); + properties.put("fs.selected.impl ", "org.apache.hadoop.hdfs.DistributedFileSystem"); + Assertions.assertThrows(AnalysisException.class, + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, "", properties)); + Config.enable_local_tvf = true; + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, + "", properties)); + Config.enable_local_tvf = false; + // BE consumes canonical defaultFS as its NameNode; the old alias can redefine + // that key inside Hadoop. Do not infer this indirect provider from the FE's raw map. + properties.put("fs.default.name", "org.apache.hadoop.fs.LocalFileSystem"); + properties.put("fs.hdfs.impl", "${fs.defaultFS}"); + Assertions.assertThrows(AnalysisException.class, + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, "", properties)); + properties.put("fs.hdfs.impl", "${env:TVF_TEST_UNRESOLVED_IMPL}"); + Assertions.assertThrows(AnalysisException.class, + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, "", properties)); + Config.enable_local_tvf = true; + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, + "", properties)); + Config.enable_local_tvf = false; + properties.clear(); + properties.put("fs.defaultFS", "hdfs://namenode:8020"); + properties.put("fs.hdfs.impl\0ignored", "org.apache.hadoop.fs.LocalFileSystem\0ignored"); + Assertions.assertThrows(AnalysisException.class, + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, "", properties)); + } + + @Test + void cleanupEntryRejectsHadoopLocalStorageBeforeFilesystemCreation() throws Exception { + InsertIntoTVFCommand command = (InsertIntoTVFCommand) new NereidsParser().parseSingle( + "INSERT INTO s3('file_path'='/tmp/local_gate_', 'format'='csv') SELECT 1"); + Method cleanup = InsertIntoTVFCommand.class.getDeclaredMethod( + "deleteExistingFilesInFE", String.class, Map.class); + cleanup.setAccessible(true); + Map properties = new HashMap<>(); + properties.put("file_path", "/tmp/local_gate_"); + properties.put("format", "csv"); + properties.put("delete_existing_files", "true"); + properties.put("fs.defaultFS", "file:///"); + try (MockedStatic factory = Mockito.mockStatic(FileSystemFactory.class)) { + // The nominal S3 name must not hide Hadoop storage selected for FE-side cleanup. + InvocationTargetException exception = Assertions.assertThrows(InvocationTargetException.class, + () -> cleanup.invoke(command, "s3", properties)); + Assertions.assertInstanceOf(AnalysisException.class, exception.getCause()); + Assertions.assertTrue(exception.getCause().getMessage().contains("LOCAL TVF is disabled")); + factory.verifyNoInteractions(); + } + } + + @Test + void loadUserCannotReachHadoopLocalCleanupOrExecution() { + ConnectContext context = MemoTestUtils.createConnectContext(); + String sql = "INSERT INTO hdfs('file_path'='/tmp/local_gate_', 'format'='csv'," + + "'delete_existing_files'='true', 'fs.defaultFS'='file:///') SELECT 1"; + MemoTestUtils.createStatementContext(context, sql); + InsertIntoTVFCommand command = (InsertIntoTVFCommand) new NereidsParser().parseSingle(sql); + Env env = Mockito.spy(Env.getCurrentEnv()); + AccessControllerManager accessManager = Mockito.mock(AccessControllerManager.class); + Mockito.doReturn(accessManager).when(env).getAccessManager(); + Mockito.when(accessManager.checkGlobalPriv(context, PrivPredicate.LOAD)).thenReturn(true); + StmtExecutor executor = Mockito.mock(StmtExecutor.class); + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class, Mockito.CALLS_REAL_METHODS); + MockedStatic factory = Mockito.mockStatic(FileSystemFactory.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> command.run(context, executor)); + Assertions.assertTrue(exception.getMessage().contains("LOCAL TVF is disabled")); + Mockito.verify(accessManager).checkGlobalPriv(context, PrivPredicate.ADMIN); + Mockito.verify(accessManager).checkGlobalPriv(context, PrivPredicate.LOAD); + Mockito.verifyNoInteractions(executor); + factory.verifyNoInteractions(); + } + } + + @Test + void disabledViewFsLocalMountsRejectInsertAndExplain() { + for (String prefix : Arrays.asList("", "EXPLAIN ")) { + for (String cleanup : Arrays.asList("false", "true")) { + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> analyzeInsert(prefix + "INSERT INTO hdfs('file_path'='/local/sub/prefix_'," + + "'format'='csv','delete_existing_files'='" + cleanup + "'," + + "'fs.defaultFS'='viewfs://cluster'," + + "'fs.viewfs.mounttable.cluster.link./local'='file:///tmp') SELECT 1")); + Assertions.assertTrue(exception.getMessage().contains("LOCAL TVF is disabled")); + } + } + } + + @Test + void viewFsChecksSelectedTargetsAndPreservesRemoteMounts() throws Exception { + Map properties = new HashMap<>(); + properties.put("file_path", "/remote/prefix_"); + properties.put("format", "csv"); + properties.put("fs.defaultFS", "viewfs://cluster"); + properties.put("fs.viewfs.mounttable.cluster.link./remote", "hdfs://namenode:8020/base"); + properties.put("fs.viewfs.mounttable.cluster.link./local", "file:///tmp"); + properties.put("fs.viewfs.mounttable.cluster.linkFallback", "hdfs://namenode:8020/fallback"); + new TVFTableSink(new PlanNodeId(0), "hdfs", properties, Collections.emptyList()).bindDataSink(); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, + "/locality/prefix_", properties)); + // A prefix naming a mount point generates a sibling filename; a trailing slash writes inside it. + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, + "/local", properties)); + Assertions.assertInstanceOf(LogicalTVFTableSink.class, + analyzeInsert("INSERT INTO hdfs('file_path'='/local', 'format'='csv'," + + "'delete_existing_files'='true', 'fs.defaultFS'='viewfs://cluster'," + + "'fs.viewfs.mounttable.cluster.link./local'='file:///tmp'," + + "'fs.viewfs.mounttable.cluster.linkFallback'='hdfs://namenode:8020/base') SELECT 1")); + Assertions.assertThrows(AnalysisException.class, + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, "/local/", properties)); + properties.put("file_path", "/local/sub/prefix_"); + TVFTableSink sink = new TVFTableSink(new PlanNodeId(0), "hdfs", properties, Collections.emptyList()); + Assertions.assertThrows(AnalysisException.class, sink::bindDataSink); + Config.enable_local_tvf = true; + sink.bindDataSink(); + Config.enable_local_tvf = false; + properties.remove("fs.viewfs.mounttable.cluster.link./local"); + properties.put("fs.viewfs.mounttable.cluster.link./local/", "file:///tmp"); + Assertions.assertThrows(AnalysisException.class, + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, + "/local/sub/prefix_", properties)); + properties.remove("fs.viewfs.mounttable.cluster.link./local/"); + properties.put("fs.viewfs.mounttable.cluster.link./local", "file:///tmp"); + properties.put("fs.viewfs.mounttable.cluster.link./local/remote", "hdfs://namenode:8020/base"); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, + "/local/remote/prefix_", properties)); + properties.put("fs.defaultFS", "viewfs://cl%75ster"); + Assertions.assertThrows(AnalysisException.class, + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, + "/local/sub/prefix_", properties)); + properties.put("fs.defaultFS", "viewfs://cluster_name"); + properties.put("fs.viewfs.ignore.port.in.mount.table.name", "true"); + properties.put("fs.viewfs.mounttable.cluster_name.link./remote", "hdfs://namenode:8020/base"); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, + "/remote/prefix_", properties)); + properties.put("fs.defaultFS", "viewfs://cluster:8020"); + properties.put("fs.viewfs.ignore.port.in.mount.table.name", "true"); + Assertions.assertThrows(AnalysisException.class, + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, + "/local/sub/prefix_", properties)); + properties.clear(); + properties.put("fs.defaultFS", "viewfs:///"); + properties.put("fs.viewfs.mounttable.default.name.key", "named"); + properties.put("fs.viewfs.mounttable.named.link./local", "file:///tmp"); + Assertions.assertThrows(AnalysisException.class, + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, + "/local/sub/prefix_", properties)); + properties.clear(); + properties.put("fs.defaultFS", "viewfs://cluster"); + properties.put("fs.viewfs.mounttable.cluster.linkFallback", "file:///tmp"); + Assertions.assertThrows(AnalysisException.class, + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, + "/unmapped/prefix_", properties)); + properties.remove("fs.viewfs.mounttable.cluster.linkFallback"); + for (String key : Arrays.asList("linkMerge./local", + "linkNfly.minReplication=2,repairOnRead=true./local")) { + String mountKey = "fs.viewfs.mounttable.cluster." + key; + properties.put(mountKey, "hdfs://namenode:8020/base,file:///tmp"); + Assertions.assertThrows(AnalysisException.class, + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, + "/local/sub/prefix_", properties)); + properties.put(mountKey, "hdfs://namenode:8020/base,hdfs://other:8020/base"); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, + "/local/sub/prefix_", properties)); + properties.remove(mountKey); + } + properties.put("fs.viewfs.mounttable.cluster.linkMergeSlash", "file:///tmp"); + Assertions.assertThrows(AnalysisException.class, + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, + "/local/sub/prefix_", properties)); + properties.put("fs.viewfs.mounttable.cluster.linkMergeSlash", "hdfs://namenode:8020/base,archive"); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, + "/local/sub/prefix_", properties)); + properties.remove("fs.viewfs.mounttable.cluster.linkMergeSlash"); + properties.put("fs.viewfs.mounttable.cluster.link./local", "viewfs://inner/root"); + properties.put("fs.viewfs.mounttable.inner.link./root", "file:///tmp"); + Assertions.assertThrows(AnalysisException.class, + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, + "/local/sub/prefix_", properties)); + properties.put("fs.viewfs.mounttable.inner.link./root", "hdfs://namenode:8020/base"); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, + "/local/sub/prefix_", properties)); + properties.put("fs.viewfs.mounttable.inner.link./root", "viewfs://cluster/local"); + Assertions.assertThrows(AnalysisException.class, + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, + "/local/sub/prefix_", properties)); + properties.clear(); + properties.put("fs.defaultFS", "viewfs://A"); + properties.put("fs.viewfs.mounttable.A.link./a", "viewfs://B/base"); + properties.put("fs.viewfs.mounttable.A.link./a/b/c", "hdfs://namenode:8020/remote"); + properties.put("fs.viewfs.mounttable.B.link./base/d", "file:///tmp"); + properties.put("fs.viewfs.mounttable.B.link./base/b/d", "hdfs://namenode:8020/remote"); + properties.put("fs.viewfs.mounttable.B.link./base/z", "hdfs://namenode:8020/remote"); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, + "/a/z/sub/prefix_", properties)); + Assertions.assertThrows(AnalysisException.class, + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, + "/a/b/d/sub/prefix_", properties)); + properties.clear(); + properties.put("fs.defaultFS", "hdfs://cluster:8020"); + properties.put("fs.hdfs.impl", "org.apache.hadoop.fs.viewfs.ViewFileSystemOverloadScheme"); + properties.put("fs.viewfs.mounttable.cluster:8020.link./local", "hdfs://namenode:8020/base"); + properties.put("fs.viewfs.mounttable.cluster.link./local", "file:///tmp"); + Assertions.assertThrows(AnalysisException.class, + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, + "/local/sub/prefix_", properties)); + properties.clear(); + properties.put("fs.defaultFS", "viewfs://inherited"); + Assertions.assertThrows(AnalysisException.class, + () -> HadoopTvfAdmission.check(HadoopTvfAdmission.Access.NATIVE_WRITE, "/unknown/prefix_", properties)); + } + + @Test + void explicitRemoteViewFsResourceIsNormalizedBeforeAdmission() throws Exception { + String originalConfigDir = Config.hadoop_config_dir; + String originalConfigProperty = System.getProperty("doris.hadoop.config.dir"); + Path directory = Files.createTempDirectory("tvf-remote-viewfs-"); + Path resource = directory.resolve("remote.xml"); + try { + Files.write(resource, ("" + + "fs.viewfs.mounttable.cluster.link./data" + + "hdfs://remote/base") + .getBytes(StandardCharsets.UTF_8)); + Config.hadoop_config_dir = directory.toString() + "/"; + String statement = "INSERT INTO hdfs('file_path'='/data/sub/prefix_', 'format'='csv'," + + "'fs.defaultFS'='viewfs://cluster', 'hadoop.config.resources'='remote.xml') SELECT 1"; + try (MockedStatic factory = Mockito.mockStatic(FileSystemFactory.class)) { + for (String prefix : Arrays.asList("", "EXPLAIN ")) { + Assertions.assertInstanceOf(LogicalTVFTableSink.class, analyzeInsert(prefix + statement)); + } + Map properties = new HashMap<>(); + properties.put("file_path", "/data/sub/prefix_"); + properties.put("format", "csv"); + properties.put("fs.defaultFS", "viewfs://cluster"); + properties.put("hadoop.config.resources", "remote.xml"); + TVFTableSink sink = new TVFTableSink(new PlanNodeId(0), "hdfs", properties, + Collections.emptyList()); + Assertions.assertDoesNotThrow(sink::bindDataSink); + factory.verifyNoInteractions(); + } + } finally { + Config.hadoop_config_dir = originalConfigDir; + if (originalConfigProperty == null) { + System.clearProperty("doris.hadoop.config.dir"); + } else { + System.setProperty("doris.hadoop.config.dir", originalConfigProperty); + } + Files.deleteIfExists(resource); + Files.deleteIfExists(directory); + } + } + + @Test + void viewFsResourcesAndCleanupRejectBeforeFilesystemCreation() throws Exception { + String originalConfigDir = Config.hadoop_config_dir; + String originalConfigProperty = System.getProperty("doris.hadoop.config.dir"); + Path directory = Files.createTempDirectory("tvf-viewfs-admission-"); + Path resource = directory.resolve("viewfs.xml"); + try { + Files.write(resource, ("fs.defaultFS" + + "viewfs://cluster" + + "fs.viewfs.mounttable.cluster.link./local" + + "file:///tmp").getBytes(StandardCharsets.UTF_8)); + Config.hadoop_config_dir = directory.toString() + "/"; + Map properties = new HashMap<>(); + properties.put("file_path", "/local/sub/prefix_"); + properties.put("format", "csv"); + properties.put("fs.hdfs.support", "true"); + properties.put("hadoop.config.resources", resource.getFileName().toString()); + TVFTableSink sink = new TVFTableSink(new PlanNodeId(0), "hdfs", properties, + Collections.emptyList()); + Assertions.assertThrows(AnalysisException.class, sink::bindDataSink); + InsertIntoTVFCommand command = (InsertIntoTVFCommand) new NereidsParser().parseSingle( + "INSERT INTO s3('file_path'='/local/sub/prefix_', 'format'='csv') SELECT 1"); + Method cleanup = InsertIntoTVFCommand.class.getDeclaredMethod( + "deleteExistingFilesInFE", String.class, Map.class); + cleanup.setAccessible(true); + properties.put("delete_existing_files", "true"); + try (MockedStatic factory = Mockito.mockStatic(FileSystemFactory.class)) { + InvocationTargetException exception = Assertions.assertThrows(InvocationTargetException.class, + () -> cleanup.invoke(command, "s3", properties)); + Assertions.assertInstanceOf(AnalysisException.class, exception.getCause()); + factory.verifyNoInteractions(); + } + } finally { + Config.hadoop_config_dir = originalConfigDir; + if (originalConfigProperty == null) { + System.clearProperty("doris.hadoop.config.dir"); + } else { + System.setProperty("doris.hadoop.config.dir", originalConfigProperty); + } + Files.deleteIfExists(resource); + Files.deleteIfExists(directory); + } + } + + @Test + void loadUserCannotExecuteViewFsLocalWritesOrCleanup() { + ConnectContext context = MemoTestUtils.createConnectContext(); + String sql = "INSERT INTO hdfs('file_path'='/local/sub/prefix_', 'format'='csv'," + + "'delete_existing_files'='true','fs.defaultFS'='viewfs://cluster'," + + "'fs.viewfs.mounttable.cluster.link./local'='file:///tmp') SELECT 1"; + MemoTestUtils.createStatementContext(context, sql); + InsertIntoTVFCommand command = (InsertIntoTVFCommand) new NereidsParser().parseSingle(sql); + Env env = Mockito.spy(Env.getCurrentEnv()); + AccessControllerManager accessManager = Mockito.mock(AccessControllerManager.class); + Mockito.doReturn(accessManager).when(env).getAccessManager(); + Mockito.when(accessManager.checkGlobalPriv(context, PrivPredicate.LOAD)).thenReturn(true); + StmtExecutor executor = Mockito.mock(StmtExecutor.class); + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class, Mockito.CALLS_REAL_METHODS); + MockedStatic factory = Mockito.mockStatic(FileSystemFactory.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> command.run(context, executor)); + Assertions.assertTrue(exception.getMessage().contains("LOCAL TVF is disabled")); + Mockito.verify(accessManager).checkGlobalPriv(context, PrivPredicate.ADMIN); + Mockito.verify(accessManager).checkGlobalPriv(context, PrivPredicate.LOAD); + Mockito.verifyNoInteractions(executor); + factory.verifyNoInteractions(); + } + } + + @Test + void nativeUriPrefixesCannotSelectAnUncheckedLocalMount() throws Exception { + String properties = "'file_path'='hdfs://unrelated/remote/prefix_', 'format'='csv'," + + "'fs.defaultFS'='viewfs://cluster'," + + "'fs.viewfs.mounttable.cluster.link./remote'='hdfs://remote/base'," + + "'fs.viewfs.mounttable.cluster.link./hdfs:'='file:///tmp'"; + for (String prefix : Arrays.asList("", "EXPLAIN ")) { + Assertions.assertThrows(AnalysisException.class, + () -> analyzeInsert(prefix + "INSERT INTO hdfs(" + properties + ") SELECT 1")); + } + Map props = new HashMap<>(); + props.put("file_path", "hdfs://unrelated/remote/prefix_"); + props.put("format", "csv"); + props.put("fs.defaultFS", "viewfs://cluster"); + props.put("fs.viewfs.mounttable.cluster.link./remote", "hdfs://remote/base"); + props.put("fs.viewfs.mounttable.cluster.link./hdfs:", "file:///tmp"); + TVFTableSink sink = new TVFTableSink(new PlanNodeId(0), "hdfs", props, Collections.emptyList()); + Assertions.assertThrows(AnalysisException.class, sink::bindDataSink); + } + + @Test + void relativeViewFsCleanupRejectsBeforeFilesystemCreation() throws Exception { + Map properties = new HashMap<>(); + properties.put("file_path", "data/sub/prefix_"); + properties.put("format", "csv"); + properties.put("fs.hdfs.support", "true"); + properties.put("fs.defaultFS", "viewfs://cluster"); + properties.put("hadoop.username", "tvf"); + properties.put("fs.viewfs.mounttable.cluster.link./data", "hdfs://remote/base"); + properties.put("fs.viewfs.mounttable.cluster.link./user/tvf/data", "file:///tmp"); + InsertIntoTVFCommand command = (InsertIntoTVFCommand) new NereidsParser().parseSingle( + "INSERT INTO hdfs('file_path'='data/sub/prefix_', 'format'='csv') SELECT 1"); + Method cleanup = InsertIntoTVFCommand.class.getDeclaredMethod( + "deleteExistingFilesInFE", String.class, Map.class); + cleanup.setAccessible(true); + try (MockedStatic factory = Mockito.mockStatic(FileSystemFactory.class)) { + InvocationTargetException exception = Assertions.assertThrows(InvocationTargetException.class, + () -> cleanup.invoke(command, "hdfs", properties)); + Assertions.assertInstanceOf(AnalysisException.class, exception.getCause()); + factory.verifyNoInteractions(); + properties.put("file_path", " /local/prefix_"); + properties.put("fs.viewfs.mounttable.cluster.link./local", "file:///tmp"); + properties.put("fs.viewfs.mounttable.cluster.linkFallback", "hdfs://remote/base"); + exception = Assertions.assertThrows(InvocationTargetException.class, + () -> cleanup.invoke(command, "hdfs", properties)); + Assertions.assertInstanceOf(AnalysisException.class, exception.getCause()); + factory.verifyNoInteractions(); + } + } + + @Test + void emptyViewFsTargetRejectsBeforeCleanupFilesystemCreation() throws Exception { + Map properties = new HashMap<>(); + properties.put("file_path", "/data/sub/prefix_"); + properties.put("format", "csv"); + properties.put("fs.defaultFS", "viewfs://cluster"); + properties.put("fs.viewfs.mounttable.cluster.link./data", ""); + properties.put("fs.viewfs.mounttable.cluster.linkFallback", "file:///tmp"); + InsertIntoTVFCommand command = (InsertIntoTVFCommand) new NereidsParser().parseSingle( + "INSERT INTO hdfs('file_path'='/data/sub/prefix_', 'format'='csv') SELECT 1"); + Method cleanup = InsertIntoTVFCommand.class.getDeclaredMethod( + "deleteExistingFilesInFE", String.class, Map.class); + cleanup.setAccessible(true); + try (MockedStatic factory = Mockito.mockStatic(FileSystemFactory.class)) { + InvocationTargetException exception = Assertions.assertThrows(InvocationTargetException.class, + () -> cleanup.invoke(command, "hdfs", properties)); + Assertions.assertInstanceOf(AnalysisException.class, exception.getCause()); + factory.verifyNoInteractions(); + } + } + + @Test + void remoteHostPortCleanupRemainsAvailableWithLocalDisabled() throws Exception { + Map properties = new HashMap<>(); + properties.put("file_path", "/exports/prefix_"); + properties.put("format", "csv"); + properties.put("fs.hdfs.support", "true"); + properties.put("fs.defaultFS", "namenode:8020"); + InsertIntoTVFCommand command = (InsertIntoTVFCommand) new NereidsParser().parseSingle( + "INSERT INTO hdfs('file_path'='/exports/prefix_', 'format'='csv') SELECT 1"); + Method cleanup = InsertIntoTVFCommand.class.getDeclaredMethod( + "deleteExistingFilesInFE", String.class, Map.class); + cleanup.setAccessible(true); + org.apache.doris.fs.remote.RemoteFileSystem remote = Mockito.mock( + org.apache.doris.fs.remote.RemoteFileSystem.class); + Mockito.when(remote.deleteDirectory("/exports/")).thenReturn(org.apache.doris.backup.Status.OK); + try (MockedStatic factory = Mockito.mockStatic(FileSystemFactory.class)) { + factory.when(() -> FileSystemFactory.get(Mockito.any( + org.apache.doris.datasource.property.storage.StorageProperties.class))).thenReturn(remote); + Assertions.assertDoesNotThrow(() -> cleanup.invoke(command, "hdfs", properties)); + Mockito.verify(remote).deleteDirectory("/exports/"); + } + } + + private Plan analyzeInsert(String sql) { + ConnectContext context = MemoTestUtils.createConnectContext(); + LogicalPlan parsed = new NereidsParser().parseSingle(sql); + if (parsed instanceof ExplainCommand) { + parsed = ((ExplainCommand) parsed).getLogicalPlan(); + } + InsertIntoTVFCommand command = (InsertIntoTVFCommand) parsed; + return PlanChecker.from(context).analyze(command.getExplainPlan(context)).getPlan(); + } @Test void duplicateOutputNamesKeepDistinctPositions() { + Config.enable_local_tvf = true; ConnectContext context = MemoTestUtils.createConnectContext(); InsertIntoTVFCommand command = (InsertIntoTVFCommand) new NereidsParser().parseSingle( "INSERT INTO local(" diff --git a/fe/fe-core/src/test/java/org/apache/doris/tablefunction/HadoopTvfAdmissionTest.java b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/HadoopTvfAdmissionTest.java new file mode 100644 index 00000000000000..2b7e787acf1d88 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/HadoopTvfAdmissionTest.java @@ -0,0 +1,461 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.tablefunction; + +import org.apache.doris.common.Config; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.tablefunction.HadoopTvfAdmission.Access; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.HashMap; +import java.util.Map; + +class HadoopTvfAdmissionTest { + private boolean originalEnable; + + @BeforeEach + void setUp() { + originalEnable = Config.enable_local_tvf; + Config.enable_local_tvf = false; + } + + @AfterEach + void tearDown() { + Config.enable_local_tvf = originalEnable; + } + + @Test + void javaMountKeysPreserveNulWhileNativeKeysAreCStringPrefixes() { + Map properties = viewFs(); + properties.put("fs.viewfs.mounttable.cluster.link./data\0ignored", "hdfs://remote/base"); + properties.put("fs.viewfs.mounttable.cluster.linkFallback", "file:///tmp"); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, "/data/sub/prefix_", properties)); + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + Access.JAVA_DELETE, "/data/sub/", properties)); + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + Access.HADOOP_READ, "viewfs://cluster/data/sub/file.csv", properties)); + } + + @Test + void javaImplementationKeysAreNotTruncatedAtNul() { + Map properties = new HashMap<>(); + properties.put("fs.defaultFS", "hdfs://remote"); + properties.put("fs.hdfs.impl\0ignored", "org.apache.hadoop.fs.LocalFileSystem\0ignored"); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.JAVA_DELETE, "/data/sub/", properties)); + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, "/data/sub/prefix_", properties)); + } + + @Test + void nativeAuthorityCompletionUsesTheRemainingLegacyConfiguration() { + Map properties = new HashMap<>(); + properties.put("fs.defaultFS", "viewfs:///"); + properties.put("fs.default.name", "viewfs://local"); + properties.put("fs.viewfs.mounttable.default.link./data", "hdfs://remote/base"); + properties.put("fs.viewfs.mounttable.local.link./data", "file:///tmp"); + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, "/data/sub/prefix_", properties)); + properties.put("fs.viewfs.mounttable.local.link./data", "hdfs://remote/base"); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, "/data/sub/prefix_", properties)); + } + + @Test + void distributedViewWrappersCannotBeMistakenForDirectRemoteProviders() { + Map properties = new HashMap<>(); + properties.put("fs.defaultFS", "hdfs://cluster"); + properties.put("fs.hdfs.impl", "org.apache.hadoop.hdfs.ViewDistributedFileSystem"); + properties.put("fs.viewfs.mounttable.cluster.link./data", "file:///tmp"); + for (Access access : Access.values()) { + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + access, "hdfs://cluster/data/sub/file.csv", properties)); + } + Config.enable_local_tvf = true; + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, "/data/sub/prefix_", properties)); + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + Access.HADOOP_READ, "hdfs://cluster/data/sub/file.csv", properties)); + } + + @Test + void hadoopReadsRejectLocalDelegationEvenWithLocalOptIn() { + Map properties = viewFs(); + properties.put("fs.viewfs.mounttable.cluster.link./data", "file:///tmp"); + for (boolean enabled : new boolean[] {false, true}) { + Config.enable_local_tvf = enabled; + AnalysisException error = Assertions.assertThrows(AnalysisException.class, + () -> HadoopTvfAdmission.check(Access.HADOOP_READ, + "viewfs://cluster/data/sub/file.csv", properties)); + Assertions.assertTrue(error.getMessage().contains("Use local()")); + } + } + + @Test + void remoteReadsDoNotRejectAnUnrelatedLocalSiblingMount() { + Map properties = viewFs(); + properties.put("fs.viewfs.mounttable.cluster.link./data", "hdfs://remote/base"); + properties.put("fs.viewfs.mounttable.cluster.link./local", "file:///tmp"); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.HADOOP_READ, "viewfs://cluster/data/file.csv", properties)); + } + + @Test + void readPatternsCheckEveryIntersectingMountBeforeListing() { + Map properties = viewFs(); + properties.put("fs.viewfs.mounttable.cluster.link./data", "hdfs://remote/base"); + properties.put("fs.viewfs.mounttable.cluster.link./data/local", "file:///tmp"); + for (String pattern : new String[] {"/data/*", "/data/?/file.csv", "/data/{a,b}/file.csv", "/data"}) { + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + Access.HADOOP_READ, "viewfs://cluster" + pattern, properties)); + } + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.HADOOP_READ, "viewfs://cluster/data/remote/file.csv", properties)); + properties.put("fs.viewfs.mounttable.cluster.link./data/local", "hdfs://remote/other"); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.HADOOP_READ, "viewfs://cluster/data/*", properties)); + } + + @Test + void schemaAndDataReadNameNodesAreCheckedIndependently() { + Map properties = new HashMap<>(); + properties.put("fs.defaultFS", "viewfs://schema"); + properties.put("fs.viewfs.mounttable.schema.link./viewfs:", "file:///tmp"); + properties.put("fs.viewfs.mounttable.data.link./data", "hdfs://remote/base"); + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + Access.HADOOP_READ, "viewfs://data/data/file.csv", properties)); + properties.put("fs.viewfs.mounttable.schema.link./viewfs:", "hdfs://remote/schema"); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.HADOOP_READ, "viewfs://data/data/file.csv", properties)); + } + + @Test + void unknownImplementationsAndRuntimeMountsNeedExplicitWriteOptIn() { + Map properties = new HashMap<>(); + properties.put("fs.defaultFS", "hdfs://remote"); + properties.put("fs.hdfs.impl", "example.DynamicFileSystem"); + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, "/data/prefix_", properties)); + Config.enable_local_tvf = true; + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, "/data/prefix_", properties)); + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + Access.HADOOP_READ, "hdfs://remote/data/file.csv", properties)); + } + + @Test + void nativeProjectionRejectsConflictingCStringKeys() { + Map properties = viewFs(); + properties.put("fs.viewfs.mounttable.cluster.link./data", "hdfs://remote/base"); + properties.put("fs.viewfs.mounttable.cluster.link./data\0ignored", "file:///tmp"); + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, "/data/sub/prefix_", properties)); + } + + @Test + void nativeNameNodeCaseCollisionsCannotChooseAnUncheckedProvider() { + Map properties = new HashMap<>(); + properties.put("fs.defaultFS", "hdfs://remote"); + properties.put("fs.DEFAULTFS", "file:///"); + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, "/data/prefix_", properties)); + } + + @Test + void nestedRemoteReadPatternsReuseTheSameMountInterpreter() { + Map properties = viewFs(); + properties.put("fs.viewfs.mounttable.cluster.link./data", "viewfs://inner/base"); + properties.put("fs.viewfs.mounttable.inner.link./base", "hdfs://remote/base"); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.HADOOP_READ, "viewfs://cluster/data/*", properties)); + properties.put("fs.viewfs.mounttable.inner.link./base/private", "file:///tmp"); + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + Access.HADOOP_READ, "viewfs://cluster/data/*", properties)); + } + + @Test + void emptyNativeWriteNameNodeKeepsExistingAnalysisWithoutSelectingLocal() { + Map properties = new HashMap<>(); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, "remote/data", properties)); + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + Access.HADOOP_READ, "hdfs://remote/data", properties)); + properties.put("fs.defaultFS", "default"); + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, "remote/data", properties)); + properties.put("fs.defaultFS", "\0ignored"); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, "remote/data", properties)); + } + + @Test + void builtInRemoteProviderImplementationsRemainAvailable() { + for (String[] provider : new String[][] { + {"oss", "com.aliyun.jindodata.oss.JindoOssFileSystem"}, + {"obs", "org.apache.hadoop.fs.obs.OBSFileSystem"}}) { + Map properties = new HashMap<>(); + properties.put("fs.defaultFS", provider[0] + "://bucket"); + properties.put("fs." + provider[0] + ".impl", provider[1]); + for (Access access : Access.values()) { + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + access, provider[0] + "://bucket/data/file.csv", properties)); + } + } + } + + @Test + void nativePathsAreRootedBeforeHadoopUriParsing() { + Map properties = viewFs(); + properties.put("fs.viewfs.mounttable.cluster.link./remote", "hdfs://remote/base"); + properties.put("fs.viewfs.mounttable.cluster.link./hdfs:", "file:///tmp"); + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, "hdfs://unrelated/remote/prefix_", properties)); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, "remote/prefix_", properties)); + + properties.put("fs.defaultFS", "viewfs://schema"); + properties.put("fs.viewfs.mounttable.data.link./remote", "hdfs://remote/base"); + properties.put("fs.viewfs.mounttable.schema.link./remote", "hdfs://remote/base"); + properties.put("fs.viewfs.mounttable.schema.link./viewfs:", "file:///tmp"); + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + Access.HADOOP_READ, "viewfs://data/remote/file.csv", properties)); + properties.put("fs.viewfs.mounttable.schema.link./viewfs:", "hdfs://remote/schema"); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.HADOOP_READ, "viewfs://data/remote/file.csv", properties)); + } + + @Test + void nonPrefixNameNodeMatchesRequireAnEstablishedNativePath() { + String nameNode = "viewfs://clüster"; + Map properties = new HashMap<>(); + properties.put("fs.defaultFS", nameNode); + properties.put("fs.viewfs.mounttable.clüster.link./x", "hdfs://remote/base"); + properties.put("fs.viewfs.mounttable.clüster.link./local", "file:///tmp"); + String path = "abcdefghijklmnopx/local/prefix_" + nameNode + "/a"; + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, path, properties)); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, nameNode + "/x/prefix_", properties)); + Config.enable_local_tvf = true; + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, path, properties)); + } + + @Test + void relativeJavaViewFsPathsCannotAssumeTheFilesystemRoot() { + Map properties = viewFs(); + properties.put("fs.viewfs.mounttable.cluster.link./data", "hdfs://remote/base"); + properties.put("fs.viewfs.mounttable.cluster.link./user/tvf/data", "file:///tmp"); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, "data/sub/prefix_", properties)); + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + Access.JAVA_DELETE, "data/sub/", properties)); + properties.put("fs.defaultFS", "hdfs://remote"); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.JAVA_DELETE, "data/sub/", properties)); + properties.put("fs.defaultFS", "viewfs://cluster"); + Config.enable_local_tvf = true; + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.JAVA_DELETE, "data/sub/", properties)); + } + + @Test + void javaListingChecksTheTrimmedConsumerPath() { + Map properties = viewFs(); + properties.put("fs.viewfs.mounttable.cluster.link./data", "file:///tmp"); + properties.put("fs.viewfs.mounttable.cluster.link./data/file.csv /", "hdfs://remote/base"); + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + Access.HADOOP_READ, "viewfs://cluster/data/file.csv ", properties)); + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + Access.JAVA_DELETE, " viewfs://cluster/data/sub/", properties)); + properties.put("fs.viewfs.mounttable.cluster.link./data", "hdfs://remote/base"); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.HADOOP_READ, "viewfs://cluster/data/file.csv ", properties)); + } + + @Test + void selectedMountTargetsMustNotBeEmpty() { + Map properties = viewFs(); + properties.put("fs.viewfs.mounttable.cluster.linkFallback", "file:///tmp"); + for (String target : new String[] {"", ","}) { + properties.put("fs.viewfs.mounttable.cluster.link./data", target); + for (Access access : new Access[] {Access.NATIVE_WRITE, Access.JAVA_DELETE}) { + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + access, "/data/sub/prefix_", properties)); + } + } + properties.put("fs.viewfs.mounttable.cluster.link./data", "hdfs://remote/base,"); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, "/data/sub/prefix_", properties)); + properties.remove("fs.viewfs.mounttable.cluster.link./data"); + properties.put("fs.viewfs.mounttable.cluster.linkMerge./data", "hdfs://remote/base,"); + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, "/data/sub/prefix_", properties)); + properties.remove("fs.viewfs.mounttable.cluster.linkMerge./data"); + properties.put("fs.viewfs.mounttable.cluster.link./data", "hdfs://remote/base"); + properties.put("fs.viewfs.mounttable.cluster.link./unused", ""); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, "/data/sub/prefix_", properties)); + } + + @Test + void nativeWriterRejectsCaseAndNulMapCollisions() { + Map properties = viewFs(); + String upper = "fs.viewfs.mounttable.CLUSTER.link./data\0A"; + String lower = "fs.viewfs.mounttable.cluster.link./data\0b"; + properties.put(upper, "hdfs://remote/base"); + properties.put(lower, "hdfs://remote/base"); + properties.put("fs.viewfs.mounttable.cluster.linkFallback", "file:///tmp"); + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, "/data/sub/prefix_", properties)); + properties.put("fs.viewfs.mounttable.cluster.link./public", "hdfs://remote/public"); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.HADOOP_READ, "viewfs://cluster/public/file.csv", properties)); + properties.remove(upper); + properties.put(upper + "long", "hdfs://remote/base"); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, "/data/sub/prefix_", properties)); + properties.remove(upper + "long"); + properties.put(upper, "hdfs://remote/base"); + Config.enable_local_tvf = true; + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, "/data/sub/prefix_", properties)); + Config.enable_local_tvf = false; + properties.remove(upper); + properties.remove(lower); + properties.put("fs.viewfs.mounttable.cluster.link./data", "hdfs://remote/base"); + properties.put("dfs.test.VALUE\0a", "one"); + properties.put("dfs.test.value\0b", "two"); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, "/data/sub/prefix_", properties)); + } + + @Test + void crossMountGlobsRequireAnExplicitRemoteRoute() { + Map properties = viewFs(); + properties.put("fs.viewfs.mounttable.cluster.link./data", "hdfs://remote/base"); + properties.put("fs.viewfs.mount.links.as.symlinks", "false"); + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + Access.HADOOP_READ, "viewfs://cluster/*/*.csv", properties)); + properties.put("fs.viewfs.mounttable.cluster.linkFallback", "hdfs://remote/fallback"); + for (String pattern : new String[] {"/*/*.csv", "/d*/*.csv", "/{data,other}/*.csv"}) { + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.HADOOP_READ, "viewfs://cluster" + pattern, properties)); + } + properties.put("fs.viewfs.mounttable.cluster.link./private", "file:///tmp"); + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + Access.HADOOP_READ, "viewfs://cluster/*/*.csv", properties)); + properties.remove("fs.viewfs.mounttable.cluster.link./private"); + properties.put("fs.viewfs.mounttable.cluster.linkFallback", "file:///tmp"); + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + Access.HADOOP_READ, "viewfs://cluster/*/*.csv", properties)); + properties.remove("fs.viewfs.mounttable.cluster.linkFallback"); + properties.put("fs.viewfs.mounttable.cluster.link./data", ""); + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + Access.HADOOP_READ, "viewfs://cluster/*/*.csv", properties)); + } + + @Test + void defaultFilesystemHostPortShorthandRemainsRemote() { + for (String key : new String[] {"fs.defaultFS", "fs.default.name"}) { + Map properties = new HashMap<>(); + properties.put(key, "namenode:8020"); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.JAVA_DELETE, "/exports/", properties)); + properties.put(key, "local"); + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + Access.JAVA_DELETE, "/exports/", properties)); + } + } + + @Test + void singleViewFsTargetsKeepCommasWhileMultiTargetsStillRejectLocal() { + for (String key : new String[] {"link./data", "linkFallback", "linkMergeSlash"}) { + Map properties = viewFs(); + properties.put("fs.viewfs.mounttable.cluster." + key, "hdfs://remote/base,archive"); + for (Access access : Access.values()) { + String path = access == Access.HADOOP_READ + ? "viewfs://cluster/data/file.csv" : "/data/prefix_"; + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check(access, path, properties)); + } + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.HADOOP_READ, "viewfs://cluster/data/*.csv", properties)); + } + for (String key : new String[] {"linkMerge./data", "linkNfly.minReplication=2./data"}) { + Map properties = viewFs(); + properties.put("fs.viewfs.mounttable.cluster." + key, "hdfs://remote/base,file:///tmp"); + for (Access access : Access.values()) { + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + access, "viewfs://cluster/data/file.csv", properties)); + } + } + } + + @Test + void nativeRoutesRejectSupplementaryCharactersBeforeJniDecoding() { + Map properties = viewFs(); + properties.put("fs.viewfs.mounttable.cluster.link./data\uD83D\uDE00AAA", "hdfs://remote/base"); + properties.put("fs.viewfs.mounttable.cluster.linkFallback", "file:///tmp"); + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, "/data\uD83D\uDE00AAA/prefix_", properties)); + // Java deletion does not cross JNI and preserves this otherwise valid URI identity. + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.JAVA_DELETE, "/data\uD83D\uDE00AAA/", properties)); + Map bmp = viewFs(); + bmp.put("fs.viewfs.mounttable.cluster.link./数据", "hdfs://remote/目录"); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, "/数据/prefix_", bmp)); + Config.enable_local_tvf = true; + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, "/data\uD83D\uDE00AAA/prefix_", properties)); + } + + @Test + void nativeCharacterChecksCoverRoutingKeysValuesAndReadPaths() { + Map properties = viewFs(); + properties.put("fs.viewfs.mounttable.cluster.link./data", "hdfs://remote/base"); + for (String value : new String[] {"hdfs://remote/base\uD83D\uDE00AAA", "hdfs://remote/base\uD800"}) { + properties.put("fs.viewfs.mounttable.cluster.link./data", value); + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, "/data/prefix_", properties)); + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + Access.HADOOP_READ, "viewfs://cluster/data/file.csv", properties)); + } + properties.put("fs.viewfs.mounttable.cluster.link./data", "hdfs://remote/base"); + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + Access.HADOOP_READ, "viewfs://cluster/data/\uD83D\uDE00AAA.csv", properties)); + properties.put("fs.viewfs.mounttable.cluster.link./unused\uD83D\uDE00AAA", "hdfs://remote/unused"); + Assertions.assertThrows(AnalysisException.class, () -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, "/data/prefix_", properties)); + properties.remove("fs.viewfs.mounttable.cluster.link./unused\uD83D\uDE00AAA"); + properties.put("fs.viewfs.mounttable.cluster.link./data", "hdfs://remote/base\0\uD83D\uDE00AAA"); + Assertions.assertDoesNotThrow(() -> HadoopTvfAdmission.check( + Access.NATIVE_WRITE, "/data/prefix_", properties)); + } + + private static Map viewFs() { + Map properties = new HashMap<>(); + properties.put("fs.defaultFS", "viewfs://cluster"); + return properties; + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/tablefunction/HadoopTvfReadAdmissionTest.java b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/HadoopTvfReadAdmissionTest.java new file mode 100644 index 00000000000000..b9032d0fc2c429 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/HadoopTvfReadAdmissionTest.java @@ -0,0 +1,345 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.tablefunction; + +import org.apache.doris.common.Config; +import org.apache.doris.common.FeConstants; +import org.apache.doris.fs.FileSystemFactory; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.thrift.TBrokerFileStatus; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +class HadoopTvfReadAdmissionTest { + private boolean originalEnable; + private boolean originalUnitTest; + + @BeforeEach + void setUp() { + originalEnable = Config.enable_local_tvf; + originalUnitTest = FeConstants.runningUnitTest; + FeConstants.runningUnitTest = false; + } + + @AfterEach + void tearDown() { + Config.enable_local_tvf = originalEnable; + FeConstants.runningUnitTest = originalUnitTest; + } + + @Test + void hdfsAndFileRejectLocalRoutesBeforeFilesystemCreation() { + Map properties = properties("local"); + for (boolean enabled : new boolean[] {false, true}) { + Config.enable_local_tvf = enabled; + try (MockedStatic factory = Mockito.mockStatic(FileSystemFactory.class)) { + Assertions.assertThrows(AnalysisException.class, + () -> new HdfsTableValuedFunction(properties)); + Assertions.assertThrows(AnalysisException.class, + () -> new FileTableValuedFunction(properties)); + factory.verifyNoInteractions(); + } + } + } + + @Test + void remoteReadAdmissionAllowsTheExistingListingPath() throws Exception { + Config.enable_local_tvf = false; + HdfsTableValuedFunction tvf = new ListedHdfsTvf(properties("remote"), false); + Assertions.assertEquals(1, tvf.getFileStatuses().size()); + Assertions.assertEquals("viewfs://review/remote/listed.csv", tvf.getFileStatuses().get(0).getPath()); + } + + @Test + void returnedFilePathsAreCheckedBeforeSchemaOrDataAccess() { + Config.enable_local_tvf = true; + Assertions.assertThrows(AnalysisException.class, + () -> new ListedHdfsTvf(properties("remote"), true)); + } + + @Test + void fileKeepsTheBuiltInOssHdfsStorageSelection() throws Exception { + FeConstants.runningUnitTest = true; + Config.enable_local_tvf = false; + Map properties = new HashMap<>(); + properties.put("uri", "oss://bucket/file.csv"); + properties.put("fs.defaultFS", "oss://bucket"); + properties.put("fs.oss-hdfs.support", "true"); + properties.put("oss.endpoint", "cn-beijing.oss-dls.aliyuncs.com"); + properties.put("oss.access_key", "test-access-key"); + properties.put("oss.secret_key", "test-secret-key"); + properties.put("format", "csv"); + properties.put("csv_schema", "id:int"); + FileTableValuedFunction file = new FileTableValuedFunction(properties); + Assertions.assertEquals(org.apache.doris.thrift.TFileType.FILE_HDFS, file.getTFileType()); + Assertions.assertEquals("com.aliyun.jindodata.oss.JindoOssFileSystem", + file.getBackendConnectProperties().get("fs.oss.impl")); + S3TableValuedFunction s3 = new S3TableValuedFunction(properties); + Assertions.assertEquals("com.aliyun.jindodata.oss.JindoOssFileSystem", + s3.getBackendConnectProperties().get("fs.oss.impl")); + } + + @Test + void listingPathTrimmingCannotSelectAnUncheckedLocalMount() { + Config.enable_local_tvf = false; + Map properties = properties("local"); + properties.put("uri", "viewfs://review/local/file.csv "); + properties.put("fs.viewfs.mounttable.review.link./local/file.csv /", "hdfs://remote/base"); + try (MockedStatic factory = Mockito.mockStatic(FileSystemFactory.class)) { + Assertions.assertThrows(AnalysisException.class, () -> new HdfsTableValuedFunction(properties)); + Assertions.assertThrows(AnalysisException.class, () -> new FileTableValuedFunction(properties)); + factory.verifyNoInteractions(); + } + } + + @Test + void s3CsvWithHadoopStorageRejectsLocalRoutesBeforeListing() { + Map properties = properties("local"); + properties.put("fs.hdfs.support", "true"); + for (boolean enabled : new boolean[] {false, true}) { + Config.enable_local_tvf = enabled; + try (MockedStatic factory = Mockito.mockStatic(FileSystemFactory.class)) { + Assertions.assertThrows(AnalysisException.class, + () -> new ListedS3Tvf(properties, "local")); + factory.verifyNoInteractions(); + } + } + } + + @Test + void s3HadoopProvidedFilesAreCheckedInBothConstructorBranches() { + Config.enable_local_tvf = true; + Map properties = properties("remote"); + properties.put("fs.hdfs.support", "true"); + TBrokerFileStatus local = new TBrokerFileStatus("viewfs://review/local/file.csv", false, 10, true); + for (boolean unitTest : new boolean[] {false, true}) { + FeConstants.runningUnitTest = unitTest; + try (MockedStatic factory = Mockito.mockStatic(FileSystemFactory.class)) { + Assertions.assertThrows(AnalysisException.class, + () -> new S3TableValuedFunction(properties, Collections.singletonList(local))); + factory.verifyNoInteractions(); + } + } + } + + @Test + void s3HadoopRemoteCsvKeepsDeclaredSchemaAndHdfsRanges() throws Exception { + Config.enable_local_tvf = false; + Map properties = properties("remote"); + properties.put("fs.hdfs.support", "true"); + S3TableValuedFunction tvf = new ListedS3Tvf(properties, "remote"); + Assertions.assertEquals("id", tvf.getTableColumns().get(0).getName()); + Assertions.assertEquals(1, tvf.getFileStatuses().size()); + String path = tvf.getFileStatuses().get(0).getPath(); + Assertions.assertEquals("viewfs://review/remote/listed.csv", path); + Assertions.assertEquals(org.apache.doris.thrift.TFileType.FILE_HDFS, + org.apache.doris.common.util.LocationPath.getTFileTypeForBE(path)); + } + + @Test + void explicitRemoteFallbackKeepsCrossMountGlobAdmission() throws Exception { + Config.enable_local_tvf = false; + Map properties = properties("remote"); + properties.put("uri", "viewfs://review/*/*.csv"); + properties.remove("fs.viewfs.mounttable.review.link./local"); + properties.put("fs.viewfs.mount.links.as.symlinks", "false"); + properties.put("fs.viewfs.mounttable.review.linkFallback", "hdfs://remote/fallback"); + Assertions.assertEquals(1, new ListedHdfsTvf(properties, false).getFileStatuses().size()); + FeConstants.runningUnitTest = true; + Assertions.assertDoesNotThrow(() -> new FileTableValuedFunction(properties)); + } + + @Test + void s3LocalAdapterRejectsBeforeListingEvenWithDeclaredSchema() { + Map properties = new HashMap<>(); + properties.put("file_path", "/tmp/local_gate_dir"); + properties.put("fs.local.support", "true"); + properties.put("format", "csv"); + properties.put("csv_schema", "id:int"); + for (boolean enabled : new boolean[] {false, true}) { + Config.enable_local_tvf = enabled; + try (MockedStatic factory = Mockito.mockStatic(FileSystemFactory.class)) { + Exception error = Assertions.assertThrows(Exception.class, + () -> new S3TableValuedFunction(properties)); + Assertions.assertTrue(error.getMessage().contains("Use local()"), error.getMessage()); + factory.verifyNoInteractions(); + } + } + } + + @Test + void s3BrokerHadoopRangesCannotUseLocalImplementation() { + Config.enable_local_tvf = false; + Map properties = new HashMap<>(); + properties.put("uri", "hdfs://remote/srv/private.csv"); + properties.put("fs.defaultFS", "hdfs://remote"); + properties.put("fs.broker.support", "true"); + properties.put("broker.name", "review-broker"); + properties.put("fs.hdfs.impl", "org.apache.hadoop.fs.RawLocalFileSystem"); + properties.put("format", "csv"); + properties.put("csv_schema", "id:int"); + try (MockedStatic factory = Mockito.mockStatic(FileSystemFactory.class)) { + Assertions.assertThrows(AnalysisException.class, + () -> new S3TableValuedFunction(properties)); + factory.verifyNoInteractions(); + } + } + + @Test + void declaredSchemaDoesNotOpenTheUnusedLocalDefault() throws Exception { + Config.enable_local_tvf = false; + FeConstants.runningUnitTest = true; + Map properties = new HashMap<>(); + properties.put("uri", "hdfs://remote/data.csv"); + properties.put("fs.defaultFS", "file:///"); + properties.put("format", "csv"); + properties.put("csv_schema", "id:int"); + HdfsTableValuedFunction hdfs = new HdfsTableValuedFunction(properties); + Assertions.assertEquals("id", hdfs.getTableColumns().get(0).getName()); + FileTableValuedFunction file = new FileTableValuedFunction(properties); + Assertions.assertEquals("id", file.getTableColumns().get(0).getName()); + properties.put("fs.hdfs.support", "true"); + Assertions.assertEquals("id", new S3TableValuedFunction(properties).getTableColumns().get(0).getName()); + properties.remove("csv_schema"); + Assertions.assertThrows(AnalysisException.class, () -> new HdfsTableValuedFunction(properties)); + } + + @Test + void s3ChecksSuppliedRangeConsumersIndependentlyFromBrokerListing() throws Exception { + Map properties = new HashMap<>(); + properties.put("uri", "s3://bucket/listing.csv"); + properties.put("fs.broker.support", "true"); + properties.put("broker.name", "review-broker"); + properties.put("fs.defaultFS", "hdfs://remote"); + properties.put("fs.hdfs.impl", "org.apache.hadoop.fs.RawLocalFileSystem"); + properties.put("format", "csv"); + properties.put("csv_schema", "id:int"); + for (boolean unitTest : new boolean[] {false, true}) { + FeConstants.runningUnitTest = unitTest; + for (String path : new String[] {"hdfs://remote/private.csv", "file:///tmp/private.csv"}) { + List files = Collections.singletonList( + new TBrokerFileStatus(path, false, 10, true)); + Assertions.assertThrows(AnalysisException.class, + () -> new SuppliedS3Tvf(properties, files)); + } + } + FeConstants.runningUnitTest = true; + properties.put("fs.hdfs.impl", "org.apache.hadoop.hdfs.DistributedFileSystem"); + S3TableValuedFunction remote = new S3TableValuedFunction(properties, Collections.singletonList( + new TBrokerFileStatus("hdfs://remote/public.csv", false, 10, true))); + Assertions.assertEquals(1, remote.getFileStatuses().size()); + Assertions.assertEquals("id", remote.getTableColumns().get(0).getName()); + } + + private static class SuppliedS3Tvf extends S3TableValuedFunction { + SuppliedS3Tvf(Map properties, List files) + throws org.apache.doris.common.AnalysisException { + super(properties, files); + } + + @Override + protected void parseFile(List files) { + fileStatuses.addAll(files); + } + } + + @Test + void schemaDiscoveryAlsoChecksTheReturnedFilesDefaultRoute() throws Exception { + Config.enable_local_tvf = false; + Map properties = new HashMap<>(); + properties.put("uri", "hdfs://initial/data.csv"); + properties.put("fs.defaultFS", "viewfs://schema"); + properties.put("fs.viewfs.mounttable.schema.link./hdfs:/initial", "hdfs://remote/base"); + properties.put("fs.viewfs.mounttable.schema.linkFallback", "file:///tmp"); + properties.put("format", "csv"); + // The fake listing separates initial/data/schema identities without filesystem IO. + Assertions.assertThrows(AnalysisException.class, () -> new SchemaListedHdfsTvf(properties)); + properties.put("csv_schema", "id:int"); + Assertions.assertEquals("id", new SchemaListedHdfsTvf(properties).getTableColumns().get(0).getName()); + } + + private static class SchemaListedHdfsTvf extends HdfsTableValuedFunction { + SchemaListedHdfsTvf(Map properties) throws org.apache.doris.common.AnalysisException { + super(properties); + } + + @Override + protected void parseFile() { + fileStatuses.add(new TBrokerFileStatus("hdfs://returned/data.csv", false, 10, true)); + } + } + + private static Map properties(String mount) { + Map properties = new HashMap<>(); + properties.put("uri", "viewfs://review/" + mount + "/file.csv"); + properties.put("fs.defaultFS", "viewfs://review"); + properties.put("fs.viewfs.mounttable.review.link./remote", "hdfs://remote/base"); + properties.put("fs.viewfs.mounttable.review.link./local", "file:///tmp"); + properties.put("format", "csv"); + properties.put("csv_schema", "id:int"); + return properties; + } + + private static class ListedS3Tvf extends S3TableValuedFunction { + ListedS3Tvf(Map properties, String mount) + throws org.apache.doris.common.AnalysisException { + super(withListing(properties, mount)); + } + + private static Map withListing(Map properties, String mount) { + properties.put("fs.test.listing", mount); + return properties; + } + + @Override + protected void parseFile(List files) { + String mount = backendConnectProperties.get("fs.test.listing"); + fileStatuses.add(new TBrokerFileStatus("viewfs://review/" + mount + "/listed.csv", false, 10, true)); + } + } + + private static class ListedHdfsTvf extends HdfsTableValuedFunction { + // Constructor-time listing is the existing TVF lifecycle. Select the fake listing through + // statement properties, which are available before super() calls parseFile(). + ListedHdfsTvf(Map properties, boolean localListing) + throws org.apache.doris.common.AnalysisException { + super(withListing(properties, localListing)); + } + + private static Map withListing(Map properties, boolean localListing) { + properties.put("fs.test.listing", localListing ? "local" : "remote"); + return properties; + } + + @Override + protected void parseFile() { + String mount = backendConnectProperties.get("fs.test.listing"); + fileStatuses.add(new TBrokerFileStatus("viewfs://review/" + mount + "/listed.csv", false, 10, true)); + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/tablefunction/LocalTableValuedFunctionEnableTest.java b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/LocalTableValuedFunctionEnableTest.java new file mode 100644 index 00000000000000..556ae26fed314e --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/LocalTableValuedFunctionEnableTest.java @@ -0,0 +1,226 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.tablefunction; + +import org.apache.doris.catalog.Env; +import org.apache.doris.common.Config; +import org.apache.doris.mysql.privilege.AccessControllerManager; +import org.apache.doris.mysql.privilege.PrivPredicate; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.proto.InternalService.PGlobResponse; +import org.apache.doris.proto.InternalService.PGlobResponse.PFileInfo; +import org.apache.doris.proto.Types.PStatus; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.SessionVariable; +import org.apache.doris.resource.computegroup.ComputeGroup; +import org.apache.doris.rpc.BackendServiceProxy; +import org.apache.doris.system.Backend; +import org.apache.doris.system.SystemInfoService; +import org.apache.doris.thrift.TNetworkAddress; + +import com.google.common.collect.ImmutableMap; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.MockedConstruction; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +import java.util.Collections; +import java.util.Map; +import java.util.concurrent.CompletableFuture; + +public class LocalTableValuedFunctionEnableTest { + private static final Map PROPERTIES = ImmutableMap.of( + "file_path", "local_privilege_test.csv", "backend_id", "1", "format", "csv"); + + private final Env env = Mockito.mock(Env.class); + private final ConnectContext context = Mockito.mock(ConnectContext.class); + private final AccessControllerManager accessManager = Mockito.mock(AccessControllerManager.class); + private final SystemInfoService systemInfo = Mockito.mock(SystemInfoService.class); + private final BackendServiceProxy proxy = Mockito.mock(BackendServiceProxy.class); + private boolean originalEnableLocalTvf; + private MockedStatic mockedEnv; + private MockedStatic mockedContext; + private MockedStatic mockedProxy; + + @BeforeEach + public void setUp() throws Exception { + originalEnableLocalTvf = Config.enable_local_tvf; + Config.enable_local_tvf = false; + mockedEnv = Mockito.mockStatic(Env.class); + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + mockedEnv.when(Env::getCurrentSystemInfo).thenReturn(systemInfo); + mockedContext = Mockito.mockStatic(ConnectContext.class); + mockedContext.when(ConnectContext::get).thenReturn(context); + mockedProxy = Mockito.mockStatic(BackendServiceProxy.class); + mockedProxy.when(BackendServiceProxy::getInstance).thenReturn(proxy); + Mockito.when(env.getAccessManager()).thenReturn(accessManager); + Mockito.when(context.getSessionVariable()).thenReturn(new SessionVariable()); + Mockito.when(context.getQueryTimeoutS()).thenReturn(60); + + Backend backend = Mockito.mock(Backend.class); + Mockito.when(backend.getId()).thenReturn(1L); + Mockito.when(backend.getHost()).thenReturn("127.0.0.1"); + Mockito.when(backend.isAlive()).thenReturn(true); + Mockito.when(backend.isQueryAvailable()).thenReturn(true); + Mockito.when(backend.isLoadAvailable()).thenReturn(true); + ComputeGroup computeGroup = Mockito.mock(ComputeGroup.class); + Mockito.when(context.getComputeGroup()).thenReturn(computeGroup); + Mockito.when(computeGroup.getBackendList()).thenReturn(Collections.singletonList(backend)); + Mockito.when(systemInfo.getBackend(1L)).thenReturn(backend); + Mockito.when(backend.getBrpcAddress()).thenReturn(new TNetworkAddress("127.0.0.1", 8060)); + Mockito.when(proxy.glob(Mockito.any(), Mockito.any())).thenReturn(CompletableFuture.completedFuture( + PGlobResponse.newBuilder().setStatus(PStatus.newBuilder().setStatusCode(0)).build())); + } + + @AfterEach + public void tearDown() { + Config.enable_local_tvf = originalEnableLocalTvf; + mockedProxy.close(); + mockedContext.close(); + mockedEnv.close(); + } + + @Test + public void testDisabledRejectsLocalBeforePropertyParsing() { + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> new LocalTableValuedFunction(ImmutableMap.of())); + Assertions.assertTrue(exception.getMessage().contains("enable_local_tvf=true")); + Mockito.verifyNoInteractions(accessManager, systemInfo, proxy); + } + + @Test + public void testDisabledRejectsCsvAndLanceBeforeFileAccess() { + for (String format : new String[] {"csv", "lance"}) { + Map properties = ImmutableMap.of( + "file_path", "local_enable_test.csv", "backend_id", "1", "format", format); + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> new LocalTableValuedFunction(properties)); + Assertions.assertTrue(exception.getMessage().contains("LOCAL TVF is disabled")); + } + Mockito.verifyNoInteractions(accessManager, systemInfo, proxy); + } + + @Test + public void testCanonicalGlobPathsKeepTheirExactWhitespace() throws Exception { + Config.enable_local_tvf = true; + Mockito.when(accessManager.checkGlobalPriv(context, PrivPredicate.ADMIN)).thenReturn(true); + String[] paths = {"/secure/name.csv ", "/secure/ name.csv", "/secure/\tname.csv\t"}; + PGlobResponse.Builder response = PGlobResponse.newBuilder() + .setStatus(PStatus.newBuilder().setStatusCode(0)); + for (String path : paths) { + response.addFiles(PFileInfo.newBuilder().setFile(path).setSize(4)); + } + Mockito.when(proxy.glob(Mockito.any(), Mockito.any())) + .thenReturn(CompletableFuture.completedFuture(response.build())); + LocalTableValuedFunction local = new LocalTableValuedFunction(PROPERTIES); + FileTableValuedFunction file = new FileTableValuedFunction(PROPERTIES); + for (int i = 0; i < paths.length; i++) { + Assertions.assertEquals(paths[i], local.getFileStatuses().get(i).getPath()); + Assertions.assertEquals(paths[i], file.getFileStatuses().get(i).getPath()); + } + } + + @Test + public void testEnabledLocalReachesExistingFileAccess() throws Exception { + Config.enable_local_tvf = true; + Mockito.when(accessManager.checkGlobalPriv(context, PrivPredicate.ADMIN)).thenReturn(true); + new LocalTableValuedFunction(PROPERTIES); + Mockito.verify(proxy).glob(Mockito.any(), Mockito.any()); + } + + @Test + public void testDisabledRejectsFileLocalBeforeFileAccess() { + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> new FileTableValuedFunction(PROPERTIES)); + Assertions.assertTrue(exception.getMessage().contains("LOCAL TVF is disabled")); + Mockito.verifyNoInteractions(accessManager, systemInfo, proxy); + } + + @Test + public void testEnabledFileLocalReachesExistingFileAccess() throws Exception { + Config.enable_local_tvf = true; + Mockito.when(accessManager.checkGlobalPriv(context, PrivPredicate.ADMIN)).thenReturn(true); + new FileTableValuedFunction(PROPERTIES); + Mockito.verify(proxy).glob(Mockito.any(), Mockito.any()); + } + + @Test + public void testDisabledLocalDoesNotBlockFileS3() { + Map properties = ImmutableMap.builder() + .put("uri", "s3://test-bucket/local_enable_test.csv") + .put("s3.endpoint", "s3.us-east-1.amazonaws.com") + .put("s3.region", "us-east-1") + .put("s3.access_key", "test-access-key") + .put("s3.secret_key", "test-secret-key") + .put("format", "csv") + .build(); + try (MockedConstruction mockedS3 = + Mockito.mockConstruction(S3TableValuedFunction.class)) { + Assertions.assertDoesNotThrow(() -> new FileTableValuedFunction(properties)); + Assertions.assertEquals(1, mockedS3.constructed().size()); + } + Mockito.verifyNoInteractions(accessManager, systemInfo, proxy); + } + + @Test + public void testEnabledLancePreservesExistingBackendSelection() throws Exception { + Config.enable_local_tvf = true; + Mockito.when(accessManager.checkGlobalPriv(context, PrivPredicate.ADMIN)).thenReturn(true); + new LocalTableValuedFunction(ImmutableMap.of( + "file_path", "local_enable_test.lance", "backend_id", "1", "format", "lance")); + Mockito.verifyNoInteractions(proxy); + } + + @Test + public void testEnabledLocalRejectsUnprivilegedUserBeforeFileAccess() { + Config.enable_local_tvf = true; + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> new LocalTableValuedFunction(PROPERTIES)); + Assertions.assertTrue(exception.getMessage().contains("ADMIN/OPERATOR")); + Mockito.verifyNoInteractions(systemInfo, proxy); + } + + @Test + public void testOperatorCanAccessLocalFiles() throws Exception { + Config.enable_local_tvf = true; + Mockito.when(accessManager.checkGlobalPriv(context, PrivPredicate.OPERATOR)).thenReturn(true); + new LocalTableValuedFunction(PROPERTIES); + Mockito.verify(proxy).glob(Mockito.any(), Mockito.any()); + } + + @Test + public void testCheckAuthRejectsRevokedPrivileges() throws Exception { + Config.enable_local_tvf = true; + Mockito.when(accessManager.checkGlobalPriv(context, PrivPredicate.ADMIN)).thenReturn(true); + LocalTableValuedFunction local = new LocalTableValuedFunction(PROPERTIES); + Mockito.when(accessManager.checkGlobalPriv(context, PrivPredicate.ADMIN)).thenReturn(false); + Assertions.assertThrows(AnalysisException.class, () -> local.checkAuth(context)); + } + + @Test + public void testEnabledFileRejectsUnprivilegedUserBeforeFileAccess() { + Config.enable_local_tvf = true; + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> new FileTableValuedFunction(PROPERTIES)); + Assertions.assertTrue(exception.getMessage().contains("ADMIN/OPERATOR")); + Mockito.verifyNoInteractions(systemInfo, proxy); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/tablefunction/S3TableValuedFunctionTest.java b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/S3TableValuedFunctionTest.java index 2d79b5a776a0de..8cb2249479cd88 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/tablefunction/S3TableValuedFunctionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/S3TableValuedFunctionTest.java @@ -20,6 +20,7 @@ import org.apache.doris.common.AnalysisException; import org.apache.doris.common.FeConstants; import org.apache.doris.common.util.S3Util; +import org.apache.doris.datasource.lance.metadata.LanceMetadataLoader; import org.apache.doris.fs.FileSystemFactory; import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.trees.expressions.Properties; @@ -42,6 +43,7 @@ import java.util.Arrays; import java.util.Collections; +import java.util.HashMap; import java.util.Map; import java.util.concurrent.CompletableFuture; @@ -71,6 +73,134 @@ public void tearDown() { } } + @Test + public void testLanceRejectsLocalStorageBeforeOpeningDataset() { + org.apache.doris.common.AnalysisException exception = Assertions.assertThrows( + org.apache.doris.common.AnalysisException.class, + () -> new S3TableValuedFunction(ImmutableMap.of( + "file_path", "dataset.lance", "format", "lance"))); + Assertions.assertTrue(exception.getMessage().contains("requires remote storage")); + } + + @Test + public void testLanceRejectsBrokerLocalUrisBeforeOpeningDataset() { + FeConstants.runningUnitTest = false; + try (MockedStatic loader = Mockito.mockStatic(LanceMetadataLoader.class)) { + for (String uri : Arrays.asList("dataset.lance", "/tmp/dataset.lance", "../dataset.lance", + "//localhost/tmp/dataset.lance", "file:///tmp/dataset.lance", "file:/tmp/dataset.lance", + "file:dataset.lance", "FILE:///tmp/dataset.lance", "file://localhost/tmp/dataset.lance", + "file-object-store://ignored/tmp/dataset.lance", "file+uring://ignored/tmp/dataset.lance", + "FILE-OBJECT-STORE://ignored/tmp/dataset.lance", "FILE+URING://ignored/tmp/dataset.lance", + "c://ignored/tmp/dataset.lance", "memory://dataset.lance", "shared-memory://dataset.lance", + "s3://bucket:-0/../../dataset.lance", "s3://bucket:-00/../../dataset.lance", + "s3://bucket:+0/../../dataset.lance", "s3://[::ffff:127.0.0.01]/../../dataset.lance", + "s3://bucket:bad/../../dataset.lance", "s3://bucket:65536/../../dataset.lance", + "s3://:9000/dataset.lance", "s3://user@/dataset.lance", + "s3://[::1%25eth0]/dataset.lance", "https://bucket:bad/../../dataset.lance", + "", "s3:/bucket/dataset.lance", "s3://")) { + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> new S3TableValuedFunction(ImmutableMap.of( + "format", "lance", "uri", uri, "fs.broker.support", "true")), uri); + Assertions.assertTrue(exception.getMessage().contains("requires remote storage"), uri); + } + loader.verifyNoInteractions(); + } + } + + @Test + public void testLanceRejectsGuessedBrokerFileUrisBeforeOpeningDataset() { + FeConstants.runningUnitTest = false; + try (MockedStatic loader = Mockito.mockStatic(LanceMetadataLoader.class)) { + for (String uri : Arrays.asList("file:///tmp/dataset.lance", "file://localhost/tmp/dataset.lance")) { + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> new S3TableValuedFunction(ImmutableMap.of( + "format", "lance", "uri", uri, "broker.name", "test"))); + Assertions.assertTrue(exception.getMessage().contains("requires remote storage")); + } + loader.verifyNoInteractions(); + } + } + + @Test + public void testLanceRejectsBrokerWithoutUriBeforeOpeningDataset() { + FeConstants.runningUnitTest = false; + try (MockedStatic loader = Mockito.mockStatic(LanceMetadataLoader.class)) { + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> new S3TableValuedFunction(ImmutableMap.of("format", "lance", "fs.broker.support", "true"))); + Assertions.assertTrue(exception.getMessage().contains("requires remote storage")); + loader.verifyNoInteractions(); + } + } + + @Test + public void testLanceRemoteUrisReachMetadataLoader() throws Exception { + FeConstants.runningUnitTest = false; + for (String uri : Arrays.asList("s3://bucket/dataset.lance", "s3+ddb://bucket/dataset.lance", + "oss://bucket/dataset.lance", "gs://bucket/dataset.lance", "az://container/dataset.lance", + "abfss://container@account.dfs.core.windows.net/dataset.lance", + "s3://bucket:0/dataset.lance", "s3://bucket:00080/dataset.lance", + "s3://bucket:65535/dataset.lance", "s3://bucket:/dataset.lance", + "s3://[::1]/dataset.lance", "s3://[::ffff:127.0.0.1]/dataset.lance", + "gs://bucket_with_underscore/dataset.lance", "s3://bucket:9000/dataset.lance", + "goosefs://bucket/dataset.lance", "goosefs:///dataset.lance", "goosefs:/dataset.lance", + "cos://bucket/dataset.lance", + "hf://datasets/repo/data.lance", + "tos://bucket/dataset.lance", "S3://bucket/dataset.lance")) { + assertLanceMetadataLoad(ImmutableMap.of( + "format", "lance", "uri", uri, "fs.broker.support", "true"), uri); + } + } + + @Test + public void testLanceHttpStorageUrisAreNormalizedBeforeValidation() throws Exception { + FeConstants.runningUnitTest = false; + Map s3Properties = new HashMap<>(PROPERTIES); + s3Properties.put("format", "lance"); + s3Properties.put("uri", "https://bucket.s3.us-west-2.amazonaws.com/dataset.lance"); + assertLanceMetadataLoad(s3Properties, "s3://bucket/dataset.lance"); + assertLanceMetadataLoad(ImmutableMap.of( + "format", "lance", "uri", "https://bucket.oss-cn-hangzhou.aliyuncs.com/dataset.lance", + "oss.endpoint", "oss-cn-hangzhou.aliyuncs.com", "oss.access_key", "ak", "oss.secret_key", "sk"), + "s3://bucket/dataset.lance"); + } + + @Test + public void testLanceSpacePathsReachMetadataThroughS3AndFile() throws Exception { + FeConstants.runningUnitTest = false; + for (String uri : Arrays.asList("s3://bucket/sample data.lance", "s3://bucket/sample%20data.lance")) { + Map properties = new HashMap<>(PROPERTIES); + properties.put("format", "lance"); + properties.remove("file_path"); + properties.put("uri", uri); + assertLanceMetadataLoad(properties, uri); + try (MockedStatic loader = Mockito.mockStatic(LanceMetadataLoader.class)) { + IllegalStateException reachedLoader = new IllegalStateException("file remote metadata reached"); + loader.when(() -> LanceMetadataLoader.loadLatestForTvf(Mockito.eq(uri), Mockito.anyList())) + .thenThrow(reachedLoader); + RuntimeException exception = Assertions.assertThrows(RuntimeException.class, + () -> new FileTableValuedFunction(properties)); + Throwable cause = exception; + while (cause.getCause() != null) { + cause = cause.getCause(); + } + Assertions.assertSame(reachedLoader, cause); + loader.verify(() -> LanceMetadataLoader.loadLatestForTvf(Mockito.eq(uri), Mockito.anyList())); + } + } + } + + private void assertLanceMetadataLoad(Map properties, String uri) throws Exception { + try (MockedStatic loader = Mockito.mockStatic(LanceMetadataLoader.class)) { + IllegalStateException reachedLoader = new IllegalStateException("remote metadata loader reached"); + loader.when(() -> LanceMetadataLoader.loadLatestForTvf(Mockito.eq(uri), Mockito.anyList())) + .thenThrow(reachedLoader); + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> new S3TableValuedFunction(properties)); + Assertions.assertSame(reachedLoader, exception.getCause(), uri); + loader.verify(() -> LanceMetadataLoader.loadLatestForTvf(Mockito.eq(uri), Mockito.anyList())); + } + } + @Test public void testExactFilesKeepLiteralPathsAndSkipListing() throws Exception { FeConstants.runningUnitTest = false; diff --git a/gensrc/thrift/PlanNodes.thrift b/gensrc/thrift/PlanNodes.thrift index 5a9a55cabe56ef..f76f20b0f03d7e 100644 --- a/gensrc/thrift/PlanNodes.thrift +++ b/gensrc/thrift/PlanNodes.thrift @@ -598,6 +598,10 @@ struct TLanceFileDesc { // so these tasks filter their rows without repeating global scalar-index evaluation. // Unset preserves Lance's default; an explicit scalar segment must not be combined with false. 6: optional bool use_scalar_index + // Independent LOCAL/S3 TVF requests enforce storage admission for manifest-selected bases. + // Unset preserves legacy/catalog routing; old LOCAL TVFs are recognized by version zero. + 7: optional bool is_tvf + } struct TLanceScanParams { diff --git a/regression-test/pipeline/cloud_p0/conf/fe_custom.conf b/regression-test/pipeline/cloud_p0/conf/fe_custom.conf index d6c2070aa939e2..7139bfa6e02aa8 100644 --- a/regression-test/pipeline/cloud_p0/conf/fe_custom.conf +++ b/regression-test/pipeline/cloud_p0/conf/fe_custom.conf @@ -1,3 +1,6 @@ +# Exercise LOCAL TVF explicitly; production defaults to disabled. +enable_local_tvf=true + stream_load_default_timeout_second = 72000 replication_num_forced_in_cloud_mode = true ignore_unsupported_properties_in_cloud_mode = true diff --git a/regression-test/pipeline/cloud_p1/conf/fe_custom.conf b/regression-test/pipeline/cloud_p1/conf/fe_custom.conf index 157f3a07a09b47..e671d0a7dd15b2 100644 --- a/regression-test/pipeline/cloud_p1/conf/fe_custom.conf +++ b/regression-test/pipeline/cloud_p1/conf/fe_custom.conf @@ -1,3 +1,6 @@ +# Exercise LOCAL TVF explicitly; production defaults to disabled. +enable_local_tvf=true + stream_load_default_timeout_second = 72000 replication_num_forced_in_cloud_mode = true ignore_unsupported_properties_in_cloud_mode = true diff --git a/regression-test/pipeline/external/conf/fe.conf b/regression-test/pipeline/external/conf/fe.conf index 4466998ae67163..c0160fa91f07b5 100644 --- a/regression-test/pipeline/external/conf/fe.conf +++ b/regression-test/pipeline/external/conf/fe.conf @@ -1,3 +1,6 @@ +# Exercise LOCAL TVF explicitly; production defaults to disabled. +enable_local_tvf=true + # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/regression-test/pipeline/nonConcurrent/conf/fe.conf b/regression-test/pipeline/nonConcurrent/conf/fe.conf index e89a00d209ccf7..737e305a1bacf4 100644 --- a/regression-test/pipeline/nonConcurrent/conf/fe.conf +++ b/regression-test/pipeline/nonConcurrent/conf/fe.conf @@ -1,3 +1,6 @@ +# Exercise LOCAL TVF explicitly; production defaults to disabled. +enable_local_tvf=true + # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/regression-test/pipeline/p0/conf/fe.conf b/regression-test/pipeline/p0/conf/fe.conf index 3d86c5c95e6332..494bc15cfb9bd1 100644 --- a/regression-test/pipeline/p0/conf/fe.conf +++ b/regression-test/pipeline/p0/conf/fe.conf @@ -1,3 +1,6 @@ +# Exercise LOCAL TVF explicitly; production defaults to disabled. +enable_local_tvf=true + # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/regression-test/pipeline/p1/conf/fe.conf b/regression-test/pipeline/p1/conf/fe.conf index f20d51b62e6575..04037acd3644b9 100644 --- a/regression-test/pipeline/p1/conf/fe.conf +++ b/regression-test/pipeline/p1/conf/fe.conf @@ -1,3 +1,6 @@ +# Exercise LOCAL TVF explicitly; production defaults to disabled. +enable_local_tvf=true + # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/regression-test/pipeline/vault_p0/conf/fe_custom.conf b/regression-test/pipeline/vault_p0/conf/fe_custom.conf index 9b622795168540..88cff4dd920ec5 100644 --- a/regression-test/pipeline/vault_p0/conf/fe_custom.conf +++ b/regression-test/pipeline/vault_p0/conf/fe_custom.conf @@ -1,3 +1,6 @@ +# Exercise LOCAL TVF explicitly; production defaults to disabled. +enable_local_tvf=true + stream_load_default_timeout_second = 72000 replication_num_forced_in_cloud_mode = true ignore_unsupported_properties_in_cloud_mode = true diff --git a/regression-test/suites/account_p0/test_local_tvf_enable.groovy b/regression-test/suites/account_p0/test_local_tvf_enable.groovy new file mode 100644 index 00000000000000..3cc8bf71ce3d92 --- /dev/null +++ b/regression-test/suites/account_p0/test_local_tvf_enable.groovy @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_local_tvf_enable", "p0") { + def enabled = (sql "ADMIN SHOW FRONTEND CONFIG LIKE 'enable_local_tvf'")[0][1].toBoolean() + for (def functionName : ["LOCAL", "FILE"]) { + for (def prefix : ["", "EXPLAIN "]) { + test { + sql """${prefix}SELECT * FROM ${functionName}( + "file_path" = "local_enable_test.csv", + "backend_id" = "-1", + "format" = "csv" + )""" + // With opt-in enabled, the existing backend validation must be reached. + exception (enabled ? "'backend_id' is required" : "LOCAL TVF is disabled") + } + } + } + if (!enabled) { + for (def prefix : ["", "EXPLAIN "]) { + test { + sql """${prefix}INSERT INTO LOCAL( + "file_path" = "/tmp/local_enable_test_", + "backend_id" = "-1", + "format" = "csv" + ) SELECT 1""" + exception "LOCAL TVF is disabled" + } + } + } + +} diff --git a/thirdparty/build-thirdparty.sh b/thirdparty/build-thirdparty.sh index 710c2b68dba684..4ddf0560b2a544 100755 --- a/thirdparty/build-thirdparty.sh +++ b/thirdparty/build-thirdparty.sh @@ -2161,6 +2161,32 @@ install_paimon_rust() { ) } +# Publish the Lance headers and archive as one recoverable installation. +install_lance_c() { + ( + set -e + local archive="$1" + local headers="$2" + local destination="${TP_INSTALL_DIR}/include/lance" + local incomplete="${TP_INSTALL_DIR}/lib64/.lance-installing" + mkdir -p "${TP_INSTALL_DIR}/include" "${TP_INSTALL_DIR}/lib64" + local staged + staged="$(mktemp -d "${TP_INSTALL_DIR}/include/.lance.tmp.XXXXXX")" + trap 'rm -rf "${staged}"' EXIT + cp -pR "${headers}/." "${staged}/" + test -s "${staged}/lance.h" + test -s "${staged}/lance.hpp" + test -s "${archive}" + # The archive and header-directory renames cannot be atomic together. + # Keep this marker after any interrupted publication until a retry repairs both. + touch "${incomplete}" + install_rust_archive "${archive}" + rm -rf "${destination}" + mv "${staged}" "${destination}" + rm -f "${incomplete}" + ) +} + build_lance_c() { check_if_source_exist "${LANCE_C_SOURCE}" cd "${TP_SOURCE_DIR}/${LANCE_C_SOURCE}" @@ -2232,10 +2258,7 @@ build_lance_c() { fi env "${cargo_env[@]}" "${cargo_bin}" "${cargo_args[@]}" - mkdir -p "${TP_INSTALL_DIR}/include" "${TP_INSTALL_DIR}/lib64" - rm -rf "${TP_INSTALL_DIR}/include/lance" - cp -av include/lance "${TP_INSTALL_DIR}/include/" - install_rust_archive "${BUILD_DIR}/release/liblance_c.a" + install_lance_c "${BUILD_DIR}/release/liblance_c.a" "${PWD}/include/lance" } # paimon-rust diff --git a/thirdparty/download-thirdparty.sh b/thirdparty/download-thirdparty.sh index 198e328c79cdd2..03ea798d8c81f0 100755 --- a/thirdparty/download-thirdparty.sh +++ b/thirdparty/download-thirdparty.sh @@ -719,9 +719,11 @@ if [[ " ${TP_ARCHIVES[*]} " =~ " AZURE " ]]; then fi # Foyer remains a local patch until its cache interface is accepted upstream. +# TVF storage-base inspection is an additive C ABI patch on the pinned SDK. # All search fixes are supplied by the immutable lance-c dependency revision. if [[ " ${TP_ARCHIVES[*]} " =~ " LANCE_C " ]]; then - foyer_patch_checksum="$(cksum < "${TP_PATCH_DIR}/lance-c-foyer.patch")" + foyer_patch_checksum="$(cat "${TP_PATCH_DIR}/lance-c-foyer.patch" \ + "${TP_PATCH_DIR}/lance-c-storage-bases.patch" | cksum)" foyer_patch_marker="${TP_SOURCE_DIR}/${LANCE_C_SOURCE}/${PATCHED_MARK}_foyer" # A new local patch must also replace previously patched cached sources. # Empty markers from older builds cannot identify the applied patch version. @@ -734,6 +736,8 @@ if [[ " ${TP_ARCHIVES[*]} " =~ " LANCE_C " ]]; then if [[ ! -f "${PATCHED_MARK}_foyer" ]]; then patch --batch --forward --reject-file=- --fuzz=0 --no-backup-if-mismatch -s \ -p1 <"${TP_PATCH_DIR}/lance-c-foyer.patch" + patch --batch --forward --reject-file=- --fuzz=0 --no-backup-if-mismatch -s \ + -p1 <"${TP_PATCH_DIR}/lance-c-storage-bases.patch" printf '%s\n' "${foyer_patch_checksum}" > "${PATCHED_MARK}_foyer" fi cd - diff --git a/thirdparty/patches/lance-c-storage-bases.patch b/thirdparty/patches/lance-c-storage-bases.patch new file mode 100644 index 00000000000000..f5747e89af46b8 --- /dev/null +++ b/thirdparty/patches/lance-c-storage-bases.patch @@ -0,0 +1,362 @@ +--- a/Cargo.toml ++++ b/Cargo.toml +@@ -51,6 +51,7 @@ + object_store = "0.14.2" + pin-project = "1.0" + prost = "0.14" ++serde_json = "1" + snafu = "0.9" + uuid = { version = "1", features = ["v4"] } + +--- a/Cargo.lock ++++ b/Cargo.lock +@@ -4442,6 +4442,7 @@ + "opendal", + "pin-project", + "prost", ++ "serde_json", + "snafu", + "tempfile", + "tokio", +--- a/include/lance/lance.h ++++ b/include/lance/lance.h +@@ -291,6 +291,31 @@ + ); + + /* ─── Dataset lifecycle ─── */ ++ ++/** Metadata-only base inspection contract; header/archive must both provide this API. */ ++#define LANCE_STORAGE_BASES_JSON_VERSION 1 ++ ++/** ++ * Return the primary and all registered bases of the opened snapshot as JSON. ++ * ++ * This inspects SDK-decoded metadata and extracts paths using the SDK provider; ++ * it never constructs a base object store or reads any base object. It includes ++ * registrations regardless of the current projection or whether they are unused. ++ * ++ * JSON: {"format_version":1,"bases":[{"id":0,"primary":false,"uri":"...","scheme":"file", ++ * "is_local":true,"local_path":"/absolute/path","is_dataset_root":true}]}. ++ * Nonlocal entries have local_path:null. local_path is already decoded exactly as ++ * the SDK reader uses it; callers MUST NOT URI-decode it again. The entries are ++ * primary is first with id:null and primary:true; registrations follow sorted ++ * by id. This does not enumerate absolute blob URIs encoded inside data ++ * values, and is not a policy hook for blob payload materialization. ++ * ++ * @return Owned UTF-8 string, to be freed with lance_free_string(), or NULL on ++ * error (use lance_last_error_code/message). Safe with shared sessions. ++ */ ++const char* lance_dataset_storage_bases_json(const LanceDataset* dataset); ++ ++ + + /** + * Open a Lance dataset. +--- a/include/lance/lance.hpp ++++ b/include/lance/lance.hpp +@@ -820,6 +820,15 @@ + handle_.get(), stream, batch_size) != 0) { + check_error(); + } ++ } ++ ++ /// Owned JSON describing the SDK-decoded primary and registered storage bases. ++ /// Metadata-only: this does not instantiate delegated object stores. ++ std::string storage_bases_json() const { ++ const char* json = lance_dataset_storage_bases_json(handle_.get()); ++ if (!json) check_error(); ++ std::unique_ptr owned(json, &lance_free_string); ++ return std::string(owned.get()); + } + + /// Export the schema as an Arrow C Data Interface struct. +--- a/src/dataset.rs ++++ b/src/dataset.rs +@@ -20,6 +20,9 @@ + use crate::runtime::block_on; + use crate::session::LanceSession; + use crate::stream_guard::guarded_ffi_stream_from_reader; ++ ++/// Version of the metadata-only storage-base inspection JSON contract. ++pub const LANCE_STORAGE_BASES_JSON_VERSION: u32 = 1; + + /// Opaque handle representing an opened Lance dataset. + pub struct LanceDataset { +@@ -219,6 +222,82 @@ + // Metadata (in-memory, sync only) + // --------------------------------------------------------------------------- + ++/// Inspect registered base metadata before any scanner/index/delegated reader runs. ++#[unsafe(no_mangle)] ++pub unsafe extern "C" fn lance_dataset_storage_bases_json( ++ dataset: *const LanceDataset, ++) -> *const c_char { ++ ffi_try!(unsafe { storage_bases_json_inner(dataset) }, null) ++} ++ ++fn storage_base_description( ++ base: &lance_table::format::BasePath, ++ registry: Arc, ++) -> Result { ++ // These are the SDK's own parse/extract operations. Do not instantiate a ++ // provider merely to discover whether the manifest selected local storage. ++ let url = lance_io::object_store::uri_to_url(&base.path)?; ++ let path = lance_io::object_store::ObjectStore::extract_path_from_uri(registry, &base.path)?; ++ // The C session uses the pinned SDK's default registry, whose local aliases ++ // all resolve to FileStoreProvider. There is no C provider-registration API. ++ let is_local = matches!(url.scheme(), "file" | "file-object-store" | "file+uring"); ++ let local_path = is_local.then(|| lance_io::local::to_local_path(&path)); ++ Ok(serde_json::json!({ ++ "id": base.id, ++ "primary": false, ++ "uri": base.path, ++ "scheme": url.scheme(), ++ "is_local": is_local, ++ "local_path": local_path, ++ "is_dataset_root": base.is_dataset_root, ++ })) ++} ++ ++unsafe fn storage_bases_json_inner(dataset: *const LanceDataset) -> Result<*const c_char> { ++ if dataset.is_null() { ++ return Err(lance_core::Error::invalid_input_source( ++ "dataset is NULL".into(), ++ )); ++ } ++ let snapshot = unsafe { &*dataset }.snapshot(); ++ let registry = snapshot.session().store_registry(); ++ let mut bases = snapshot.manifest().base_paths.values().collect::>(); ++ bases.sort_unstable_by_key(|base| base.id); ++ let mut entries = bases ++ .into_iter() ++ .map(|base| storage_base_description(base, registry.clone())) ++ .collect::>>()?; ++ // object_store(None) returns the already-opened primary Arc; it never opens ++ // a delegated base. branch_location carries the actual parsed SDK base path. ++ let primary_store = block_on(snapshot.object_store(None))?; ++ let primary_path = snapshot.branch_location().path; ++ let primary_is_local = matches!( ++ primary_store.scheme(), ++ "file" | "file-object-store" | "file+uring" ++ ); ++ entries.insert( ++ 0, ++ serde_json::json!({ ++ "id": serde_json::Value::Null, ++ "primary": true, ++ "uri": snapshot.uri(), ++ "scheme": primary_store.scheme(), ++ "is_local": primary_is_local, ++ "local_path": primary_is_local.then(|| lance_io::local::to_local_path(&primary_path)), ++ "is_dataset_root": true, ++ }), ++ ); ++ let json = serde_json::json!({ ++ "format_version": LANCE_STORAGE_BASES_JSON_VERSION, ++ "bases": entries, ++ }) ++ .to_string(); ++ let owned = std::ffi::CString::new(json).map_err(|_| { ++ lance_core::Error::invalid_input_source("storage base metadata contains NUL".into()) ++ })?; ++ Ok(owned.into_raw() as *const c_char) ++} ++ + /// Return the version number of this dataset snapshot. + /// Returns 0 on error — check `lance_last_error_code()`. + #[unsafe(no_mangle)] +@@ -588,3 +667,121 @@ + assert_eq!(block_on(snap.count_rows(None)).unwrap(), 2); + } + } ++ ++#[cfg(test)] ++mod storage_base_metadata_tests { ++ use super::*; ++ use lance_io::object_store::ObjectStoreRegistry; ++ use lance_table::format::BasePath; ++ ++ #[test] ++ fn base_inspection_uses_sdk_local_identity_without_opening_files() { ++ let registry = Arc::new(ObjectStoreRegistry::default()); ++ for (uri, expected) in [ ++ ( ++ "file:///nonexistent/sdbsec/a%20b", ++ "/nonexistent/sdbsec/a b", ++ ), ++ ( ++ "file:///nonexistent/sdbsec/a%2520b", ++ "/nonexistent/sdbsec/a%20b", ++ ), ++ ( ++ "file-object-store:///nonexistent/sdbsec/a%20b", ++ "/nonexistent/sdbsec/a b", ++ ), ++ ] { ++ let base = BasePath::new(7, uri.into(), None, false); ++ let value = storage_base_description(&base, registry.clone()).unwrap(); ++ assert_eq!(value["id"], 7); ++ assert_eq!(value["is_local"], true); ++ assert_eq!(value["local_path"], expected); ++ assert_eq!(value["is_dataset_root"], false); ++ } ++ } ++ ++ #[test] ++ fn base_inspection_preserves_remote_base_without_opening_provider() { ++ let registry = Arc::new(ObjectStoreRegistry::default()); ++ let base = BasePath::new(3, "s3://unreachable.invalid/clone-base".into(), None, true); ++ let value = storage_base_description(&base, registry).unwrap(); ++ assert_eq!(value["is_local"], false); ++ assert_eq!(value["local_path"], serde_json::Value::Null); ++ assert_eq!(value["uri"], base.path); ++ assert_eq!(value["is_dataset_root"], true); ++ } ++ ++ #[cfg(target_os = "linux")] ++ #[test] ++ fn io_uring_local_alias_is_identified_without_opening_provider() { ++ let base = BasePath::new( ++ 8, ++ "file+uring:///nonexistent/sdbsec/a%20b".into(), ++ None, ++ false, ++ ); ++ let value = ++ storage_base_description(&base, Arc::new(ObjectStoreRegistry::default())).unwrap(); ++ assert_eq!(value["is_local"], true); ++ assert_eq!(value["local_path"], "/nonexistent/sdbsec/a b"); ++ } ++ ++ #[test] ++ fn metadata_snapshot_contains_primary_and_every_registered_base() { ++ use arrow_array::{Int64Array, RecordBatch}; ++ use arrow_schema::{DataType, Field, Schema}; ++ let temp = tempfile::tempdir().unwrap(); ++ let uri = temp.path().join("primary with spaces.lance"); ++ let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)])); ++ let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(Int64Array::from(vec![1]))]) ++ .unwrap(); ++ let mut dataset = block_on(Dataset::write( ++ arrow::record_batch::RecordBatchIterator::new(vec![Ok(batch)], schema), ++ uri.to_str().unwrap(), ++ None, ++ )) ++ .unwrap(); ++ // Add unreferenced registrations in memory: listing every registered base ++ // must not instantiate the unreachable remote provider or open local data. ++ let manifest = Arc::make_mut(&mut dataset.manifest); ++ manifest.base_paths.insert( ++ 9, ++ BasePath::new(9, "s3://unreachable.invalid/base".into(), None, true), ++ ); ++ manifest.base_paths.insert( ++ 4, ++ BasePath::new( ++ 4, ++ "file:///nonexistent/sdbsec/base%20path".into(), ++ None, ++ false, ++ ), ++ ); ++ let handle = LanceDataset { ++ inner: RwLock::new(Arc::new(dataset)), ++ data_cache: None, ++ }; ++ let raw = unsafe { lance_dataset_storage_bases_json(&handle) }; ++ assert!(!raw.is_null()); ++ let value: serde_json::Value = ++ serde_json::from_str(unsafe { std::ffi::CStr::from_ptr(raw) }.to_str().unwrap()) ++ .unwrap(); ++ unsafe { crate::error::lance_free_string(raw) }; ++ assert_eq!(value["format_version"], 1); ++ let entries = value["bases"].as_array().unwrap(); ++ assert_eq!(entries.len(), 3); ++ assert_eq!(entries[0]["primary"], true); ++ assert_eq!(entries[0]["id"], serde_json::Value::Null); ++ assert_eq!(entries[0]["is_local"], true); ++ assert_eq!(entries[0]["local_path"], uri.to_str().unwrap()); ++ assert_eq!(entries[1]["id"], 4); ++ assert_eq!(entries[1]["local_path"], "/nonexistent/sdbsec/base path"); ++ assert_eq!(entries[2]["id"], 9); ++ assert_eq!(entries[2]["is_local"], false); ++ } ++ ++ #[test] ++ fn null_dataset_base_inspection_fails() { ++ assert!(unsafe { lance_dataset_storage_bases_json(std::ptr::null()) }.is_null()); ++ } ++} +--- a/tests/c_api_test.rs ++++ b/tests/c_api_test.rs +@@ -436,6 +436,70 @@ + + // Closing NULL is safe. + unsafe { lance_dataset_close(ptr::null_mut()) }; ++} ++ ++#[test] ++fn test_storage_bases_json_null_dataset_reports_error() { ++ let result = unsafe { lance_dataset_storage_bases_json(ptr::null()) }; ++ assert!(result.is_null()); ++ assert_eq!(lance_last_error_code(), LanceErrorCode::InvalidArgument); ++ let message = lance_last_error_message(); ++ assert!(!message.is_null()); ++ unsafe { lance_free_string(message) }; ++} ++ ++#[test] ++fn test_storage_bases_json_primary_snapshot_with_multiple_fragments() { ++ let tmp = tempfile::tempdir().unwrap(); ++ let uri = tmp ++ .path() ++ .join("base metadata with spaces.lance") ++ .to_str() ++ .unwrap() ++ .to_string(); ++ let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)])); ++ for fragment in 0..2 { ++ let batch = RecordBatch::try_new( ++ schema.clone(), ++ vec![Arc::new(Int32Array::from(vec![fragment]))], ++ ) ++ .unwrap(); ++ let params = lance::dataset::WriteParams { ++ mode: if fragment == 0 { ++ lance::dataset::WriteMode::Create ++ } else { ++ lance::dataset::WriteMode::Append ++ }, ++ ..Default::default() ++ }; ++ lance_c::runtime::block_on(Dataset::write( ++ arrow::record_batch::RecordBatchIterator::new(vec![Ok(batch)], schema.clone()), ++ &uri, ++ Some(params), ++ )) ++ .unwrap(); ++ } ++ let c_uri = c_str(&uri); ++ let dataset = unsafe { lance_dataset_open(c_uri.as_ptr(), ptr::null(), 0) }; ++ assert!(!dataset.is_null()); ++ assert_eq!(unsafe { lance_dataset_fragment_count(dataset) }, 2); ++ let raw = unsafe { lance_dataset_storage_bases_json(dataset) }; ++ assert!(!raw.is_null()); ++ assert_eq!(lance_last_error_code(), LanceErrorCode::Ok); ++ let value: serde_json::Value = ++ serde_json::from_str(unsafe { std::ffi::CStr::from_ptr(raw) }.to_str().unwrap()).unwrap(); ++ unsafe { lance_free_string(raw) }; ++ assert_eq!(value["format_version"], LANCE_STORAGE_BASES_JSON_VERSION); ++ let bases = value["bases"].as_array().unwrap(); ++ assert_eq!(bases.len(), 1); ++ assert_eq!(bases[0]["primary"], true); ++ assert_eq!(bases[0]["id"], serde_json::Value::Null); ++ assert_eq!(bases[0]["is_local"], true); ++ assert_eq!(bases[0]["local_path"], uri); ++ assert_eq!(bases[0]["is_dataset_root"], true); ++ // Inspection does not consume or mutate the Dataset handle. ++ assert_eq!(unsafe { lance_dataset_fragment_count(dataset) }, 2); ++ unsafe { lance_dataset_close(dataset) }; + } + + #[test] diff --git a/thirdparty/test/lance-build-test.py b/thirdparty/test/lance-build-test.py new file mode 100644 index 00000000000000..c3a60bd338e002 --- /dev/null +++ b/thirdparty/test/lance-build-test.py @@ -0,0 +1,329 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +"""Run with python3 thirdparty/test/lance-build-test.py; no Rust build or downloads.""" + +import os +from pathlib import Path +import re +import shutil +import subprocess +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[2] +SYMBOL = "lance_dataset_storage_bases_json" +HEADER = "#define LANCE_STORAGE_BASES_JSON_VERSION 1\n" + + +class LanceBuildTest(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.work = Path(self.temp.name) + self.prefix = self.work / "thirdparty" + self.env = dict(os.environ) + self.env.pop("NM", None) + self.env.update(DORIS_HOME=str(ROOT), DORIS_THIRDPARTY=str(self.prefix), + BUILD_BE="1", CLEAN="0", PARALLEL="2") + self.write("thirdparty/installed/lib64/unrelated.a", "keep unrelated dependency") + + def write(self, name, text): + path = self.work / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(text) + return path + + def executable(self, name, text): + path = self.write(name, "#!/usr/bin/env bash\nset -e\n" + text) + path.chmod(0o755) + return path + + def run_bash(self, script): + return subprocess.run(["bash", "-eo", "pipefail", "-c", script], env=self.env, + cwd=self.work, text=True, capture_output=True) + + def archive(self, name, symbol=SYMBOL, undefined=False): + source = self.write(name + ".c", (f"extern void {symbol}(void);\n" + f"void caller(void) {{ {symbol}(); }}\n") if undefined + else f"void {symbol}(void) {{}}\n") + obj = source.with_suffix(".o") + target = source.with_suffix(".a") + subprocess.run(["cc", "-c", str(source), "-o", str(obj)], check=True, + capture_output=True) + subprocess.run(["ar", "rcs", str(target), str(obj)], check=True, capture_output=True) + return target + + def installed(self, header=HEADER, symbol=SYMBOL, undefined=False): + self.write("thirdparty/installed/include/lance/lance.h", header) + archive = self.archive("fixtures/installed", symbol, undefined) + shutil.copy2(archive, self.prefix / "installed/lib64/liblance_c.a") + + def gate(self): + script = (ROOT / "build.sh").read_text() + return script[script.index("# Lance TVF storage policy needs"): + script.index("update_submodule() {")] + + def compatible(self): + functions = self.gate().split('if [[ "${BUILD_BE}" -eq 1 ]]')[0] + return self.run_bash(functions + '\nlance_c_is_compatible "$DORIS_THIRDPARTY/installed"') + + def matching_source(self, repair=True): + self.prefix.mkdir(parents=True, exist_ok=True) + shutil.copy2(ROOT / "thirdparty/vars.sh", self.prefix / "vars.sh") + self.write("thirdparty/download-thirdparty.sh", "# lance-c-storage-bases.patch\n") + patch = self.prefix / "patches/lance-c-storage-bases.patch" + patch.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(ROOT / "thirdparty/patches/lance-c-storage-bases.patch", patch) + new_header = self.write("fixtures/new.h", HEADER) + new_archive = self.archive("fixtures/new") + self.env.update(BUILD_LOG=str(self.work / "build.log"), NEW_HEADER=str(new_header), + NEW_ARCHIVE=str(new_archive)) + script = '''install_lance_c() { :; } +build_paimon_rust() { :; } +install_paimon_rust() { :; } +printf '%s\\n' "$*" > "$BUILD_LOG" +printf '%s|%s|%s\\n' "${LANCE_C_CARGO:-}" "${PAIMON_RUST_CARGO:-}" "${RUSTUP_TOOLCHAIN:-}" > "${BUILD_LOG}.toolchain" +''' + if repair: + script += '''cp "$NEW_HEADER" "$DORIS_THIRDPARTY/installed/include/lance/lance.h" +cp "$NEW_ARCHIVE" "$DORIS_THIRDPARTY/installed/lib64/liblance_c.a" +rm -f "$DORIS_THIRDPARTY/installed/lib64/.lance-installing" +''' + self.executable("thirdparty/build-thirdparty.sh", script) + + def assert_unrelated_preserved(self): + self.assertEqual((self.prefix / "installed/lib64/unrelated.a").read_text(), + "keep unrelated dependency") + + def test_complete_installed_only_pair_is_reused(self): + self.installed() + result = self.run_bash(self.gate()) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertFalse((self.work / "build.log").exists()) + self.assert_unrelated_preserved() + + def test_old_header_new_archive_and_old_archive_new_header_fail_closed(self): + for header, symbol, undefined in [("/* old header */\n", SYMBOL, False), + (HEADER, "old_lance_api", False), + (HEADER, SYMBOL, True), + (HEADER.replace("VERSION 1", "VERSION 2"), SYMBOL, False)]: + with self.subTest(header=header, symbol=symbol, undefined=undefined): + self.installed(header, symbol, undefined) + archive = self.prefix / "installed/lib64/liblance_c.a" + before = archive.read_bytes() + result = self.run_bash(self.gate()) + self.assertNotEqual(result.returncode, 0) + self.assertIn("Refresh the compilation image", result.stderr) + self.assertEqual(before, archive.read_bytes()) + self.assert_unrelated_preserved() + + def test_incomplete_installed_pair_is_not_reused(self): + for defect in ("marker", "missing_header", "empty_header", "empty_archive"): + with self.subTest(defect=defect): + self.installed() + marker = self.prefix / "installed/lib64/.lance-installing" + marker.unlink(missing_ok=True) + header = self.prefix / "installed/include/lance/lance.h" + archive = self.prefix / "installed/lib64/liblance_c.a" + if defect == "marker": + marker.touch() + elif defect == "missing_header": + header.unlink() + elif defect == "empty_header": + header.write_text("") + else: + archive.write_text("") + result = self.run_bash(self.gate()) + self.assertNotEqual(result.returncode, 0) + self.assertIn("header/archive pair", result.stderr) + self.assertEqual(marker.exists(), defect == "marker") + self.assert_unrelated_preserved() + + def test_fe_only_does_not_check_the_new_lance_abi(self): + self.installed("/* old header */\n", "old_lance_api") + self.env.update(BUILD_BE="0", NM=str(self.executable("bin/nm", "exit 77\n"))) + result = self.run_bash(self.gate()) + self.assertEqual(result.returncode, 0, result.stderr) + self.assert_unrelated_preserved() + + def test_complete_matching_source_rebuilds_paired_rust_dependencies_and_repairs_marker(self): + for clean in ("0", "1"): + with self.subTest(clean=clean): + self.installed("/* old header */\n", "old_lance_api") + self.write("thirdparty/installed/lib64/.lance-installing", "") + self.matching_source() + self.env["CLEAN"] = clean + result = self.run_bash(self.gate()) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual((self.work / "build.log").read_text().strip(), "-j 2 lance_c paimon_rust") + self.assertFalse((self.prefix / "installed/lib64/.lance-installing").exists()) + self.assertEqual(self.compatible().returncode, 0) + self.assert_unrelated_preserved() + + def test_abi_repair_uses_one_cargo_command_for_both_dependencies(self): + self.installed("/* old header */\n", "old_lance_api") + self.matching_source() + self.env.update(LANCE_C_CARGO="/selected/cargo", RUSTUP_TOOLCHAIN="1.97.1") + result = self.run_bash(self.gate()) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual((self.work / "build.log.toolchain").read_text().strip(), + "/selected/cargo|/selected/cargo|1.97.1") + self.assert_unrelated_preserved() + + def test_abi_repair_rejects_conflicting_per_library_cargo_overrides(self): + self.installed("/* old header */\n", "old_lance_api") + self.matching_source() + self.env.update(LANCE_C_CARGO="/new/cargo", PAIMON_RUST_CARGO="/old/cargo") + result = self.run_bash(self.gate()) + self.assertNotEqual(result.returncode, 0) + self.assertIn("same cargo command", result.stderr) + self.assertFalse((self.work / "build.log").exists()) + self.assert_unrelated_preserved() + + def test_old_or_incomplete_source_is_not_used_for_rebuild(self): + for defect in ("patch", "pin", "download", "installer"): + with self.subTest(defect=defect): + self.installed("/* old header */\n", "old_lance_api") + self.matching_source() + (self.work / "build.log").unlink(missing_ok=True) + if defect == "patch": + self.write("thirdparty/patches/lance-c-storage-bases.patch", "old patch\n") + elif defect == "pin": + with (self.prefix / "vars.sh").open("a") as handle: + handle.write('\nLANCE_C_SOURCE="another-source"\n') + elif defect == "download": + self.write("thirdparty/download-thirdparty.sh", "# old download recipe\n") + else: + self.write("thirdparty/build-thirdparty.sh", "# old installer\n") + result = self.run_bash(self.gate()) + self.assertNotEqual(result.returncode, 0) + self.assertIn("matching complete third-party sources", result.stderr) + self.assertFalse((self.work / "build.log").exists()) + self.assert_unrelated_preserved() + + def test_rebuild_failure_or_incompatible_output_does_not_continue(self): + for mode in ("failure", "old_output"): + with self.subTest(mode=mode): + self.installed("/* old header */\n", "old_lance_api") + self.matching_source(repair=False) + if mode == "failure": + with (self.prefix / "build-thirdparty.sh").open("a") as handle: + handle.write("exit 23\n") + result = self.run_bash(self.gate()) + self.assertNotEqual(result.returncode, 0) + if mode == "old_output": + self.assertIn("Rebuilt Lance SDK header/archive pair", result.stderr) + self.assert_unrelated_preserved() + + def test_nm_symbol_spelling_definition_and_stream_completion(self): + self.installed() + cases = [ + ("gnu", f"printf '00000000 T {SYMBOL}\\n'\n", True), + ("darwin", f"printf '00000000 T _{SYMBOL}\\n'\n", True), + ("weak", f"printf '00000000 W {SYMBOL}\\n'\n", True), + ("undefined", f"printf ' U {SYMBOL}\\n'\n", False), + ("darwin-undefined", f"printf ' U _{SYMBOL}\\n'\n", False), + ("nm-failure", f"printf '00000000 T {SYMBOL}\\n'; exit 2\n", False), + ("long", f"printf '\\nmember.o:\\n00000000 T {SYMBOL}\\n'\n" + "for ((i=0; i<20000; i++)); do printf '00000000 T other_symbol_%s\\n' \"$i\"; done\n", True), + ] + for name, script, expected in cases: + with self.subTest(name=name): + self.env["NM"] = str(self.executable("bin/nm", script)) + result = self.compatible() + self.assertEqual(result.returncode == 0, expected, result.stderr) + + def installer(self): + script = (ROOT / "thirdparty/build-thirdparty.sh").read_text() + return "\n".join(re.search(r"^" + name + r"\(\) \{\n.*?^\}", script, re.M | re.S)[0] + for name in ("install_rust_archive", "install_lance_c")) + + def test_header_archive_pair_publication_and_retry(self): + archive = self.write("build/liblance_c.a", "new archive") + headers = self.work / "build/include/lance" + self.write("build/include/lance/lance.h", "new C header") + self.write("build/include/lance/lance.hpp", "new C++ header") + self.env.update(TP_INSTALL_DIR=str(self.prefix / "installed"), STRIP_TP_LIB="ON", + KERNEL="Linux", ARCHIVE=str(archive), HEADERS=str(headers)) + target_archive = self.prefix / "installed/lib64/liblance_c.a" + target_headers = self.prefix / "installed/include/lance" + marker = self.prefix / "installed/lib64/.lance-installing" + failures = { + "header-copy": 'cp() { if [[ "$1" = -pR ]]; then ' + 'printf partial > "${@: -1}/lance.h"; return 1; fi; command cp "$@"; }', + "archive-copy": 'cp() { if [[ "$1" = -p && "$2" = "$ARCHIVE" ]]; then ' + 'printf partial > "${@: -1}"; return 1; fi; command cp "$@"; }', + "archive-strip": 'strip() { printf damaged > "${@: -1}"; return 1; }', + "archive-publish": 'mv() { if [[ "${@: -1}" = */liblance_c.a ]]; then return 1; fi; ' + 'command mv "$@"; }', + "header-publish": 'mv() { if [[ "${@: -1}" = */include/lance ]]; then return 1; fi; ' + 'command mv "$@"; }', + "success": "", + } + for replacement in (False, True): + for stage, injection in failures.items(): + with self.subTest(replacement=replacement, stage=stage): + for path in (target_archive, marker): + path.unlink(missing_ok=True) + if target_headers.exists(): + shutil.rmtree(target_headers) + if replacement: + self.write("thirdparty/installed/include/lance/lance.h", "old C header") + self.write("thirdparty/installed/include/lance/lance.hpp", "old C++ header") + target_archive.write_text("old archive") + result = self.run_bash(self.installer() + '\nstrip() { :; }\n' + injection + + '\ninstall_lance_c "$ARCHIVE" "$HEADERS"') + if stage == "success": + self.assertEqual(result.returncode, 0, result.stderr) + self.assertFalse(marker.exists()) + else: + self.assertNotEqual(result.returncode, 0) + self.assertEqual(marker.exists(), stage != "header-copy") + if replacement and stage != "header-publish": + self.assertEqual((target_headers / "lance.h").read_text(), "old C header") + retry = self.run_bash(self.installer() + '\nstrip() { :; }\n' + + 'install_lance_c "$ARCHIVE" "$HEADERS"') + self.assertEqual(retry.returncode, 0, retry.stderr) + self.assertFalse(marker.exists()) + self.assertEqual((target_headers / "lance.h").read_text(), "new C header") + self.assertEqual((target_headers / "lance.hpp").read_text(), "new C++ header") + self.assertEqual(target_archive.read_text(), "new archive") + self.assertEqual(list(target_headers.parent.glob(".lance.tmp.*")), []) + self.assertEqual(list(target_archive.parent.glob("liblance_c.a.tmp.*")), []) + self.assert_unrelated_preserved() + + def test_unstaged_headers_do_not_begin_publication(self): + archive = self.write("build/liblance_c.a", "new archive") + headers = self.work / "build/include/lance" + self.write("build/include/lance/lance.h", "new C header") + self.env.update(TP_INSTALL_DIR=str(self.prefix / "installed"), STRIP_TP_LIB="OFF", + KERNEL="Linux", ARCHIVE=str(archive), HEADERS=str(headers)) + self.write("thirdparty/installed/include/lance/lance.h", "old header") + self.write("thirdparty/installed/lib64/liblance_c.a", "old archive") + result = self.run_bash(self.installer() + '\ninstall_lance_c "$ARCHIVE" "$HEADERS"') + self.assertNotEqual(result.returncode, 0) + self.assertEqual((self.prefix / "installed/include/lance/lance.h").read_text(), "old header") + self.assertEqual((self.prefix / "installed/lib64/liblance_c.a").read_text(), "old archive") + self.assertFalse((self.prefix / "installed/lib64/.lance-installing").exists()) + self.assert_unrelated_preserved() + + +if __name__ == "__main__": + unittest.main() diff --git a/thirdparty/test/lance-prefilter-patch-test.sh b/thirdparty/test/lance-prefilter-patch-test.sh index ab3587c56d1fc0..21b6e2bba6c5cd 100755 --- a/thirdparty/test/lance-prefilter-patch-test.sh +++ b/thirdparty/test/lance-prefilter-patch-test.sh @@ -75,7 +75,17 @@ check_sources() { || fail "missing upstream C++ prewarm API" grep -q 'fn test_prewarm_index_vector_segments_reuse_shared_session' "${source}/tests/c_api_test.rs" \ || fail "missing upstream shared-session prewarm regression" - [[ -f "${source}/patched_mark_foyer" ]] || fail "missing Foyer patch marker" + grep -q '^#define LANCE_STORAGE_BASES_JSON_VERSION 1$' "${source}/include/lance/lance.h" \ + || fail "missing storage-base metadata ABI version" + grep -q 'lance_dataset_storage_bases_json' "${source}/include/lance/lance.h" \ + || fail "missing storage-base C API" + grep -q 'std::string storage_bases_json() const' "${source}/include/lance/lance.hpp" \ + || fail "missing storage-base C++ API" + grep -q 'pub unsafe extern "C" fn lance_dataset_storage_bases_json' "${source}/src/dataset.rs" \ + || fail "missing storage-base implementation" + grep -q 'fn test_storage_bases_json_primary_snapshot_with_multiple_fragments' "${source}/tests/c_api_test.rs" \ + || fail "missing storage-base snapshot regression" + [[ -f "${source}/patched_mark_foyer" ]] || fail "missing Foyer/storage-base patch marker" } prepare "${tmpdir}/fresh"