diff --git a/be/CMakeLists.txt b/be/CMakeLists.txt index 13f6ebe986c887..5c38d5cf3af829 100644 --- a/be/CMakeLists.txt +++ b/be/CMakeLists.txt @@ -692,7 +692,7 @@ set(DORIS_LINK_LIBS ) set(absl_DIR ${THIRDPARTY_DIR}/lib/cmake/absl) -find_package(absl) +find_package(absl CONFIG REQUIRED PATHS ${absl_DIR} NO_DEFAULT_PATH) # COMMON_THIRDPARTY are thirdparty dependencies that can run on all platform # When adding new dependencies, If you don’t know if it can run on all platforms, @@ -722,6 +722,14 @@ add_definitions(-DUSE_DORIS_HADOOP_HDFS) if (absl_FOUND) set(COMMON_THIRDPARTY ${COMMON_THIRDPARTY} + # Public dependencies of the statically linked google-cloud-cpp common library. + absl::base + absl::memory + absl::optional + absl::span + absl::str_format + absl::time + absl::variant absl::cord absl::cord_internal absl::cordz_functions diff --git a/be/cmake/thirdparty.cmake b/be/cmake/thirdparty.cmake index abc1c7c5f3d7c8..6fddbea795bae7 100644 --- a/be/cmake/thirdparty.cmake +++ b/be/cmake/thirdparty.cmake @@ -69,6 +69,9 @@ add_thirdparty(lz4) add_thirdparty(lzo2) add_thirdparty(thrift) add_thirdparty(thriftnb) +add_thirdparty(google_cloud_cpp_oauth2) +add_thirdparty(google_cloud_cpp_rest_internal) +add_thirdparty(google_cloud_cpp_common) add_thirdparty(crc32c) add_thirdparty(libevent_core LIBNAME "lib/libevent_core.a") diff --git a/be/src/io/fs/gcs_signed_url_provider.cpp b/be/src/io/fs/gcs_signed_url_provider.cpp new file mode 100644 index 00000000000000..85e08264f29710 --- /dev/null +++ b/be/src/io/fs/gcs_signed_url_provider.cpp @@ -0,0 +1,193 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "io/fs/gcs_signed_url_provider.h" + +#include +#include + +#include +#include +#include + +#include "cpp/obj-client/auth/gcp/gcp_token_provider.h" +#include "cpp/obj-client/auth/gcp/gcs_signed_url.h" +#include "cpp/sync_point.h" +#include "service/http/http_client.h" +#include "util/url_coding.h" + +namespace doris::io { +namespace { + +constexpr std::string_view IAM_CREDENTIALS_ENDPOINT = + "https://iamcredentials.googleapis.com/v1/projects/-/serviceAccounts/"; + +std::string iam_error_message(const rapidjson::Document& document) { + if (!document.IsObject() || !document.HasMember("error") || !document["error"].IsObject()) { + return "unparseable error response"; + } + const auto& error = document["error"]; + std::string message; + if (error.HasMember("status") && error["status"].IsString()) { + message = error["status"].GetString(); + } + if (error.HasMember("message") && error["message"].IsString()) { + if (!message.empty()) { + message.append(": "); + } + message.append(error["message"].GetString()); + } + if (message.empty()) { + return "error response did not contain status or message"; + } + constexpr size_t MAX_ERROR_MESSAGE_SIZE = 1024; + if (message.size() > MAX_ERROR_MESSAGE_SIZE) { + message.resize(MAX_ERROR_MESSAGE_SIZE); + message.append("..."); + } + return message; +} + +bool is_unreserved(unsigned char c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '-' || + c == '_' || c == '.' || c == '~'; +} + +std::string percent_encode(std::string_view value) { + constexpr char HEX[] = "0123456789ABCDEF"; + std::string encoded; + encoded.reserve(value.size()); + for (unsigned char c : value) { + if (is_unreserved(c)) { + encoded.push_back(static_cast(c)); + continue; + } + encoded.push_back('%'); + encoded.push_back(HEX[c >> 4]); + encoded.push_back(HEX[c & 0x0F]); + } + return encoded; +} + +Status call_iam_sign_blob(std::string_view access_token, std::string_view service_account, + std::string_view string_to_sign, int64_t request_timeout_ms, + const std::string& ca_cert_file_path, std::string* signature) { + std::string encoded_payload; + base64_encode(std::string(string_to_sign), &encoded_payload); + std::string request_body = fmt::format(R"({{"payload":"{}"}})", encoded_payload); + std::string endpoint = + std::string(IAM_CREDENTIALS_ENDPOINT) + percent_encode(service_account) + ":signBlob"; + + HttpClient client; + // Keep the response body for non-2xx replies so IAM permission and + // service-account errors are actionable to operators. + RETURN_IF_ERROR(client.init(endpoint, false, HttpClient::AuthTokenMode::NONE)); + if (!ca_cert_file_path.empty()) { + RETURN_IF_ERROR(client.set_ca_cert_file(ca_cert_file_path)); + } + client.set_authorization("Bearer " + std::string(access_token)); + client.set_content_type("application/json"); + client.set_timeout_ms(request_timeout_ms > 0 ? request_timeout_ms : 10000); + + TEST_SYNC_POINT_RETURN_WITH_VALUE("GcsV4Signer::sign_blob", Status::OK(), access_token, + service_account, string_to_sign, signature); + std::string response; + RETURN_IF_ERROR(client.execute_post_request(request_body, &response)); + + rapidjson::Document document; + document.Parse(response.data(), response.size()); + const auto http_status = client.get_http_status(); + if (http_status < 200 || http_status >= 300) { + return Status::HttpError("IAM signBlob failed with HTTP {}: {}", http_status, + iam_error_message(document)); + } + if (document.HasParseError() || !document.IsObject() || !document.HasMember("signedBlob") || + !document["signedBlob"].IsString()) { + return Status::InternalError("IAM signBlob returned an invalid response"); + } + if (!base64_decode(document["signedBlob"].GetString(), signature)) { + return Status::InternalError("IAM signBlob returned an invalid base64 signature"); + } + return Status::OK(); +} + +} // namespace + +Status resolve_gcs_signer_email(const GcpCredentialConfig& credential, + const GcpTokenProvider& token_provider, std::string* signer_email) { + *signer_email = credential.impersonation_service_account.empty() + ? token_provider.get_service_account_email() + : credential.impersonation_service_account; + if (!is_valid_gcp_service_account_email(*signer_email)) { + return Status::InvalidArgument( + "GCS V4 signing requires a service account identity; configure " + "gs.impersonation_service_account for credentials without one"); + } + return Status::OK(); +} + +Status generate_gcs_v4_signed_url(const GcsV4SignedUrlProviderOptions& options, + const GcpCredentialConfig& credential, + const std::shared_ptr& token_provider, + std::string* signed_url) { + if (signed_url == nullptr) { + return Status::InvalidArgument("signed_url output must not be null"); + } + signed_url->clear(); + + if (token_provider == nullptr) { + return Status::InternalError("GCS V4 signing token provider is not initialized"); + } + + try { + auto token = token_provider->get_token(); + if (!token.has_value()) { + return Status::InternalError("failed to obtain OAuth token for IAM signBlob"); + } + + // Resolve identity from the same credentials that supplied the token. DEFAULT may + // select a VM, a key file, or another ADC source; never infer it from VM presence. + std::string signer_email; + RETURN_IF_ERROR(resolve_gcs_signer_email(credential, *token_provider, &signer_email)); + + auto result = build_gcs_v4_signed_url( + {.endpoint = options.endpoint, + .bucket = options.bucket, + .key = options.key, + .signer_email = signer_email, + .expiration_secs = options.expiration_secs}, + std::chrono::system_clock::now(), [&](std::string_view string_to_sign) { + std::string signature; + auto status = call_iam_sign_blob(*token, signer_email, string_to_sign, + options.request_timeout_ms, + options.ca_cert_file_path, &signature); + if (!status.ok()) { + return GcsSignBlobResult {.error = status.to_string()}; + } + return GcsSignBlobResult {.signature = std::move(signature)}; + }); + if (!result.ok()) { + return Status::InternalError("failed to build GCS V4 signed URL: {}", result.error); + } + *signed_url = std::move(result.signed_url); + return Status::OK(); + } catch (const std::exception& e) { + return Status::InternalError("failed to initialize GCS URL signer: {}", e.what()); + } +} + +} // namespace doris::io diff --git a/be/src/io/fs/gcs_signed_url_provider.h b/be/src/io/fs/gcs_signed_url_provider.h new file mode 100644 index 00000000000000..872622059223bb --- /dev/null +++ b/be/src/io/fs/gcs_signed_url_provider.h @@ -0,0 +1,54 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "common/status.h" +#include "cpp/obj-client/auth/gcp/gcp_auth.h" + +namespace doris { +class GcpTokenProvider; +} + +namespace doris::io { + +struct GcsV4SignedUrlProviderOptions { + std::string endpoint; + std::string bucket; + std::string key; + int64_t expiration_secs = 0; + int64_t request_timeout_ms = 10000; + std::string ca_cert_file_path; +}; + +// Resolve the identity selected by ADC or explicit impersonation, without falling back +// to an unrelated VM service account when the selected credentials have no identity. +Status resolve_gcs_signer_email(const GcpCredentialConfig& credential, + const GcpTokenProvider& token_provider, std::string* signer_email); + +// Resolves the signing service account, obtains an OAuth token, and invokes +// IAM Credentials signBlob before delegating URL construction to common auth. +Status generate_gcs_v4_signed_url(const GcsV4SignedUrlProviderOptions& options, + const GcpCredentialConfig& credential, + const std::shared_ptr& token_provider, + std::string* signed_url); + +} // namespace doris::io diff --git a/be/src/io/fs/s3_file_system.cpp b/be/src/io/fs/s3_file_system.cpp index d307335d15a517..1635c166617d5a 100644 --- a/be/src/io/fs/s3_file_system.cpp +++ b/be/src/io/fs/s3_file_system.cpp @@ -85,6 +85,7 @@ Status ObjClientHolder::reset(const S3ClientConf& conf) { { std::shared_lock lock(_mtx); reset_conf = _conf; + reset_conf.provider = conf.provider; reset_conf.ak = conf.ak; reset_conf.sk = conf.sk; reset_conf.token = conf.token; @@ -98,6 +99,7 @@ Status ObjClientHolder::reset(const S3ClientConf& conf) { reset_conf.role_arn = conf.role_arn; reset_conf.external_id = conf.external_id; reset_conf.cred_provider_type = conf.cred_provider_type; + reset_conf.credential = conf.credential; // Compare full-field equality of the merged conf, not get_hash(): the hash is // an XOR of crc32s and distinct configurations can collide, which would skip a @@ -444,7 +446,9 @@ Status S3FileSystem::download_impl(const Path& remote_file, const Path& local_fi // whether to return a public endpoint. std::string S3FileSystem::generate_presigned_url(const Path& path, int64_t expiration_secs, bool is_public_endpoint) const { - std::string key = fmt::format("{}/{}", _prefix, path.native()); + // An empty prefix must not add a separator absent from the uploaded object key. + std::string key = + _prefix.empty() ? path.native() : fmt::format("{}/{}", _prefix, path.native()); std::shared_ptr client; if (is_public_endpoint && _client->s3_client_conf().endpoint.ends_with(OSS_PRIVATE_ENDPOINT_SUFFIX)) { diff --git a/be/src/runtime/runtime_state.cpp b/be/src/runtime/runtime_state.cpp index a6185aa10b1658..1a69a1ce4cbec3 100644 --- a/be/src/runtime/runtime_state.cpp +++ b/be/src/runtime/runtime_state.cpp @@ -599,6 +599,11 @@ std::string RuntimeState::get_error_log_file_path() { auto presigned_url = s3_error_fs->generate_presigned_url(remote_error_log_file_path, EXPIRATION_SECONDS, config::use_public_endpoint_for_error_log); + if (presigned_url.empty()) { + LOG(WARNING) << "Fail to generate presigned URL for S3 error log; retain local path " + << local_error_log_file_path; + return local_error_log_file_path; + } std::lock_guard load_lock(_load_error_log_lock); _error_log_file_path = std::move(presigned_url); return _error_log_file_path; diff --git a/be/src/service/http/http_client.cpp b/be/src/service/http/http_client.cpp index 9e44a6af6a5f73..5c79c85628c461 100644 --- a/be/src/service/http/http_client.cpp +++ b/be/src/service/http/http_client.cpp @@ -34,6 +34,7 @@ #include "common/cast_set.h" #include "common/config.h" #include "common/status.h" +#include "cpp/sync_point.h" #include "io/fs/local_file_system.h" #include "runtime/cluster_info.h" #include "runtime/exec_env.h" @@ -356,7 +357,8 @@ void HttpClient::reset() { _method = GET; } -Status HttpClient::init(const std::string& url, bool set_fail_on_error) { +Status HttpClient::init(const std::string& url, bool set_fail_on_error, + AuthTokenMode auth_token_mode) { reset(); if (_curl == nullptr) { _curl = curl_easy_init(); @@ -426,9 +428,20 @@ Status HttpClient::init(const std::string& url, bool set_fail_on_error) { return Status::InternalError("fail to set CURLOPT_URL"); } + if (auth_token_mode == AuthTokenMode::CLUSTER) { #ifndef BE_TEST - set_auth_token(ExecEnv::GetInstance()->cluster_info()->curr_auth_token); + set_auth_token(ExecEnv::GetInstance()->cluster_info()->curr_auth_token); #endif + } + return Status::OK(); +} + +Status HttpClient::set_ca_cert_file(const std::string& path) { + auto code = curl_easy_setopt(_curl, CURLOPT_CAINFO, path.c_str()); + if (code != CURLE_OK) { + return Status::InternalError("failed to set CA certificate file: {}", _to_errmsg(code)); + } + TEST_SYNC_POINT_CALLBACK("HttpClient::set_ca_cert_file", &path); return Status::OK(); } diff --git a/be/src/service/http/http_client.h b/be/src/service/http/http_client.h index de0dd202ad37f8..68b414631efbf0 100644 --- a/be/src/service/http/http_client.h +++ b/be/src/service/http/http_client.h @@ -44,6 +44,8 @@ struct HttpTransferMetrics { // Helper class to access HTTP resource class HttpClient { public: + enum class AuthTokenMode { CLUSTER, NONE }; + HttpClient(); ~HttpClient(); @@ -56,7 +58,8 @@ class HttpClient { // this function must call before other function, // you can call this multiple times to reuse this object - Status init(const std::string& url, bool set_fail_on_error = true); + Status init(const std::string& url, bool set_fail_on_error = true, + AuthTokenMode auth_token_mode = AuthTokenMode::CLUSTER); // Release request-owned options and callbacks while retaining reusable connections. void reset(); @@ -95,7 +98,9 @@ class HttpClient { curl_easy_setopt(_curl, CURLOPT_COPYPOSTFIELDS, post_body.c_str()); } - // Currently, only fake SSL configurations are supported + // Override the trusted CA bundle after init(), which resets curl options. + Status set_ca_cert_file(const std::string& path); + void use_untrusted_ssl() { curl_easy_setopt(_curl, CURLOPT_SSL_VERIFYPEER, 0L); curl_easy_setopt(_curl, CURLOPT_SSL_VERIFYHOST, 0L); diff --git a/be/src/util/obj_credential.cpp b/be/src/util/obj_credential.cpp new file mode 100644 index 00000000000000..6308350c0e441f --- /dev/null +++ b/be/src/util/obj_credential.cpp @@ -0,0 +1,76 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "cpp/obj-client/auth/obj_credential.h" + +#include +#include + +namespace doris { +namespace { + +GcpCredentialConfig to_gcp_credential_config(const cloud::GcpCredentialPB& credential) { + GcpCredentialConfig config; + switch (credential.credential_provider_type()) { + case cloud::GcpCredentialPB::DEFAULT: + config.provider_type = GcpCredentialProviderType::Default; + break; + case cloud::GcpCredentialPB::COMPUTE_ENGINE: + config.provider_type = GcpCredentialProviderType::ComputeEngine; + break; + } + config.impersonation_service_account = credential.impersonation_service_account(); + return config; +} + +GcpCredentialConfig to_gcp_credential_config(const TGcpCredential& credential) { + GcpCredentialConfig config; + if (credential.__isset.credential_provider_type) { + switch (credential.credential_provider_type) { + case TGcpCredentialProviderType::DEFAULT: + config.provider_type = GcpCredentialProviderType::Default; + break; + case TGcpCredentialProviderType::COMPUTE_ENGINE: + config.provider_type = GcpCredentialProviderType::ComputeEngine; + break; + } + } + if (credential.__isset.impersonation_service_account) { + config.impersonation_service_account = credential.impersonation_service_account; + } + return config; +} + +} // namespace + +void convert_obj_credential(const cloud::ObjectStoreInfoPB& source, CredentialConfig* credential) { + if (!source.has_credential() || !source.credential().has_gcp_credential()) { + *credential = std::monostate {}; + return; + } + *credential = to_gcp_credential_config(source.credential().gcp_credential()); +} + +void convert_obj_credential(const TCredential& source, CredentialConfig* credential) { + if (!source.__isset.gcp_credential) { + *credential = std::monostate {}; + return; + } + *credential = to_gcp_credential_config(source.gcp_credential); +} + +} // namespace doris diff --git a/be/src/util/s3_util.cpp b/be/src/util/s3_util.cpp index cd5d019a18f2c5..f3487fd9cc98d0 100644 --- a/be/src/util/s3_util.cpp +++ b/be/src/util/s3_util.cpp @@ -51,12 +51,16 @@ #endif #include "cloud/config.h" #include "cpp/aws_logger.h" +#include "cpp/obj-client/auth/gcp/gcp_token_provider.h" +#include "cpp/obj-client/auth/obj_credential.h" +#include "cpp/obj-client/auth/obj_s3_client_factory.h" #include "cpp/obj-client/rate_limited_obj_storage_client.h" #include "cpp/obj-client/s3_obj_storage_client.h" #include "cpp/obj_retry_strategy.h" #include "cpp/sync_point.h" #include "cpp/util.h" #include "exec/scan/scanner_scheduler.h" +#include "io/fs/gcs_signed_url_provider.h" #include "runtime/exec_env.h" #include "util/s3_rate_limiter_manager.h" #include "util/s3_uri.h" @@ -64,6 +68,32 @@ namespace doris { namespace { +// Adapter between the IO storage-domain enum and the credential layer. Keep it +// local: neither layer should own or depend on the other's enum definition. +ObjCredentialProvider to_obj_credential_provider(io::ObjStorageProvider provider) { + switch (provider) { + case io::ObjStorageProvider::AWS: + return ObjCredentialProvider::Aws; + case io::ObjStorageProvider::AZURE: + return ObjCredentialProvider::Azure; + case io::ObjStorageProvider::BOS: + return ObjCredentialProvider::Bos; + case io::ObjStorageProvider::COS: + return ObjCredentialProvider::Cos; + case io::ObjStorageProvider::GCP: + return ObjCredentialProvider::Gcp; + case io::ObjStorageProvider::OBS: + return ObjCredentialProvider::Obs; + case io::ObjStorageProvider::OSS: + return ObjCredentialProvider::Oss; + case io::ObjStorageProvider::TOS: + return ObjCredentialProvider::Tos; + case io::ObjStorageProvider::UNKNOWN: + return ObjCredentialProvider::Unknown; + } + return ObjCredentialProvider::Unknown; +} + doris::Status is_s3_conf_valid(const S3ClientConf& conf) { if (conf.endpoint.empty()) { return Status::InvalidArgument("Invalid s3 conf, empty endpoint"); @@ -72,6 +102,20 @@ doris::Status is_s3_conf_valid(const S3ClientConf& conf) { return Status::InvalidArgument("Invalid s3 conf, empty region"); } + ObjCredentialValidationContext credential_context { + .provider = to_obj_credential_provider(conf.provider), + .ak = conf.ak, + .sk = conf.sk, + .token = conf.token, + .has_aws_role_arn = !conf.role_arn.empty(), + .has_aws_external_id = !conf.external_id.empty(), + .has_aws_credential_provider = conf.cred_provider_type != CredProviderType::Default, + }; + if (auto error = validate_obj_credential_config(conf.credential, credential_context); + error.has_value()) { + return Status::InvalidArgument(*error); + } + if (conf.role_arn.empty()) { // Allow anonymous access when both ak and sk are empty bool hasAk = !conf.ak.empty(); @@ -151,6 +195,7 @@ std::string build_azure_tls_debug_context(const std::string& selected_ca_file) { constexpr char USE_PATH_STYLE[] = "use_path_style"; constexpr char AZURE_PROVIDER_STRING[] = "AZURE"; +constexpr char GCP_PROVIDER_STRING[] = "GCP"; constexpr char S3_PROVIDER[] = "provider"; constexpr char S3_AK[] = "AWS_ACCESS_KEY"; constexpr char S3_SK[] = "AWS_SECRET_KEY"; @@ -389,17 +434,54 @@ Result> S3ClientFactory::_create_s3_client return ResultError(Status::InvalidArgument("failed to create AWS credential provider: {}", credentials.error)); } - std::shared_ptr new_client = std::make_shared( - std::move(credentials.provider), std::move(aws_config), - Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, - s3_conf.use_virtual_addressing); + auto new_client = make_s3_client( + s3_conf.credential, + S3ClientBuildContext { + .fallback_provider = std::move(credentials.provider), + .config = std::move(aws_config), + .payload_signing_policy = + Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, + .use_virtual_addressing = s3_conf.use_virtual_addressing, + .ca_cert_path = ca_cert_file_path, + }); + std::shared_ptr gcs_signing_token_provider; + if (const auto* gcp_credential = std::get_if(&s3_conf.credential)) { + // signBlob is authorized by the source identity, not by the optional + // impersonated target identity. Keep one refreshable cloud-platform + // token provider for the lifetime of the S3 client. + auto signing_caller_credential = *gcp_credential; + signing_caller_credential.impersonation_service_account.clear(); + gcs_signing_token_provider = std::make_shared( + signing_caller_credential, ca_cert_file_path, GcpTokenScope::CloudPlatform); + } + + S3ObjStorageClient::SignedUrlGenerator signed_url_generator; + if (const auto* credential = std::get_if(&s3_conf.credential)) { + signed_url_generator = [conf = s3_conf, credential = *credential, ca_cert_file_path, + token_provider = std::move(gcs_signing_token_provider)]( + const ObjStoragePath& path, int64_t expiration_secs) { + std::string signed_url; + auto status = + io::generate_gcs_v4_signed_url({.endpoint = conf.endpoint, + .bucket = path.bucket, + .key = path.key, + .expiration_secs = expiration_secs, + .request_timeout_ms = conf.request_timeout_ms, + .ca_cert_file_path = ca_cert_file_path}, + credential, token_provider, &signed_url); + if (!status.ok()) { + LOG(WARNING) << "failed to generate GCS V4 signed URL: " << status; + return std::string {}; + } + return signed_url; + }; + } auto provider_client = std::make_shared( - std::move(new_client), ObjStorageEndpointInfo { - .endpoint = s3_conf.endpoint, - .ak = s3_conf.ak, - .sk = s3_conf.sk, - }); + std::move(new_client), + ObjStorageEndpointInfo { + .endpoint = s3_conf.endpoint, .ak = s3_conf.ak, .sk = s3_conf.sk}, + std::move(signed_url_generator)); LOG_INFO("create one s3 client with {}", s3_conf.to_string()); return provider_client; } @@ -446,6 +528,8 @@ Status S3ClientFactory::convert_properties_to_s3_conf( // S3 Provider properties should be case insensitive. if (0 == strcasecmp(it->second.c_str(), AZURE_PROVIDER_STRING)) { s3_conf->client_conf.provider = io::ObjStorageProvider::AZURE; + } else if (0 == strcasecmp(it->second.c_str(), GCP_PROVIDER_STRING)) { + s3_conf->client_conf.provider = io::ObjStorageProvider::GCP; } } @@ -478,6 +562,11 @@ Status S3ClientFactory::convert_properties_to_s3_conf( s3_conf->client_conf.cred_provider_type = cred_provider_type_from_string(it->second); } + if (auto error = parse_obj_credential_properties(properties, &s3_conf->client_conf.credential); + error.has_value()) { + return Status::InvalidArgument(*error); + } + if (auto st = is_s3_conf_valid(s3_conf->client_conf); !st.ok()) { return st; } @@ -537,6 +626,7 @@ S3Conf S3Conf::get_s3_conf(const cloud::ObjectStoreInfoPB& info) { if (info.has_cred_provider_type()) { ret.client_conf.cred_provider_type = cred_provider_type_from_pb(info.cred_provider_type()); } + convert_obj_credential(info, &ret.client_conf.credential); io::ObjStorageProvider type = io::ObjStorageProvider::AWS; switch (info.provider()) { @@ -598,6 +688,9 @@ S3Conf S3Conf::get_s3_conf(const TS3StorageParam& param) { ret.client_conf.cred_provider_type = cred_provider_type_from_thrift(param.cred_provider_type); } + if (param.__isset.credential) { + convert_obj_credential(param.credential, &ret.client_conf.credential); + } io::ObjStorageProvider type = io::ObjStorageProvider::AWS; switch (param.provider) { diff --git a/be/src/util/s3_util.h b/be/src/util/s3_util.h index b2ca49d4751cb9..6856d205ff5241 100644 --- a/be/src/util/s3_util.h +++ b/be/src/util/s3_util.h @@ -39,6 +39,7 @@ #include "core/string_ref.h" #include "cpp/aws_common.h" #include "cpp/obj-client/auth/aws_credential_factory.h" +#include "cpp/obj-client/auth/obj_credential.h" #include "cpp/obj-client/obj_storage_client.h" namespace Aws::S3 { @@ -74,6 +75,7 @@ struct S3ClientConf { CredProviderType cred_provider_type = CredProviderType::Default; std::string role_arn; std::string external_id; + CredentialConfig credential {}; // True when this client is bound to a Doris internal object storage bucket // (a storage vault in cloud mode). S3ClientFactory wraps such clients with the // shared rate limiter; external buckets (S3 load, TVF, external catalogs) are @@ -102,6 +104,7 @@ struct S3ClientConf { hash_code ^= static_cast(cred_provider_type); hash_code ^= crc32_hash(role_arn); hash_code ^= crc32_hash(external_id); + hash_combine_credential(&hash_code, credential); hash_code ^= is_internal_bucket; return hash_code; } @@ -110,10 +113,12 @@ struct S3ClientConf { return fmt::format( "(ak={}, token={}, endpoint={}, region={}, bucket={}, max_connections={}, " "request_timeout_ms={}, connect_timeout_ms={}, use_virtual_addressing={}, " - "cred_provider_type={},role_arn={}, external_id={}, is_internal_bucket={}", + "cred_provider_type={},role_arn={}, external_id={}, credential_type={}, " + "is_internal_bucket={}", hide_access_key(ak), token.empty() ? "" : "******", endpoint, region, bucket, max_connections, request_timeout_ms, connect_timeout_ms, use_virtual_addressing, - cred_provider_type, role_arn, external_id, is_internal_bucket); + cred_provider_type, role_arn, external_id, credential_type_name(credential), + is_internal_bucket); } }; diff --git a/be/test/common/auth/gcp/gcp_token_provider_test.cpp b/be/test/common/auth/gcp/gcp_token_provider_test.cpp new file mode 100644 index 00000000000000..9ce2988e812ba9 --- /dev/null +++ b/be/test/common/auth/gcp/gcp_token_provider_test.cpp @@ -0,0 +1,219 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "cpp/obj-client/auth/gcp/gcp_token_provider.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "service/http/ev_http_server.h" +#include "service/http/http_channel.h" +#include "service/http/http_handler.h" +#include "service/http/http_headers.h" +#include "service/http/http_request.h" +#include "service/http/http_status.h" + +namespace doris { +namespace { + +class OAuthTokenHandler : public HttpHandler { +public: + void handle(HttpRequest* request) override { + std::lock_guard lock(mutex); + requests.push_back(request->get_request_body()); + request->add_output_header(HttpHeaders::CONTENT_TYPE, "application/json"); + if (reject) { + HttpChannel::send_reply(request, HttpStatus::UNAUTHORIZED, + R"({"error":"invalid_grant"})"); + return; + } + HttpChannel::send_reply( + request, + R"({"access_token":"oauth-access-token","expires_in":3600,"token_type":"Bearer"})"); + } + + std::mutex mutex; + std::vector requests; + bool reject = false; +}; + +class GcpTokenProviderTest : public testing::Test { +protected: + void SetUp() override { + for (size_t i = 0; i < ENV_NAMES.size(); ++i) { + const char* value = std::getenv(ENV_NAMES[i]); + saved_env[i] = value == nullptr ? std::nullopt : std::optional(value); + unsetenv(ENV_NAMES[i]); + } + char path[] = "/tmp/doris_gcp_adc_XXXXXX"; + int fd = mkstemp(path); + ASSERT_GE(fd, 0); + close(fd); + adc_path = path; + ASSERT_EQ(setenv("GOOGLE_APPLICATION_CREDENTIALS", adc_path.c_str(), 1), 0); + ASSERT_TRUE(server.register_handler(POST, "/token", &handler)); + server.start(); + ASSERT_GT(server.get_real_port(), 0); + token_uri = "http://127.0.0.1:" + std::to_string(server.get_real_port()) + "/token"; + + // This is a freshly generated test key, never a real Google identity or secret. + std::unique_ptr context( + EVP_PKEY_CTX_new_id(EVP_PKEY_RSA, nullptr), EVP_PKEY_CTX_free); + ASSERT_NE(context, nullptr); + ASSERT_EQ(EVP_PKEY_keygen_init(context.get()), 1); + ASSERT_EQ(EVP_PKEY_CTX_set_rsa_keygen_bits(context.get(), 2048), 1); + EVP_PKEY* raw_key = nullptr; + ASSERT_EQ(EVP_PKEY_keygen(context.get(), &raw_key), 1); + std::unique_ptr key(raw_key, EVP_PKEY_free); + std::unique_ptr bio(BIO_new(BIO_s_mem()), BIO_free); + ASSERT_NE(bio, nullptr); + ASSERT_EQ(PEM_write_bio_PrivateKey(bio.get(), key.get(), nullptr, nullptr, 0, nullptr, + nullptr), + 1); + char* pem = nullptr; + auto length = BIO_get_mem_data(bio.get(), &pem); + private_key.assign(pem, length); + write_service_account(); + } + + void TearDown() override { + std::remove(adc_path.c_str()); + for (size_t i = 0; i < ENV_NAMES.size(); ++i) { + if (saved_env[i]) { + setenv(ENV_NAMES[i], saved_env[i]->c_str(), 1); + } else { + unsetenv(ENV_NAMES[i]); + } + } + } + + void write_service_account() { + rapidjson::Document document; + document.SetObject(); + auto& allocator = document.GetAllocator(); + document.AddMember("type", "service_account", allocator); + document.AddMember("client_email", "test@example.iam.gserviceaccount.com", allocator); + document.AddMember("private_key_id", "test-key-id", allocator); + document.AddMember("private_key", rapidjson::Value(private_key.c_str(), allocator), + allocator); + document.AddMember("token_uri", rapidjson::Value(token_uri.c_str(), allocator), allocator); + rapidjson::StringBuffer buffer; + rapidjson::Writer writer(buffer); + document.Accept(writer); + std::ofstream output(adc_path); + output << buffer.GetString(); + ASSERT_TRUE(output.good()); + } + + void expect_exchange_scope(const std::string& expected_scope) { + std::lock_guard lock(handler.mutex); + ASSERT_EQ(handler.requests.size(), 1); + const auto& body = handler.requests.front(); + EXPECT_NE(body.find("grant_type="), std::string::npos); + auto assertion = body.find("assertion="); + ASSERT_NE(assertion, std::string::npos); + auto start = body.find('.', assertion); + ASSERT_NE(start, std::string::npos); + auto end = body.find('.', start + 1); + ASSERT_NE(end, std::string::npos); + auto encoded = body.substr(start + 1, end - start - 1); + std::replace(encoded.begin(), encoded.end(), '-', '+'); + std::replace(encoded.begin(), encoded.end(), '_', '/'); + while (encoded.size() % 4 != 0) { + encoded.push_back('='); + } + std::string decoded(encoded.size(), '\0'); + int length = EVP_DecodeBlock(reinterpret_cast(decoded.data()), + reinterpret_cast(encoded.data()), + encoded.size()); + ASSERT_GT(length, 0); + rapidjson::Document payload; + payload.Parse(decoded.c_str()); + ASSERT_FALSE(payload.HasParseError()); + ASSERT_TRUE(payload.HasMember("scope")); + EXPECT_EQ(std::string(payload["scope"].GetString()), expected_scope); + ASSERT_TRUE(payload.HasMember("aud")); + EXPECT_EQ(std::string(payload["aud"].GetString()), token_uri); + } + + static constexpr std::array ENV_NAMES { + "GOOGLE_APPLICATION_CREDENTIALS", "GOOGLE_GCLOUD_ADC_PATH_OVERRIDE", + "GOOGLE_CLOUD_CPP_EXPERIMENTAL_DISABLE_SELF_SIGNED_JWT"}; + std::array, ENV_NAMES.size()> saved_env; + std::string adc_path; + std::string private_key; + std::string token_uri; + OAuthTokenHandler handler; + // The port-only constructor initializes the worker event bases before start(). + EvHttpServer server {0}; +}; + +TEST_F(GcpTokenProviderTest, ServiceAccountAdcExchangesAndCachesOAuthToken) { + GcpTokenProvider provider(GcpCredentialConfig {}); + EXPECT_EQ(provider.get_token(), "oauth-access-token"); + EXPECT_EQ(provider.get_token(), "oauth-access-token"); + EXPECT_EQ(provider.get_service_account_email(), "test@example.iam.gserviceaccount.com"); + expect_exchange_scope("https://www.googleapis.com/auth/devstorage.read_write"); + EXPECT_EQ(std::getenv("GOOGLE_CLOUD_CPP_EXPERIMENTAL_DISABLE_SELF_SIGNED_JWT"), nullptr); +} + +TEST_F(GcpTokenProviderTest, WellKnownAdcExchangesOAuthTokenWithCloudPlatformScope) { + unsetenv("GOOGLE_APPLICATION_CREDENTIALS"); + ASSERT_EQ(setenv("GOOGLE_GCLOUD_ADC_PATH_OVERRIDE", adc_path.c_str(), 1), 0); + GcpTokenProvider provider(GcpCredentialConfig {}, "", GcpTokenScope::CloudPlatform); + EXPECT_EQ(provider.get_token(), "oauth-access-token"); + expect_exchange_scope("https://www.googleapis.com/auth/cloud-platform"); +} + +TEST_F(GcpTokenProviderTest, OAuthFailureDoesNotReturnSelfSignedJwt) { + { + std::lock_guard lock(handler.mutex); + handler.reject = true; + } + GcpTokenProvider provider(GcpCredentialConfig {}); + EXPECT_FALSE(provider.get_token().has_value()); + expect_exchange_scope("https://www.googleapis.com/auth/devstorage.read_write"); +} + +TEST_F(GcpTokenProviderTest, InvalidServiceAccountAdcDoesNotFallBackToAnotherIdentity) { + std::ofstream(adc_path) + << R"({"type":"service_account","client_email":"test@example.iam.gserviceaccount.com"})"; + GcpTokenProvider provider(GcpCredentialConfig {}); + EXPECT_FALSE(provider.get_token().has_value()); + std::lock_guard lock(handler.mutex); + EXPECT_TRUE(handler.requests.empty()); +} + +} // namespace +} // namespace doris diff --git a/be/test/common/auth/gcp/gcs_signed_url_test.cpp b/be/test/common/auth/gcp/gcs_signed_url_test.cpp new file mode 100644 index 00000000000000..5425fa7c979642 --- /dev/null +++ b/be/test/common/auth/gcp/gcs_signed_url_test.cpp @@ -0,0 +1,148 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "cpp/obj-client/auth/gcp/gcs_signed_url.h" + +#include + +#include +#include + +namespace doris { +namespace { + +TEST(GcsSignedUrlTest, BuildsV4UrlAndPassesCanonicalStringToSigner) { + GcsV4SignedUrlOptions options { + .endpoint = "storage.googleapis.com/", + .bucket = "test-bucket", + .key = "error_log/a b+%.csv", + .signer_email = "signer@my-project.iam.gserviceaccount.com", + .expiration_secs = 60, + }; + std::string captured_string_to_sign; + auto result = build_gcs_v4_signed_url( + options, std::chrono::system_clock::time_point {}, + [&](std::string_view string_to_sign) { + captured_string_to_sign = string_to_sign; + return GcsSignBlobResult {.signature = std::string("\x00\xab\xff", 3)}; + }); + + ASSERT_TRUE(result.ok()) << result.error; + EXPECT_EQ(result.signed_url, + "https://storage.googleapis.com/test-bucket/error_log/a%20b%2B%25.csv?" + "X-Goog-Algorithm=GOOG4-RSA-SHA256&" + "X-Goog-Credential=signer%40my-project.iam.gserviceaccount.com%2F19700101%2Fauto%" + "2Fstorage%2Fgoog4_request&X-Goog-Date=19700101T000000Z&X-Goog-Expires=60&" + "X-Goog-SignedHeaders=host&X-Goog-Signature=00abff"); + + EXPECT_EQ(captured_string_to_sign, + "GOOG4-RSA-SHA256\n19700101T000000Z\n" + "19700101/auto/storage/goog4_request\n" + "f741db2e699530357ae34622ae689c7ec43e10e418af0ff7f1487beb1f144ce5"); +} + +TEST(GcsSignedUrlTest, DefaultHttpsPortProducesTheSameSignatureAndUrl) { + GcsV4SignedUrlOptions options { + .endpoint = "https://storage.googleapis.com", + .bucket = "test-bucket", + .key = "error_log/id", + .signer_email = "signer@my-project.iam.gserviceaccount.com", + .expiration_secs = 60, + }; + std::string without_port; + auto expected = build_gcs_v4_signed_url(options, std::chrono::system_clock::time_point {}, + [&](std::string_view value) { + without_port = value; + return GcsSignBlobResult {.signature = "test"}; + }); + ASSERT_TRUE(expected.ok()) << expected.error; + for (const auto* endpoint : + {"https://storage.googleapis.com:443", "storage.googleapis.com:443/"}) { + options.endpoint = endpoint; + std::string with_port; + auto result = build_gcs_v4_signed_url(options, std::chrono::system_clock::time_point {}, + [&](std::string_view value) { + with_port = value; + return GcsSignBlobResult {.signature = "test"}; + }); + ASSERT_TRUE(result.ok()) << result.error; + EXPECT_EQ(result.signed_url, expected.signed_url); + EXPECT_EQ(with_port, without_port); + } +} + +TEST(GcsSignedUrlTest, RejectsBucketEndpointsBeforeSigning) { + GcsV4SignedUrlOptions options { + .endpoint = "", + .bucket = "test-bucket", + .key = "error_log/id", + .signer_email = "signer@my-project.iam.gserviceaccount.com", + .expiration_secs = 60, + }; + for (const auto* endpoint : {"https://test-bucket.storage.googleapis.com", + "https://test-bucket.us-central1-storage.googleapis.com", + "https://test-bucket.storage.us-central1.rep.googleapis.com"}) { + options.endpoint = endpoint; + bool signer_called = false; + auto result = build_gcs_v4_signed_url(options, std::chrono::system_clock::time_point {}, + [&](std::string_view) { + signer_called = true; + return GcsSignBlobResult {.signature = "test"}; + }); + EXPECT_FALSE(result.ok()) << endpoint; + EXPECT_FALSE(signer_called); + } +} + +TEST(GcsSignedUrlTest, PreservesLiteralLeadingSlashInObjectKey) { + GcsV4SignedUrlOptions options { + .endpoint = "https://storage.googleapis.com", + .bucket = "test-bucket", + .key = "/error_log/id", + .signer_email = "signer@my-project.iam.gserviceaccount.com", + .expiration_secs = 60, + }; + auto result = build_gcs_v4_signed_url( + options, std::chrono::system_clock::time_point {}, + [](std::string_view) { return GcsSignBlobResult {.signature = "test"}; }); + ASSERT_TRUE(result.ok()) << result.error; + EXPECT_TRUE(result.signed_url.starts_with( + "https://storage.googleapis.com/test-bucket//error_log/id?")); +} + +TEST(GcsSignedUrlTest, RejectsInvalidExpirationBeforeSigning) { + GcsV4SignedUrlOptions options { + .endpoint = "storage.googleapis.com", + .bucket = "test-bucket", + .key = "error_log/file", + .signer_email = "signer@my-project.iam.gserviceaccount.com", + .expiration_secs = 7 * 24 * 60 * 60 + 1, + }; + bool signer_called = false; + auto result = build_gcs_v4_signed_url(options, std::chrono::system_clock::time_point {}, + [&](std::string_view) { + signer_called = true; + return GcsSignBlobResult {}; + }); + + EXPECT_FALSE(result.ok()); + EXPECT_FALSE(signer_called); + EXPECT_TRUE(result.signed_url.empty()); +} + +} // namespace +} // namespace doris diff --git a/be/test/io/fs/gcs_signed_url_provider_test.cpp b/be/test/io/fs/gcs_signed_url_provider_test.cpp new file mode 100644 index 00000000000000..503cefea9fff89 --- /dev/null +++ b/be/test/io/fs/gcs_signed_url_provider_test.cpp @@ -0,0 +1,155 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "io/fs/gcs_signed_url_provider.h" + +#include + +#include +#include + +#include "cpp/obj-client/auth/gcp/gcp_token_provider.h" +#include "cpp/sync_point.h" + +namespace doris::io { +namespace { + +class SelectedAccountCredentials : public google::cloud::oauth2_internal::Credentials { +public: + explicit SelectedAccountCredentials(std::string email) : _email(std::move(email)) {} + + google::cloud::StatusOr GetToken( + std::chrono::system_clock::time_point now) override { + return google::cloud::AccessToken {"test-token", now + std::chrono::hours(1)}; + } + + std::string AccountEmail() const override { return _email; } + +private: + std::string _email; +}; + +class GcsSignedUrlProviderTest : public testing::Test { +protected: + void TearDown() override { + SyncPoint::get_instance()->clear_call_back("GcsV4Signer::sign_blob"); + SyncPoint::get_instance()->clear_call_back("HttpClient::set_ca_cert_file"); + SyncPoint::get_instance()->disable_processing(); + } +}; + +TEST_F(GcsSignedUrlProviderTest, DefaultSignsErrorLogWithResolvedIdentity) { + auto tokens = std::make_shared( + std::make_shared("vm@my-project.iam.gserviceaccount.com")); + bool called = false; + SyncPoint::get_instance()->set_call_back( + "GcsV4Signer::sign_blob", [&](std::vector&& args) { + called = true; + EXPECT_EQ(try_any_cast(args[0]), "test-token"); + EXPECT_EQ(try_any_cast(args[1]), + "vm@my-project.iam.gserviceaccount.com"); + EXPECT_EQ(try_any_cast(args[2]).find("GOOG4-RSA-SHA256"), 0); + *try_any_cast(args[3]) = "test-signature"; + auto* result = try_any_cast_ret(args); + result->second = true; + }); + SyncPoint::get_instance()->enable_processing(); + GcpCredentialConfig credential; + credential.provider_type = GcpCredentialProviderType::Default; + GcsV4SignedUrlProviderOptions options {.endpoint = "storage.googleapis.com", + .bucket = "bucket", + .key = "load-errors/error.log", + .expiration_secs = 300, + .ca_cert_file_path = ""}; + std::string url; + auto status = generate_gcs_v4_signed_url(options, credential, tokens, &url); + ASSERT_TRUE(status.ok()) << status; + EXPECT_TRUE(called); + EXPECT_NE(url.find("vm%40my-project.iam.gserviceaccount.com"), std::string::npos); + EXPECT_NE(url.find("X-Goog-Signature="), std::string::npos); +} + +TEST_F(GcsSignedUrlProviderTest, SignBlobUsesConfiguredCaBundle) { + auto tokens = std::make_shared( + std::make_shared("vm@my-project.iam.gserviceaccount.com")); + GcsV4SignedUrlProviderOptions options {.endpoint = "storage.googleapis.com", + .bucket = "bucket", + .key = "load-errors/error.log", + .expiration_secs = 300, + .ca_cert_file_path = "/custom/certs/proxy-ca.pem"}; + bool called = false; + SyncPoint::get_instance()->set_call_back( + "HttpClient::set_ca_cert_file", [&](std::vector&& args) { + called = true; + EXPECT_EQ(*try_any_cast(args[0]), options.ca_cert_file_path); + }); + SyncPoint::get_instance()->set_call_back( + "GcsV4Signer::sign_blob", [&](std::vector&& args) { + // Exercise HTTP client setup but avoid sending a live IAM request. + *try_any_cast(args[3]) = "test-signature"; + try_any_cast_ret(args)->second = true; + }); + SyncPoint::get_instance()->enable_processing(); + std::string url; + auto status = generate_gcs_v4_signed_url(options, GcpCredentialConfig {}, tokens, &url); + ASSERT_TRUE(status.ok()) << status; + EXPECT_TRUE(called); + EXPECT_NE(url.find("X-Goog-Signature="), std::string::npos); +} + +TEST_F(GcsSignedUrlProviderTest, DefaultAndComputeUseTheSelectedAccount) { + // Represents ADC selecting either the VM account or a service-account file. + // Both must use the account attached to the token, without a separate metadata lookup. + for (auto source : + {GcpCredentialProviderType::Default, GcpCredentialProviderType::ComputeEngine}) { + for (const auto* email : + {"vm@my-project.iam.gserviceaccount.com", "file@my-project.iam.gserviceaccount.com"}) { + GcpTokenProvider tokens(std::make_shared(email)); + EXPECT_EQ(tokens.get_token(), "test-token"); + GcpCredentialConfig credential; + credential.provider_type = source; + std::string signer; + ASSERT_TRUE(resolve_gcs_signer_email(credential, tokens, &signer).ok()); + EXPECT_EQ(signer, email); + } + } +} + +TEST_F(GcsSignedUrlProviderTest, AccountlessAdcRequiresExplicitImpersonation) { + GcpTokenProvider tokens(std::make_shared("")); + GcpCredentialConfig credential; + credential.provider_type = GcpCredentialProviderType::Default; + std::string signer; + EXPECT_FALSE(resolve_gcs_signer_email(credential, tokens, &signer).ok()); + + credential.impersonation_service_account = "target@my-project.iam.gserviceaccount.com"; + ASSERT_TRUE(resolve_gcs_signer_email(credential, tokens, &signer).ok()); + EXPECT_EQ(signer, credential.impersonation_service_account); +} + +TEST_F(GcsSignedUrlProviderTest, ImpersonationOverridesSourceAccount) { + GcpTokenProvider tokens(std::make_shared( + "source@my-project.iam.gserviceaccount.com")); + GcpCredentialConfig credential; + credential.impersonation_service_account = "target@my-project.iam.gserviceaccount.com"; + std::string signer; + ASSERT_TRUE(resolve_gcs_signer_email(credential, tokens, &signer).ok()); + EXPECT_EQ(signer, credential.impersonation_service_account); +} + +} // namespace +} // namespace doris::io diff --git a/be/test/io/fs/s3_obj_storage_client_test.cpp b/be/test/io/fs/s3_obj_storage_client_test.cpp index 4bafbc4dde71f7..f43fd498de7a7a 100644 --- a/be/test/io/fs/s3_obj_storage_client_test.cpp +++ b/be/test/io/fs/s3_obj_storage_client_test.cpp @@ -25,6 +25,17 @@ namespace doris { +TEST(S3ObjStorageSignedUrlTest, DelegatesProviderSpecificSigning) { + S3ObjStorageClient client(nullptr, {}, [](const ObjStoragePath& path, int64_t expiration) { + EXPECT_EQ(path.bucket, "bucket"); + EXPECT_EQ(path.key, "nested/key"); + EXPECT_EQ(expiration, 300); + return std::string {"https://storage.googleapis.com/bucket/nested/key?signed"}; + }); + EXPECT_EQ(client.generate_presigned_url({.bucket = "bucket", .key = "nested/key"}, 300), + "https://storage.googleapis.com/bucket/nested/key?signed"); +} + class S3ObjStorageClientTest : public testing::Test { protected: static std::shared_ptr obj_storage_client; diff --git a/be/test/io/s3_client_factory_test.cpp b/be/test/io/s3_client_factory_test.cpp index 835b32bdaef7e5..936564816dab5f 100644 --- a/be/test/io/s3_client_factory_test.cpp +++ b/be/test/io/s3_client_factory_test.cpp @@ -18,18 +18,47 @@ #include #include #include +#include #include +#include +#include #include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include +#include +#include +#include +#include +#include +#include #include #include +#include #include #include #include #include #include +#include #include +#include #include #include @@ -38,8 +67,13 @@ #include "cpp/aws_common.h" #include "cpp/custom_aws_credentials_provider_chain.h" #include "cpp/obj-client/auth/aws_credential_factory.h" +#include "cpp/obj-client/auth/gcp/gcp_auth.h" +#include "cpp/obj-client/auth/gcp/gcp_s3_client.h" +#include "cpp/obj-client/auth/gcp/gcs_signed_url.h" +#include "cpp/obj-client/rate_limited_obj_storage_client.h" #include "cpp/obj-client/s3_obj_storage_client.h" #include "cpp/sync_point.h" +#include "io/fs/err_utils.h" #include "io/fs/s3_file_system.h" #include "testutil/container_credentials_endpoint.h" #include "util/s3_rate_limiter_manager.h" @@ -126,6 +160,44 @@ class RateLimiterConfigGuard { size_t _bytes_limit; }; +S3ClientConf make_gcp_native_conf(std::string impersonation_service_account = {}) { + auto conf = make_factory_conf("storage.googleapis.com", true); + conf.provider = io::ObjStorageProvider::GCP; + conf.region = "us-east1"; + conf.cred_provider_type = CredProviderType::Default; + conf.credential = GcpCredentialConfig { + .provider_type = GcpCredentialProviderType::Default, + .impersonation_service_account = std::move(impersonation_service_account), + }; + return conf; +} + +class FixedTokenGcpS3Client final : public GcpS3Client { +public: + explicit FixedTokenGcpS3Client(std::string endpoint = "https://storage.googleapis.com") + : GcpS3Client(GcpCredentialConfig {}, "", make_config(endpoint), + Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, true) {} + + using GcpS3Client::BuildHttpRequest; + using GcpS3Client::authorize_request; + + std::optional token = "test-gcp-token"; + mutable int token_requests = 0; + +protected: + std::optional fetch_token() const override { + ++token_requests; + return token; + } + +private: + static Aws::Client::ClientConfiguration make_config(const std::string& endpoint) { + Aws::Client::ClientConfiguration config; + config.endpointOverride = endpoint; + return config; + } +}; + class SyncPointProcessingGuard { public: SyncPointProcessingGuard() : _was_enabled(SyncPoint::get_instance()->get_enable()) { @@ -282,6 +354,247 @@ TEST_F(S3ClientFactoryTest, RateLimitResponseDistinguishesBytesFromProviderThrot EXPECT_NE(response.status.msg.find("s3 get request exceeds bytes limit"), std::string::npos); } +TEST_F(S3ClientFactoryTest, ObjClientHolderResetRefreshesNativeCredential) { + auto initial_conf = make_gcp_native_conf(); + auto rotated_conf = make_gcp_native_conf("rotated@my-project.iam.gserviceaccount.com"); + std::vector created_confs; + auto client = std::make_shared(std::shared_ptr {}); + S3ClientFactory::instance().set_client_creator_for_test( + [&](const S3ClientConf& conf) -> std::shared_ptr { + created_confs.push_back(conf); + return client; + }); + + io::ObjClientHolder holder(initial_conf); + ASSERT_TRUE(holder.init().ok()); + ASSERT_TRUE(holder.reset(rotated_conf).ok()); + + ASSERT_EQ(created_confs.size(), 2); + EXPECT_EQ(created_confs.back(), rotated_conf); + EXPECT_EQ(holder.s3_client_conf(), rotated_conf); +} + +TEST_F(S3ClientFactoryTest, ObjClientHolderResetSwitchesS3AndGcpProviders) { + auto native_conf = make_gcp_native_conf(); + auto s3_conf = native_conf; + s3_conf.provider = io::ObjStorageProvider::AWS; + s3_conf.credential = std::monostate {}; + + std::vector created_confs; + auto client = std::make_shared(std::shared_ptr {}); + S3ClientFactory::instance().set_client_creator_for_test( + [&](const S3ClientConf& conf) -> std::shared_ptr { + created_confs.push_back(conf); + return client; + }); + + io::ObjClientHolder holder(s3_conf); + ASSERT_TRUE(holder.init().ok()); + // Existing policy resources take this reset path after receiving the new FE version. + ASSERT_TRUE(holder.reset(native_conf).ok()); + EXPECT_EQ(holder.s3_client_conf(), native_conf); + ASSERT_TRUE(holder.reset(s3_conf).ok()); + EXPECT_EQ(holder.s3_client_conf(), s3_conf); + ASSERT_EQ(created_confs.size(), 3); + EXPECT_EQ(created_confs[1], native_conf); + EXPECT_EQ(created_confs[2], s3_conf); +} + +TEST_F(S3ClientFactoryTest, ObjClientHolderResetSwitchesNativeAndStaticCredentials) { + auto native_conf = make_gcp_native_conf(); + auto static_conf = native_conf; + static_conf.credential = std::monostate {}; + static_conf.ak = "access-key"; + static_conf.sk = "secret-key"; + + std::vector created_confs; + auto client = std::make_shared(std::shared_ptr {}); + S3ClientFactory::instance().set_client_creator_for_test( + [&](const S3ClientConf& conf) -> std::shared_ptr { + created_confs.push_back(conf); + return client; + }); + + io::ObjClientHolder holder(native_conf); + ASSERT_TRUE(holder.init().ok()); + ASSERT_TRUE(holder.reset(static_conf).ok()); + ASSERT_TRUE(std::holds_alternative(holder.s3_client_conf().credential)); + EXPECT_EQ(holder.s3_client_conf().ak, "access-key"); + + ASSERT_TRUE(holder.reset(native_conf).ok()); + ASSERT_EQ(created_confs.size(), 3); + EXPECT_EQ(created_confs[1], static_conf); + EXPECT_EQ(created_confs[2], native_conf); + EXPECT_EQ(holder.s3_client_conf(), native_conf); +} + +TEST_F(S3ClientFactoryTest, CredentialTypeNamesAreExplicit) { + CredentialConfig none; + CredentialConfig gcp = GcpCredentialConfig {}; + + EXPECT_EQ(credential_type_name(none), "none"); + EXPECT_EQ(credential_type_name(gcp), "gcp"); +} + +TEST_F(S3ClientFactoryTest, GcpBearerHeaderSurvivesAnonymousAwsSigning) { + FixedTokenGcpS3Client client; + Aws::S3::Model::GetObjectRequest request; + request.SetBucket("bucket"); + request.SetKey("key"); + auto http_request = Aws::Http::CreateHttpRequest( + Aws::Http::URI("https://storage.googleapis.com/bucket/key"), + Aws::Http::HttpMethod::HTTP_GET, + Aws::Utils::Stream::DefaultResponseStreamFactoryMethod); + + ASSERT_FALSE(client.authorize_request(request, request.GetBucket()).has_value()); + client.BuildHttpRequest(request, http_request); + EXPECT_EQ(http_request->GetHeaderValue("Authorization"), "Bearer test-gcp-token"); + + // AWSAuthV4Signer reads this before it checks for anonymous credentials. + http_request->SetServiceSpecificParameters( + std::make_shared()); + Aws::Client::AWSAuthV4Signer signer( + std::make_shared(), "s3", "us-east-1"); + EXPECT_TRUE(signer.SignRequest(*http_request)); + EXPECT_EQ(http_request->GetHeaderValue("Authorization"), "Bearer test-gcp-token"); +} + +TEST_F(S3ClientFactoryTest, GcpRejectsUntrustedEndpointsBeforeFetchingToken) { + for (const auto* endpoint : + {"https://attacker.example", "http://storage.googleapis.com", + "https://storage.googleapis.com.attacker.example", + "https://storage.googleapis.com@attacker.example", "https://storage.googleapis.com:8443", + "https://bucket.storage.googleapis.com", + "https://bucket.us-central1-storage.googleapis.com", + "https://bucket.storage.us-central1.rep.googleapis.com", ""}) { + FixedTokenGcpS3Client client(endpoint); + Aws::S3::Model::GetObjectRequest request; + request.SetBucket("bucket"); + request.SetKey("key"); + auto outcome = client.GetObject(request); + ASSERT_FALSE(outcome.IsSuccess()) << endpoint; + EXPECT_EQ(outcome.GetError().GetExceptionName(), "GcpEndpointError"); + EXPECT_EQ(client.token_requests, 0); + } + FixedTokenGcpS3Client client; + Aws::S3::Model::GetObjectRequest request; + request.SetBucket("arn:aws:s3:us-east-1:123456789012:accesspoint/example"); + EXPECT_FALSE(client.GetObject(request).IsSuccess()); + EXPECT_EQ(client.token_requests, 0); +} + +TEST_F(S3ClientFactoryTest, GcpAcceptsGoogleStorageEndpoints) { + for (const auto* endpoint : + {"https://storage.googleapis.com", "https://storage.googleapis.com:443/", + "https://us-central1-storage.googleapis.com", + "https://storage.us-central1.rep.googleapis.com"}) { + EXPECT_TRUE(is_valid_gcp_storage_endpoint(endpoint)) << endpoint; + } +} + +TEST_F(S3ClientFactoryTest, GcpErrorLogPresigningUsesUploadedKey) { + for (const std::string prefix : {"", "instance-prefix"}) { + S3Conf conf; + conf.bucket = "test-bucket"; + conf.prefix = prefix; + io::S3FileSystem fs(std::move(conf), "gcp-error-log-test"); + const std::string expected_key = + prefix.empty() ? "error_log/id" : "instance-prefix/error_log/id"; + io::Path upload_path; + ASSERT_TRUE(fs.absolute_path("error_log/id", upload_path).ok()); + S3URI upload_uri(upload_path.native()); + ASSERT_TRUE(upload_uri.parse().ok()); + ASSERT_EQ(upload_uri.get_key(), expected_key); + + fs.client_holder()->_client = std::make_shared( + nullptr, ObjStorageEndpointInfo {}, + [&](const ObjStoragePath& path, int64_t expiration_secs) { + EXPECT_EQ(path.key, upload_uri.get_key()); + GcsV4SignedUrlOptions options { + .endpoint = "https://storage.googleapis.com", + .bucket = path.bucket, + .key = path.key, + .signer_email = "signer@test.iam.gserviceaccount.com", + .expiration_secs = expiration_secs, + }; + auto result = build_gcs_v4_signed_url( + options, std::chrono::system_clock::time_point {}, + [](std::string_view) { + return GcsSignBlobResult {.signature = "test"}; + }); + EXPECT_TRUE(result.ok()) << result.error; + return result.signed_url; + }); + EXPECT_TRUE(fs.generate_presigned_url("error_log/id", 60, false) + .starts_with("https://storage.googleapis.com/test-bucket/" + + expected_key + "?")); + } +} + +TEST_F(S3ClientFactoryTest, GcpTokenFailureRejectsEveryObjectStorageOperation) { + FixedTokenGcpS3Client client; + for (const auto& token : {std::optional {}, std::optional {""}}) { + client.token = token; + auto check = [](const auto& outcome) { + ASSERT_FALSE(outcome.IsSuccess()); + EXPECT_EQ(outcome.GetError().GetExceptionName(), "GcpAuthenticationError"); + EXPECT_EQ(outcome.GetError().GetResponseCode(), + Aws::Http::HttpResponseCode::UNAUTHORIZED); + EXPECT_FALSE(outcome.GetError().ShouldRetry()); + EXPECT_EQ(doris::s3fs_error(outcome.GetError(), "GCS request failed").code, + ObjStorageStatus::PERMISSION_DENIED); + }; + check(client.AbortMultipartUpload(Aws::S3::Model::AbortMultipartUploadRequest {})); + check(client.CompleteMultipartUpload(Aws::S3::Model::CompleteMultipartUploadRequest {})); + check(client.CreateMultipartUpload(Aws::S3::Model::CreateMultipartUploadRequest {})); + check(client.DeleteObject(Aws::S3::Model::DeleteObjectRequest {})); + check(client.DeleteObjects(Aws::S3::Model::DeleteObjectsRequest {})); + check(client.GetBucketLifecycleConfiguration( + Aws::S3::Model::GetBucketLifecycleConfigurationRequest {})); + check(client.GetBucketVersioning(Aws::S3::Model::GetBucketVersioningRequest {})); + check(client.GetObject(Aws::S3::Model::GetObjectRequest {})); + check(client.HeadObject(Aws::S3::Model::HeadObjectRequest {})); + check(client.ListObjectsV2(Aws::S3::Model::ListObjectsV2Request {})); + check(client.PutObject(Aws::S3::Model::PutObjectRequest {})); + check(client.UploadPart(Aws::S3::Model::UploadPartRequest {})); + } +} + +TEST_F(S3ClientFactoryTest, UnauthorizedObjectStoreResponseIsPermissionDenied) { + Aws::S3::S3Error error; + error.SetResponseCode(Aws::Http::HttpResponseCode::UNAUTHORIZED); + error.SetExceptionName("InvalidAuthenticationCredentials"); + error.SetMessage("missing or invalid bearer token"); + + auto status = doris::s3fs_error(error, "GCS request failed"); + EXPECT_EQ(status.code, ObjStorageStatus::PERMISSION_DENIED); +} + +TEST_F(S3ClientFactoryTest, GcpServiceAccountEmailValidation) { + const std::vector valid_emails = { + "target@my-project.iam.gserviceaccount.com", + "123456789-compute@developer.gserviceaccount.com", + "my-project@appspot.gserviceaccount.com", + }; + for (auto email : valid_emails) { + EXPECT_TRUE(is_valid_gcp_service_account_email(email)) << email; + } + + const std::vector invalid_emails = { + "", + "@my-project.iam.gserviceaccount.com", + "target@.iam.gserviceaccount.com", + "target@@my-project.iam.gserviceaccount.com", + "target@evil.developer.gserviceaccount.com", + "target@evil.appspot.gserviceaccount.com", + "target@my-project.iam.gserviceaccount.com.evil.example", + "target account@my-project.iam.gserviceaccount.com", + }; + for (auto email : invalid_emails) { + EXPECT_FALSE(is_valid_gcp_service_account_email(email)) << email; + } +} + TEST_F(S3ClientFactoryTest, AwsCredentialsProvider) { S3ClientFactory& factory = S3ClientFactory::instance(); S3ClientConf anonymous_conf; @@ -491,6 +804,156 @@ TEST_F(S3ClientFactoryTest, ConvertPropertiesToS3ConfProviderTypeMatrix) { } } +TEST_F(S3ClientFactoryTest, ConvertPropertiesToS3ConfGcpCredentialModes) { + S3URI s3_uri("s3://test-bucket/test-prefix"); + ASSERT_TRUE(s3_uri.parse().ok()); + + std::map base_properties { + {"AWS_ENDPOINT", "storage.googleapis.com"}, + {"AWS_REGION", "us-east1"}, + {"provider", "GCP"}, + }; + + struct TestCase { + const char* provider_type; + const char* impersonation_service_account; + GcpCredentialProviderType expected_provider_type; + }; + + // Both ADC and Compute Engine metadata credentials may be used directly or + // as the source credential for service-account impersonation. + std::vector cases = { + {"DEFAULT", "", GcpCredentialProviderType::Default}, + {"default", "", GcpCredentialProviderType::Default}, + {"DEFAULT", "target@my-project.iam.gserviceaccount.com", + GcpCredentialProviderType::Default}, + {"COMPUTE_ENGINE", "", GcpCredentialProviderType::ComputeEngine}, + {" compute_engine ", "", GcpCredentialProviderType::ComputeEngine}, + {"COMPUTE_ENGINE", "target@my-project.iam.gserviceaccount.com", + GcpCredentialProviderType::ComputeEngine}, + }; + + for (const auto& test_case : cases) { + auto properties = base_properties; + properties["gs.credential_provider_type"] = test_case.provider_type; + if (test_case.impersonation_service_account[0] != '\0') { + properties["gs.impersonation_service_account"] = + test_case.impersonation_service_account; + } + + S3Conf s3_conf; + ASSERT_TRUE( + S3ClientFactory::convert_properties_to_s3_conf(properties, s3_uri, &s3_conf).ok()) + << "provider_type=" << test_case.provider_type; + ASSERT_EQ(s3_conf.client_conf.provider, io::ObjStorageProvider::GCP); + ASSERT_TRUE(std::holds_alternative(s3_conf.client_conf.credential)); + const auto& credential = std::get(s3_conf.client_conf.credential); + EXPECT_EQ(credential.provider_type, test_case.expected_provider_type); + EXPECT_EQ(credential.impersonation_service_account, + test_case.impersonation_service_account); + } +} + +TEST_F(S3ClientFactoryTest, ConvertPropertiesToS3ConfGcpDefaultImpersonation) { + S3URI s3_uri("s3://test-bucket/test-prefix"); + ASSERT_TRUE(s3_uri.parse().ok()); + std::map properties { + {"AWS_ENDPOINT", "storage.googleapis.com"}, + {"AWS_REGION", "us-east1"}, + {"provider", "GCP"}, + {"gs.impersonation_service_account", "target@my-project.iam.gserviceaccount.com"}, + }; + S3Conf s3_conf; + ASSERT_TRUE(S3ClientFactory::convert_properties_to_s3_conf(properties, s3_uri, &s3_conf).ok()); + const auto& credential = std::get(s3_conf.client_conf.credential); + EXPECT_EQ(credential.provider_type, GcpCredentialProviderType::Default); + EXPECT_EQ(credential.impersonation_service_account, + "target@my-project.iam.gserviceaccount.com"); + + properties["gs.impersonation_service_account"] = ""; + ASSERT_TRUE(S3ClientFactory::convert_properties_to_s3_conf(properties, s3_uri, &s3_conf).ok()); + const auto& direct_credential = std::get(s3_conf.client_conf.credential); + EXPECT_EQ(direct_credential.provider_type, GcpCredentialProviderType::Default); + EXPECT_TRUE(direct_credential.impersonation_service_account.empty()); +} + +TEST_F(S3ClientFactoryTest, ConvertPropertiesToS3ConfRejectsConflictingGcpCredentials) { + S3URI s3_uri("s3://test-bucket/test-prefix"); + ASSERT_TRUE(s3_uri.parse().ok()); + + std::map base_properties { + {"AWS_ENDPOINT", "storage.googleapis.com"}, + {"AWS_REGION", "us-east1"}, + {"provider", "GCP"}, + {"gs.credential_provider_type", "DEFAULT"}, + }; + + const std::vector> conflicts = { + {"AWS_ACCESS_KEY", "ak"}, + {"AWS_SECRET_KEY", "sk"}, + {"AWS_TOKEN", "token"}, + {"AWS_ROLE_ARN", "arn:aws:iam::123456789012:role/test-role"}, + {"AWS_EXTERNAL_ID", "external-id"}, + {"AWS_CREDENTIALS_PROVIDER_TYPE", "INSTANCE_PROFILE"}, + }; + for (const auto& [key, value] : conflicts) { + auto properties = base_properties; + properties[key] = value; + S3Conf s3_conf; + auto status = S3ClientFactory::convert_properties_to_s3_conf(properties, s3_uri, &s3_conf); + ASSERT_FALSE(status.ok()) << key; + EXPECT_NE(status.to_string().find("cannot be combined"), std::string::npos) << key; + } +} + +TEST_F(S3ClientFactoryTest, ConvertPropertiesToS3ConfRejectsInvalidGcpCredentialModes) { + S3URI s3_uri("s3://test-bucket/test-prefix"); + ASSERT_TRUE(s3_uri.parse().ok()); + + std::map base_properties { + {"AWS_ENDPOINT", "storage.googleapis.com"}, + {"AWS_REGION", "us-east1"}, + {"provider", "GCP"}, + }; + for (const auto& provider_type : {"ADC", "", "unknown"}) { + auto properties = base_properties; + properties["gs.credential_provider_type"] = provider_type; + S3Conf s3_conf; + auto status = S3ClientFactory::convert_properties_to_s3_conf(properties, s3_uri, &s3_conf); + ASSERT_FALSE(status.ok()) << provider_type; + EXPECT_NE(status.to_string().find("gs.credential_provider_type"), std::string::npos) + << provider_type; + EXPECT_NE(status.to_string().find("DEFAULT"), std::string::npos) << provider_type; + EXPECT_NE(status.to_string().find("COMPUTE_ENGINE"), std::string::npos) << provider_type; + } + + for (const auto& account : {" ", "invalid-account"}) { + auto properties = base_properties; + properties["gs.impersonation_service_account"] = account; + S3Conf s3_conf; + ASSERT_FALSE( + S3ClientFactory::convert_properties_to_s3_conf(properties, s3_uri, &s3_conf).ok()); + } +} + +TEST_F(S3ClientFactoryTest, ConvertPropertiesToS3ConfRejectsGcpCredentialWithAwsProvider) { + S3URI s3_uri("s3://test-bucket/test-prefix"); + ASSERT_TRUE(s3_uri.parse().ok()); + + for (const auto& provider : {"S3", "AWS"}) { + std::map properties { + {"AWS_ENDPOINT", "storage.googleapis.com"}, + {"AWS_REGION", "us-east1"}, + {"provider", provider}, + {"gs.credential_provider_type", "DEFAULT"}, + }; + S3Conf s3_conf; + auto status = S3ClientFactory::convert_properties_to_s3_conf(properties, s3_uri, &s3_conf); + ASSERT_FALSE(status.ok()) << provider; + EXPECT_NE(status.to_string().find("provider=GCP"), std::string::npos) << provider; + } +} + TEST_F(S3ClientFactoryTest, ConvertPropertiesToS3ConfCredentialValidation) { S3URI s3_uri("s3://test-bucket/test-prefix"); ASSERT_TRUE(s3_uri.parse().ok()); diff --git a/be/test/service/http/http_client_test.cpp b/be/test/service/http/http_client_test.cpp index f23b010b7c092e..6cc320e2d4b9e2 100644 --- a/be/test/service/http/http_client_test.cpp +++ b/be/test/service/http/http_client_test.cpp @@ -530,6 +530,14 @@ class HttpClientTestSimplePostHandler : public HttpHandler { } }; +class ExternalAuthHandler : public HttpHandler { +public: + void handle(HttpRequest* req) override { + HttpChannel::send_reply(req, req->header(HttpHeaders::AUTH_TOKEN) + "|" + + req->header(HttpHeaders::AUTHORIZATION)); + } +}; + class HttpNotFoundHandler : public HttpHandler { public: void handle(HttpRequest* req) override { @@ -567,6 +575,7 @@ static std::string address = ""; static HttpClientTestSimpleGetHandler s_simple_get_handler; static HttpClientTestSimplePostHandler s_simple_post_handler; static HttpNotFoundHandler s_not_found_handler; +static ExternalAuthHandler s_external_auth_handler; static HttpDownloadFileHandler s_download_file_handler; static HttpBatchDownloadFileHandler s_batch_download_file_handler; @@ -615,6 +624,7 @@ class HttpClientTest : public testing::Test { s_server->register_handler(HEAD, "/simple_get", &s_simple_get_handler); s_server->register_handler(POST, "/simple_post", &s_simple_post_handler); s_server->register_handler(GET, "/not_found", &s_not_found_handler); + s_server->register_handler(GET, "/external_auth", &s_external_auth_handler); s_server->register_handler(HEAD, "/download_file", &s_download_file_handler); s_server->register_handler(HEAD, "/api/_tablet/_batch_download", &s_batch_download_file_handler); @@ -656,6 +666,18 @@ TEST_F(HttpClientTest, get_normal) { EXPECT_EQ(5, len); } +TEST_F(HttpClientTest, external_request_omits_cluster_token) { + HttpClient client; + ASSERT_TRUE(client.init(hostname + "/external_auth").ok()); + client.set_auth_token("previous-cluster-token"); + ASSERT_TRUE( + client.init(hostname + "/external_auth", false, HttpClient::AuthTokenMode::NONE).ok()); + client.set_authorization("Bearer google-token"); + std::string response; + ASSERT_TRUE(client.execute(&response).ok()); + EXPECT_EQ(response, "|Bearer google-token"); +} + TEST_F(HttpClientTest, download) { HttpClient client; auto st = client.init(hostname + "/simple_get"); diff --git a/cloud/CMakeLists.txt b/cloud/CMakeLists.txt index e0b44d9cdff1d2..32983430c60e05 100644 --- a/cloud/CMakeLists.txt +++ b/cloud/CMakeLists.txt @@ -139,6 +139,9 @@ set(GPERFTOOLS_HOME "${THIRDPARTY_DIR}/gperftools") # Set all libraries include(thirdparty) +set(absl_DIR ${THIRDPARTY_DIR}/lib/cmake/absl) +find_package(absl CONFIG REQUIRED PATHS ${absl_DIR} NO_DEFAULT_PATH) + find_program(THRIFT_COMPILER thrift ${CMAKE_SOURCE_DIR}/bin) # Check if functions are supported in this platform. All flags will generated @@ -321,6 +324,19 @@ else() add_definitions(-DUSE_LIBHDFS3) endif() +# google-cloud-cpp is linked as static archives in thirdparty.cmake, so its +# public Abseil dependencies must be present in the final link group as well. +set(COMMON_THIRDPARTY + ${COMMON_THIRDPARTY} + absl::base + absl::memory + absl::optional + absl::span + absl::str_format + absl::time + absl::variant +) + set(DORIS_DEPENDENCIES ${DORIS_DEPENDENCIES} ${WL_START_GROUP} diff --git a/cloud/cmake/thirdparty.cmake b/cloud/cmake/thirdparty.cmake index a383b263b9c63d..d945ceaa45eb2f 100644 --- a/cloud/cmake/thirdparty.cmake +++ b/cloud/cmake/thirdparty.cmake @@ -72,6 +72,10 @@ add_thirdparty(brpc LIB64) add_thirdparty(rocksdb) # For local storage mocking add_thirdparty(libz LIBNAME "lib/libz.a") # Required by google::protobuf add_thirdparty(curl) +add_thirdparty(google_cloud_cpp_oauth2) +add_thirdparty(google_cloud_cpp_rest_internal) +add_thirdparty(google_cloud_cpp_common) +add_thirdparty(crc32c) add_thirdparty(zstd LIB64) add_thirdparty(fmt) # begin aws libs diff --git a/cloud/src/common/CMakeLists.txt b/cloud/src/common/CMakeLists.txt index 9f29e1a183b0a3..e356861dd728d9 100644 --- a/cloud/src/common/CMakeLists.txt +++ b/cloud/src/common/CMakeLists.txt @@ -32,6 +32,7 @@ set(COMMON_FILES version_metrics.cpp kms.cpp network_util.cpp + auth/obj_credential.cpp ../server/oss/cloud_server_starter_factory.cpp ) diff --git a/cloud/src/common/auth/obj_credential.cpp b/cloud/src/common/auth/obj_credential.cpp new file mode 100644 index 00000000000000..e2db4815bc56ed --- /dev/null +++ b/cloud/src/common/auth/obj_credential.cpp @@ -0,0 +1,244 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "common/auth/obj_credential.h" + +#include + +#include + +#include "cpp/obj-client/auth/obj_credential.h" + +namespace doris { +namespace { + +GcpCredentialConfig to_gcp_credential_config(const cloud::GcpCredentialPB& credential) { + GcpCredentialConfig config; + switch (credential.credential_provider_type()) { + case cloud::GcpCredentialPB::DEFAULT: + config.provider_type = GcpCredentialProviderType::Default; + break; + case cloud::GcpCredentialPB::COMPUTE_ENGINE: + config.provider_type = GcpCredentialProviderType::ComputeEngine; + break; + } + config.impersonation_service_account = credential.impersonation_service_account(); + return config; +} + +} // namespace + +void convert_obj_credential(const cloud::ObjectStoreInfoPB& source, CredentialConfig* credential) { + if (!source.has_credential() || !source.credential().has_gcp_credential()) { + *credential = std::monostate {}; + return; + } + *credential = to_gcp_credential_config(source.credential().gcp_credential()); +} + +} // namespace doris + +namespace doris::cloud { +namespace { + +// Extension checklist for a new passwordless provider: +// 1. add a provider-specific validate__obj_credential(); +// 2. dispatch it from validate_obj_credential(); and +// 3. reject requests that set more than one provider credential. +// Generic create/alter normalization does not need provider-specific branches. +ObjCredentialProvider to_obj_credential_provider(const ObjectStoreInfoPB& obj) { + if (!obj.has_provider()) { + return ObjCredentialProvider::Unknown; + } + switch (obj.provider()) { + case ObjectStoreInfoPB::S3: + return ObjCredentialProvider::Aws; + case ObjectStoreInfoPB::AZURE: + return ObjCredentialProvider::Azure; + case ObjectStoreInfoPB::BOS: + return ObjCredentialProvider::Bos; + case ObjectStoreInfoPB::COS: + return ObjCredentialProvider::Cos; + case ObjectStoreInfoPB::GCP: + return ObjCredentialProvider::Gcp; + case ObjectStoreInfoPB::OBS: + return ObjCredentialProvider::Obs; + case ObjectStoreInfoPB::OSS: + return ObjCredentialProvider::Oss; + case ObjectStoreInfoPB::TOS: + return ObjCredentialProvider::Tos; + case ObjectStoreInfoPB::UNKONWN: + return ObjCredentialProvider::Unknown; + } + return ObjCredentialProvider::Unknown; +} + +ObjCredentialValidationContext credential_validation_context(const ObjectStoreInfoPB& obj) { + return ObjCredentialValidationContext { + .provider = to_obj_credential_provider(obj), + .ak = obj.ak(), + .sk = obj.sk(), + .has_aws_role_arn = !obj.role_arn().empty(), + .has_aws_external_id = !obj.external_id().empty(), + .has_aws_credential_provider = obj.has_cred_provider_type(), + .has_encrypted_access_keys = obj.has_encryption_info(), + }; +} + +// A provider-native credential supersedes both common static credentials and +// provider-specific authentication fields outside the credential envelope. +void clear_legacy_credentials(ObjectStoreInfoPB* obj) { + obj->clear_ak(); + obj->clear_sk(); + obj->clear_encryption_info(); + obj->clear_cred_provider_type(); + obj->clear_role_arn(); + obj->clear_external_id(); +} + +std::optional validate_required_storage_fields(const ObjectStoreInfoPB& obj, + std::string_view provider) { + if (obj.bucket().empty() || obj.endpoint().empty() || obj.region().empty()) { + return std::string(provider) + " storage conf requires bucket, endpoint and region"; + } + return std::nullopt; +} + +std::optional validate_gcp_obj_credential(const ObjectStoreInfoPB& obj) { + const auto& credential = obj.credential().gcp_credential(); + if (!credential.has_credential_provider_type()) { + return "GCP credential provider type is required"; + } + if (auto error = validate_required_storage_fields(obj, "GCP"); error.has_value()) { + return error; + } + + CredentialConfig config; + convert_obj_credential(obj, &config); + return validate_obj_credential_config(config, credential_validation_context(obj)); +} + +} // namespace + +bool has_obj_credential(const ObjectStoreInfoPB& obj) { + return obj.has_credential(); +} + +std::optional validate_obj_credential(const ObjectStoreInfoPB& obj) { + if (!obj.has_credential()) { + return "credential is not set"; + } + + // Add one provider-specific branch for every new field in + // ObjectStoreCredentialPB. Reject multiple fields before dispatch when a + // second provider is introduced. The common visitor validates only the + // normalized credential itself. + if (obj.credential().has_gcp_credential()) { + return validate_gcp_obj_credential(obj); + } + return "unsupported object storage credential"; +} + +std::optional validate_obj_authentication(const ObjectStoreInfoPB& obj) { + if (has_obj_credential(obj)) { + return validate_obj_credential(obj); + } + if (obj.has_role_arn() && obj.role_arn().empty()) { + return "AWS role ARN cannot be empty"; + } + + const auto context = credential_validation_context(obj); + if (context.has_static_credentials() && context.has_aws_authentication()) { + return "access keys cannot be combined with AWS role or credential provider"; + } + if (!context.has_aws_authentication()) { + return std::nullopt; + } + if (!obj.has_provider() || obj.provider() != ObjectStoreInfoPB::S3) { + return "AWS role and credential provider require provider=S3"; + } + if (obj.has_external_id() && obj.external_id().size() > 0 && obj.role_arn().empty()) { + return "AWS external ID requires a role ARN"; + } + return std::nullopt; +} + +std::optional validate_and_normalize_obj_credential(ObjectStoreInfoPB* obj) { + if (auto error = validate_obj_credential(*obj); error.has_value()) { + return error; + } + clear_legacy_credentials(obj); + return std::nullopt; +} + +std::optional apply_obj_credential(const ObjectStoreInfoPB& update, + ObjectStoreInfoPB* target) { + if (!has_obj_credential(update)) { + return "credential is not set"; + } + if (credential_validation_context(update).has_conflicting_credentials()) { + return "credentials cannot be combined with other credentials"; + } + + // Merge into a copy first so a rejected alter never partially mutates target. + ObjectStoreInfoPB candidate = *target; + copy_obj_credential(update, &candidate); + if (update.credential().has_gcp_credential()) { + const auto& patch = update.credential().gcp_credential(); + if (!patch.has_credential_provider_type() && !patch.has_impersonation_service_account()) { + return "GCP credential update must specify a provider type or service account"; + } + if (!target->credential().has_gcp_credential() && !patch.has_credential_provider_type() && + patch.impersonation_service_account().empty()) { + return "clearing impersonation requires an existing native GCP credential; " + "specify a provider type to replace HMAC credentials"; + } + auto* credential = candidate.mutable_credential()->mutable_gcp_credential(); + if (target->credential().has_gcp_credential()) { + credential->CopyFrom(target->credential().gcp_credential()); + credential->MergeFrom(patch); + } + // Only a new native credential needs a default; an existing source is + // preserved when the patch changes just the impersonation target. + if (!credential->has_credential_provider_type()) { + credential->set_credential_provider_type(GcpCredentialPB::DEFAULT); + } + if (credential->impersonation_service_account().empty()) { + credential->clear_impersonation_service_account(); + } + } + clear_legacy_credentials(&candidate); + if (auto error = validate_obj_credential(candidate); error.has_value()) { + return error; + } + + clear_legacy_credentials(target); + copy_obj_credential(candidate, target); + return std::nullopt; +} + +void copy_obj_credential(const ObjectStoreInfoPB& source, ObjectStoreInfoPB* target) { + if (&source == target) { + return; + } + target->clear_credential(); + if (source.has_credential()) { + target->mutable_credential()->CopyFrom(source.credential()); + } +} + +} // namespace doris::cloud diff --git a/cloud/src/common/auth/obj_credential.h b/cloud/src/common/auth/obj_credential.h new file mode 100644 index 00000000000000..a1ad7714e141a6 --- /dev/null +++ b/cloud/src/common/auth/obj_credential.h @@ -0,0 +1,65 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include + +namespace doris::cloud { + +// This helper keeps provider-specific credential branches out of MetaService. +// Provider authentication implementations belong in common/cpp/obj-client/auth; this +// Cloud layer owns vault-field validation, compatibility cleanup, and protobuf +// persistence mutations. + +class ObjectStoreInfoPB; + +// Whether a provider-native credential envelope is present. +bool has_obj_credential(const ObjectStoreInfoPB& obj); + +// TODO: Move AWS authentication fields into ObjectStoreInfoPB.credential so AWS +// follows the same validation and mutation path. Keep cred_provider_type, +// role_arn, and external_id in their current fields for now to preserve +// compatibility with stored vault metadata and existing clients. AK/SK remain +// the common static-credential representation for S3-compatible providers. + +// Validates the active ObjectStoreInfoPB.credential case without modifying obj. +// Add a provider-specific validation case when extending the credential envelope. +std::optional validate_obj_credential(const ObjectStoreInfoPB& obj); + +// Validates the complete authentication state after create/alter mutations. In +// addition to provider-native credentials, this rejects cross-provider AWS +// authentication and mutually exclusive authentication modes. +std::optional validate_obj_authentication(const ObjectStoreInfoPB& obj); + +// Used by create paths: validates first, then removes common static credentials +// and provider-specific authentication fields that must not coexist with the +// selected provider-native credential. +std::optional validate_and_normalize_obj_credential(ObjectStoreInfoPB* obj); + +// Used by alter paths: merges explicitly present GCP fields into the stored +// credential, with an empty service account clearing impersonation. Validates +// the result against target's storage fields before changing authentication. +std::optional apply_obj_credential(const ObjectStoreInfoPB& update, + ObjectStoreInfoPB* target); + +// Replaces only the credential envelope. An unset source clears target's credential; +// all non-credential fields are preserved. +void copy_obj_credential(const ObjectStoreInfoPB& source, ObjectStoreInfoPB* target); + +} // namespace doris::cloud diff --git a/cloud/src/meta-service/meta_service_resource.cpp b/cloud/src/meta-service/meta_service_resource.cpp index 0284d4246efcc1..792d05fb90c30e 100644 --- a/cloud/src/meta-service/meta_service_resource.cpp +++ b/cloud/src/meta-service/meta_service_resource.cpp @@ -19,6 +19,7 @@ #include #include #include +#include #include #include @@ -31,6 +32,7 @@ #include #include +#include "common/auth/obj_credential.h" #include "common/bvars.h" #include "common/config.h" #include "common/encryption_util.h" @@ -361,7 +363,14 @@ static int alter_instance_obj_store_info_by_id(InstanceInfoPB& instance, } if (role_arn.empty()) { - if (it.ak() == ak && it.sk() == sk) { + if (has_obj_credential(it) && (ak.empty() || sk.empty())) { + code = MetaServiceCode::INVALID_ARGUMENT; + msg = "nonempty access key and secret key are required to replace native " + "credentials"; + return -1; + } + // Resubmitting keys must also repair a previously persisted mixed auth state. + if (it.ak() == ak && it.sk() == sk && !has_obj_credential(it)) { code = MetaServiceCode::OK; msg = "ak/sk not changed"; return 1; @@ -369,6 +378,7 @@ static int alter_instance_obj_store_info_by_id(InstanceInfoPB& instance, it.clear_role_arn(); it.clear_external_id(); it.clear_cred_provider_type(); + it.clear_credential(); it.set_ak(std::string(ak)); it.set_sk(std::string(sk)); @@ -495,7 +505,13 @@ static int update_instance_ak_sk(InstanceInfoPB& instance, const UpdateAkSkReque proto_to_json(*request); return -1; } - if (it.ak() == ak && it.sk() == sk) { + if (has_obj_credential(it) && (ak.empty() || sk.empty())) { + code = MetaServiceCode::INVALID_ARGUMENT; + msg = "nonempty access key and secret key are required to replace native " + "credentials"; + return -1; + } + if (it.ak() == ak && it.sk() == sk && !has_obj_credential(it)) { code = MetaServiceCode::INVALID_ARGUMENT; msg = "ak sk eq original, please check it"; return -1; @@ -504,6 +520,7 @@ static int update_instance_ak_sk(InstanceInfoPB& instance, const UpdateAkSkReque it.set_user_id(user_id); it.set_ak(ak); it.set_sk(sk); + it.clear_credential(); it.mutable_encryption_info()->CopyFrom(encryption_info); update_record << "update obj_info's ak sk without user_id, instance_id: " << instance.instance_id() << " obj_info_id: " << it.id() @@ -514,7 +531,13 @@ static int update_instance_ak_sk(InstanceInfoPB& instance, const UpdateAkSkReque } if (it.user_id() == user_id) { has_found_alter_obj_info = true; - if (it.ak() == ak && it.sk() == sk) { + if (has_obj_credential(it) && (ak.empty() || sk.empty())) { + code = MetaServiceCode::INVALID_ARGUMENT; + msg = "nonempty access key and secret key are required to replace native " + "credentials"; + return -1; + } + if (it.ak() == ak && it.sk() == sk && !has_obj_credential(it)) { code = MetaServiceCode::INVALID_ARGUMENT; msg = "ak sk eq original, please check it"; return -1; @@ -522,6 +545,7 @@ static int update_instance_ak_sk(InstanceInfoPB& instance, const UpdateAkSkReque it.set_mtime(time); it.set_ak(ak); it.set_sk(sk); + it.clear_credential(); it.mutable_encryption_info()->CopyFrom(encryption_info); update_record << "update obj_info's ak sk, instance_id: " << instance.instance_id() << " obj_info_id: " << it.id() << " user_id: " << user_id @@ -765,7 +789,7 @@ static bool has_non_empty_role_arn(const ObjectStoreInfoPB& obj) { } static bool use_credential_provider(const ObjectStoreInfoPB& obj) { - return obj.has_cred_provider_type() || has_non_empty_role_arn(obj); + return obj.has_cred_provider_type() || has_non_empty_role_arn(obj) || has_obj_credential(obj); } static void create_object_info_with_encrypt(const InstanceInfoPB& instance, ObjectStoreInfoPB* obj, @@ -781,14 +805,23 @@ static void create_object_info_with_encrypt(const InstanceInfoPB& instance, Obje std::string external_endpoint = obj->has_external_endpoint() ? obj->external_endpoint() : ""; std::string region = obj->has_region() ? obj->region() : ""; - if (obj->has_role_arn()) { - if (obj->role_arn().empty() || !obj->has_cred_provider_type() || !obj->has_provider() || + if (has_obj_credential(*obj)) { + if (auto error = validate_and_normalize_obj_credential(obj); error.has_value()) { + code = MetaServiceCode::INVALID_ARGUMENT; + msg = *error; + return; + } + } else if (obj->has_role_arn()) { + if (obj->role_arn().empty() || !obj->has_provider() || obj->provider() != ObjectStoreInfoPB::S3 || bucket.empty() || endpoint.empty() || region.empty()) { code = MetaServiceCode::INVALID_ARGUMENT; msg = "s3 conf info err with role_arn or cred provider, please check it"; return; } + if (!obj->has_cred_provider_type()) { + obj->set_cred_provider_type(CredProviderTypePB::INSTANCE_PROFILE); + } } else { // ATTN: prefix may be empty if (plain_ak.empty() || plain_sk.empty() || bucket.empty() || endpoint.empty() || @@ -1071,7 +1104,11 @@ static int alter_s3_storage_vault_by_id(InstanceInfoPB& instance, std::unique_pt msg = ss.str(); return -1; } - auto* storage_vault_names = instance.mutable_storage_vault_names(); + // Apply all mutations to a candidate. The caller can safely abandon the + // transaction on any validation/serialization error without leaking a + // partial rename into the InstanceInfoPB it later persists. + InstanceInfoPB candidate_instance = instance; + auto* storage_vault_names = candidate_instance.mutable_storage_vault_names(); auto* name_ptr = storage_vault_names->Mutable(pos); DCHECK(name_ptr != nullptr); const std::string old_name = *name_ptr; @@ -1081,8 +1118,8 @@ static int alter_s3_storage_vault_by_id(InstanceInfoPB& instance, std::unique_pt target_vault_id, name, old_name); return -1; } - std::string vault_id = instance.resource_ids().Get(pos); - auto vault_key = storage_vault_key({instance.instance_id(), vault_id}); + std::string vault_id = candidate_instance.resource_ids().Get(pos); + auto vault_key = storage_vault_key({candidate_instance.instance_id(), vault_id}); std::string val; auto err = txn->get(vault_key, &val); @@ -1118,7 +1155,7 @@ static int alter_s3_storage_vault_by_id(InstanceInfoPB& instance, std::unique_pt return -1; } - if (vault_exist(instance, vault.alter_name())) { + if (vault_exist(candidate_instance, vault.alter_name())) { code = MetaServiceCode::ALREADY_EXISTED; msg = fmt::format("vault_name={} already existed", vault.alter_name()); return -1; @@ -1126,14 +1163,16 @@ static int alter_s3_storage_vault_by_id(InstanceInfoPB& instance, std::unique_pt new_vault.set_name(vault.alter_name()); *name_ptr = vault.alter_name(); - if (instance.default_storage_vault_id() == vault_id) { - instance.set_default_storage_vault_name(vault.alter_name()); + if (candidate_instance.default_storage_vault_id() == vault_id) { + candidate_instance.set_default_storage_vault_name(vault.alter_name()); } } - if (obj_info.has_role_arn() && (obj_info.has_ak() || obj_info.has_sk())) { + if ((obj_info.has_role_arn() || obj_info.has_cred_provider_type() || + has_obj_credential(obj_info)) && + (obj_info.has_ak() || obj_info.has_sk())) { code = MetaServiceCode::INVALID_ARGUMENT; - msg = "invaild argument, both set ak/sk and role_arn is not allowed"; + msg = "invalid argument, access keys cannot be combined with role or credentials"; LOG(WARNING) << msg; return -1; } @@ -1147,6 +1186,12 @@ static int alter_s3_storage_vault_by_id(InstanceInfoPB& instance, std::unique_pt } if (obj_info.has_ak()) { + if (has_obj_credential(new_vault.obj_info()) && + (obj_info.ak().empty() || obj_info.sk().empty())) { + code = MetaServiceCode::INVALID_ARGUMENT; + msg = "Replacing native credentials requires nonempty accesskey and secretkey"; + return -1; + } EncryptionInfoPB encryption_info = new_vault.obj_info().encryption_info(); AkSkPair new_ak_sk_pair {new_vault.obj_info().ak(), new_vault.obj_info().sk()}; @@ -1162,28 +1207,72 @@ static int alter_s3_storage_vault_by_id(InstanceInfoPB& instance, std::unique_pt new_vault.mutable_obj_info()->clear_role_arn(); new_vault.mutable_obj_info()->clear_external_id(); new_vault.mutable_obj_info()->clear_cred_provider_type(); + new_vault.mutable_obj_info()->clear_credential(); new_vault.mutable_obj_info()->set_ak(new_ak_sk_pair.first); new_vault.mutable_obj_info()->set_sk(new_ak_sk_pair.second); new_vault.mutable_obj_info()->mutable_encryption_info()->CopyFrom(encryption_info); } - if (obj_info.has_role_arn()) { - new_vault.mutable_obj_info()->clear_ak(); - new_vault.mutable_obj_info()->clear_sk(); - new_vault.mutable_obj_info()->clear_encryption_info(); + if (obj_info.has_role_arn() || obj_info.has_cred_provider_type()) { + // Only selecting AWS authentication replaces other credentials. Clearing + // an unused role must preserve static keys and native GCP credentials. + if (!obj_info.role_arn().empty() || obj_info.has_cred_provider_type()) { + new_vault.mutable_obj_info()->clear_ak(); + new_vault.mutable_obj_info()->clear_sk(); + new_vault.mutable_obj_info()->clear_encryption_info(); + new_vault.mutable_obj_info()->clear_credential(); + } - new_vault.mutable_obj_info()->set_role_arn(obj_info.role_arn()); - new_vault.mutable_obj_info()->set_cred_provider_type(get_cred_provider_type(obj_info)); - if (obj_info.has_external_id()) { + if (obj_info.has_role_arn()) { + if (obj_info.role_arn().empty()) { + new_vault.mutable_obj_info()->clear_role_arn(); + // An external ID has no meaning without a role. Clearing the role + // explicitly clears its dependent value as well. + new_vault.mutable_obj_info()->clear_external_id(); + } else { + new_vault.mutable_obj_info()->set_role_arn(obj_info.role_arn()); + if (!new_vault.obj_info().has_cred_provider_type() && + !obj_info.has_cred_provider_type()) { + new_vault.mutable_obj_info()->set_cred_provider_type( + CredProviderTypePB::INSTANCE_PROFILE); + } + } + } + if (obj_info.has_cred_provider_type()) { + new_vault.mutable_obj_info()->set_cred_provider_type(obj_info.cred_provider_type()); + } + } + + // external_id is an independent ALTER field. It may be changed without + // requiring FE to resend role_arn and cred_provider_type. + if (obj_info.has_external_id()) { + if (obj_info.external_id().empty()) { + new_vault.mutable_obj_info()->clear_external_id(); + } else { new_vault.mutable_obj_info()->set_external_id(obj_info.external_id()); } } + if (has_obj_credential(obj_info)) { + if (auto error = apply_obj_credential(obj_info, new_vault.mutable_obj_info()); + error.has_value()) { + code = MetaServiceCode::INVALID_ARGUMENT; + msg = *error; + return -1; + } + } + if (obj_info.has_use_path_style()) { new_vault.mutable_obj_info()->set_use_path_style(obj_info.use_path_style()); } + if (auto error = validate_obj_authentication(new_vault.obj_info()); error.has_value()) { + code = MetaServiceCode::INVALID_ARGUMENT; + msg = *error; + return -1; + } + auto now_time = std::chrono::system_clock::now(); uint64_t time = std::chrono::duration_cast(now_time.time_since_epoch()).count(); @@ -1198,6 +1287,7 @@ static int alter_s3_storage_vault_by_id(InstanceInfoPB& instance, std::unique_pt } txn->put(vault_key, val); + instance.Swap(&candidate_instance); LOG(INFO) << "put vault_id=" << vault_id << ", vault_key=" << hex(vault_key) << ", origin vault=" << encryt_sk(hide_ak(origin_vault_info)) << ", new vault=" << encryt_sk(hide_ak(new_vault_info)); @@ -1258,9 +1348,20 @@ static int extract_object_storage_info(const AlterObjStoreInfoRequest* request, auto& [ak, sk, bucket, prefix, endpoint, external_endpoint, region, use_path_style, role_arn, external_id] = obj_desc; - bool use_credential_provider_for_add_vault = - request->op() == AlterObjStoreInfoRequest::ADD_S3_VAULT && obj.has_cred_provider_type(); - if (!obj.has_role_arn() && !use_credential_provider_for_add_vault) { + bool native_obj_add = + request->op() == AlterObjStoreInfoRequest::ADD_OBJ_INFO && has_obj_credential(obj); + if (native_obj_add) { + if (auto error = validate_obj_credential(obj); error.has_value()) { + code = MetaServiceCode::INVALID_ARGUMENT; + msg = *error; + return -1; + } + } + bool use_credential_provider_for_add = + (request->op() == AlterObjStoreInfoRequest::ADD_S3_VAULT && + use_credential_provider(obj)) || + native_obj_add; + if (!obj.has_role_arn() && !use_credential_provider_for_add) { if (!obj.has_ak() || !obj.has_sk()) { code = MetaServiceCode::INVALID_ARGUMENT; msg = "s3 obj info err " + proto_to_json(*request); @@ -1325,7 +1426,10 @@ static ObjectStoreInfoPB object_info_pb_factory(ObjectStorageDesc& obj_desc, last_item.set_role_arn(role_arn); last_item.set_external_id(external_id); } - last_item.set_cred_provider_type(get_cred_provider_type(obj)); + if (obj.has_cred_provider_type() || has_non_empty_role_arn(obj)) { + last_item.set_cred_provider_type(get_cred_provider_type(obj)); + } + copy_obj_credential(obj, &last_item); } last_item.set_bucket(bucket); // format prefix, such as `/aa/bb/`, `aa/bb//`, `//aa/bb`, ` /aa/bb` -> `aa/bb` @@ -1496,7 +1600,12 @@ void MetaServiceImpl::alter_storage_vault(google::protobuf::RpcController* contr } if (use_credential_provider(obj)) { - if (!obj.has_provider() || obj.provider() != ObjectStoreInfoPB::S3) { + bool valid_aws = (obj.has_cred_provider_type() || has_non_empty_role_arn(obj)) && + !has_obj_credential(obj) && obj.has_provider() && + obj.provider() == ObjectStoreInfoPB::S3; + bool valid_credential = + has_obj_credential(obj) && !validate_obj_credential(obj).has_value(); + if (!valid_aws && !valid_credential) { code = MetaServiceCode::INVALID_ARGUMENT; msg = "s3 conf info err with credentials_provider_type, please check it"; return; @@ -1507,7 +1616,10 @@ void MetaServiceImpl::alter_storage_vault(google::protobuf::RpcController* contr for (auto& it : objs) { if (bucket == it.bucket() && prefix == it.prefix() && endpoint == it.endpoint() && region == it.region() && ak == it.ak() && sk == it.sk() && - obj.provider() == it.provider() && external_endpoint == it.external_endpoint()) { + obj.provider() == it.provider() && external_endpoint == it.external_endpoint() && + obj.has_credential() == it.has_credential() && + google::protobuf::util::MessageDifferencer::Equivalent(obj.credential(), + it.credential())) { // err, anything not changed code = MetaServiceCode::INVALID_ARGUMENT; msg = "original obj infos has a same conf, please check it"; @@ -1620,7 +1732,9 @@ void MetaServiceImpl::alter_storage_vault(google::protobuf::RpcController* contr break; } case AlterObjStoreInfoRequest::ALTER_S3_VAULT: { - alter_s3_storage_vault(instance, txn, request->vault(), code, msg, response); + if (alter_s3_storage_vault(instance, txn, request->vault(), code, msg, response) != 0) { + return; + } break; } case AlterObjStoreInfoRequest::ALTER_HDFS_VAULT: { @@ -1897,8 +2011,8 @@ void MetaServiceImpl::alter_obj_store_info(google::protobuf::RpcController* cont return; } // ATTN: prefix may be empty - if (((ak.empty() || sk.empty()) && role_arn.empty()) || bucket.empty() || - endpoint.empty() || region.empty() || prefix.empty()) { + if (((ak.empty() || sk.empty()) && role_arn.empty() && !has_obj_credential(obj)) || + bucket.empty() || endpoint.empty() || region.empty() || prefix.empty()) { code = MetaServiceCode::INVALID_ARGUMENT; msg = "s3 conf info err, please check it"; return; @@ -1908,7 +2022,10 @@ void MetaServiceImpl::alter_obj_store_info(google::protobuf::RpcController* cont for (auto& it : objs) { if (bucket == it.bucket() && prefix == it.prefix() && endpoint == it.endpoint() && region == it.region() && ak == it.ak() && sk == it.sk() && - obj.provider() == it.provider() && external_endpoint == it.external_endpoint()) { + obj.provider() == it.provider() && external_endpoint == it.external_endpoint() && + obj.has_credential() == it.has_credential() && + google::protobuf::util::MessageDifferencer::Equivalent(obj.credential(), + it.credential())) { // err, anything not changed code = MetaServiceCode::INVALID_ARGUMENT; msg = "original obj infos has a same conf, please check it"; diff --git a/cloud/src/recycler/s3_accessor.cpp b/cloud/src/recycler/s3_accessor.cpp index fe8455b4907f28..0413044d3fafb8 100644 --- a/cloud/src/recycler/s3_accessor.cpp +++ b/cloud/src/recycler/s3_accessor.cpp @@ -48,6 +48,8 @@ #include "cpp/obj-client/azure_obj_storage_client.h" #endif #include "cpp/aws_logger.h" +#include "cpp/obj-client/auth/obj_credential.h" +#include "cpp/obj-client/auth/obj_s3_client_factory.h" #include "cpp/obj-client/rate_limited_obj_storage_client.h" #include "cpp/obj-client/s3_obj_storage_client.h" #include "cpp/obj_retry_strategy.h" @@ -301,6 +303,8 @@ std::optional S3Conf::from_obj_store_info(const ObjectStoreInfoPB& obj_i } } + convert_obj_credential(obj_info, &s3_conf.credential); + s3_conf.endpoint = obj_info.endpoint(); s3_conf.region = obj_info.region(); s3_conf.bucket = obj_info.bucket(); @@ -460,10 +464,16 @@ int S3Accessor::init() { LOG(WARNING) << "failed to create AWS credential provider: " << credentials.error; return -1; } - auto s3_client = std::make_shared( - std::move(credentials.provider), std::move(aws_config), - Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, - conf_.use_virtual_addressing /* useVirtualAddressing */); + auto s3_client = doris::make_s3_client( + conf_.credential, + doris::S3ClientBuildContext { + .fallback_provider = std::move(credentials.provider), + .config = std::move(aws_config), + .payload_signing_policy = + Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, + .use_virtual_addressing = conf_.use_virtual_addressing, + .ca_cert_path = _ca_cert_file_path, + }); auto client = std::make_shared(std::move(s3_client), ObjStorageEndpointInfo { .endpoint = conf_.endpoint, diff --git a/cloud/src/recycler/s3_accessor.h b/cloud/src/recycler/s3_accessor.h index 922d2fdfdae5c0..5a0b48628a01c7 100644 --- a/cloud/src/recycler/s3_accessor.h +++ b/cloud/src/recycler/s3_accessor.h @@ -22,9 +22,11 @@ #include #include #include +#include #include "cpp/aws_common.h" #include "cpp/obj-client/auth/aws_credential_factory.h" +#include "cpp/obj-client/auth/obj_credential.h" #include "cpp/obj-client/obj_storage_client.h" #include "recycler/storage_vault_accessor.h" @@ -87,6 +89,7 @@ struct S3Conf { CredProviderType cred_provider_type = CredProviderType::Default; std::string role_arn; std::string external_id; + doris::CredentialConfig credential {}; enum Provider : uint8_t { S3, diff --git a/cloud/test/CMakeLists.txt b/cloud/test/CMakeLists.txt index da31c75448c4ff..fb9b1df742a280 100644 --- a/cloud/test/CMakeLists.txt +++ b/cloud/test/CMakeLists.txt @@ -81,6 +81,8 @@ add_executable(log_test log_test.cpp) add_executable(resource_test resource_test.cpp) +add_executable(obj_credential_test obj_credential_test.cpp) + add_executable(rpc_kv_bvar_test rpc_kv_bvar_test.cpp) add_executable(http_encode_key_test http_encode_key_test.cpp) @@ -176,6 +178,8 @@ target_link_libraries(log_test ${TEST_LINK_LIBS}) target_link_libraries(resource_test ${TEST_LINK_LIBS}) +target_link_libraries(obj_credential_test ${TEST_LINK_LIBS}) + target_link_libraries(rpc_kv_bvar_test ${TEST_LINK_LIBS}) target_link_libraries(http_encode_key_test ${TEST_LINK_LIBS}) diff --git a/cloud/test/meta_service_test.cpp b/cloud/test/meta_service_test.cpp index ec393bae1de7e5..8442eee2f6e0bf 100644 --- a/cloud/test/meta_service_test.cpp +++ b/cloud/test/meta_service_test.cpp @@ -35,6 +35,7 @@ #include #include #include +#include #include #include "common/config.h" @@ -734,10 +735,28 @@ TEST(MetaServiceTest, AlterS3StorageVaultTest) { InstanceInfoPB instance; instance.add_storage_vault_names(vault.name()); instance.add_resource_ids(vault.id()); + + constexpr char gcp_vault_name[] = "test_alter_gcp_vault"; + StorageVaultPB gcp_vault; + gcp_vault.set_name(gcp_vault_name); + gcp_vault.set_id("3"); + auto* gcp_obj_info = gcp_vault.mutable_obj_info(); + gcp_obj_info->set_id("3"); + gcp_obj_info->set_provider(ObjectStoreInfoPB::GCP); + gcp_obj_info->set_bucket("gcp-bucket"); + gcp_obj_info->set_endpoint("storage.googleapis.com"); + gcp_obj_info->set_region("us-east1"); + gcp_obj_info->mutable_credential()->mutable_gcp_credential()->set_credential_provider_type( + GcpCredentialPB::COMPUTE_ENGINE); + gcp_obj_info->mutable_credential()->mutable_gcp_credential()->set_impersonation_service_account( + "original@my-project.iam.gserviceaccount.com"); + instance.add_storage_vault_names(gcp_vault.name()); + instance.add_resource_ids(gcp_vault.id()); instance.set_instance_id("GetObjStoreInfoTestInstance"); val = instance.SerializeAsString(); txn->put(key, val); txn->put(storage_vault_key({instance.instance_id(), "2"}), vault.SerializeAsString()); + txn->put(storage_vault_key({instance.instance_id(), "3"}), gcp_vault.SerializeAsString()); ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK); txn = nullptr; @@ -752,6 +771,259 @@ TEST(MetaServiceTest, AlterS3StorageVaultTest) { i.ParseFromString(val); }; + auto get_test_vault = [&](std::string_view vault_id, StorageVaultPB& stored_vault) { + std::unique_ptr txn; + ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK); + std::string val; + ASSERT_EQ(txn->get(storage_vault_key({instance.instance_id(), vault_id}), &val), + TxnErrorCode::TXN_OK); + ASSERT_TRUE(stored_vault.ParseFromString(val)); + }; + + // Clearing an unused AWS role must preserve the active credentials, + // including encryption metadata, through the RPC and persistence boundary. + for (const auto* vault_id : {"2", "3"}) { + StorageVaultPB stored; + get_test_vault(vault_id, stored); + AlterObjStoreInfoRequest req; + req.set_cloud_unique_id("test_cloud_unique_id"); + req.set_op(AlterObjStoreInfoRequest::ALTER_S3_VAULT); + req.mutable_vault()->set_name(stored.name()); + auto* update = req.mutable_vault()->mutable_obj_info(); + auto alter_vault = [&]() { + brpc::Controller cntl; + AlterObjStoreInfoResponse res; + meta_service->alter_storage_vault( + reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, + nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK) << res.status().msg(); + }; + if (stored.obj_info().has_ak()) { + update->set_ak("ak"); + update->set_sk("sk"); + alter_vault(); + get_test_vault(vault_id, stored); + ASSERT_TRUE(stored.obj_info().has_encryption_info()); + } else { + ASSERT_TRUE(stored.obj_info().credential().has_gcp_credential()); + } + stored.mutable_obj_info()->clear_mtime(); + const auto original = stored.obj_info().SerializeAsString(); + update->Clear(); + update->set_role_arn(""); + // Retrying the same field clear must also preserve the credentials. + for (int attempt = 0; attempt < 2; ++attempt) { + alter_vault(); + get_test_vault(vault_id, stored); + stored.mutable_obj_info()->clear_mtime(); + ASSERT_EQ(stored.obj_info().SerializeAsString(), original); + } + } + + // Retain the prerequisite regression: an unpaired static key must not commit a rename. + { + AlterObjStoreInfoRequest req; + req.set_cloud_unique_id("test_cloud_unique_id"); + req.set_op(AlterObjStoreInfoRequest::ALTER_S3_VAULT); + req.mutable_vault()->set_name(vault_name); + req.mutable_vault()->set_alter_name("rename_must_not_commit"); + req.mutable_vault()->mutable_obj_info()->set_ak("unpaired-key"); + + brpc::Controller cntl; + AlterObjStoreInfoResponse res; + meta_service->alter_storage_vault( + reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::INVALID_ARGUMENT) << res.status().msg(); + + InstanceInfoPB stored_instance; + get_test_instance(stored_instance); + ASSERT_EQ(stored_instance.storage_vault_names(0), vault_name); + StorageVaultPB stored_vault; + get_test_vault("2", stored_vault); + ASSERT_EQ(stored_vault.name(), vault_name); + } + + // A rejected credential update must not commit a rename that was applied + // earlier in the request. + { + AlterObjStoreInfoRequest req; + req.set_cloud_unique_id("test_cloud_unique_id"); + req.set_op(AlterObjStoreInfoRequest::ALTER_S3_VAULT); + req.mutable_vault()->set_name(vault_name); + req.mutable_vault()->set_alter_name("rename_must_not_commit"); + req.mutable_vault() + ->mutable_obj_info() + ->mutable_credential() + ->mutable_gcp_credential() + ->set_credential_provider_type(GcpCredentialPB::DEFAULT); + + brpc::Controller cntl; + AlterObjStoreInfoResponse res; + meta_service->alter_storage_vault( + reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::INVALID_ARGUMENT) << res.status().msg(); + + InstanceInfoPB stored_instance; + get_test_instance(stored_instance); + ASSERT_EQ(stored_instance.storage_vault_names(0), vault_name); + StorageVaultPB stored_vault; + get_test_vault("2", stored_vault); + ASSERT_EQ(stored_vault.name(), vault_name); + } + + // A GCP-native vault cannot be changed to either AWS role or AWS + // credential-provider authentication while its provider remains GCP. + { + AlterObjStoreInfoRequest req; + req.set_cloud_unique_id("test_cloud_unique_id"); + req.set_op(AlterObjStoreInfoRequest::ALTER_S3_VAULT); + req.mutable_vault()->set_name(gcp_vault_name); + auto* update = req.mutable_vault()->mutable_obj_info(); + update->set_role_arn("arn:aws:iam::123456789012:role/test-role"); + update->set_cred_provider_type(CredProviderTypePB::INSTANCE_PROFILE); + + brpc::Controller cntl; + AlterObjStoreInfoResponse res; + meta_service->alter_storage_vault( + reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::INVALID_ARGUMENT) << res.status().msg(); + ASSERT_NE(res.status().msg().find("provider=S3"), std::string::npos); + } + + { + AlterObjStoreInfoRequest req; + req.set_cloud_unique_id("test_cloud_unique_id"); + req.set_op(AlterObjStoreInfoRequest::ALTER_S3_VAULT); + req.mutable_vault()->set_name(gcp_vault_name); + req.mutable_vault()->mutable_obj_info()->set_cred_provider_type( + CredProviderTypePB::CONTAINER); + + brpc::Controller cntl; + AlterObjStoreInfoResponse res; + meta_service->alter_storage_vault( + reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::INVALID_ARGUMENT) << res.status().msg(); + ASSERT_NE(res.status().msg().find("provider=S3"), std::string::npos); + + InstanceInfoPB stored_instance; + get_test_instance(stored_instance); + ASSERT_EQ(stored_instance.storage_vault_names(1), gcp_vault_name); + StorageVaultPB stored_vault; + get_test_vault("3", stored_vault); + ASSERT_TRUE(stored_vault.obj_info().has_credential()); + ASSERT_TRUE(stored_vault.obj_info().credential().has_gcp_credential()); + ASSERT_FALSE(stored_vault.obj_info().has_role_arn()); + ASSERT_FALSE(stored_vault.obj_info().has_cred_provider_type()); + } + + // Verify partial GCP updates at the RPC and persistence boundary. + { + AlterObjStoreInfoRequest req; + req.set_cloud_unique_id("test_cloud_unique_id"); + req.set_op(AlterObjStoreInfoRequest::ALTER_S3_VAULT); + req.mutable_vault()->set_name(gcp_vault_name); + auto alter_gcp = [&]() { + brpc::Controller cntl; + AlterObjStoreInfoResponse res; + meta_service->alter_storage_vault( + reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, + nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK) << res.status().msg(); + }; + auto* update = req.mutable_vault()->mutable_obj_info(); + auto* patch = update->mutable_credential()->mutable_gcp_credential(); + patch->set_impersonation_service_account("updated@my-project.iam.gserviceaccount.com"); + alter_gcp(); + StorageVaultPB stored; + get_test_vault("3", stored); + ASSERT_EQ(stored.obj_info().credential().gcp_credential().credential_provider_type(), + GcpCredentialPB::COMPUTE_ENGINE); + ASSERT_EQ(stored.obj_info().credential().gcp_credential().impersonation_service_account(), + "updated@my-project.iam.gserviceaccount.com"); + + patch->Clear(); + patch->set_credential_provider_type(GcpCredentialPB::DEFAULT); + alter_gcp(); + get_test_vault("3", stored); + ASSERT_EQ(stored.obj_info().credential().gcp_credential().credential_provider_type(), + GcpCredentialPB::DEFAULT); + ASSERT_EQ(stored.obj_info().credential().gcp_credential().impersonation_service_account(), + "updated@my-project.iam.gserviceaccount.com"); + + patch->set_credential_provider_type(GcpCredentialPB::COMPUTE_ENGINE); + alter_gcp(); + patch->Clear(); + patch->set_impersonation_service_account(""); + alter_gcp(); + get_test_vault("3", stored); + ASSERT_EQ(stored.obj_info().credential().gcp_credential().credential_provider_type(), + GcpCredentialPB::COMPUTE_ENGINE); + ASSERT_FALSE(stored.obj_info() + .credential() + .gcp_credential() + .has_impersonation_service_account()); + + // Rejected patches must preserve both the stored vault and its name. + const auto original = stored.SerializeAsString(); + patch->set_impersonation_service_account("invalid-account"); + req.mutable_vault()->set_alter_name("invalid_gcp_rename"); + brpc::Controller cntl; + AlterObjStoreInfoResponse res; + meta_service->alter_storage_vault( + reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::INVALID_ARGUMENT); + get_test_vault("3", stored); + ASSERT_EQ(stored.SerializeAsString(), original); + InstanceInfoPB stored_instance; + get_test_instance(stored_instance); + ASSERT_EQ(stored_instance.storage_vault_names(1), gcp_vault_name); + req.mutable_vault()->clear_alter_name(); + + // Empty or half-empty key replacements must not remove native authentication + // or persist the rename that was applied to the transaction's candidate. + for (int empty_keys = 1; empty_keys <= 3; ++empty_keys) { + AlterObjStoreInfoRequest empty_req; + empty_req.set_cloud_unique_id("test_cloud_unique_id"); + empty_req.set_op(AlterObjStoreInfoRequest::ALTER_S3_VAULT); + empty_req.mutable_vault()->set_name(gcp_vault_name); + empty_req.mutable_vault()->set_alter_name("empty_key_rename"); + auto* empty_update = empty_req.mutable_vault()->mutable_obj_info(); + empty_update->set_ak((empty_keys & 1) ? "" : "replacement-ak"); + empty_update->set_sk((empty_keys & 2) ? "" : "replacement-sk"); + brpc::Controller empty_cntl; + AlterObjStoreInfoResponse empty_res; + meta_service->alter_storage_vault( + reinterpret_cast<::google::protobuf::RpcController*>(&empty_cntl), &empty_req, + &empty_res, nullptr); + ASSERT_EQ(empty_res.status().code(), MetaServiceCode::INVALID_ARGUMENT); + get_test_vault("3", stored); + ASSERT_EQ(stored.SerializeAsString(), original); + get_test_instance(stored_instance); + ASSERT_EQ(stored_instance.storage_vault_names(1), gcp_vault_name); + } + + // Switching from encrypted static keys creates a new native credential. + update->Clear(); + update->set_ak("gcp-ak"); + update->set_sk("gcp-sk"); + alter_gcp(); + get_test_vault("3", stored); + ASSERT_FALSE(stored.obj_info().has_credential()); + ASSERT_TRUE(stored.obj_info().has_encryption_info()); + update->Clear(); + update->mutable_credential()->mutable_gcp_credential()->set_impersonation_service_account( + "updated@my-project.iam.gserviceaccount.com"); + alter_gcp(); + get_test_vault("3", stored); + ASSERT_EQ(stored.obj_info().credential().gcp_credential().credential_provider_type(), + GcpCredentialPB::DEFAULT); + ASSERT_EQ(stored.obj_info().credential().gcp_credential().impersonation_service_account(), + "updated@my-project.iam.gserviceaccount.com"); + ASSERT_FALSE(stored.obj_info().has_ak()); + ASSERT_FALSE(stored.obj_info().has_sk()); + ASSERT_FALSE(stored.obj_info().has_encryption_info()); + } + { AlterObjStoreInfoRequest req; req.set_cloud_unique_id("test_cloud_unique_id"); @@ -4678,6 +4950,10 @@ TEST(MetaServiceTest, FilterCopyFilesTest) { auto stage_id = "test_stage_id"; int64_t table_id = 100; [[maybe_unused]] auto sp = SyncPoint::get_instance(); + DORIS_CLOUD_DEFER { + sp->disable_processing(); + sp->clear_all_call_backs(); + }; sp->set_call_back("get_instance_id", [&](auto&& args) { auto* ret = try_any_cast_ret(args); ret->first = instance_id; @@ -4948,6 +5224,122 @@ TEST(MetaServiceTest, CalcSyncVersionsTest) { } } +TEST(MetaServiceTest, NativeGcpLegacyAddReadback) { + auto meta_service = get_meta_service(); + brpc::Controller cntl; + std::string key; + instance_key({"test_instance"}, &key); + InstanceInfoPB instance; + instance.set_instance_id("test_instance"); + std::unique_ptr txn; + ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK); + txn->put(key, instance.SerializeAsString()); + ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK); + + AlterObjStoreInfoRequest req; + req.set_cloud_unique_id("test_cloud_unique_id"); + req.set_op(AlterObjStoreInfoRequest::ADD_OBJ_INFO); + auto* obj = req.mutable_obj(); + obj->set_provider(ObjectStoreInfoPB::GCP); + obj->set_bucket("native-bucket"); + obj->set_prefix("instance-prefix"); + obj->set_endpoint("storage.googleapis.com"); + obj->set_region("us-east1"); + auto* credential = obj->mutable_credential()->mutable_gcp_credential(); + credential->set_credential_provider_type(GcpCredentialPB::COMPUTE_ENGINE); + credential->set_impersonation_service_account("target@test.iam.gserviceaccount.com"); + + // Reject mixed authentication before persisting anything. + obj->set_ak("unexpected-key"); + AlterObjStoreInfoResponse rejected; + meta_service->alter_obj_store_info(&cntl, &req, &rejected, nullptr); + ASSERT_EQ(rejected.status().code(), MetaServiceCode::INVALID_ARGUMENT); + obj->clear_ak(); + AlterObjStoreInfoResponse res; + meta_service->alter_obj_store_info(&cntl, &req, &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK) << res.status().msg(); + + txn.reset(); + ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK); + std::string val; + ASSERT_EQ(txn->get(key, &val), TxnErrorCode::TXN_OK); + ASSERT_TRUE(instance.ParseFromString(val)); + ASSERT_EQ(instance.obj_info_size(), 1); + ASSERT_EQ(instance.obj_info(0).credential().SerializeAsString(), + obj->credential().SerializeAsString()); + ASSERT_FALSE(instance.obj_info(0).has_ak()); + ASSERT_FALSE(instance.obj_info(0).has_encryption_info()); + + // A changed account or credential source at the same location is a new configuration. + const auto original_credential = instance.obj_info(0).credential().SerializeAsString(); + int expected_count = 1; + credential->set_impersonation_service_account("rotated@test.iam.gserviceaccount.com"); + for (auto provider : {GcpCredentialPB::COMPUTE_ENGINE, GcpCredentialPB::DEFAULT}) { + credential->set_credential_provider_type(provider); + res.Clear(); + meta_service->alter_obj_store_info(&cntl, &req, &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK) << res.status().msg(); + ++expected_count; + + // An exact repeat must still be rejected without appending another entry. + res.Clear(); + meta_service->alter_obj_store_info(&cntl, &req, &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::INVALID_ARGUMENT); + EXPECT_EQ(res.status().msg(), "original obj infos has a same conf, please check it"); + + txn.reset(); + ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK); + ASSERT_EQ(txn->get(key, &val), TxnErrorCode::TXN_OK); + ASSERT_TRUE(instance.ParseFromString(val)); + ASSERT_EQ(instance.obj_info_size(), expected_count); + EXPECT_EQ(instance.obj_info(0).credential().SerializeAsString(), original_credential); + EXPECT_EQ(instance.obj_info(expected_count - 1).credential().SerializeAsString(), + obj->credential().SerializeAsString()); + } +} + +TEST(MetaServiceTest, AddRoleOnlyVaultDefaultsToInstanceProfile) { + auto meta_service = get_meta_service(); + brpc::Controller cntl; + std::string key; + instance_key({"test_instance"}, &key); + InstanceInfoPB instance; + instance.set_instance_id("test_instance"); + instance.set_enable_storage_vault(true); + std::unique_ptr txn; + ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK); + txn->put(key, instance.SerializeAsString()); + ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK); + + AlterObjStoreInfoRequest req; + req.set_cloud_unique_id("test_cloud_unique_id"); + req.set_op(AlterObjStoreInfoRequest::ADD_S3_VAULT); + req.mutable_vault()->set_name("role_only_vault"); + auto* obj = req.mutable_obj(); + obj->set_provider(ObjectStoreInfoPB::S3); + obj->set_bucket("bucket"); + obj->set_endpoint("s3.us-east-1.amazonaws.com"); + obj->set_region("us-east-1"); + obj->set_role_arn("arn:aws:iam::123456789012:role/test"); + ASSERT_FALSE(obj->has_cred_provider_type()); + AlterObjStoreInfoResponse res; + meta_service->alter_storage_vault(&cntl, &req, &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK) << res.status().msg(); + + txn.reset(); + ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK); + std::string val; + ASSERT_EQ(txn->get(key, &val), TxnErrorCode::TXN_OK); + ASSERT_TRUE(instance.ParseFromString(val)); + ASSERT_EQ(instance.resource_ids_size(), 1); + ASSERT_EQ(txn->get(storage_vault_key({"test_instance", instance.resource_ids(0)}), &val), + TxnErrorCode::TXN_OK); + StorageVaultPB vault; + ASSERT_TRUE(vault.ParseFromString(val)); + EXPECT_EQ(vault.obj_info().role_arn(), obj->role_arn()); + EXPECT_EQ(vault.obj_info().cred_provider_type(), CredProviderTypePB::INSTANCE_PROFILE); +} + TEST(MetaServiceTest, StageTest) { auto meta_service = get_meta_service(); brpc::Controller cntl; @@ -11896,7 +12288,8 @@ TEST(MetaServiceTest, UpdateAkSkTest) { std::string cipher_sk = "JUkuTDctR+ckJtnPkLScWaQZRcOtWBhsLLpnCRxQLxr734qB8cs6gNLH6grE1FxO"; std::string plain_sk = "Hx60p12123af234541nsVsffdfsdfghsdfhsdf34t"; - auto update = [&](bool with_user_id, bool with_wrong_user_id) { + auto update = [&](bool with_user_id, bool with_wrong_user_id, bool with_gcp_credential = false, + bool with_existing_keys = false) { std::unique_ptr txn; ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK); std::string key; @@ -11910,6 +12303,20 @@ TEST(MetaServiceTest, UpdateAkSkTest) { } obj_info.set_ak("ak"); obj_info.set_sk("sk"); + if (with_gcp_credential) { + obj_info.set_provider(ObjectStoreInfoPB::GCP); + auto* credential = obj_info.mutable_credential()->mutable_gcp_credential(); + credential->set_credential_provider_type(GcpCredentialPB::COMPUTE_ENGINE); + credential->set_impersonation_service_account("test@project.iam.gserviceaccount.com"); + obj_info.clear_ak(); + obj_info.clear_sk(); + if (with_existing_keys) { + // Reproduce the mixed state persisted before this fix. Resubmitting + // the same keys must still remove the native credential. + obj_info.set_ak("new_ak"); + obj_info.set_sk(cipher_sk); + } + } InstanceInfoPB instance; instance.add_obj_info()->CopyFrom(obj_info); val = instance.SerializeAsString(); @@ -11928,12 +12335,34 @@ TEST(MetaServiceTest, UpdateAkSkTest) { ram_user.set_sk(plain_sk); req.add_internal_bucket_user()->CopyFrom(ram_user); + if (with_gcp_credential && !with_wrong_user_id) { + // Empty replacement keys must not erase the active native credential. + for (int empty_keys = 1; empty_keys <= 3; ++empty_keys) { + UpdateAkSkRequest invalid_req = req; + auto* keys = invalid_req.mutable_internal_bucket_user(0); + if (empty_keys & 1) keys->set_ak(""); + if (empty_keys & 2) keys->set_sk(""); + brpc::Controller invalid_cntl; + UpdateAkSkResponse invalid_res; + meta_service->update_ak_sk( + reinterpret_cast<::google::protobuf::RpcController*>(&invalid_cntl), + &invalid_req, &invalid_res, nullptr); + ASSERT_EQ(invalid_res.status().code(), MetaServiceCode::INVALID_ARGUMENT); + InstanceInfoPB unchanged; + get_test_instance(unchanged); + ASSERT_EQ(unchanged.obj_info(0).SerializeAsString(), obj_info.SerializeAsString()); + } + } + brpc::Controller cntl; UpdateAkSkResponse res; meta_service->update_ak_sk(reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, nullptr); if (with_wrong_user_id) { ASSERT_EQ(res.status().code(), MetaServiceCode::INVALID_ARGUMENT); + InstanceInfoPB unchanged; + get_test_instance(unchanged); + ASSERT_EQ(unchanged.obj_info(0).SerializeAsString(), obj_info.SerializeAsString()); } else { ASSERT_EQ(res.status().code(), MetaServiceCode::OK); InstanceInfoPB update_instance; @@ -11941,12 +12370,25 @@ TEST(MetaServiceTest, UpdateAkSkTest) { ASSERT_EQ(update_instance.obj_info(0).user_id(), "111"); ASSERT_EQ(update_instance.obj_info(0).ak(), "new_ak"); ASSERT_EQ(update_instance.obj_info(0).sk(), cipher_sk); + ASSERT_FALSE(update_instance.obj_info(0).has_credential()); + ASSERT_TRUE(update_instance.obj_info(0).has_encryption_info()); + if (with_gcp_credential) { + ASSERT_EQ(update_instance.obj_info(0).provider(), ObjectStoreInfoPB::GCP); + } } }; update(false, false); update(true, false); update(true, true); + update(false, false, true); + update(true, false, true); + update(true, true, true); + update(false, false, true, true); + update(true, false, true, true); + + sp->disable_processing(); + sp->clear_all_call_backs(); } TEST(MetaServiceTest, AlterIamTest) { @@ -13194,6 +13636,70 @@ TEST(MetaServiceTest, AlterObjInfoTest) { ASSERT_FALSE(instance.obj_info(0).has_external_id()); } + // Both legacy ALTER operations must replace native GCP authentication, + // including a mixed state left by an earlier key update using the same keys. + for (auto op : {AlterObjStoreInfoRequest::ALTER_OBJ_INFO, + AlterObjStoreInfoRequest::LEGACY_UPDATE_AK_SK}) { + for (bool with_existing_keys : {false, true}) { + InstanceInfoPB native_instance; + auto* obj = native_instance.add_obj_info(); + obj->set_id("1"); + obj->set_provider(ObjectStoreInfoPB::GCP); + auto* credential = obj->mutable_credential()->mutable_gcp_credential(); + credential->set_credential_provider_type(GcpCredentialPB::COMPUTE_ENGINE); + credential->set_impersonation_service_account("test@project.iam.gserviceaccount.com"); + if (with_existing_keys) { + obj->set_ak("new_ak"); + obj->set_sk(cipher_sk); + } + ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK); + txn->put(key, native_instance.SerializeAsString()); + ASSERT_EQ(txn->commit(), TxnErrorCode::TXN_OK); + + AlterObjStoreInfoRequest req; + req.set_cloud_unique_id("test_cloud_unique_id"); + req.set_op(op); + req.mutable_obj()->set_id("1"); + req.mutable_obj()->set_ak("new_ak"); + req.mutable_obj()->set_sk(plain_sk); + // A role-only clear and either empty key must leave the stored + // native credential intact, including when legacy keys also exist. + for (int empty_keys = 0; empty_keys <= 3; ++empty_keys) { + AlterObjStoreInfoRequest invalid_req = req; + if (empty_keys == 0) { + invalid_req.mutable_obj()->clear_ak(); + invalid_req.mutable_obj()->clear_sk(); + invalid_req.mutable_obj()->set_role_arn(""); + } else { + if (empty_keys & 1) invalid_req.mutable_obj()->set_ak(""); + if (empty_keys & 2) invalid_req.mutable_obj()->set_sk(""); + } + brpc::Controller invalid_cntl; + AlterObjStoreInfoResponse invalid_res; + meta_service->alter_obj_store_info( + reinterpret_cast<::google::protobuf::RpcController*>(&invalid_cntl), + &invalid_req, &invalid_res, nullptr); + ASSERT_EQ(invalid_res.status().code(), MetaServiceCode::INVALID_ARGUMENT); + InstanceInfoPB unchanged; + get_test_instance(unchanged); + ASSERT_EQ(unchanged.obj_info(0).SerializeAsString(), obj->SerializeAsString()); + } + brpc::Controller cntl; + AlterObjStoreInfoResponse res; + meta_service->alter_obj_store_info( + reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, + nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK) << res.status().msg(); + InstanceInfoPB stored; + get_test_instance(stored); + ASSERT_EQ(stored.obj_info(0).provider(), ObjectStoreInfoPB::GCP); + ASSERT_EQ(stored.obj_info(0).ak(), "new_ak"); + ASSERT_EQ(stored.obj_info(0).sk(), cipher_sk); + ASSERT_TRUE(stored.obj_info(0).has_encryption_info()); + ASSERT_FALSE(stored.obj_info(0).has_credential()); + } + } + SyncPoint::get_instance()->disable_processing(); SyncPoint::get_instance()->clear_all_call_backs(); } @@ -13299,6 +13805,87 @@ TEST(MetaServiceTest, AlterS3StorageVaultWithRoleArnTest) { ASSERT_EQ(get_obj.name(), new_vault_name) << get_obj.obj_info().ShortDebugString(); } + // Role, external ID, and credential provider are independent ALTER fields. + // Updating one must preserve the other two. + { + AlterObjStoreInfoRequest req; + req.set_cloud_unique_id("test_cloud_unique_id"); + req.set_op(AlterObjStoreInfoRequest::ALTER_S3_VAULT); + req.mutable_vault()->set_name("new_test_alter_s3_vault_111"); + req.mutable_vault()->mutable_obj_info()->set_cred_provider_type( + CredProviderTypePB::CONTAINER); + + brpc::Controller cntl; + AlterObjStoreInfoResponse res; + meta_service->alter_storage_vault( + reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK) << res.status().msg(); + + std::unique_ptr txn; + ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK); + std::string val; + ASSERT_EQ(txn->get(storage_vault_key({"GetObjStoreInfoTestInstance", "2"}), &val), + TxnErrorCode::TXN_OK); + StorageVaultPB stored_vault; + ASSERT_TRUE(stored_vault.ParseFromString(val)); + ASSERT_EQ(stored_vault.obj_info().role_arn(), "arn:aws:iam::12311321:role/test-alter-role"); + ASSERT_EQ(stored_vault.obj_info().external_id(), "external_id_123123"); + ASSERT_EQ(stored_vault.obj_info().cred_provider_type(), CredProviderTypePB::CONTAINER); + } + + { + AlterObjStoreInfoRequest req; + req.set_cloud_unique_id("test_cloud_unique_id"); + req.set_op(AlterObjStoreInfoRequest::ALTER_S3_VAULT); + req.mutable_vault()->set_name("new_test_alter_s3_vault_111"); + req.mutable_vault()->mutable_obj_info()->set_role_arn( + "arn:aws:iam::12311321:role/replacement-role"); + + brpc::Controller cntl; + AlterObjStoreInfoResponse res; + meta_service->alter_storage_vault( + reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK) << res.status().msg(); + + std::unique_ptr txn; + ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK); + std::string val; + ASSERT_EQ(txn->get(storage_vault_key({"GetObjStoreInfoTestInstance", "2"}), &val), + TxnErrorCode::TXN_OK); + StorageVaultPB stored_vault; + ASSERT_TRUE(stored_vault.ParseFromString(val)); + ASSERT_EQ(stored_vault.obj_info().role_arn(), + "arn:aws:iam::12311321:role/replacement-role"); + ASSERT_EQ(stored_vault.obj_info().external_id(), "external_id_123123"); + ASSERT_EQ(stored_vault.obj_info().cred_provider_type(), CredProviderTypePB::CONTAINER); + } + + { + AlterObjStoreInfoRequest req; + req.set_cloud_unique_id("test_cloud_unique_id"); + req.set_op(AlterObjStoreInfoRequest::ALTER_S3_VAULT); + req.mutable_vault()->set_name("new_test_alter_s3_vault_111"); + req.mutable_vault()->mutable_obj_info()->set_external_id("replacement-external-id"); + + brpc::Controller cntl; + AlterObjStoreInfoResponse res; + meta_service->alter_storage_vault( + reinterpret_cast<::google::protobuf::RpcController*>(&cntl), &req, &res, nullptr); + ASSERT_EQ(res.status().code(), MetaServiceCode::OK) << res.status().msg(); + + std::unique_ptr txn; + ASSERT_EQ(meta_service->txn_kv()->create_txn(&txn), TxnErrorCode::TXN_OK); + std::string val; + ASSERT_EQ(txn->get(storage_vault_key({"GetObjStoreInfoTestInstance", "2"}), &val), + TxnErrorCode::TXN_OK); + StorageVaultPB stored_vault; + ASSERT_TRUE(stored_vault.ParseFromString(val)); + ASSERT_EQ(stored_vault.obj_info().role_arn(), + "arn:aws:iam::12311321:role/replacement-role"); + ASSERT_EQ(stored_vault.obj_info().external_id(), "replacement-external-id"); + ASSERT_EQ(stored_vault.obj_info().cred_provider_type(), CredProviderTypePB::CONTAINER); + } + std::string cipher_sk = "JUkuTDctR+ckJtnPkLScWaQZRcOtWBhsLLpnCRxQLxr734qB8cs6gNLH6grE1FxO"; std::string plain_sk = "Hx60p12123af234541nsVsffdfsdfghsdfhsdf34t"; diff --git a/cloud/test/obj_credential_test.cpp b/cloud/test/obj_credential_test.cpp new file mode 100644 index 00000000000000..9f33e5ab219139 --- /dev/null +++ b/cloud/test/obj_credential_test.cpp @@ -0,0 +1,155 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "common/auth/obj_credential.h" + +#include +#include + +namespace doris::cloud { +namespace { + +constexpr char ACCOUNT[] = "original@my-project.iam.gserviceaccount.com"; +constexpr char NEW_ACCOUNT[] = "updated@my-project.iam.gserviceaccount.com"; + +ObjectStoreInfoPB make_gcp_vault() { + ObjectStoreInfoPB obj; + obj.set_provider(ObjectStoreInfoPB::GCP); + obj.set_bucket("test-bucket"); + obj.set_endpoint("storage.googleapis.com"); + obj.set_region("us-east1"); + auto* credential = obj.mutable_credential()->mutable_gcp_credential(); + credential->set_credential_provider_type(GcpCredentialPB::COMPUTE_ENGINE); + credential->set_impersonation_service_account(ACCOUNT); + return obj; +} + +TEST(ObjCredentialTest, AlterTargetPreservesSourceAndStorageFields) { + auto stored = make_gcp_vault(); + auto expected = stored; + expected.mutable_credential()->mutable_gcp_credential()->set_impersonation_service_account( + NEW_ACCOUNT); + ObjectStoreInfoPB update; + update.mutable_credential()->mutable_gcp_credential()->set_impersonation_service_account( + NEW_ACCOUNT); + auto error = apply_obj_credential(update, &stored); + ASSERT_FALSE(error.has_value()) << error.value_or(""); + EXPECT_EQ(stored.SerializeAsString(), expected.SerializeAsString()); +} + +TEST(ObjCredentialTest, AlterSourcePreservesTarget) { + for (auto source : {GcpCredentialPB::COMPUTE_ENGINE, GcpCredentialPB::DEFAULT}) { + auto stored = make_gcp_vault(); + ObjectStoreInfoPB update; + update.mutable_credential()->mutable_gcp_credential()->set_credential_provider_type(source); + auto error = apply_obj_credential(update, &stored); + ASSERT_FALSE(error.has_value()) << error.value_or(""); + EXPECT_EQ(stored.credential().gcp_credential().credential_provider_type(), source); + EXPECT_EQ(stored.credential().gcp_credential().impersonation_service_account(), ACCOUNT); + } +} + +TEST(ObjCredentialTest, ExplicitEmptyTargetClearsOnlyImpersonation) { + auto stored = make_gcp_vault(); + ObjectStoreInfoPB update; + update.mutable_credential()->mutable_gcp_credential()->set_impersonation_service_account(""); + auto error = apply_obj_credential(update, &stored); + ASSERT_FALSE(error.has_value()) << error.value_or(""); + EXPECT_EQ(stored.credential().gcp_credential().credential_provider_type(), + GcpCredentialPB::COMPUTE_ENGINE); + EXPECT_FALSE(stored.credential().gcp_credential().has_impersonation_service_account()); +} + +TEST(ObjCredentialTest, SwitchFromStaticDefaultsSourceAndRemovesLegacyCredentials) { + for (auto account : {NEW_ACCOUNT, ""}) { + for (bool explicit_source : {false, true}) { + auto stored = make_gcp_vault(); + stored.clear_credential(); + stored.set_ak("encrypted-ak"); + stored.set_sk("encrypted-sk"); + stored.mutable_encryption_info()->set_key_id(1); + const auto original = stored.SerializeAsString(); + ObjectStoreInfoPB update; + auto* credential = update.mutable_credential()->mutable_gcp_credential(); + credential->set_impersonation_service_account(account); + if (explicit_source) { + credential->set_credential_provider_type(GcpCredentialPB::COMPUTE_ENGINE); + } + auto error = apply_obj_credential(update, &stored); + if (!explicit_source && std::string(account).empty()) { + ASSERT_TRUE(error.has_value()); + EXPECT_EQ(stored.SerializeAsString(), original); + continue; + } + ASSERT_FALSE(error.has_value()) << error.value_or(""); + EXPECT_TRUE(stored.credential().gcp_credential().has_credential_provider_type()); + EXPECT_EQ(stored.credential().gcp_credential().credential_provider_type(), + explicit_source ? GcpCredentialPB::COMPUTE_ENGINE : GcpCredentialPB::DEFAULT); + EXPECT_EQ(stored.credential().gcp_credential().impersonation_service_account(), + account); + EXPECT_FALSE(stored.has_ak()); + EXPECT_FALSE(stored.has_sk()); + EXPECT_FALSE(stored.has_encryption_info()); + EXPECT_FALSE(stored.has_cred_provider_type()); + EXPECT_FALSE(stored.has_role_arn()); + EXPECT_FALSE(stored.has_external_id()); + } + } +} + +TEST(ObjCredentialTest, RejectedPatchDoesNotMutateStoredCredential) { + for (bool native : {false, true}) { + auto stored = make_gcp_vault(); + if (!native) { + stored.clear_credential(); + stored.set_ak("encrypted-ak"); + stored.set_sk("encrypted-sk"); + stored.mutable_encryption_info()->set_key_id(1); + } + const auto original = stored.SerializeAsString(); + ObjectStoreInfoPB update; + update.mutable_credential()->mutable_gcp_credential()->set_impersonation_service_account( + "invalid-account"); + EXPECT_TRUE(apply_obj_credential(update, &stored).has_value()); + EXPECT_EQ(stored.SerializeAsString(), original); + + update.mutable_credential()->mutable_gcp_credential()->set_impersonation_service_account( + NEW_ACCOUNT); + update.set_ak("conflicting-ak"); + EXPECT_TRUE(apply_obj_credential(update, &stored).has_value()); + EXPECT_EQ(stored.SerializeAsString(), original); + } +} + +TEST(ObjCredentialTest, RejectsEmptyPatchAndCrossProviderUpdate) { + auto stored = make_gcp_vault(); + auto original = stored.SerializeAsString(); + ObjectStoreInfoPB update; + update.mutable_credential()->mutable_gcp_credential(); + EXPECT_TRUE(apply_obj_credential(update, &stored).has_value()); + EXPECT_EQ(stored.SerializeAsString(), original); + + stored.set_provider(ObjectStoreInfoPB::S3); + original = stored.SerializeAsString(); + update.mutable_credential()->mutable_gcp_credential()->set_impersonation_service_account( + NEW_ACCOUNT); + EXPECT_TRUE(apply_obj_credential(update, &stored).has_value()); + EXPECT_EQ(stored.SerializeAsString(), original); +} + +} // namespace +} // namespace doris::cloud diff --git a/cloud/test/s3_accessor_test.cpp b/cloud/test/s3_accessor_test.cpp index fa2bfd1c7efe6e..ef69a0d45ab647 100644 --- a/cloud/test/s3_accessor_test.cpp +++ b/cloud/test/s3_accessor_test.cpp @@ -27,6 +27,7 @@ #include #include +#include "common/auth/obj_credential.h" #include "common/config.h" #include "common/configbase.h" #include "common/logging.h" @@ -396,6 +397,44 @@ TEST_F(S3AccessorTest, path_style_test) { } } +TEST(S3AccessorConfigTest, GcpCredentialFromObjectStoreInfo) { + ObjectStoreInfoPB obj_info; + obj_info.set_provider(ObjectStoreInfoPB::GCP); + obj_info.set_endpoint("storage.googleapis.com"); + obj_info.set_region("us-east1"); + obj_info.set_bucket("test-bucket"); + auto* gcp_credential = obj_info.mutable_credential()->mutable_gcp_credential(); + gcp_credential->set_credential_provider_type(GcpCredentialPB::COMPUTE_ENGINE); + gcp_credential->set_impersonation_service_account("target@my-project.iam.gserviceaccount.com"); + + auto s3_conf = S3Conf::from_obj_store_info(obj_info); + ASSERT_TRUE(s3_conf.has_value()); + EXPECT_EQ(s3_conf->provider, S3Conf::GCS); + ASSERT_TRUE(std::holds_alternative(s3_conf->credential)); + const auto& credential = std::get(s3_conf->credential); + EXPECT_EQ(credential.provider_type, doris::GcpCredentialProviderType::ComputeEngine); + EXPECT_EQ(credential.impersonation_service_account, + "target@my-project.iam.gserviceaccount.com"); +} + +TEST(S3AccessorConfigTest, ValidateGcpStorageVaultCredential) { + ObjectStoreInfoPB obj_info; + obj_info.set_provider(ObjectStoreInfoPB::GCP); + obj_info.set_endpoint("storage.googleapis.com"); + obj_info.set_region("us-east1"); + obj_info.set_bucket("test-bucket"); + obj_info.mutable_credential()->mutable_gcp_credential()->set_credential_provider_type( + GcpCredentialPB::DEFAULT); + + EXPECT_FALSE(validate_obj_credential(obj_info).has_value()); + + obj_info.set_ak("ak"); + obj_info.set_sk("sk"); + auto error = validate_obj_credential(obj_info); + ASSERT_TRUE(error.has_value()); + EXPECT_NE(error->find("cannot be combined"), std::string::npos); +} + class S3AccessorRoleTest : public testing::Test { static void SetUpTestSuite() { if (!std::getenv("AWS_ROLE_ARN") || !std::getenv("AWS_EXTERNAL_ID") || diff --git a/common/cpp/obj-client/auth/gcp/gcp_auth.cpp b/common/cpp/obj-client/auth/gcp/gcp_auth.cpp new file mode 100644 index 00000000000000..8928e202a34845 --- /dev/null +++ b/common/cpp/obj-client/auth/gcp/gcp_auth.cpp @@ -0,0 +1,108 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "cpp/obj-client/auth/gcp/gcp_auth.h" + +#include +#include +#include +#include + +namespace doris { + +bool is_valid_gcp_storage_endpoint(std::string_view endpoint) { + // Only service-base XML API endpoints: the S3 clients add the bucket themselves. + static const std::regex pattern( + R"(https://(storage\.googleapis\.com|[a-z0-9-]+-storage\.googleapis\.com|storage\.[a-z0-9-]+\.rep\.googleapis\.com)(:443)?/?)", + std::regex::icase); + return std::regex_match(endpoint.begin(), endpoint.end(), pattern); +} + +bool is_valid_gcp_service_account_email(std::string_view email) { + constexpr std::string_view user_managed_suffix = ".iam.gserviceaccount.com"; + constexpr std::string_view compute_engine_domain = "developer.gserviceaccount.com"; + constexpr std::string_view app_engine_domain = "appspot.gserviceaccount.com"; + auto at = email.find('@'); + if (at == std::string_view::npos || at == 0 || + email.find('@', at + 1) != std::string_view::npos || + std::any_of(email.begin(), email.end(), [](unsigned char c) { return std::isspace(c); })) { + return false; + } + + auto domain = email.substr(at + 1); + return domain == compute_engine_domain || domain == app_engine_domain || + (domain.size() > user_managed_suffix.size() && domain.ends_with(user_managed_suffix)); +} + +std::optional parse_gcp_credential_provider_type( + std::string_view provider_type) { + std::string normalized(provider_type); + normalized.erase(normalized.begin(), + std::find_if(normalized.begin(), normalized.end(), + [](unsigned char c) { return !std::isspace(c); })); + normalized.erase(std::find_if(normalized.rbegin(), normalized.rend(), + [](unsigned char c) { return !std::isspace(c); }) + .base(), + normalized.end()); + std::transform(normalized.begin(), normalized.end(), normalized.begin(), + [](unsigned char c) { return static_cast(std::toupper(c)); }); + if (normalized == "DEFAULT") { + return GcpCredentialProviderType::Default; + } + if (normalized == "COMPUTE_ENGINE") { + return GcpCredentialProviderType::ComputeEngine; + } + return std::nullopt; +} + +std::optional validate_gcp_credential(const GcpCredentialConfig& credential) { + if (credential.impersonation_service_account.empty()) { + return std::nullopt; + } + if (!is_valid_gcp_service_account_email(credential.impersonation_service_account)) { + return "Invalid GCP service account email"; + } + return std::nullopt; +} + +GcpCredentialParseResult parse_gcp_credential_properties( + const ObjCredentialPropertyGetter& get_property) { + auto provider_type_value = get_property(GCP_CREDENTIAL_PROVIDER_TYPE); + auto impersonation_service_account = get_property(GCP_IMPERSONATION_SERVICE_ACCOUNT); + if (!provider_type_value.has_value() && !impersonation_service_account.has_value()) { + return {}; + } + auto provider_type = provider_type_value.has_value() + ? parse_gcp_credential_provider_type(*provider_type_value) + : GcpCredentialProviderType::Default; + if (!provider_type.has_value()) { + return {.error = "invalid " + std::string(GCP_CREDENTIAL_PROVIDER_TYPE) + " value \"" + + *provider_type_value + + "\"; supported values are DEFAULT and COMPUTE_ENGINE"}; + } + + GcpCredentialConfig config {.provider_type = *provider_type}; + if (impersonation_service_account.has_value()) { + config.impersonation_service_account = std::move(*impersonation_service_account); + } + if (auto error = validate_gcp_credential(config); error.has_value()) { + return {.error = std::move(error)}; + } + return {.credential = std::move(config)}; +} + +} // namespace doris diff --git a/common/cpp/obj-client/auth/gcp/gcp_auth.h b/common/cpp/obj-client/auth/gcp/gcp_auth.h new file mode 100644 index 00000000000000..8f05d7a580a44d --- /dev/null +++ b/common/cpp/obj-client/auth/gcp/gcp_auth.h @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "cpp/obj-client/auth/obj_credential_common.h" + +namespace doris { + +// Provider-specific authentication lives under cpp/obj-client/auth/. Future +// providers should use sibling packages and expose only their normalized config, +// validation, and client-setup entry points there. + +inline constexpr std::string_view GCP_CREDENTIAL_PROVIDER_TYPE = "gs.credential_provider_type"; +inline constexpr std::string_view GCP_IMPERSONATION_SERVICE_ACCOUNT = + "gs.impersonation_service_account"; + +enum class GcpCredentialProviderType { + Default, + ComputeEngine, +}; + +struct GcpCredentialConfig { + GcpCredentialProviderType provider_type = GcpCredentialProviderType::Default; + std::string impersonation_service_account {}; + + bool operator==(const GcpCredentialConfig&) const = default; +}; + +struct GcpCredentialParseResult { + std::optional credential = std::nullopt; + std::optional error = std::nullopt; +}; + +bool is_valid_gcp_service_account_email(std::string_view email); +bool is_valid_gcp_storage_endpoint(std::string_view endpoint); +std::optional parse_gcp_credential_provider_type( + std::string_view provider_type); +std::optional validate_gcp_credential(const GcpCredentialConfig& credential); +GcpCredentialParseResult parse_gcp_credential_properties( + const ObjCredentialPropertyGetter& get_property); + +} // namespace doris diff --git a/common/cpp/obj-client/auth/gcp/gcp_s3_client.cpp b/common/cpp/obj-client/auth/gcp/gcp_s3_client.cpp new file mode 100644 index 00000000000000..fe553cc1ada851 --- /dev/null +++ b/common/cpp/obj-client/auth/gcp/gcp_s3_client.cpp @@ -0,0 +1,223 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "cpp/obj-client/auth/gcp/gcp_s3_client.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace doris { +namespace { + +Aws::Client::ClientConfiguration gcp_http_config(Aws::Client::ClientConfiguration config) { + // Do not forward bearer headers on a redirect or allow TLS verification to be disabled. + config.followRedirects = Aws::Client::FollowRedirectsPolicy::NEVER; + config.verifySSL = true; + config.scheme = Aws::Http::Scheme::HTTPS; + return config; +} + +bool trusted_endpoint(const Aws::Client::ClientConfiguration& config) { + std::string endpoint(config.endpointOverride.c_str()); + if (endpoint.find("://") == std::string::npos) { + endpoint = "https://" + endpoint; + } + return is_valid_gcp_storage_endpoint(endpoint); +} + +} // namespace + +GcpS3Client::GcpS3Client(const GcpCredentialConfig& credential, std::string ca_cert_path, + Aws::Client::ClientConfiguration aws_config, + Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy payload_signing_policy, + bool use_virtual_addressing) + : Aws::S3::S3Client(std::make_shared(), + gcp_http_config(aws_config), payload_signing_policy, + use_virtual_addressing), + _trusted_endpoint(trusted_endpoint(aws_config)), + _token_provider(credential, ca_cert_path) {} + +std::optional GcpS3Client::authorize_request( + Aws::AmazonWebServiceRequest& request, std::string_view bucket) const { + // GCS bucket names cannot contain ARN, URL or authority delimiters. In particular, + // do not let the AWS endpoint resolver route an access-point ARN to an AWS host. + bool valid_bucket = std::all_of(bucket.begin(), bucket.end(), [](unsigned char c) { + return (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9') || c == '-' || c == '_' || c == '.'; + }); + if (!_trusted_endpoint || !valid_bucket) { + Aws::Client::AWSError error( + Aws::S3::S3Errors::ACCESS_DENIED, "GcpEndpointError", + "Native GCP OAuth requires a trusted Google Cloud Storage HTTPS " + "endpoint on port 443 and a GCS bucket name", + false); + error.SetResponseCode(Aws::Http::HttpResponseCode::FORBIDDEN); + return Aws::S3::S3Error(std::move(error)); + } + auto token = fetch_token(); + if (!token.has_value() || token->empty()) { + // Return through the SDK outcome path before any object-storage HTTP request. + // The token provider has already logged the underlying credential error. + Aws::Client::AWSError error(Aws::S3::S3Errors::ACCESS_DENIED, + "GcpAuthenticationError", + "Failed to obtain GCP access token", false); + error.SetResponseCode(Aws::Http::HttpResponseCode::UNAUTHORIZED); + return Aws::S3::S3Error(std::move(error)); + } + request.SetAdditionalCustomHeaderValue("Authorization", + Aws::String("Bearer ") + token->c_str()); + return std::nullopt; +} + +Aws::S3::Model::AbortMultipartUploadOutcome GcpS3Client::AbortMultipartUpload( + const Aws::S3::Model::AbortMultipartUploadRequest& request) const { + auto authorized_request = request; + if (auto error = authorize_request(authorized_request, request.GetBucket())) { + return std::move(*error); + } + return Aws::S3::S3Client::AbortMultipartUpload(authorized_request); +} + +Aws::S3::Model::CompleteMultipartUploadOutcome GcpS3Client::CompleteMultipartUpload( + const Aws::S3::Model::CompleteMultipartUploadRequest& request) const { + auto authorized_request = request; + if (auto error = authorize_request(authorized_request, request.GetBucket())) { + return std::move(*error); + } + return Aws::S3::S3Client::CompleteMultipartUpload(authorized_request); +} + +Aws::S3::Model::CreateMultipartUploadOutcome GcpS3Client::CreateMultipartUpload( + const Aws::S3::Model::CreateMultipartUploadRequest& request) const { + auto authorized_request = request; + if (auto error = authorize_request(authorized_request, request.GetBucket())) { + return std::move(*error); + } + return Aws::S3::S3Client::CreateMultipartUpload(authorized_request); +} + +Aws::S3::Model::DeleteObjectOutcome GcpS3Client::DeleteObject( + const Aws::S3::Model::DeleteObjectRequest& request) const { + auto authorized_request = request; + if (auto error = authorize_request(authorized_request, request.GetBucket())) { + return std::move(*error); + } + return Aws::S3::S3Client::DeleteObject(authorized_request); +} + +Aws::S3::Model::DeleteObjectsOutcome GcpS3Client::DeleteObjects( + const Aws::S3::Model::DeleteObjectsRequest& request) const { + auto authorized_request = request; + if (auto error = authorize_request(authorized_request, request.GetBucket())) { + return std::move(*error); + } + return Aws::S3::S3Client::DeleteObjects(authorized_request); +} + +Aws::S3::Model::GetBucketLifecycleConfigurationOutcome GcpS3Client::GetBucketLifecycleConfiguration( + const Aws::S3::Model::GetBucketLifecycleConfigurationRequest& request) const { + auto authorized_request = request; + if (auto error = authorize_request(authorized_request, request.GetBucket())) { + return std::move(*error); + } + return Aws::S3::S3Client::GetBucketLifecycleConfiguration(authorized_request); +} + +Aws::S3::Model::GetBucketVersioningOutcome GcpS3Client::GetBucketVersioning( + const Aws::S3::Model::GetBucketVersioningRequest& request) const { + auto authorized_request = request; + if (auto error = authorize_request(authorized_request, request.GetBucket())) { + return std::move(*error); + } + return Aws::S3::S3Client::GetBucketVersioning(authorized_request); +} + +Aws::S3::Model::GetObjectOutcome GcpS3Client::GetObject( + const Aws::S3::Model::GetObjectRequest& request) const { + auto authorized_request = request; + if (auto error = authorize_request(authorized_request, request.GetBucket())) { + return std::move(*error); + } + return Aws::S3::S3Client::GetObject(authorized_request); +} + +Aws::S3::Model::HeadObjectOutcome GcpS3Client::HeadObject( + const Aws::S3::Model::HeadObjectRequest& request) const { + auto authorized_request = request; + if (auto error = authorize_request(authorized_request, request.GetBucket())) { + return std::move(*error); + } + return Aws::S3::S3Client::HeadObject(authorized_request); +} + +Aws::S3::Model::ListObjectsV2Outcome GcpS3Client::ListObjectsV2( + const Aws::S3::Model::ListObjectsV2Request& request) const { + auto authorized_request = request; + if (auto error = authorize_request(authorized_request, request.GetBucket())) { + return std::move(*error); + } + return Aws::S3::S3Client::ListObjectsV2(authorized_request); +} + +Aws::S3::Model::PutObjectOutcome GcpS3Client::PutObject( + const Aws::S3::Model::PutObjectRequest& request) const { + auto authorized_request = request; + if (auto error = authorize_request(authorized_request, request.GetBucket())) { + return std::move(*error); + } + return Aws::S3::S3Client::PutObject(authorized_request); +} + +Aws::S3::Model::UploadPartOutcome GcpS3Client::UploadPart( + const Aws::S3::Model::UploadPartRequest& request) const { + auto authorized_request = request; + if (auto error = authorize_request(authorized_request, request.GetBucket())) { + return std::move(*error); + } + return Aws::S3::S3Client::UploadPart(authorized_request); +} + +std::optional GcpS3Client::fetch_token() const { + return _token_provider.get_token(); +} + +} // namespace doris diff --git a/common/cpp/obj-client/auth/gcp/gcp_s3_client.h b/common/cpp/obj-client/auth/gcp/gcp_s3_client.h new file mode 100644 index 00000000000000..8d880ebf60f97f --- /dev/null +++ b/common/cpp/obj-client/auth/gcp/gcp_s3_client.h @@ -0,0 +1,93 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +#include +#include +#include + +#include "cpp/obj-client/auth/gcp/gcp_auth.h" +#include "cpp/obj-client/auth/gcp/gcp_token_provider.h" + +namespace doris { + +// GCS accepts OAuth2 bearer tokens on its S3-compatible XML API. The client uses +// anonymous AWS credentials so AWSAuthV4Signer returns without adding SigV4, and +// each object-storage operation obtains its bearer token before entering the SDK. +// Keep these overrides in sync with the operations used by S3ObjStorageClient. +class GcpS3Client : public Aws::S3::S3Client { +public: + GcpS3Client(const GcpCredentialConfig& credential, std::string ca_cert_path, + Aws::Client::ClientConfiguration aws_config, + Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy payload_signing_policy, + bool use_virtual_addressing); + + Aws::S3::Model::AbortMultipartUploadOutcome AbortMultipartUpload( + const Aws::S3::Model::AbortMultipartUploadRequest& request) const override; + + Aws::S3::Model::CompleteMultipartUploadOutcome CompleteMultipartUpload( + const Aws::S3::Model::CompleteMultipartUploadRequest& request) const override; + + Aws::S3::Model::CreateMultipartUploadOutcome CreateMultipartUpload( + const Aws::S3::Model::CreateMultipartUploadRequest& request) const override; + + Aws::S3::Model::DeleteObjectOutcome DeleteObject( + const Aws::S3::Model::DeleteObjectRequest& request) const override; + + Aws::S3::Model::DeleteObjectsOutcome DeleteObjects( + const Aws::S3::Model::DeleteObjectsRequest& request) const override; + + Aws::S3::Model::GetBucketLifecycleConfigurationOutcome GetBucketLifecycleConfiguration( + const Aws::S3::Model::GetBucketLifecycleConfigurationRequest& request) const override; + + Aws::S3::Model::GetBucketVersioningOutcome GetBucketVersioning( + const Aws::S3::Model::GetBucketVersioningRequest& request) const override; + + Aws::S3::Model::GetObjectOutcome GetObject( + const Aws::S3::Model::GetObjectRequest& request) const override; + + Aws::S3::Model::HeadObjectOutcome HeadObject( + const Aws::S3::Model::HeadObjectRequest& request) const override; + + Aws::S3::Model::ListObjectsV2Outcome ListObjectsV2( + const Aws::S3::Model::ListObjectsV2Request& request) const override; + + Aws::S3::Model::PutObjectOutcome PutObject( + const Aws::S3::Model::PutObjectRequest& request) const override; + + Aws::S3::Model::UploadPartOutcome UploadPart( + const Aws::S3::Model::UploadPartRequest& request) const override; + +protected: + std::optional authorize_request(Aws::AmazonWebServiceRequest& request, + std::string_view bucket) const; + + // Test seam for asserting the outgoing header without relying on ADC or metadata. + virtual std::optional fetch_token() const; + +private: + const bool _trusted_endpoint; + GcpTokenProvider _token_provider; +}; + +} // namespace doris diff --git a/common/cpp/obj-client/auth/gcp/gcp_token_provider.cpp b/common/cpp/obj-client/auth/gcp/gcp_token_provider.cpp new file mode 100644 index 00000000000000..42174b05e8ec98 --- /dev/null +++ b/common/cpp/obj-client/auth/gcp/gcp_token_provider.cpp @@ -0,0 +1,153 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "cpp/obj-client/auth/gcp/gcp_token_provider.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace doris { +namespace { + +constexpr char GCS_READ_WRITE_SCOPE[] = "https://www.googleapis.com/auth/devstorage.read_write"; +constexpr char CLOUD_PLATFORM_SCOPE[] = "https://www.googleapis.com/auth/cloud-platform"; + +std::shared_ptr make_default_storage_credentials( + const google::cloud::Options& options) { + namespace oauth2 = google::cloud::oauth2_internal; + auto path = oauth2::GoogleAdcFilePathFromEnvVarOrEmpty(); + if (path.empty()) { + path = oauth2::GoogleAdcFilePathFromWellKnownPathOrEmpty(); + } + std::ifstream input(path); + std::string contents(std::istreambuf_iterator {input}, {}); + rapidjson::Document document; + document.Parse(contents.data(), contents.size()); + // The generic SDK adapter emits scoped self-signed JWTs for service-account + // JSON keys, which GCS rejects. Keep other ADC types (and their errors) on + // the SDK's normal resolution path, including federation and VM identities. + if (document.HasParseError() || !document.IsObject() || !document.HasMember("type") || + !document["type"].IsString() || + std::string_view(document["type"].GetString(), document["type"].GetStringLength()) != + "service_account") { + return google::cloud::rest_internal::MapCredentials( + *google::cloud::MakeGoogleDefaultCredentials(options)); + } + auto info = oauth2::ParseServiceAccountCredentials(contents, path); + if (!info) { + return std::make_shared(info.status()); + } + info->enable_self_signed_jwt = false; + const auto& scopes = options.get(); + info->scopes = std::set(scopes.begin(), scopes.end()); + return oauth2::Decorate(std::make_shared( + *info, options, + [](const google::cloud::Options& opts) { + return google::cloud::rest_internal::MakeDefaultRestClient( + "", opts); + }), + options); +} + +std::shared_ptr make_credentials( + const GcpCredentialConfig& credential, const std::string& ca_cert_path, + GcpTokenScope token_scope) { + google::cloud::Options options; + if (!ca_cert_path.empty()) { + options.set(ca_cert_path); + } + + // A source credential used for impersonation calls the IAM Credentials API + // and therefore needs cloud-platform. The resulting target token uses the + // caller-selected scope: object read/write for storage requests, or + // cloud-platform when it will call IAM signBlob. + const char* requested_scope = token_scope == GcpTokenScope::CloudPlatform + ? CLOUD_PLATFORM_SCOPE + : GCS_READ_WRITE_SCOPE; + options.set({credential.impersonation_service_account.empty() + ? requested_scope + : CLOUD_PLATFORM_SCOPE}); + + std::shared_ptr credentials; + switch (credential.provider_type) { + case GcpCredentialProviderType::Default: + // A direct service-account ADC token goes to GCS. Impersonation sources + // instead authenticate to IAM, which accepts the SDK's self-signed JWTs. + if (credential.impersonation_service_account.empty()) { + return make_default_storage_credentials(options); + } + credentials = google::cloud::MakeGoogleDefaultCredentials(options); + break; + case GcpCredentialProviderType::ComputeEngine: + credentials = google::cloud::MakeComputeEngineCredentials(options); + break; + } + if (!credential.impersonation_service_account.empty()) { + auto impersonation_options = options; + impersonation_options.set({requested_scope}); + credentials = google::cloud::MakeImpersonateServiceAccountCredentials( + std::move(credentials), credential.impersonation_service_account, + std::move(impersonation_options)); + } + return google::cloud::rest_internal::MapCredentials(*credentials); +} + +} // namespace + +GcpTokenProvider::GcpTokenProvider(const GcpCredentialConfig& credential, + const std::string& ca_cert_path, GcpTokenScope token_scope) + : GcpTokenProvider(make_credentials(credential, ca_cert_path, token_scope)) {} + +GcpTokenProvider::GcpTokenProvider( + std::shared_ptr credentials) + : _credentials(std::move(credentials)) {} + +GcpTokenProvider::~GcpTokenProvider() = default; + +std::optional GcpTokenProvider::get_token() const { + // GcpS3Client and presign callers may share one provider across request + // threads. Serialize access even if the underlying implementation changes + // its thread-safety guarantees in a future google-cloud-cpp release. + std::lock_guard lock(_credentials_mutex); + auto token = _credentials->GetToken(std::chrono::system_clock::now()); + if (!token) { + LOG_EVERY_N(WARNING, 100) << "failed to obtain GCP access token: " + << token.status().message(); + return std::nullopt; + } + return token->token; +} + +std::string GcpTokenProvider::get_service_account_email() const { + std::lock_guard lock(_credentials_mutex); + return _credentials->AccountEmail(); +} + +} // namespace doris diff --git a/common/cpp/obj-client/auth/gcp/gcp_token_provider.h b/common/cpp/obj-client/auth/gcp/gcp_token_provider.h new file mode 100644 index 00000000000000..eb94166cc3fec4 --- /dev/null +++ b/common/cpp/obj-client/auth/gcp/gcp_token_provider.h @@ -0,0 +1,58 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include +#include +#include + +#include "cpp/obj-client/auth/gcp/gcp_auth.h" + +namespace doris { + +enum class GcpTokenScope { + StorageReadWrite, + CloudPlatform, +}; + +// Use one resolved credential for tokens and signing identity. The Google auth +// library handles ADC selection, token caching and refresh internally. +class GcpTokenProvider { +public: + explicit GcpTokenProvider(const GcpCredentialConfig& credential, + const std::string& ca_cert_path = "", + GcpTokenScope token_scope = GcpTokenScope::StorageReadWrite); + explicit GcpTokenProvider( + std::shared_ptr credentials); + ~GcpTokenProvider(); + + GcpTokenProvider(const GcpTokenProvider&) = delete; + GcpTokenProvider& operator=(const GcpTokenProvider&) = delete; + + std::optional get_token() const; + std::string get_service_account_email() const; + +private: + std::shared_ptr _credentials; + mutable std::mutex _credentials_mutex; +}; + +} // namespace doris diff --git a/common/cpp/obj-client/auth/gcp/gcs_signed_url.cpp b/common/cpp/obj-client/auth/gcp/gcs_signed_url.cpp new file mode 100644 index 00000000000000..3a5e18a953965c --- /dev/null +++ b/common/cpp/obj-client/auth/gcp/gcs_signed_url.cpp @@ -0,0 +1,193 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "cpp/obj-client/auth/gcp/gcs_signed_url.h" + +#include +#include + +#include +#include +#include +#include +#include + +#include "cpp/obj-client/auth/gcp/gcp_auth.h" + +namespace doris { +namespace { + +constexpr std::string_view SIGNING_ALGORITHM = "GOOG4-RSA-SHA256"; +constexpr std::string_view SIGNED_HEADERS = "host"; +constexpr std::string_view SIGNING_REGION = "auto"; +constexpr std::string_view SIGNING_SERVICE = "storage"; +constexpr std::string_view SIGNING_TERMINATOR = "goog4_request"; +constexpr int64_t MAX_EXPIRATION_SECONDS = 7 * 24 * 60 * 60; + +struct Endpoint { + std::string scheme; + std::string authority; +}; + +bool is_unreserved(unsigned char c) { + return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '-' || + c == '_' || c == '.' || c == '~'; +} + +std::string percent_encode(std::string_view value, bool preserve_slash) { + constexpr char HEX[] = "0123456789ABCDEF"; + std::string encoded; + encoded.reserve(value.size()); + for (unsigned char c : value) { + if (is_unreserved(c) || (preserve_slash && c == '/')) { + encoded.push_back(static_cast(c)); + continue; + } + encoded.push_back('%'); + encoded.push_back(HEX[c >> 4]); + encoded.push_back(HEX[c & 0x0F]); + } + return encoded; +} + +std::string hex_encode(std::string_view value) { + constexpr char HEX[] = "0123456789abcdef"; + std::string encoded; + encoded.reserve(value.size() * 2); + for (unsigned char c : value) { + encoded.push_back(HEX[c >> 4]); + encoded.push_back(HEX[c & 0x0F]); + } + return encoded; +} + +std::string sha256_hex(std::string_view value) { + std::array digest {}; + SHA256(reinterpret_cast(value.data()), value.size(), digest.data()); + return hex_encode( + std::string_view(reinterpret_cast(digest.data()), digest.size())); +} + +std::optional parse_endpoint(std::string endpoint, Endpoint* parsed) { + if (endpoint.empty()) { + return "GCS endpoint must not be empty"; + } + auto scheme_end = endpoint.find("://"); + if (scheme_end == std::string::npos) { + parsed->scheme = "https"; + } else { + parsed->scheme = endpoint.substr(0, scheme_end); + endpoint.erase(0, scheme_end + 3); + } + if (!is_valid_gcp_storage_endpoint(parsed->scheme + "://" + endpoint)) { + return "GCS signed URL requires a Google Storage HTTPS service-base endpoint without a " + "bucket"; + } + if (endpoint.ends_with('/')) { + endpoint.pop_back(); + } + // HTTP clients omit the default TLS port from Host. Sign the same authority. + if (endpoint.ends_with(":443")) { + endpoint.resize(endpoint.size() - 4); + } + parsed->authority = std::move(endpoint); + return std::nullopt; +} + +std::optional format_signing_time(std::chrono::system_clock::time_point now, + std::string* timestamp, std::string* date) { + auto time = std::chrono::system_clock::to_time_t(now); + std::tm utc {}; +#if defined(_WIN32) + if (gmtime_s(&utc, &time) != 0) { + return "failed to convert GCS signing time to UTC"; + } +#else + if (gmtime_r(&time, &utc) == nullptr) { + return "failed to convert GCS signing time to UTC"; + } +#endif + char timestamp_buffer[17]; + char date_buffer[9]; + if (std::strftime(timestamp_buffer, sizeof(timestamp_buffer), "%Y%m%dT%H%M%SZ", &utc) == 0 || + std::strftime(date_buffer, sizeof(date_buffer), "%Y%m%d", &utc) == 0) { + return "failed to format GCS signing time"; + } + *timestamp = timestamp_buffer; + *date = date_buffer; + return std::nullopt; +} + +} // namespace + +GcsV4SignedUrlResult build_gcs_v4_signed_url(const GcsV4SignedUrlOptions& options, + std::chrono::system_clock::time_point now, + const GcsSignBlobFunction& sign_blob) { + if (options.bucket.empty()) { + return {.error = "GCS bucket must not be empty"}; + } + if (!is_valid_gcp_service_account_email(options.signer_email)) { + return {.error = "invalid GCS signing service account email"}; + } + if (options.expiration_secs <= 0 || options.expiration_secs > MAX_EXPIRATION_SECONDS) { + return {.error = fmt::format("GCS signed URL expiration must be in [1, {}] seconds", + MAX_EXPIRATION_SECONDS)}; + } + if (!sign_blob) { + return {.error = "GCS signBlob callback must not be empty"}; + } + + Endpoint endpoint; + if (auto error = parse_endpoint(options.endpoint, &endpoint); error.has_value()) { + return {.error = std::move(*error)}; + } + std::string timestamp; + std::string date; + if (auto error = format_signing_time(now, ×tamp, &date); error.has_value()) { + return {.error = std::move(*error)}; + } + + const std::string canonical_uri = + "/" + percent_encode(options.bucket, false) + "/" + percent_encode(options.key, true); + const std::string credential_scope = + fmt::format("{}/{}/{}/{}", date, SIGNING_REGION, SIGNING_SERVICE, SIGNING_TERMINATOR); + const std::string credential = options.signer_email + "/" + credential_scope; + const std::string canonical_query = fmt::format( + "X-Goog-Algorithm={}&X-Goog-Credential={}&X-Goog-Date={}&" + "X-Goog-Expires={}&X-Goog-SignedHeaders={}", + SIGNING_ALGORITHM, percent_encode(credential, false), timestamp, + options.expiration_secs, SIGNED_HEADERS); + const std::string canonical_request = + fmt::format("GET\n{}\n{}\nhost:{}\n\n{}\nUNSIGNED-PAYLOAD", canonical_uri, + canonical_query, endpoint.authority, SIGNED_HEADERS); + const std::string string_to_sign = fmt::format("{}\n{}\n{}\n{}", SIGNING_ALGORITHM, timestamp, + credential_scope, sha256_hex(canonical_request)); + + auto sign_result = sign_blob(string_to_sign); + if (!sign_result.ok()) { + return {.error = std::move(sign_result.error)}; + } + if (sign_result.signature.empty()) { + return {.error = "signBlob returned an empty signature"}; + } + + return {.signed_url = fmt::format("{}://{}{}?{}&X-Goog-Signature={}", endpoint.scheme, + endpoint.authority, canonical_uri, canonical_query, + hex_encode(sign_result.signature))}; +} + +} // namespace doris diff --git a/common/cpp/obj-client/auth/gcp/gcs_signed_url.h b/common/cpp/obj-client/auth/gcp/gcs_signed_url.h new file mode 100644 index 00000000000000..8cc152925b4551 --- /dev/null +++ b/common/cpp/obj-client/auth/gcp/gcs_signed_url.h @@ -0,0 +1,62 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include + +namespace doris { + +struct GcsV4SignedUrlOptions { + std::string endpoint; + std::string bucket; + std::string key; + // The transport layer resolves this identity before invoking the builder. + std::string signer_email; + int64_t expiration_secs = 0; +}; + +struct GcsSignBlobResult { + std::string signature {}; + std::string error {}; + + bool ok() const noexcept { return error.empty(); } +}; + +// Returns the raw signature bytes for the supplied GCS V4 string-to-sign. The +// callback intentionally contains no transport-specific type so this signing +// algorithm can be shared by BE and cloud targets. +using GcsSignBlobFunction = std::function; + +struct GcsV4SignedUrlResult { + std::string signed_url {}; + std::string error {}; + + bool ok() const noexcept { return error.empty(); } +}; + +// Builds a GCS V4 signed URL from a caller-supplied signing function. Resolving +// credentials and invoking IAM signBlob belong to the caller's transport layer. +GcsV4SignedUrlResult build_gcs_v4_signed_url(const GcsV4SignedUrlOptions& options, + std::chrono::system_clock::time_point now, + const GcsSignBlobFunction& sign_blob); + +} // namespace doris diff --git a/common/cpp/obj-client/auth/obj_credential.cpp b/common/cpp/obj-client/auth/obj_credential.cpp new file mode 100644 index 00000000000000..8317059e34e890 --- /dev/null +++ b/common/cpp/obj-client/auth/obj_credential.cpp @@ -0,0 +1,134 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "cpp/obj-client/auth/obj_credential.h" + +#include +#include +#include + +namespace doris { +namespace { + +struct CredentialParseResult { + CredentialConfig credential {}; + std::optional error = std::nullopt; +}; + +// Adapt provider-owned parsers to the generic registry. A provider parser +// returns no credential when none of its properties are present and an error +// for a partially configured credential. +CredentialParseResult parse_gcp_properties(const ObjCredentialPropertyGetter& get_property) { + auto result = parse_gcp_credential_properties(get_property); + if (result.error.has_value()) { + return {.error = std::move(result.error)}; + } + if (!result.credential.has_value()) { + return {}; + } + return {.credential = std::move(*result.credential)}; +} + +using CredentialPropertyParser = + CredentialParseResult (*)(const ObjCredentialPropertyGetter& get_property); + +// Register exactly one parser per non-monostate CredentialConfig alternative. +// The loop below also rejects properties from multiple credential providers. +// Add parse_oss_credential_properties/parse_azure_credential_properties here +// when those passwordless modes are introduced. +constexpr std::array CREDENTIAL_PROPERTY_PARSERS { + CredentialPropertyParser {parse_gcp_properties}, +}; + +static_assert(CREDENTIAL_PROPERTY_PARSERS.size() + 1 == std::variant_size_v); + +// These visitors are the runtime extension points. Do not add a catch-all +// template overload: explicit overloads make a new CredentialConfig type fail +// to compile until its hashing and validation are implemented. +struct CredentialHashVisitor { + uint64_t* hash; + + void operator()(const std::monostate&) const {} + + void operator()(const GcpCredentialConfig& credential) const { + *hash ^= static_cast(credential.provider_type) + 1; + *hash ^= std::hash {}(credential.impersonation_service_account); + } +}; + +struct CredentialValidationVisitor { + const ObjCredentialValidationContext& context; + + std::optional operator()(const std::monostate&) const { return std::nullopt; } + + std::optional operator()(const GcpCredentialConfig& credential) const { + if (context.provider != ObjCredentialProvider::Gcp) { + return "GCP credentials require provider=GCP"; + } + if (context.has_conflicting_credentials()) { + return "GCP credentials cannot be combined with other credentials"; + } + return validate_gcp_credential(credential); + } +}; + +struct CredentialTypeNameVisitor { + std::string_view operator()(const std::monostate&) const { return "none"; } + std::string_view operator()(const GcpCredentialConfig&) const { return "gcp"; } +}; + +} // namespace + +void hash_combine_credential(uint64_t* hash, const CredentialConfig& credential) { + *hash ^= credential.index(); + std::visit(CredentialHashVisitor {hash}, credential); +} + +std::string_view credential_type_name(const CredentialConfig& credential) { + return std::visit(CredentialTypeNameVisitor {}, credential); +} + +std::optional validate_obj_credential_config( + const CredentialConfig& credential, const ObjCredentialValidationContext& context) { + return std::visit(CredentialValidationVisitor {context}, credential); +} + +std::optional parse_obj_credential_properties( + const ObjCredentialPropertyGetter& get_property, CredentialConfig* credential) { + CredentialConfig selected; + + for (auto parser : CREDENTIAL_PROPERTY_PARSERS) { + auto result = parser(get_property); + if (result.error.has_value()) { + return std::move(result.error); + } + if (std::holds_alternative(result.credential)) { + continue; + } + if (!std::holds_alternative(selected)) { + return "multiple object storage credential types are configured"; + } + selected = std::move(result.credential); + } + + if (!std::holds_alternative(selected)) { + *credential = std::move(selected); + } + return std::nullopt; +} + +} // namespace doris diff --git a/common/cpp/obj-client/auth/obj_credential.h b/common/cpp/obj-client/auth/obj_credential.h new file mode 100644 index 00000000000000..6d32385588d5f8 --- /dev/null +++ b/common/cpp/obj-client/auth/obj_credential.h @@ -0,0 +1,70 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "cpp/obj-client/auth/gcp/gcp_auth.h" +#include "cpp/obj-client/auth/obj_credential_common.h" + +namespace doris { + +class TCredential; + +namespace cloud { +class ObjectStoreInfoPB; +} + +// This abstraction keeps provider-specific passwordless authentication out of +// S3 utilities and provides one runtime credential model shared by BE and Cloud. +// To add another provider (for example OSS or Azure): +// 1. add CredentialConfig in its own _auth.h; +// 2. add that type to this variant; and +// 3. implement the corresponding hash/validate visitors, client builder, and +// protocol/property conversions. +// The explicit visitors intentionally make an incomplete extension fail to compile. +using CredentialConfig = std::variant; + +void hash_combine_credential(uint64_t* hash, const CredentialConfig& credential); + +std::string_view credential_type_name(const CredentialConfig& credential); + +std::optional validate_obj_credential_config( + const CredentialConfig& credential, const ObjCredentialValidationContext& context); + +std::optional parse_obj_credential_properties( + const ObjCredentialPropertyGetter& get_property, CredentialConfig* credential); + +template +std::optional parse_obj_credential_properties( + const std::map& properties, + CredentialConfig* credential) { + return parse_obj_credential_properties(make_property_getter(properties), credential); +} + +void convert_obj_credential(const cloud::ObjectStoreInfoPB& source, CredentialConfig* credential); +void convert_obj_credential(const TCredential& source, CredentialConfig* credential); + +} // namespace doris diff --git a/common/cpp/obj-client/auth/obj_credential_common.h b/common/cpp/obj-client/auth/obj_credential_common.h new file mode 100644 index 00000000000000..0e855fe57f1354 --- /dev/null +++ b/common/cpp/obj-client/auth/obj_credential_common.h @@ -0,0 +1,88 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +namespace doris { + +// Credential-layer provider identity. Storage and protocol enums are converted +// to this type only at their respective validation boundaries. +enum class ObjCredentialProvider { + Unknown, + Aws, + Azure, + Bos, + Cos, + Gcp, + Obs, + Oss, + Tos, +}; + +// TODO: Add AwsCredentialConfig to CredentialConfig so AWS uses the same +// provider-neutral path. The AWS-specific role ARN, external ID, and credential- +// provider fields remain outside this abstraction for compatibility with existing +// configurations and protocol clients. AK/SK and session tokens are common static +// credentials for S3-compatible providers and are not part of this migration. +struct ObjCredentialValidationContext { + ObjCredentialProvider provider = ObjCredentialProvider::Unknown; + std::string_view ak {}; + std::string_view sk {}; + std::string_view token {}; + bool has_aws_role_arn = false; + bool has_aws_external_id = false; + bool has_aws_credential_provider = false; + bool has_encrypted_access_keys = false; + + bool has_static_credentials() const { + return !ak.empty() || !sk.empty() || !token.empty() || has_encrypted_access_keys; + } + + bool has_aws_authentication() const { + return has_aws_role_arn || has_aws_external_id || has_aws_credential_provider; + } + + // A provider-native credential is mutually exclusive with both common + // static credentials and provider-specific authentication fields. + bool has_conflicting_credentials() const { + return has_static_credentials() || has_aws_authentication(); + } +}; + +// Return an owned string so provider parsers never depend on a caller's storage +// or on the lifetime of a temporary normalized property value. +using ObjCredentialPropertyGetter = std::function(std::string_view)>; + +template +ObjCredentialPropertyGetter make_property_getter(const PropertyMap& properties) { + // Preserve the caller's comparator (including StringCaseMap's + // case-insensitive lookup) instead of normalizing property names here. + return [&properties](std::string_view key) -> std::optional { + auto it = properties.find(std::string(key)); + if (it == properties.end()) { + return std::nullopt; + } + return it->second; + }; +} + +} // namespace doris diff --git a/common/cpp/obj-client/auth/obj_s3_client_factory.cpp b/common/cpp/obj-client/auth/obj_s3_client_factory.cpp new file mode 100644 index 00000000000000..45ffa7e3bd4159 --- /dev/null +++ b/common/cpp/obj-client/auth/obj_s3_client_factory.cpp @@ -0,0 +1,54 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "cpp/obj-client/auth/obj_s3_client_factory.h" + +#include + +#include + +#include "cpp/obj-client/auth/gcp/gcp_s3_client.h" + +namespace doris { +namespace { + +// Keep the overloads explicit so adding another CredentialConfig alternative +// fails to compile until its client construction path is implemented here. +struct S3ClientBuilder { + S3ClientBuildContext& context; + + std::shared_ptr operator()(const std::monostate&) const { + return std::make_shared( + context.fallback_provider, std::move(context.config), + context.payload_signing_policy, context.use_virtual_addressing); + } + + std::shared_ptr operator()(const GcpCredentialConfig& credential) const { + return std::make_shared( + credential, context.ca_cert_path, std::move(context.config), + context.payload_signing_policy, context.use_virtual_addressing); + } +}; + +} // namespace + +std::shared_ptr make_s3_client(const CredentialConfig& credential, + S3ClientBuildContext context) { + return std::visit(S3ClientBuilder {context}, credential); +} + +} // namespace doris diff --git a/common/cpp/obj-client/auth/obj_s3_client_factory.h b/common/cpp/obj-client/auth/obj_s3_client_factory.h new file mode 100644 index 00000000000000..bd4d104de958cb --- /dev/null +++ b/common/cpp/obj-client/auth/obj_s3_client_factory.h @@ -0,0 +1,50 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include +#include + +#include "cpp/obj-client/auth/obj_credential.h" + +namespace Aws::S3 { +class S3Client; +} + +namespace doris { + +// Neutral inputs shared by BE's S3ClientConf and Cloud's S3Conf. Provider-native +// authentication is selected before construction because AWS credentials providers +// cannot be replaced safely on an already constructed client. +struct S3ClientBuildContext { + std::shared_ptr fallback_provider; + Aws::Client::ClientConfiguration config; + Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy payload_signing_policy = + Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never; + bool use_virtual_addressing = true; + std::string ca_cert_path; +}; + +std::shared_ptr make_s3_client(const CredentialConfig& credential, + S3ClientBuildContext context); + +} // namespace doris diff --git a/common/cpp/obj-client/s3_obj_storage_client.cpp b/common/cpp/obj-client/s3_obj_storage_client.cpp index df9210f612ee38..ceea3cfea6a0c8 100644 --- a/common/cpp/obj-client/s3_obj_storage_client.cpp +++ b/common/cpp/obj-client/s3_obj_storage_client.cpp @@ -501,6 +501,9 @@ ObjStorageResponse S3ObjStorageClient::delete_object(const ObjStoragePath& opts) std::string S3ObjStorageClient::generate_presigned_url(const ObjStoragePath& opts, int64_t expiration_secs) { + if (_signed_url_generator) { + return _signed_url_generator(opts, expiration_secs); + } return _client->GeneratePresignedUrl(opts.bucket, opts.key, Aws::Http::HttpMethod::HTTP_GET, expiration_secs); } diff --git a/common/cpp/obj-client/s3_obj_storage_client.h b/common/cpp/obj-client/s3_obj_storage_client.h index 1f18bde7431ba1..dc45530fe4ddb0 100644 --- a/common/cpp/obj-client/s3_obj_storage_client.h +++ b/common/cpp/obj-client/s3_obj_storage_client.h @@ -67,6 +67,7 @@ #include #include +#include #include #include @@ -89,9 +90,14 @@ ObjStorageStatus s3fs_error(const Aws::S3::S3Error& err, std::string_view msg); class S3ObjStorageClient final : public ObjStorageClient { public: + using SignedUrlGenerator = std::function; + S3ObjStorageClient(std::shared_ptr client, - ObjStorageEndpointInfo config = {}) - : _config(std::move(config)), _client(std::move(client)) {} + ObjStorageEndpointInfo config = {}, + SignedUrlGenerator signed_url_generator = {}) + : _config(std::move(config)), + _client(std::move(client)), + _signed_url_generator(std::move(signed_url_generator)) {} ~S3ObjStorageClient() override = default; ObjStorageUploadResult create_multipart_upload(const ObjStoragePath& opts) override; ObjStorageResponse put_object(const ObjStoragePath& opts, std::string_view stream) override; @@ -125,6 +131,7 @@ class S3ObjStorageClient final : public ObjStorageClient { private: ObjStorageEndpointInfo _config; std::shared_ptr _client; + SignedUrlGenerator _signed_url_generator; }; } // namespace doris diff --git a/fe/be-java-extensions/hadoop-deps/pom.xml b/fe/be-java-extensions/hadoop-deps/pom.xml index 0f7c94cf621eca..ef8b3a89a66f87 100644 --- a/fe/be-java-extensions/hadoop-deps/pom.xml +++ b/fe/be-java-extensions/hadoop-deps/pom.xml @@ -73,6 +73,11 @@ under the License. org.apache.hadoop hadoop-common + + com.google.cloud.bigdataoss + gcs-connector + shaded + org.apache.hadoop hadoop-azure diff --git a/fe/be-java-extensions/iceberg-metadata-scanner/pom.xml b/fe/be-java-extensions/iceberg-metadata-scanner/pom.xml index 8b63a2428d4b92..7e27fab2c35f05 100644 --- a/fe/be-java-extensions/iceberg-metadata-scanner/pom.xml +++ b/fe/be-java-extensions/iceberg-metadata-scanner/pom.xml @@ -41,6 +41,18 @@ under the License. + + + com.google.cloud.bigdataoss + gcs-connector + shaded + runtime + + + org.apache.doris + fe-iceberg-common + ${revision} + org.apache.doris jni-spi diff --git a/fe/be-java-extensions/iceberg-metadata-scanner/src/test/java/org/apache/doris/iceberg/IcebergPluginTest.java b/fe/be-java-extensions/iceberg-metadata-scanner/src/test/java/org/apache/doris/iceberg/IcebergPluginTest.java index 0b82ae6f898779..7965c280b42f21 100644 --- a/fe/be-java-extensions/iceberg-metadata-scanner/src/test/java/org/apache/doris/iceberg/IcebergPluginTest.java +++ b/fe/be-java-extensions/iceberg-metadata-scanner/src/test/java/org/apache/doris/iceberg/IcebergPluginTest.java @@ -32,6 +32,8 @@ import org.apache.iceberg.Schema; import org.apache.iceberg.Table; import org.apache.iceberg.TableScan; +import org.apache.iceberg.aws.s3.S3FileIO; +import org.apache.iceberg.aws.s3.S3FileIOProperties; import org.apache.iceberg.hadoop.HadoopTables; import org.apache.iceberg.io.CloseableIterable; import org.apache.iceberg.types.Types; @@ -88,6 +90,40 @@ private static DorisPlugin loadPlugin() { } /** The path the plugin registry takes: services file, plugin class, factory list. */ + @Test + public void reconstructsNativeGcpFileIoAfterSerialization() throws Exception { + Map properties = new HashMap<>(); + properties.put(S3FileIOProperties.CLIENT_FACTORY, + "org.apache.doris.connector.iceberg.GcpS3FileIOAwsClientFactory"); + properties.put("provider", "GCP"); + properties.put("s3.endpoint", "https://storage.googleapis.com"); + properties.put("s3.region", "us-central1"); + properties.put("gs.credential_provider_type", "COMPUTE_ENGINE"); + properties.put("gs.impersonation_service_account", "target@project.iam.gserviceaccount.com"); + try (S3FileIO original = new S3FileIO()) { + original.initialize(properties); + // Metadata tasks carry this FileIO across to BE, where the first manifest read rebuilds its client. + String wire = SerializationUtil.serializeToBase64(original); + try (S3FileIO restored = IcebergSerializationCompat.deserializeFromBase64(wire)) { + Assertions.assertEquals(properties, restored.properties()); + Assertions.assertNotNull(restored.client()); + Assertions.assertEquals("s3://bucket/manifest.avro", + restored.newInputFile("s3://bucket/manifest.avro").location()); + } + } + } + + @Test + public void instantiatesNativeGcsFilesystem() throws Exception { + Configuration conf = new Configuration(); + conf.set("fs.gs.impl", "com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem"); + // This only checks construction. No resources are opened until initialize(), and close() + // requires the filesystem root that initialize() sets up. + FileSystem fs = FileSystem.getFileSystemClass("gs", conf).getDeclaredConstructor().newInstance(); + Assertions.assertEquals("com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem", + fs.getClass().getName()); + } + @Test public void isDiscoverableThroughServiceLoader() { Assertions.assertTrue(loadPlugin() instanceof IcebergPlugin); @@ -132,6 +168,7 @@ public void resolvesEveryFilesystemSchemeAScanCanArriveOn() throws IOException { expected.put("hdfs", "org.apache.hadoop.hdfs.DistributedFileSystem"); expected.put("webhdfs", "org.apache.hadoop.hdfs.web.WebHdfsFileSystem"); expected.put("s3a", "org.apache.hadoop.fs.s3a.S3AFileSystem"); + expected.put("gs", "com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem"); expected.put("obs", "org.apache.hadoop.fs.obs.OBSFileSystem"); expected.put("abfs", "org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem"); expected.put("wasb", "org.apache.hadoop.fs.azure.NativeAzureFileSystem"); diff --git a/fe/fe-connector/fe-connector-iceberg/pom.xml b/fe/fe-connector/fe-connector-iceberg/pom.xml index 282499f29883f5..d17f5c467e0d45 100644 --- a/fe/fe-connector/fe-connector-iceberg/pom.xml +++ b/fe/fe-connector/fe-connector-iceberg/pom.xml @@ -41,6 +41,23 @@ under the License. + + org.apache.doris + fe-filesystem-gcs-common + ${revision} + + + + org.apache.doris + fe-iceberg-common + ${revision} + + + + com.google.cloud.bigdataoss + gcs-connector + shaded + ${project.groupId} fe-connector-spi diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergCatalogFactory.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergCatalogFactory.java index 9620d6fec20840..4f75779e494c51 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergCatalogFactory.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergCatalogFactory.java @@ -22,6 +22,9 @@ import org.apache.doris.connector.metastore.iceberg.jdbc.IcebergJdbcMetaStoreProperties; import org.apache.doris.connector.metastore.iceberg.rest.IcebergRestMetaStoreProperties; import org.apache.doris.connector.spi.DorisConnectorException; +import org.apache.doris.filesystem.gcs.auth.GcpCredential; +import org.apache.doris.filesystem.gcs.auth.GcsAuth; +import org.apache.doris.filesystem.gcs.auth.GcsAuthResolver; import org.apache.doris.filesystem.properties.S3CompatibleFileSystemProperties; import org.apache.doris.filesystem.properties.StorageProperties; @@ -450,6 +453,40 @@ public static Map buildCatalogProperties(IcebergCatalogPropertie // s3tables: bespoke instantiation. Preserve the skeleton's base+impl routing. break; } + chosenS3.ifPresent(storage -> storage.toBackendProperties().ifPresent(backend -> { + Optional auth = GcsAuthResolver.resolve(backend.toMap()); + if (auth.filter(GcsAuth::isAnonymous).isPresent()) { + // Glue and REST metadata can require AWS authentication even when GCS objects are public. + opts.put(S3FileIOProperties.CLIENT_FACTORY, AnonymousS3FileIOAwsClientFactory.class.getName()); + } + auth.flatMap(GcsAuth::getNativeCredential).ifPresent(credential -> { + // HadoopCatalog resolves its namespace filesystem independently of S3FileIO. + // Native GCS credentials configure fs.gs.*, so compatibility warehouse schemes + // must select that same filesystem before HadoopCatalog initializes. + String warehouse = opts.get(CatalogProperties.WAREHOUSE_LOCATION); + if (IcebergCatalogProperties.TYPE_HADOOP.equals(flavor) && warehouse != null) { + if (warehouse.regionMatches(true, 0, "s3://", 0, 5)) { + opts.put(CatalogProperties.WAREHOUSE_LOCATION, "gs://" + warehouse.substring(5)); + } else if (warehouse.regionMatches(true, 0, "s3a://", 0, 6)) { + opts.put(CatalogProperties.WAREHOUSE_LOCATION, "gs://" + warehouse.substring(6)); + } + } + // Glue already emitted the object-store settings. Its shared client.region belongs + // to the metadata client; the GCP FileIO factory resolves its own storage region. + if (!IcebergCatalogProperties.TYPE_GLUE.equals(flavor)) { + putS3FileIODialect(opts, storage); + } + opts.put("provider", "GCP"); + opts.put(GcpCredential.CREDENTIAL_PROVIDER_TYPE, credential.getCredentialProviderType().name()); + putIfNotBlank(opts, GcpCredential.IMPERSONATION_SERVICE_ACCOUNT, + credential.getImpersonationServiceAccount()); + // Keep explicit FileIO choices. By default, S3 aliases use the GCP client factory, + // while gs/HDFS paths use HadoopFileIO with the native fs.gs.* configuration + // (the plugin does not bundle iceberg-gcp's GCSFileIO). + opts.putIfAbsent(CatalogProperties.FILE_IO_IMPL, "org.apache.iceberg.io.ResolvingFileIO"); + opts.put(S3FileIOProperties.CLIENT_FACTORY, GcpS3FileIOAwsClientFactory.class.getName()); + }); + })); // The iceberg SDK forbids both "type" and "catalog-impl"; legacy buildIcebergCatalog removes "type". opts.remove(CatalogUtil.ICEBERG_CATALOG_TYPE); return opts; diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnector.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnector.java index afc79d395440d9..2d372f21eadb0d 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnector.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnector.java @@ -1083,16 +1083,13 @@ private Catalog createCatalog() { catalogOptions.get("client-pool-cache-keys"))); break; } - case IcebergCatalogProperties.TYPE_GLUE: - // Legacy IcebergGlueMetaStoreProperties builds the catalog with conf=null. - conf = null; - break; case IcebergCatalogProperties.TYPE_JDBC: maybeRegisterJdbcDriver(); conf = IcebergCatalogFactory.buildHadoopConfiguration(properties, storageHadoopConfig); break; default: - // rest / hadoop: a storage Configuration from the fe-filesystem-bound storage + raw + // rest / hadoop / glue: FileIO needs the bound storage configuration, including + // native GCS credentials when ResolvingFileIO falls back to HadoopFileIO, plus raw // fs./dfs./hadoop. passthrough. conf = IcebergCatalogFactory.buildHadoopConfiguration(properties, storageHadoopConfig); break; diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergCatalogFactoryTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergCatalogFactoryTest.java index 3fc0f6c3d9934f..4fae7f3286e324 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergCatalogFactoryTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergCatalogFactoryTest.java @@ -21,13 +21,25 @@ import org.apache.doris.connector.metastore.iceberg.jdbc.IcebergJdbcMetaStoreProperties; import org.apache.doris.connector.metastore.iceberg.rest.IcebergRestMetaStoreProperties; import org.apache.doris.connector.spi.DorisConnectorException; +import org.apache.doris.filesystem.gcs.GcsFileSystemProperties; import org.apache.doris.filesystem.properties.S3CompatibleFileSystemProperties; import org.apache.doris.filesystem.properties.StorageProperties; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RawLocalFileSystem; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.iceberg.CatalogUtil; import org.apache.iceberg.aws.AwsClientProperties; import org.apache.iceberg.aws.HttpClientProperties; +import org.apache.iceberg.aws.s3.S3FileIOProperties; +import org.apache.iceberg.aws.s3.S3InputFile; +import org.apache.iceberg.hadoop.HadoopFileIO; +import org.apache.iceberg.hadoop.HadoopInputFile; +import org.apache.iceberg.io.FileIO; +import org.apache.iceberg.io.InputFile; +import org.apache.iceberg.io.SeekableInputStream; import org.apache.iceberg.rest.responses.ConfigResponse; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -40,6 +52,8 @@ import software.amazon.awssdk.auth.credentials.WebIdentityTokenFileCredentialsProvider; import software.amazon.awssdk.services.s3.S3Client; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -1111,4 +1125,117 @@ public void assembleHiveConfFailsLoudOnMissingFile(@TempDir java.nio.file.Path t } } + @Test + void nativeGcpCatalogUsesBearerFileIoAndNativeHadoopConfiguration() { + for (String source : new String[] {"DEFAULT", "COMPUTE_ENGINE"}) { + Map raw = Map.of("provider", "GCP", "warehouse", "gs://bucket/warehouse", + "gs.endpoint", "https://storage.googleapis.com", + "gs.credential_provider_type", source, + "gs.impersonation_service_account", "target@test.iam.gserviceaccount.com"); + GcsFileSystemProperties storage = GcsFileSystemProperties.of(raw); + for (String catalogType : List.of("hadoop", "rest", "jdbc", "hms")) { + Map properties = new HashMap<>(raw); + properties.put("iceberg.catalog.type", catalogType); + properties.put("uri", "jdbc".equals(catalogType) ? "jdbc:postgresql://localhost/iceberg" + : "hms".equals(catalogType) ? "thrift://localhost:9083" : "http://localhost:8181"); + properties.put("jdbc.user", "user"); + properties.put("jdbc.password", "password"); + properties.put("iceberg.jdbc.uri", "jdbc:postgresql://localhost/iceberg"); + Map options = IcebergCatalogFactory.buildCatalogProperties( + IcebergCatalogProperties.of(properties), Optional.of(storage), false); + Assertions.assertEquals("org.apache.iceberg.io.ResolvingFileIO", options.get("io-impl")); + Assertions.assertEquals(GcpS3FileIOAwsClientFactory.class.getName(), + options.get(S3FileIOProperties.CLIENT_FACTORY)); + Assertions.assertEquals(source, options.get("gs.credential_provider_type")); + Assertions.assertEquals("https://storage.googleapis.com", options.get("s3.endpoint")); + Assertions.assertEquals("target@test.iam.gserviceaccount.com", + options.get("gs.impersonation_service_account")); + } + Assertions.assertEquals("com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem", + storage.toHadoopConfigurationMap().get("fs.gs.impl")); + } + } + + @Test + void nativeGcpCatalogPreservesExplicitFileIo(@TempDir java.nio.file.Path temp) throws Exception { + Map raw = new HashMap<>(Map.of("provider", "GCP", + "warehouse", "gs://bucket/warehouse", "iceberg.catalog.type", "rest", + "uri", "http://localhost:8181", "gs.credential_provider_type", "COMPUTE_ENGINE", + "io-impl", HadoopFileIO.class.getName())); + GcsFileSystemProperties storage = GcsFileSystemProperties.of(raw); + Map options = IcebergCatalogFactory.buildCatalogProperties( + IcebergCatalogProperties.of(raw), Optional.of(storage), false); + Assertions.assertEquals(HadoopFileIO.class.getName(), options.get("io-impl")); + Configuration conf = IcebergCatalogFactory.buildHadoopConfiguration(raw, + storage.toHadoopConfigurationMap()); + java.nio.file.Path metadata = temp.resolve("metadata.json"); + Files.writeString(metadata, "metadata", StandardCharsets.UTF_8); + try (FileIO io = CatalogUtil.loadFileIO(options.get("io-impl"), options, conf); + SeekableInputStream in = io.newInputFile(metadata.toUri().toString()).newStream()) { + Assertions.assertEquals("metadata", new String(in.readAllBytes(), StandardCharsets.UTF_8)); + } + } + + @Test + void nativeGcpRestCatalogRoutesMixedLocationsByScheme() throws Exception { + Map raw = Map.of("provider", "GCP", "warehouse", "gs://bucket/warehouse", + "iceberg.catalog.type", "rest", "uri", "http://localhost:8181", + "gs.credential_provider_type", "COMPUTE_ENGINE", + "gs.impersonation_service_account", "target@test.iam.gserviceaccount.com"); + GcsFileSystemProperties storage = GcsFileSystemProperties.of(raw); + Map options = IcebergCatalogFactory.buildCatalogProperties( + IcebergCatalogProperties.of(raw), Optional.of(storage), false); + Configuration conf = IcebergCatalogFactory.buildHadoopConfiguration(raw, + storage.toHadoopConfigurationMap()); + // Replace the Hadoop transports only, so the real FileIO dispatch runs without cloud/HDFS services. + for (String scheme : List.of("gs", "hdfs")) { + conf.set("fs." + scheme + ".impl", RawLocalFileSystem.class.getName()); + conf.setBoolean("fs." + scheme + ".impl.disable.cache", true); + } + try (FileIO io = CatalogUtil.loadFileIO(options.get("io-impl"), options, conf)) { + for (String location : List.of("gs://bucket/metadata.json", "hdfs://namenode/metadata.json")) { + InputFile input = io.newInputFile(location, 10); + Assertions.assertInstanceOf(HadoopInputFile.class, input, location); + HadoopInputFile hadoopInput = (HadoopInputFile) input; + try (FileSystem fs = hadoopInput.getFileSystem()) { + Assertions.assertInstanceOf(RawLocalFileSystem.class, fs); + Assertions.assertEquals("COMPUTE_ENGINE", fs.getConf().get("fs.gs.auth.type")); + Assertions.assertEquals("target@test.iam.gserviceaccount.com", + fs.getConf().get("fs.gs.auth.impersonation.service.account")); + } + } + for (String scheme : List.of("s3", "s3a")) { + Assertions.assertInstanceOf(S3InputFile.class, io.newInputFile(scheme + "://bucket/metadata.json", 10)); + } + } + } + + @Test + void nativeGcpWarehouseAliasesSelectGoogleHadoopFileSystem() throws Exception { + for (String scheme : List.of("gs", "s3", "s3a", "S3A")) { + Map raw = Map.of("provider", "GCP", "warehouse", scheme + "://bucket/warehouse", + "iceberg.catalog.type", "hadoop", "gs.credential_provider_type", "COMPUTE_ENGINE"); + GcsFileSystemProperties storage = GcsFileSystemProperties.of(raw); + Map options = IcebergCatalogFactory.buildCatalogProperties( + IcebergCatalogProperties.of(raw), Optional.of(storage), false); + String warehouse = options.get("warehouse"); + Assertions.assertEquals("gs://bucket/warehouse", warehouse); + Configuration conf = IcebergCatalogFactory.buildHadoopConfiguration(raw, + storage.toHadoopConfigurationMap()); + Assertions.assertEquals("com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem", + FileSystem.getFileSystemClass(new Path(warehouse).toUri().getScheme(), conf).getName()); + } + } + + @Test + void anonymousGcpCatalogUsesUnsignedFileIo() { + Map raw = Map.of("provider", "GCP", "warehouse", "gs://bucket/warehouse", + "iceberg.catalog.type", "hadoop", "gs.credential_provider_type", "ANONYMOUS"); + Map options = IcebergCatalogFactory.buildCatalogProperties( + IcebergCatalogProperties.of(raw), + Optional.of(GcsFileSystemProperties.of(raw)), false); + Assertions.assertEquals(AnonymousS3FileIOAwsClientFactory.class.getName(), + options.get(S3FileIOProperties.CLIENT_FACTORY)); + Assertions.assertFalse(options.containsKey(AwsClientProperties.CLIENT_CREDENTIALS_PROVIDER)); + } } diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorGlueFileIOTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorGlueFileIOTest.java new file mode 100644 index 00000000000000..2fb33077bf0a50 --- /dev/null +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorGlueFileIOTest.java @@ -0,0 +1,110 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.iceberg; + +import org.apache.doris.filesystem.gcs.GcsFileSystemProperties; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.RawLocalFileSystem; +import org.apache.iceberg.TableOperations; +import org.apache.iceberg.aws.glue.GlueCatalog; +import org.apache.iceberg.catalog.TableIdentifier; +import org.apache.iceberg.hadoop.HadoopInputFile; +import org.apache.iceberg.io.FileIO; +import org.apache.iceberg.io.ResolvingFileIO; +import org.apache.iceberg.io.SeekableInputStream; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.net.URI; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +class IcebergConnectorGlueFileIOTest { + + @Test + void gluePassesNativeGcsConfigurationToHadoopFileIO(@TempDir java.nio.file.Path temp) throws Exception { + java.nio.file.Path metadata = temp.resolve("metadata.json"); + Files.writeString(metadata, "metadata", StandardCharsets.UTF_8); + for (String source : List.of("DEFAULT", "COMPUTE_ENGINE")) { + for (String account : List.of("", "reader@test.iam.gserviceaccount.com")) { + Map properties = new HashMap<>(Map.of( + "iceberg.catalog.type", "glue", + "provider", "GCP", + "warehouse", "gs://test-bucket/warehouse", + "gs.credential_provider_type", source, + "gs.impersonation_service_account", account, + "glue.access_key", "test-ak", + "glue.secret_key", "test-sk", + "glue.region", "us-east-1", + "glue.endpoint", "http://127.0.0.1:1")); + // Replace only the Hadoop transport. Keep the real connector, GlueCatalog, + // table operations and ResolvingFileIO -> HadoopFileIO dispatch. + properties.put("fs.gs.impl", LocalGcsFileSystem.class.getName()); + properties.put("fs.gs.impl.disable.cache", "true"); + RecordingConnectorContext context = new RecordingConnectorContext(); + context.storageProperties = List.of(GcsFileSystemProperties.of(properties)); + try (IcebergConnector connector = new IcebergConnector(properties, context)) { + connector.getMetadata(null); + Field catalogField = IcebergConnector.class.getDeclaredField("icebergCatalog"); + catalogField.setAccessible(true); + GlueCatalog catalog = (GlueCatalog) catalogField.get(connector); + // Construct table IO without contacting Glue to fetch table metadata. + Method newTableOps = GlueCatalog.class.getDeclaredMethod("newTableOps", TableIdentifier.class); + newTableOps.setAccessible(true); + TableOperations operations = (TableOperations) newTableOps.invoke( + catalog, TableIdentifier.of("db", "table")); + FileIO io = operations.io(); + Assertions.assertInstanceOf(ResolvingFileIO.class, io); + Configuration conf = ((ResolvingFileIO) io).getConf(); + Assertions.assertNotNull(conf, "Glue must receive the bound storage Hadoop configuration"); + String authType = "DEFAULT".equals(source) ? "APPLICATION_DEFAULT" : "COMPUTE_ENGINE"; + Assertions.assertEquals(authType, conf.get("fs.gs.auth.type")); + Assertions.assertEquals(account, conf.get("fs.gs.auth.impersonation.service.account")); + Assertions.assertEquals("https://storage.googleapis.com/", conf.get("fs.gs.storage.root.url")); + + HadoopInputFile input = Assertions.assertInstanceOf(HadoopInputFile.class, + io.newInputFile(new Path("gs", "test-bucket", metadata.toString()).toString())); + try (FileSystem fs = input.getFileSystem(); SeekableInputStream stream = input.newStream()) { + Assertions.assertInstanceOf(LocalGcsFileSystem.class, fs); + Assertions.assertEquals(authType, fs.getConf().get("fs.gs.auth.type")); + Assertions.assertEquals(account, + fs.getConf().get("fs.gs.auth.impersonation.service.account")); + Assertions.assertEquals("metadata", new String(stream.readAllBytes(), StandardCharsets.UTF_8)); + } + } + } + } + } + + /** Serves gs://test-bucket paths from the local test directory without cloud credentials or I/O. */ + public static class LocalGcsFileSystem extends RawLocalFileSystem { + @Override + public URI getUri() { + return URI.create("gs://test-bucket/"); + } + } +} diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergGlueGcsAuthenticationTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergGlueGcsAuthenticationTest.java new file mode 100644 index 00000000000000..ccf5775a013c27 --- /dev/null +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergGlueGcsAuthenticationTest.java @@ -0,0 +1,184 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.iceberg; + +import org.apache.doris.connector.iceberg.glue.ConfigurationAWSCredentialsProvider2x; +import org.apache.doris.filesystem.gcs.GcsFileSystemProperties; + +import com.sun.net.httpserver.HttpServer; +import org.apache.iceberg.aws.AwsClientFactories; +import org.apache.iceberg.aws.AwsClientProperties; +import org.apache.iceberg.aws.HttpClientProperties; +import org.apache.iceberg.aws.S3FileIOAwsClientFactories; +import org.apache.iceberg.aws.s3.S3FileIO; +import org.apache.iceberg.aws.s3.S3FileIOAwsClientFactory; +import org.apache.iceberg.aws.s3.S3FileIOProperties; +import org.apache.iceberg.util.SerializationUtil; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.services.glue.GlueClient; +import software.amazon.awssdk.services.s3.S3AsyncClient; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +class IcebergGlueGcsAuthenticationTest { + @Test + void nativeGcsPreservesGlueRegionAndSigningCredentials() throws Exception { + try (RequestRecorder recorder = new RequestRecorder()) { + for (String mode : List.of("DEFAULT", "COMPUTE_ENGINE")) { + for (String account : List.of("", "reader@test.iam.gserviceaccount.com")) { + Map raw = properties(mode, recorder.endpoint()); + raw.put("gs.impersonation_service_account", account); + Map options = catalogOptions(raw); + Assertions.assertEquals("us-west-2", options.get(AwsClientProperties.CLIENT_REGION)); + Assertions.assertEquals(GcpS3FileIOAwsClientFactory.class.getName(), + options.get(S3FileIOProperties.CLIENT_FACTORY)); + assertGlueRequestIsSigned(options, recorder); + } + } + Assertions.assertEquals(4, recorder.glueRequests.get()); + } + } + + @Test + void anonymousGcsPreservesGlueCredentialsAndSendsUnsignedObjectRequests() throws Exception { + try (RequestRecorder recorder = new RequestRecorder()) { + Map options = catalogOptions(properties("ANONYMOUS", recorder.endpoint())); + Assertions.assertEquals(ConfigurationAWSCredentialsProvider2x.class.getName(), + options.get(AwsClientProperties.CLIENT_CREDENTIALS_PROVIDER)); + assertGlueRequestIsSigned(options, recorder); + assertObjectRequestsAreUnsigned(options, recorder); + // Initializing FileIO must leave the catalog's credential provider intact. + assertGlueRequestIsSigned(options, recorder); + } + } + + @Test + void anonymousGcsFileIoDoesNotAssumeTheGlueRole() throws Exception { + try (RequestRecorder recorder = new RequestRecorder()) { + Map raw = properties("ANONYMOUS", recorder.endpoint()); + raw.remove("glue.access_key"); + raw.remove("glue.secret_key"); + raw.put("glue.role_arn", "arn:aws:iam::123456789012:role/glue-reader"); + Map options = catalogOptions(raw); + Assertions.assertEquals("org.apache.iceberg.aws.AssumeRoleAwsClientFactory", + options.get("client.factory")); + assertObjectRequestsAreUnsigned(options, recorder); + Assertions.assertEquals(0, recorder.glueRequests.get()); + } + } + + private static Map properties(String mode, String endpoint) { + Map raw = new HashMap<>(Map.of( + "iceberg.catalog.type", "glue", "provider", "GCP", + "warehouse", "gs://bucket/warehouse", "gs.credential_provider_type", mode, + "glue.access_key", "glue-test-ak", "glue.secret_key", "glue-test-sk", + "glue.region", "us-west-2", "glue.endpoint", endpoint)); + // Keep the local transport test independent of the optional Apache HTTP client. + raw.put(HttpClientProperties.CLIENT_TYPE, HttpClientProperties.CLIENT_TYPE_URLCONNECTION); + if ("ANONYMOUS".equals(mode)) { + raw.put("gs.endpoint", endpoint); + raw.put("use_path_style", "true"); + // Exercise the Java async client without the optional native CRT dependency. + raw.put(S3FileIOProperties.S3_CRT_ENABLED, "false"); + } + return raw; + } + + private static Map catalogOptions(Map raw) { + return IcebergCatalogFactory.buildCatalogProperties(IcebergCatalogProperties.of(raw), + Optional.of(GcsFileSystemProperties.of(raw)), false); + } + + private static void assertGlueRequestIsSigned(Map options, RequestRecorder recorder) { + try (GlueClient glue = AwsClientFactories.from(options).glue()) { + Assertions.assertTrue(glue.getDatabases(b -> { }).databaseList().isEmpty()); + Assertions.assertEquals("us-west-2", glue.serviceClientConfiguration().region().id()); + } + String authorization = recorder.glueAuthorization.get(); + Assertions.assertNotNull(authorization); + Assertions.assertTrue(authorization.contains("Credential=glue-test-ak/"), authorization); + Assertions.assertTrue(authorization.contains("/us-west-2/glue/aws4_request"), authorization); + } + + private static void assertObjectRequestsAreUnsigned(Map options, RequestRecorder recorder) { + try (S3FileIO io = new S3FileIO()) { + io.initialize(options); + Assertions.assertEquals(8L, io.newInputFile("s3://bucket/object").getLength()); + } + Assertions.assertEquals(1, recorder.objectRequests.get()); + Assertions.assertNull(recorder.objectAuthorization.get()); + S3FileIOAwsClientFactory factory = S3FileIOAwsClientFactories.initialize(options); + factory = SerializationUtil.deserializeFromBytes(SerializationUtil.serializeToBytes(factory)); + try (S3AsyncClient client = factory.s3Async()) { + Assertions.assertEquals(8L, client.headObject(b -> b.bucket("bucket").key("object")) + .join().contentLength()); + } + Assertions.assertEquals(2, recorder.objectRequests.get()); + Assertions.assertNull(recorder.objectAuthorization.get()); + } + + private static final class RequestRecorder implements AutoCloseable { + private final HttpServer server; + private final AtomicReference glueAuthorization = new AtomicReference<>(); + private final AtomicReference objectAuthorization = new AtomicReference<>(); + private final AtomicInteger glueRequests = new AtomicInteger(); + private final AtomicInteger objectRequests = new AtomicInteger(); + + private RequestRecorder() throws IOException { + server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + server.createContext("/", exchange -> { + try (exchange) { + exchange.getRequestBody().readAllBytes(); + if ("HEAD".equals(exchange.getRequestMethod())) { + objectRequests.incrementAndGet(); + objectAuthorization.set(exchange.getRequestHeaders().getFirst("Authorization")); + exchange.getResponseHeaders().set("Content-Length", "8"); + exchange.sendResponseHeaders(200, -1); + } else { + glueRequests.incrementAndGet(); + glueAuthorization.set(exchange.getRequestHeaders().getFirst("Authorization")); + byte[] response = "{\"DatabaseList\":[]}".getBytes(StandardCharsets.UTF_8); + exchange.getResponseHeaders().set("Content-Type", "application/x-amz-json-1.1"); + exchange.sendResponseHeaders(200, response.length); + exchange.getResponseBody().write(response); + } + } + }); + server.start(); + } + + private String endpoint() { + return "http://127.0.0.1:" + server.getAddress().getPort(); + } + + @Override + public void close() { + server.stop(0); + } + } + +} diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergRestGcsAuthenticationTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergRestGcsAuthenticationTest.java new file mode 100644 index 00000000000000..632ce72c06d83f --- /dev/null +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergRestGcsAuthenticationTest.java @@ -0,0 +1,149 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.iceberg; + +import org.apache.doris.filesystem.gcs.GcsFileSystemProperties; + +import com.sun.net.httpserver.HttpServer; +import org.apache.iceberg.aws.AwsClientProperties; +import org.apache.iceberg.aws.AwsProperties; +import org.apache.iceberg.aws.HttpClientProperties; +import org.apache.iceberg.aws.s3.S3FileIO; +import org.apache.iceberg.aws.s3.S3FileIOProperties; +import org.apache.iceberg.rest.RESTCatalog; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; +import software.amazon.awssdk.auth.credentials.SystemPropertyCredentialsProvider; + +import java.net.InetSocketAddress; +import java.nio.charset.StandardCharsets; +import java.util.HashMap; +import java.util.Map; +import java.util.Optional; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +class IcebergRestGcsAuthenticationTest { + @Test + void anonymousGcsPreservesRestDefaultAndSelectedCredentialProviders() { + Map raw = properties("http://127.0.0.1"); + Map options = catalogOptions(raw); + Assertions.assertFalse(options.containsKey(AwsClientProperties.CLIENT_CREDENTIALS_PROVIDER)); + Assertions.assertInstanceOf(DefaultCredentialsProvider.class, + new AwsProperties(options).restCredentialsProvider()); + raw.put("iceberg.rest.credentials_provider_type", "SYSTEM_PROPERTIES"); + options = catalogOptions(raw); + Assertions.assertEquals(SystemPropertyCredentialsProvider.class.getName(), + options.get(AwsClientProperties.CLIENT_CREDENTIALS_PROVIDER)); + Assertions.assertInstanceOf(SystemPropertyCredentialsProvider.class, + new AwsProperties(options).restCredentialsProvider()); + Assertions.assertEquals(AnonymousS3FileIOAwsClientFactory.class.getName(), + options.get(S3FileIOProperties.CLIENT_FACTORY)); + } + + @Test + void anonymousGcsObjectsDoNotDisableRestRequestSigning() throws Exception { + AtomicReference restAuthorization = new AtomicReference<>(); + AtomicReference objectAuthorization = new AtomicReference<>(); + AtomicInteger restRequests = new AtomicInteger(); + AtomicInteger objectRequests = new AtomicInteger(); + HttpServer server = HttpServer.create(new InetSocketAddress("127.0.0.1", 0), 0); + server.createContext("/", exchange -> { + try (exchange) { + exchange.getRequestBody().readAllBytes(); + String authorization = exchange.getRequestHeaders().getFirst("Authorization"); + if ("HEAD".equals(exchange.getRequestMethod())) { + objectRequests.incrementAndGet(); + objectAuthorization.set(authorization); + exchange.getResponseHeaders().set("Content-Length", "8"); + exchange.sendResponseHeaders(200, -1); + } else { + restRequests.incrementAndGet(); + restAuthorization.set(authorization); + byte[] response = "{\"defaults\":{},\"overrides\":{}}".getBytes(StandardCharsets.UTF_8); + exchange.getResponseHeaders().set("Content-Type", "application/json"); + exchange.sendResponseHeaders(200, response.length); + exchange.getResponseBody().write(response); + } + } + }); + server.start(); + try { + String endpoint = "http://127.0.0.1:" + server.getAddress().getPort(); + Map raw = properties(endpoint); + raw.put(AwsClientProperties.CLIENT_CREDENTIALS_PROVIDER, MetadataCredentialsProvider.class.getName()); + Map options = catalogOptions(raw); + assertRestRequestIsSigned(options, restAuthorization); + try (S3FileIO io = new S3FileIO()) { + io.initialize(options); + Assertions.assertEquals(8L, io.newInputFile("s3://bucket/object").getLength()); + } + Assertions.assertEquals(1, objectRequests.get()); + Assertions.assertNull(objectAuthorization.get()); + // Creating an object client must not mutate the REST catalog's credential configuration. + assertRestRequestIsSigned(options, restAuthorization); + Assertions.assertTrue(restRequests.get() >= 2); + } finally { + server.stop(0); + } + } + + private static void assertRestRequestIsSigned(Map options, + AtomicReference authorization) throws Exception { + authorization.set(null); + try (RESTCatalog catalog = new RESTCatalog()) { + catalog.initialize("rest-gcs", options); + } + String header = authorization.get(); + Assertions.assertNotNull(header); + Assertions.assertTrue(header.contains("Credential=rest-test-ak/"), header); + Assertions.assertTrue(header.contains("/us-west-2/execute-api/aws4_request"), header); + } + + private static Map properties(String endpoint) { + Map raw = new HashMap<>(Map.of( + "iceberg.catalog.type", "rest", "iceberg.rest.uri", endpoint, + "iceberg.rest.signing-name", "execute-api", "iceberg.rest.sigv4-enabled", "true", + "iceberg.rest.signing-region", "us-west-2", "provider", "GCP", + "gs.credential_provider_type", "ANONYMOUS", "gs.endpoint", endpoint, + "use_path_style", "true", S3FileIOProperties.S3_CRT_ENABLED, "false")); + // Keep the local transport test independent of the optional Apache HTTP client. + raw.put(HttpClientProperties.CLIENT_TYPE, HttpClientProperties.CLIENT_TYPE_URLCONNECTION); + return raw; + } + + private static Map catalogOptions(Map raw) { + return IcebergCatalogFactory.buildCatalogProperties(IcebergCatalogProperties.of(raw), + Optional.of(GcsFileSystemProperties.of(raw)), false); + } + + public static final class MetadataCredentialsProvider implements AwsCredentialsProvider { + public static MetadataCredentialsProvider create() { + return new MetadataCredentialsProvider(); + } + + @Override + public AwsCredentials resolveCredentials() { + return AwsBasicCredentials.create("rest-test-ak", "rest-test-sk"); + } + } +} diff --git a/fe/fe-connector/fe-connector-spi/src/test/java/org/apache/doris/connector/spi/ConnectorPluginSurfaceTest.java b/fe/fe-connector/fe-connector-spi/src/test/java/org/apache/doris/connector/spi/ConnectorPluginSurfaceTest.java index 83944f581856e6..cab3d00cb4b4be 100644 --- a/fe/fe-connector/fe-connector-spi/src/test/java/org/apache/doris/connector/spi/ConnectorPluginSurfaceTest.java +++ b/fe/fe-connector/fe-connector-spi/src/test/java/org/apache/doris/connector/spi/ConnectorPluginSurfaceTest.java @@ -88,9 +88,9 @@ public void connectorApiMajorTracksTheRecordedSurfaceChange() throws IOException Assertions.assertNotNull(in, "missing connector plugin API version resource"); version.load(in); } - // Major 19 adds ConnectorColumn#isSourceNullable()/withSourceNullable(boolean); major 20 adds - // deleteUsingMetadata to the connector write surface. - Assertions.assertEquals("20.0", version.getProperty("api.version")); + // Major 20 adds deleteUsingMetadata to the connector write surface; major 21 adds + // typed filesystem credential descriptions to that surface. + Assertions.assertEquals("21.0", version.getProperty("api.version")); } /** Root entry points plus provider/handle types returned to connector plugins. */ diff --git a/fe/fe-connector/pom.xml b/fe/fe-connector/pom.xml index d1d8f2fb4c445f..1d91dc0c412092 100644 --- a/fe/fe-connector/pom.xml +++ b/fe/fe-connector/pom.xml @@ -55,7 +55,7 @@ under the License. of the latter two means bumping this property as well (and fe-extension-spi means bumping all five families). --> - 20.0 + 21.0 diff --git a/fe/fe-core/pom.xml b/fe/fe-core/pom.xml index c19b65251e512a..a590f7647f7fd1 100644 --- a/fe/fe-core/pom.xml +++ b/fe/fe-core/pom.xml @@ -202,6 +202,11 @@ under the License. + + org.apache.doris + fe-storage-protocol + ${revision} + ${project.groupId} diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/BrokerDesc.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/BrokerDesc.java index dad70399e5fe7f..025d3f07d4814b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/BrokerDesc.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/BrokerDesc.java @@ -121,6 +121,20 @@ public String getFileLocation(String location) throws UserException { return (null != storageAdapter) ? storageAdapter.validateAndNormalizeUri(location) : location; } + /** + * Normalize EXPORT and OUTFILE destinations for the BE writer. + */ + public String getFileLocationForExport(String location) throws UserException { + // BE parses HTTP(S) paths as path-style, while FE defaults to virtual-hosted parsing. + // Preserve these paths and the existing path handling for non-S3 backends. + if (StorageType.S3.equals(getStorageType()) + && !StringUtils.startsWithIgnoreCase(location, "http://") + && !StringUtils.startsWithIgnoreCase(location, "https://")) { + return getFileLocation(location); + } + return location; + } + public static BrokerDesc createForStreamLoad() { return new BrokerDesc("", StorageType.STREAM, null); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java index 68ef656f1d1b26..449c3e862d247c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/OutFileClause.java @@ -603,6 +603,11 @@ private void analyzeBrokerDesc(Map copiedProps) throws UserExcep } String brokerName = copiedProps.get(PROP_BROKER_NAME); brokerDesc = new BrokerDesc(brokerName, copiedProps); + try { + filePath = brokerDesc.getFileLocationForExport(filePath); + } catch (RuntimeException e) { + throw new AnalysisException("Invalid outfile path: " + filePath + ", " + e.getMessage(), e); + } /* * Note on HDFS export behavior and URI handling: * diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/HdfsStorageVault.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/HdfsStorageVault.java index c5851405958e0d..62167919196afc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/HdfsStorageVault.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/HdfsStorageVault.java @@ -38,6 +38,7 @@ import java.io.IOException; import java.util.Arrays; import java.util.HashSet; +import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.stream.Collectors; @@ -103,6 +104,10 @@ public void modifyProperties(ImmutableMap newProperties) throws for (Map.Entry kv : newProperties.entrySet()) { replaceIfEffectiveValue(this.properties, kv.getKey(), kv.getValue()); } + // Reject object-storage properties before they can select a different filesystem + // during the connectivity check, even when that check is disabled. + // Leave Kerberos validation in filesystem binding to preserve its error message. + this.properties.keySet().forEach(HdfsStorageVault::checkForbiddenProperty); checkConnectivity(this.properties); } @@ -192,23 +197,7 @@ public static Cloud.HdfsVaultInfo generateHdfsParam(Map properti } else if (property.getKey().equalsIgnoreCase(StorageVault.PropertyKey.VAULT_NAME)) { continue; } else { - // Get rid of copy and paste from create s3 vault stmt - Preconditions.checkArgument( - !property.getKey().toLowerCase().contains(S3StorageVault.PropertyKey.REGION), - "Invalid argument %s", property.getKey()); - Preconditions.checkArgument( - !property.getKey().toLowerCase().contains(S3StorageVault.PropertyKey.ENDPOINT), - "Invalid argument %s", property.getKey()); - Preconditions.checkArgument( - !property.getKey().toLowerCase().contains(S3StorageVault.PropertyKey.ROOT_PATH), - "Invalid argument %s", property.getKey()); - Preconditions.checkArgument( - !property.getKey().toLowerCase().contains(S3StorageVault.PropertyKey.PROVIDER), - "Invalid argument %s", property.getKey()); - Preconditions.checkArgument( - !property.getKey().toLowerCase().contains(S3StorageVault.PropertyKey.BUCKET), - "Invalid argument %s", property.getKey()); - + checkForbiddenProperty(property.getKey()); if (!NON_HDFS_CONF_PROPERTY_KEYS.contains(property.getKey().toLowerCase())) { Cloud.HdfsBuildConf.HdfsConfKVPair.Builder conf = Cloud.HdfsBuildConf.HdfsConfKVPair.newBuilder(); conf.setKey(property.getKey()); @@ -219,4 +208,19 @@ public static Cloud.HdfsVaultInfo generateHdfsParam(Map properti } return hdfsVaultInfoBuilder.setBuildConf(hdfsConfBuilder.build()).build(); } + + private static void checkForbiddenProperty(String key) { + // Get rid of copy and paste from create s3 vault stmt. + String lowerCaseKey = key.toLowerCase(Locale.ROOT); + Preconditions.checkArgument(!lowerCaseKey.contains(S3StorageVault.PropertyKey.REGION), + "Invalid argument %s", key); + Preconditions.checkArgument(!lowerCaseKey.contains(S3StorageVault.PropertyKey.ENDPOINT), + "Invalid argument %s", key); + Preconditions.checkArgument(!lowerCaseKey.contains(S3StorageVault.PropertyKey.ROOT_PATH), + "Invalid argument %s", key); + Preconditions.checkArgument(!lowerCaseKey.contains(S3StorageVault.PropertyKey.PROVIDER), + "Invalid argument %s", key); + Preconditions.checkArgument(!lowerCaseKey.contains(S3StorageVault.PropertyKey.BUCKET), + "Invalid argument %s", key); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java index 0c70cd67ce4a26..e21faeef95fdb9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/S3Resource.java @@ -23,6 +23,7 @@ import org.apache.doris.common.util.DatasourcePrintableMap; import org.apache.doris.common.util.S3Util; import org.apache.doris.datasource.storage.S3ResourceCompat; +import org.apache.doris.datasource.storage.StorageAdapter; import org.apache.doris.filesystem.UploadPartResult; import org.apache.doris.filesystem.spi.ObjFileSystem; import org.apache.doris.filesystem.spi.ObjStorage; @@ -45,6 +46,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Optional; /** @@ -93,7 +95,7 @@ public String getProperty(String propertyKey) { @Override protected void setProperties(ImmutableMap newProperties) throws DdlException { Preconditions.checkState(newProperties != null); - this.properties = Maps.newHashMap(newProperties); + this.properties = StorageAdapter.normalizeProperties(newProperties, newProperties); // check properties S3ResourceCompat.requiredS3PingProperties(properties); @@ -219,71 +221,89 @@ protected static void pingS3(String bucketName, String rootPath, Map properties) throws DdlException { + public synchronized void modifyProperties(Map newProperties) throws DdlException { + // Serialize the snapshot, validation and publication. A lock only around publication + // would allow a concurrent ALTER to replace a successful update with an older snapshot. + Map properties = new HashMap<>(newProperties); + Map selectionProperties = new HashMap<>(this.properties); + selectionProperties.putAll(properties); + if (Strings.isNullOrEmpty(properties.get("provider"))) { + // Empty ALTER values retain the persisted provider, like other resource properties. + selectionProperties.put("provider", this.properties.get("provider")); + } + // Normalize the patch independently so its aliases win over persisted canonical values. + S3ResourceCompat.convertToStdProperties(properties); + Map normalizedUpdates = StorageAdapter.normalizeProperties(properties, selectionProperties); + // Interpret persisted aliases using their original provider before applying a provider change. + Map effectiveProperties = + StorageAdapter.normalizeProperties(this.properties, this.properties); + S3ResourceCompat.convertToStdProperties(effectiveProperties); + for (Map.Entry update : normalizedUpdates.entrySet()) { + // Empty updates are ignored, except when clearing a session token or impersonation account. + replaceIfEffectiveValue(effectiveProperties, update.getKey(), update.getValue()); + if (S3ResourceCompat.SESSION_TOKEN.equals(update.getKey()) + || S3ResourceCompat.Env.TOKEN.equals(update.getKey()) + || StorageAdapter.isClearableProperty(update.getKey())) { + effectiveProperties.put(update.getKey(), update.getValue()); + } + } if (references.containsValue(ReferenceType.POLICY)) { // can't change, because remote fs use it info to find data. List cantChangeProperties = Arrays.asList(S3ResourceCompat.ENDPOINT, S3ResourceCompat.REGION, S3ResourceCompat.ROOT_PATH, S3ResourceCompat.BUCKET, S3ResourceCompat.Env.ENDPOINT, S3ResourceCompat.Env.REGION, S3ResourceCompat.Env.ROOT_PATH, S3ResourceCompat.Env.BUCKET); - Optional any = cantChangeProperties.stream().filter(properties::containsKey).findAny(); + Optional any = cantChangeProperties.stream() + .filter(key -> normalizedUpdates.containsKey(key) + || !Objects.equals(this.properties.get(key), effectiveProperties.get(key))) + .findAny(); if (any.isPresent()) { throw new DdlException("current not support modify property : " + any.get()); } } - // compatible with old version, Need convert if modified properties map uses old properties. - S3ResourceCompat.convertToStdProperties(properties); - if (!Strings.isNullOrEmpty(properties.get(S3ResourceCompat.ENDPOINT))) { - properties.put(S3ResourceCompat.Env.ENDPOINT, properties.get(S3ResourceCompat.ENDPOINT)); + if (!Strings.isNullOrEmpty(effectiveProperties.get(S3ResourceCompat.ENDPOINT))) { + effectiveProperties.put(S3ResourceCompat.Env.ENDPOINT, effectiveProperties.get(S3ResourceCompat.ENDPOINT)); } - boolean needCheck = isNeedCheck(properties); + for (Map.Entry kv : normalizedUpdates.entrySet()) { + if (kv.getKey().equalsIgnoreCase(S3ResourceCompat.ROLE_ARN) + && !Strings.isNullOrEmpty(kv.getValue())) { + effectiveProperties.remove(S3ResourceCompat.ACCESS_KEY); + effectiveProperties.remove(S3ResourceCompat.Env.ACCESS_KEY); + effectiveProperties.remove(S3ResourceCompat.SECRET_KEY); + effectiveProperties.remove(S3ResourceCompat.Env.SECRET_KEY); + } + if (kv.getKey().equalsIgnoreCase(S3ResourceCompat.ACCESS_KEY) + && !Strings.isNullOrEmpty(kv.getValue())) { + effectiveProperties.remove(S3ResourceCompat.ROLE_ARN); + effectiveProperties.remove(S3ResourceCompat.Env.ROLE_ARN); + effectiveProperties.remove(S3ResourceCompat.EXTERNAL_ID); + effectiveProperties.remove(S3ResourceCompat.Env.EXTERNAL_ID); + } + } + StorageAdapter.resolveAuthentication(effectiveProperties); + boolean needCheck = isNeedCheck(effectiveProperties); if (LOG.isDebugEnabled()) { LOG.debug("s3 info need check validity : {}", needCheck); } if (needCheck) { - S3ResourceCompat.requiredS3PingProperties(this.properties); - Map changedProperties = new HashMap<>(this.properties); - changedProperties.putAll(properties); + S3ResourceCompat.requiredS3PingProperties(effectiveProperties); + Map changedProperties = new HashMap<>(effectiveProperties); String endpoint = S3Util.buildEndpointUrl(changedProperties.get(S3ResourceCompat.ENDPOINT)); changedProperties.put(S3ResourceCompat.ENDPOINT, endpoint); changedProperties.put(S3ResourceCompat.Env.ENDPOINT, endpoint); - String bucketName = properties.getOrDefault(S3ResourceCompat.BUCKET, - this.properties.get(S3ResourceCompat.BUCKET)); - String rootPath = properties.getOrDefault(S3ResourceCompat.ROOT_PATH, - this.properties.get(S3ResourceCompat.ROOT_PATH)); - - pingS3(bucketName, rootPath, changedProperties); + pingS3(effectiveProperties.get(S3ResourceCompat.BUCKET), + effectiveProperties.get(S3ResourceCompat.ROOT_PATH), + changedProperties); } - // modify properties writeLock(); - - for (Map.Entry kv : properties.entrySet()) { - replaceIfEffectiveValue(this.properties, kv.getKey(), kv.getValue()); - if (kv.getKey().equals(S3ResourceCompat.Env.TOKEN) - || kv.getKey().equals(S3ResourceCompat.SESSION_TOKEN)) { - this.properties.put(kv.getKey(), kv.getValue()); - } - - if (kv.getKey().equalsIgnoreCase(S3ResourceCompat.ROLE_ARN) - && !Strings.isNullOrEmpty(kv.getValue())) { - this.properties.remove(S3ResourceCompat.ACCESS_KEY); - this.properties.remove(S3ResourceCompat.Env.ACCESS_KEY); - this.properties.remove(S3ResourceCompat.SECRET_KEY); - this.properties.remove(S3ResourceCompat.Env.SECRET_KEY); - } - - if (kv.getKey().equalsIgnoreCase(S3ResourceCompat.ACCESS_KEY) - && !Strings.isNullOrEmpty(kv.getValue())) { - this.properties.remove(S3ResourceCompat.ROLE_ARN); - this.properties.remove(S3ResourceCompat.Env.ROLE_ARN); - this.properties.remove(S3ResourceCompat.EXTERNAL_ID); - this.properties.remove(S3ResourceCompat.Env.EXTERNAL_ID); - } + try { + this.properties = effectiveProperties; + ++version; + } finally { + writeUnlock(); } - ++version; - writeUnlock(); - super.modifyProperties(properties); + super.modifyProperties(effectiveProperties); } private CloudCredentialWithEndpoint getS3PingCredentials(Map properties) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVaultMgr.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVaultMgr.java index 4662c72d7ff621..4f3d1c07c30969 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVaultMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/StorageVaultMgr.java @@ -26,6 +26,8 @@ import org.apache.doris.common.UserException; import org.apache.doris.common.lock.MonitoredReentrantReadWriteLock; import org.apache.doris.datasource.storage.CloudObjectStoreAdapter; +import org.apache.doris.datasource.storage.S3ResourceCompat; +import org.apache.doris.datasource.storage.StorageAdapter; import org.apache.doris.nereids.trees.plans.commands.CreateStorageVaultCommand; import org.apache.doris.proto.InternalService.PAlterVaultSyncRequest; import org.apache.doris.rpc.BackendServiceProxy; @@ -145,16 +147,42 @@ private void updateDefaultStorageVaultCache(Pair newDefaultVault } } - private Cloud.StorageVaultPB.Builder buildAlterS3VaultRequest(Map properties, String name) + private Cloud.StorageVaultPB.Builder buildS3VaultRequest(Map properties, String name) throws Exception { - Cloud.ObjectStoreInfoPB.Builder objBuilder = CloudObjectStoreAdapter.getObjStoreInfoPB(properties); - Cloud.StorageVaultPB.Builder alterObjVaultBuilder = Cloud.StorageVaultPB.newBuilder(); - alterObjVaultBuilder.setName(name); - alterObjVaultBuilder.setObjInfo(objBuilder.build()); + Cloud.StorageVaultPB.Builder builder = Cloud.StorageVaultPB.newBuilder() + .setName(name).setObjInfo(CloudObjectStoreAdapter.getObjStoreInfoPB(properties)); if (properties.containsKey(StorageVault.PropertyKey.VAULT_NAME)) { - alterObjVaultBuilder.setAlterName(properties.get(StorageVault.PropertyKey.VAULT_NAME)); + builder.setAlterName(properties.get(StorageVault.PropertyKey.VAULT_NAME)); } - return alterObjVaultBuilder; + return builder; + } + + private Cloud.StorageVaultPB.Builder buildAlterS3VaultRequest(Map properties, String name) + throws Exception { + Cloud.StorageVaultPB.Builder builder = buildS3VaultRequest(properties, name); + Cloud.ObjectStoreInfoPB.Builder objBuilder = builder.getObjInfoBuilder(); + CloudObjectStoreAdapter.applyCredentialPatch(objBuilder, properties); + // CREATE may infer GCP from native credentials. ALTER must not send an inferred + // provider because MS rejects changes to this immutable storage field. + if (!properties.containsKey("provider")) { + objBuilder.clearProvider(); + } + // getObjStoreInfoPB supplies INSTANCE_PROFILE as the create-time default + // when role_arn is present. ALTER is a patch operation, so only send the + // credential provider when the user explicitly changed it. + if (!properties.containsKey(S3ResourceCompat.CREDENTIALS_PROVIDER_TYPE) + && !properties.containsKey(S3ResourceCompat.Env.CREDENTIALS_PROVIDER_TYPE)) { + objBuilder.clearCredProviderType(); + } + // Empty strings are meaningful for ALTER: they explicitly clear the + // stored role or external ID. The generic create builder omits them. + if (properties.containsKey(S3ResourceCompat.ROLE_ARN)) { + objBuilder.setRoleArn(properties.get(S3ResourceCompat.ROLE_ARN)); + } + if (properties.containsKey(S3ResourceCompat.EXTERNAL_ID)) { + objBuilder.setExternalId(properties.get(S3ResourceCompat.EXTERNAL_ID)); + } + return builder; } private Cloud.StorageVaultPB.Builder buildAlterHdfsVaultRequest(Map properties, String name) @@ -182,9 +210,15 @@ private Cloud.StorageVaultPB.Builder buildAlterStorageVaultRequest(StorageVaultT return builder; } - private Cloud.StorageVaultPB.Builder buildAlterStorageVaultRequest(StorageVault vault) throws Exception { - Cloud.StorageVaultPB.Builder builder = buildAlterStorageVaultRequest(vault.getType(), - vault.getCopiedProperties(), vault.getName()); + private Cloud.StorageVaultPB.Builder buildCreateStorageVaultRequest(StorageVault vault) throws Exception { + Cloud.StorageVaultPB.Builder builder; + if (vault.getType() == StorageVaultType.S3) { + builder = buildS3VaultRequest(vault.getCopiedProperties(), vault.getName()); + } else if (vault.getType() == StorageVaultType.HDFS) { + builder = buildAlterHdfsVaultRequest(vault.getCopiedProperties(), vault.getName()); + } else { + throw new DdlException("Unknown storage vault type"); + } Cloud.StorageVaultPB.PathFormat.Builder pathBuilder = Cloud.StorageVaultPB.PathFormat.newBuilder(); pathBuilder.setShardNum(vault.getNumShard()); pathBuilder.setPathVersion(vault.getPathVersion()); @@ -201,7 +235,8 @@ public void alterStorageVault(StorageVaultType type, Map propert .setRequestIp(FrontendOptions.getLocalHostAddressCached()); if (type == StorageVaultType.S3) { properties.keySet().stream() - .filter(key -> !S3StorageVault.ALLOW_ALTER_PROPERTIES.contains(key)) + .filter(key -> !S3StorageVault.ALLOW_ALTER_PROPERTIES.contains(key) + && !StorageAdapter.isModifiableCredentialProperty(key)) .findAny() .ifPresent(key -> { throw new IllegalArgumentException("Alter property " + key + " is not allowed."); @@ -209,8 +244,8 @@ public void alterStorageVault(StorageVaultType type, Map propert request.setOp(Operation.ALTER_S3_VAULT); } else if (type == StorageVaultType.HDFS) { properties.keySet().stream() - // Exact literals of the legacy S3Properties.S3_PREFIX ("s3.") and - // StorageProperties.FS_PROVIDER_KEY ("provider") constants. + // Exact literals of the legacy S3ResourceCompat.S3_PREFIX ("s3.") and + // "provider" ("provider") constants. .filter(key -> HdfsStorageVault.FORBID_ALTER_PROPERTIES.contains(key) || key.toLowerCase().contains("s3.") || key.toLowerCase().contains("provider")) @@ -321,7 +356,7 @@ public StorageVaultType getStorageVaultTypeByName(String vaultName) throws DdlEx @VisibleForTesting public void createHdfsVault(StorageVault vault) throws Exception { - Cloud.StorageVaultPB.Builder alterHdfsInfoBuilder = buildAlterStorageVaultRequest(vault); + Cloud.StorageVaultPB.Builder alterHdfsInfoBuilder = buildCreateStorageVaultRequest(vault); Cloud.AlterObjStoreInfoRequest.Builder requestBuilder = Cloud.AlterObjStoreInfoRequest.newBuilder().setRequestIp(FrontendOptions.getLocalHostAddressCached()); requestBuilder.setOp(Cloud.AlterObjStoreInfoRequest.Operation.ADD_HDFS_INFO); @@ -371,7 +406,7 @@ private void alterSyncVaultTask() { } public void createS3Vault(StorageVault vault) throws Exception { - Cloud.StorageVaultPB.Builder s3StorageVaultBuilder = buildAlterStorageVaultRequest(vault); + Cloud.StorageVaultPB.Builder s3StorageVaultBuilder = buildCreateStorageVaultRequest(vault); Cloud.AlterObjStoreInfoRequest.Builder requestBuilder = Cloud.AlterObjStoreInfoRequest.newBuilder().setRequestIp(FrontendOptions.getLocalHostAddressCached()); requestBuilder.setOp(Cloud.AlterObjStoreInfoRequest.Operation.ADD_S3_VAULT); diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/storage/ObjectInfo.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/storage/ObjectInfo.java index dc3959f01d1624..1494810c4d147a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/storage/ObjectInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/storage/ObjectInfo.java @@ -38,6 +38,7 @@ public class ObjectInfo { // only used for aws private final String externalId; private final String token; + private final Cloud.ObjectStoreCredentialPB credential; // Used to get sts token public ObjectInfo(Cloud.ObjectStoreInfoPB.Provider provider, String ak, String sk, @@ -56,19 +57,26 @@ public ObjectInfo(Cloud.ObjectStoreInfoPB objectStoreInfoPB) { this(objectStoreInfoPB.getProvider(), objectStoreInfoPB.getAk(), objectStoreInfoPB.getSk(), objectStoreInfoPB.getBucket(), objectStoreInfoPB.getEndpoint(), objectStoreInfoPB.getRegion(), objectStoreInfoPB.getPrefix(), null, objectStoreInfoPB.getRoleArn(), - objectStoreInfoPB.getExternalId(), null); + objectStoreInfoPB.getExternalId(), null, nativeCredential(objectStoreInfoPB)); } public ObjectInfo(Cloud.ObjectStoreInfoPB objectStoreInfoPB, String roleName, String arn, String externalId, String token) { this(objectStoreInfoPB.getProvider(), objectStoreInfoPB.getAk(), objectStoreInfoPB.getSk(), objectStoreInfoPB.getBucket(), objectStoreInfoPB.getEndpoint(), objectStoreInfoPB.getRegion(), - objectStoreInfoPB.getPrefix(), roleName, arn, externalId, token); + objectStoreInfoPB.getPrefix(), roleName, arn, externalId, token, nativeCredential(objectStoreInfoPB)); } public ObjectInfo(Cloud.ObjectStoreInfoPB.Provider provider, String ak, String sk, String bucket, String endpoint, String region, String prefix, String roleName, String arn, String externalId, String token) { + this(provider, ak, sk, bucket, endpoint, region, prefix, roleName, arn, externalId, token, null); + } + + private ObjectInfo(Cloud.ObjectStoreInfoPB.Provider provider, String ak, String sk, String bucket, + String endpoint, String region, String prefix, String roleName, String arn, String externalId, + String token, Cloud.ObjectStoreCredentialPB credential) { + this.credential = credential; this.provider = provider; this.ak = ak; this.sk = sk; @@ -86,6 +94,14 @@ public Cloud.ObjectStoreInfoPB.Provider getProvider() { return provider; } + private static Cloud.ObjectStoreCredentialPB nativeCredential(Cloud.ObjectStoreInfoPB info) { + return info.hasCredential() ? info.getCredential() : null; + } + + public Cloud.ObjectStoreCredentialPB getCredential() { + return credential; + } + public String getAk() { return ak; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/cloud/storage/ObjectInfoAdapter.java b/fe/fe-core/src/main/java/org/apache/doris/cloud/storage/ObjectInfoAdapter.java index e38aacfa6a6ba5..c1fa1ecaed6fde 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/cloud/storage/ObjectInfoAdapter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/cloud/storage/ObjectInfoAdapter.java @@ -22,6 +22,7 @@ import org.apache.doris.cloud.proto.Cloud.StagePB.StageAccessType; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Config; +import org.apache.doris.datasource.storage.ObjectStorageCredentialAdapter; import org.apache.doris.datasource.storage.StorageAdapter; import org.apache.doris.filesystem.spi.ObjFileSystem; import org.apache.doris.filesystem.spi.StsCredentials; @@ -70,12 +71,22 @@ private ObjectInfoAdapter() {} * it separately to {@code listObjectsWithPrefix} / {@code headObjectWithMeta}. */ public static StorageAdapter toStorageAdapter(ObjectInfo obj) { + if (obj.getCredential() != null) { + Map properties = buildS3CompatibleProps(obj); + properties.put("provider", obj.getProvider().name()); + properties.putAll(ObjectStorageCredentialAdapter.toProperties( + obj.getProvider().name(), obj.getCredential())); + return StorageAdapter.of(properties); + } switch (obj.getProvider()) { case OSS: return StorageAdapter.ofProvider("OSS", buildS3CompatibleProps(obj)); case S3: - case GCP: return StorageAdapter.ofProvider("S3", buildS3CompatibleProps(obj)); + case GCP: + Map gcpProperties = buildS3CompatibleProps(obj); + gcpProperties.put("provider", "GCP"); + return StorageAdapter.ofProvider("GCS", gcpProperties); case COS: return StorageAdapter.ofProvider("COS", buildS3CompatibleProps(obj)); case OBS: diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/storage/CloudObjectStoreAdapter.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/storage/CloudObjectStoreAdapter.java index b74680c79d7159..20a1780308fa36 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/storage/CloudObjectStoreAdapter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/storage/CloudObjectStoreAdapter.java @@ -21,12 +21,15 @@ import org.apache.doris.cloud.proto.Cloud.CredProviderTypePB; import org.apache.doris.cloud.proto.Cloud.ObjectStoreInfoPB.Provider; import org.apache.doris.datasource.property.common.AwsCredentialsProviderMode; +import org.apache.doris.filesystem.auth.ObjectStorageAuthentication; import org.apache.doris.filesystem.properties.S3CompatibleFileSystemProperties; import com.google.common.base.Preconditions; import com.google.common.base.Strings; +import java.util.Locale; import java.util.Map; +import java.util.Optional; /** * fe-core-only cloud meta-service glue: builds {@link Cloud.ObjectStoreInfoPB} from the @@ -60,6 +63,10 @@ private CloudObjectStoreAdapter() { /** Direct move of legacy {@code S3Properties.getObjStoreInfoPB}. */ public static Cloud.ObjectStoreInfoPB.Builder getObjStoreInfoPB(Map properties) { + Optional authentication = StorageAdapter.resolveAuthentication(properties); + if (authentication.filter(ObjectStorageAuthentication::isAnonymous).isPresent()) { + throw new IllegalArgumentException("Anonymous GCS authentication is not supported for storage vaults."); + } Cloud.ObjectStoreInfoPB.Builder builder = Cloud.ObjectStoreInfoPB.newBuilder(); if (properties.containsKey(ENDPOINT)) { builder.setEndpoint(properties.get(ENDPOINT)); @@ -86,7 +93,9 @@ public static Cloud.ObjectStoreInfoPB.Builder getObjStoreInfoPB(Map credential.applyTo(builder)); return builder; } + /** ALTER carries only explicitly supplied credential fields, including empty values. */ + public static void applyCredentialPatch(Cloud.ObjectStoreInfoPB.Builder builder, Map properties) { + builder.clearCredential(); + StorageAdapter.resolveAuthentication(properties) + .flatMap(ObjectStorageAuthentication::getCredentialUpdates) + .ifPresent(credential -> ObjectStorageCredentialAdapter.fromCredential(credential).applyTo(builder)); + } + private static boolean hasCredentialsProviderType(Map properties) { return properties.containsKey(CREDENTIALS_PROVIDER_TYPE) || properties.containsKey(ENV_CREDENTIALS_PROVIDER_TYPE); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/storage/ObjectStorageCredentialAdapter.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/storage/ObjectStorageCredentialAdapter.java new file mode 100644 index 00000000000000..de189be6d11c90 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/storage/ObjectStorageCredentialAdapter.java @@ -0,0 +1,69 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.storage; + +import org.apache.doris.cloud.proto.Cloud.ObjectStoreCredentialPB; +import org.apache.doris.cloud.proto.Cloud.ObjectStoreInfoPB; +import org.apache.doris.filesystem.auth.NativeObjectStorageCredential; +import org.apache.doris.filesystem.auth.ObjectStorageAuthentication; +import org.apache.doris.storage.protocol.ObjectStorageCredentialTranslator; +import org.apache.doris.thrift.TCredential; +import org.apache.doris.thrift.TObjStorageType; +import org.apache.doris.thrift.TS3StorageParam; + +import java.util.Map; +import java.util.Optional; + +/** Connects provider-resolved identities to the host protocol adapter without provider-specific logic. */ +public final class ObjectStorageCredentialAdapter { + private final NativeObjectStorageCredential credential; + + private ObjectStorageCredentialAdapter(NativeObjectStorageCredential credential) { + this.credential = credential; + } + + public static Optional fromAuthentication( + Optional authentication) { + return authentication.flatMap(ObjectStorageAuthentication::getCredential) + .map(ObjectStorageCredentialAdapter::fromCredential); + } + + public static ObjectStorageCredentialAdapter fromCredential(NativeObjectStorageCredential credential) { + return new ObjectStorageCredentialAdapter(credential); + } + + public static Map toProperties(String provider, ObjectStoreCredentialPB credential) { + return StorageAdapter.credentialToProperties(ObjectStorageCredentialTranslator.fromCloud(provider, credential)); + } + + public void applyTo(ObjectStoreInfoPB.Builder builder) { + ObjectStoreCredentialPB encoded = ObjectStorageCredentialTranslator.toCloud(credential); + if (!builder.hasProvider()) { + builder.setProvider(ObjectStoreInfoPB.Provider.valueOf(credential.getProvider())); + } + builder.setCredential(encoded); + } + + public void applyTo(TS3StorageParam param) { + TCredential encoded = ObjectStorageCredentialTranslator.toThrift(credential); + if (!param.isSetProvider()) { + param.setProvider(TObjStorageType.valueOf(credential.getProvider())); + } + param.setCredential(encoded); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/storage/S3ResourceCompat.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/storage/S3ResourceCompat.java index aa313ca01660c4..ebfcddb41faf05 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/storage/S3ResourceCompat.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/storage/S3ResourceCompat.java @@ -57,7 +57,7 @@ public final class S3ResourceCompat { public static final List REQUIRED_FIELDS = Arrays.asList(ENDPOINT); public static final List PROVIDERS = - Arrays.asList("COS", "OSS", "S3", "OBS", "BOS", "AZURE", "GCP", "TOS"); + Arrays.asList("COS", "OSS", "S3", "OBS", "BOS", "AZURE", "GCP", "GCS", "TOS"); private static final Pattern IPV4_PORT_PATTERN = Pattern.compile("((?:\\d{1,3}\\.){3}\\d{1,3}:\\d{1,5})"); @@ -111,7 +111,7 @@ public static void requiredS3Properties(Map properties) throws D if (StringUtils.isNotBlank(properties.get(FS_PROVIDER_KEY))) { // S3 Provider properties should be case insensitive. if (!PROVIDERS.stream().anyMatch(s -> s.equals(properties.get(FS_PROVIDER_KEY).toUpperCase()))) { - throw new DdlException("Provider must be one of OSS, OBS, AZURE, BOS, COS, S3, GCP"); + throw new DdlException("Provider must be one of OSS, OBS, AZURE, BOS, COS, S3, GCP, GCS, TOS"); } } @@ -136,6 +136,7 @@ public static void optionalS3Property(Map properties) { } public static void convertToStdProperties(Map properties) { + StorageAdapter.resolveAuthentication(properties); if (properties.containsKey(Env.ENDPOINT)) { properties.putIfAbsent(ENDPOINT, properties.get(Env.ENDPOINT)); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/storage/S3ThriftAdapter.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/storage/S3ThriftAdapter.java index 5b979ef1d1fc04..1d02992b145a48 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/storage/S3ThriftAdapter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/storage/S3ThriftAdapter.java @@ -18,13 +18,17 @@ package org.apache.doris.datasource.storage; import org.apache.doris.datasource.property.common.AwsCredentialsProviderMode; +import org.apache.doris.filesystem.auth.ObjectStorageAuthentication; import org.apache.doris.filesystem.properties.S3CompatibleFileSystemProperties; import org.apache.doris.thrift.TCredProviderType; +import org.apache.doris.thrift.TObjStorageType; import org.apache.doris.thrift.TS3StorageParam; import org.apache.commons.lang3.StringUtils; +import java.util.Locale; import java.util.Map; +import java.util.Optional; /** * fe-core-only Thrift glue: builds {@link TS3StorageParam} from the persisted user-facing @@ -66,7 +70,25 @@ private S3ThriftAdapter() { /** Direct move of legacy {@code S3Properties.getS3TStorageParam}. */ public static TS3StorageParam getS3TStorageParam(Map properties) { + Optional authentication = StorageAdapter.resolveAuthentication(properties); TS3StorageParam s3Info = new TS3StorageParam(); + if (StringUtils.isNotBlank(properties.get("provider"))) { + String provider = properties.get("provider").trim().toUpperCase(Locale.ROOT); + // Filesystem dialect names such as MINIO have no Thrift enum value. + // They use the same AWS-compatible backend as S3 and must not abort a policy batch. + switch (provider) { + case "S3": + case "MINIO": + case "OZONE": + s3Info.setProvider(TObjStorageType.AWS); + break; + case "GCS": + s3Info.setProvider(TObjStorageType.GCP); + break; + default: + s3Info.setProvider(TObjStorageType.valueOf(provider)); + } + } if (properties.containsKey(ROLE_ARN)) { s3Info.setRoleArn(properties.get(ROLE_ARN)); @@ -95,6 +117,13 @@ public static TS3StorageParam getS3TStorageParam(Map properties) ? DEFAULT_CONNECTION_TIMEOUT_MS : connTimeoutMs)); String usePathStyle = properties.getOrDefault(USE_PATH_STYLE, "false"); s3Info.setUsePathStyle(Boolean.parseBoolean(usePathStyle)); + ObjectStorageCredentialAdapter.fromAuthentication(authentication) + .ifPresent(credential -> credential.applyTo(s3Info)); + if (authentication.filter(ObjectStorageAuthentication::isAnonymous).isPresent() + || (authentication.isEmpty() && getCredentialsProviderMode(properties, + AwsCredentialsProviderMode.INSTANCE_PROFILE) == AwsCredentialsProviderMode.ANONYMOUS)) { + s3Info.setCredProviderType(TCredProviderType.ANONYMOUS); + } return s3Info; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/storage/StorageAdapter.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/storage/StorageAdapter.java index 0c42c095d33cff..1ddde0b6883a8c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/storage/StorageAdapter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/storage/StorageAdapter.java @@ -19,6 +19,8 @@ import org.apache.doris.common.Config; import org.apache.doris.datasource.property.common.AwsCredentialsProviderMode; +import org.apache.doris.filesystem.auth.NativeObjectStorageCredential; +import org.apache.doris.filesystem.auth.ObjectStorageAuthentication; import org.apache.doris.filesystem.properties.FileSystemProperties; import org.apache.doris.filesystem.properties.FsCacheKeys; import org.apache.doris.filesystem.properties.HadoopStorageProperties; @@ -39,6 +41,7 @@ import java.util.Locale; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.Set; /** @@ -141,6 +144,59 @@ private StorageAdapter(FileSystemProperties spi, Map origProps, checkAzureOauth2OnlyForIcebergRest(); } + /** Provider-owned normalization also accepts partial resource/vault ALTER maps. */ + public static Map normalizeProperties(Map properties, + Map context) { + Map normalized = new HashMap<>(properties); + for (FileSystemProvider provider : manager().getProviders()) { + normalized = provider.normalizeProperties(normalized, context); + } + return normalized; + } + + public static Optional resolveAuthentication(Map properties) { + ObjectStorageAuthentication selected = null; + for (FileSystemProvider provider : manager().getProviders()) { + Optional auth = provider.resolveAuthentication(properties); + if (auth.isPresent()) { + if (selected != null) { + throw new IllegalArgumentException("Multiple filesystem providers claimed object authentication"); + } + selected = auth.get(); + } + } + return Optional.ofNullable(selected); + } + + public static boolean isModifiableCredentialProperty(String key) { + return manager().getProviders().stream() + .anyMatch(provider -> provider.modifiableCredentialPropertyKeys().contains(key)); + } + + public static boolean isClearableProperty(String key) { + return manager().getProviders().stream().anyMatch(provider -> provider.clearablePropertyKeys().contains(key)); + } + + /** Restore user properties through the plugin owning the decoded identity description. */ + public static Map credentialToProperties(NativeObjectStorageCredential credential) { + Map selected = null; + for (FileSystemProvider provider : manager().getProviders()) { + Optional> properties = provider.credentialToProperties(credential); + if (properties.isPresent()) { + if (selected != null) { + throw new StoragePropertiesException( + "Multiple credential property owners for provider: " + credential.getProvider()); + } + selected = properties.get(); + } + } + if (selected == null) { + throw new StoragePropertiesException("Filesystem plugin for credential properties is not available: " + + credential.getProvider()); + } + return selected; + } + /** * Binds the primary provider for the raw user properties, mirroring the legacy * {@code StorageProperties.createPrimary}. @@ -170,6 +226,9 @@ public static List ofAll(Map origProps) { */ public static StorageAdapter ofProvider(String providerName, Map origProps) { Map props = origProps == null ? new HashMap<>() : origProps; + Map selection = new HashMap<>(props); + selection.put("provider", providerName); + manager().validateProperties(selection); for (FileSystemProvider provider : manager().getProviders()) { if (provider.name().equalsIgnoreCase(providerName)) { return new StorageAdapter(provider.bind(withHadoopConfigDir(props)), props); @@ -500,14 +559,15 @@ private Map alignS3FamilyBackendMap(S3CompatibleFileSystemProper // AWS_EXTERNAL_ID pass through unconditionally when set (the SPI already emits them). aligned.put("AWS_CREDENTIALS_PROVIDER_TYPE", s3CredentialsMode.getMode()); } else { - // Align fe-core, ledger 2.4-5: non-S3 dialects never emit role keys, and emit - // AWS_CREDENTIALS_PROVIDER_TYPE=ANONYMOUS exactly when both AK and SK are blank. + // Non-S3 dialects never emit role keys. Preserve an explicit credential mode + // supplied by the filesystem plugin when no static or native credentials are present. aligned.remove("AWS_ROLE_ARN"); aligned.remove("AWS_EXTERNAL_ID"); - if (s3.hasStaticCredentials()) { + if (s3.hasStaticCredentials() + || resolveAuthentication(aligned).map(ObjectStorageAuthentication::isNative).orElse(false)) { aligned.remove("AWS_CREDENTIALS_PROVIDER_TYPE"); } else { - aligned.put("AWS_CREDENTIALS_PROVIDER_TYPE", AwsCredentialsProviderMode.ANONYMOUS.name()); + aligned.putIfAbsent("AWS_CREDENTIALS_PROVIDER_TYPE", AwsCredentialsProviderMode.ANONYMOUS.name()); } } return aligned; diff --git a/fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemPluginManager.java b/fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemPluginManager.java index 4c3d59548307cf..4e72cd2427aa6b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemPluginManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/fs/FileSystemPluginManager.java @@ -45,6 +45,7 @@ import java.util.Map; import java.util.ServiceLoader; import java.util.Set; +import java.util.TreeSet; import java.util.concurrent.CopyOnWriteArrayList; /** @@ -54,8 +55,8 @@ * 1. ServiceLoader scan (classpath-based built-ins / test overrides) * 2. DirectoryPluginRuntimeManager scan (production plugin directories) * - *

The first provider that returns {@code supports(props) == true} is used. - * Classpath providers have higher priority than directory-loaded providers. + *

The first supporting provider is used unless a matching provider supersedes its declared + * heuristic fallbacks. Classpath providers have higher priority than directory-loaded providers. */ public class FileSystemPluginManager { @@ -200,8 +201,16 @@ private boolean hasProviderNamed(String name) { * @throws IOException if no provider supports the properties, or creation fails */ public FileSystem createFileSystem(Map properties) throws IOException { + validateProperties(properties); + Set fallbacks = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); + for (FileSystemProvider provider : providers) { + if (!provider.fallbackProviders().isEmpty() && provider.supports(properties)) { + fallbacks.addAll(provider.fallbackProviders()); + } + } for (FileSystemProvider provider : providers) { - if (provider.supports(properties)) { + if ((!fallbacks.contains(provider.name()) || provider.supportsExplicit(properties)) + && provider.supports(properties)) { return provider.create(properties); } } @@ -228,32 +237,61 @@ public List getProviders() { private static final String DEPRECATED_OSS_HDFS_SUPPORT = "oss.hdfs.enabled"; + /** Validate plugin-owned keys even if routing would otherwise ignore them. */ + public void validateProperties(Map properties) { + for (FileSystemProvider provider : providers) { + provider.validateProperties(properties); + } + } + + private Set fallbackProviders(Map properties, Map probeView, + boolean useGuess) { + Set fallbacks = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); + for (FileSystemProvider provider : providers) { + if (!provider.fallbackProviders().isEmpty() + && (provider.supportsExplicit(properties) || (useGuess && provider.supportsGuess(probeView)))) { + fallbacks.addAll(provider.fallbackProviders()); + } + } + return fallbacks; + } + + private static boolean matches(FileSystemProvider provider, Map properties, + Map probeView, boolean useGuess, Set fallbacks) { + return provider.supportsExplicit(properties) + || (useGuess && !fallbacks.contains(provider.name()) && provider.supportsGuess(probeView)); + } + /** * Binds the first matching provider for raw user properties, mirroring * {@code StorageProperties.createPrimary}: fixed priority, explicit {@code fs..support} - * flags first-class, heuristic guesses globally disabled once any explicit flag is present, + * flags first-class, heuristic guesses disabled by an explicit flag, + * matched plugins superseding their declared heuristic fallbacks, * and no default fallback — throws when nothing matches. */ public FileSystemProperties bindPrimary(Map properties) { + validateProperties(properties); boolean useGuess = !hasAnyExplicitFsSupport(properties); Map probeView = withProbeContext(properties); + Set fallbacks = fallbackProviders(properties, probeView, useGuess); for (StorageRegistry.Provider meta : StorageRegistry.Provider.values()) { FileSystemProvider provider = providerByName(meta.name()); if (provider == null) { continue; } - if (provider.supportsExplicit(properties) || (useGuess && provider.supportsGuess(probeView))) { + if (matches(provider, properties, probeView, useGuess, fallbacks)) { return provider.bind(properties); } } // Providers not in the StorageRegistry.Provider table (out-of-tree plugins) are consulted // after every known provider, in registration order — adding a plugin needs no fe-core - // priority-list change and can never preempt the built-in set. + // priority-list change. A matched plugin can supersede its declared fallback providers. for (FileSystemProvider provider : providers) { if (StorageRegistry.Provider.byName(provider.name()).isPresent()) { continue; } - FileSystemProperties bound = tryBindUnlisted(provider, properties, useGuess, probeView); + FileSystemProperties bound = tryBindUnlisted(provider, properties, + useGuess && !fallbacks.contains(provider.name()), probeView); if (bound != null) { return bound; } @@ -270,8 +308,10 @@ public FileSystemProperties bindPrimary(Map properties) { * default HDFS binding is inserted at index 0 (fe-core's default-HDFS fallback). */ public List bindAll(Map properties) { + validateProperties(properties); boolean useGuess = !hasAnyExplicitFsSupport(properties); Map probeView = withProbeContext(properties); + Set fallbacks = fallbackProviders(properties, probeView, useGuess); List result = new ArrayList<>(); boolean ossHdfsMatched = false; boolean hdfsFamilyMatched = false; @@ -291,7 +331,7 @@ public List bindAll(Map properties) { if (meta == StorageRegistry.Provider.HDFS && hdfsFamilyMatched) { continue; } - if (provider.supportsExplicit(properties) || (useGuess && provider.supportsGuess(probeView))) { + if (matches(provider, properties, probeView, useGuess, fallbacks)) { result.add(provider.bind(properties)); if (meta == StorageRegistry.Provider.OSS_HDFS) { ossHdfsMatched = true; @@ -310,7 +350,8 @@ public List bindAll(Map properties) { if (StorageRegistry.Provider.byName(provider.name()).isPresent()) { continue; } - FileSystemProperties bound = tryBindUnlisted(provider, properties, useGuess, probeView); + FileSystemProperties bound = tryBindUnlisted(provider, properties, + useGuess && !fallbacks.contains(provider.name()), probeView); if (bound != null) { result.add(bound); } @@ -379,7 +420,7 @@ public static Map withProbeContext(Map propertie private FileSystemProvider providerByName(String name) { for (FileSystemProvider p : providers) { - if (name.equalsIgnoreCase(p.name())) { + if (p.name().equalsIgnoreCase(name)) { return p; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExportCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExportCommand.java index 19663686f636d4..38fc69b4e23c38 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExportCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ExportCommand.java @@ -277,7 +277,11 @@ private ExportJob generateExportJob(ConnectContext ctx, Map file exportJob.setWhereExpression(this.expr); exportJob.setWhereStr(this.expr.isPresent() ? this.expr.get().toSql() : ""); // set path - exportJob.setExportPath(this.path); + try { + exportJob.setExportPath(this.brokerDesc.get().getFileLocationForExport(this.path)); + } catch (RuntimeException e) { + throw new AnalysisException("Invalid export path: " + this.path + ", " + e.getMessage(), e); + } // set column separator String columnSeparator = Separator.convertSeparator(fileProperties.getOrDefault( diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowCreateStorageVaultCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowCreateStorageVaultCommand.java index 1c322f9d8b0e84..7645ea5802912f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowCreateStorageVaultCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowCreateStorageVaultCommand.java @@ -28,6 +28,7 @@ import org.apache.doris.common.Config; import org.apache.doris.common.FeConstants; import org.apache.doris.common.util.DatasourcePrintableMap; +import org.apache.doris.datasource.storage.ObjectStorageCredentialAdapter; import org.apache.doris.mysql.privilege.AccessControllerManager; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.nereids.trees.plans.PlanType; @@ -130,8 +131,14 @@ private String getObjectCreateStmt(Cloud.ObjectStoreInfoPB objectInfo) { properties.put("s3.region", objectInfo.getRegion()); properties.put("s3.root.path", objectInfo.getPrefix()); properties.put("s3.bucket", objectInfo.getBucket()); - properties.put("s3.access_key", objectInfo.getAk()); - properties.put("s3.secret_key", objectInfo.getSk()); + if (objectInfo.hasCredential()) { + properties.putAll(ObjectStorageCredentialAdapter.toProperties( + objectInfo.getProvider().name(), objectInfo.getCredential())); + } else { + // Masking even empty keys would produce nonempty HMAC credentials conflicting with native auth. + properties.put("s3.access_key", objectInfo.getAk()); + properties.put("s3.secret_key", objectInfo.getSk()); + } properties.put("provider", objectInfo.getProvider().name()); properties.put("use_path_style", String.valueOf(objectInfo.getUsePathStyle())); if (objectInfo.hasExternalEndpoint()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/task/PushStoragePolicyTask.java b/fe/fe-core/src/main/java/org/apache/doris/task/PushStoragePolicyTask.java index 0b9a51b39948ef..63cee8f9a9e67f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/task/PushStoragePolicyTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/task/PushStoragePolicyTask.java @@ -84,15 +84,18 @@ public TPushStoragePolicyReq toThrift() { resource.forEach(r -> { TStorageResource item = new TStorageResource(); r.readLock(); - item.setId(r.getId()); - item.setName(r.getName()); - item.setVersion(r.getVersion()); - if (r.getType() == ResourceType.S3) { - item.setS3StorageParam(S3ThriftAdapter.getS3TStorageParam(r.getCopiedProperties())); - } else if (r.getType() == ResourceType.HDFS) { - item.setHdfsStorageParam(HdfsResource.generateHdfsParam(r.getCopiedProperties())); + try { + item.setId(r.getId()); + item.setName(r.getName()); + item.setVersion(r.getVersion()); + if (r.getType() == ResourceType.S3) { + item.setS3StorageParam(S3ThriftAdapter.getS3TStorageParam(r.getCopiedProperties())); + } else if (r.getType() == ResourceType.HDFS) { + item.setHdfsStorageParam(HdfsResource.generateHdfsParam(r.getCopiedProperties())); + } + } finally { + r.readUnlock(); } - r.readUnlock(); tStorageResources.add(item); }); ret.setResource(tStorageResources); diff --git a/fe/fe-core/src/test/java/org/apache/doris/analysis/StorageDescPersistTest.java b/fe/fe-core/src/test/java/org/apache/doris/analysis/StorageDescPersistTest.java index b029e9695099a5..8b9ad26cdb0b32 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/analysis/StorageDescPersistTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/analysis/StorageDescPersistTest.java @@ -21,6 +21,7 @@ import org.apache.doris.load.EtlJobType; import org.apache.doris.load.loadv2.BrokerLoadJob; import org.apache.doris.persist.gson.GsonUtils; +import org.apache.doris.thrift.TFileType; import com.google.common.collect.Maps; import org.junit.jupiter.api.Assertions; @@ -75,6 +76,25 @@ public void testBrokerLoadJobRestoreS3StoragePropertiesAfterGsonRoundTrip() thro restoredBrokerDesc.getStorageAdapter().getOrigProps().get("s3.bucket")); } + @Test + public void testGcpNativeCredentialForExportBroker() throws Exception { + Map properties = Maps.newHashMap(); + properties.put("provider", "GCP"); + properties.put("s3.region", "us-east1"); + properties.put("gs.credential_provider_type", "DEFAULT"); + + // EXPORT forwards this BrokerDesc map to BE Writer. Verify the shared FE conversion + // preserves the provider-specific credential while keeping the S3-compatible file type. + BrokerDesc brokerDesc = new BrokerDesc("S3", StorageBackend.StorageType.S3, properties); + Assertions.assertEquals("GCS", brokerDesc.getStorageAdapter().getSpiProperties().providerName()); + Assertions.assertEquals(TFileType.FILE_S3, brokerDesc.getFileType()); + Assertions.assertEquals("GCP", brokerDesc.getBackendConfigProperties().get("provider")); + Assertions.assertEquals("DEFAULT", brokerDesc.getBackendConfigProperties() + .get("gs.credential_provider_type")); + Assertions.assertEquals("s3://export-bucket/path", + brokerDesc.getFileLocation("gs://export-bucket/path")); + } + private static void setField(Class clazz, Object target, String fieldName, Object value) throws ReflectiveOperationException { Field field = clazz.getDeclaredField(fieldName); diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/S3ResourceTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/S3ResourceTest.java index 57dff9d4d38ed9..476a2c76c4fa3b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/S3ResourceTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/S3ResourceTest.java @@ -17,18 +17,28 @@ package org.apache.doris.catalog; +import org.apache.doris.cloud.proto.Cloud; import org.apache.doris.common.DdlException; import org.apache.doris.common.FeConstants; import org.apache.doris.common.FeMetaVersion; import org.apache.doris.common.UserException; import org.apache.doris.datasource.storage.CloudObjectStoreAdapter; import org.apache.doris.datasource.storage.S3ResourceCompat; +import org.apache.doris.datasource.storage.S3ThriftAdapter; +import org.apache.doris.datasource.storage.StorageAdapter; +import org.apache.doris.filesystem.properties.S3CompatibleFileSystemProperties; import org.apache.doris.meta.MetaContext; import org.apache.doris.mysql.privilege.AccessControllerManager; import org.apache.doris.mysql.privilege.PrivPredicate; import org.apache.doris.nereids.trees.plans.commands.CreateResourceCommand; import org.apache.doris.nereids.trees.plans.commands.info.CreateResourceInfo; +import org.apache.doris.policy.StoragePolicy; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.task.PushStoragePolicyTask; +import org.apache.doris.thrift.TGcpCredential; +import org.apache.doris.thrift.TObjStorageType; +import org.apache.doris.thrift.TPushStoragePolicyReq; +import org.apache.doris.thrift.TS3StorageParam; import com.google.common.base.Strings; import com.google.common.collect.ImmutableMap; @@ -41,13 +51,21 @@ import org.mockito.MockedStatic; import org.mockito.Mockito; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.io.DataInputStream; import java.io.DataOutputStream; +import java.lang.reflect.Field; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.HashMap; +import java.util.List; import java.util.Map; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; public class S3ResourceTest { private static final Logger LOG = LogManager.getLogger(S3ResourceTest.class); @@ -213,6 +231,69 @@ public void testSerialization() throws Exception { Files.deleteIfExists(path); } + @Test + public void testConcurrentAltersPreserveBothUpdates() throws Exception { + CountDownLatch firstReady = new CountDownLatch(1); + CountDownLatch releaseFirst = new CountDownLatch(1); + CountDownLatch secondStarted = new CountDownLatch(1); + AtomicBoolean first = new AtomicBoolean(true); + AtomicReference failure = new AtomicReference<>(); + S3Resource resource = new S3Resource("concurrent_alter") { + @Override + public void writeLock() { + if (first.compareAndSet(true, false)) { + firstReady.countDown(); + try { + Assertions.assertTrue(releaseFirst.await(10, TimeUnit.SECONDS)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new AssertionError(e); + } + } + super.writeLock(); + } + }; + resource.setProperties(ImmutableMap.copyOf(s3Properties)); + Thread updateConnections = new Thread(() -> { + try { + resource.modifyProperties(ImmutableMap.of(S3ResourceCompat.MAX_CONNECTIONS, "77")); + } catch (Throwable e) { + failure.compareAndSet(null, e); + } + }); + Thread updateTimeout = new Thread(() -> { + secondStarted.countDown(); + try { + resource.modifyProperties(ImmutableMap.of(S3ResourceCompat.REQUEST_TIMEOUT_MS, "8888")); + } catch (Throwable e) { + failure.compareAndSet(null, e); + } + }); + try { + updateConnections.start(); + Assertions.assertTrue(firstReady.await(10, TimeUnit.SECONDS)); + updateTimeout.start(); + Assertions.assertTrue(secondStarted.await(10, TimeUnit.SECONDS)); + // The second ALTER either waits for the first one's monitor, or (before the fix) + // finishes and is subsequently overwritten by the first one's stale snapshot. + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(10); + while (updateTimeout.isAlive() && updateTimeout.getState() != Thread.State.BLOCKED + && System.nanoTime() < deadline) { + Thread.yield(); + } + Assertions.assertTrue(!updateTimeout.isAlive() || updateTimeout.getState() == Thread.State.BLOCKED); + } finally { + releaseFirst.countDown(); + updateConnections.join(10000); + updateTimeout.join(10000); + } + Assertions.assertFalse(updateConnections.isAlive()); + Assertions.assertFalse(updateTimeout.isAlive()); + Assertions.assertNull(failure.get()); + Assertions.assertEquals("77", resource.getProperty(S3ResourceCompat.MAX_CONNECTIONS)); + Assertions.assertEquals("8888", resource.getProperty(S3ResourceCompat.REQUEST_TIMEOUT_MS)); + } + @Test public void testModifyProperties() throws Exception { ImmutableMap properties = ImmutableMap.of( @@ -323,4 +404,777 @@ public void testPingS3WithRoleArn() { Assertions.assertTrue(false, e.getMessage()); } } + + @Test + public void testGsEndpointAndAlterAliases() throws Exception { + Map properties = new HashMap<>(); + properties.put("provider", "gcp"); + properties.put("s3.endpoint", "https://old.example.com"); + properties.put("gs.endpoint", "https://storage.googleapis.com"); + properties.put("s3.region", "us-central1"); + properties.put("s3.bucket", "bucket"); + properties.put("s3.root.path", "prefix"); + properties.put("s3.access_key", "access"); + properties.put("s3.secret_key", "secret"); + properties.put("s3_validity_check", "false"); + properties.put("type", "s3"); + S3Resource resource = new S3Resource("gcp_resource"); + resource.setProperties(ImmutableMap.copyOf(properties)); + Assertions.assertEquals("us-central1", resource.getProperty(S3ResourceCompat.REGION)); + Assertions.assertEquals("bucket", resource.getProperty(S3ResourceCompat.BUCKET)); + Assertions.assertEquals("prefix", resource.getProperty(S3ResourceCompat.ROOT_PATH)); + Assertions.assertEquals("https://storage.googleapis.com", resource.getProperty(S3ResourceCompat.ENDPOINT)); + Assertions.assertEquals("https://storage.googleapis.com", + S3ThriftAdapter.getS3TStorageParam(resource.getCopiedProperties()).getEndpoint()); + Assertions.assertFalse(resource.getCopiedProperties().containsKey("gs.endpoint")); + resource.modifyProperties(ImmutableMap.of("gs.endpoint", "https://custom.example.com", + "s3_validity_check", "false")); + Assertions.assertEquals("https://custom.example.com", resource.getProperty(S3ResourceCompat.ENDPOINT)); + resource.modifyProperties(ImmutableMap.of("s3.endpoint", "https://storage.googleapis.com", + "s3_validity_check", "false")); + Assertions.assertEquals("https://storage.googleapis.com", resource.getProperty(S3ResourceCompat.ENDPOINT)); + Assertions.assertEquals("https://storage.googleapis.com", properties.get("gs.endpoint")); + Assertions.assertEquals("https://old.example.com", properties.get("s3.endpoint")); + } + + @Test + public void testExplicitS3DefaultChainSurvivesGcsPolicyAndVaultCreation() throws Exception { + Map properties = new HashMap<>(); + properties.put("provider", "S3"); + properties.put("type", "s3"); + properties.put("s3.endpoint", "https://storage.googleapis.com"); + properties.put("s3.region", "us-central1"); + properties.put("s3.bucket", "bucket"); + properties.put("s3.root.path", "prefix"); + AtomicBoolean pinged = new AtomicBoolean(); + try (MockedStatic resourceMock = Mockito.mockStatic(S3Resource.class, Mockito.CALLS_REAL_METHODS); + MockedStatic envMock = Mockito.mockStatic(Env.class)) { + envMock.when(Env::getCurrentEnv).thenReturn(Mockito.mock(Env.class)); + resourceMock.when(() -> S3Resource.pingS3(Mockito.anyString(), Mockito.anyString(), Mockito.anyMap())) + .thenAnswer(invocation -> { + Map pingProperties = invocation.getArgument(2); + StorageAdapter adapter = StorageAdapter.of(pingProperties); + Assertions.assertEquals("S3", adapter.getSpiProperties().providerName()); + Assertions.assertEquals("DEFAULT", adapter.getBackendConfigProperties() + .get("AWS_CREDENTIALS_PROVIDER_TYPE")); + pinged.set(true); + return null; + }); + CreateResourceCommand command = new CreateResourceCommand( + new CreateResourceInfo(false, false, "s3_vault", ImmutableMap.copyOf(properties))); + command.getInfo().analyzeResourceType(); + S3StorageVault vault = new S3StorageVault("s3_vault", false, false, command); + Assertions.assertTrue(pinged.get()); + Map stored = vault.getCopiedProperties(); + Assertions.assertFalse(S3ThriftAdapter.getS3TStorageParam(stored).isSetCredProviderType()); + Assertions.assertFalse(CloudObjectStoreAdapter.getObjStoreInfoPB(stored).hasCredProviderType()); + // Anonymous remains an explicit choice and remains forbidden for GCS vaults. + stored.put("s3.credentials_provider_type", "ANONYMOUS"); + Assertions.assertEquals(org.apache.doris.thrift.TCredProviderType.ANONYMOUS, + S3ThriftAdapter.getS3TStorageParam(stored).getCredProviderType()); + Assertions.assertThrows(IllegalArgumentException.class, + () -> CloudObjectStoreAdapter.getObjStoreInfoPB(stored)); + } + } + + @Test + public void testInferredGcpAliasesForResourceAndVault() throws Exception { + Map properties = new HashMap<>(); + properties.put("type", "s3"); + properties.put("gs.endpoint", "https://storage.googleapis.com"); + properties.put("gs.credential_provider_type", "DEFAULT"); + properties.put("s3.bucket", "bucket"); + properties.put("s3.root.path", "prefix"); + properties.put("s3_validity_check", "false"); + S3Resource resource = new S3Resource("inferred_gcp"); + resource.setProperties(ImmutableMap.copyOf(properties)); + Assertions.assertEquals("GCP", resource.getProperty("provider")); + Assertions.assertEquals(properties.get("gs.endpoint"), resource.getProperty("s3.endpoint")); + resource.modifyProperties(ImmutableMap.of("gs.connection.timeout", "789")); + Assertions.assertEquals("789", resource.getProperty("s3.connection.timeout")); + try (MockedStatic envMock = Mockito.mockStatic(Env.class)) { + envMock.when(Env::getCurrentEnv).thenReturn(Mockito.mock(Env.class)); + CreateResourceCommand command = new CreateResourceCommand( + new CreateResourceInfo(false, false, "gcp_vault", ImmutableMap.copyOf(properties))); + command.getInfo().analyzeResourceType(); + S3StorageVault vault = new S3StorageVault("gcp_vault", false, false, command); + Assertions.assertEquals("https://storage.googleapis.com", + CloudObjectStoreAdapter.getObjStoreInfoPB(vault.getCopiedProperties()).getEndpoint()); + Assertions.assertTrue(CloudObjectStoreAdapter.getObjStoreInfoPB(vault.getCopiedProperties()) + .getCredential().hasGcpCredential()); + } + // Legacy persisted resources can have an inferred identity without a provider field. + Field field = S3Resource.class.getDeclaredField("properties"); + field.setAccessible(true); + Map legacy = resource.getCopiedProperties(); + legacy.remove("provider"); + field.set(resource, legacy); + resource.modifyProperties(ImmutableMap.of("gs.connection.timeout", "987")); + Assertions.assertEquals("987", resource.getProperty("s3.connection.timeout")); + Assertions.assertEquals("GCP", resource.getProperty("provider")); + } + + @Test + public void testGsEndpointForStorageVault() throws Exception { + Env env = Mockito.mock(Env.class); + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + for (String provider : new String[] {"GCP", "gcp", "S3", "OSS"}) { + Map properties = new HashMap<>(s3Properties); + properties.put("provider", provider); + properties.put("gs.endpoint", "https://storage.googleapis.com"); + properties.put("gs.access_key", "gcp-access"); + properties.put("gs.secret_key", "gcp-secret"); + properties.put("gs.use_path_style", "true"); + properties.put("s3.root.path", "prefix"); + CreateResourceCommand command = new CreateResourceCommand( + new CreateResourceInfo(false, false, "gcp_vault", ImmutableMap.copyOf(properties))); + command.getInfo().analyzeResourceType(); + S3StorageVault vault = new S3StorageVault("gcp_vault", false, false, command); + vault.checkCreationProperties(ImmutableMap.copyOf(properties)); + Assertions.assertEquals("prefix", vault.getCopiedProperties().get(S3ResourceCompat.ROOT_PATH)); + String expectedEndpoint = "GCP".equalsIgnoreCase(provider) + ? "https://storage.googleapis.com" : s3Endpoint; + Assertions.assertEquals(expectedEndpoint, + CloudObjectStoreAdapter.getObjStoreInfoPB(vault.getCopiedProperties()).getEndpoint()); + Assertions.assertEquals("GCP".equalsIgnoreCase(provider) ? "gcp-access" : s3AccessKey, + CloudObjectStoreAdapter.getObjStoreInfoPB(vault.getCopiedProperties()).getAk()); + Assertions.assertEquals("GCP".equalsIgnoreCase(provider) ? "gcp-secret" : s3SecretKey, + CloudObjectStoreAdapter.getObjStoreInfoPB(vault.getCopiedProperties()).getSk()); + Assertions.assertEquals("GCP".equalsIgnoreCase(provider), + CloudObjectStoreAdapter.getObjStoreInfoPB(vault.getCopiedProperties()).getUsePathStyle()); + } + } + } + + @Test + public void testStorageVaultRejectsNonAllowlistedAliases() throws Exception { + StorageVaultMgr manager = new StorageVaultMgr(null); + for (String key : new String[] {"gs.endpoint", S3ResourceCompat.ENDPOINT, S3ResourceCompat.Env.ENDPOINT, + "gs.access_key", "gs.secret_key", "gs.session_token", "gs.connection.maximum", + "gs.connection.request.timeout", "gs.connection.timeout", "gs.use_path_style", + "gs.force_parsing_by_standard_uri"}) { + try { + manager.alterStorageVault(StorageVault.StorageVaultType.S3, + ImmutableMap.of(key, "https://custom.example.com"), "gcp_vault"); + Assertions.fail("Storage Vault must reject properties outside its ALTER whitelist: " + key); + } catch (IllegalArgumentException exception) { + Assertions.assertEquals("Alter property " + key + " is not allowed.", exception.getMessage()); + } + } + } + + @Test + public void testBlankGsEndpointPreservesResourceAliases() throws Exception { + for (String blank : new String[] {"", " \t"}) { + for (String endpointKey : new String[] {S3ResourceCompat.ENDPOINT, S3ResourceCompat.Env.ENDPOINT}) { + Map properties = new HashMap<>(s3Properties); + properties.put("provider", "GCP"); + properties.remove(S3ResourceCompat.Env.ENDPOINT); + properties.put(endpointKey, s3Endpoint); + properties.put("gs.endpoint", blank); + properties.put("gs.connection.timeout", blank); + S3Resource resource = new S3Resource("gcp_resource"); + resource.setProperties(ImmutableMap.copyOf(properties)); + Assertions.assertEquals(s3Endpoint, resource.getProperty(S3ResourceCompat.ENDPOINT)); + Assertions.assertFalse(resource.getCopiedProperties().containsKey("gs.endpoint")); + Assertions.assertEquals(Integer.parseInt(s3ConnTimeoutMs), + S3ThriftAdapter.getS3TStorageParam(resource.getCopiedProperties()).getConnTimeoutMs()); + Map updated = new HashMap<>(); + updated.put("gs.endpoint", blank); + updated.put("s3_validity_check", "false"); + resource.modifyProperties(updated); + Assertions.assertEquals(s3Endpoint, resource.getProperty(S3ResourceCompat.ENDPOINT)); + Assertions.assertEquals(blank, updated.get("gs.endpoint")); + Assertions.assertFalse(resource.getCopiedProperties().containsKey("gs.connection.timeout")); + } + } + } + + @Test + public void testGsEndpointCannotAlterPolicyResource() throws Exception { + S3Resource resource = new S3Resource("gcp_resource"); + Map properties = new HashMap<>(s3Properties); + properties.put("provider", "GCP"); + resource.setProperties(ImmutableMap.copyOf(properties)); + resource.references.put("policy", Resource.ReferenceType.POLICY); + try { + resource.modifyProperties(ImmutableMap.of("gs.endpoint", "https://custom.example.com")); + Assertions.fail("A gs.endpoint alias must not bypass the policy endpoint restriction"); + } catch (DdlException exception) { + Assertions.assertTrue(exception.getMessage().contains(S3ResourceCompat.ENDPOINT)); + } + Assertions.assertEquals(s3Endpoint, resource.getProperty(S3ResourceCompat.ENDPOINT)); + } + + @Test + public void testGsEndpointDoesNotOverrideOtherProviders() throws Exception { + for (String provider : new String[] {"S3", "OSS", "AZURE"}) { + Map properties = new HashMap<>(s3Properties); + if (!provider.isEmpty()) { + properties.put("provider", provider); + } + properties.put("gs.endpoint", "https://storage.googleapis.com"); + S3Resource resource = new S3Resource("other_resource"); + resource.setProperties(ImmutableMap.copyOf(properties)); + Assertions.assertEquals(s3Endpoint, resource.getProperty(S3ResourceCompat.ENDPOINT)); + resource.modifyProperties(ImmutableMap.of("gs.endpoint", "https://custom.example.com")); + Assertions.assertEquals(s3Endpoint, resource.getProperty(S3ResourceCompat.ENDPOINT)); + Assertions.assertEquals(s3Endpoint, resource.getProperty(S3ResourceCompat.Env.ENDPOINT)); + } + } + + @Test + public void testGsEndpointUsesEffectiveProviderOnAlter() throws Exception { + S3Resource resource = new S3Resource("gcp_resource"); + Map properties = new HashMap<>(s3Properties); + properties.put("provider", "GCP"); + resource.setProperties(ImmutableMap.copyOf(properties)); + resource.modifyProperties(ImmutableMap.of("provider", "S3", "gs.endpoint", "https://ignored.example.com")); + Assertions.assertEquals("S3", resource.getProperty("provider")); + Assertions.assertEquals(s3Endpoint, resource.getProperty(S3ResourceCompat.ENDPOINT)); + resource.modifyProperties(ImmutableMap.of("provider", "gcp", "gs.endpoint", "https://storage.googleapis.com")); + Assertions.assertEquals("gcp", resource.getProperty("provider")); + Assertions.assertEquals("https://storage.googleapis.com", resource.getProperty(S3ResourceCompat.ENDPOINT)); + resource.modifyProperties(ImmutableMap.of("provider", "", "gs.endpoint", "https://custom.example.com")); + Assertions.assertEquals("gcp", resource.getProperty("provider")); + Assertions.assertEquals("https://custom.example.com", resource.getProperty(S3ResourceCompat.ENDPOINT)); + } + + @Test + public void testInactiveGsAliasesCannotReappearAfterProviderAlter() throws Exception { + for (boolean replayLegacyAliases : new boolean[] {false, true}) { + S3Resource resource = new S3Resource("s3_resource"); + Map properties = new HashMap<>(s3Properties); + properties.put("provider", "S3"); + resource.setProperties(ImmutableMap.copyOf(properties)); + Map aliases = ImmutableMap.of("gs.endpoint", "https://ignored.example.com", + "gs.access_key", "ignored-access", "gs.secret_key", "ignored-secret"); + if (replayLegacyAliases) { + // Simulate the dormant aliases persisted by an older FE. + Map stored = resource.getCopiedProperties(); + stored.putAll(aliases); + Field field = S3Resource.class.getDeclaredField("properties"); + field.setAccessible(true); + field.set(resource, stored); + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try (DataOutputStream output = new DataOutputStream(bytes)) { + resource.write(output); + } + try (DataInputStream input = new DataInputStream(new ByteArrayInputStream(bytes.toByteArray()))) { + resource = (S3Resource) Resource.read(input); + } + } else { + resource.modifyProperties(aliases); + Assertions.assertFalse(resource.getCopiedProperties().keySet().stream() + .anyMatch(key -> key.startsWith("gs."))); + } + resource.modifyProperties(ImmutableMap.of("provider", "GCP")); + TS3StorageParam thrift = S3ThriftAdapter.getS3TStorageParam(resource.getCopiedProperties()); + Assertions.assertEquals(s3Endpoint, thrift.getEndpoint()); + Assertions.assertEquals(s3AccessKey, thrift.getAk()); + Assertions.assertEquals(s3SecretKey, thrift.getSk()); + Assertions.assertFalse(resource.getCopiedProperties().keySet().stream() + .anyMatch(key -> key.startsWith("gs."))); + } + } + + @Test + public void testLegacyGcpDefaultChainSurvivesReplayAndPolicyPush() throws Exception { + for (String key : new String[] {"s3.credentials_provider_type", "AWS_CREDENTIALS_PROVIDER_TYPE"}) { + Map properties = new HashMap<>(s3Properties); + properties.remove("AWS_ACCESS_KEY"); + properties.remove("AWS_SECRET_KEY"); + properties.put("provider", "GCP"); + properties.put(key, "DEFAULT"); + S3Resource resource = new S3Resource("gcp_default_chain"); + resource.setProperties(ImmutableMap.copyOf(properties)); + resource.modifyProperties(ImmutableMap.of(S3ResourceCompat.CONNECTION_TIMEOUT_MS, "2000")); + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try (DataOutputStream output = new DataOutputStream(bytes)) { + resource.write(output); + } + S3Resource restored; + try (DataInputStream input = new DataInputStream(new ByteArrayInputStream(bytes.toByteArray()))) { + restored = (S3Resource) Resource.read(input); + } + Assertions.assertEquals(resource.getCopiedProperties(), restored.getCopiedProperties()); + Assertions.assertEquals("DEFAULT", StorageAdapter.of(restored.getCopiedProperties()) + .getBackendConfigProperties().get("AWS_CREDENTIALS_PROVIDER_TYPE")); + PushStoragePolicyTask task = new PushStoragePolicyTask(1L, List.of(), List.of(restored), List.of()); + TS3StorageParam thrift = task.toThrift().getResource().get(0).getS3StorageParam(); + Assertions.assertFalse(thrift.isSetCredential()); + Assertions.assertFalse(thrift.isSetCredProviderType()); + Assertions.assertEquals(2000, thrift.getConnTimeoutMs()); + } + } + + @Test + public void testLegacyGcpStaticKeysSurviveReplayAndBatchedPolicyPush() throws Exception { + for (String key : new String[] {"s3.credentials_provider_type", "AWS_CREDENTIALS_PROVIDER_TYPE"}) { + Map properties = new HashMap<>(s3Properties); + properties.put("provider", "GCP"); + S3Resource resource = new S3Resource("legacy_gcp_static_keys"); + resource.id = 1L; + resource.version = 1L; + resource.setProperties(ImmutableMap.copyOf(properties)); + // Recreate metadata accepted by older FEs without invoking current CREATE validation. + Map legacyProperties = resource.getCopiedProperties(); + legacyProperties.put(key, "ANONYMOUS"); + Field field = S3Resource.class.getDeclaredField("properties"); + field.setAccessible(true); + field.set(resource, legacyProperties); + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try (DataOutputStream output = new DataOutputStream(bytes)) { + resource.write(output); + } + S3Resource restored; + try (DataInputStream input = new DataInputStream(new ByteArrayInputStream(bytes.toByteArray()))) { + restored = (S3Resource) Resource.read(input); + } + ResourceMgr resourceMgr = new ResourceMgr(); + resourceMgr.replayCreateResource(restored); + Assertions.assertEquals(legacyProperties, restored.getCopiedProperties()); + Assertions.assertSame(restored, resourceMgr.getResource(restored.getName())); + + S3Resource unrelated = new S3Resource("unrelated_resource"); + unrelated.id = 2L; + unrelated.version = 1L; + unrelated.setProperties(ImmutableMap.copyOf(s3Properties)); + resourceMgr.replayCreateResource(unrelated); + StoragePolicy policy = Mockito.mock(StoragePolicy.class); + Mockito.when(policy.getId()).thenReturn(3L); + Mockito.when(policy.getPolicyName()).thenReturn("unrelated_policy"); + Mockito.when(policy.getStorageResource()).thenReturn(unrelated.getName()); + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + Env env = Mockito.mock(Env.class); + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + Mockito.when(env.getResourceMgr()).thenReturn(resourceMgr); + PushStoragePolicyTask task = new PushStoragePolicyTask(1L, List.of(policy), + List.of(restored, unrelated), List.of(9L)); + TPushStoragePolicyReq request = task.toThrift(); + Assertions.assertEquals(2, request.getResourceSize()); + Assertions.assertEquals(1L, request.getResource().get(0).getId()); + Assertions.assertEquals(2L, request.getResource().get(1).getId()); + Assertions.assertEquals(1, request.getStoragePolicySize()); + Assertions.assertEquals(3L, request.getStoragePolicy().get(0).getId()); + Assertions.assertEquals(2L, request.getStoragePolicy().get(0).getResourceId()); + Assertions.assertEquals(List.of(9L), request.getDroppedStoragePolicy()); + TS3StorageParam thrift = request.getResource().get(0).getS3StorageParam(); + Assertions.assertEquals(TObjStorageType.GCP, thrift.getProvider()); + Assertions.assertEquals(s3AccessKey, thrift.getAk()); + Assertions.assertEquals(s3SecretKey, thrift.getSk()); + Assertions.assertFalse(thrift.isSetCredential()); + Assertions.assertFalse(thrift.isSetCredProviderType()); + Assertions.assertEquals(s3AccessKey, request.getResource().get(1).getS3StorageParam().getAk()); + } + } + } + + @Test + public void testMinioAlterSurvivesReplayAndBatchedPolicyPush() throws Exception { + S3Resource minio = new S3Resource("minio_resource"); + minio.id = 1L; + minio.version = 0L; + minio.setProperties(ImmutableMap.copyOf(s3Properties)); + minio.modifyProperties(ImmutableMap.of("provider", "MINIO", "s3_validity_check", "false")); + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try (DataOutputStream output = new DataOutputStream(bytes)) { + minio.write(output); + } + S3Resource restored; + try (DataInputStream input = new DataInputStream(new ByteArrayInputStream(bytes.toByteArray()))) { + restored = (S3Resource) Resource.read(input); + } + Assertions.assertEquals("MINIO", restored.getProperty("provider")); + S3Resource unrelated = new S3Resource("unrelated_resource"); + unrelated.id = 2L; + unrelated.version = 0L; + unrelated.setProperties(ImmutableMap.copyOf(s3Properties)); + unrelated.modifyProperties(ImmutableMap.of(S3ResourceCompat.CONNECTION_TIMEOUT_MS, "2000")); + // The MinIO resource is unreferenced but participates in the same update as other resources/drops. + PushStoragePolicyTask task = new PushStoragePolicyTask(1L, List.of(), + List.of(restored, unrelated), List.of(9L)); + TPushStoragePolicyReq request = task.toThrift(); + Assertions.assertEquals(2, request.getResourceSize()); + Assertions.assertEquals(1L, request.getResource().get(0).getId()); + Assertions.assertEquals(2L, request.getResource().get(1).getId()); + Assertions.assertEquals(List.of(9L), request.getDroppedStoragePolicy()); + TS3StorageParam minioThrift = request.getResource().get(0).getS3StorageParam(); + Assertions.assertEquals(TObjStorageType.AWS, minioThrift.getProvider()); + Assertions.assertEquals(s3AccessKey, minioThrift.getAk()); + Assertions.assertEquals(s3SecretKey, minioThrift.getSk()); + Assertions.assertEquals(2000, request.getResource().get(1).getS3StorageParam().getConnTimeoutMs()); + } + + @Test + public void testPolicyPushReleasesResourceLockOnInvalidAuthentication() throws Exception { + S3Resource resource = new S3Resource("invalid_gcp_resource"); + Map properties = new HashMap<>(s3Properties); + properties.put("provider", "GCP"); + resource.setProperties(ImmutableMap.copyOf(properties)); + Map invalid = resource.getCopiedProperties(); + invalid.put("gs.credential_provider_type", "DEFAULT"); + Field field = S3Resource.class.getDeclaredField("properties"); + field.setAccessible(true); + field.set(resource, invalid); + S3Resource tracked = Mockito.spy(resource); + PushStoragePolicyTask task = new PushStoragePolicyTask(1L, List.of(), List.of(tracked), List.of()); + Assertions.assertThrows(IllegalArgumentException.class, task::toThrift); + Mockito.verify(tracked).readLock(); + Mockito.verify(tracked).readUnlock(); + } + + @Test + public void testGsAliasesMatchConnectorAndResourceProtocols() throws Exception { + for (String provider : new String[] {"GCP", "gcp", "GCS", "gcs"}) { + Map properties = new HashMap<>(s3Properties); + properties.put("provider", provider); + properties.put("gs.endpoint", "https://storage.googleapis.com"); + properties.put("gs.access_key", "gcp-access"); + properties.put("gs.secret_key", "gcp-secret"); + properties.put("gs.session_token", "gcp-token"); + properties.put("gs.connection.maximum", "123"); + properties.put("gs.connection.request.timeout", "456"); + properties.put("gs.connection.timeout", "789"); + properties.put("gs.use_path_style", "true"); + properties.put("gs.force_parsing_by_standard_uri", "true"); + properties.put(S3ResourceCompat.SECRET_KEY, "other-secret"); + properties.put(S3ResourceCompat.CONNECTION_TIMEOUT_MS, "999"); + Map original = new HashMap<>(properties); + S3CompatibleFileSystemProperties connector = (S3CompatibleFileSystemProperties) StorageAdapter.of(properties).getSpiProperties(); + + S3Resource resource = new S3Resource("gcp_resource"); + resource.setProperties(ImmutableMap.copyOf(properties)); + Map normalized = resource.getCopiedProperties(); + Assertions.assertFalse(normalized.keySet().stream().anyMatch(key -> key.startsWith("gs."))); + Assertions.assertEquals("true", normalized.get("force_parsing_by_standard_uri")); + TS3StorageParam thrift = S3ThriftAdapter.getS3TStorageParam(normalized); + Assertions.assertEquals(connector.getEndpoint(), thrift.getEndpoint()); + Assertions.assertEquals(connector.getAccessKey(), thrift.getAk()); + Assertions.assertEquals(connector.getSecretKey(), thrift.getSk()); + Assertions.assertEquals(connector.getSessionToken(), thrift.getToken()); + Assertions.assertEquals(Integer.parseInt(connector.getMaxConnections()), thrift.getMaxConn()); + Assertions.assertEquals(Integer.parseInt(connector.getRequestTimeoutMs()), thrift.getRequestTimeoutMs()); + Assertions.assertEquals(Integer.parseInt(connector.getConnectionTimeoutMs()), thrift.getConnTimeoutMs()); + Assertions.assertEquals(Boolean.parseBoolean(connector.getUsePathStyle()), thrift.isUsePathStyle()); + Assertions.assertEquals(Cloud.ObjectStoreInfoPB.Provider.GCP, + CloudObjectStoreAdapter.getObjStoreInfoPB(normalized).getProvider()); + Assertions.assertEquals(connector.getAccessKey(), CloudObjectStoreAdapter.getObjStoreInfoPB(normalized).getAk()); + Assertions.assertEquals(connector.getSecretKey(), CloudObjectStoreAdapter.getObjStoreInfoPB(normalized).getSk()); + Assertions.assertEquals(thrift.isUsePathStyle(), CloudObjectStoreAdapter.getObjStoreInfoPB(normalized).getUsePathStyle()); + Assertions.assertEquals(original, properties); + + resource.modifyProperties(ImmutableMap.of("gs.secret_key", "new-secret", + "gs.connection.timeout", "321", "gs.use_path_style", "false")); + thrift = S3ThriftAdapter.getS3TStorageParam(resource.getCopiedProperties()); + Assertions.assertEquals("new-secret", thrift.getSk()); + Assertions.assertEquals(321, thrift.getConnTimeoutMs()); + Assertions.assertFalse(thrift.isUsePathStyle()); + resource.modifyProperties(ImmutableMap.of(S3ResourceCompat.SECRET_KEY, "canonical-secret", + S3ResourceCompat.SESSION_TOKEN, "")); + connector = (S3CompatibleFileSystemProperties) StorageAdapter.of(resource.getCopiedProperties()).getSpiProperties(); + Assertions.assertEquals("canonical-secret", connector.getSecretKey()); + Assertions.assertEquals("", connector.getSessionToken()); + } + } + + @Test + public void testGcsProviderAlterRotatesKeysAcrossReplayAndPolicyPush() throws Exception { + for (String provider : new String[] {"GCS", "gcs"}) { + Map properties = new HashMap<>(s3Properties); + properties.put("provider", "GCP"); + S3Resource resource = new S3Resource("gcs_resource"); + resource.setProperties(ImmutableMap.copyOf(properties)); + resource.references.put("policy", Resource.ReferenceType.POLICY); + resource.modifyProperties(ImmutableMap.of("provider", provider, + "gs.access_key", "rotated-access", "gs.secret_key", "rotated-secret", + "s3_validity_check", "false")); + Assertions.assertEquals("rotated-access", resource.getProperty(S3ResourceCompat.ACCESS_KEY)); + Assertions.assertEquals("rotated-secret", resource.getProperty(S3ResourceCompat.SECRET_KEY)); + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try (DataOutputStream output = new DataOutputStream(bytes)) { + resource.write(output); + } + S3Resource restored; + try (DataInputStream input = new DataInputStream(new ByteArrayInputStream(bytes.toByteArray()))) { + restored = (S3Resource) Resource.read(input); + } + Assertions.assertEquals(resource.getCopiedProperties(), restored.getCopiedProperties()); + Assertions.assertEquals(provider, restored.getProperty("provider")); + Assertions.assertFalse(restored.getCopiedProperties().containsKey("gs.access_key")); + Assertions.assertFalse(restored.getCopiedProperties().containsKey("gs.secret_key")); + S3CompatibleFileSystemProperties connector = (S3CompatibleFileSystemProperties) + StorageAdapter.of(restored.getCopiedProperties()).getSpiProperties(); + Assertions.assertEquals("rotated-access", connector.getAccessKey()); + Assertions.assertEquals("rotated-secret", connector.getSecretKey()); + PushStoragePolicyTask task = new PushStoragePolicyTask(1L, List.of(), List.of(restored), List.of()); + TS3StorageParam thrift = task.toThrift().getResource().get(0).getS3StorageParam(); + Assertions.assertEquals(TObjStorageType.GCP, thrift.getProvider()); + Assertions.assertEquals("rotated-access", thrift.getAk()); + Assertions.assertEquals("rotated-secret", thrift.getSk()); + } + } + + @Test + public void testBlankGsCredentialAndConnectionAliases() throws Exception { + for (String blank : new String[] {"", " \t"}) { + Map properties = new HashMap<>(s3Properties); + properties.put("provider", "GCP"); + properties.put("gs.access_key", blank); + properties.put("gs.secret_key", blank); + properties.put("gs.session_token", blank); + properties.put(S3ResourceCompat.SESSION_TOKEN, "canonical-token"); + properties.put("gs.connection.maximum", blank); + properties.put(S3ResourceCompat.MAX_CONNECTIONS, "123"); + properties.put("gs.use_path_style", blank); + properties.put(S3ResourceCompat.USE_PATH_STYLE, "true"); + S3Resource resource = new S3Resource("gcp_resource"); + resource.setProperties(ImmutableMap.copyOf(properties)); + resource.modifyProperties(ImmutableMap.of("gs.secret_key", blank, "gs.connection.maximum", blank)); + TS3StorageParam thrift = S3ThriftAdapter.getS3TStorageParam(resource.getCopiedProperties()); + Assertions.assertEquals(s3AccessKey, thrift.getAk()); + Assertions.assertEquals(s3SecretKey, thrift.getSk()); + Assertions.assertEquals("canonical-token", thrift.getToken()); + Assertions.assertEquals(123, thrift.getMaxConn()); + Assertions.assertTrue(thrift.isUsePathStyle()); + Assertions.assertFalse(resource.getCopiedProperties().keySet().stream().anyMatch(key -> key.startsWith("gs."))); + } + } + + @Test + public void testGsAliasesDoNotOverrideOtherProviders() throws Exception { + for (String provider : new String[] {"S3", "OSS", "AZURE", ""}) { + Map properties = new HashMap<>(s3Properties); + properties.put("provider", provider); + properties.put("gs.access_key", "ignored-access"); + properties.put("gs.secret_key", "ignored-secret"); + properties.put("gs.connection.timeout", "789"); + properties.put("gs.use_path_style", "true"); + S3Resource resource = new S3Resource("other_resource"); + resource.setProperties(ImmutableMap.copyOf(properties)); + resource.modifyProperties(ImmutableMap.of("gs.secret_key", "also-ignored", + "gs.connection.timeout", "321")); + TS3StorageParam thrift = S3ThriftAdapter.getS3TStorageParam(resource.getCopiedProperties()); + Assertions.assertEquals(s3AccessKey, thrift.getAk()); + Assertions.assertEquals(s3SecretKey, thrift.getSk()); + Assertions.assertEquals(Integer.parseInt(s3ConnTimeoutMs), thrift.getConnTimeoutMs()); + Assertions.assertFalse(thrift.isUsePathStyle()); + Assertions.assertFalse(resource.getCopiedProperties().keySet().stream() + .anyMatch(key -> key.startsWith("gs."))); + } + } + + @Test + public void testAlterClearsGcpImpersonation() throws Exception { + MetaContext previousContext = MetaContext.get(); + MetaContext metaContext = new MetaContext(); + metaContext.setMetaVersion(FeMetaVersion.VERSION_CURRENT); + metaContext.setThreadLocalInfo(); + try { + for (String providerType : new String[] {"DEFAULT", "COMPUTE_ENGINE"}) { + try (MockedStatic resourceMock = Mockito.mockStatic( + S3Resource.class, Mockito.CALLS_REAL_METHODS)) { + S3Resource resource = createGcpResourceWithImpersonation(providerType); + resource.modifyProperties(ImmutableMap.of(S3ResourceCompat.CONNECTION_TIMEOUT_MS, "2000")); + Assertions.assertEquals("reader@project.iam.gserviceaccount.com", + resource.getProperty("gs.impersonation_service_account")); + AtomicBoolean pinged = new AtomicBoolean(); + resourceMock.when(() -> S3Resource.pingS3(Mockito.anyString(), Mockito.anyString(), Mockito.anyMap())) + .thenAnswer(invocation -> { + Map pingProperties = invocation.getArgument(2); + Assertions.assertEquals("", pingProperties.get("gs.impersonation_service_account")); + TGcpCredential credential = S3ThriftAdapter.getS3TStorageParam(pingProperties) + .getCredential().getGcpCredential(); + Assertions.assertEquals(providerType, credential.getCredentialProviderType().name()); + Assertions.assertFalse(credential.isSetImpersonationServiceAccount()); + pinged.set(true); + return null; + }); + long previousVersion = resource.getVersion(); + resource.modifyProperties(ImmutableMap.of("gs.impersonation_service_account", "", + S3ResourceCompat.VALIDITY_CHECK, "true")); + Assertions.assertTrue(pinged.get()); + Assertions.assertEquals(previousVersion + 1, resource.getVersion()); + Assertions.assertEquals("", resource.getProperty("gs.impersonation_service_account")); + + ByteArrayOutputStream bytes = new ByteArrayOutputStream(); + try (DataOutputStream output = new DataOutputStream(bytes)) { + resource.write(output); + } + try (DataInputStream input = new DataInputStream(new ByteArrayInputStream(bytes.toByteArray()))) { + S3Resource restored = (S3Resource) Resource.read(input); + Assertions.assertEquals(resource.getCopiedProperties(), restored.getCopiedProperties()); + TGcpCredential credential = S3ThriftAdapter.getS3TStorageParam(restored.getCopiedProperties()) + .getCredential().getGcpCredential(); + Assertions.assertEquals(providerType, credential.getCredentialProviderType().name()); + Assertions.assertFalse(credential.isSetImpersonationServiceAccount()); + } + } + } + } finally { + if (previousContext == null) { + MetaContext.remove(); + } else { + previousContext.setThreadLocalInfo(); + } + } + } + + @Test + public void testFailedAlterPreservesGcpImpersonation() throws Exception { + try (MockedStatic resourceMock = Mockito.mockStatic(S3Resource.class, Mockito.CALLS_REAL_METHODS)) { + resourceMock.when(() -> S3Resource.pingS3(Mockito.anyString(), Mockito.anyString(), Mockito.anyMap())) + .thenAnswer(invocation -> { + Map pingProperties = invocation.getArgument(2); + Assertions.assertEquals("", pingProperties.get("gs.impersonation_service_account")); + throw new DdlException("source credential cannot access bucket"); + }); + for (String providerType : new String[] {"DEFAULT", "COMPUTE_ENGINE"}) { + S3Resource resource = createGcpResourceWithImpersonation(providerType); + Map previousProperties = resource.getCopiedProperties(); + long previousVersion = resource.getVersion(); + Assertions.assertThrows(DdlException.class, () -> resource.modifyProperties( + ImmutableMap.of("gs.impersonation_service_account", "", + S3ResourceCompat.VALIDITY_CHECK, "true"))); + Assertions.assertEquals(previousProperties, resource.getCopiedProperties()); + Assertions.assertEquals(previousVersion, resource.getVersion()); + } + } + } + + private S3Resource createGcpResourceWithImpersonation(String providerType) throws DdlException { + Map properties = new HashMap<>(); + properties.put("provider", "GCP"); + properties.put(S3ResourceCompat.ENDPOINT, "https://storage.googleapis.com"); + properties.put(S3ResourceCompat.REGION, "us-central1"); + properties.put(S3ResourceCompat.BUCKET, "bucket"); + properties.put(S3ResourceCompat.ROOT_PATH, "prefix"); + properties.put(S3ResourceCompat.VALIDITY_CHECK, "false"); + properties.put("gs.credential_provider_type", providerType); + properties.put("gs.impersonation_service_account", "reader@project.iam.gserviceaccount.com"); + S3Resource resource = new S3Resource("gcp_resource"); + resource.setProperties(ImmutableMap.copyOf(properties)); + return resource; + } + + @Test + public void testGsCredentialAliasesCannotBypassNativeAuthValidation() throws Exception { + Map properties = new HashMap<>(); + properties.put("provider", "GCP"); + properties.put("gs.endpoint", "https://storage.googleapis.com"); + properties.put(S3ResourceCompat.BUCKET, "bucket"); + properties.put(S3ResourceCompat.VALIDITY_CHECK, "false"); + properties.put("gs.credential_provider_type", "DEFAULT"); + for (String key : new String[] {"gs.access_key", "gs.secret_key", "gs.session_token", + S3ResourceCompat.ACCESS_KEY, S3ResourceCompat.SECRET_KEY, S3ResourceCompat.SESSION_TOKEN}) { + S3Resource resource = new S3Resource("gcp_resource"); + Map conflicting = new HashMap<>(properties); + conflicting.put(key, "conflicting-credential"); + Assertions.assertThrows(IllegalArgumentException.class, + () -> resource.setProperties(ImmutableMap.copyOf(conflicting))); + + resource.setProperties(ImmutableMap.copyOf(properties)); + Map before = resource.getCopiedProperties(); + Assertions.assertThrows(IllegalArgumentException.class, + () -> resource.modifyProperties(ImmutableMap.of(key, "conflicting-credential"))); + Assertions.assertEquals(before, resource.getCopiedProperties()); + Assertions.assertEquals("DEFAULT", resource.getProperty("gs.credential_provider_type")); + } + S3Resource resource = new S3Resource("gcp_resource"); + properties.remove("gs.credential_provider_type"); + properties.put("gs.access_key", "access"); + properties.put("gs.secret_key", "secret"); + resource.setProperties(ImmutableMap.copyOf(properties)); + Assertions.assertThrows(IllegalArgumentException.class, () -> resource.modifyProperties( + ImmutableMap.of("gs.credential_provider_type", "DEFAULT"))); + // Empty AK/SK updates are ignored by Resource ALTER and cannot clear stored credentials. + Assertions.assertThrows(IllegalArgumentException.class, () -> resource.modifyProperties( + ImmutableMap.of("gs.credential_provider_type", "DEFAULT", + S3ResourceCompat.ACCESS_KEY, "", S3ResourceCompat.SECRET_KEY, ""))); + } + + @Test + public void testAlterOverridesStoredGsAliases() throws Exception { + Map properties = new HashMap<>(s3Properties); + properties.put("provider", "GCP"); + S3Resource resource = new S3Resource("gcp_resource"); + resource.setProperties(ImmutableMap.copyOf(properties)); + // Simulate a resource persisted before gs.* normalization was introduced. + Field field = S3Resource.class.getDeclaredField("properties"); + field.setAccessible(true); + Map legacy = resource.getCopiedProperties(); + legacy.put("gs.access_key", "old-access"); + legacy.put("gs.secret_key", "old-secret"); + legacy.put("gs.session_token", "old-token"); + legacy.put("gs.connection.timeout", "789"); + field.set(resource, legacy); + resource.modifyProperties(ImmutableMap.of(S3ResourceCompat.ACCESS_KEY, "new-access", + S3ResourceCompat.SECRET_KEY, "new-secret", + S3ResourceCompat.SESSION_TOKEN, "")); + Map normalized = resource.getCopiedProperties(); + S3CompatibleFileSystemProperties connector = (S3CompatibleFileSystemProperties) StorageAdapter.of(normalized).getSpiProperties(); + Assertions.assertEquals("new-access", connector.getAccessKey()); + Assertions.assertEquals("new-secret", connector.getSecretKey()); + Assertions.assertEquals("", connector.getSessionToken()); + Assertions.assertEquals(connector.getAccessKey(), S3ThriftAdapter.getS3TStorageParam(normalized).getAk()); + Assertions.assertEquals(connector.getSecretKey(), S3ThriftAdapter.getS3TStorageParam(normalized).getSk()); + Assertions.assertEquals(connector.getSessionToken(), S3ThriftAdapter.getS3TStorageParam(normalized).getToken()); + Assertions.assertEquals(connector.getSecretKey(), CloudObjectStoreAdapter.getObjStoreInfoPB(normalized).getSk()); + Assertions.assertEquals(789, S3ThriftAdapter.getS3TStorageParam(normalized).getConnTimeoutMs()); + Assertions.assertFalse(normalized.containsKey("gs.access_key")); + Assertions.assertFalse(normalized.containsKey("gs.secret_key")); + Assertions.assertFalse(normalized.containsKey("gs.session_token")); + Assertions.assertFalse(normalized.containsKey("gs.connection.timeout")); + + // Within the same ALTER input, gs.* still takes precedence over s3.*. + resource.modifyProperties(ImmutableMap.of("gs.secret_key", "updated-gs-secret", + S3ResourceCompat.SECRET_KEY, "ignored-s3-secret")); + Assertions.assertEquals("updated-gs-secret", resource.getProperty(S3ResourceCompat.SECRET_KEY)); + } + + @Test + public void testBlankStoredGsAliasesFallBackToAlterValues() throws Exception { + for (String blank : new String[] {"", " \t"}) { + Map properties = new HashMap<>(s3Properties); + properties.put("provider", "GCP"); + S3Resource resource = new S3Resource("gcp_resource"); + resource.setProperties(ImmutableMap.copyOf(properties)); + Field field = S3Resource.class.getDeclaredField("properties"); + field.setAccessible(true); + Map legacy = resource.getCopiedProperties(); + legacy.put("gs.secret_key", blank); + legacy.put("gs.session_token", blank); + field.set(resource, legacy); + + resource.modifyProperties(ImmutableMap.of(S3ResourceCompat.SECRET_KEY, "new-secret", + S3ResourceCompat.SESSION_TOKEN, "")); + Map normalized = resource.getCopiedProperties(); + S3CompatibleFileSystemProperties connector = (S3CompatibleFileSystemProperties) StorageAdapter.of(normalized).getSpiProperties(); + Assertions.assertEquals("new-secret", connector.getSecretKey()); + Assertions.assertEquals("", connector.getSessionToken()); + Assertions.assertEquals(connector.getSecretKey(), S3ThriftAdapter.getS3TStorageParam(normalized).getSk()); + Assertions.assertFalse(normalized.containsKey("gs.secret_key")); + Assertions.assertFalse(normalized.containsKey("gs.session_token")); + } + } + + @Test + public void testStoredGsEndpointCannotChangePolicyResourceOnAlter() throws Exception { + Map properties = new HashMap<>(s3Properties); + properties.put("provider", "GCP"); + S3Resource resource = new S3Resource("gcp_resource"); + resource.setProperties(ImmutableMap.copyOf(properties)); + resource.references.put("policy", Resource.ReferenceType.POLICY); + Field field = S3Resource.class.getDeclaredField("properties"); + field.setAccessible(true); + Map legacy = resource.getCopiedProperties(); + legacy.put("gs.endpoint", "https://different.example.com"); + field.set(resource, legacy); + + Assertions.assertThrows(DdlException.class, () -> resource.modifyProperties( + ImmutableMap.of(S3ResourceCompat.SECRET_KEY, "new-secret"))); + Assertions.assertEquals(legacy, resource.getCopiedProperties()); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/cloud/catalog/HdfsStorageVaultTest.java b/fe/fe-core/src/test/java/org/apache/doris/cloud/catalog/HdfsStorageVaultTest.java index 2657aede6ad857..03d292b7442d4d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/cloud/catalog/HdfsStorageVaultTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/cloud/catalog/HdfsStorageVaultTest.java @@ -27,6 +27,7 @@ import org.apache.doris.cloud.rpc.MetaServiceProxy; import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; +import org.apache.doris.datasource.storage.StorageAdapter; import org.apache.doris.nereids.trees.plans.commands.CreateStorageVaultCommand; import org.apache.doris.system.SystemInfoService; @@ -42,6 +43,8 @@ import org.mockito.Mockito; import java.util.HashSet; +import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.Set; @@ -262,6 +265,48 @@ public void testCheckConnectivityException() { }); } + @Test + public void testRejectS3PropertiesBeforeConnectivity() { + try (MockedStatic adapter = Mockito.mockStatic(StorageAdapter.class)) { + for (String key : new String[] {"s3.region", "s3.endpoint", "s3.root.path", "provider", "s3.bucket"}) { + for (String invalidKey : new String[] {key, key.toUpperCase(Locale.ROOT)}) { + for (String validityCheck : new String[] {"true", "false"}) { + ImmutableMap properties = ImmutableMap.of( + "FS.DEFAULTFS", "hdfs://localhost:10000", + HdfsStorageVault.PropertyKey.VAULT_PATH_PREFIX, "testInvalidProperties", + HdfsStorageVault.S3_VALIDITY_CHECK, validityCheck, + invalidKey, "invalid"); + HdfsStorageVault vault = new HdfsStorageVault("testHdfsVault", false, false); + IllegalArgumentException exception = Assertions.assertThrows( + IllegalArgumentException.class, () -> vault.modifyProperties(properties)); + Assertions.assertEquals("Invalid argument " + invalidKey, exception.getMessage()); + } + } + } + adapter.verifyNoInteractions(); + } + } + + @Test + public void testKerberosValidationBeforeConnectivity() { + for (Map credentials : List.>of( + ImmutableMap.of(), + ImmutableMap.of(HdfsStorageVault.PropertyKey.HADOOP_KERBEROS_PRINCIPAL, "hadoop/localhost@EXAMPLE.COM"), + ImmutableMap.of(HdfsStorageVault.PropertyKey.HADOOP_KERBEROS_KEYTAB, "/path/to/test.keytab"))) { + ImmutableMap properties = ImmutableMap.builder() + .put(HdfsStorageVault.PropertyKey.HADOOP_FS_NAME, "hdfs://localhost:10000") + .put(HdfsStorageVault.PropertyKey.VAULT_PATH_PREFIX, "testKerberosProperties") + .put(HdfsStorageVault.PropertyKey.HADOOP_SECURITY_AUTHENTICATION, "kerberos") + .putAll(credentials) + .build(); + HdfsStorageVault vault = new HdfsStorageVault("testHdfsVault", false, false); + IllegalArgumentException exception = Assertions.assertThrows( + IllegalArgumentException.class, () -> vault.modifyProperties(properties)); + Assertions.assertEquals("HDFS authentication type is kerberos, but principal or keytab is not set.", + exception.getMessage()); + } + } + @Test public void testIgnoreCheckConnectivity() throws DdlException { ImmutableMap properties = ImmutableMap.of( diff --git a/fe/fe-core/src/test/java/org/apache/doris/cloud/catalog/S3StorageVaultTest.java b/fe/fe-core/src/test/java/org/apache/doris/cloud/catalog/S3StorageVaultTest.java new file mode 100644 index 00000000000000..a407a9c848fba0 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/cloud/catalog/S3StorageVaultTest.java @@ -0,0 +1,246 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.cloud.catalog; + +import org.apache.doris.catalog.Env; +import org.apache.doris.catalog.StorageVault; +import org.apache.doris.catalog.StorageVault.StorageVaultType; +import org.apache.doris.catalog.StorageVaultMgr; +import org.apache.doris.cloud.proto.Cloud; +import org.apache.doris.cloud.rpc.MetaServiceProxy; +import org.apache.doris.common.DdlException; +import org.apache.doris.datasource.storage.S3ResourceCompat; +import org.apache.doris.nereids.trees.plans.commands.CreateStorageVaultCommand; +import org.apache.doris.system.SystemInfoService; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.atomic.AtomicReference; + +public class S3StorageVaultTest { + private static final String ROLE = "arn:aws:iam::123456789012:role/test-role"; + private static final String ACCOUNT = "target@my-project.iam.gserviceaccount.com"; + + // Capture the actual CREATE/ALTER RPC, then stop before cache updates and backend synchronization. + private Cloud.ObjectStoreInfoPB captureRequest(Map auth, boolean isAlter) throws Exception { + AtomicReference captured = new AtomicReference<>(); + Env env = Mockito.mock(Env.class); + MetaServiceProxy proxy = Mockito.mock(MetaServiceProxy.class); + try (MockedStatic mockedProxy = Mockito.mockStatic(MetaServiceProxy.class); + MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedProxy.when(MetaServiceProxy::getInstance).thenReturn(proxy); + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + Mockito.when(proxy.alterStorageVault(Mockito.any(Cloud.AlterObjStoreInfoRequest.class))) + .thenAnswer(invocation -> { + captured.set(invocation.getArgument(0)); + return Cloud.AlterObjStoreInfoResponse.newBuilder().setStatus( + Cloud.MetaServiceResponseStatus.newBuilder() + .setCode(Cloud.MetaServiceCode.INVALID_ARGUMENT) + .setMsg("captured vault request")).build(); + }); + StorageVaultMgr mgr = new StorageVaultMgr(new SystemInfoService()); + if (isAlter) { + Assertions.assertThrows(DdlException.class, + () -> mgr.alterStorageVault(StorageVaultType.S3, new HashMap<>(auth), "iam_vault")); + } else { + Map properties = new HashMap<>(auth); + boolean isGcp = "GCP".equals(auth.get("provider")) + || auth.keySet().stream().anyMatch(key -> key.toLowerCase().startsWith("gs.")); + properties.put("type", "S3"); + properties.put("provider", isGcp ? "GCP" : "S3"); + properties.put(S3ResourceCompat.ENDPOINT, isGcp ? "storage.googleapis.com" : "s3.us-east-1.amazonaws.com"); + properties.put(S3ResourceCompat.REGION, "us-east-1"); + properties.put(S3ResourceCompat.BUCKET, "test-bucket"); + properties.put(S3ResourceCompat.ROOT_PATH, "test-root"); + properties.put(S3ResourceCompat.VALIDITY_CHECK, "false"); + CreateStorageVaultCommand command = new CreateStorageVaultCommand(false, "iam_vault", properties); + command.setStorageVaultType(StorageVaultType.S3); + StorageVault vault = StorageVault.fromCommand(command); + Assertions.assertThrows(DdlException.class, () -> mgr.createS3Vault(vault)); + } + Assertions.assertNotNull(captured.get(), "The request must reach the MS RPC boundary"); + Assertions.assertEquals(isAlter ? Cloud.AlterObjStoreInfoRequest.Operation.ALTER_S3_VAULT + : Cloud.AlterObjStoreInfoRequest.Operation.ADD_S3_VAULT, captured.get().getOp()); + if (isAlter) { + Assertions.assertFalse(captured.get().getVault().getObjInfo().hasProvider(), + "ALTER must not send an inferred immutable provider"); + } + return captured.get().getVault().getObjInfo(); + } + } + + @Test + public void testAwsCreatePreservesDefaultAndExplicitProvider() throws Exception { + Map properties = new HashMap<>(); + properties.put(S3ResourceCompat.ROLE_ARN, ROLE); + Cloud.ObjectStoreInfoPB obj = captureRequest(properties, false); + Assertions.assertTrue(obj.hasCredProviderType()); + Assertions.assertEquals(Cloud.CredProviderTypePB.INSTANCE_PROFILE, obj.getCredProviderType()); + for (String key : new String[] {S3ResourceCompat.CREDENTIALS_PROVIDER_TYPE, + S3ResourceCompat.Env.CREDENTIALS_PROVIDER_TYPE}) { + Map explicit = new HashMap<>(properties); + explicit.put(key, "CONTAINER"); + obj = captureRequest(explicit, false); + Assertions.assertTrue(obj.hasCredProviderType()); + Assertions.assertEquals(Cloud.CredProviderTypePB.CONTAINER, obj.getCredProviderType()); + } + } + + @Test + public void testAwsAlterPreservesOmissionAndExplicitClears() throws Exception { + Map properties = new HashMap<>(); + properties.put(S3ResourceCompat.ROLE_ARN, ROLE); + Assertions.assertFalse(captureRequest(properties, true).hasCredProviderType()); + properties.put(S3ResourceCompat.CREDENTIALS_PROVIDER_TYPE, "DEFAULT"); + Cloud.ObjectStoreInfoPB obj = captureRequest(properties, true); + Assertions.assertTrue(obj.hasCredProviderType()); + Assertions.assertEquals(Cloud.CredProviderTypePB.DEFAULT, obj.getCredProviderType()); + properties.clear(); + properties.put(S3ResourceCompat.ROLE_ARN, ""); + properties.put(S3ResourceCompat.EXTERNAL_ID, ""); + obj = captureRequest(properties, true); + Assertions.assertTrue(obj.hasRoleArn()); + Assertions.assertEquals("", obj.getRoleArn()); + Assertions.assertTrue(obj.hasExternalId()); + Assertions.assertEquals("", obj.getExternalId()); + Assertions.assertFalse(obj.hasCredProviderType()); + } + + @Test + public void testGcpCreateDefaultsAndExplicitProvider() throws Exception { + Map properties = new HashMap<>(); + properties.put("gs.impersonation_service_account", ACCOUNT); + Cloud.GcpCredentialPB credential = captureRequest(properties, false).getCredential().getGcpCredential(); + Assertions.assertTrue(credential.hasCredentialProviderType()); + Assertions.assertEquals(Cloud.GcpCredentialPB.CredentialProviderType.DEFAULT, + credential.getCredentialProviderType()); + Assertions.assertEquals(ACCOUNT, credential.getImpersonationServiceAccount()); + properties.put("gs.credential_provider_type", "COMPUTE_ENGINE"); + credential = captureRequest(properties, false).getCredential().getGcpCredential(); + Assertions.assertTrue(credential.hasCredentialProviderType()); + Assertions.assertEquals(Cloud.GcpCredentialPB.CredentialProviderType.COMPUTE_ENGINE, + credential.getCredentialProviderType()); + } + + @Test + public void testGcpEmptyTargetDefaultsToDefault() throws Exception { + Map properties = new HashMap<>(); + properties.put("gs.impersonation_service_account", ""); + Cloud.GcpCredentialPB credential = captureRequest(properties, false).getCredential().getGcpCredential(); + Assertions.assertTrue(credential.hasCredentialProviderType()); + Assertions.assertEquals(Cloud.GcpCredentialPB.CredentialProviderType.DEFAULT, + credential.getCredentialProviderType()); + Assertions.assertFalse(credential.hasImpersonationServiceAccount()); + + credential = captureRequest(properties, true).getCredential().getGcpCredential(); + Assertions.assertFalse(credential.hasCredentialProviderType()); + Assertions.assertTrue(credential.hasImpersonationServiceAccount()); + Assertions.assertEquals("", credential.getImpersonationServiceAccount()); + } + + @Test + public void testGcpAlterPreservesOmittedFields() throws Exception { + Map properties = new HashMap<>(); + properties.put("gs.impersonation_service_account", ACCOUNT); + Cloud.GcpCredentialPB credential = captureRequest(properties, true).getCredential().getGcpCredential(); + Assertions.assertFalse(credential.hasCredentialProviderType()); + Assertions.assertTrue(credential.hasImpersonationServiceAccount()); + Assertions.assertEquals(ACCOUNT, credential.getImpersonationServiceAccount()); + properties.clear(); + properties.put("gs.credential_provider_type", "COMPUTE_ENGINE"); + credential = captureRequest(properties, true).getCredential().getGcpCredential(); + Assertions.assertTrue(credential.hasCredentialProviderType()); + Assertions.assertEquals(Cloud.GcpCredentialPB.CredentialProviderType.COMPUTE_ENGINE, + credential.getCredentialProviderType()); + Assertions.assertFalse(credential.hasImpersonationServiceAccount()); + properties.put("gs.impersonation_service_account", ACCOUNT); + credential = captureRequest(properties, true).getCredential().getGcpCredential(); + Assertions.assertTrue(credential.hasCredentialProviderType()); + Assertions.assertEquals(Cloud.GcpCredentialPB.CredentialProviderType.COMPUTE_ENGINE, + credential.getCredentialProviderType()); + Assertions.assertTrue(credential.hasImpersonationServiceAccount()); + Assertions.assertEquals(ACCOUNT, credential.getImpersonationServiceAccount()); + properties.clear(); + properties.put(S3ResourceCompat.USE_PATH_STYLE, "true"); + Cloud.ObjectStoreInfoPB obj = captureRequest(properties, true); + Assertions.assertFalse(obj.hasCredential()); + Assertions.assertFalse(obj.hasCredProviderType()); + } + + @Test + public void testGcpCreateWithoutAuthenticationDefaultsToAdc() throws Exception { + Map properties = new HashMap<>(); + properties.put("provider", "GCP"); + Cloud.ObjectStoreInfoPB obj = captureRequest(properties, false); + Assertions.assertTrue(obj.hasCredential()); + Assertions.assertEquals(Cloud.GcpCredentialPB.CredentialProviderType.DEFAULT, + obj.getCredential().getGcpCredential().getCredentialProviderType()); + Assertions.assertFalse(obj.getCredential().getGcpCredential().hasImpersonationServiceAccount()); + } + + @Test + public void testGcpAnonymousAliasesAreRejectedBeforeCreateRpc() throws Exception { + Env env = Mockito.mock(Env.class); + MetaServiceProxy proxy = Mockito.mock(MetaServiceProxy.class); + try (MockedStatic mockedProxy = Mockito.mockStatic(MetaServiceProxy.class); + MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedProxy.when(MetaServiceProxy::getInstance).thenReturn(proxy); + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + StorageVaultMgr mgr = new StorageVaultMgr(new SystemInfoService()); + for (String key : new String[] {"gs.credential_provider_type", + S3ResourceCompat.CREDENTIALS_PROVIDER_TYPE, S3ResourceCompat.Env.CREDENTIALS_PROVIDER_TYPE}) { + Map properties = new HashMap<>(); + properties.put("type", "S3"); + properties.put("provider", "GCP"); + properties.put(S3ResourceCompat.ENDPOINT, "storage.googleapis.com"); + properties.put(S3ResourceCompat.REGION, "us-east1"); + properties.put(S3ResourceCompat.BUCKET, "test-bucket"); + properties.put(S3ResourceCompat.ROOT_PATH, "test-root"); + properties.put(S3ResourceCompat.VALIDITY_CHECK, "false"); + properties.put(key, " anonymous "); + CreateStorageVaultCommand command = new CreateStorageVaultCommand(false, "anonymous_gcp", properties); + command.setStorageVaultType(StorageVaultType.S3); + StorageVault vault = StorageVault.fromCommand(command); + IllegalArgumentException error = Assertions.assertThrows(IllegalArgumentException.class, + () -> mgr.createS3Vault(vault), key); + Assertions.assertTrue(error.getMessage().contains("storage vaults"), key); + } + Mockito.verifyNoInteractions(proxy); + } + } + + @Test + public void testGcpStorageOnlyAlterDoesNotInjectAdc() throws Exception { + Map properties = new HashMap<>(); + properties.put("s3.endpoint", "storage.googleapis.com"); + // Inspect the builder directly: ALTER's public API may reject immutable storage fields. + java.lang.reflect.Method method = StorageVaultMgr.class.getDeclaredMethod( + "buildAlterS3VaultRequest", Map.class, String.class); + method.setAccessible(true); + StorageVaultMgr mgr = new StorageVaultMgr(new SystemInfoService()); + Cloud.StorageVaultPB.Builder request = (Cloud.StorageVaultPB.Builder) method.invoke(mgr, properties, "gcp"); + Assertions.assertFalse(request.getObjInfo().hasCredential()); + Assertions.assertFalse(request.getObjInfo().hasProvider()); + } + +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/cloud/storage/ObjectInfoAdapterTest.java b/fe/fe-core/src/test/java/org/apache/doris/cloud/storage/ObjectInfoAdapterTest.java index ed26ad4a0aab41..af1ac11b37e4ea 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/cloud/storage/ObjectInfoAdapterTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/cloud/storage/ObjectInfoAdapterTest.java @@ -83,4 +83,20 @@ public void testOssStageBindingCarriesEveryField() { Assertions.assertEquals("stage-sk", oss.getSecretKey()); Assertions.assertEquals("stage-token", oss.getSessionToken()); } + + @Test + public void testNativeGcpStagePreservesCredentialSourceAndImpersonation() { + Cloud.ObjectStoreInfoPB.Builder info = Cloud.ObjectStoreInfoPB.newBuilder() + .setProvider(Cloud.ObjectStoreInfoPB.Provider.GCP) + .setEndpoint("storage.googleapis.com").setRegion("us-east1").setBucket("stage-bucket"); + info.getCredentialBuilder().getGcpCredentialBuilder() + .setCredentialProviderType(Cloud.GcpCredentialPB.CredentialProviderType.COMPUTE_ENGINE) + .setImpersonationServiceAccount("target@test.iam.gserviceaccount.com"); + StorageAdapter adapter = ObjectInfoAdapter.toStorageAdapter(new ObjectInfo(info.build())); + Assertions.assertEquals("COMPUTE_ENGINE", + adapter.getBackendConfigProperties().get("gs.credential_provider_type")); + Assertions.assertEquals("target@test.iam.gserviceaccount.com", + adapter.getBackendConfigProperties().get("gs.impersonation_service_account")); + Assertions.assertFalse(adapter.getBackendConfigProperties().containsKey("AWS_CREDENTIALS_PROVIDER_TYPE")); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/storage/S3ThriftAdapterParityTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/storage/S3ThriftAdapterParityTest.java index 5bc69f8b37c13b..f13c6fdf0d63d2 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/storage/S3ThriftAdapterParityTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/storage/S3ThriftAdapterParityTest.java @@ -18,6 +18,7 @@ package org.apache.doris.datasource.storage; import org.apache.doris.thrift.TCredProviderType; +import org.apache.doris.thrift.TObjStorageType; import org.apache.doris.thrift.TS3StorageParam; import org.junit.jupiter.api.Assertions; @@ -133,6 +134,30 @@ public void testAssumeRoleWithLegacyEnvProviderTypeKey() { "AWS_CREDENTIALS_PROVIDER_TYPE", "SYSTEM_PROPERTIES"))); } + @Test + public void testFilesystemProviderAliasesUseBackendTypes() { + for (String provider : new String[] {"S3", "MINIO", "minio", " OZONE ", "AWS"}) { + TS3StorageParam thrift = S3ThriftAdapter.getS3TStorageParam(map( + "provider", provider, + "s3.endpoint", "http://minio.example.com:9000", + "s3.access_key", "myAk", + "s3.secret_key", "mySk")); + Assertions.assertEquals(TObjStorageType.AWS, thrift.getProvider()); + Assertions.assertEquals("myAk", thrift.getAk()); + Assertions.assertEquals("mySk", thrift.getSk()); + } + for (String provider : new String[] {"GCP", "GCS"}) { + Assertions.assertEquals(TObjStorageType.GCP, + S3ThriftAdapter.getS3TStorageParam(map("provider", provider)).getProvider()); + } + for (TObjStorageType provider : new TObjStorageType[] { + TObjStorageType.OSS, TObjStorageType.COS, TObjStorageType.OBS, + TObjStorageType.BOS, TObjStorageType.TOS, TObjStorageType.AZURE}) { + Assertions.assertEquals(provider, + S3ThriftAdapter.getS3TStorageParam(map("provider", provider.name())).getProvider()); + } + } + @Test public void testEmptyMap() { Assertions.assertEquals(expected(null, null), diff --git a/fe/fe-core/src/test/java/org/apache/doris/fs/FileSystemPluginManagerTest.java b/fe/fe-core/src/test/java/org/apache/doris/fs/FileSystemPluginManagerTest.java index 29feec06d54b2e..af6a47c1f2b881 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/fs/FileSystemPluginManagerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/fs/FileSystemPluginManagerTest.java @@ -123,6 +123,157 @@ public void bindAll_returnsEmptyListWhenNoProviderSupports() { // modules: runtime dependencies removed in Phase 4 P4.1"). End-to-end binding against the real // providers is therefore covered by P1-T06 (docker / full plugin classpath), not here. + @Test + public void specificPluginSupersedesFallbackInPrimaryAndAllBindings() throws Exception { + for (boolean specificFirst : new boolean[] {false, true}) { + FileSystemPluginManager manager = new FileSystemPluginManager(); + FileSystemProvider specific = routingProvider("custom", Set.of("s3")); + FileSystemProvider general = routingProvider("S3", Set.of()); + manager.registerProvider(specificFirst ? specific : general); + manager.registerProvider(specificFirst ? general : specific); + Map props = Map.of("custom.match", "true", "S3.match", "true"); + Assertions.assertEquals("custom", manager.bindPrimary(props).providerName()); + Assertions.assertEquals(1, manager.bindAll(props).size()); + Assertions.assertEquals("custom", manager.bindAll(props).get(0).providerName()); + Assertions.assertEquals("custom", Assertions.assertThrows(UnsupportedOperationException.class, + () -> manager.createFileSystem(props)).getMessage()); + + // A nonmatching specific plugin must not remove the general fallback. + Assertions.assertEquals("S3", manager.bindPrimary(Map.of("S3.match", "true")).providerName()); + } + } + + @Test + public void explicitFlagsWinAndMultipleExplicitFlagsRemainSupported() { + FileSystemPluginManager manager = new FileSystemPluginManager(); + manager.registerProvider(routingProvider("custom", Set.of("s3"))); + manager.registerProvider(routingProvider("S3", Set.of())); + Map props = new HashMap<>(Map.of("custom.match", "true", "S3.match", "true")); + props.put("fs.s3.support", "true"); + Assertions.assertEquals("S3", manager.bindPrimary(props).providerName()); + Assertions.assertEquals(1, manager.bindAll(props).size()); + Assertions.assertEquals("S3", Assertions.assertThrows(UnsupportedOperationException.class, + () -> manager.createFileSystem(props)).getMessage()); + props.put("fs.custom.support", "true"); + Assertions.assertEquals(2, manager.bindAll(props).size()); + } + + @Test + public void objectStorageGuessPreservesDefaultHdfsAndExplicitFlagsDisableIt() { + FileSystemPluginManager manager = new FileSystemPluginManager(); + manager.registerProvider(routingProvider("S3", Set.of())); + manager.registerProvider(routingProvider("HDFS", Set.of())); + List bindings = manager.bindAll(Map.of("provider", "s3", "S3.match", "true")); + Assertions.assertEquals(2, bindings.size()); + Assertions.assertEquals("HDFS", bindings.get(0).providerName()); + Assertions.assertEquals("S3", bindings.get(1).providerName()); + Assertions.assertEquals(1, manager.bindAll(Map.of("fs.s3.support", "true")).size()); + } + + @Test + public void providerNameDoesNotOverrideExistingMatchOrder() throws Exception { + FileSystemPluginManager manager = new FileSystemPluginManager(); + manager.registerProvider(routingProvider("S3", Set.of())); + manager.registerProvider(routingProvider("HDFS", Set.of())); + Map props = new HashMap<>(Map.of( + "provider", "S3", "S3.match", "true", "HDFS.match", "true")); + Assertions.assertEquals("HDFS", manager.bindPrimary(props).providerName()); + Assertions.assertEquals(2, manager.bindAll(props).size()); + Assertions.assertEquals("HDFS", Assertions.assertThrows(UnsupportedOperationException.class, + () -> manager.createFileSystem(props)).getMessage()); + + props.put("fs.hdfs.support", "true"); + Assertions.assertEquals("HDFS", manager.bindPrimary(props).providerName()); + Assertions.assertEquals(1, manager.bindAll(props).size()); + Assertions.assertEquals("HDFS", Assertions.assertThrows(UnsupportedOperationException.class, + () -> manager.createFileSystem(props)).getMessage()); + } + + @Test + public void unrelatedGuessesAreNotEvaluatedBeforePrimaryMatch() { + FileSystemPluginManager manager = new FileSystemPluginManager(); + manager.registerProvider(routingProvider("S3", Set.of())); + manager.registerProvider(new FileSystemProvider() { + @Override + public String name() { + return "unused"; + } + + @Override + public boolean supports(Map properties) { + throw new AssertionError("A later unrelated provider must not be probed"); + } + + @Override + public FileSystem create(Map properties) { + throw new AssertionError("A later unrelated provider must not be created"); + } + + @Override + public boolean supportsGuess(Map properties) { + throw new AssertionError("A later unrelated provider must not be probed"); + } + }); + Assertions.assertEquals("S3", manager.bindPrimary(Map.of("S3.match", "true")).providerName()); + } + + @Test + public void nonSelectedPluginStillValidatesItsOwnedProperties() { + FileSystemPluginManager manager = new FileSystemPluginManager(); + manager.registerProvider(routingProvider("custom", Set.of())); + manager.registerProvider(routingProvider("S3", Set.of())); + Map props = Map.of("provider", "S3", "custom.invalid", "true"); + Assertions.assertThrows(IllegalArgumentException.class, () -> manager.bindPrimary(props)); + Assertions.assertThrows(IllegalArgumentException.class, () -> manager.bindAll(props)); + Assertions.assertThrows(IllegalArgumentException.class, () -> manager.createFileSystem(props)); + } + + private static FileSystemProvider routingProvider(String name, Set fallbacks) { + return new FileSystemProvider() { + @Override + public Set fallbackProviders() { + return fallbacks; + } + + @Override + public void validateProperties(Map properties) { + if (properties.containsKey(name + ".invalid")) { + throw new IllegalArgumentException("Invalid " + name + " properties"); + } + } + + @Override + public boolean supportsExplicit(Map properties) { + return "true".equals(properties.get("fs." + name.toLowerCase(java.util.Locale.ROOT) + ".support")); + } + + @Override + public boolean supportsGuess(Map properties) { + return properties.containsKey(name + ".match"); + } + + @Override + public boolean supports(Map properties) { + return supportsExplicit(properties) || supportsGuess(properties); + } + + @Override + public FileSystemProperties bind(Map properties) { + return new FakeFsProps(name); + } + + @Override + public FileSystem create(Map properties) { + throw new UnsupportedOperationException(name); + } + + @Override + public String name() { + return name; + } + }; + } + // ---- helpers ---- private static FileSystemProvider bindingProvider( diff --git a/fe/fe-core/src/test/java/org/apache/doris/fs/GcsGuessRoutingTest.java b/fe/fe-core/src/test/java/org/apache/doris/fs/GcsGuessRoutingTest.java new file mode 100644 index 00000000000000..d90272469ca9bd --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/fs/GcsGuessRoutingTest.java @@ -0,0 +1,209 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.fs; + +import org.apache.doris.cloud.proto.Cloud.ObjectStoreInfoPB; +import org.apache.doris.datasource.storage.CloudObjectStoreAdapter; +import org.apache.doris.datasource.storage.ObjectStorageCredentialAdapter; +import org.apache.doris.datasource.storage.S3ThriftAdapter; +import org.apache.doris.datasource.storage.StorageAdapter; +import org.apache.doris.thrift.TS3StorageParam; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; + +public class GcsGuessRoutingTest { + private static final String[] ENDPOINTS = { + "https://storage.googleapis.com", "https://storage.googleapis.com:443", + "https://storage.googleapis.com/", "storage.googleapis.com:443/", + "https://storage.us-east1.rep.googleapis.com", "https://us-east1-storage.googleapis.com"}; + + @Test + public void testExplicitS3KeepsAwsAuthenticationInPrimaryAndAllBindings() { + for (String endpoint : ENDPOINTS) { + for (String provider : new String[] {"S3", "s3"}) { + Map props = properties(endpoint); + props.put("provider", provider); + assertS3Bindings(props, "DEFAULT"); + props.put("s3.access_key", "ak"); + props.put("s3.secret_key", "sk"); + assertS3Bindings(props, "DEFAULT"); + props.remove("s3.access_key"); + props.remove("s3.secret_key"); + props.put("s3.credentials_provider_type", "ANONYMOUS"); + assertS3Bindings(props, "ANONYMOUS"); + } + } + } + + @Test + public void testProviderNamesPreserveLegacySchemalessUriError() { + for (String provider : new String[] {"S3", "OSS", "COS", "OBS", "AZURE", "GCS", "GCP"}) { + Map props = properties("https://storage.googleapis.com"); + props.put("provider", provider); + props.put("uri", "bucket/regression/load/data/example_0.csv"); + props.put("s3.access_key", "ak"); + props.put("s3.secret_key", "sk"); + IllegalArgumentException primary = Assertions.assertThrows( + IllegalArgumentException.class, () -> StorageAdapter.of(props), provider); + Assertions.assertTrue(primary.getMessage().contains("Invalid uri: bucket/"), primary.getMessage()); + IllegalArgumentException all = Assertions.assertThrows( + IllegalArgumentException.class, () -> StorageAdapter.ofAll(props), provider); + Assertions.assertTrue(all.getMessage().contains("Invalid uri: bucket/"), all.getMessage()); + } + } + + @Test + public void testGcpAndGcsAliasesSelectTheSameBindings() { + for (String provider : new String[] {"GCP", "GCS", "gcp", "gcs"}) { + Map props = properties("https://storage.googleapis.com"); + props.put("provider", provider); + bindings(props, "GCS"); + props.put("s3.access_key", "ak"); + props.put("s3.secret_key", "sk"); + bindings(props, "GCS"); + } + } + + @Test + public void testExplicitHdfsFlagIsNotOverriddenByProviderName() { + Map props = properties("https://storage.googleapis.com"); + props.put("provider", "S3"); + props.put("fs.hdfs.support", "true"); + props.put("uri", "hdfs://namenode/path"); + Assertions.assertEquals("HDFS", StorageAdapter.of(props).getStorageName()); + List all = StorageAdapter.ofAll(props); + Assertions.assertEquals(1, all.size()); + Assertions.assertEquals("HDFS", all.get(0).getStorageName()); + } + + @Test + public void testInferredGcsEndpointsBindNativeAndHmacWithoutMinio() { + for (String endpoint : ENDPOINTS) { + Map props = properties(endpoint); + for (StorageAdapter adapter : bindings(props, "GCS")) { + Assertions.assertEquals("DEFAULT", + adapter.getBackendConfigProperties().get("gs.credential_provider_type")); + } + props.put("s3.access_key", "ak"); + props.put("s3.secret_key", "sk"); + for (StorageAdapter adapter : bindings(props, "GCS")) { + Assertions.assertFalse(adapter.getBackendConfigProperties().containsKey("gs.credential_provider_type")); + Assertions.assertEquals("ak", adapter.getBackendConfigProperties().get("AWS_ACCESS_KEY")); + } + } + } + + @Test + public void testInferredNativeGcsStillValidatesEndpoint() { + for (String endpoint : new String[] {"http://storage.googleapis.com", + "https://storage.googleapis.com:8443", "https://storage.googleapis.com/bucket", + "https://storage.googleapis.com?query=value", "https://bucket.storage.googleapis.com"}) { + Map props = properties(endpoint); + Assertions.assertThrows(IllegalArgumentException.class, () -> StorageAdapter.of(props), endpoint); + Assertions.assertThrows(IllegalArgumentException.class, () -> StorageAdapter.ofAll(props), endpoint); + } + } + + @Test + public void testNativeGcsHintsWithoutEndpointDoNotBindS3OrMinio() { + for (Map.Entry hint : Map.of( + "provider", "GCP", "gs.credential_provider_type", "DEFAULT", + "gs.impersonation_service_account", "target@test.iam.gserviceaccount.com", + "warehouse", "gs://bucket/warehouse").entrySet()) { + Map props = new HashMap<>(); + props.put("s3.region", "us-east1"); + props.put("_STORAGE_TYPE_", "S3"); + props.put(hint.getKey(), hint.getValue()); + bindings(props, "GCS"); + } + bindings(Map.of("s3.region", "us-east1", "gs.impersonation_service_account", ""), "GCS"); + bindings(Map.of("s3.region", "us-east-1"), "S3"); + } + + @Test + public void testNativeCredentialsCannotBeIgnoredByS3Selection() { + for (Map selection : List.of(Map.of("provider", "S3"), + Map.of("fs.s3.support", "true"))) { + Map props = new HashMap<>(selection); + props.put("gs.credential_provider_type", "DEFAULT"); + props.put("s3.region", "us-east-1"); + Assertions.assertThrows(IllegalArgumentException.class, () -> StorageAdapter.of(props)); + Assertions.assertThrows(IllegalArgumentException.class, () -> StorageAdapter.ofAll(props)); + } + Assertions.assertThrows(IllegalArgumentException.class, + () -> StorageAdapter.ofProvider("S3", Map.of("s3.region", "us-east-1", + "gs.credential_provider_type", "DEFAULT"))); + } + + @Test + public void testTypedIdentityConnectsPluginCloudAndBeWithoutLosingAlterPresence() { + Map properties = Map.of("provider", "GCP", + "gs.credential_provider_type", "COMPUTE_ENGINE", + "gs.impersonation_service_account", "target@test.iam.gserviceaccount.com"); + ObjectStoreInfoPB.Builder cloud = CloudObjectStoreAdapter.getObjStoreInfoPB(properties); + TS3StorageParam thrift = S3ThriftAdapter.getS3TStorageParam(properties); + Assertions.assertEquals("COMPUTE_ENGINE", cloud.getCredential().getGcpCredential() + .getCredentialProviderType().name()); + Assertions.assertEquals("COMPUTE_ENGINE", thrift.getCredential().getGcpCredential() + .getCredentialProviderType().name()); + Assertions.assertEquals("target@test.iam.gserviceaccount.com", thrift.getCredential() + .getGcpCredential().getImpersonationServiceAccount()); + Assertions.assertEquals(Map.of("gs.credential_provider_type", "COMPUTE_ENGINE", + "gs.impersonation_service_account", "target@test.iam.gserviceaccount.com"), + ObjectStorageCredentialAdapter.toProperties("GCP", cloud.getCredential())); + CloudObjectStoreAdapter.applyCredentialPatch(cloud, Map.of("gs.impersonation_service_account", "")); + Assertions.assertFalse(cloud.getCredential().getGcpCredential().hasCredentialProviderType()); + Assertions.assertTrue(cloud.getCredential().getGcpCredential().hasImpersonationServiceAccount()); + Assertions.assertEquals("", cloud.getCredential().getGcpCredential().getImpersonationServiceAccount()); + CloudObjectStoreAdapter.applyCredentialPatch(cloud, Map.of("provider", "GCP")); + Assertions.assertFalse(cloud.hasCredential()); + } + + private static Map properties(String endpoint) { + Map props = new HashMap<>(); + props.put("s3.endpoint", endpoint); + props.put("s3.region", "us-east1"); + props.put("uri", "s3://bucket/key"); + return props; + } + + private static void assertS3Bindings(Map props, String mode) { + for (StorageAdapter adapter : bindings(props, "S3")) { + Map backend = adapter.getBackendConfigProperties(); + Assertions.assertEquals(mode, backend.get("AWS_CREDENTIALS_PROVIDER_TYPE")); + Assertions.assertFalse(backend.containsKey("gs.credential_provider_type")); + } + } + + private static List bindings(Map props, String expectedProvider) { + StorageAdapter primary = StorageAdapter.of(props); + Assertions.assertEquals(expectedProvider, primary.getSpiProperties().providerName()); + // bindAll also supplies the default HDFS filesystem for catalog callers. + List objectBindings = StorageAdapter.ofAll(props).stream() + .filter(adapter -> !"HDFS".equals(adapter.getStorageName())).collect(Collectors.toList()); + Assertions.assertEquals(1, objectBindings.size(), props.toString()); + Assertions.assertEquals(expectedProvider, objectBindings.get(0).getSpiProperties().providerName()); + return List.of(primary, objectBindings.get(0)); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/fs/ObjectStorageCredentialAdapterTest.java b/fe/fe-core/src/test/java/org/apache/doris/fs/ObjectStorageCredentialAdapterTest.java new file mode 100644 index 00000000000000..8baaf550536d74 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/fs/ObjectStorageCredentialAdapterTest.java @@ -0,0 +1,117 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.fs; + +import org.apache.doris.cloud.proto.Cloud.ObjectStoreInfoPB; +import org.apache.doris.datasource.storage.ObjectStorageCredentialAdapter; +import org.apache.doris.datasource.storage.StorageAdapter; +import org.apache.doris.filesystem.FileSystem; +import org.apache.doris.filesystem.auth.GcpCredentialSpec; +import org.apache.doris.filesystem.auth.NativeObjectStorageCredential; +import org.apache.doris.filesystem.properties.FileSystemProperties; +import org.apache.doris.filesystem.spi.FileSystemProvider; +import org.apache.doris.foundation.property.StoragePropertiesException; +import org.apache.doris.thrift.TObjStorageType; +import org.apache.doris.thrift.TS3StorageParam; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Map; +import java.util.Optional; + +class ObjectStorageCredentialAdapterTest { + @AfterEach + void restoreBuiltinRegistry() { + StorageAdapter.initPluginManager(null); + } + + @Test + void hostTranslatesTypedCredentialsAndPluginOwnsPropertyNames() { + FileSystemPluginManager manager = new FileSystemPluginManager(); + manager.registerProvider(provider("GCS")); + StorageAdapter.initPluginManager(manager); + GcpCredentialSpec spec = new GcpCredentialSpec(GcpCredentialSpec.ProviderType.COMPUTE_ENGINE, null); + ObjectStorageCredentialAdapter adapter = ObjectStorageCredentialAdapter.fromCredential(spec); + ObjectStoreInfoPB.Builder cloud = ObjectStoreInfoPB.newBuilder(); + adapter.applyTo(cloud); + Assertions.assertEquals(ObjectStoreInfoPB.Provider.GCP, cloud.getProvider()); + Assertions.assertEquals("COMPUTE_ENGINE", cloud.getCredential().getGcpCredential() + .getCredentialProviderType().name()); + TS3StorageParam thrift = new TS3StorageParam(); + adapter.applyTo(thrift); + Assertions.assertEquals(TObjStorageType.GCP, thrift.getProvider()); + Assertions.assertEquals("COMPUTE_ENGINE", thrift.getCredential().getGcpCredential() + .getCredentialProviderType().name()); + Assertions.assertEquals(Map.of("plugin.identity", "COMPUTE_ENGINE"), + ObjectStorageCredentialAdapter.toProperties("GCP", cloud.getCredential())); + } + + @Test + void alterDoesNotInventSourceOrDropExplicitClear() { + ObjectStoreInfoPB.Builder cloud = ObjectStoreInfoPB.newBuilder(); + TS3StorageParam thrift = new TS3StorageParam(); + ObjectStorageCredentialAdapter adapter = ObjectStorageCredentialAdapter.fromCredential( + new GcpCredentialSpec(null, "")); + adapter.applyTo(cloud); + adapter.applyTo(thrift); + Assertions.assertFalse(cloud.getCredential().getGcpCredential().hasCredentialProviderType()); + Assertions.assertFalse(thrift.getCredential().getGcpCredential().isSetCredentialProviderType()); + Assertions.assertTrue(cloud.getCredential().getGcpCredential().hasImpersonationServiceAccount()); + Assertions.assertTrue(thrift.getCredential().getGcpCredential().isSetImpersonationServiceAccount()); + Assertions.assertEquals("", cloud.getCredential().getGcpCredential().getImpersonationServiceAccount()); + Assertions.assertEquals("", thrift.getCredential().getGcpCredential().getImpersonationServiceAccount()); + } + + @Test + void missingOrAmbiguousPropertyOwnerFailsExplicitly() { + FileSystemPluginManager manager = new FileSystemPluginManager(); + StorageAdapter.initPluginManager(manager); + GcpCredentialSpec spec = new GcpCredentialSpec(GcpCredentialSpec.ProviderType.DEFAULT, null); + Assertions.assertThrows(StoragePropertiesException.class, () -> StorageAdapter.credentialToProperties(spec)); + manager.registerProvider(provider("first")); + manager.registerProvider(provider("second")); + Assertions.assertThrows(StoragePropertiesException.class, () -> StorageAdapter.credentialToProperties(spec)); + } + + private FileSystemProvider provider(String name) { + return new FileSystemProvider() { + @Override + public String name() { + return name; + } + + @Override + public boolean supports(Map properties) { + return false; + } + + @Override + public FileSystem create(Map properties) { + throw new UnsupportedOperationException("No filesystem I/O in credential conversion"); + } + + @Override + public Optional> credentialToProperties(NativeObjectStorageCredential credential) { + GcpCredentialSpec spec = (GcpCredentialSpec) credential; + return Optional.of(Map.of("plugin.identity", spec.getProviderType().orElseThrow().name())); + } + }; + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/OutFileTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/OutFileTest.java index 5b6d4e4aaecedb..5016cd0c0c6dff 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/OutFileTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/OutFileTest.java @@ -35,8 +35,11 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; import java.lang.reflect.Field; +import java.util.Map; public class OutFileTest extends TestWithFeService implements PlanPatternMatchSupported { private final NereidsParser parser = new NereidsParser(); @@ -94,9 +97,108 @@ public void testHdfsOutFileCarriesDefaultFs() throws Exception { Field field = ResultFileSink.class.getDeclaredField("fileSinkOptions"); field.setAccessible(true); TResultFileSinkOptions sinkOptions = (TResultFileSinkOptions) field.get(fragment.getSink()); + Assertions.assertEquals("hdfs://127.0.0.1:8020/tmp/outfile_test_", sinkOptions.getFilePath()); Assertions.assertEquals("hdfs://127.0.0.1:8020", sinkOptions.getBrokerProperties().get("fs.defaultFS")); } + @Test + public void testGcsOutFileNormalizesPathAndPreservesCredentials() throws Exception { + for (String scheme : new String[] {"gs", "s3"}) { + for (String provider : new String[] {"DEFAULT", "COMPUTE_ENGINE"}) { + for (String serviceAccount : new String[] {"", "target@test-project.iam.gserviceaccount.com"}) { + String properties = "\"provider\" = \"GCP\"," + + "\"s3.endpoint\" = \"https://storage.googleapis.com\"," + + "\"s3.region\" = \"us-central1\"," + + "\"gs.credential_provider_type\" = \"" + provider + "\""; + if (!serviceAccount.isEmpty()) { + properties += ",\"gs.impersonation_service_account\" = \"" + serviceAccount + "\""; + } + TResultFileSinkOptions options = getFileSinkOptions("select * from T1 into outfile '" + + scheme + "://outfile-bucket/nested/result_' format as csv properties (" + + properties + ")"); + Assertions.assertEquals("s3://outfile-bucket/nested/result_", options.getFilePath()); + Map backendProperties = options.getBrokerProperties(); + Assertions.assertEquals("GCP", backendProperties.get("provider")); + Assertions.assertEquals("https://storage.googleapis.com", backendProperties.get("AWS_ENDPOINT")); + // GCSProperties does not bind s3.region and retains its compatibility default. + Assertions.assertEquals("us-east1", backendProperties.get("AWS_REGION")); + Assertions.assertEquals(provider, backendProperties.get("gs.credential_provider_type")); + Assertions.assertEquals(serviceAccount, + backendProperties.getOrDefault("gs.impersonation_service_account", "")); + } + } + } + } + + @ParameterizedTest + @CsvSource({ + "http, false", "https, false", "http, true", "https, true", + "HTTP, false", "HTTPS, false" + }) + public void testS3HttpOutFilePreservesPath(String scheme, boolean usePathStyle) throws Exception { + String path = scheme + "://s3.us-east-1.amazonaws.com/outfile-bucket/nested/result_"; + TResultFileSinkOptions options = getFileSinkOptions("select * from T1 into outfile '" + + path + "' format as csv properties (" + + "\"s3.endpoint\" = \"https://s3.us-east-1.amazonaws.com\"," + + "\"s3.region\" = \"us-east-1\"," + + "\"s3.access_key\" = \"test-ak\",\"s3.secret_key\" = \"test-sk\"," + + "\"use_path_style\" = \"" + usePathStyle + "\")"); + Assertions.assertEquals(path, options.getFilePath()); + } + + @Test + public void testHdfsOutFilePreservesPathWithoutAuthority() throws Exception { + String path = "hdfs:/tmp/outfile_test_"; + TResultFileSinkOptions options = getFileSinkOptions("select * from T1 into outfile '" + + path + "' format as csv properties (\"hadoop.username\" = \"doris\")"); + Assertions.assertEquals(path, options.getFilePath()); + } + + @Test + public void testS3OutFilePreservesPath() throws Exception { + TResultFileSinkOptions options = getFileSinkOptions("select * from T1 into outfile '" + + "s3://outfile-bucket/nested/result_' format as csv properties (" + + "\"s3.endpoint\" = \"https://s3.us-east-1.amazonaws.com\"," + + "\"s3.region\" = \"us-east-1\"," + + "\"s3.access_key\" = \"test-ak\",\"s3.secret_key\" = \"test-sk\")"); + Assertions.assertEquals("s3://outfile-bucket/nested/result_", options.getFilePath()); + Assertions.assertEquals("test-ak", options.getBrokerProperties().get("AWS_ACCESS_KEY")); + Assertions.assertEquals("test-sk", options.getBrokerProperties().get("AWS_SECRET_KEY")); + } + + @ParameterizedTest + @CsvSource({ + "oss://outfile-bucket, https://oss-cn-hangzhou.aliyuncs.com, cn-hangzhou", + "oss://outfile-bucket.oss-cn-hangzhou.aliyuncs.com, https://oss-cn-hangzhou.aliyuncs.com, cn-hangzhou", + "s3://outfile-bucket.oss-cn-hangzhou.aliyuncs.com, https://oss-cn-hangzhou.aliyuncs.com, cn-hangzhou", + "cos://outfile-bucket, https://cos.ap-guangzhou.myqcloud.com, ap-guangzhou", + "cosn://outfile-bucket, https://cos.ap-guangzhou.myqcloud.com, ap-guangzhou", + "obs://outfile-bucket, https://obs.cn-north-4.myhuaweicloud.com, cn-north-4", + "bos://outfile-bucket, https://s3.us-east-1.amazonaws.com, us-east-1", + "s3a://outfile-bucket, https://s3.us-east-1.amazonaws.com, us-east-1", + "s3n://outfile-bucket, https://s3.us-east-1.amazonaws.com, us-east-1" + }) + public void testS3CompatibleOutFileNormalizesPath(String location, String endpoint, String region) throws Exception { + TResultFileSinkOptions options = getFileSinkOptions("select * from T1 into outfile '" + + location + "/nested/result_' format as csv properties (" + + "\"s3.endpoint\" = \"" + endpoint + "\"," + + "\"s3.region\" = \"" + region + "\"," + + "\"s3.access_key\" = \"test-ak\",\"s3.secret_key\" = \"test-sk\")"); + Assertions.assertEquals("s3://outfile-bucket/nested/result_", options.getFilePath()); + Assertions.assertEquals(endpoint, options.getBrokerProperties().get("AWS_ENDPOINT")); + Assertions.assertEquals(region, options.getBrokerProperties().get("AWS_REGION")); + Assertions.assertEquals("test-ak", options.getBrokerProperties().get("AWS_ACCESS_KEY")); + Assertions.assertEquals("test-sk", options.getBrokerProperties().get("AWS_SECRET_KEY")); + } + + private TResultFileSinkOptions getFileSinkOptions(String sql) throws Exception { + PlanFragment fragment = getOutputFragment(sql); + Assertions.assertTrue(fragment.getSink() instanceof ResultFileSink); + Field field = ResultFileSink.class.getDeclaredField("fileSinkOptions"); + field.setAccessible(true); + return (TResultFileSinkOptions) field.get(fragment.getSink()); + } + @Test public void testNativeUuidOrcSchemaUsesBinaryRecursively() throws Exception { createTable("CREATE TABLE uuid_outfile (id INT, u UUID, a ARRAY," diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ExportCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ExportCommandTest.java index 981b7be805b898..4007ee6634db43 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ExportCommandTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ExportCommandTest.java @@ -17,18 +17,131 @@ package org.apache.doris.nereids.trees.plans.commands; +import org.apache.doris.analysis.BrokerDesc; +import org.apache.doris.analysis.StorageBackend.StorageType; +import org.apache.doris.catalog.info.TableNameInfo; import org.apache.doris.common.jmockit.Deencapsulation; import org.apache.doris.datasource.property.fileformat.ParquetFileFormatProperties; +import org.apache.doris.load.ExportJob; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.utframe.TestWithFeService; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; +import java.lang.reflect.Method; import java.util.Collections; +import java.util.HashMap; import java.util.Map; import java.util.Optional; -public class ExportCommandTest { +public class ExportCommandTest extends TestWithFeService { + @Override + protected void runBeforeAll() throws Exception { + createDatabase("export_test"); + connectContext.setDatabase("export_test"); + createTable("CREATE TABLE T1 (id bigint) DUPLICATE KEY(id) " + + "DISTRIBUTED BY HASH(id) BUCKETS 1 PROPERTIES (\"replication_num\" = \"1\")"); + } + + @ParameterizedTest + @CsvSource({ + "s3://export-bucket, https://s3.us-east-1.amazonaws.com, us-east-1", + "oss://export-bucket, https://oss-cn-hangzhou.aliyuncs.com, cn-hangzhou", + "oss://export-bucket.oss-cn-hangzhou.aliyuncs.com, https://oss-cn-hangzhou.aliyuncs.com, cn-hangzhou", + "s3://export-bucket.oss-cn-hangzhou.aliyuncs.com, https://oss-cn-hangzhou.aliyuncs.com, cn-hangzhou", + "cos://export-bucket, https://cos.ap-guangzhou.myqcloud.com, ap-guangzhou", + "cosn://export-bucket, https://cos.ap-guangzhou.myqcloud.com, ap-guangzhou", + "obs://export-bucket, https://obs.cn-north-4.myhuaweicloud.com, cn-north-4", + "bos://export-bucket, https://s3.us-east-1.amazonaws.com, us-east-1", + "s3a://export-bucket, https://s3.us-east-1.amazonaws.com, us-east-1", + "s3n://export-bucket, https://s3.us-east-1.amazonaws.com, us-east-1" + }) + public void testS3CompatibleExportNormalizesPath(String location, String endpoint, String region) throws Exception { + BrokerDesc broker = new BrokerDesc(null, s3Properties(endpoint, region)); + ExportJob job = generateExportJob(location + "/nested/result_", broker); + Assertions.assertEquals("s3://export-bucket/nested/result_", job.getExportPath()); + Map backendProperties = job.getBrokerDesc().getBackendConfigProperties(); + Assertions.assertEquals(endpoint, backendProperties.get("AWS_ENDPOINT")); + Assertions.assertEquals(region, backendProperties.get("AWS_REGION")); + Assertions.assertEquals("test-ak", backendProperties.get("AWS_ACCESS_KEY")); + Assertions.assertEquals("test-sk", backendProperties.get("AWS_SECRET_KEY")); + } + + @Test + public void testGcsExportNormalizesPathAndPreservesCredentials() throws Exception { + for (String scheme : new String[] {"gs", "s3"}) { + for (String provider : new String[] {"DEFAULT", "COMPUTE_ENGINE"}) { + for (String serviceAccount : new String[] {"", "target@test-project.iam.gserviceaccount.com"}) { + Map properties = new HashMap<>(); + properties.put("provider", "GCP"); + properties.put("s3.endpoint", "https://storage.googleapis.com"); + properties.put("gs.credential_provider_type", provider); + if (!serviceAccount.isEmpty()) { + properties.put("gs.impersonation_service_account", serviceAccount); + } + ExportJob job = generateExportJob(scheme + "://export-bucket/nested/result_", + new BrokerDesc(null, properties)); + Assertions.assertEquals("s3://export-bucket/nested/result_", job.getExportPath()); + Map backendProperties = job.getBrokerDesc().getBackendConfigProperties(); + Assertions.assertEquals("GCP", backendProperties.get("provider")); + Assertions.assertEquals("https://storage.googleapis.com", backendProperties.get("AWS_ENDPOINT")); + Assertions.assertEquals(provider, backendProperties.get("gs.credential_provider_type")); + Assertions.assertEquals(serviceAccount, + backendProperties.getOrDefault("gs.impersonation_service_account", "")); + } + } + } + } + + @ParameterizedTest + @CsvSource({ + "http, false", "https, false", "http, true", "https, true", + "HTTP, false", "HTTPS, false" + }) + public void testHttpExportPreservesPath(String scheme, boolean usePathStyle) throws Exception { + Map properties = s3Properties("https://s3.us-east-1.amazonaws.com", "us-east-1"); + properties.put("use_path_style", Boolean.toString(usePathStyle)); + String path = scheme + "://s3.us-east-1.amazonaws.com/export-bucket/nested/result_"; + Assertions.assertEquals(path, generateExportJob(path, new BrokerDesc(null, properties)).getExportPath()); + } + + @Test + public void testNonS3ExportPreservesPath() throws Exception { + String hdfsPath = "hdfs:/tmp/export_test_"; + Assertions.assertEquals(hdfsPath, generateExportJob(hdfsPath, + new BrokerDesc(null, ImmutableMap.of("hadoop.username", "doris"))).getExportPath()); + String brokerPath = "oss://export-bucket/nested/result_"; + Assertions.assertEquals(brokerPath, generateExportJob(brokerPath, + new BrokerDesc("broker", StorageType.BROKER, ImmutableMap.of())).getExportPath()); + String localPath = "file:///tmp/export_test_"; + Assertions.assertEquals(localPath, generateExportJob(localPath, + new BrokerDesc("local", StorageType.LOCAL, null)).getExportPath()); + } + + private Map s3Properties(String endpoint, String region) { + Map properties = new HashMap<>(); + properties.put("s3.endpoint", endpoint); + properties.put("s3.region", region); + properties.put("s3.access_key", "test-ak"); + properties.put("s3.secret_key", "test-sk"); + return properties; + } + + private ExportJob generateExportJob(String path, BrokerDesc broker) throws Exception { + ExportCommand command = new ExportCommand(ImmutableList.of("T1"), ImmutableList.of(), Optional.empty(), + path, ImmutableMap.of(), Optional.of(broker)); + // Inspect the job before registration so the test never starts an export task. + Method method = ExportCommand.class.getDeclaredMethod("generateExportJob", + ConnectContext.class, Map.class, TableNameInfo.class); + method.setAccessible(true); + return (ExportJob) method.invoke(command, connectContext, ImmutableMap.of(), + new TableNameInfo("internal", "export_test", "T1")); + } @Test void testEnableInt96TimestampsProperty() { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/LoadCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/LoadCommandTest.java index 60ee030dcf3624..a15130636d8f03 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/LoadCommandTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/LoadCommandTest.java @@ -84,6 +84,46 @@ protected void runBeforeAll() throws Exception { createTable(createTableBitmapSql); } + @Test + public void testGcpNativeCredentialForS3Load() { + String loadSql = "LOAD LABEL gcp_native_load(" + + " DATA INFILE(\"gs://load-bucket/customer\")" + + " INTO TABLE customer" + + ") WITH S3(" + + " \"provider\" = \"GCP\"," + + " \"gs.endpoint\" = \"https://storage.googleapis.com\"," + + " \"gs.credential_provider_type\" = \"COMPUTE_ENGINE\"," + + " \"gs.impersonation_service_account\" = " + + "\"target@my-project.iam.gserviceaccount.com\"" + + ");"; + + LoadCommand command = (LoadCommand) new NereidsParser().parseSingle(loadSql); + BrokerDesc brokerDesc = command.getBrokerDesc(); + Assertions.assertEquals("us-east1", brokerDesc.getBackendConfigProperties().get("AWS_REGION")); + Assertions.assertEquals("GCP", brokerDesc.getBackendConfigProperties().get("provider")); + Assertions.assertEquals("COMPUTE_ENGINE", brokerDesc.getBackendConfigProperties() + .get("gs.credential_provider_type")); + Assertions.assertEquals("target@my-project.iam.gserviceaccount.com", + brokerDesc.getBackendConfigProperties().get("gs.impersonation_service_account")); + } + + @Test + public void testGcpNativeCredentialValidationHappensBeforeLoadSubmission() { + String loadSql = "LOAD LABEL invalid_gcp_native_load(" + + " DATA INFILE(\"gs://load-bucket/customer\")" + + " INTO TABLE customer" + + ") WITH S3(" + + " \"provider\" = \"GCP\"," + + " \"gs.credential_provider_type\" = \"\"" + + ");"; + + IllegalArgumentException exception = Assertions.assertThrows(IllegalArgumentException.class, + () -> new NereidsParser().parseSingle(loadSql)); + Assertions.assertTrue(exception.getMessage().contains("gs.credential_provider_type")); + Assertions.assertTrue(exception.getMessage().contains("DEFAULT")); + Assertions.assertTrue(exception.getMessage().contains("COMPUTE_ENGINE")); + } + @Test public void testLoadCommandBitmap() { String loadSql1 = "LOAD LABEL load_bitmap_table_test( " diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ShowCreateStorageVaultCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ShowCreateStorageVaultCommandTest.java new file mode 100644 index 00000000000000..eb2e2472638c52 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ShowCreateStorageVaultCommandTest.java @@ -0,0 +1,89 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.commands; + +import org.apache.doris.cloud.proto.Cloud; +import org.apache.doris.common.jmockit.Deencapsulation; +import org.apache.doris.common.util.DatasourcePrintableMap; +import org.apache.doris.datasource.storage.CloudObjectStoreAdapter; +import org.apache.doris.nereids.parser.NereidsParser; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.Map; + +public class ShowCreateStorageVaultCommandTest { + @Test + public void testNativeGcpIdentityRoundTrip() { + for (Cloud.GcpCredentialPB.CredentialProviderType providerType : List.of( + Cloud.GcpCredentialPB.CredentialProviderType.DEFAULT, + Cloud.GcpCredentialPB.CredentialProviderType.COMPUTE_ENGINE)) { + for (String account : List.of("", "target@project.iam.gserviceaccount.com")) { + Cloud.GcpCredentialPB.Builder credential = Cloud.GcpCredentialPB.newBuilder() + .setCredentialProviderType(providerType); + if (!account.isEmpty()) { + credential.setImpersonationServiceAccount(account); + } + Cloud.ObjectStoreInfoPB.Builder original = objectInfo(Cloud.ObjectStoreInfoPB.Provider.GCP); + original.getCredentialBuilder().setGcpCredential(credential); + + Map properties = shownProperties(original.build()); + Assertions.assertEquals(providerType.name(), properties.get("gs.credential_provider_type")); + Assertions.assertEquals(account, properties.getOrDefault("gs.impersonation_service_account", "")); + Assertions.assertFalse(properties.containsKey("s3.access_key")); + Assertions.assertFalse(properties.containsKey("s3.secret_key")); + Assertions.assertEquals(original.build(), CloudObjectStoreAdapter.getObjStoreInfoPB(properties).build(), + "SHOW CREATE must preserve the native identity and connection properties"); + } + } + } + + @Test + public void testLegacyKeysRemainMasked() { + for (Cloud.ObjectStoreInfoPB.Provider provider : List.of( + Cloud.ObjectStoreInfoPB.Provider.GCP, Cloud.ObjectStoreInfoPB.Provider.S3)) { + Map properties = shownProperties(objectInfo(provider) + .setAk("access-key").setSk("secret-key").build()); + Assertions.assertEquals(DatasourcePrintableMap.PASSWORD_MASK, properties.get("s3.access_key")); + Assertions.assertEquals(DatasourcePrintableMap.PASSWORD_MASK, properties.get("s3.secret_key")); + Assertions.assertFalse(properties.containsKey("gs.credential_provider_type")); + Assertions.assertFalse(properties.containsKey("gs.impersonation_service_account")); + } + } + + private Cloud.ObjectStoreInfoPB.Builder objectInfo(Cloud.ObjectStoreInfoPB.Provider provider) { + return Cloud.ObjectStoreInfoPB.newBuilder() + .setProvider(provider) + .setEndpoint("storage.googleapis.com") + .setExternalEndpoint("storage.googleapis.com") + .setRegion("us-central1") + .setBucket("bucket") + .setPrefix("vault") + .setUsePathStyle(true); + } + + private Map shownProperties(Cloud.ObjectStoreInfoPB info) { + ShowCreateStorageVaultCommand command = new ShowCreateStorageVaultCommand("gcp_vault"); + String ddl = Deencapsulation.invoke(command, "getObjectCreateStmt", info); + CreateStorageVaultCommand parsed = (CreateStorageVaultCommand) new NereidsParser().parseSingle(ddl); + Assertions.assertEquals("gcp_vault", parsed.getVaultName()); + return parsed.getProperties(); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunctionTest.java b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunctionTest.java index 92b024c2fb8b79..5218e7e7815367 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunctionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/tablefunction/ExternalFileTableValuedFunctionTest.java @@ -25,6 +25,7 @@ import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Config; +import org.apache.doris.common.FeConstants; import org.apache.doris.common.Pair; import org.apache.doris.common.util.FileFormatConstants; import org.apache.doris.common.util.FileFormatUtils; @@ -167,6 +168,53 @@ public void testHiveParquetTimeZoneRejectsAmbiguousShortAlias() { Assertions.assertTrue(exception.getMessage().contains("short timezone aliases are not supported")); } + @Test + public void testGcpNativeCredentialForS3Tvf() throws AnalysisException { + boolean previousRunningUnitTest = FeConstants.runningUnitTest; + FeConstants.runningUnitTest = true; + try { + Map properties = Maps.newHashMap(); + properties.put("uri", "gs://tvf-bucket/path/file.parquet"); + properties.put("format", "parquet"); + properties.put("provider", "GCP"); + properties.put("gs.endpoint", "https://storage.googleapis.com"); + properties.put("gs.credential_provider_type", "COMPUTE_ENGINE"); + properties.put("gs.impersonation_service_account", + "target@my-project.iam.gserviceaccount.com"); + + S3TableValuedFunction tvf = new S3TableValuedFunction(properties); + Assertions.assertEquals("s3://tvf-bucket/path/file.parquet", tvf.getFilePath()); + Assertions.assertEquals("us-east1", tvf.getBackendConnectProperties().get("AWS_REGION")); + Assertions.assertEquals("GCP", tvf.getBackendConnectProperties().get("provider")); + Assertions.assertEquals("COMPUTE_ENGINE", tvf.getBackendConnectProperties() + .get("gs.credential_provider_type")); + Assertions.assertEquals("target@my-project.iam.gserviceaccount.com", + tvf.getBackendConnectProperties().get("gs.impersonation_service_account")); + } finally { + FeConstants.runningUnitTest = previousRunningUnitTest; + } + } + + @Test + public void testGcpNativeCredentialValidationHappensBeforeTvfAccess() { + boolean previousRunningUnitTest = FeConstants.runningUnitTest; + FeConstants.runningUnitTest = true; + try { + Map properties = Maps.newHashMap(); + properties.put("uri", "gs://tvf-bucket/path/file.parquet"); + properties.put("format", "parquet"); + properties.put("provider", "GCP"); + properties.put("gs.credential_provider_type", "DEFAULT"); + properties.put("s3.session_token", "session-token"); + + IllegalArgumentException exception = Assertions.assertThrows(IllegalArgumentException.class, + () -> new S3TableValuedFunction(properties)); + Assertions.assertTrue(exception.getMessage().contains("session token")); + } finally { + FeConstants.runningUnitTest = previousRunningUnitTest; + } + } + @Test public void testCsvSchemaUuid() throws Exception { List columns = Lists.newArrayList(); diff --git a/fe/fe-filesystem/README.md b/fe/fe-filesystem/README.md index eb8e34ff06631a..95d5f8b4216e1c 100644 --- a/fe/fe-filesystem/README.md +++ b/fe/fe-filesystem/README.md @@ -141,6 +141,64 @@ At runtime, `DirectoryPluginRuntimeManager` creates an isolated `ClassLoader` pe `software.amazon.awssdk.*`, `org.apache.hadoop.*`) are loaded parent-first to avoid cross- ClassLoader cast failures. +### Native object-storage identities and protocol adapters + +The credential path has three layers with one-way dependencies: + +``` +GCS plugin -> filesystem API identity description <- fe-storage-protocol -> fe-grpc / fe-thrift + ^ ^ + | | + fe-core ---------------------+ +``` + +The GCS plugin owns user property aliases, validation and authentication. It returns an immutable +`GcpCredentialSpec` through `ObjectStorageAuthentication`. The shared API contains only data types; +it imports no SDK or generated protocol classes. Resolved credentials and partial ALTER updates are +separate values. An absent field stays absent, while an explicitly empty impersonation account clears +that setting. HMAC and anonymous modes do not contain native credentials. + +The host-side `fe-storage-protocol` module converts those descriptions directly to Cloud protobuf and +BE Thrift objects. It owns wire-schema mapping and depends on the filesystem API and generated +protocol modules, never on filesystem plugin implementations. There is no intermediate serialization, +reflection-based field mapping, or plugin-supplied wire codec. `fe-core` invokes the generic translator +and attaches the result to its request envelopes without interpreting GCP fields. + +For SHOW CREATE and Cloud stages, the protocol adapter decodes a description and the owning plugin's +`credentialToProperties()` restores user property names. Unknown wire providers, mismatched description +types, and missing or ambiguous property owners fail explicitly. + +Adding a native wire variant requires a data description, a mapping in `fe-storage-protocol`, and +matching BE/Cloud support. Filesystem plugins cannot independently extend the receiver's wire schema. +The filesystem plugin API is 5.0 and connector plugin API is 15.0 because shared authentication types +changed. Plugins must be rebuilt against the matching host APIs. + +Native OAuth accepts only the Google Storage service-base HTTPS endpoints recognized by +`GcsEndpoint` (port 443, without a bucket, path, or query). Arbitrary S3-compatible endpoints +and raw `fs.gs.auth.*` overrides are not supported in native mode. HMAC authentication retains +its existing endpoint behavior. BE error-log downloads use IAM `signBlob`; FE presigned +uploads remain unsupported with native OAuth. + +### Overlapping provider matches + +The binding registry preserves the existing provider order and `fs..support` flag semantics. +It does not treat a `provider` property as a global override; each plugin owns its interpretation +of that property. A provider can declare `fallbackProviders()` to suppress general-purpose +heuristic matches when it also matches. This applies to both primary and multi-provider binding; +explicit filesystem flags are never suppressed. Providers without this declaration retain their +existing multi-match behavior. The default HDFS binding and HDFS URI validation are preserved. + +`validateProperties()` runs on every loaded provider before selection, so invalid provider-owned +credentials cannot silently disappear when another plugin is chosen. It must be deterministic and +perform no I/O. Forced provider binding validates with the effective provider name too. + +GCS owns its endpoint/credential recognition and declares S3 and MinIO as heuristic fallbacks. +Within this family, `provider=S3` retains S3 authentication and disables GCS/MinIO guessing; +`provider=GCS` and `provider=GCP` are both recognized by GCS. These choices do not override the +existing HDFS, OSS, COS, OBS, or Azure routing order. +S3 and its shared protocol library do not depend on GCS authentication classes. Filesystem plugin +API 5.0 includes these two default hooks; all plugins must be rebuilt for that host API. + ### S3 Delegation Pattern Several cloud providers (OSS, COS, OBS) are S3-compatible. Instead of duplicating the S3 diff --git a/fe/fe-filesystem/fe-filesystem-api/src/main/java/org/apache/doris/filesystem/auth/GcpCredentialSpec.java b/fe/fe-filesystem/fe-filesystem-api/src/main/java/org/apache/doris/filesystem/auth/GcpCredentialSpec.java new file mode 100644 index 00000000000000..5711ccc184a79e --- /dev/null +++ b/fe/fe-filesystem/fe-filesystem-api/src/main/java/org/apache/doris/filesystem/auth/GcpCredentialSpec.java @@ -0,0 +1,73 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.filesystem.auth; + +import java.util.Objects; +import java.util.Optional; + +/** + * GCP identity description shared by the GCS plugin and the host protocol adapter. + * Validation of user properties and credential loading belong to the GCS plugin. + * Null fields mean absent; an empty impersonation account is an explicit ALTER clear. + */ +public final class GcpCredentialSpec implements NativeObjectStorageCredential { + public enum ProviderType { + DEFAULT, COMPUTE_ENGINE + } + + private final ProviderType providerType; + private final String impersonationServiceAccount; + + public GcpCredentialSpec(ProviderType providerType, String impersonationServiceAccount) { + this.providerType = providerType; + this.impersonationServiceAccount = impersonationServiceAccount; + } + + @Override + public String getProvider() { + return "GCP"; + } + + public Optional getProviderType() { + return Optional.ofNullable(providerType); + } + + public Optional getImpersonationServiceAccount() { + return Optional.ofNullable(impersonationServiceAccount); + } + + @Override + public boolean isEmpty() { + return providerType == null && impersonationServiceAccount == null; + } + + @Override + public boolean equals(Object other) { + if (!(other instanceof GcpCredentialSpec)) { + return false; + } + GcpCredentialSpec that = (GcpCredentialSpec) other; + return providerType == that.providerType + && Objects.equals(impersonationServiceAccount, that.impersonationServiceAccount); + } + + @Override + public int hashCode() { + return Objects.hash(providerType, impersonationServiceAccount); + } +} diff --git a/fe/fe-filesystem/fe-filesystem-api/src/main/java/org/apache/doris/filesystem/auth/NativeObjectStorageCredential.java b/fe/fe-filesystem/fe-filesystem-api/src/main/java/org/apache/doris/filesystem/auth/NativeObjectStorageCredential.java new file mode 100644 index 00000000000000..4a83e8e9078ed2 --- /dev/null +++ b/fe/fe-filesystem/fe-filesystem-api/src/main/java/org/apache/doris/filesystem/auth/NativeObjectStorageCredential.java @@ -0,0 +1,30 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.filesystem.auth; + +/** + * Immutable, protocol-independent description of a native object-storage identity. + * Implementations contain data only: no clients, credential loading, or wire serialization. + * New wire variants also require support in the host protocol adapter and the receiving BE/Cloud. + */ +public interface NativeObjectStorageCredential { + String getProvider(); + + /** True for an ALTER patch with no supplied fields. */ + boolean isEmpty(); +} diff --git a/fe/fe-filesystem/fe-filesystem-api/src/main/java/org/apache/doris/filesystem/auth/ObjectStorageAuthentication.java b/fe/fe-filesystem/fe-filesystem-api/src/main/java/org/apache/doris/filesystem/auth/ObjectStorageAuthentication.java new file mode 100644 index 00000000000000..5448da998cd6c9 --- /dev/null +++ b/fe/fe-filesystem/fe-filesystem-api/src/main/java/org/apache/doris/filesystem/auth/ObjectStorageAuthentication.java @@ -0,0 +1,80 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.filesystem.auth; + +import java.util.Objects; +import java.util.Optional; + +/** Provider-resolved authentication, independent of user property aliases, SDKs and wire formats. */ +public final class ObjectStorageAuthentication { + private final String provider; + private final boolean anonymous; + private final NativeObjectStorageCredential credential; + private final NativeObjectStorageCredential credentialUpdates; + + private ObjectStorageAuthentication(String provider, boolean anonymous, + NativeObjectStorageCredential credential, NativeObjectStorageCredential credentialUpdates) { + this.provider = Objects.requireNonNull(provider, "provider"); + this.anonymous = anonymous; + this.credential = credential; + this.credentialUpdates = credentialUpdates; + } + + public static ObjectStorageAuthentication nativeAuthentication(NativeObjectStorageCredential credential, + NativeObjectStorageCredential updates) { + Objects.requireNonNull(credential, "credential"); + Objects.requireNonNull(updates, "updates"); + if (credential.isEmpty()) { + throw new IllegalArgumentException("Resolved native credentials must not be empty"); + } + if (!credential.getProvider().equals(updates.getProvider())) { + throw new IllegalArgumentException("Credential and ALTER patch must identify the same provider"); + } + return new ObjectStorageAuthentication(credential.getProvider(), false, credential, + updates.isEmpty() ? null : updates); + } + + public static ObjectStorageAuthentication hmac(String provider) { + return new ObjectStorageAuthentication(provider, false, null, null); + } + + public static ObjectStorageAuthentication anonymous(String provider) { + return new ObjectStorageAuthentication(provider, true, null, null); + } + + public String getProvider() { + return provider; + } + + public boolean isAnonymous() { + return anonymous; + } + + public boolean isNative() { + return credential != null; + } + + public Optional getCredential() { + return Optional.ofNullable(credential); + } + + /** Only explicitly supplied native fields, including empty values that clear persisted fields. */ + public Optional getCredentialUpdates() { + return Optional.ofNullable(credentialUpdates); + } +} diff --git a/fe/fe-filesystem/fe-filesystem-api/src/test/java/org/apache/doris/filesystem/auth/ObjectStorageAuthenticationTest.java b/fe/fe-filesystem/fe-filesystem-api/src/test/java/org/apache/doris/filesystem/auth/ObjectStorageAuthenticationTest.java new file mode 100644 index 00000000000000..8458e06f43af7b --- /dev/null +++ b/fe/fe-filesystem/fe-filesystem-api/src/test/java/org/apache/doris/filesystem/auth/ObjectStorageAuthenticationTest.java @@ -0,0 +1,83 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.filesystem.auth; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +class ObjectStorageAuthenticationTest { + @Test + void absentAndClearedFieldsAreDifferentValues() { + GcpCredentialSpec absent = new GcpCredentialSpec(null, null); + GcpCredentialSpec cleared = new GcpCredentialSpec(null, ""); + Assertions.assertTrue(absent.isEmpty()); + Assertions.assertFalse(cleared.isEmpty()); + Assertions.assertNotEquals(absent, cleared); + Assertions.assertTrue(absent.getImpersonationServiceAccount().isEmpty()); + Assertions.assertEquals("", cleared.getImpersonationServiceAccount().orElseThrow()); + Assertions.assertEquals(cleared, new GcpCredentialSpec(null, "")); + Assertions.assertEquals(cleared.hashCode(), new GcpCredentialSpec(null, "").hashCode()); + } + + @Test + void resolvedIdentityAndPartialUpdateHaveSeparatePresence() { + GcpCredentialSpec resolved = new GcpCredentialSpec(GcpCredentialSpec.ProviderType.DEFAULT, null); + ObjectStorageAuthentication initial = ObjectStorageAuthentication.nativeAuthentication( + resolved, new GcpCredentialSpec(null, null)); + Assertions.assertTrue(initial.isNative()); + Assertions.assertFalse(initial.isAnonymous()); + Assertions.assertEquals("GCP", initial.getProvider()); + Assertions.assertEquals(resolved, initial.getCredential().orElseThrow()); + Assertions.assertTrue(initial.getCredentialUpdates().isEmpty()); + ObjectStorageAuthentication alter = ObjectStorageAuthentication.nativeAuthentication( + resolved, new GcpCredentialSpec(null, "")); + Assertions.assertEquals(new GcpCredentialSpec(null, ""), alter.getCredentialUpdates().orElseThrow()); + } + + @Test + void invalidNativeAuthenticationCannotBeConstructed() { + Assertions.assertThrows(IllegalArgumentException.class, () -> ObjectStorageAuthentication.nativeAuthentication( + new GcpCredentialSpec(null, null), new GcpCredentialSpec(null, null))); + NativeObjectStorageCredential foreign = new NativeObjectStorageCredential() { + @Override + public String getProvider() { + return "OTHER"; + } + + @Override + public boolean isEmpty() { + return false; + } + }; + Assertions.assertThrows(IllegalArgumentException.class, () -> ObjectStorageAuthentication.nativeAuthentication( + new GcpCredentialSpec(GcpCredentialSpec.ProviderType.DEFAULT, null), foreign)); + } + + @Test + void nonNativeModesCannotContainNativeFields() { + ObjectStorageAuthentication hmac = ObjectStorageAuthentication.hmac("GCP"); + ObjectStorageAuthentication anonymous = ObjectStorageAuthentication.anonymous("GCP"); + Assertions.assertFalse(hmac.isAnonymous()); + Assertions.assertTrue(anonymous.isAnonymous()); + for (ObjectStorageAuthentication auth : new ObjectStorageAuthentication[] {hmac, anonymous}) { + Assertions.assertFalse(auth.isNative()); + Assertions.assertTrue(auth.getCredential().isEmpty()); + Assertions.assertTrue(auth.getCredentialUpdates().isEmpty()); + } + } +} diff --git a/fe/fe-filesystem/fe-filesystem-gcs-common/pom.xml b/fe/fe-filesystem/fe-filesystem-gcs-common/pom.xml new file mode 100644 index 00000000000000..b6357bbf3185b3 --- /dev/null +++ b/fe/fe-filesystem/fe-filesystem-gcs-common/pom.xml @@ -0,0 +1,42 @@ + + + + 4.0.0 + + + org.apache.doris + fe-filesystem + ${revision} + ../pom.xml + + + fe-filesystem-gcs-common + Doris GCS authentication and property rules + GCS property and authentication rules shared by GCS and Iceberg; not part of the host API. + + + org.junit.jupiter + junit-jupiter + test + + + diff --git a/fe/fe-filesystem/fe-filesystem-gcs-common/src/main/java/org/apache/doris/filesystem/gcs/auth/GcpCredential.java b/fe/fe-filesystem/fe-filesystem-gcs-common/src/main/java/org/apache/doris/filesystem/gcs/auth/GcpCredential.java new file mode 100644 index 00000000000000..c32cd4d271a4f5 --- /dev/null +++ b/fe/fe-filesystem/fe-filesystem-gcs-common/src/main/java/org/apache/doris/filesystem/gcs/auth/GcpCredential.java @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.filesystem.gcs.auth; + + + +/** Validated GCP identity used by the GCS authentication implementation. */ +public final class GcpCredential { + public static final String CREDENTIAL_PROVIDER_TYPE = "gs.credential_provider_type"; + public static final String IMPERSONATION_SERVICE_ACCOUNT = "gs.impersonation_service_account"; + + private static final String SERVICE_ACCOUNT_PATTERN = + "^[^@\\s]+@(?:[^@\\s]+\\.iam\\.gserviceaccount\\.com|" + + "developer\\.gserviceaccount\\.com|appspot\\.gserviceaccount\\.com)$"; + + private final GcpCredentialProviderType providerType; + private final String impersonationServiceAccount; + + public GcpCredential(GcpCredentialProviderType providerType, String impersonationServiceAccount) { + this.providerType = providerType; + this.impersonationServiceAccount = impersonationServiceAccount == null ? "" : impersonationServiceAccount; + validate(); + } + + public GcpCredentialProviderType getCredentialProviderType() { + return providerType; + } + + public String getImpersonationServiceAccount() { + return impersonationServiceAccount; + } + + private void validate() { + if (providerType == GcpCredentialProviderType.ANONYMOUS) { + throw new IllegalArgumentException("Anonymous access does not use a native GCP credential"); + } + if (impersonationServiceAccount.isEmpty()) { + return; + } + if (!impersonationServiceAccount.matches(SERVICE_ACCOUNT_PATTERN)) { + throw new IllegalArgumentException("Invalid GCP service account email: " + + impersonationServiceAccount); + } + } + +} diff --git a/fe/fe-filesystem/fe-filesystem-gcs-common/src/main/java/org/apache/doris/filesystem/gcs/auth/GcpCredentialProviderType.java b/fe/fe-filesystem/fe-filesystem-gcs-common/src/main/java/org/apache/doris/filesystem/gcs/auth/GcpCredentialProviderType.java new file mode 100644 index 00000000000000..f8bc0c781fadf0 --- /dev/null +++ b/fe/fe-filesystem/fe-filesystem-gcs-common/src/main/java/org/apache/doris/filesystem/gcs/auth/GcpCredentialProviderType.java @@ -0,0 +1,35 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.filesystem.gcs.auth; + +import java.util.Locale; + +public enum GcpCredentialProviderType { + DEFAULT, + COMPUTE_ENGINE, + ANONYMOUS; + + public static GcpCredentialProviderType fromString(String propertyName, String value) { + try { + return valueOf(value.trim().toUpperCase(Locale.ROOT)); + } catch (RuntimeException e) { + throw new IllegalArgumentException("Invalid " + propertyName + " value: " + value + + ". Only DEFAULT, COMPUTE_ENGINE and ANONYMOUS are supported.", e); + } + } +} diff --git a/fe/fe-filesystem/fe-filesystem-gcs-common/src/main/java/org/apache/doris/filesystem/gcs/auth/GcsAuth.java b/fe/fe-filesystem/fe-filesystem-gcs-common/src/main/java/org/apache/doris/filesystem/gcs/auth/GcsAuth.java new file mode 100644 index 00000000000000..2d45f2d4301807 --- /dev/null +++ b/fe/fe-filesystem/fe-filesystem-gcs-common/src/main/java/org/apache/doris/filesystem/gcs/auth/GcsAuth.java @@ -0,0 +1,50 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.filesystem.gcs.auth; + +import java.util.Optional; + +/** Immutable authentication choice shared by GCS clients, cache identity and protocol builders. */ +public final class GcsAuth { + public enum Mode { + HMAC, ADC, COMPUTE_ENGINE, ANONYMOUS + } + + private final Mode mode; + private final GcpCredential nativeCredential; + + GcsAuth(Mode mode, String impersonationServiceAccount) { + this.mode = mode; + this.nativeCredential = mode == Mode.ADC || mode == Mode.COMPUTE_ENGINE + ? new GcpCredential(mode == Mode.ADC ? GcpCredentialProviderType.DEFAULT + : GcpCredentialProviderType.COMPUTE_ENGINE, impersonationServiceAccount) + : null; + } + + public Mode getMode() { + return mode; + } + + public boolean isAnonymous() { + return mode == Mode.ANONYMOUS; + } + + public Optional getNativeCredential() { + return Optional.ofNullable(nativeCredential); + } +} diff --git a/fe/fe-filesystem/fe-filesystem-gcs-common/src/main/java/org/apache/doris/filesystem/gcs/auth/GcsAuthResolver.java b/fe/fe-filesystem/fe-filesystem-gcs-common/src/main/java/org/apache/doris/filesystem/gcs/auth/GcsAuthResolver.java new file mode 100644 index 00000000000000..dce0d7729bf04b --- /dev/null +++ b/fe/fe-filesystem/fe-filesystem-gcs-common/src/main/java/org/apache/doris/filesystem/gcs/auth/GcsAuthResolver.java @@ -0,0 +1,169 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.filesystem.gcs.auth; + + +import java.util.Locale; +import java.util.Map; +import java.util.Optional; + +/** + * Resolves GCS authentication without creating clients or loading credentials. + * Vault ALTER still preserves field presence when constructing its patch from this result. + */ +public final class GcsAuthResolver { + private static final String CREDENTIAL_PROVIDER_TYPE = GcpCredential.CREDENTIAL_PROVIDER_TYPE; + private static final String IMPERSONATION_SERVICE_ACCOUNT = GcpCredential.IMPERSONATION_SERVICE_ACCOUNT; + private static final String[] ACCESS_KEYS = {"gs.access_key", "s3.access_key", "AWS_ACCESS_KEY", "access_key"}; + private static final String[] SECRET_KEYS = {"gs.secret_key", "s3.secret_key", "AWS_SECRET_KEY", "secret_key"}; + private static final String[] TOKENS = {"gs.session_token", "s3.session_token", "AWS_TOKEN", "session_token"}; + private static final String[] AWS_PROVIDERS = {"s3.credentials_provider_type", "AWS_CREDENTIALS_PROVIDER_TYPE"}; + private static final String[] AWS_ROLES = {"s3.role_arn", "AWS_ROLE_ARN", "s3.external_id", "AWS_EXTERNAL_ID"}; + + private GcsAuthResolver() { + } + + public static Optional resolve(Map properties) { + boolean hasNativeProperties = hasNativeCredentialProperties(properties); + boolean hasGcsSelector = guessIsGcs(properties) + || "true".equalsIgnoreCase(getPropertyIgnoreCase(properties, "fs.gcs.support")); + boolean hasAccessKey = hasNonBlankProperty(properties, ACCESS_KEYS); + boolean hasSecretKey = hasNonBlankProperty(properties, SECRET_KEYS); + boolean hasLegacyAnonymous = hasLegacyProvider(properties, "ANONYMOUS"); + String provider = getPropertyIgnoreCase(properties, "provider"); + if (isNotBlank(provider) && !"GCP".equalsIgnoreCase(provider) && !"GCS".equalsIgnoreCase(provider)) { + if (hasNativeProperties) { + throw new IllegalArgumentException("Native GCP authentication requires provider=GCP, but found: " + + provider); + } + // Legacy GCS configurations may explicitly select the S3-compatible protocol. + // Preserve explicit HMAC/anonymous authentication without enabling native ADC for S3. + // Omitted credentials still belong to S3's default chain, even at a GCS endpoint. + if (!"S3".equalsIgnoreCase(provider) || !hasGcsSelector + || !(hasAccessKey || hasSecretKey || hasLegacyAnonymous)) { + return Optional.empty(); + } + } + if (!hasNativeProperties && !hasGcsSelector) { + return Optional.empty(); + } + + String providerType = getPropertyIgnoreCase(properties, CREDENTIAL_PROVIDER_TYPE); + String impersonation = getPropertyIgnoreCase(properties, IMPERSONATION_SERVICE_ACCOUNT); + GcsAuth.Mode mode; + if (hasNativeProperties) { + GcpCredentialProviderType type = providerType == null ? GcpCredentialProviderType.DEFAULT + : GcpCredentialProviderType.fromString(CREDENTIAL_PROVIDER_TYPE, providerType); + mode = type == GcpCredentialProviderType.ANONYMOUS ? GcsAuth.Mode.ANONYMOUS + : type == GcpCredentialProviderType.COMPUTE_ENGINE ? GcsAuth.Mode.COMPUTE_ENGINE : GcsAuth.Mode.ADC; + } else if (hasAccessKey || hasSecretKey) { + // Legacy static credentials take precedence over a dormant anonymous selector. + mode = GcsAuth.Mode.HMAC; + } else if (hasLegacyAnonymous) { + mode = GcsAuth.Mode.ANONYMOUS; + } else if (hasLegacyProvider(properties, "DEFAULT")) { + // Explicit legacy DEFAULT uses ambient HMAC credentials through the AWS default chain. + mode = GcsAuth.Mode.HMAC; + } else { + mode = GcsAuth.Mode.ADC; + } + + // The storage properties validate paired HMAC keys. Raw Vault patches may update only one key. + if (mode != GcsAuth.Mode.HMAC) { + if (hasAccessKey || hasSecretKey || hasNonBlankProperty(properties, TOKENS)) { + throw new IllegalArgumentException(CREDENTIAL_PROVIDER_TYPE + + " cannot be used together with access key, secret key, or session token."); + } + if (hasNonBlankProperty(properties, AWS_ROLES) + || ((hasNativeProperties || mode != GcsAuth.Mode.ANONYMOUS) + && hasNonBlankProperty(properties, AWS_PROVIDERS))) { + throw new IllegalArgumentException("Native GCP authentication cannot be used together with " + + "AWS role, external ID, or credentials provider properties."); + } + } + if (mode == GcsAuth.Mode.ANONYMOUS && impersonation != null && !impersonation.isEmpty()) { + throw new IllegalArgumentException("ANONYMOUS cannot be used with " + IMPERSONATION_SERVICE_ACCOUNT); + } + return Optional.of(new GcsAuth(mode, impersonation)); + } + + public static boolean hasNativeCredentialProperties(Map properties) { + return properties.keySet().stream().anyMatch(key -> CREDENTIAL_PROVIDER_TYPE.equalsIgnoreCase(key) + || IMPERSONATION_SERVICE_ACCOUNT.equalsIgnoreCase(key)); + } + + // Storage selection also uses these signals, before authentication is resolved. + public static boolean guessIsGcs(Map properties) { + for (Map.Entry entry : properties.entrySet()) { + String key = entry.getKey().toLowerCase(Locale.ROOT); + String value = entry.getValue(); + if ("provider".equals(key) && ("GCP".equalsIgnoreCase(value) || "GCS".equalsIgnoreCase(value))) { + return true; + } + if (CREDENTIAL_PROVIDER_TYPE.equals(key) || IMPERSONATION_SERVICE_ACCOUNT.equals(key)) { + return true; + } + if (("uri".equals(key) || "warehouse".equals(key)) + && value != null && value.regionMatches(true, 0, "gs://", 0, 5)) { + return true; + } + if ("gs.endpoint".equals(key) && isNotBlank(value)) { + return true; + } + if (("s3.endpoint".equals(key) || "aws_endpoint".equals(key) || "endpoint".equals(key)) + && value != null && GcsEndpoint.isGcsEndpoint(value)) { + return true; + } + } + return false; + } + + private static boolean isNotBlank(String value) { + return value != null && !value.isBlank(); + } + + private static String trim(String value) { + return value == null ? null : value.trim(); + } + + private static boolean hasLegacyProvider(Map properties, String providerType) { + for (String name : AWS_PROVIDERS) { + if (providerType.equalsIgnoreCase(trim(getPropertyIgnoreCase(properties, name)))) { + return true; + } + } + return false; + } + + private static boolean hasNonBlankProperty(Map properties, String... names) { + for (String name : names) { + if (isNotBlank(getPropertyIgnoreCase(properties, name))) { + return true; + } + } + return false; + } + + private static String getPropertyIgnoreCase(Map properties, String name) { + return properties.entrySet().stream() + .filter(entry -> name.equalsIgnoreCase(entry.getKey())) + .map(Map.Entry::getValue) + .findFirst() + .orElse(null); + } +} diff --git a/fe/fe-filesystem/fe-filesystem-gcs-common/src/main/java/org/apache/doris/filesystem/gcs/auth/GcsEndpoint.java b/fe/fe-filesystem/fe-filesystem-gcs-common/src/main/java/org/apache/doris/filesystem/gcs/auth/GcsEndpoint.java new file mode 100644 index 00000000000000..5395a459f82cd5 --- /dev/null +++ b/fe/fe-filesystem/fe-filesystem-gcs-common/src/main/java/org/apache/doris/filesystem/gcs/auth/GcsEndpoint.java @@ -0,0 +1,73 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.filesystem.gcs.auth; + +import java.net.URI; +import java.util.regex.Pattern; + +/** Shared GCS host recognition and native OAuth endpoint validation. */ +public final class GcsEndpoint { + // Endpoint overrides must be service-base hosts; SDK resolvers add the bucket. + private static final Pattern STORAGE_BASE_HOST = Pattern.compile( + "(?:storage\\.googleapis\\.com|[a-z0-9-]+-storage\\.googleapis\\.com|" + + "storage\\.[a-z0-9-]+\\.rep\\.googleapis\\.com)", Pattern.CASE_INSENSITIVE); + // Resolved virtual-hosted requests legitimately contain the bucket prefix. + private static final Pattern STORAGE_HOST = Pattern.compile( + "(?:[a-z0-9-]+\\.)*" + STORAGE_BASE_HOST.pattern(), Pattern.CASE_INSENSITIVE); + + private GcsEndpoint() { + } + + /** Identifies GCS by host; authentication-specific restrictions are checked after binding. */ + public static boolean isGcsEndpoint(String endpoint) { + try { + String host = parse(endpoint).getHost(); + return host != null && STORAGE_HOST.matcher(host).matches(); + } catch (IllegalArgumentException e) { + // A malformed URI cannot identify a filesystem provider. + return false; + } + } + + private static URI parse(String endpoint) { + return URI.create(endpoint.contains("://") ? endpoint : "https://" + endpoint); + } + + public static URI validateEndpoint(String endpoint) { + URI uri = parse(endpoint); + validateRequestUri(uri); + if (!STORAGE_BASE_HOST.matcher(uri.getHost()).matches()) { + throw new IllegalArgumentException( + "Native GCP OAuth endpoint must be a service-base host without a bucket"); + } + if (uri.getRawQuery() != null || (!uri.getRawPath().isEmpty() && !"/".equals(uri.getRawPath()))) { + throw new IllegalArgumentException("Native GCP OAuth endpoint must not contain a path or query"); + } + return uri; + } + + public static void validateRequestUri(URI uri) { + if (!"https".equalsIgnoreCase(uri.getScheme()) || uri.getHost() == null + || !STORAGE_HOST.matcher(uri.getHost()).matches() + || (uri.getPort() != -1 && uri.getPort() != 443) + || uri.getRawUserInfo() != null || uri.getRawFragment() != null) { + throw new IllegalArgumentException( + "Native GCP OAuth requires a trusted Google Cloud Storage HTTPS endpoint on port 443"); + } + } +} diff --git a/fe/fe-filesystem/fe-filesystem-gcs-common/src/test/java/org/apache/doris/filesystem/gcs/auth/GcsAuthResolverTest.java b/fe/fe-filesystem/fe-filesystem-gcs-common/src/test/java/org/apache/doris/filesystem/gcs/auth/GcsAuthResolverTest.java new file mode 100644 index 00000000000000..9b6a39f9e658c5 --- /dev/null +++ b/fe/fe-filesystem/fe-filesystem-gcs-common/src/test/java/org/apache/doris/filesystem/gcs/auth/GcsAuthResolverTest.java @@ -0,0 +1,206 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.filesystem.gcs.auth; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.HashMap; +import java.util.Map; + +public class GcsAuthResolverTest { + @Test + public void testDefaultAndExplicitModes() { + Map props = new HashMap<>(); + Assertions.assertFalse(GcsAuthResolver.resolve(props).isPresent()); + props.put("provider", "GCP"); + Assertions.assertEquals(GcsAuth.Mode.ADC, GcsAuthResolver.resolve(props).get().getMode()); + for (String provider : new String[] {"DEFAULT", "COMPUTE_ENGINE", "ANONYMOUS"}) { + props.put(GcpCredential.CREDENTIAL_PROVIDER_TYPE, provider); + GcsAuth auth = GcsAuthResolver.resolve(props).get(); + Assertions.assertEquals(provider.equals("DEFAULT") ? GcsAuth.Mode.ADC : GcsAuth.Mode.valueOf(provider), + auth.getMode()); + Assertions.assertEquals(!provider.equals("ANONYMOUS"), auth.getNativeCredential().isPresent()); + } + } + + @Test + public void testImpersonationAndFactoryUseResolvedCredentials() { + Map props = new HashMap<>(); + props.put(GcpCredential.IMPERSONATION_SERVICE_ACCOUNT, "target@test.iam.gserviceaccount.com"); + GcsAuth auth = GcsAuthResolver.resolve(props).get(); + Assertions.assertEquals(GcsAuth.Mode.ADC, auth.getMode()); + GcpCredential credential = auth.getNativeCredential().orElseThrow(); + Assertions.assertEquals(auth.getNativeCredential().get().getCredentialProviderType(), + credential.getCredentialProviderType()); + Assertions.assertEquals("target@test.iam.gserviceaccount.com", credential.getImpersonationServiceAccount()); + } + + @Test + public void testHmacAliasesDoNotAcquireAdc() { + for (String[] keys : new String[][] {{"gs.access_key", "gs.secret_key"}, + {"s3.access_key", "s3.secret_key"}, {"AWS_ACCESS_KEY", "AWS_SECRET_KEY"}, + {"ACCESS_KEY", "SECRET_KEY"}}) { + Map props = new HashMap<>(); + props.put("provider", "GCP"); + props.put(keys[0], "ak"); + props.put(keys[1], "sk"); + Assertions.assertEquals(GcsAuth.Mode.HMAC, GcsAuthResolver.resolve(props).get().getMode()); + Assertions.assertFalse(GcsAuthResolver.resolve(props).get().getNativeCredential().isPresent()); + // Vault ALTER may supply just one HMAC field; it must not inject native credentials. + props.remove(keys[1]); + Assertions.assertEquals(GcsAuth.Mode.HMAC, GcsAuthResolver.resolve(props).get().getMode()); + } + } + + @Test + public void testLegacyS3HmacRequiresGcsSelector() { + Map props = new HashMap<>(); + props.put("provider", "S3"); + props.put("s3.access_key", "ak"); + props.put("s3.secret_key", "sk"); + Assertions.assertFalse(GcsAuthResolver.resolve(props).isPresent()); + props.put("s3.endpoint", "storage.googleapis.com"); + Assertions.assertEquals(GcsAuth.Mode.HMAC, GcsAuthResolver.resolve(props).get().getMode()); + for (String provider : new String[] {"AZURE", "OSS"}) { + props.put("provider", provider); + Assertions.assertFalse(GcsAuthResolver.resolve(props).isPresent()); + } + props.put("provider", "S3"); + props.put(GcpCredential.CREDENTIAL_PROVIDER_TYPE, "DEFAULT"); + Assertions.assertThrows(IllegalArgumentException.class, () -> GcsAuthResolver.resolve(props)); + } + + @Test + public void testExplicitS3OmittedAuthenticationPreservesDefaultChain() { + Map props = new HashMap<>(); + props.put("provider", "s3"); + Assertions.assertFalse(GcsAuthResolver.resolve(props).isPresent()); + props.put("s3.endpoint", "https://storage.googleapis.com"); + Assertions.assertFalse(GcsAuthResolver.resolve(props).isPresent()); + + props.put("s3.credentials_provider_type", "DEFAULT"); + Assertions.assertFalse(GcsAuthResolver.resolve(props).isPresent()); + props.put("s3.credentials_provider_type", "ANONYMOUS"); + Assertions.assertEquals(GcsAuth.Mode.ANONYMOUS, GcsAuthResolver.resolve(props).get().getMode()); + props.remove("s3.credentials_provider_type"); + props.put("provider", "GCP"); + Assertions.assertEquals(GcsAuth.Mode.ADC, GcsAuthResolver.resolve(props).get().getMode()); + props.remove("provider"); + Assertions.assertEquals(GcsAuth.Mode.ADC, GcsAuthResolver.resolve(props).get().getMode()); + } + + @Test + public void testLegacyGcpDefaultChainDoesNotAcquireAdc() { + for (String key : new String[] {"s3.credentials_provider_type", "AWS_CREDENTIALS_PROVIDER_TYPE"}) { + Map props = new HashMap<>(); + props.put("provider", "GCP"); + props.put(key, " default "); + GcsAuth auth = GcsAuthResolver.resolve(props).orElseThrow(); + Assertions.assertEquals(GcsAuth.Mode.HMAC, auth.getMode()); + Assertions.assertFalse(auth.getNativeCredential().isPresent()); + Assertions.assertFalse(auth.isAnonymous()); + // Native selectors still conflict with an explicitly selected AWS credential chain. + props.put(GcpCredential.CREDENTIAL_PROVIDER_TYPE, "DEFAULT"); + Assertions.assertThrows(IllegalArgumentException.class, () -> GcsAuthResolver.resolve(props)); + props.remove(GcpCredential.CREDENTIAL_PROVIDER_TYPE); + props.put(GcpCredential.IMPERSONATION_SERVICE_ACCOUNT, "target@test.iam.gserviceaccount.com"); + Assertions.assertThrows(IllegalArgumentException.class, () -> GcsAuthResolver.resolve(props)); + } + } + + @Test + public void testLegacyStaticKeysTakePrecedenceOverAnonymous() { + for (String selector : new String[] {"s3.credentials_provider_type", "AWS_CREDENTIALS_PROVIDER_TYPE"}) { + for (String[] keys : new String[][] {{"gs.access_key", "gs.secret_key"}, + {"s3.access_key", "s3.secret_key"}, {"AWS_ACCESS_KEY", "AWS_SECRET_KEY"}, + {"ACCESS_KEY", "SECRET_KEY"}}) { + Map props = new HashMap<>(); + props.put("provider", "GCP"); + props.put(selector, " anonymous "); + props.put(keys[0], "ak"); + props.put(keys[1], "sk"); + GcsAuth auth = GcsAuthResolver.resolve(props).orElseThrow(); + Assertions.assertEquals(GcsAuth.Mode.HMAC, auth.getMode()); + Assertions.assertFalse(auth.isAnonymous()); + Assertions.assertTrue(auth.getNativeCredential().isEmpty()); + for (String mode : new String[] {"DEFAULT", "COMPUTE_ENGINE", "ANONYMOUS"}) { + props.put(GcpCredential.CREDENTIAL_PROVIDER_TYPE, mode); + Assertions.assertThrows(IllegalArgumentException.class, () -> GcsAuthResolver.resolve(props)); + } + props.remove(GcpCredential.CREDENTIAL_PROVIDER_TYPE); + props.put(GcpCredential.IMPERSONATION_SERVICE_ACCOUNT, "target@test.iam.gserviceaccount.com"); + Assertions.assertThrows(IllegalArgumentException.class, () -> GcsAuthResolver.resolve(props)); + props.remove(GcpCredential.IMPERSONATION_SERVICE_ACCOUNT); + // Partial vault patches must also retain HMAC authentication. + props.remove(keys[1]); + Assertions.assertEquals(GcsAuth.Mode.HMAC, GcsAuthResolver.resolve(props).orElseThrow().getMode()); + props.remove(keys[0]); + props.put(keys[1], "sk"); + Assertions.assertEquals(GcsAuth.Mode.HMAC, GcsAuthResolver.resolve(props).orElseThrow().getMode()); + props.remove(keys[1]); + Assertions.assertTrue(GcsAuthResolver.resolve(props).orElseThrow().isAnonymous()); + } + } + } + + @Test + public void testOtherProvidersAndConflictingNativeProperties() { + for (String provider : new String[] {"S3", "AZURE", "OSS"}) { + Map props = new HashMap<>(); + props.put("provider", provider); + props.put("s3.endpoint", "storage.googleapis.com"); + Assertions.assertFalse(GcsAuthResolver.resolve(props).isPresent()); + props.put(GcpCredential.CREDENTIAL_PROVIDER_TYPE, "DEFAULT"); + Assertions.assertThrows(IllegalArgumentException.class, () -> GcsAuthResolver.resolve(props)); + } + Map props = new HashMap<>(); + props.put(GcpCredential.CREDENTIAL_PROVIDER_TYPE, ""); + Assertions.assertThrows(IllegalArgumentException.class, () -> GcsAuthResolver.resolve(props)); + } + + @Test + public void testEndpointAliasesResolveGcsByUriHost() { + for (String key : new String[] {"s3.endpoint", "AWS_ENDPOINT", "endpoint", "ENDPOINT"}) { + for (String endpoint : new String[] {"storage.googleapis.com", "storage.googleapis.com:443", + "https://storage.googleapis.com/", "https://storage.googleapis.com:443/", + "HTTPS://STORAGE.GOOGLEAPIS.COM", "https://storage.us-east1.rep.googleapis.com", + "https://us-east1-storage.googleapis.com", "https://bucket.storage.googleapis.com"}) { + Map props = new HashMap<>(); + props.put(key, endpoint); + Assertions.assertTrue(GcsAuthResolver.guessIsGcs(props), endpoint); + Assertions.assertEquals(GcsAuth.Mode.ADC, GcsAuthResolver.resolve(props).orElseThrow().getMode()); + props.put("s3.access_key", "ak"); + props.put("s3.secret_key", "sk"); + Assertions.assertEquals(GcsAuth.Mode.HMAC, GcsAuthResolver.resolve(props).orElseThrow().getMode()); + } + } + } + + @Test + public void testEndpointInferenceRejectsForeignAndMalformedHosts() { + for (String endpoint : new String[] {"https://notstorage.googleapis.com", + "https://storage.googleapis.com.example.com", "https://example.com/storage.googleapis.com", + "https://storage.googleapis.com@example.com", "https://[invalid", ""}) { + Map props = Map.of("s3.endpoint", endpoint); + Assertions.assertFalse(GcsAuthResolver.guessIsGcs(props), endpoint); + Assertions.assertFalse(GcsAuthResolver.resolve(props).isPresent(), endpoint); + } + } + +} diff --git a/fe/fe-filesystem/fe-filesystem-gcs/pom.xml b/fe/fe-filesystem/fe-filesystem-gcs/pom.xml index 6c276177773547..a2b6bdb6d69794 100644 --- a/fe/fe-filesystem/fe-filesystem-gcs/pom.xml +++ b/fe/fe-filesystem/fe-filesystem-gcs/pom.xml @@ -36,6 +36,16 @@ under the License. client in fe-filesystem-s3-base. + + org.apache.doris + fe-filesystem-gcs-common + ${revision} + + + com.google.auth + google-auth-library-oauth2-http + 1.46.0 + org.apache.doris fe-filesystem-spi diff --git a/fe/fe-filesystem/fe-filesystem-gcs/src/main/java/org/apache/doris/filesystem/gcs/GcpAuth.java b/fe/fe-filesystem/fe-filesystem-gcs/src/main/java/org/apache/doris/filesystem/gcs/GcpAuth.java new file mode 100644 index 00000000000000..3f3f1c4492e06b --- /dev/null +++ b/fe/fe-filesystem/fe-filesystem-gcs/src/main/java/org/apache/doris/filesystem/gcs/GcpAuth.java @@ -0,0 +1,154 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.filesystem.gcs; + +import org.apache.doris.filesystem.gcs.auth.GcpCredential; +import org.apache.doris.filesystem.gcs.auth.GcsEndpoint; + +import com.google.auth.oauth2.ComputeEngineCredentials; +import com.google.auth.oauth2.GoogleCredentials; +import com.google.auth.oauth2.ImpersonatedCredentials; +import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider; +import software.amazon.awssdk.core.client.config.ClientOverrideConfiguration; +import software.amazon.awssdk.core.client.config.SdkAdvancedClientOption; +import software.amazon.awssdk.core.interceptor.Context; +import software.amazon.awssdk.core.interceptor.ExecutionAttributes; +import software.amazon.awssdk.core.interceptor.ExecutionInterceptor; +import software.amazon.awssdk.core.retry.RetryPolicy; +import software.amazon.awssdk.core.retry.backoff.EqualJitterBackoffStrategy; +import software.amazon.awssdk.core.signer.NoOpSigner; +import software.amazon.awssdk.http.SdkHttpRequest; +import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.S3Configuration; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.net.URI; +import java.time.Duration; +import java.util.Collections; +import java.util.List; +import java.util.Map; + +/** Creates GCP OAuth2 credentials and adapts them to the S3-compatible client. */ +public final class GcpAuth { + public static final String DEVSTORAGE_SCOPE = + "https://www.googleapis.com/auth/devstorage.read_write"; + private static final String CLOUD_PLATFORM_SCOPE = "https://www.googleapis.com/auth/cloud-platform"; + private static final int IMPERSONATION_LIFETIME_SECONDS = 3600; + + private GcpAuth() { + } + + public static S3Client createS3Client(GcsFileSystemProperties properties) throws IOException { + return S3Client.builder() + .httpClient(UrlConnectionHttpClient.builder().socketTimeout(Duration.ofSeconds(30)) + .connectionTimeout(Duration.ofSeconds(30)).build()) + .endpointOverride(endpoint(properties)) + .credentialsProvider(AnonymousCredentialsProvider.create()) + .region(Region.of(properties.getRegion())) + .overrideConfiguration(clientConfiguration(properties)) + .serviceConfiguration(serviceConfiguration(properties)) + .build(); + } + + public static S3AsyncClient createS3AsyncClient(GcsFileSystemProperties properties) throws IOException { + return S3AsyncClient.builder() + .endpointOverride(endpoint(properties)) + .credentialsProvider(AnonymousCredentialsProvider.create()) + .region(Region.of(properties.getRegion())) + .overrideConfiguration(clientConfiguration(properties)) + .serviceConfiguration(serviceConfiguration(properties)) + .build(); + } + + private static URI endpoint(GcsFileSystemProperties properties) { + return validateEndpoint(properties.getEndpoint()); + } + + static URI validateEndpoint(String endpoint) { + return GcsEndpoint.validateEndpoint(endpoint); + } + + private static S3Configuration serviceConfiguration(GcsFileSystemProperties properties) { + return S3Configuration.builder().chunkedEncodingEnabled(false) + .pathStyleAccessEnabled(properties.isUsePathStyle()).build(); + } + + private static ClientOverrideConfiguration clientConfiguration(GcsFileSystemProperties properties) + throws IOException { + return ClientOverrideConfiguration.builder() + .retryPolicy(RetryPolicy.builder().numRetries(3) + .backoffStrategy(EqualJitterBackoffStrategy.builder() + .baseDelay(Duration.ofSeconds(1)) + .maxBackoffTime(Duration.ofMinutes(1)).build()).build()) + .putAdvancedOption(SdkAdvancedClientOption.SIGNER, new NoOpSigner()) + .addExecutionInterceptor(bearerTokenInterceptor(createCredentials( + properties.getAuth().getNativeCredential().orElseThrow()))) + .build(); + } + + public static GoogleCredentials createCredentials(GcpCredential credential) throws IOException { + GoogleCredentials source; + switch (credential.getCredentialProviderType()) { + case DEFAULT: + source = GoogleCredentials.getApplicationDefault(); + break; + case COMPUTE_ENGINE: + source = ComputeEngineCredentials.create(); + break; + default: + throw new IllegalArgumentException("Unsupported GCP credential provider: " + + credential.getCredentialProviderType()); + } + if (credential.getImpersonationServiceAccount().isEmpty()) { + return source.createScoped(Collections.singletonList(DEVSTORAGE_SCOPE)); + } + // The source token calls the IAM Credentials API; the impersonated + // target token itself only needs object read/write access. + source = source.createScoped(Collections.singletonList(CLOUD_PLATFORM_SCOPE)); + return ImpersonatedCredentials.create(source, credential.getImpersonationServiceAccount(), null, + Collections.singletonList(DEVSTORAGE_SCOPE), IMPERSONATION_LIFETIME_SECONDS); + } + + public static ExecutionInterceptor bearerTokenInterceptor(GoogleCredentials credentials) { + return new ExecutionInterceptor() { + @Override + public SdkHttpRequest modifyHttpRequest(Context.ModifyHttpRequest context, + ExecutionAttributes executionAttributes) { + try { + // Validate the resolved request host before obtaining or attaching process credentials. + GcsEndpoint.validateRequestUri(context.httpRequest().getUri()); + Map> metadata = credentials.getRequestMetadata(); + List authorization = metadata.get("Authorization"); + if (authorization == null || authorization.isEmpty() + || authorization.get(0) == null || authorization.get(0).isEmpty()) { + throw new IOException("GCP credentials did not provide an Authorization header"); + } + return context.httpRequest().toBuilder() + .putHeader("Authorization", authorization.get(0)) + .build(); + } catch (IOException e) { + throw new UncheckedIOException("Failed to obtain a GCP access token", e); + } + } + }; + } +} diff --git a/fe/fe-filesystem/fe-filesystem-gcs/src/main/java/org/apache/doris/filesystem/gcs/GcsFileSystemProperties.java b/fe/fe-filesystem/fe-filesystem-gcs/src/main/java/org/apache/doris/filesystem/gcs/GcsFileSystemProperties.java index f24e7b3b77a49c..513b68127a0a64 100644 --- a/fe/fe-filesystem/fe-filesystem-gcs/src/main/java/org/apache/doris/filesystem/gcs/GcsFileSystemProperties.java +++ b/fe/fe-filesystem/fe-filesystem-gcs/src/main/java/org/apache/doris/filesystem/gcs/GcsFileSystemProperties.java @@ -17,6 +17,11 @@ package org.apache.doris.filesystem.gcs; +import org.apache.doris.filesystem.gcs.auth.GcpCredential; +import org.apache.doris.filesystem.gcs.auth.GcpCredentialProviderType; +import org.apache.doris.filesystem.gcs.auth.GcsAuth; +import org.apache.doris.filesystem.gcs.auth.GcsAuthResolver; +import org.apache.doris.filesystem.properties.FsCacheKeys; import org.apache.doris.filesystem.s3.AbstractDelegatingS3Properties; import org.apache.doris.foundation.property.ConnectorProperty; @@ -26,12 +31,11 @@ import java.util.Set; /** - * Provider-owned GCS properties (S3 interoperability / XML API, HMAC credentials only). + * Provider-owned GCS properties supporting HMAC, native OAuth2 and anonymous access. * *

The public aliases, ordering, defaults, and static-credential validation follow fe-core - * GCSProperties. The Google native SDK (JSON API) is intentionally NOT used: it only accepts - * OAuth2 credentials, while Doris GCS configurations carry HMAC keys. AWS-only credential - * mechanisms (role ARN, instance profile) are rejected at binding time. + * GCSProperties. Object access uses the S3-compatible XML API; native IAM Hadoop operations + * use the GCS connector. AWS-only role and instance-profile options are rejected at binding time. */ public final class GcsFileSystemProperties extends AbstractDelegatingS3Properties { @@ -105,20 +109,63 @@ public final class GcsFileSystemProperties extends AbstractDelegatingS3Propertie description = "Whether to force standard URI parsing.") private String forceParsingByStandardUrl = "false"; + private GcsAuth auth; + + @ConnectorProperty(names = {GcpCredential.CREDENTIAL_PROVIDER_TYPE}, required = false, + description = "GCP credential source: DEFAULT, COMPUTE_ENGINE or ANONYMOUS; defaults to ADC without HMAC.") + private String credentialProviderType = ""; + + @ConnectorProperty(names = {GcpCredential.IMPERSONATION_SERVICE_ACCOUNT}, required = false, + description = "Target service account email used for GCP impersonation.") + private String impersonationServiceAccount = ""; + private GcsFileSystemProperties(Map rawProperties) { super(rawProperties); } public static GcsFileSystemProperties of(Map properties) { GcsFileSystemProperties props = new GcsFileSystemProperties(properties); + props.auth = GcsAuthResolver.resolve(properties) + .orElseGet(() -> GcsAuthResolver.resolve(withGcpProvider(properties)).orElseThrow()); props.bindAndCollect(); props.validate(); return props; } + @Override + protected Map credentialProperties() { + // Unlike S3's legacy aliases, GCS never uses Glue/REST credentials for object access. + Map storage = new HashMap<>(rawProperties()); + storage.remove("glue.role_arn"); + storage.remove("glue.external_id"); + storage.remove("glue.credentials_provider_type"); + storage.remove("iceberg.rest.credentials_provider_type"); + return storage; + } + @Override public void validate() { validateHmacDialect("gs", "GCS"); + if (auth.getNativeCredential().isPresent()) { + GcpAuth.validateEndpoint(getEndpoint()); + // Connectors overlay raw Hadoop properties after this canonical configuration. + // Keep the OAuth destination on the validated gs.endpoint path. + for (String key : Set.of("fs.gs.storage.root.url", "fs.gs.storage.service.path", + "fs.gs.token.server.url")) { + if (rawProperties().containsKey(key)) { + throw new IllegalArgumentException("Native GCP authentication does not allow " + key + + "; use the native GCP credential and endpoint properties instead"); + } + } + // Raw auth overrides would make Hadoop use a different identity from object clients. + for (String key : rawProperties().keySet()) { + if (key.startsWith("fs.gs.auth.")) { + throw new IllegalArgumentException("Native GCP authentication does not allow " + key + + "; configure authentication with " + GcpCredential.CREDENTIAL_PROVIDER_TYPE + + " and " + GcpCredential.IMPERSONATION_SERVICE_ACCOUNT + " instead"); + } + } + } } @Override @@ -134,10 +181,63 @@ public Map toMap() { return Collections.unmodifiableMap(kv); } + private static Map withGcpProvider(Map properties) { + Map selected = new HashMap<>(properties); + selected.put("provider", "GCP"); + return selected; + } + + public GcsAuth getAuth() { + return auth; + } + + @Override + public Map matchedProperties() { + Map matched = new HashMap<>(super.matchedProperties()); + if (auth.getMode() != GcsAuth.Mode.HMAC) { + matched.put(GcpCredential.CREDENTIAL_PROVIDER_TYPE, auth.isAnonymous() ? "ANONYMOUS" + : auth.getNativeCredential().orElseThrow().getCredentialProviderType().name()); + matched.put(GcpCredential.IMPERSONATION_SERVICE_ACCOUNT, + auth.getNativeCredential().map(GcpCredential::getImpersonationServiceAccount).orElse("")); + } + return Collections.unmodifiableMap(matched); + } + + @Override + protected void customizeS3CompatibleKv(Map kv) { + if (auth.getNativeCredential().isPresent()) { + GcpCredential credential = auth.getNativeCredential().get(); + kv.remove("AWS_CREDENTIALS_PROVIDER_TYPE"); + kv.put(GcpCredential.CREDENTIAL_PROVIDER_TYPE, credential.getCredentialProviderType().name()); + putIfNotBlank(kv, GcpCredential.IMPERSONATION_SERVICE_ACCOUNT, + credential.getImpersonationServiceAccount()); + } + } + + @Override + public Map toHadoopConfigurationMap() { + if (auth.getNativeCredential().isEmpty()) { + return super.toHadoopConfigurationMap(); + } + GcpCredential credential = auth.getNativeCredential().get(); + Map cfg = new HashMap<>(); + cfg.put("fs.gs.impl", "com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem"); + String storageRoot = getEndpoint(); + if (!storageRoot.contains("://")) { + storageRoot = "https://" + storageRoot; + } + cfg.put("fs.gs.storage.root.url", storageRoot.endsWith("/") ? storageRoot : storageRoot + "/"); + cfg.put("fs.gs.auth.type", credential.getCredentialProviderType() == GcpCredentialProviderType.DEFAULT + ? "APPLICATION_DEFAULT" : "COMPUTE_ENGINE"); + cfg.put("fs.gs.auth.impersonation.service.account", credential.getImpersonationServiceAccount()); + FsCacheKeys.putFsCacheKeys(cfg, this); + return Collections.unmodifiableMap(cfg); + } + @Override protected void customizeHadoopConfiguration(Map cfg) { cfg.put("fs.gs.impl", "org.apache.hadoop.fs.s3a.S3AFileSystem"); - if (!hasStaticCredentials()) { + if (auth.isAnonymous()) { cfg.put("fs.s3a.aws.credentials.provider", "org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider"); } diff --git a/fe/fe-filesystem/fe-filesystem-gcs/src/main/java/org/apache/doris/filesystem/gcs/GcsFileSystemProvider.java b/fe/fe-filesystem/fe-filesystem-gcs/src/main/java/org/apache/doris/filesystem/gcs/GcsFileSystemProvider.java index d9b11f4d3445dc..31dc2ee37dcb0f 100644 --- a/fe/fe-filesystem/fe-filesystem-gcs/src/main/java/org/apache/doris/filesystem/gcs/GcsFileSystemProvider.java +++ b/fe/fe-filesystem/fe-filesystem-gcs/src/main/java/org/apache/doris/filesystem/gcs/GcsFileSystemProvider.java @@ -18,15 +18,21 @@ package org.apache.doris.filesystem.gcs; import org.apache.doris.filesystem.FileSystem; +import org.apache.doris.filesystem.auth.GcpCredentialSpec; +import org.apache.doris.filesystem.auth.NativeObjectStorageCredential; +import org.apache.doris.filesystem.auth.ObjectStorageAuthentication; +import org.apache.doris.filesystem.gcs.auth.GcpCredential; +import org.apache.doris.filesystem.gcs.auth.GcsAuthResolver; import org.apache.doris.filesystem.s3.S3CompatSignals; import org.apache.doris.filesystem.s3.S3FileSystem; import org.apache.doris.filesystem.s3.S3FileSystemProperties; -import org.apache.doris.filesystem.s3.S3ObjStorage; import org.apache.doris.filesystem.spi.FileSystemProvider; import org.apache.doris.foundation.property.ConnectorPropertiesUtils; import java.io.IOException; +import java.util.HashMap; import java.util.Map; +import java.util.Optional; import java.util.Set; /** @@ -36,7 +42,7 @@ * *

Selected explicitly by {@code provider=GCS}/{@code GCP} or {@code fs.gcs.support=true}; * otherwise, when the user declared no filesystem explicitly, by - * {@link S3CompatSignals#guessIsGcs} (a {@code gs.endpoint}, or an endpoint on + * {@link GcsAuthResolver#guessIsGcs} (a {@code gs.endpoint}, or an endpoint on * {@code storage.googleapis.com}). Delegates I/O to {@link S3FileSystem}. */ public class GcsFileSystemProvider implements FileSystemProvider { @@ -44,6 +50,103 @@ public class GcsFileSystemProvider implements FileSystemProvider RESOURCE_ALIASES = Map.ofEntries( + Map.entry("gs.endpoint", "s3.endpoint"), + Map.entry("gs.access_key", "s3.access_key"), + Map.entry("gs.secret_key", "s3.secret_key"), + Map.entry("gs.session_token", "s3.session_token"), + Map.entry("gs.connection.maximum", "s3.connection.maximum"), + Map.entry("gs.connection.request.timeout", "s3.connection.request.timeout"), + Map.entry("gs.connection.timeout", "s3.connection.timeout"), + Map.entry("gs.use_path_style", "use_path_style"), + Map.entry("gs.force_parsing_by_standard_uri", "force_parsing_by_standard_uri")); + + @Override + public Map normalizeProperties(Map properties, Map context) { + GcsAuthResolver.resolve(properties); + Map normalized = new HashMap<>(properties); + String selected = context.get("provider"); + if (selected == null || selected.isBlank()) { + if (!GcsAuthResolver.guessIsGcs(context)) { + normalized.keySet().removeAll(RESOURCE_ALIASES.keySet()); + return normalized; + } + selected = "GCP"; + normalized.put("provider", selected); + } + if ("GCP".equalsIgnoreCase(selected) || STORAGE_TYPE_GCS.equalsIgnoreCase(selected)) { + RESOURCE_ALIASES.forEach((alias, key) -> { + String value = normalized.remove(alias); + if (value != null && !value.isBlank()) { + normalized.put(key, value); + } + }); + } else { + // Inactive aliases must not become effective after a later provider-only ALTER. + normalized.keySet().removeAll(RESOURCE_ALIASES.keySet()); + } + return normalized; + } + + @Override + public Optional resolveAuthentication(Map properties) { + return GcsAuthResolver.resolve(properties).map(auth -> { + if (auth.isAnonymous()) { + return ObjectStorageAuthentication.anonymous("GCP"); + } + if (auth.getNativeCredential().isEmpty()) { + return ObjectStorageAuthentication.hmac("GCP"); + } + GcpCredential nativeCredential = auth.getNativeCredential().orElseThrow(); + GcpCredentialSpec.ProviderType type = GcpCredentialSpec.ProviderType.valueOf( + nativeCredential.getCredentialProviderType().name()); + String principal = nativeCredential.getImpersonationServiceAccount(); + GcpCredentialSpec resolved = new GcpCredentialSpec(type, principal.isEmpty() ? null : principal); + GcpCredentialSpec updates = new GcpCredentialSpec( + properties.containsKey(GcpCredential.CREDENTIAL_PROVIDER_TYPE) ? type : null, + properties.containsKey(GcpCredential.IMPERSONATION_SERVICE_ACCOUNT) ? principal : null); + return ObjectStorageAuthentication.nativeAuthentication(resolved, updates); + }); + } + + @Override + public void validateProperties(Map properties) { + GcsAuthResolver.resolve(properties); + if (GcsAuthResolver.hasNativeCredentialProperties(properties) + && S3CompatSignals.hasExplicitS3Request(properties)) { + throw new IllegalArgumentException("Native GCP authentication cannot be used with S3 properties."); + } + } + + @Override + public Set fallbackProviders() { + return Set.of("S3", "MINIO"); + } + + @Override + public Optional> credentialToProperties(NativeObjectStorageCredential credential) { + if (!(credential instanceof GcpCredentialSpec)) { + return Optional.empty(); + } + GcpCredentialSpec gcp = (GcpCredentialSpec) credential; + Map properties = new HashMap<>(); + properties.put(GcpCredential.CREDENTIAL_PROVIDER_TYPE, + gcp.getProviderType().orElse(GcpCredentialSpec.ProviderType.DEFAULT).name()); + gcp.getImpersonationServiceAccount().filter(principal -> !principal.isEmpty()) + .ifPresent(principal -> properties.put(GcpCredential.IMPERSONATION_SERVICE_ACCOUNT, principal)); + return Optional.of(properties); + } + + @Override + public Set modifiableCredentialPropertyKeys() { + return Set.of(GcpCredential.CREDENTIAL_PROVIDER_TYPE, GcpCredential.IMPERSONATION_SERVICE_ACCOUNT); + } + + @Override + public Set clearablePropertyKeys() { + return Set.of(GcpCredential.IMPERSONATION_SERVICE_ACCOUNT); + } + @Override public boolean supports(Map properties) { String providerValue = properties.get(S3CompatSignals.PROVIDER_KEY); @@ -53,7 +156,7 @@ public boolean supports(Map properties) { || S3CompatSignals.isFsSupport(properties, FS_GCS_SUPPORT)) { return true; } - return S3CompatSignals.guessAllowed(properties) && S3CompatSignals.guessIsGcs(properties); + return S3CompatSignals.guessAllowed(properties) && GcsAuthResolver.guessIsGcs(properties); } @Override @@ -63,9 +166,16 @@ public GcsFileSystemProperties bind(Map properties) { @Override public FileSystem create(GcsFileSystemProperties properties) throws IOException { - S3FileSystemProperties delegate = S3FileSystemProperties.of(properties.toS3CompatibleKv()); + Map delegateProperties = new HashMap<>(properties.toS3CompatibleKv()); + if (properties.getAuth().getNativeCredential().isPresent()) { + // The delegate only supplies URI/connection settings. GcsObjStorage supplies OAuth. + delegateProperties.remove(GcpCredential.CREDENTIAL_PROVIDER_TYPE); + delegateProperties.remove(GcpCredential.IMPERSONATION_SERVICE_ACCOUNT); + delegateProperties.put("AWS_CREDENTIALS_PROVIDER_TYPE", "ANONYMOUS"); + } + S3FileSystemProperties delegate = S3FileSystemProperties.of(delegateProperties); return new S3FileSystem(delegate, - new S3ObjStorage(delegate, properties.getSupportedSchemes())); + new GcsObjStorage(delegate, properties)); } @Override @@ -75,7 +185,7 @@ public boolean supportsExplicit(Map properties) { @Override public boolean supportsGuess(Map properties) { - return S3CompatSignals.guessIsGcs(properties); + return !S3CompatSignals.hasExplicitS3Request(properties) && GcsAuthResolver.guessIsGcs(properties); } @Override diff --git a/fe/fe-filesystem/fe-filesystem-gcs/src/main/java/org/apache/doris/filesystem/gcs/GcsObjStorage.java b/fe/fe-filesystem/fe-filesystem-gcs/src/main/java/org/apache/doris/filesystem/gcs/GcsObjStorage.java new file mode 100644 index 00000000000000..f1f9bfb38cb25b --- /dev/null +++ b/fe/fe-filesystem/fe-filesystem-gcs/src/main/java/org/apache/doris/filesystem/gcs/GcsObjStorage.java @@ -0,0 +1,58 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.filesystem.gcs; + +import org.apache.doris.filesystem.s3.S3FileSystemProperties; +import org.apache.doris.filesystem.s3.S3ObjStorage; + +import software.amazon.awssdk.services.s3.S3Client; + +import java.io.IOException; + +/** S3-compatible GCS client with refreshable Google OAuth credentials. */ +final class GcsObjStorage extends S3ObjStorage { + private final GcsFileSystemProperties properties; + + GcsObjStorage(S3FileSystemProperties delegate, GcsFileSystemProperties properties) { + super(delegate, properties.getSupportedSchemes()); + this.properties = properties; + } + + @Override + public String getPresignedUrl(String objectKey) throws IOException { + if (properties.getAuth().getNativeCredential().isPresent()) { + throw new UnsupportedOperationException("Presigned URLs are not supported for native GCP OAuth " + + "credentials; GCS V4 signing with IAM signBlob is required"); + } + return super.getPresignedUrl(objectKey); + } + + @Override + protected S3Client buildClient() throws IOException { + if (properties.getAuth().getNativeCredential().isEmpty()) { + return super.buildClient(); + } + ClassLoader previous = Thread.currentThread().getContextClassLoader(); + Thread.currentThread().setContextClassLoader(GcsObjStorage.class.getClassLoader()); + try { + return GcpAuth.createS3Client(properties); + } finally { + Thread.currentThread().setContextClassLoader(previous); + } + } +} diff --git a/fe/fe-filesystem/fe-filesystem-gcs/src/test/java/org/apache/doris/filesystem/gcs/GcpAuthTest.java b/fe/fe-filesystem/fe-filesystem-gcs/src/test/java/org/apache/doris/filesystem/gcs/GcpAuthTest.java new file mode 100644 index 00000000000000..bd93d4d0d19990 --- /dev/null +++ b/fe/fe-filesystem/fe-filesystem-gcs/src/test/java/org/apache/doris/filesystem/gcs/GcpAuthTest.java @@ -0,0 +1,131 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.filesystem.gcs; + +import org.apache.doris.filesystem.gcs.auth.GcpCredential; +import org.apache.doris.filesystem.gcs.auth.GcpCredentialProviderType; + +import com.google.auth.oauth2.AccessToken; +import com.google.auth.oauth2.ComputeEngineCredentials; +import com.google.auth.oauth2.GoogleCredentials; +import com.google.auth.oauth2.ImpersonatedCredentials; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; +import software.amazon.awssdk.core.interceptor.Context; +import software.amazon.awssdk.core.interceptor.ExecutionAttributes; +import software.amazon.awssdk.core.interceptor.ExecutionInterceptor; +import software.amazon.awssdk.http.SdkHttpMethod; +import software.amazon.awssdk.http.SdkHttpRequest; + +import java.net.URI; +import java.util.Collections; +import java.util.Date; + +public class GcpAuthTest { + @Test + public void testComputeEngineCredentialsUseMetadataProviderDirectly() throws Exception { + GoogleCredentials credentials = GcpAuth.createCredentials( + new GcpCredential(GcpCredentialProviderType.COMPUTE_ENGINE, "")); + + Assertions.assertInstanceOf(ComputeEngineCredentials.class, credentials); + } + + @Test + public void testComputeEngineImpersonationWrapsMetadataCredentials() throws Exception { + GoogleCredentials credentials = GcpAuth.createCredentials( + new GcpCredential(GcpCredentialProviderType.COMPUTE_ENGINE, + "target@my-project.iam.gserviceaccount.com")); + + Assertions.assertInstanceOf(ImpersonatedCredentials.class, credentials); + } + + @Test + public void testBearerTokenInterceptorUsesCredentialRequestMetadata() { + GoogleCredentials credentials = GoogleCredentials.create( + new AccessToken("metadata-token", new Date(System.currentTimeMillis() + 3_600_000))); + ExecutionInterceptor interceptor = GcpAuth.bearerTokenInterceptor(credentials); + Context.ModifyHttpRequest context = Mockito.mock(Context.ModifyHttpRequest.class); + SdkHttpRequest request = SdkHttpRequest.builder() + .uri(URI.create("https://storage.googleapis.com/bucket/key")) + .method(SdkHttpMethod.GET) + .build(); + Mockito.when(context.httpRequest()).thenReturn(request); + + SdkHttpRequest modified = interceptor.modifyHttpRequest(context, new ExecutionAttributes()); + + Assertions.assertEquals("Bearer metadata-token", + modified.firstMatchingHeader("Authorization").orElse(null)); + } + + @Test + public void testBearerTokenInterceptorRejectsMissingAuthorizationMetadata() throws Exception { + GoogleCredentials credentials = Mockito.mock(GoogleCredentials.class); + Mockito.when(credentials.getRequestMetadata()).thenReturn(Collections.emptyMap()); + ExecutionInterceptor interceptor = GcpAuth.bearerTokenInterceptor(credentials); + Context.ModifyHttpRequest context = Mockito.mock(Context.ModifyHttpRequest.class); + SdkHttpRequest request = SdkHttpRequest.builder() + .uri(URI.create("https://storage.googleapis.com/bucket/key")) + .method(SdkHttpMethod.GET) + .build(); + Mockito.when(context.httpRequest()).thenReturn(request); + + RuntimeException exception = Assertions.assertThrows(RuntimeException.class, + () -> interceptor.modifyHttpRequest(context, new ExecutionAttributes())); + Assertions.assertTrue(exception.getMessage().contains("Failed to obtain a GCP access token")); + } + + @Test + public void testEndpointOverridesRejectBucketHostsButResolvedRequestsAcceptThem() { + for (String endpoint : new String[] {"https://storage.googleapis.com", + "https://storage.googleapis.com:443/", "https://us-central1-storage.googleapis.com", + "https://storage.us-central1.rep.googleapis.com"}) { + Assertions.assertEquals(URI.create(endpoint), GcpAuth.validateEndpoint(endpoint)); + } + for (String endpoint : new String[] {"https://bucket.storage.googleapis.com", + "https://bucket.us-central1-storage.googleapis.com", + "https://bucket.storage.us-central1.rep.googleapis.com"}) { + Assertions.assertThrows(IllegalArgumentException.class, () -> GcpAuth.validateEndpoint(endpoint)); + // These hosts are still valid after the SDK adds the bucket to a service-base endpoint. + GoogleCredentials credentials = GoogleCredentials.create( + new AccessToken("metadata-token", new Date(System.currentTimeMillis() + 3_600_000))); + Context.ModifyHttpRequest context = Mockito.mock(Context.ModifyHttpRequest.class); + Mockito.when(context.httpRequest()).thenReturn(SdkHttpRequest.builder() + .uri(URI.create(endpoint + "/key")).method(SdkHttpMethod.GET).build()); + Assertions.assertEquals("Bearer metadata-token", GcpAuth.bearerTokenInterceptor(credentials) + .modifyHttpRequest(context, new ExecutionAttributes()) + .firstMatchingHeader("Authorization").orElse(null)); + } + } + + @Test + public void testUntrustedResolvedRequestIsRejectedBeforeFetchingToken() { + GoogleCredentials credentials = Mockito.mock(GoogleCredentials.class); + ExecutionInterceptor interceptor = GcpAuth.bearerTokenInterceptor(credentials); + Context.ModifyHttpRequest context = Mockito.mock(Context.ModifyHttpRequest.class); + for (String endpoint : new String[] {"https://attacker.example/bucket/key", + "http://storage.googleapis.com/bucket/key", "https://storage.googleapis.com.attacker.example/key", + "https://storage.googleapis.com:8443/key"}) { + Mockito.when(context.httpRequest()).thenReturn(SdkHttpRequest.builder() + .uri(URI.create(endpoint)).method(SdkHttpMethod.GET).build()); + Assertions.assertThrows(IllegalArgumentException.class, + () -> interceptor.modifyHttpRequest(context, new ExecutionAttributes())); + } + Mockito.verifyNoInteractions(credentials); + } +} diff --git a/fe/fe-filesystem/fe-filesystem-gcs/src/test/java/org/apache/doris/filesystem/gcs/GcsCredentialDescriptionTest.java b/fe/fe-filesystem/fe-filesystem-gcs/src/test/java/org/apache/doris/filesystem/gcs/GcsCredentialDescriptionTest.java new file mode 100644 index 00000000000000..8c82b703ef6418 --- /dev/null +++ b/fe/fe-filesystem/fe-filesystem-gcs/src/test/java/org/apache/doris/filesystem/gcs/GcsCredentialDescriptionTest.java @@ -0,0 +1,122 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.filesystem.gcs; + +import org.apache.doris.filesystem.auth.GcpCredentialSpec; +import org.apache.doris.filesystem.auth.NativeObjectStorageCredential; +import org.apache.doris.filesystem.auth.ObjectStorageAuthentication; +import org.apache.doris.filesystem.gcs.auth.GcsAuthResolver; +import org.apache.doris.filesystem.spi.FileSystemProvider; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.net.URL; +import java.net.URLClassLoader; +import java.util.Map; + +class GcsCredentialDescriptionTest { + private final GcsFileSystemProvider provider = new GcsFileSystemProvider(); + + @Test + void nativeIdentityRoundTripsThroughUserPropertiesWithoutWireTypes() { + for (GcpCredentialSpec.ProviderType type : GcpCredentialSpec.ProviderType.values()) { + Map properties = Map.of("gs.credential_provider_type", type.name(), + "gs.impersonation_service_account", "target@test.iam.gserviceaccount.com"); + ObjectStorageAuthentication auth = provider.resolveAuthentication(properties).orElseThrow(); + GcpCredentialSpec expected = new GcpCredentialSpec(type, "target@test.iam.gserviceaccount.com"); + Assertions.assertEquals(expected, auth.getCredential().orElseThrow()); + Assertions.assertEquals(expected, auth.getCredentialUpdates().orElseThrow()); + Assertions.assertEquals(properties, provider.credentialToProperties(expected).orElseThrow()); + } + } + + @Test + void clearingImpersonationDoesNotAddProviderTypeToAlter() { + ObjectStorageAuthentication auth = provider.resolveAuthentication(Map.of( + "provider", "GCP", "gs.impersonation_service_account", "")).orElseThrow(); + Assertions.assertEquals(new GcpCredentialSpec(GcpCredentialSpec.ProviderType.DEFAULT, null), + auth.getCredential().orElseThrow()); + Assertions.assertEquals(new GcpCredentialSpec(null, ""), auth.getCredentialUpdates().orElseThrow()); + } + + @Test + void onlyGcpDescriptionsAreRestoredAndAbsentSourceUsesLegacyDefault() { + Assertions.assertEquals(Map.of("gs.credential_provider_type", "DEFAULT"), + provider.credentialToProperties(new GcpCredentialSpec(null, null)).orElseThrow()); + NativeObjectStorageCredential foreign = new NativeObjectStorageCredential() { + @Override + public String getProvider() { + return "OTHER"; + } + + @Override + public boolean isEmpty() { + return false; + } + }; + Assertions.assertTrue(provider.credentialToProperties(foreign).isEmpty()); + } + + @Test + void pluginCrossesTheSharedApiWithoutAccessToHostWireClasses() throws Exception { + URL[] urls = {GcsFileSystemProvider.class.getProtectionDomain().getCodeSource().getLocation(), + GcsAuthResolver.class.getProtectionDomain().getCodeSource().getLocation()}; + try (URLClassLoader loader = new URLClassLoader(urls, getClass().getClassLoader()) { + @Override + protected synchronized Class loadClass(String name, boolean resolve) throws ClassNotFoundException { + if (name.startsWith("org.apache.doris.thrift.") || name.startsWith("org.apache.doris.cloud.proto.") + || name.startsWith("org.apache.doris.storage.protocol.")) { + throw new ClassNotFoundException("Protocol classes are unavailable to this plugin: " + name); + } + if (name.startsWith("org.apache.doris.filesystem.gcs.")) { + Class loaded = findLoadedClass(name); + if (loaded == null) { + loaded = findClass(name); + } + if (resolve) { + resolveClass(loaded); + } + return loaded; + } + return super.loadClass(name, resolve); + } + }) { + FileSystemProvider isolated = (FileSystemProvider) loader.loadClass( + GcsFileSystemProvider.class.getName()).getConstructor().newInstance(); + Assertions.assertNotSame(GcsFileSystemProvider.class, isolated.getClass()); + NativeObjectStorageCredential credential = isolated.resolveAuthentication( + Map.of("provider", "GCP")).orElseThrow().getCredential().orElseThrow(); + Assertions.assertEquals(GcpCredentialSpec.class, credential.getClass()); + Assertions.assertEquals(Map.of("gs.credential_provider_type", "DEFAULT"), + isolated.credentialToProperties(credential).orElseThrow()); + } + } + + @Test + void hmacAndAnonymousAuthenticationDoNotProduceNativeCredentials() { + for (Map properties : java.util.List.of( + Map.of("provider", "GCP", "s3.access_key", "ak", "s3.secret_key", "sk"), + Map.of("provider", "GCP", "gs.credential_provider_type", "ANONYMOUS"))) { + ObjectStorageAuthentication auth = provider.resolveAuthentication(properties).orElseThrow(); + Assertions.assertFalse(auth.isNative()); + Assertions.assertTrue(auth.getCredential().isEmpty()); + Assertions.assertTrue(auth.getCredentialUpdates().isEmpty()); + } + } +} diff --git a/fe/fe-filesystem/fe-filesystem-gcs/src/test/java/org/apache/doris/filesystem/gcs/GcsFileSystemPropertiesTest.java b/fe/fe-filesystem/fe-filesystem-gcs/src/test/java/org/apache/doris/filesystem/gcs/GcsFileSystemPropertiesTest.java index 2802018a50d26f..9b5d92b01a56ec 100644 --- a/fe/fe-filesystem/fe-filesystem-gcs/src/test/java/org/apache/doris/filesystem/gcs/GcsFileSystemPropertiesTest.java +++ b/fe/fe-filesystem/fe-filesystem-gcs/src/test/java/org/apache/doris/filesystem/gcs/GcsFileSystemPropertiesTest.java @@ -22,6 +22,7 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import java.util.HashMap; import java.util.List; import java.util.Map; @@ -90,11 +91,12 @@ void toS3CompatibleKv_delegatesHmacToCanonicalAwsKeys() { } @Test - void toS3CompatibleKv_fallsBackToAnonymousWithoutStaticCredentials() { + void toS3CompatibleKv_defaultsToAdcWithoutStaticCredentials() { Map kv = GcsFileSystemProperties.of(Map.of( "gs.endpoint", "https://storage.googleapis.com")).toS3CompatibleKv(); - Assertions.assertEquals("ANONYMOUS", kv.get("AWS_CREDENTIALS_PROVIDER_TYPE")); + Assertions.assertEquals("DEFAULT", kv.get("gs.credential_provider_type")); + Assertions.assertFalse(kv.containsKey("AWS_CREDENTIALS_PROVIDER_TYPE")); } @Test @@ -107,13 +109,41 @@ void toMap_tagsGcpProviderForBackendParity() { } @Test - void toHadoopConfigurationMap_anonymousWhenNoStaticCredentials() { + void toHadoopConfigurationMap_usesNativeAdcWithoutStaticCredentials() { Map cfg = GcsFileSystemProperties.of(Map.of( "gs.endpoint", "https://storage.googleapis.com")).toHadoopConfigurationMap(); - Assertions.assertEquals("org.apache.hadoop.fs.s3a.S3AFileSystem", cfg.get("fs.gs.impl")); - Assertions.assertEquals("org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider", - cfg.get("fs.s3a.aws.credentials.provider")); + Assertions.assertEquals("com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem", cfg.get("fs.gs.impl")); + Assertions.assertEquals("APPLICATION_DEFAULT", cfg.get("fs.gs.auth.type")); + Assertions.assertFalse(cfg.containsKey("fs.s3a.aws.credentials.provider")); + } + + @Test + void nativeHadoopUsesConfiguredEndpoint() { + for (String endpoint : new String[] {"storage.googleapis.com", "https://storage.googleapis.com/", + "https://storage.us-central1.rep.googleapis.com", "https://us-central1-storage.googleapis.com"}) { + GcsFileSystemProperties props = GcsFileSystemProperties.of(Map.of( + "gs.endpoint", endpoint, "gs.credential_provider_type", "COMPUTE_ENGINE")); + String expected = endpoint.contains("://") ? endpoint : "https://" + endpoint; + Assertions.assertEquals(expected.endsWith("/") ? expected : expected + "/", + props.toHadoopConfigurationMap().get("fs.gs.storage.root.url")); + } + } + + @Test + void nativeOauthRejectsUntrustedEndpointsButPreservesHmacAndAnonymous() { + for (String endpoint : new String[] {"https://attacker.example", "http://storage.googleapis.com", + "https://storage.googleapis.com.attacker.example", "https://storage.googleapis.com@attacker.example", + "https://storage.googleapis.com:8443", "https://bucket.storage.googleapis.com", + "https://bucket.us-central1-storage.googleapis.com", + "https://bucket.storage.us-central1.rep.googleapis.com"}) { + Assertions.assertThrows(IllegalArgumentException.class, + () -> GcsFileSystemProperties.of(Map.of("gs.endpoint", endpoint))); + Assertions.assertDoesNotThrow(() -> GcsFileSystemProperties.of(Map.of( + "gs.endpoint", endpoint, "gs.access_key", "ak", "gs.secret_key", "sk"))); + Assertions.assertDoesNotThrow(() -> GcsFileSystemProperties.of(Map.of( + "gs.endpoint", endpoint, "gs.credential_provider_type", "ANONYMOUS"))); + } } @Test @@ -166,7 +196,7 @@ void of_rejectsInvalidUsePathStyle() { void of_rejectsSessionTokenWithoutStaticCredentials() { IllegalArgumentException e = Assertions.assertThrows(IllegalArgumentException.class, () -> GcsFileSystemProperties.of(Map.of("gs.session_token", "token"))); - Assertions.assertTrue(e.getMessage().contains("gs.session_token"), e.getMessage()); + Assertions.assertTrue(e.getMessage().contains("session token"), e.getMessage()); } @Test @@ -187,6 +217,48 @@ void validate_rejectsAwsOnlyCredentialOptions() { Assertions.assertTrue(e.getMessage().contains("HMAC"), e.getMessage()); } + @Test + void metadataCredentialsDoNotSelectOrInvalidateGcsAuthentication() { + Map metadata = Map.of( + "glue.role_arn", "arn:aws:iam::1:role/metadata", + "glue.external_id", "metadata-external-id", + "glue.credentials_provider_type", "INSTANCE_PROFILE", + "iceberg.rest.credentials_provider_type", "SYSTEM_PROPERTIES"); + for (String mode : List.of("DEFAULT", "COMPUTE_ENGINE", "ANONYMOUS", "HMAC")) { + Map raw = new HashMap<>(metadata); + raw.put("provider", "GCP"); + if ("HMAC".equals(mode)) { + raw.put("gs.access_key", "gcs-ak"); + raw.put("gs.secret_key", "gcs-sk"); + } else { + raw.put("gs.credential_provider_type", mode); + } + GcsFileSystemProperties properties = GcsFileSystemProperties.of(raw); + Map backend = properties.toMap(); + Assertions.assertEquals(raw, properties.rawProperties()); + if ("ANONYMOUS".equals(mode)) { + Assertions.assertEquals("ANONYMOUS", backend.get("AWS_CREDENTIALS_PROVIDER_TYPE")); + } else { + Assertions.assertFalse(backend.containsKey("AWS_CREDENTIALS_PROVIDER_TYPE")); + } + Assertions.assertEquals("", properties.getRoleArn()); + Assertions.assertEquals("", properties.getExternalId()); + for (String key : metadata.keySet()) { + Assertions.assertFalse(backend.containsKey(key), key); + } + // Actual AWS storage credentials must still conflict with GCS authentication. + for (Map.Entry credential : Map.of( + "s3.role_arn", "arn:aws:iam::1:role/storage", + "sts.role_arn", "arn:aws:iam::1:role/storage", + "s3.credentials_provider_type", "INSTANCE_PROFILE").entrySet()) { + Map conflicting = new HashMap<>(raw); + conflicting.put(credential.getKey(), credential.getValue()); + Assertions.assertThrows(IllegalArgumentException.class, + () -> GcsFileSystemProperties.of(conflicting), mode + ": " + credential.getKey()); + } + } + } + @Test void toString_masksSecretsButNotAccessKey() { String rendered = GcsFileSystemProperties.of(Map.of( @@ -220,4 +292,112 @@ void region_isFixedAndNotUserConfigurable() { "gs.region", "europe-west1", "AWS_REGION", "europe-west1")).getRegion()); } + + @Test + void nativeModesPropagateToBackendHadoopAndCache() { + for (String mode : new String[] {"DEFAULT", "COMPUTE_ENGINE"}) { + for (String account : new String[] {"", "target@test.iam.gserviceaccount.com"}) { + GcsFileSystemProperties props = GcsFileSystemProperties.of(Map.of( + "provider", "GCP", "gs.credential_provider_type", mode, + "gs.impersonation_service_account", account)); + Assertions.assertEquals(mode, props.toMap().get("gs.credential_provider_type")); + Assertions.assertEquals(account, props.toMap().getOrDefault("gs.impersonation_service_account", "")); + Assertions.assertFalse(props.toMap().containsKey("AWS_CREDENTIALS_PROVIDER_TYPE")); + Assertions.assertEquals(mode.equals("DEFAULT") ? "APPLICATION_DEFAULT" : mode, + props.toHadoopConfigurationMap().get("fs.gs.auth.type")); + Assertions.assertEquals(account, + props.toHadoopConfigurationMap().get("fs.gs.auth.impersonation.service.account")); + Assertions.assertEquals(mode, props.matchedProperties().get("gs.credential_provider_type")); + Assertions.assertEquals(account, props.matchedProperties().get("gs.impersonation_service_account")); + } + } + } + + @Test + void nativeHadoopEndpointCannotBeOverriddenByRawProperties() { + for (String mode : new String[] {"DEFAULT", "COMPUTE_ENGINE"}) { + for (String key : new String[] {"fs.gs.storage.root.url", "fs.gs.storage.service.path", + "fs.gs.token.server.url"}) { + IllegalArgumentException error = Assertions.assertThrows(IllegalArgumentException.class, + () -> GcsFileSystemProperties.of(Map.of("provider", "GCP", + "gs.credential_provider_type", mode, key, "https://untrusted.example/"))); + Assertions.assertTrue(error.getMessage().contains(key)); + } + } + GcsFileSystemProperties nativeProps = GcsFileSystemProperties.of(Map.of("provider", "GCP", + "gs.endpoint", "https://us-central1-storage.googleapis.com")); + Assertions.assertEquals("https://us-central1-storage.googleapis.com/", + nativeProps.toHadoopConfigurationMap().get("fs.gs.storage.root.url")); + // The restriction applies to native OAuth; preserve legacy HMAC Hadoop configuration. + Assertions.assertDoesNotThrow(() -> GcsFileSystemProperties.of(Map.of("provider", "GCP", + "gs.access_key", "ak", "gs.secret_key", "sk", "fs.gs.storage.root.url", "https://custom.example/"))); + } + + @Test + void nativeHadoopAuthCannotBeOverriddenByRawProperties() { + List> nativeModes = List.of( + Map.of("provider", "GCP"), + Map.of("provider", "GCP", "gs.credential_provider_type", "DEFAULT"), + Map.of("provider", "GCP", "gs.credential_provider_type", "COMPUTE_ENGINE")); + Map overrides = Map.of( + "fs.gs.auth.type", "UNAUTHENTICATED", + "fs.gs.auth.impersonation.service.account", "other@test.iam.gserviceaccount.com", + "fs.gs.auth.service.account.json.keyfile", "/tmp/other-service-account.json"); + for (Map mode : nativeModes) { + for (String account : new String[] {"", "target@test.iam.gserviceaccount.com"}) { + overrides.forEach((key, value) -> { + Map properties = new HashMap<>(mode); + if (!account.isEmpty()) { + properties.put("gs.impersonation_service_account", account); + } + properties.put(key, value); + IllegalArgumentException error = Assertions.assertThrows(IllegalArgumentException.class, + () -> GcsFileSystemProperties.of(properties)); + Assertions.assertTrue(error.getMessage().contains(key)); + Assertions.assertTrue(error.getMessage().contains("gs.credential_provider_type")); + Assertions.assertTrue(error.getMessage().contains("gs.impersonation_service_account")); + }); + } + } + // An empty raw value must not clear the selected impersonation identity either. + Assertions.assertThrows(IllegalArgumentException.class, () -> GcsFileSystemProperties.of(Map.of( + "provider", "GCP", "gs.impersonation_service_account", "target@test.iam.gserviceaccount.com", + "fs.gs.auth.impersonation.service.account", ""))); + } + + @Test + void nativeHadoopAuthValidationPreservesOtherModesAndUnrelatedProperties() { + Assertions.assertDoesNotThrow(() -> GcsFileSystemProperties.of(Map.of( + "provider", "GCP", "gs.access_key", "ak", "gs.secret_key", "sk", + "fs.gs.auth.type", "UNAUTHENTICATED"))); + Assertions.assertDoesNotThrow(() -> GcsFileSystemProperties.of(Map.of( + "provider", "GCP", "gs.credential_provider_type", "ANONYMOUS", + "fs.gs.auth.type", "UNAUTHENTICATED"))); + GcsFileSystemProperties props = GcsFileSystemProperties.of(Map.of( + "provider", "GCP", "gs.credential_provider_type", "COMPUTE_ENGINE", + "gs.impersonation_service_account", "target@test.iam.gserviceaccount.com", + "fs.gs.http.max.retry", "3")); + Assertions.assertEquals("COMPUTE_ENGINE", props.toHadoopConfigurationMap().get("fs.gs.auth.type")); + Assertions.assertEquals("target@test.iam.gserviceaccount.com", + props.toHadoopConfigurationMap().get("fs.gs.auth.impersonation.service.account")); + } + + @Test + void explicitAnonymousUsesUnsignedBackendAndHadoop() { + GcsFileSystemProperties props = GcsFileSystemProperties.of(Map.of( + "provider", "GCP", "gs.credential_provider_type", "ANONYMOUS")); + Assertions.assertEquals("ANONYMOUS", props.toMap().get("AWS_CREDENTIALS_PROVIDER_TYPE")); + Assertions.assertFalse(props.toMap().containsKey("gs.credential_provider_type")); + Assertions.assertEquals("org.apache.hadoop.fs.s3a.AnonymousAWSCredentialsProvider", + props.toHadoopConfigurationMap().get("fs.s3a.aws.credentials.provider")); + } + + @Test + void inferredAdcAndExplicitAdcHaveSameAuthenticationCacheIdentity() { + GcsFileSystemProperties implicit = GcsFileSystemProperties.of(Map.of("provider", "GCP")); + GcsFileSystemProperties explicit = GcsFileSystemProperties.of(Map.of( + "provider", "GCP", "gs.credential_provider_type", "DEFAULT")); + Assertions.assertEquals(implicit.matchedProperties(), explicit.matchedProperties()); + Assertions.assertEquals(implicit.toMap(), explicit.toMap()); + } } diff --git a/fe/fe-filesystem/fe-filesystem-gcs/src/test/java/org/apache/doris/filesystem/gcs/GcsFileSystemProviderTest.java b/fe/fe-filesystem/fe-filesystem-gcs/src/test/java/org/apache/doris/filesystem/gcs/GcsFileSystemProviderTest.java index ce8f2049423ce9..809402292afebe 100644 --- a/fe/fe-filesystem/fe-filesystem-gcs/src/test/java/org/apache/doris/filesystem/gcs/GcsFileSystemProviderTest.java +++ b/fe/fe-filesystem/fe-filesystem-gcs/src/test/java/org/apache/doris/filesystem/gcs/GcsFileSystemProviderTest.java @@ -18,6 +18,8 @@ package org.apache.doris.filesystem.gcs; import org.apache.doris.filesystem.FileSystem; +import org.apache.doris.filesystem.auth.GcpCredentialSpec; +import org.apache.doris.filesystem.auth.ObjectStorageAuthentication; import org.apache.doris.filesystem.s3.S3FileSystem; import org.apache.doris.filesystem.s3.S3ObjStorage; @@ -184,8 +186,115 @@ void sensitivePropertyKeys_coverSecretAliasesButNotAccessKey() { Assertions.assertFalse(keys.contains("gs.access_key"), keys.toString()); } + @Test + void validatesOwnedCredentialsEvenWhenS3IsSelected() { + for (Map selection : java.util.List.of(Map.of("provider", "S3"), + Map.of("fs.s3.support", "true"))) { + Map props = new HashMap<>(selection); + props.put("gs.credential_provider_type", "DEFAULT"); + Assertions.assertThrows(IllegalArgumentException.class, () -> provider.validateProperties(props)); + } + Assertions.assertDoesNotThrow(() -> provider.validateProperties(Map.of("provider", "S3"))); + Assertions.assertDoesNotThrow(() -> provider.validateProperties(Map.of("provider", "GCP"))); + Assertions.assertEquals(Set.of("S3", "MINIO"), provider.fallbackProviders()); + Assertions.assertTrue(provider.supportsGuess(Map.of("gs.impersonation_service_account", ""))); + } + @Test void name_isGcs() { Assertions.assertEquals("GCS", provider.name()); } + + @Test + void nativeOAuthRejectsAwsPresigningBeforeCredentialLoading() { + GcsFileSystemProperties props = GcsFileSystemProperties.of(Map.of( + "provider", "GCP", "s3.bucket", "bucket")); + GcsObjStorage storage = new GcsObjStorage( + org.apache.doris.filesystem.s3.S3FileSystemProperties.of(Map.of( + "s3.endpoint", "storage.googleapis.com", "s3.region", "us-east1", + "AWS_CREDENTIALS_PROVIDER_TYPE", "ANONYMOUS")), props); + Assertions.assertThrows(UnsupportedOperationException.class, () -> storage.getPresignedUrl("key")); + } + + @Test + void supportsGuess_respectsExplicitS3AndRecognizesRegionalEndpoints() { + Map props = new HashMap<>(); + for (String endpoint : new String[] {"https://storage.googleapis.com:443/", + "https://storage.us-east1.rep.googleapis.com"}) { + props.put("s3.endpoint", endpoint); + Assertions.assertTrue(provider.supportsGuess(props)); + props.put("provider", "S3"); + Assertions.assertFalse(provider.supportsGuess(props)); + props.remove("provider"); + } + } + + @Test + void normalizePropertiesUsesEffectiveProviderForPartialAlter() { + Map patch = Map.of("gs.endpoint", "storage.googleapis.com", + "gs.connection.maximum", "25", "gs.impersonation_service_account", ""); + for (String selected : new String[] {"GCP", "gcp", "GCS", "gcs"}) { + Map stored = Map.of("provider", selected, "s3.endpoint", "old.endpoint"); + Map normalized = provider.normalizeProperties(patch, stored); + Assertions.assertEquals(Map.of("s3.endpoint", "storage.googleapis.com", "s3.connection.maximum", "25", + "gs.impersonation_service_account", ""), normalized); + Assertions.assertEquals("old.endpoint", stored.get("s3.endpoint")); + } + Assertions.assertTrue(patch.containsKey("gs.endpoint")); + Assertions.assertTrue(provider.clearablePropertyKeys().contains("gs.impersonation_service_account")); + Assertions.assertTrue(provider.modifiableCredentialPropertyKeys().contains("gs.credential_provider_type")); + Assertions.assertFalse(provider.modifiableCredentialPropertyKeys().contains("s3.endpoint")); + } + + @Test + void normalizePropertiesPreservesSelectionAndValidatesRawAliases() { + Map inferred = Map.of("gs.endpoint", "storage.googleapis.com"); + Assertions.assertEquals(Map.of("provider", "GCP", "s3.endpoint", "storage.googleapis.com"), + provider.normalizeProperties(inferred, inferred)); + Map s3 = Map.of("provider", "S3", "s3.endpoint", "storage.googleapis.com"); + Assertions.assertEquals(s3, provider.normalizeProperties(s3, s3)); + Assertions.assertTrue(provider.resolveAuthentication(s3).isEmpty()); + Map conflict = Map.of("gs.credential_provider_type", "DEFAULT", + "gs.access_key", "ak", "gs.secret_key", "sk"); + Assertions.assertThrows(IllegalArgumentException.class, + () -> provider.normalizeProperties(conflict, conflict)); + } + + @Test + void nativeAuthenticationSeparatesCreateDefaultsFromExplicitAlterFields() { + ObjectStorageAuthentication defaults = provider.resolveAuthentication(Map.of("provider", "GCP")).get(); + Assertions.assertEquals("GCP", defaults.getProvider()); + Assertions.assertTrue(defaults.isNative()); + Assertions.assertFalse(defaults.isAnonymous()); + Assertions.assertEquals(new GcpCredentialSpec(GcpCredentialSpec.ProviderType.DEFAULT, null), + defaults.getCredential().orElseThrow()); + Assertions.assertTrue(defaults.getCredentialUpdates().isEmpty()); + + ObjectStorageAuthentication clear = provider.resolveAuthentication( + Map.of("gs.impersonation_service_account", "")).get(); + Assertions.assertEquals(defaults.getCredential(), clear.getCredential()); + Assertions.assertEquals(new GcpCredentialSpec(null, ""), clear.getCredentialUpdates().orElseThrow()); + + Map properties = Map.of("gs.credential_provider_type", "COMPUTE_ENGINE", + "gs.impersonation_service_account", "reader@project.iam.gserviceaccount.com"); + ObjectStorageAuthentication explicit = provider.resolveAuthentication(properties).get(); + Assertions.assertEquals(explicit.getCredential(), explicit.getCredentialUpdates()); + Assertions.assertEquals(properties, + provider.credentialToProperties(explicit.getCredential().orElseThrow()).orElseThrow()); + } + + @Test + void hmacAndAnonymousDoNotPublishNativeWireCredentials() { + ObjectStorageAuthentication hmac = provider.resolveAuthentication(Map.of( + "provider", "GCP", "gs.access_key", "ak", "gs.secret_key", "sk")).get(); + Assertions.assertFalse(hmac.isNative()); + Assertions.assertFalse(hmac.isAnonymous()); + Assertions.assertTrue(hmac.getCredentialUpdates().isEmpty()); + ObjectStorageAuthentication anonymous = provider.resolveAuthentication(Map.of( + "provider", "GCP", "gs.credential_provider_type", "ANONYMOUS")).get(); + Assertions.assertTrue(anonymous.isAnonymous()); + Assertions.assertFalse(anonymous.isNative()); + Assertions.assertTrue(anonymous.getCredentialUpdates().isEmpty()); + } + } diff --git a/fe/fe-filesystem/fe-filesystem-minio/src/main/java/org/apache/doris/filesystem/minio/MinioFileSystemProvider.java b/fe/fe-filesystem/fe-filesystem-minio/src/main/java/org/apache/doris/filesystem/minio/MinioFileSystemProvider.java index f08e741dd39b2f..92e16d6b4d5c5d 100644 --- a/fe/fe-filesystem/fe-filesystem-minio/src/main/java/org/apache/doris/filesystem/minio/MinioFileSystemProvider.java +++ b/fe/fe-filesystem/fe-filesystem-minio/src/main/java/org/apache/doris/filesystem/minio/MinioFileSystemProvider.java @@ -89,6 +89,10 @@ public boolean supportsExplicit(Map properties) { @Override public boolean supportsGuess(Map properties) { + // Match the raw supports() path: an explicit S3 request must not also bind MinIO. + if (S3CompatSignals.hasExplicitS3Request(properties)) { + return false; + } // Verbatim port of fe-core MinioProperties.guessIsMe: MinIO is the "any other // S3-compatible" fallback — it claims the map iff none of Azure/COS/OSS/S3 would claim // it AND at least one identifier key is present. The sibling guesses are replicated diff --git a/fe/fe-filesystem/fe-filesystem-minio/src/test/java/org/apache/doris/filesystem/minio/MinioFileSystemProviderTest.java b/fe/fe-filesystem/fe-filesystem-minio/src/test/java/org/apache/doris/filesystem/minio/MinioFileSystemProviderTest.java index 1058dedc7fab0b..33c7405936eacf 100644 --- a/fe/fe-filesystem/fe-filesystem-minio/src/test/java/org/apache/doris/filesystem/minio/MinioFileSystemProviderTest.java +++ b/fe/fe-filesystem/fe-filesystem-minio/src/test/java/org/apache/doris/filesystem/minio/MinioFileSystemProviderTest.java @@ -108,6 +108,19 @@ void supports_rejectsForeignDialects() { Assertions.assertFalse(provider.supports(plainS3)); } + @Test + void supportsGuess_doesNotAlsoBindExplicitS3() { + for (String endpoint : new String[] {"https://storage.us-east1.rep.googleapis.com", + "http://minio.local:9000"}) { + Map props = new HashMap<>(); + props.put("s3.endpoint", endpoint); + Assertions.assertTrue(provider.supportsGuess(props), endpoint); + props.put("provider", "S3"); + Assertions.assertFalse(provider.supportsGuess(props), endpoint); + Assertions.assertFalse(provider.supports(props), endpoint); + } + } + @Test void supportsGuess_yieldsToAzureSovereignCloudHostSuffixes() { // The Azure claim is host-suffix based (all sovereign clouds, blob AND dfs), not a diff --git a/fe/fe-filesystem/fe-filesystem-s3-base/src/main/java/org/apache/doris/filesystem/s3/AbstractDelegatingS3Properties.java b/fe/fe-filesystem/fe-filesystem-s3-base/src/main/java/org/apache/doris/filesystem/s3/AbstractDelegatingS3Properties.java index 3e502a2828d008..a95fc0bd45be4d 100644 --- a/fe/fe-filesystem/fe-filesystem-s3-base/src/main/java/org/apache/doris/filesystem/s3/AbstractDelegatingS3Properties.java +++ b/fe/fe-filesystem/fe-filesystem-s3-base/src/main/java/org/apache/doris/filesystem/s3/AbstractDelegatingS3Properties.java @@ -106,12 +106,17 @@ protected final void validateHmacDialect(String aliasPrefix, String displayName) "Property " + aliasPrefix + ".endpoint is required.") .check(this::hasInvalidUsePathStyle, "use_path_style must be true or false, got: '" + getUsePathStyle() + "'") - .check(() -> S3CompatSignals.hasAwsOnlyCredentialOptions(rawProperties), + .check(() -> S3CompatSignals.hasAwsOnlyCredentialOptions(credentialProperties()), displayName + " supports only HMAC access_key/secret_key credentials; " + "role_arn/instance-profile style options are not supported") .validate("Invalid " + displayName + " filesystem properties"); } + /** Raw credential aliases belonging to this storage, excluding independent metadata clients. */ + protected Map credentialProperties() { + return rawProperties; + } + @Override public StorageKind kind() { return StorageKind.OBJECT_STORAGE; @@ -175,7 +180,7 @@ public final Map toS3CompatibleKv() { // requested mode (validation restricts it to DEFAULT/ANONYMOUS): a user asking for DEFAULT // keeps the credential chain instead of being silently downgraded to unsigned ANONYMOUS. if (!hasStaticCredentials()) { - String requested = S3CompatSignals.requestedCredentialsProviderType(rawProperties); + String requested = S3CompatSignals.requestedCredentialsProviderType(credentialProperties()); kv.put("AWS_CREDENTIALS_PROVIDER_TYPE", requested != null ? requested : "ANONYMOUS"); } customizeS3CompatibleKv(kv); diff --git a/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3FileSystemProvider.java b/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3FileSystemProvider.java index a4f95a85fd0a66..38e434945394ef 100644 --- a/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3FileSystemProvider.java +++ b/fe/fe-filesystem/fe-filesystem-s3/src/main/java/org/apache/doris/filesystem/s3/S3FileSystemProvider.java @@ -115,6 +115,11 @@ public boolean supportsExplicit(Map properties) { @Override public boolean supportsGuess(Map properties) { + // Keep an explicit S3 request on the S3 credential path even at a GCS/MinIO endpoint. + // This is local to the S3 family; other providers retain their existing match order. + if (S3CompatSignals.hasExplicitS3Request(properties)) { + return true; + } // Port of fe-core S3Properties.guessIsMe on raw props. String endpoint = null; for (String name : GUESS_ENDPOINT_NAMES) { diff --git a/fe/fe-filesystem/fe-filesystem-s3/src/test/java/org/apache/doris/filesystem/s3/S3FileSystemProviderTest.java b/fe/fe-filesystem/fe-filesystem-s3/src/test/java/org/apache/doris/filesystem/s3/S3FileSystemProviderTest.java index c81f119386bc97..1d4e771a777009 100644 --- a/fe/fe-filesystem/fe-filesystem-s3/src/test/java/org/apache/doris/filesystem/s3/S3FileSystemProviderTest.java +++ b/fe/fe-filesystem/fe-filesystem-s3/src/test/java/org/apache/doris/filesystem/s3/S3FileSystemProviderTest.java @@ -127,6 +127,18 @@ void supports_yieldsToExplicitlyRequestedDedicatedDialects() { } } + @Test + void supportsGuess_keepsExplicitS3AtGcsAndMinioEndpoints() { + for (String endpoint : new String[] {"https://storage.googleapis.com", + "https://storage.us-east1.rep.googleapis.com", "http://minio.local:9000"}) { + for (String name : new String[] {"S3", "s3"}) { + Map props = Map.of("provider", name, "s3.endpoint", endpoint); + Assertions.assertTrue(provider.supportsGuess(props), endpoint); + Assertions.assertTrue(provider.supports(props), endpoint); + } + } + } + @Test void supports_keepsGoogleapisMapWhenUserExplicitlyRequestsS3() { // An explicit fs.s3.support=true / provider=S3 overrides the dialect guess: the user has diff --git a/fe/fe-filesystem/fe-filesystem-spi/src/main/java/org/apache/doris/filesystem/spi/FileSystemProvider.java b/fe/fe-filesystem/fe-filesystem-spi/src/main/java/org/apache/doris/filesystem/spi/FileSystemProvider.java index 7e4859dbb147a5..b4be3bfb1f6e57 100644 --- a/fe/fe-filesystem/fe-filesystem-spi/src/main/java/org/apache/doris/filesystem/spi/FileSystemProvider.java +++ b/fe/fe-filesystem/fe-filesystem-spi/src/main/java/org/apache/doris/filesystem/spi/FileSystemProvider.java @@ -20,13 +20,17 @@ import org.apache.doris.extension.spi.Plugin; import org.apache.doris.extension.spi.PluginFactory; import org.apache.doris.filesystem.FileSystem; +import org.apache.doris.filesystem.auth.NativeObjectStorageCredential; +import org.apache.doris.filesystem.auth.ObjectStorageAuthentication; import org.apache.doris.filesystem.properties.FileSystemCapability; import org.apache.doris.filesystem.properties.FileSystemProperties; import java.io.IOException; import java.util.Collections; import java.util.EnumSet; +import java.util.HashMap; import java.util.Map; +import java.util.Optional; import java.util.Set; /** @@ -44,6 +48,55 @@ */ public interface FileSystemProvider

extends PluginFactory { + /** + * Normalize provider-owned aliases without binding clients or requiring a complete property map. + * Context is the effective CREATE/ALTER selection; properties may be a partial ALTER patch. + * Implementations return a new map and preserve explicit credential-clearing values. + */ + default Map normalizeProperties(Map properties, Map context) { + return new HashMap<>(properties); + } + + /** Resolve authentication without I/O, including validation of conflicting raw aliases. */ + default Optional resolveAuthentication(Map properties) { + return Optional.empty(); + } + + /** + * Validate provider-owned properties before routing, including combinations that would otherwise + * select another provider and silently ignore these properties. Must be deterministic and do no I/O. + * Called for every loaded provider, even when it does not match the map. + */ + default void validateProperties(Map properties) { + } + + /** + * Names of general-purpose providers whose heuristic matches this provider supersedes when it + * also matches. Explicit selections are never suppressed. Names are case-insensitive; the + * relationship must be acyclic. Empty by default to preserve existing multi-provider bindings. + */ + default Set fallbackProviders() { + return Collections.emptySet(); + } + + /** + * Restore provider-owned property names from a typed identity. Empty means this plugin does not + * handle the credential type. Neither user property names nor wire formats are interpreted by core. + */ + default Optional> credentialToProperties(NativeObjectStorageCredential credential) { + return Optional.empty(); + } + + /** Provider-owned credential properties accepted by a storage vault ALTER. */ + default Set modifiableCredentialPropertyKeys() { + return Collections.emptySet(); + } + + /** Properties whose explicit empty ALTER value clears the persisted setting. */ + default Set clearablePropertyKeys() { + return Collections.emptySet(); + } + /** * Returns true if this provider can handle the given properties. * Must be cheap (no network calls) and deterministic. diff --git a/fe/fe-filesystem/fe-filesystem-spi/src/test/java/org/apache/doris/filesystem/spi/FileSystemPluginSurfaceTest.java b/fe/fe-filesystem/fe-filesystem-spi/src/test/java/org/apache/doris/filesystem/spi/FileSystemPluginSurfaceTest.java index 16c91831c14485..ba8f429846c27d 100644 --- a/fe/fe-filesystem/fe-filesystem-spi/src/test/java/org/apache/doris/filesystem/spi/FileSystemPluginSurfaceTest.java +++ b/fe/fe-filesystem/fe-filesystem-spi/src/test/java/org/apache/doris/filesystem/spi/FileSystemPluginSurfaceTest.java @@ -68,6 +68,10 @@ public class FileSystemPluginSurfaceTest { /** The types a filesystem plugin implements or calls. Everything reachable on them is the contract. */ private static final List> FROZEN_TYPES = Arrays.asList( FileSystemProvider.class, + org.apache.doris.filesystem.auth.NativeObjectStorageCredential.class, + org.apache.doris.filesystem.auth.GcpCredentialSpec.class, + org.apache.doris.filesystem.auth.GcpCredentialSpec.ProviderType.class, + org.apache.doris.filesystem.auth.ObjectStorageAuthentication.class, ObjFileSystem.class, ObjStorage.class, org.apache.doris.extension.spi.Plugin.class, diff --git a/fe/fe-filesystem/fe-filesystem-spi/src/test/java/org/apache/doris/filesystem/spi/FileSystemProviderTypedPropertiesTest.java b/fe/fe-filesystem/fe-filesystem-spi/src/test/java/org/apache/doris/filesystem/spi/FileSystemProviderTypedPropertiesTest.java index 4d614f22bd3396..a63fc103000012 100644 --- a/fe/fe-filesystem/fe-filesystem-spi/src/test/java/org/apache/doris/filesystem/spi/FileSystemProviderTypedPropertiesTest.java +++ b/fe/fe-filesystem/fe-filesystem-spi/src/test/java/org/apache/doris/filesystem/spi/FileSystemProviderTypedPropertiesTest.java @@ -46,6 +46,12 @@ void provider_canBindAndCreateWithTypedProperties() throws IOException { Assertions.assertSame(properties, ((TestFileSystem) fileSystem).properties); } + @Test + void providerWithoutNativeCredentialsDoesNotClaimCredentialProperties() { + Assertions.assertTrue(new TypedProvider().credentialToProperties( + new org.apache.doris.filesystem.auth.GcpCredentialSpec(null, null)).isEmpty()); + } + private static class TypedProvider implements FileSystemProvider { @Override public boolean supports(Map properties) { diff --git a/fe/fe-filesystem/fe-filesystem-spi/src/test/resources/filesystem-plugin-surface.txt b/fe/fe-filesystem/fe-filesystem-spi/src/test/resources/filesystem-plugin-surface.txt index 819b07ca74430e..0cda5a09b8212c 100644 --- a/fe/fe-filesystem/fe-filesystem-spi/src/test/resources/filesystem-plugin-surface.txt +++ b/fe/fe-filesystem/fe-filesystem-spi/src/test/resources/filesystem-plugin-surface.txt @@ -5,19 +5,53 @@ org.apache.doris.extension.spi.PluginFactory#create():org.apache.doris.extension org.apache.doris.extension.spi.PluginFactory#create(org.apache.doris.extension.spi.PluginContext):org.apache.doris.extension.spi.Plugin org.apache.doris.extension.spi.PluginFactory#description():java.lang.String org.apache.doris.extension.spi.PluginFactory#name():java.lang.String +org.apache.doris.filesystem.auth.GcpCredentialSpec#equals(java.lang.Object):boolean +org.apache.doris.filesystem.auth.GcpCredentialSpec#getImpersonationServiceAccount():java.util.Optional +org.apache.doris.filesystem.auth.GcpCredentialSpec#getProvider():java.lang.String +org.apache.doris.filesystem.auth.GcpCredentialSpec#getProviderType():java.util.Optional +org.apache.doris.filesystem.auth.GcpCredentialSpec#hashCode():int +org.apache.doris.filesystem.auth.GcpCredentialSpec#isEmpty():boolean +org.apache.doris.filesystem.auth.GcpCredentialSpec$ProviderType#compareTo(java.lang.Enum):int +org.apache.doris.filesystem.auth.GcpCredentialSpec$ProviderType#describeConstable():java.util.Optional +org.apache.doris.filesystem.auth.GcpCredentialSpec$ProviderType#equals(java.lang.Object):boolean +org.apache.doris.filesystem.auth.GcpCredentialSpec$ProviderType#getDeclaringClass():java.lang.Class +org.apache.doris.filesystem.auth.GcpCredentialSpec$ProviderType#hashCode():int +org.apache.doris.filesystem.auth.GcpCredentialSpec$ProviderType#name():java.lang.String +org.apache.doris.filesystem.auth.GcpCredentialSpec$ProviderType#ordinal():int +org.apache.doris.filesystem.auth.GcpCredentialSpec$ProviderType#toString():java.lang.String +org.apache.doris.filesystem.auth.GcpCredentialSpec$ProviderType#valueOf(java.lang.Class,java.lang.String):java.lang.Enum +org.apache.doris.filesystem.auth.GcpCredentialSpec$ProviderType#valueOf(java.lang.String):org.apache.doris.filesystem.auth.GcpCredentialSpec$ProviderType +org.apache.doris.filesystem.auth.GcpCredentialSpec$ProviderType#values():org.apache.doris.filesystem.auth.GcpCredentialSpec$ProviderType[] +org.apache.doris.filesystem.auth.NativeObjectStorageCredential#getProvider():java.lang.String +org.apache.doris.filesystem.auth.NativeObjectStorageCredential#isEmpty():boolean +org.apache.doris.filesystem.auth.ObjectStorageAuthentication#anonymous(java.lang.String):org.apache.doris.filesystem.auth.ObjectStorageAuthentication +org.apache.doris.filesystem.auth.ObjectStorageAuthentication#getCredential():java.util.Optional +org.apache.doris.filesystem.auth.ObjectStorageAuthentication#getCredentialUpdates():java.util.Optional +org.apache.doris.filesystem.auth.ObjectStorageAuthentication#getProvider():java.lang.String +org.apache.doris.filesystem.auth.ObjectStorageAuthentication#hmac(java.lang.String):org.apache.doris.filesystem.auth.ObjectStorageAuthentication +org.apache.doris.filesystem.auth.ObjectStorageAuthentication#isAnonymous():boolean +org.apache.doris.filesystem.auth.ObjectStorageAuthentication#isNative():boolean +org.apache.doris.filesystem.auth.ObjectStorageAuthentication#nativeAuthentication(org.apache.doris.filesystem.auth.NativeObjectStorageCredential,org.apache.doris.filesystem.auth.NativeObjectStorageCredential):org.apache.doris.filesystem.auth.ObjectStorageAuthentication org.apache.doris.filesystem.spi.FileSystemProvider#bind(java.util.Map):org.apache.doris.filesystem.properties.FileSystemProperties org.apache.doris.filesystem.spi.FileSystemProvider#capabilities(org.apache.doris.filesystem.properties.FileSystemProperties):java.util.Set +org.apache.doris.filesystem.spi.FileSystemProvider#clearablePropertyKeys():java.util.Set org.apache.doris.filesystem.spi.FileSystemProvider#create():org.apache.doris.extension.spi.Plugin org.apache.doris.filesystem.spi.FileSystemProvider#create(java.util.Map):org.apache.doris.filesystem.FileSystem org.apache.doris.filesystem.spi.FileSystemProvider#create(org.apache.doris.extension.spi.PluginContext):org.apache.doris.extension.spi.Plugin org.apache.doris.filesystem.spi.FileSystemProvider#create(org.apache.doris.filesystem.properties.FileSystemProperties):org.apache.doris.filesystem.FileSystem org.apache.doris.filesystem.spi.FileSystemProvider#createUntyped(org.apache.doris.filesystem.properties.FileSystemProperties):org.apache.doris.filesystem.FileSystem +org.apache.doris.filesystem.spi.FileSystemProvider#credentialToProperties(org.apache.doris.filesystem.auth.NativeObjectStorageCredential):java.util.Optional org.apache.doris.filesystem.spi.FileSystemProvider#description():java.lang.String +org.apache.doris.filesystem.spi.FileSystemProvider#fallbackProviders():java.util.Set +org.apache.doris.filesystem.spi.FileSystemProvider#modifiableCredentialPropertyKeys():java.util.Set org.apache.doris.filesystem.spi.FileSystemProvider#name():java.lang.String +org.apache.doris.filesystem.spi.FileSystemProvider#normalizeProperties(java.util.Map,java.util.Map):java.util.Map +org.apache.doris.filesystem.spi.FileSystemProvider#resolveAuthentication(java.util.Map):java.util.Optional org.apache.doris.filesystem.spi.FileSystemProvider#sensitivePropertyKeys():java.util.Set org.apache.doris.filesystem.spi.FileSystemProvider#supports(java.util.Map):boolean org.apache.doris.filesystem.spi.FileSystemProvider#supportsExplicit(java.util.Map):boolean org.apache.doris.filesystem.spi.FileSystemProvider#supportsGuess(java.util.Map):boolean +org.apache.doris.filesystem.spi.FileSystemProvider#validateProperties(java.util.Map):void org.apache.doris.filesystem.spi.ObjFileSystem#capability(java.lang.Class):java.util.Optional org.apache.doris.filesystem.spi.ObjFileSystem#close():void org.apache.doris.filesystem.spi.ObjFileSystem#completeMultipartUpload(java.lang.String,java.lang.String,java.util.Map):void diff --git a/fe/fe-filesystem/pom.xml b/fe/fe-filesystem/pom.xml index a8368afa785571..2ff20328f23c54 100644 --- a/fe/fe-filesystem/pom.xml +++ b/fe/fe-filesystem/pom.xml @@ -53,12 +53,13 @@ under the License. bumping all five families. See plan-doc/designs/2026-07-29-plugin-api-version-check-design.md. --> - 1.0 + 5.0 fe-filesystem-api fe-filesystem-spi + fe-filesystem-gcs-common fe-filesystem-s3-base fe-filesystem-s3 fe-filesystem-gcs diff --git a/fe/fe-iceberg-common/pom.xml b/fe/fe-iceberg-common/pom.xml new file mode 100644 index 00000000000000..d139e7aa15bef2 --- /dev/null +++ b/fe/fe-iceberg-common/pom.xml @@ -0,0 +1,51 @@ + + + + 4.0.0 + + org.apache.doris + fe + ${revision} + ../pom.xml + + fe-iceberg-common + Doris Shared Iceberg FileIO + Native GCP FileIO factory shared by the FE Iceberg connector and the isolated + BE metadata scanner. Its class name is part of serialized Iceberg scan tasks. + + + org.apache.iceberg + iceberg-aws + ${iceberg.version} + + + org.apache.doris + fe-filesystem-gcs + ${revision} + + + + org.apache.doris + fe-filesystem-spi + ${revision} + + + diff --git a/fe/fe-iceberg-common/src/main/java/org/apache/doris/connector/iceberg/AnonymousS3FileIOAwsClientFactory.java b/fe/fe-iceberg-common/src/main/java/org/apache/doris/connector/iceberg/AnonymousS3FileIOAwsClientFactory.java new file mode 100644 index 00000000000000..25f027016743fe --- /dev/null +++ b/fe/fe-iceberg-common/src/main/java/org/apache/doris/connector/iceberg/AnonymousS3FileIOAwsClientFactory.java @@ -0,0 +1,61 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.iceberg; + +import org.apache.iceberg.aws.AwsClientFactories; +import org.apache.iceberg.aws.AwsClientFactory; +import org.apache.iceberg.aws.AwsClientProperties; +import org.apache.iceberg.aws.AwsProperties; +import org.apache.iceberg.aws.s3.S3FileIOAwsClientFactory; +import org.apache.iceberg.aws.s3.S3FileIOProperties; +import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.S3Client; + +import java.util.HashMap; +import java.util.Map; + +/** Keeps anonymous object access separate from the catalog's authenticated AWS metadata clients. */ +public class AnonymousS3FileIOAwsClientFactory implements S3FileIOAwsClientFactory { + private static final long serialVersionUID = 1L; + + private AwsClientFactory delegate; + + @Override + public void initialize(Map properties) { + Map fileIoProperties = new HashMap<>(properties); + // A Glue assume-role factory or explicit object keys must not override anonymous FileIO. + fileIoProperties.remove(AwsProperties.CLIENT_FACTORY); + fileIoProperties.remove(S3FileIOProperties.ACCESS_KEY_ID); + fileIoProperties.remove(S3FileIOProperties.SECRET_ACCESS_KEY); + fileIoProperties.remove(S3FileIOProperties.SESSION_TOKEN); + fileIoProperties.put(AwsClientProperties.CLIENT_CREDENTIALS_PROVIDER, + AnonymousCredentialsProvider.class.getName()); + delegate = AwsClientFactories.from(fileIoProperties); + } + + @Override + public S3Client s3() { + return delegate.s3(); + } + + @Override + public S3AsyncClient s3Async() { + return delegate.s3Async(); + } +} diff --git a/fe/fe-iceberg-common/src/main/java/org/apache/doris/connector/iceberg/GcpS3FileIOAwsClientFactory.java b/fe/fe-iceberg-common/src/main/java/org/apache/doris/connector/iceberg/GcpS3FileIOAwsClientFactory.java new file mode 100644 index 00000000000000..fac1530c53db95 --- /dev/null +++ b/fe/fe-iceberg-common/src/main/java/org/apache/doris/connector/iceberg/GcpS3FileIOAwsClientFactory.java @@ -0,0 +1,74 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.connector.iceberg; + +import org.apache.doris.filesystem.gcs.GcpAuth; +import org.apache.doris.filesystem.gcs.GcsFileSystemProperties; + +import org.apache.iceberg.aws.s3.S3FileIOAwsClientFactory; +import software.amazon.awssdk.services.s3.S3AsyncClient; +import software.amazon.awssdk.services.s3.S3Client; + +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.HashMap; +import java.util.Map; + +/** Supplies Iceberg S3FileIO clients configured with native GCP bearer authentication. */ +public class GcpS3FileIOAwsClientFactory implements S3FileIOAwsClientFactory { + private static final long serialVersionUID = 1L; + + private Map properties; + + public GcpS3FileIOAwsClientFactory() { + } + + @Override + public void initialize(Map properties) { + this.properties = new HashMap<>(properties); + } + + @Override + public S3Client s3() { + try { + return GcpAuth.createS3Client(gcsProperties()); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @Override + public S3AsyncClient s3Async() { + try { + return GcpAuth.createS3AsyncClient(gcsProperties()); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + private GcsFileSystemProperties gcsProperties() { + if (properties == null) { + throw new IllegalStateException("GCP Iceberg client factory has not been initialized"); + } + GcsFileSystemProperties storageProperties = GcsFileSystemProperties.of(properties); + if (storageProperties.getAuth().getNativeCredential().isEmpty()) { + throw new IllegalArgumentException("Native GCP Iceberg authentication requires GCS credentials"); + } + return storageProperties; + } +} diff --git a/fe/fe-storage-protocol/pom.xml b/fe/fe-storage-protocol/pom.xml new file mode 100644 index 00000000000000..9b8ed9504f7809 --- /dev/null +++ b/fe/fe-storage-protocol/pom.xml @@ -0,0 +1,55 @@ + + + + 4.0.0 + + org.apache.doris + fe + ${revision} + ../pom.xml + + fe-storage-protocol + Doris object storage protocol adapters + Host-side translation of typed filesystem identities into Doris RPC messages. + + + org.apache.doris + fe-filesystem-api + ${revision} + + + org.apache.doris + fe-grpc + ${revision} + + + org.apache.doris + fe-thrift + ${revision} + + + org.junit.jupiter + junit-jupiter + test + + + diff --git a/fe/fe-storage-protocol/src/main/java/org/apache/doris/storage/protocol/GcpCredentialCodec.java b/fe/fe-storage-protocol/src/main/java/org/apache/doris/storage/protocol/GcpCredentialCodec.java new file mode 100644 index 00000000000000..be06993bd00432 --- /dev/null +++ b/fe/fe-storage-protocol/src/main/java/org/apache/doris/storage/protocol/GcpCredentialCodec.java @@ -0,0 +1,86 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.storage.protocol; + +import org.apache.doris.cloud.proto.Cloud.GcpCredentialPB; +import org.apache.doris.cloud.proto.Cloud.ObjectStoreCredentialPB; +import org.apache.doris.filesystem.auth.GcpCredentialSpec; +import org.apache.doris.thrift.TCredential; +import org.apache.doris.thrift.TGcpCredential; +import org.apache.doris.thrift.TGcpCredentialProviderType; + +/** GCP wire-schema mapping only; property validation and credential loading stay in the GCS plugin. */ +final class GcpCredentialCodec implements ObjectStorageCredentialCodec { + @Override + public Class credentialType() { + return GcpCredentialSpec.class; + } + + @Override + public ObjectStoreCredentialPB toCloud(GcpCredentialSpec spec) { + GcpCredentialPB.Builder credential = GcpCredentialPB.newBuilder(); + spec.getProviderType().ifPresent(type -> { + GcpCredentialPB.CredentialProviderType mapped = switch (type) { + case DEFAULT -> GcpCredentialPB.CredentialProviderType.DEFAULT; + case COMPUTE_ENGINE -> GcpCredentialPB.CredentialProviderType.COMPUTE_ENGINE; + default -> throw new IllegalArgumentException("Unsupported GCP identity source: " + type); + }; + credential.setCredentialProviderType(mapped); + }); + spec.getImpersonationServiceAccount().ifPresent(credential::setImpersonationServiceAccount); + return ObjectStoreCredentialPB.newBuilder().setGcpCredential(credential).build(); + } + + @Override + public TCredential toThrift(GcpCredentialSpec spec) { + TGcpCredential credential = new TGcpCredential(); + spec.getProviderType().ifPresent(type -> { + TGcpCredentialProviderType mapped = switch (type) { + case DEFAULT -> TGcpCredentialProviderType.DEFAULT; + case COMPUTE_ENGINE -> TGcpCredentialProviderType.COMPUTE_ENGINE; + default -> throw new IllegalArgumentException("Unsupported GCP identity source: " + type); + }; + credential.setCredentialProviderType(mapped); + }); + spec.getImpersonationServiceAccount().ifPresent(credential::setImpersonationServiceAccount); + return new TCredential().setGcpCredential(credential); + } + + @Override + public GcpCredentialSpec fromCloud(ObjectStoreCredentialPB envelope) { + if (!envelope.hasGcpCredential()) { + throw new IllegalArgumentException("Missing GCP credentials in the Cloud credential envelope"); + } + GcpCredentialPB credential = envelope.getGcpCredential(); + // Proto2 retains unknown enum values as unknown fields. Treating that as absence would + // make a newer identity source silently fall back to ADC when properties are restored. + if (credential.getUnknownFields().hasField(GcpCredentialPB.CREDENTIAL_PROVIDER_TYPE_FIELD_NUMBER)) { + throw new IllegalArgumentException("Unsupported GCP identity source in Cloud credentials"); + } + GcpCredentialSpec.ProviderType type = null; + if (credential.hasCredentialProviderType()) { + type = switch (credential.getCredentialProviderType()) { + case DEFAULT -> GcpCredentialSpec.ProviderType.DEFAULT; + case COMPUTE_ENGINE -> GcpCredentialSpec.ProviderType.COMPUTE_ENGINE; + default -> throw new IllegalArgumentException("Unsupported GCP identity source in Cloud credentials"); + }; + } + return new GcpCredentialSpec(type, + credential.hasImpersonationServiceAccount() ? credential.getImpersonationServiceAccount() : null); + } +} diff --git a/fe/fe-storage-protocol/src/main/java/org/apache/doris/storage/protocol/ObjectStorageCredentialCodec.java b/fe/fe-storage-protocol/src/main/java/org/apache/doris/storage/protocol/ObjectStorageCredentialCodec.java new file mode 100644 index 00000000000000..cd9d19062d7e8c --- /dev/null +++ b/fe/fe-storage-protocol/src/main/java/org/apache/doris/storage/protocol/ObjectStorageCredentialCodec.java @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.storage.protocol; + +import org.apache.doris.cloud.proto.Cloud.ObjectStoreCredentialPB; +import org.apache.doris.filesystem.auth.NativeObjectStorageCredential; +import org.apache.doris.thrift.TCredential; + +/** Internal host adapter contract; never implemented or loaded by filesystem plugins. */ +interface ObjectStorageCredentialCodec { + Class credentialType(); + + ObjectStoreCredentialPB toCloud(C credential); + + TCredential toThrift(C credential); + + C fromCloud(ObjectStoreCredentialPB credential); +} diff --git a/fe/fe-storage-protocol/src/main/java/org/apache/doris/storage/protocol/ObjectStorageCredentialTranslator.java b/fe/fe-storage-protocol/src/main/java/org/apache/doris/storage/protocol/ObjectStorageCredentialTranslator.java new file mode 100644 index 00000000000000..975afbd0022147 --- /dev/null +++ b/fe/fe-storage-protocol/src/main/java/org/apache/doris/storage/protocol/ObjectStorageCredentialTranslator.java @@ -0,0 +1,76 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.storage.protocol; + +import org.apache.doris.cloud.proto.Cloud.ObjectStoreCredentialPB; +import org.apache.doris.filesystem.auth.NativeObjectStorageCredential; +import org.apache.doris.thrift.TCredential; + +import java.util.Locale; +import java.util.Map; + +/** + * Host-side boundary between identity descriptions and the wire variants understood by Doris. + * Adding a wire variant requires a codec here and matching receiver support, not a core factory branch + * or a serialization hook in the filesystem SPI. No user properties or SDK objects enter this module. + */ +public final class ObjectStorageCredentialTranslator { + private static final Map> CODECS = Map.of("GCP", new GcpCredentialCodec()); + + private ObjectStorageCredentialTranslator() { + } + + public static ObjectStoreCredentialPB toCloud(NativeObjectStorageCredential credential) { + return toCloud(codec(credential.getProvider()), credential); + } + + public static TCredential toThrift(NativeObjectStorageCredential credential) { + return toThrift(codec(credential.getProvider()), credential); + } + + public static NativeObjectStorageCredential fromCloud(String provider, ObjectStoreCredentialPB credential) { + return codec(provider).fromCloud(credential); + } + + private static ObjectStorageCredentialCodec codec(String provider) { + ObjectStorageCredentialCodec codec = CODECS.get(provider.toUpperCase(Locale.ROOT)); + if (codec == null) { + throw new IllegalArgumentException("Unsupported native credential protocol for provider: " + provider); + } + return codec; + } + + private static C checkedCredential( + ObjectStorageCredentialCodec codec, NativeObjectStorageCredential credential) { + if (!codec.credentialType().isInstance(credential)) { + throw new IllegalArgumentException("Unexpected credential description for provider: " + + credential.getProvider()); + } + return codec.credentialType().cast(credential); + } + + private static ObjectStoreCredentialPB toCloud( + ObjectStorageCredentialCodec codec, NativeObjectStorageCredential credential) { + return codec.toCloud(checkedCredential(codec, credential)); + } + + private static TCredential toThrift( + ObjectStorageCredentialCodec codec, NativeObjectStorageCredential credential) { + return codec.toThrift(checkedCredential(codec, credential)); + } +} diff --git a/fe/fe-storage-protocol/src/test/java/org/apache/doris/storage/protocol/ObjectStorageCredentialTranslatorTest.java b/fe/fe-storage-protocol/src/test/java/org/apache/doris/storage/protocol/ObjectStorageCredentialTranslatorTest.java new file mode 100644 index 00000000000000..c2046c5685e1eb --- /dev/null +++ b/fe/fe-storage-protocol/src/test/java/org/apache/doris/storage/protocol/ObjectStorageCredentialTranslatorTest.java @@ -0,0 +1,116 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.storage.protocol; + +import org.apache.doris.cloud.proto.Cloud.GcpCredentialPB; +import org.apache.doris.cloud.proto.Cloud.ObjectStoreCredentialPB; +import org.apache.doris.filesystem.auth.GcpCredentialSpec; +import org.apache.doris.filesystem.auth.NativeObjectStorageCredential; +import org.apache.doris.thrift.TGcpCredential; +import org.apache.doris.thrift.TGcpCredentialProviderType; + +import com.google.protobuf.UnknownFieldSet; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +class ObjectStorageCredentialTranslatorTest { + @Test + void bothWireFormatsPreserveAllSupportedIdentitySources() { + for (GcpCredentialSpec.ProviderType type : GcpCredentialSpec.ProviderType.values()) { + GcpCredentialSpec spec = new GcpCredentialSpec(type, "target@test.iam.gserviceaccount.com"); + ObjectStoreCredentialPB cloud = ObjectStorageCredentialTranslator.toCloud(spec); + TGcpCredential thrift = ObjectStorageCredentialTranslator.toThrift(spec).getGcpCredential(); + Assertions.assertEquals(GcpCredentialPB.CredentialProviderType.valueOf(type.name()), + cloud.getGcpCredential().getCredentialProviderType()); + Assertions.assertEquals(TGcpCredentialProviderType.valueOf(type.name()), thrift.getCredentialProviderType()); + Assertions.assertEquals("target@test.iam.gserviceaccount.com", thrift.getImpersonationServiceAccount()); + Assertions.assertEquals(thrift.getImpersonationServiceAccount(), + cloud.getGcpCredential().getImpersonationServiceAccount()); + Assertions.assertEquals(spec, ObjectStorageCredentialTranslator.fromCloud("gcp", cloud)); + } + } + + @Test + void partialAlterPreservesExplicitClearWithoutAddingSource() { + GcpCredentialSpec patch = new GcpCredentialSpec(null, ""); + ObjectStoreCredentialPB cloud = ObjectStorageCredentialTranslator.toCloud(patch); + TGcpCredential thrift = ObjectStorageCredentialTranslator.toThrift(patch).getGcpCredential(); + Assertions.assertFalse(cloud.getGcpCredential().hasCredentialProviderType()); + Assertions.assertFalse(thrift.isSetCredentialProviderType()); + Assertions.assertTrue(cloud.getGcpCredential().hasImpersonationServiceAccount()); + Assertions.assertTrue(thrift.isSetImpersonationServiceAccount()); + Assertions.assertEquals("", thrift.getImpersonationServiceAccount()); + Assertions.assertEquals("", cloud.getGcpCredential().getImpersonationServiceAccount()); + Assertions.assertEquals(patch, ObjectStorageCredentialTranslator.fromCloud("GCP", cloud)); + } + + @Test + void absentFieldsStayAbsentInBothDirections() { + for (GcpCredentialSpec spec : new GcpCredentialSpec[] { + new GcpCredentialSpec(GcpCredentialSpec.ProviderType.DEFAULT, null), + new GcpCredentialSpec(null, null)}) { + ObjectStoreCredentialPB cloud = ObjectStorageCredentialTranslator.toCloud(spec); + TGcpCredential thrift = ObjectStorageCredentialTranslator.toThrift(spec).getGcpCredential(); + Assertions.assertFalse(cloud.getGcpCredential().hasImpersonationServiceAccount()); + Assertions.assertFalse(thrift.isSetImpersonationServiceAccount()); + Assertions.assertEquals(spec, ObjectStorageCredentialTranslator.fromCloud("GCP", cloud)); + Assertions.assertEquals(spec.getProviderType().isPresent(), thrift.isSetCredentialProviderType()); + } + } + + @Test + void missingEnvelopeDoesNotSilentlySelectDefaultCredentials() { + Assertions.assertThrows(IllegalArgumentException.class, () -> ObjectStorageCredentialTranslator.fromCloud( + "GCP", ObjectStoreCredentialPB.getDefaultInstance())); + } + + @Test + void unknownFutureSourceDoesNotTurnIntoApplicationDefaultCredentials() throws Exception { + GcpCredentialPB unknown = GcpCredentialPB.newBuilder().setUnknownFields(UnknownFieldSet.newBuilder() + .addField(GcpCredentialPB.CREDENTIAL_PROVIDER_TYPE_FIELD_NUMBER, + UnknownFieldSet.Field.newBuilder().addVarint(100).build()).build()).build(); + ObjectStoreCredentialPB envelope = ObjectStoreCredentialPB.parseFrom( + ObjectStoreCredentialPB.newBuilder().setGcpCredential(unknown).build().toByteArray()); + Assertions.assertFalse(envelope.getGcpCredential().hasCredentialProviderType()); + Assertions.assertThrows(IllegalArgumentException.class, + () -> ObjectStorageCredentialTranslator.fromCloud("GCP", envelope)); + } + + @Test + void unsupportedProvidersAndMismatchedTypesFailExplicitly() { + for (String provider : new String[] {"OTHER", "GCP"}) { + NativeObjectStorageCredential credential = new NativeObjectStorageCredential() { + @Override + public String getProvider() { + return provider; + } + + @Override + public boolean isEmpty() { + return false; + } + }; + Assertions.assertThrows(IllegalArgumentException.class, + () -> ObjectStorageCredentialTranslator.toCloud(credential)); + Assertions.assertThrows(IllegalArgumentException.class, + () -> ObjectStorageCredentialTranslator.toThrift(credential)); + } + Assertions.assertThrows(IllegalArgumentException.class, () -> ObjectStorageCredentialTranslator.fromCloud( + "OTHER", ObjectStoreCredentialPB.getDefaultInstance())); + } +} diff --git a/fe/pom.xml b/fe/pom.xml index c2e3f62d124d4c..47ea3a21263eae 100644 --- a/fe/pom.xml +++ b/fe/pom.xml @@ -230,8 +230,10 @@ under the License. fe-kerberos fe-common fe-trino-connector-common + fe-iceberg-common fe-catalog fe-filesystem + fe-storage-protocol fe-extension-spi fe-connector fe-extension-loader @@ -387,6 +389,7 @@ under the License. 3.0.9 42.7.13 3.4.2 + 3.1.18 1.8 1.2.0 1.5.0 @@ -1489,6 +1492,19 @@ under the License. + + + com.google.cloud.bigdataoss + gcs-connector + ${gcs-connector.version} + shaded + + + * + * + + + org.apache.hadoop hadoop-azure diff --git a/gensrc/proto/cloud.proto b/gensrc/proto/cloud.proto index 6d96eba78c0d77..c40a5ad289d962 100644 --- a/gensrc/proto/cloud.proto +++ b/gensrc/proto/cloud.proto @@ -287,6 +287,24 @@ enum CredProviderTypePB { ANONYMOUS = 8; // AnonymousAWSCredentialsProvider } +message GcpCredentialPB { + enum CredentialProviderType { + DEFAULT = 0; + COMPUTE_ENGINE = 1; + } + + optional CredentialProviderType credential_provider_type = 1; + optional string impersonation_service_account = 2; +} + +// Provider credentials are grouped in a regular message so older readers can +// still observe that a credential envelope is present when a new provider is +// added. Validation must reject multiple provider credentials and ensure the +// selected credential matches ObjectStoreInfoPB.provider. +message ObjectStoreCredentialPB { + optional GcpCredentialPB gcp_credential = 1; +} + message ObjectStoreInfoPB { // presigned url use // oss,aws,cos,obs,bos @@ -320,6 +338,8 @@ message ObjectStoreInfoPB { optional CredProviderTypePB cred_provider_type = 17; optional string role_arn = 18; // aws assumed role's arn optional string external_id = 19; // aws assumed role's external_id if configure + + optional ObjectStoreCredentialPB credential = 20; } // The legacy ObjectStoreInfoPB is stored in InstanceInfoPB diff --git a/gensrc/thrift/AgentService.thrift b/gensrc/thrift/AgentService.thrift index 49907f4a4ba475..0371de4d158bc9 100644 --- a/gensrc/thrift/AgentService.thrift +++ b/gensrc/thrift/AgentService.thrift @@ -108,6 +108,23 @@ enum TCredProviderType { ANONYMOUS = 7 // AnonymousAWSCredentialsProvider } +enum TGcpCredentialProviderType { + DEFAULT = 0, + COMPUTE_ENGINE = 1 +} + +struct TGcpCredential { + 1: optional TGcpCredentialProviderType credential_provider_type + 2: optional string impersonation_service_account +} + +// Keep this struct aligned with ObjectStoreCredentialPB. Append new fields with +// new IDs; validation rejects multiple provider credentials and ensures the +// selected credential matches TS3StorageParam.provider. +struct TCredential { + 1: optional TGcpCredential gcp_credential +} + struct TS3StorageParam { 1: optional string endpoint 2: optional string region @@ -125,6 +142,7 @@ struct TS3StorageParam { 13: optional TCredProviderType cred_provider_type 14: optional string role_arn // aws assumed role's arn 15: optional string external_id // aws assumed role's external_id if configure + 16: optional TCredential credential } struct TStoragePolicy { diff --git a/regression-test/conf/regression-conf.groovy b/regression-test/conf/regression-conf.groovy index 6a5104a4f4fcb1..0baa216c1dced7 100644 --- a/regression-test/conf/regression-conf.groovy +++ b/regression-test/conf/regression-conf.groovy @@ -341,6 +341,28 @@ anymousS3Region="eu-west-3" anymousS3ExpectDataCount="8365" awsInstanceProfileRegion="us-east-1" +// Object storage IAM regression tests run one provider per invocation: AWS or GCP. +objectStorageIamProvider="" +objectStorageIamEndpoint="" +objectStorageIamRegion="" +objectStorageIamBucket="" +objectStorageIamPrefix="regression/object_storage_iam_p0" +objectStorageIamDataPath="regression/tpch/sf0.01/customer.csv.gz" +// AWS authentication. +objectStorageIamAwsRoleArn="" +objectStorageIamAwsExternalId="" +// GCP authentication. Credential provider types are comma-separated DEFAULT and/or +// COMPUTE_ENGINE. DEFAULT also tests omitted authentication properties; leaving both GCP +// settings empty tests only omitted properties. A non-empty account adds COMPUTE_ENGINE +// impersonation. The IAM data fixture must be private so anonymous reads can be rejected. +objectStorageIamGcpCredentialProviderTypes="" +objectStorageIamGcpImpersonationServiceAccount="" +// Optional public GCS fixture for anonymous TVF reads: gzip CSV with a '|' separator. +// Configure a gs:// URI and its positive row count to enable test_gcp_tvf_anonymous. +// This public fixture must differ from the private objectStorageIamDataPath fixture. +objectStorageIamGcpAnonymousUri="" +objectStorageIamGcpAnonymousExpectedRows="" + // hudi p0 external regression test config // To enable hudi test, you need first start hudi container. // See `docker/thirdparties/run-thirdparties-docker.sh -c hudi` diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Syncer.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Syncer.groovy index a87744a86a387e..3a0b74d0ad555c 100644 --- a/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Syncer.groovy +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/suite/Syncer.groovy @@ -1049,4 +1049,17 @@ class Syncer { ) """ } + + void createObjectStorageIamRepository(String name, Map iamConfig, Map authCase, boolean readOnly = false) { + suite.try_sql "DROP REPOSITORY `${name}`" + suite.sql """ + CREATE ${readOnly ? "READ ONLY" : ""} REPOSITORY `${name}` + WITH S3 + ON LOCATION "${iamConfig.scheme}://${iamConfig.bucket}/${iamConfig.prefix}/${name}" + PROPERTIES + ( + ${authCase.storageSqlProperties} + ) + """ + } } diff --git a/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/ObjectStorageIamTestUtils.groovy b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/ObjectStorageIamTestUtils.groovy new file mode 100644 index 00000000000000..e3e94fbf3662d2 --- /dev/null +++ b/regression-test/framework/src/main/groovy/org/apache/doris/regression/util/ObjectStorageIamTestUtils.groovy @@ -0,0 +1,193 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.regression.util + +import java.util.Locale + +/** + * Builds the IAM authentication matrix shared by the object_storage_iam_p0 suites. + * + *

One regression-test invocation selects exactly one cloud with + * {@code objectStorageIamProvider=AWS} or {@code objectStorageIamProvider=GCP}. Common connection + * settings are reused by both providers. Leaving {@code objectStorageIamProvider} empty disables + * the shared cases; AWS-only suites also use {@link #isProvider(Map, String)} to skip themselves + * during a GCP run. + * + *

AWS role example in regression-conf.groovy: + * 

+ * objectStorageIamProvider="AWS"
+ * objectStorageIamEndpoint="https://s3.us-east-1.amazonaws.com"
+ * objectStorageIamRegion="us-east-1"
+ * objectStorageIamBucket="my-bucket"
+ * objectStorageIamPrefix="regression/object_storage_iam_p0"
+ * objectStorageIamDataPath="regression/tpch/sf0.01/customer.csv.gz"
+ * objectStorageIamAwsRoleArn="arn:aws:iam::123456789012:role/my-role"
+ * objectStorageIamAwsExternalId="optional-external-id"
+ *
+ * + *

GCP example covering DEFAULT, COMPUTE_ENGINE, and service-account impersonation: + * 

+ * objectStorageIamProvider="GCP"
+ * objectStorageIamEndpoint="https://storage.googleapis.com"
+ * objectStorageIamRegion="us-east1"
+ * objectStorageIamBucket="my-bucket"
+ * objectStorageIamPrefix="regression/object_storage_iam_p0"
+ * objectStorageIamDataPath="regression/tpch/sf0.01/customer.csv.gz"
+ * objectStorageIamGcpCredentialProviderTypes="DEFAULT,COMPUTE_ENGINE"
+ * objectStorageIamGcpImpersonationServiceAccount="target@project.iam.gserviceaccount.com"
+ *
+ * + *

The comma-separated GCP provider list controls which direct credential cases run. DEFAULT also + * tests omitted authentication properties against the same private fixture. If both the provider + * list and impersonation account are empty, only the omitted-authentication case runs. A non-empty + * impersonation service account adds one more COMPUTE_ENGINE case. Each shared suite iterates over + * the resulting {@code authCases}. The test selector uses AWS/GCP, while SQL properties use the + * public S3/GCP provider values. + */ +class ObjectStorageIamTestUtils { + private static final List PROVIDERS = ["AWS", "GCP"] + private static final List GCP_CREDENTIAL_PROVIDER_TYPES = ["DEFAULT", "COMPUTE_ENGINE"] + private static final List REQUIRED_CONFIGS = [ + "objectStorageIamEndpoint", + "objectStorageIamRegion", + "objectStorageIamBucket", + "objectStorageIamPrefix", + "objectStorageIamDataPath" + ] + + static Map getConfig(Map configs) { + String provider = value(configs, "objectStorageIamProvider")?.toUpperCase(Locale.ROOT) + if (provider == null || provider.isEmpty()) { + return null + } + if (!PROVIDERS.contains(provider)) { + throw new IllegalArgumentException("Unsupported objectStorageIamProvider: ${provider}; " + + "supported values are ${PROVIDERS}") + } + + List missingConfigs = REQUIRED_CONFIGS.findAll { isBlank(value(configs, it)) } + if (!missingConfigs.isEmpty()) { + throw new IllegalArgumentException("Missing object storage IAM configs: ${missingConfigs}") + } + + List authCases = provider == "AWS" + ? getAwsAuthCases(configs) + : getGcpAuthCases(configs) + String endpointProperty = provider == "AWS" ? "s3.endpoint" : "gs.endpoint" + String propertyProvider = provider == "AWS" ? "S3" : "GCP" + String endpoint = value(configs, "objectStorageIamEndpoint") + String region = value(configs, "objectStorageIamRegion") + authCases.each { authCase -> + Map storageProperties = new LinkedHashMap<>() + storageProperties.put("provider", propertyProvider) + storageProperties.put(endpointProperty, endpoint) + storageProperties.put("s3.region", region) + storageProperties.putAll(authCase.properties) + authCase.storageProperties = storageProperties + authCase.storageSqlProperties = toSqlProperties(storageProperties) + } + return [ + provider: provider, + propertyProvider: propertyProvider, + scheme: provider == "AWS" ? "s3" : "gs", + endpoint: endpoint, + region: region, + bucket: value(configs, "objectStorageIamBucket"), + prefix: value(configs, "objectStorageIamPrefix"), + dataPath: value(configs, "objectStorageIamDataPath"), + roleArn: value(configs, "objectStorageIamAwsRoleArn"), + externalId: value(configs, "objectStorageIamAwsExternalId"), + authCases: authCases + ] + } + + static boolean isProvider(Map configs, String expectedProvider) { + String provider = value(configs, "objectStorageIamProvider") + return provider != null && provider.equalsIgnoreCase(expectedProvider) + } + + private static List getAwsAuthCases(Map configs) { + String roleArn = value(configs, "objectStorageIamAwsRoleArn") + if (roleArn == null || roleArn.isEmpty()) { + throw new IllegalArgumentException("objectStorageIamAwsRoleArn must be configured for AWS") + } + + Map properties = ["s3.role_arn": roleArn] + String externalId = value(configs, "objectStorageIamAwsExternalId") + if (externalId != null && !externalId.isEmpty()) { + properties.put("s3.external_id", externalId) + } + return [authCase("aws_role", properties)] + } + + private static List getGcpAuthCases(Map configs) { + String configuredTypes = value(configs, "objectStorageIamGcpCredentialProviderTypes") + List providerTypes = (configuredTypes ?: "").split(",") + .collect { it.trim().toUpperCase(Locale.ROOT) } + .findAll { !it.isEmpty() } + .unique() + List unsupportedTypes = providerTypes.findAll { + !GCP_CREDENTIAL_PROVIDER_TYPES.contains(it) + } + if (!unsupportedTypes.isEmpty()) { + throw new IllegalArgumentException( + "Unsupported objectStorageIamGcpCredentialProviderTypes: ${unsupportedTypes}; " + + "supported values are ${GCP_CREDENTIAL_PROVIDER_TYPES}") + } + + List cases = providerTypes.collect { providerType -> + authCase("gcp_${providerType.toLowerCase(Locale.ROOT)}", + ["gs.credential_provider_type": providerType]) + } + String serviceAccount = value(configs, "objectStorageIamGcpImpersonationServiceAccount") + if (serviceAccount != null && !serviceAccount.isEmpty()) { + cases.add(authCase("gcp_impersonation", [ + "gs.credential_provider_type": "COMPUTE_ENGINE", + "gs.impersonation_service_account": serviceAccount + ])) + } + if (providerTypes.contains("DEFAULT") || cases.isEmpty()) { + // Exercise the VM-attached service account without either native authentication property. + // Shared TVF, Load, Outfile, Export, Resource, Catalog and Vault suites all consume this case. + cases.add(0, authCase("gcp_default_omitted", [:])) + } + return cases + } + + private static Map authCase(String name, Map properties) { + return [ + name: name, + properties: properties, + authProperties: toSqlProperties(properties) + ] + } + + static String toSqlProperties(Map properties) { + return properties.collect { key, propertyValue -> + "\"${key}\" = \"${propertyValue}\"" + }.join(",\n") + } + + private static String value(Map configs, String key) { + return configs.get(key)?.toString()?.trim() + } + + private static boolean isBlank(String value) { + return value == null || value.isEmpty() + } +} diff --git a/regression-test/suites/object_storage_iam_p0/test_alter_resource_with_role.groovy b/regression-test/suites/object_storage_iam_p0/test_alter_resource_with_role.groovy new file mode 100644 index 00000000000000..cdf3fb5cf578eb --- /dev/null +++ b/regression-test/suites/object_storage_iam_p0/test_alter_resource_with_role.groovy @@ -0,0 +1,240 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import groovy.json.JsonSlurper +import org.apache.doris.regression.util.ObjectStorageIamTestUtils + +suite("test_alter_resource_with_role") { + if (!ObjectStorageIamTestUtils.isProvider(context.config.otherConfigs, "AWS")) { + logger.info("skip ${name} because objectStorageIamProvider is not AWS") + return + } + def config = ObjectStorageIamTestUtils.getConfig(context.config.otherConfigs) + + + if (isCloudMode()) { + logger.info("skip ${name} case, because it is cloud mode") + return + } + + def tableName = "test_alter_resource_with_role" + def randomStr = UUID.randomUUID().toString().replace("-", "") + def resourceName = "alter_resource_${randomStr}" + def policyName = "alter_policy_${randomStr}" + + def awsEndpoint = config.endpoint + def region = config.region + def bucket = config.bucket + def roleArn = config.roleArn + def externalId = config.externalId ?: "" + def prefix = config.prefix + + def awsAccessKey = context.config.awsAccessKey + def awsSecretKey = context.config.awsSecretKey + + sql """ + CREATE RESOURCE IF NOT EXISTS "${resourceName}" + PROPERTIES( + "type"="s3", + "AWS_ENDPOINT" = "${awsEndpoint}", + "AWS_REGION" = "${region}", + "AWS_BUCKET" = "${bucket}", + "AWS_ROOT_PATH" = "${prefix}/test_alter_resource_with_role/${randomStr}", + "AWS_ACCESS_KEY" = "error_ak", + "AWS_SECRET_KEY" = "error_sk", + "s3_validity_check" = "false" + ); + """ + + sql """ + CREATE STORAGE POLICY IF NOT EXISTS ${policyName} + PROPERTIES( + "storage_resource" = "${resourceName}", + "cooldown_ttl" = "1" + ) + """ + + sql """ + DROP TABLE IF EXISTS ${tableName} FORCE; + """ + + sql """ + CREATE TABLE ${tableName} + ( + siteid INT DEFAULT '10', + citycode SMALLINT NOT NULL, + username VARCHAR(32) DEFAULT '', + pv BIGINT SUM DEFAULT '0' + ) + AGGREGATE KEY(siteid, citycode, username) + DISTRIBUTED BY HASH(siteid) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "storage_policy" = "${policyName}" + ) + """ + + sql """insert into ${tableName}(siteid, citycode, username, pv) values (1, 1, "xxx", 1), + (2, 2, "yyy", 2), + (3, 3, "zzz", 3) + """ + + def result = sql """ SHOW RESOURCES WHERE NAME = "${resourceName}"; """ + log.info("result:${result}") + assertTrue(!result.toString().contains(roleArn)) + if (!externalId.isEmpty()) { + assertTrue(!result.toString().contains(externalId)) + } + assertTrue(result.toString().contains("error_ak")); + + // data_sizes is one arrayList, t is tablet + def fetchDataSize = {List data_sizes, Map t -> + def tabletId = t.TabletId + def meta_url = t.MetaUrl + def clos = { respCode, body -> + logger.info("test ttl expired resp Code {}", "${respCode}".toString()) + assertEquals("${respCode}".toString(), "200") + String out = "${body}".toString() + logger.info("test fetchBeHttp, resp body: ${out}") + def obj = new JsonSlurper().parseText(out) + data_sizes[0] = obj.local_data_size + data_sizes[1] = obj.remote_data_size + } + meta_url = meta_url.replace("header", "data_size") + + def i = meta_url.indexOf("/api") + def endPoint = meta_url.substring(0, i) + def metaUri = meta_url.substring(i) + logger.info("test fetchBeHttp, endpoint:${endPoint}, metaUri:${metaUri}") + i = endPoint.lastIndexOf('/') + endPoint = endPoint.substring(i + 1) + + httpTest { + endpoint {endPoint} + uri metaUri + op "get" + check clos + } + } + + List sizes = [-1, -1] + def tablets = sql_return_maparray """ + SHOW TABLETS FROM ${tableName} + """ + log.info( "test tablets not empty:${tablets}") + fetchDataSize(sizes, tablets[0]) + def retry = 12 + while (sizes[1] == 0 && retry-- > 0) { + log.info( "test remote size is zero, sleep 10s") + sleep(10000) + tablets = sql_return_maparray """ + SHOW TABLETS FROM ${tableName} + """ + fetchDataSize(sizes, tablets[0]) + } + assertTrue(sizes[1] == 0, "remote size expected is zero, but got ${sizes[1]}") + assertTrue(tablets.size() > 0) + + // alter resource to arn role + sql """ truncate table ${tableName} """ + sql """ALTER RESOURCE "${resourceName}" + PROPERTIES( + "AWS_ROLE_ARN" = "${roleArn}", + "AWS_EXTERNAL_ID" = "${externalId}" + );""" + + result = sql """ SHOW RESOURCES WHERE NAME = "${resourceName}"; """ + log.info("result:${result}") + assertTrue(result.toString().contains(roleArn)) + if (!externalId.isEmpty()) { + assertTrue(result.toString().contains(externalId)) + } + assertTrue(!result.toString().contains("error_ak")); + + // report_tablet_interval_seconds = 60 sleep wait for report policy + sleep(120 * 1000) + + sql """insert into ${tableName}(siteid, citycode, username, pv) values (1, 1, "xxx", 1), + (4, 4, "yyy", 5), + (5, 5, "zzz", 5), + (6, 6, "xxx", 6) + """ + + sizes = [-1, -1] + tablets = sql_return_maparray """ + SHOW TABLETS FROM ${tableName} + """ + log.info( "test tablets not empty:${tablets}") + fetchDataSize(sizes, tablets[0]) + retry = 100 + while (sizes[1] == 0 && retry-- > 0) { + log.info( "test remote size is zero, sleep 10s") + sleep(10000) + tablets = sql_return_maparray """ + SHOW TABLETS FROM ${tableName} + """ + fetchDataSize(sizes, tablets[0]) + } + assertTrue(sizes[1] != 0, "remote size is still zero, maybe some error occurred") + assertTrue(tablets.size() > 0) + log.info( "test remote size not zero") + + // alter resource to correct ak/sk + sql """ truncate table ${tableName} """ + sql """ALTER RESOURCE "${resourceName}" + PROPERTIES( + "AWS_ACCESS_KEY" = "${awsAccessKey}", + "AWS_SECRET_KEY" = "${awsSecretKey}" + );""" + + result = sql """ SHOW RESOURCES WHERE NAME = "${resourceName}"; """ + log.info("result:${result}") + assertTrue(!result.toString().contains(roleArn)) + if (!externalId.isEmpty()) { + assertTrue(!result.toString().contains(externalId)) + } + assertTrue(result.toString().contains(awsAccessKey)); + + // report_tablet_interval_seconds = 60 sleep wait for report policy + sleep(120 * 1000) + + sql """insert into ${tableName}(siteid, citycode, username, pv) values (1, 1, "xxx", 1), + (4, 4, "yyy", 5), + (5, 5, "zzz", 5), + (6, 6, "xxx", 6), + (7, 7, "aaa", 8) + """ + + sizes = [-1, -1] + tablets = sql_return_maparray """ + SHOW TABLETS FROM ${tableName} + """ + log.info( "test tablets not empty:${tablets}") + fetchDataSize(sizes, tablets[0]) + retry = 100 + while (sizes[1] == 0 && retry-- > 0) { + log.info( "test remote size is zero, sleep 10s") + sleep(10000) + tablets = sql_return_maparray """ + SHOW TABLETS FROM ${tableName} + """ + fetchDataSize(sizes, tablets[0]) + } + assertTrue(sizes[1] != 0, "remote size is still zero, maybe some error occurred") + assertTrue(tablets.size() > 0) + log.info( "test remote size not zero") +} diff --git a/regression-test/suites/object_storage_iam_p0/test_alter_storage_vault_iam.groovy b/regression-test/suites/object_storage_iam_p0/test_alter_storage_vault_iam.groovy new file mode 100644 index 00000000000000..b6c80a6b0c9733 --- /dev/null +++ b/regression-test/suites/object_storage_iam_p0/test_alter_storage_vault_iam.groovy @@ -0,0 +1,158 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.util.ObjectStorageIamTestUtils + +suite("test_alter_storage_vault_iam") { + if (!isCloudMode()) { + logger.info("skip ${name} case, because not cloud mode") + return + } + + if (!enableStoragevault()) { + logger.info("skip ${name} case, because storage vault not enabled") + return + } + def config = ObjectStorageIamTestUtils.getConfig(context.config.otherConfigs) + if (config == null) { + logger.info("skip ${name} because objectStorageIamProvider is not configured") + return + } + + def randomStr = UUID.randomUUID().toString().replace("-", "") + def s3VaultName = "object_storage_iam_" + randomStr + def firstAuthCase = config.authCases[0] + def awsAccessKey = context.config.awsAccessKey + def awsSecretKey = context.config.awsSecretKey + + sql """ + CREATE STORAGE VAULT IF NOT EXISTS ${s3VaultName} + PROPERTIES ( + "type"="S3", + ${firstAuthCase.storageSqlProperties}, + "s3.root.path" = "${config.prefix}/test_alter_storage_vault_iam/${s3VaultName}", + "s3.bucket" = "${config.bucket}", + "s3.external_endpoint" = "", + "use_path_style" = "false" + ); + """ + + sql """ + CREATE TABLE ${s3VaultName} ( + C_CUSTKEY INTEGER NOT NULL, + C_NAME INTEGER NOT NULL + ) + DUPLICATE KEY(C_CUSTKEY, C_NAME) + DISTRIBUTED BY HASH(C_CUSTKEY) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "storage_vault_name" = ${s3VaultName} + ) + """ + sql """ insert into ${s3VaultName} values(1, 1); """ + sql """ sync;""" + def result = sql """ select * from ${s3VaultName}; """ + assertEquals(result.size(), 1); + + def assertVaultProperties = { Map expected, List unexpected -> + def vaultInfos = sql "SHOW STORAGE VAULTS" + def vaultInfo = vaultInfos.find { it[0].equals(s3VaultName) } + assertTrue(vaultInfo != null, "storage vault ${s3VaultName} was not found") + def newProperties = vaultInfo[2] + logger.info("newProperties: ${newProperties}") + expected.each { key, value -> + assertTrue(newProperties.contains(value), "${key} was not updated") + } + unexpected.findAll { it != null && !it.isEmpty() }.each { value -> + assertFalse(newProperties.contains(value), "stale credential property was not removed") + } + } + + def insertAndAssert = { int rowCount -> + sql "insert into ${s3VaultName} values(${rowCount}, ${rowCount})" + sql "sync" + result = sql "select * from ${s3VaultName}" + assertEquals(result.size(), rowCount) + } + + if (config.provider == "AWS") { + sql """ + ALTER STORAGE VAULT ${s3VaultName} + PROPERTIES ( + "type"="S3", + "s3.access_key" = "${awsAccessKey}", + "s3.secret_key" = "${awsSecretKey}" + ); + """ + assertVaultProperties(["s3.access_key": awsAccessKey], [config.roleArn]) + insertAndAssert(2) + + sql """ + ALTER STORAGE VAULT ${s3VaultName} + PROPERTIES ( + "type"="S3", + ${firstAuthCase.authProperties} + ); + """ + assertVaultProperties(firstAuthCase.properties, [awsAccessKey]) + insertAndAssert(3) + } else { + def impersonationAccounts = config.authCases.collect { + it.properties.get("gs.impersonation_service_account") + }.findAll { it != null } + // Omitted authentication is a CREATE default, not a request to reset credentials on ALTER. + // Exercise a storage-only update before switching to the explicitly configured providers. + sql """ + ALTER STORAGE VAULT ${s3VaultName} PROPERTIES ( + "type" = "S3", + "use_path_style" = "true" + ) + """ + assertVaultProperties(firstAuthCase.properties ?: ["gs.credential_provider_type": "DEFAULT"], []) + insertAndAssert(2) + + def explicitCases = config.authCases.findAll { !it.properties.isEmpty() } + def transitionCases = explicitCases.isEmpty() ? [] : explicitCases + [explicitCases[0]] + transitionCases.eachWithIndex { authCase, index -> + logger.info("alter ${name} to ${authCase.name}") + def alterProperties = new LinkedHashMap(authCase.properties) + // A direct-credential transition must explicitly clear the old impersonation target. + alterProperties.putIfAbsent("gs.impersonation_service_account", "") + sql """ + ALTER STORAGE VAULT ${s3VaultName} + PROPERTIES ( + "type"="S3", + ${ObjectStorageIamTestUtils.toSqlProperties(alterProperties)} + ); + """ + def unexpected = authCase.properties.containsKey("gs.impersonation_service_account") + ? [] : impersonationAccounts + assertVaultProperties(authCase.properties, unexpected) + insertAndAssert(index * 2 + 3) + + // Unrelated ALTERs must preserve COMPUTE_ENGINE and impersonation as well as DEFAULT. + sql """ + ALTER STORAGE VAULT ${s3VaultName} PROPERTIES ( + "type" = "S3", + "use_path_style" = "${index % 2 == 0 ? 'false' : 'true'}" + ) + """ + assertVaultProperties(authCase.properties, unexpected) + insertAndAssert(index * 2 + 4) + } + } +} diff --git a/regression-test/suites/object_storage_iam_p0/test_backup_restore_iam.groovy b/regression-test/suites/object_storage_iam_p0/test_backup_restore_iam.groovy new file mode 100644 index 00000000000000..1d14c95705409c --- /dev/null +++ b/regression-test/suites/object_storage_iam_p0/test_backup_restore_iam.groovy @@ -0,0 +1,101 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.util.ObjectStorageIamTestUtils + +suite("test_backup_restore_iam") { + if (isCloudMode()) { + logger.info("skip ${name} case, because cloud mode not support") + return + } + def config = ObjectStorageIamTestUtils.getConfig(context.config.otherConfigs) + if (config == null) { + logger.info("skip ${name} because objectStorageIamProvider is not configured") + return + } + + String suiteName = "test_backup_restore_iam" + + String dbName = "${suiteName}_db" + String tableName = "${suiteName}_table" + + def syncer = getSyncer() + + sql "CREATE DATABASE IF NOT EXISTS ${dbName}" + sql "DROP TABLE IF EXISTS ${dbName}.${tableName} FORCE;" + sql """ + CREATE TABLE ${dbName}.${tableName} ( + `id` LARGEINT NOT NULL, + `count` LARGEINT SUM DEFAULT "0") + AGGREGATE KEY(`id`) + DISTRIBUTED BY HASH(`id`) BUCKETS 2 + PROPERTIES + ( + "replication_num" = "1" + ) + """ + + List values = [] + for (int i = 1; i <= 10; ++i) { + values.add("(${i}, ${i})") + } + sql "INSERT INTO ${dbName}.${tableName} VALUES ${values.join(",")}" + def result = sql "SELECT * FROM ${dbName}.${tableName}" + assertEquals(result.size(), values.size()); + + config.authCases.each { authCase -> + String suffix = UUID.randomUUID().toString().replace("-", "") + String repoName = "obj_iam_${authCase.name}_${suffix}" + String snapshotName = "obj_iam_${authCase.name}_${suffix.substring(0, 8)}" + + logger.info("run ${name} with ${authCase.name}") + syncer.createObjectStorageIamRepository(repoName, config, authCase) + + sql """ + BACKUP SNAPSHOT ${dbName}.${snapshotName} + TO `${repoName}` + ON (${tableName}) + """ + + syncer.waitSnapshotFinish(dbName) + + def snapshot = syncer.getSnapshotTimestamp(repoName, snapshotName) + assertTrue(snapshot != null) + + sql "TRUNCATE TABLE ${dbName}.${tableName}" + + sql """ + RESTORE SNAPSHOT ${dbName}.${snapshotName} + FROM `${repoName}` + ON (`${tableName}`) + PROPERTIES + ( + "backup_timestamp" = "${snapshot}", + "reserve_replica" = "true" + ) + """ + + syncer.waitAllRestoreFinish(dbName) + + result = sql "SELECT * FROM ${dbName}.${tableName}" + assertEquals(values.size(), result.size(), authCase.name) + sql "DROP REPOSITORY `${repoName}`" + } + + sql "DROP TABLE ${dbName}.${tableName} FORCE" + sql "DROP DATABASE ${dbName} FORCE" +} diff --git a/regression-test/suites/object_storage_iam_p0/test_export_iam.groovy b/regression-test/suites/object_storage_iam_p0/test_export_iam.groovy new file mode 100644 index 00000000000000..046f909b22f2b4 --- /dev/null +++ b/regression-test/suites/object_storage_iam_p0/test_export_iam.groovy @@ -0,0 +1,108 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import com.google.common.base.Strings +import org.apache.doris.regression.util.ObjectStorageIamTestUtils + +suite("test_export_iam") { + def config = ObjectStorageIamTestUtils.getConfig(context.config.otherConfigs) + if (config == null) { + logger.info("skip ${name} because objectStorageIamProvider is not configured") + return + } + + def cases = config.authCases.collect { authCase -> + return [ + name: authCase.name, + properties: """ + ${authCase.storageSqlProperties} + """ + ] + } + + def tableName = "test_export_iam" + sql "DROP TABLE IF EXISTS ${tableName} FORCE" + sql """ + CREATE TABLE ${tableName} ( + siteid INT DEFAULT '10', + citycode SMALLINT NOT NULL, + username VARCHAR(32) DEFAULT '', + pv BIGINT SUM DEFAULT '0' + ) + AGGREGATE KEY(siteid, citycode, username) + DISTRIBUTED BY HASH(siteid) BUCKETS 1 + PROPERTIES("replication_num" = "1") + """ + sql """INSERT INTO ${tableName}(siteid, citycode, username, pv) VALUES + (1, 1, "xxx", 1), (2, 2, "yyy", 2), (3, 3, "zzz", 3)""" + + cases.each { testCase -> + def randomStr = UUID.randomUUID().toString().replace("-", "") + def label = "iam_export_${testCase.name}_${randomStr}" + // Keep cleanup inside this run's UUID directory. + def exportPrefix = "${config.prefix}/test_export_iam/${testCase.name}/${randomStr}/" + + logger.info("run ${name} with ${testCase.name}") + sql """ + EXPORT TABLE ${tableName} TO "${config.scheme}://${config.bucket}/${exportPrefix}" + PROPERTIES( + "label" = "${label}", + "format" = "csv", + "column_separator" = ",", + "delete_existing_files" = "true" + ) + WITH S3 ( + ${testCase.properties} + ) + """ + + def maxTryMs = 600000 + def exportedUrl = "" + while (maxTryMs > 0) { + String[][] exportResult = sql "SHOW EXPORT WHERE LABEL = '${label}'" + if (exportResult[0][2] == "FINISHED") { + def json = parseJson(exportResult[0][11]) + assert json instanceof List + assertEquals("1", json.fileNumber[0][0], testCase.name) + exportedUrl = json.url[0][0] + break + } + if (exportResult[0][2] == "CANCELLED") { + assertTrue(false, "Export ${label} cancelled for ${testCase.name}: ${exportResult}") + } + Thread.sleep(5000) + maxTryMs -= 5000 + } + assertTrue(maxTryMs > 0, "Export ${label} timeout for ${testCase.name}") + assertFalse(Strings.isNullOrEmpty(exportedUrl), "Export returned no URL for ${testCase.name}") + + def s3Prefix = "s3://${config.bucket}/" + assertTrue(exportedUrl.startsWith(s3Prefix), "Unexpected export URL: ${exportedUrl}") + def exportedFile = "${config.scheme}://${config.bucket}/" + + "${exportedUrl.substring(s3Prefix.length())}0.csv" + def result = sql """ + SELECT COUNT(*) FROM s3( + "uri" = "${exportedFile}", + ${testCase.properties}, + "format" = "csv", + "column_separator" = ",", + "use_path_style" = "false" + ) + """ + assertEquals(3, result[0][0], testCase.name) + } +} diff --git a/regression-test/suites/object_storage_iam_p0/test_external_catalog_iam.groovy b/regression-test/suites/object_storage_iam_p0/test_external_catalog_iam.groovy new file mode 100644 index 00000000000000..4d26375c0b19e3 --- /dev/null +++ b/regression-test/suites/object_storage_iam_p0/test_external_catalog_iam.groovy @@ -0,0 +1,81 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.util.ObjectStorageIamTestUtils + +suite("test_external_catalog_iam") { + def config = ObjectStorageIamTestUtils.getConfig(context.config.otherConfigs) + if (config == null) { + logger.info("skip ${name} because objectStorageIamProvider is not configured") + return + } + + def schemes = config.provider == "GCP" ? ["gs", "s3", "s3a"] : [config.scheme] + def cases = config.authCases.collectMany { authCase -> + schemes.collect { scheme -> + return [ + name: "${authCase.name}_${scheme}", + warehouse: "${scheme}://${config.bucket}/${config.prefix}" + + "/test_external_catalog_iam/${authCase.name}_${scheme}", + properties: """ + ${authCase.storageSqlProperties} + """ + ] + } + } + + cases.eachWithIndex { testCase, index -> + def randomStr = UUID.randomUUID().toString().replace("-", "") + def catalogName = "iam_iceberg_${index}_${randomStr}" + def databaseName = "iam_db" + def tableName = "iam_table" + + logger.info("run ${name} with ${testCase.name}") + try { + sql """ + CREATE CATALOG ${catalogName} PROPERTIES ( + "type" = "iceberg", + "iceberg.catalog.type" = "hadoop", + "warehouse" = "${testCase.warehouse}/${randomStr}", + ${testCase.properties} + ) + """ + sql "CREATE DATABASE ${catalogName}.${databaseName}" + def databases = sql "SHOW DATABASES FROM ${catalogName}" + assertTrue(databases.any { it[0] == databaseName }, testCase.name) + sql """ + CREATE TABLE ${catalogName}.${databaseName}.${tableName} ( + id INT, + value STRING + ) + PROPERTIES("file_format" = "parquet") + """ + sql "INSERT INTO ${catalogName}.${databaseName}.${tableName} VALUES (1, 'one'), (2, 'two')" + def result = sql "SELECT COUNT(*) FROM ${catalogName}.${databaseName}.${tableName}" + assertEquals(2, result[0][0], testCase.name) + def rows = sql "SELECT id, value FROM ${catalogName}.${databaseName}.${tableName} ORDER BY id" + assertEquals([[1, 'one'], [2, 'two']], rows, testCase.name) + sql "DROP TABLE ${catalogName}.${databaseName}.${tableName}" + sql "DROP DATABASE ${catalogName}.${databaseName}" + // Exercise namespace deletion and recreation through Hadoop FileSystem as well. + sql "CREATE DATABASE ${catalogName}.${databaseName}" + sql "DROP DATABASE ${catalogName}.${databaseName}" + } finally { + sql "DROP CATALOG IF EXISTS ${catalogName}" + } + } +} diff --git a/regression-test/suites/object_storage_iam_p0/test_gcp_iam_property_validation.groovy b/regression-test/suites/object_storage_iam_p0/test_gcp_iam_property_validation.groovy new file mode 100644 index 00000000000000..2233a0b881112a --- /dev/null +++ b/regression-test/suites/object_storage_iam_p0/test_gcp_iam_property_validation.groovy @@ -0,0 +1,131 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_gcp_iam_property_validation") { + ["ADC", "", "unknown"].each { providerType -> + test { + sql """ + SELECT * FROM s3( + "uri" = "gs://validation-only-bucket/file.csv", + "provider" = "GCP", + "gs.credential_provider_type" = "${providerType}", + "format" = "csv" + ) LIMIT 1 + """ + exception "gs.credential_provider_type" + } + } + + [" ", "invalid-account"].each { account -> + test { + sql """ + SELECT * FROM s3( + "uri" = "gs://validation-only-bucket/file.csv", + "provider" = "GCP", + "gs.impersonation_service_account" = "${account}", + "format" = "csv" + ) LIMIT 1 + """ + exception "Invalid GCP service account email" + } + } + + [ + "gs.access_key": "gcp-access-key", + "gs.secret_key": "gcp-secret-key", + "gs.session_token": "gcp-session-token", + "s3.access_key": "access-key", + "s3.secret_key": "secret-key", + "s3.session_token": "session-token", + "s3.role_arn": "arn:aws:iam::123456789012:role/test-role", + "s3.external_id": "external-id", + "s3.credentials_provider_type": "INSTANCE_PROFILE" + ].each { key, value -> + test { + sql """ + SELECT * FROM s3( + "uri" = "gs://validation-only-bucket/file.csv", + "provider" = "GCP", + "gs.credential_provider_type" = "DEFAULT", + "${key}" = "${value}", + "format" = "csv" + ) LIMIT 1 + """ + exception "cannot be used together" + } + } + + test { + sql """ + SELECT * FROM s3( + "uri" = "gs://validation-only-bucket/file.csv", + "provider" = "S3", + "gs.credential_provider_type" = "DEFAULT", + "format" = "csv" + ) LIMIT 1 + """ + exception "requires provider=GCP" + } + + [ + "gs.access_key": "access-key", + "gs.secret_key": "secret-key", + "gs.session_token": "session-token", + "s3.role_arn": "arn:aws:iam::123456789012:role/test-role" + ].each { key, value -> + test { + sql """ + SELECT * FROM s3( + "uri" = "gs://validation-only-bucket/file.csv", + "provider" = "GCP", + "gs.credential_provider_type" = "ANONYMOUS", + "${key}" = "${value}", + "format" = "csv" + ) LIMIT 1 + """ + exception "cannot be used together" + } + } + + test { + sql """ + SELECT * FROM s3( + "uri" = "gs://validation-only-bucket/file.csv", + "gs.credential_provider_type" = "ANONYMOUS", + "gs.impersonation_service_account" = "target@project.iam.gserviceaccount.com", + "format" = "csv" + ) LIMIT 1 + """ + exception "ANONYMOUS cannot be used with gs.impersonation_service_account" + } + + // Implicit DEFAULT must enforce the same conflicts as explicit DEFAULT, before any remote I/O. + ["gs.session_token", "s3.role_arn", "s3.credentials_provider_type"].each { key -> + test { + sql """ + SELECT * FROM s3( + "uri" = "gs://validation-only-bucket/file.csv", + "provider" = "GCP", + "${key}" = "conflicting-credential", + "format" = "csv" + ) LIMIT 1 + """ + exception "cannot be used together" + } + } + +} diff --git a/regression-test/suites/object_storage_iam_p0/test_gcp_tvf_anonymous.groovy b/regression-test/suites/object_storage_iam_p0/test_gcp_tvf_anonymous.groovy new file mode 100644 index 00000000000000..459bcb4959245d --- /dev/null +++ b/regression-test/suites/object_storage_iam_p0/test_gcp_tvf_anonymous.groovy @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.util.ObjectStorageIamTestUtils + +suite("test_gcp_tvf_anonymous") { + def uri = context.config.otherConfigs.get("objectStorageIamGcpAnonymousUri")?.toString()?.trim() + if (!uri) { + logger.info("skip ${name} because objectStorageIamGcpAnonymousUri is not configured") + return + } + assertTrue(uri.startsWith("gs://"), "The anonymous fixture must be a gs:// URI") + def expectedRows = context.config.otherConfigs.get("objectStorageIamGcpAnonymousExpectedRows") + ?.toString()?.trim() + assertTrue(expectedRows != null && expectedRows.isLong() && expectedRows.toLong() > 0, + "Configure a positive objectStorageIamGcpAnonymousExpectedRows") + + // Each query exercises FE object discovery/schema inference and a BE scan of the public file. + // Include the legacy S3-compatible spelling and provider-value case/whitespace normalization. + [ + ["gs.credential_provider_type": "ANONYMOUS"], + ["gs.credential_provider_type": " anonymous "], + ["s3.credentials_provider_type": "ANONYMOUS"] + ].each { authProperties -> + def result = sql """ + SELECT COUNT(*) FROM s3( + "uri" = "${uri}", + "provider" = "GCP", + ${ObjectStorageIamTestUtils.toSqlProperties(authProperties)}, + "format" = "csv", + "compress_type" = "gz", + "column_separator" = "|" + ) + """ + assertEquals(expectedRows.toLong(), result[0][0], authProperties.toString()) + } +} diff --git a/regression-test/suites/object_storage_iam_p0/test_gcp_vault_anonymous_validation.groovy b/regression-test/suites/object_storage_iam_p0/test_gcp_vault_anonymous_validation.groovy new file mode 100644 index 00000000000000..f3d699e678d1a1 --- /dev/null +++ b/regression-test/suites/object_storage_iam_p0/test_gcp_vault_anonymous_validation.groovy @@ -0,0 +1,42 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_gcp_vault_anonymous_validation") { + if (!isCloudMode() || !enableStoragevault()) { + logger.info("skip ${name} because cloud storage vaults are not enabled") + return + } + + ["gs.credential_provider_type", "s3.credentials_provider_type", "AWS_CREDENTIALS_PROVIDER_TYPE"].each { key -> + def vaultName = "gcp_anonymous_${UUID.randomUUID().toString().replace('-', '')}" + test { + sql """ + CREATE STORAGE VAULT ${vaultName} PROPERTIES ( + "type" = "S3", + "provider" = "GCP", + "s3.endpoint" = "storage.googleapis.com", + "s3.region" = "us-east1", + "s3.bucket" = "validation-only-bucket", + "s3.root.path" = "validation-only-root", + "s3_validity_check" = "false", + "${key}" = " anonymous " + ) + """ + exception "Anonymous GCS authentication is not supported for storage vaults" + } + } +} diff --git a/regression-test/suites/object_storage_iam_p0/test_load_iam.groovy b/regression-test/suites/object_storage_iam_p0/test_load_iam.groovy new file mode 100644 index 00000000000000..e9b1517305f4db --- /dev/null +++ b/regression-test/suites/object_storage_iam_p0/test_load_iam.groovy @@ -0,0 +1,106 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.util.ObjectStorageIamTestUtils + +suite("test_load_iam") { + def config = ObjectStorageIamTestUtils.getConfig(context.config.otherConfigs) + if (config == null) { + logger.info("skip ${name} because objectStorageIamProvider is not configured") + return + } + + def cases = config.authCases.collect { authCase -> + return [ + name: authCase.name, + properties: """ + ${authCase.storageSqlProperties} + """ + ] + } + if (config.provider == "AWS") { + cases.add(0, [ + name: "aws_role_legacy_properties", + properties: """ + "AWS_ENDPOINT" = "${config.endpoint}", + "AWS_REGION" = "${config.region}", + "AWS_ROLE_ARN" = "${config.roleArn}", + "AWS_EXTERNAL_ID" = "${config.externalId ?: ''}" + """ + ]) + cases[1].name = "aws_role_s3_properties" + } + + cases.eachWithIndex { testCase, index -> + def randomStr = UUID.randomUUID().toString().replace("-", "") + def loadLabel = "iam_load_${index}_${randomStr}" + def tableName = "test_iam_load_${index}_${randomStr}" + + sql "DROP TABLE IF EXISTS ${tableName} FORCE" + sql """ + CREATE TABLE ${tableName} ( + C_CUSTKEY INTEGER NOT NULL, + C_NAME VARCHAR(25) NOT NULL, + C_ADDRESS VARCHAR(40) NOT NULL, + C_NATIONKEY INTEGER NOT NULL, + C_PHONE CHAR(15) NOT NULL, + C_ACCTBAL DECIMAL(15, 2) NOT NULL, + C_MKTSEGMENT CHAR(10) NOT NULL, + C_COMMENT VARCHAR(117) NOT NULL + ) + DUPLICATE KEY(C_CUSTKEY, C_NAME) + DISTRIBUTED BY HASH(C_CUSTKEY) BUCKETS 1 + PROPERTIES("replication_num" = "1") + """ + + logger.info("run ${name} with ${testCase.name}") + sql """ + LOAD LABEL ${loadLabel} ( + DATA INFILE("${config.scheme}://${config.bucket}/${config.dataPath}") + INTO TABLE ${tableName} + COLUMNS TERMINATED BY "|" + (c_custkey, c_name, c_address, c_nationkey, c_phone, c_acctbal, c_mktsegment, c_comment) + ) + WITH S3 ( + ${testCase.properties}, + "compress_type" = "GZ" + ) + PROPERTIES( + "timeout" = "600", + "exec_mem_limit" = "8589934592" + ) + """ + + def maxTryMs = 600000 + while (maxTryMs > 0) { + String[][] loadResult = sql """SHOW LOAD WHERE LABEL = "${loadLabel}" + ORDER BY CREATETIME DESC LIMIT 1""" + if (loadResult[0][2] == "FINISHED") { + break + } + if (loadResult[0][2] == "CANCELLED") { + assertTrue(false, "Load ${loadLabel} cancelled for ${testCase.name}: ${loadResult}") + } + Thread.sleep(5000) + maxTryMs -= 5000 + } + assertTrue(maxTryMs > 0, "Load ${loadLabel} timeout for ${testCase.name}") + + def result = sql "SELECT COUNT(*) FROM ${tableName}" + assertEquals(1500, result[0][0], testCase.name) + } +} diff --git a/regression-test/suites/object_storage_iam_p0/test_resource_iam.groovy b/regression-test/suites/object_storage_iam_p0/test_resource_iam.groovy new file mode 100644 index 00000000000000..7433179e902fbe --- /dev/null +++ b/regression-test/suites/object_storage_iam_p0/test_resource_iam.groovy @@ -0,0 +1,134 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import groovy.json.JsonSlurper +import org.apache.doris.regression.util.ObjectStorageIamTestUtils + +suite("test_resource_iam") { + if (isCloudMode()) { + logger.info("skip ${name} case, because cloud mode not support") + return + } + def config = ObjectStorageIamTestUtils.getConfig(context.config.otherConfigs) + if (config == null) { + logger.info("skip ${name} because objectStorageIamProvider is not configured") + return + } + + def randomStr = UUID.randomUUID().toString().replace("-", "") + def testObjects = [] + config.authCases.eachWithIndex { authCase, index -> + def tableName = "test_resource_iam_${index}" + def resourceName = "resource_${authCase.name}_${randomStr}" + def policyName = "policy_${authCase.name}_${randomStr}" + + logger.info("run ${name} with ${authCase.name}") + sql """ + CREATE RESOURCE IF NOT EXISTS "${resourceName}" + PROPERTIES( + "type" = "s3", + ${authCase.storageSqlProperties}, + "s3.bucket" = "${config.bucket}", + "s3.root.path" = "${config.prefix}/test_resource_iam/${authCase.name}/${randomStr}", + "s3.connection.maximum" = "50", + "s3.connection.request.timeout" = "3000", + "s3.connection.timeout" = "1000", + "s3_validity_check" = "true" + ); + """ + + sql """ + CREATE STORAGE POLICY IF NOT EXISTS ${policyName} + PROPERTIES( + "storage_resource" = "${resourceName}", + "cooldown_ttl" = "1" + ) + """ + + sql "DROP TABLE IF EXISTS ${tableName} FORCE" + sql """ + CREATE TABLE ${tableName} + ( + siteid INT DEFAULT '10', + citycode SMALLINT NOT NULL, + username VARCHAR(32) DEFAULT '', + pv BIGINT SUM DEFAULT '0' + ) + AGGREGATE KEY(siteid, citycode, username) + DISTRIBUTED BY HASH(siteid) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1", + "storage_policy" = "${policyName}" + ) + """ + + sql """insert into ${tableName}(siteid, citycode, username, pv) values (1, 1, "xxx", 1), + (2, 2, "yyy", 2), + (3, 3, "zzz", 3) + """ + testObjects.add([name: authCase.name, tableName: tableName]) + } + + // data_sizes is one arrayList, t is tablet + def fetchDataSize = {List data_sizes, Map t -> + def tabletId = t.TabletId + def meta_url = t.MetaUrl + def clos = { respCode, body -> + logger.info("test ttl expired resp Code {}", "${respCode}".toString()) + assertEquals("${respCode}".toString(), "200") + String out = "${body}".toString() + def obj = new JsonSlurper().parseText(out) + data_sizes[0] = obj.local_data_size + data_sizes[1] = obj.remote_data_size + } + meta_url = meta_url.replace("header", "data_size") + + def i = meta_url.indexOf("/api") + def endPoint = meta_url.substring(0, i) + def metaUri = meta_url.substring(i) + logger.info("test fetchBeHttp, endpoint:${endPoint}, metaUri:${metaUri}") + i = endPoint.lastIndexOf('/') + endPoint = endPoint.substring(i + 1) + + httpTest { + endpoint {endPoint} + uri metaUri + op "get" + check clos + } + } + + sleep(60000) + + testObjects.each { testObject -> + List sizes = [-1, -1] + def tablets = sql_return_maparray "SHOW TABLETS FROM ${testObject.tableName}" + log.info("test tablets not empty for ${testObject.name}: ${tablets}") + fetchDataSize(sizes, tablets[0]) + def retry = 100 + while (sizes[1] == 0 && retry-- > 0) { + log.info("test remote size is zero for ${testObject.name}, sleep 10s") + sleep(10000) + tablets = sql_return_maparray "SHOW TABLETS FROM ${testObject.tableName}" + fetchDataSize(sizes, tablets[0]) + } + assertTrue(sizes[1] != 0, + "remote size is still zero for ${testObject.name}, maybe some error occurred") + assertTrue(tablets.size() > 0) + log.info("test remote size not zero for ${testObject.name}") + } +} diff --git a/regression-test/suites/object_storage_iam_p0/test_select_into_outfile_iam.groovy b/regression-test/suites/object_storage_iam_p0/test_select_into_outfile_iam.groovy new file mode 100644 index 00000000000000..e71067ab2879b9 --- /dev/null +++ b/regression-test/suites/object_storage_iam_p0/test_select_into_outfile_iam.groovy @@ -0,0 +1,80 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.util.ObjectStorageIamTestUtils + +suite("test_select_into_outfile_iam") { + def config = ObjectStorageIamTestUtils.getConfig(context.config.otherConfigs) + if (config == null) { + logger.info("skip ${name} because objectStorageIamProvider is not configured") + return + } + + def randomStr = UUID.randomUUID().toString().replace("-", "") + def tableName = "test_select_into_outfile_iam" + + sql """ drop table if exists ${tableName} force;""" + sql """ + CREATE TABLE ${tableName} + ( + siteid INT DEFAULT '10', + citycode SMALLINT NOT NULL, + username VARCHAR(32) DEFAULT '', + pv BIGINT SUM DEFAULT '0' + ) + AGGREGATE KEY(siteid, citycode, username) + DISTRIBUTED BY HASH(siteid) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1" + ) + """ + + sql """insert into ${tableName}(siteid, citycode, username, pv) values (1, 1, "xxx", 1), + (2, 2, "yyy", 2), + (3, 3, "zzz", 3) + """ + sql """sync;""" + + def expectedRows = sql """ + SELECT CAST(siteid AS STRING), CAST(citycode AS STRING), username, CAST(pv AS STRING) + FROM ${tableName} ORDER BY siteid + """ + + config.authCases.each { authCase -> + logger.info("run ${name} with ${authCase.name}") + def outfilePath = "${config.scheme}://${config.bucket}/${config.prefix}/" + + "test_select_into_outfile_iam/${authCase.name}/${randomStr}" + sql """ + SELECT * FROM ${tableName} + INTO OUTFILE "${outfilePath}" + FORMAT AS CSV + PROPERTIES( + "column_separator" = ",", + ${authCase.storageSqlProperties} + ); + """ + def exportedRows = sql """ + SELECT c1, c2, c3, c4 FROM s3( + "uri" = "${outfilePath}*.csv", + ${authCase.storageSqlProperties}, + "format" = "csv", + "column_separator" = "," + ) ORDER BY CAST(c1 AS INT) + """ + assertEquals(expectedRows, exportedRows, authCase.name) + } +} diff --git a/regression-test/suites/object_storage_iam_p0/test_storage_vault_iam.groovy b/regression-test/suites/object_storage_iam_p0/test_storage_vault_iam.groovy new file mode 100644 index 00000000000000..9f89873ef2cd37 --- /dev/null +++ b/regression-test/suites/object_storage_iam_p0/test_storage_vault_iam.groovy @@ -0,0 +1,80 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.util.ObjectStorageIamTestUtils + +suite("test_storage_vault_iam") { + if (!isCloudMode()) { + logger.info("skip ${name} because it requires cloud mode") + return + } + if (!enableStoragevault()) { + logger.info("skip ${name} because storage vault is disabled") + return + } + + def config = ObjectStorageIamTestUtils.getConfig(context.config.otherConfigs) + if (config == null) { + logger.info("skip ${name} because objectStorageIamProvider is not configured") + return + } + + def cases = config.authCases.collect { authCase -> + return [ + name: authCase.name, + properties: """ + ${authCase.storageSqlProperties}, + "s3.bucket" = "${config.bucket}", + "s3.external_endpoint" = "", + "use_path_style" = "false" + """ + ] + } + + cases.eachWithIndex { testCase, index -> + def randomStr = UUID.randomUUID().toString().replace("-", "") + def vaultName = "iam_vault_${index}_${randomStr}" + def tableName = "test_iam_vault_${index}_${randomStr}" + + logger.info("run ${name} with ${testCase.name}") + sql """ + CREATE STORAGE VAULT ${vaultName} + PROPERTIES ( + "type" = "S3", + "s3.root.path" = "${config.prefix}/test_storage_vault_iam/${testCase.name}/${randomStr}", + ${testCase.properties} + ) + """ + sql """ + CREATE TABLE ${tableName} ( + id INT NOT NULL, + value INT NOT NULL + ) + DUPLICATE KEY(id, value) + DISTRIBUTED BY HASH(id) BUCKETS 1 + PROPERTIES( + "replication_num" = "1", + "storage_vault_name" = "${vaultName}" + ) + """ + sql "INSERT INTO ${tableName} VALUES (1, 1)" + sql "SYNC" + def result = sql "SELECT * FROM ${tableName}" + assertEquals(1, result.size(), testCase.name) + assertEquals(1, result[0][0], testCase.name) + } +} diff --git a/regression-test/suites/object_storage_iam_p0/test_tvf_anonymous.groovy b/regression-test/suites/object_storage_iam_p0/test_tvf_anonymous.groovy new file mode 100644 index 00000000000000..5cc3fd392f0957 --- /dev/null +++ b/regression-test/suites/object_storage_iam_p0/test_tvf_anonymous.groovy @@ -0,0 +1,77 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + + +import com.google.common.base.Strings; +import org.apache.doris.regression.util.ObjectStorageIamTestUtils + +suite("test_tvf_anonymous") { + if (!ObjectStorageIamTestUtils.isProvider(context.config.otherConfigs, "AWS")) { + logger.info("skip ${name} because objectStorageIamProvider is not AWS") + return + } + if (Strings.isNullOrEmpty(context.config.otherConfigs.get("enableTestTvfAnonymous"))) { + return + } + + + def region = context.config.otherConfigs.get("anymousS3Region") + def uri = context.config.otherConfigs.get("anymousS3Uri") + def expectDataCount = context.config.otherConfigs.get("anymousS3ExpectDataCount"); + //aws_credentials_provider_version + // sql """ ADMIN SET FRONTEND CONFIG ("aws_credentials_provider_version"="v1"); """ + + def result = sql """ + SELECT count(1) FROM S3 ( + "uri"="${uri}", + "format" = "csv", + "s3.region" = "${region}", + "s3.credentials_provider_type"="ANONYMOUS", + "s3.endpoint" = "https://s3.${region}.amazonaws.com", + "column_separator" = "," ); + """ + + def countValue = result[0][0] + assertTrue(countValue == expectDataCount.toInteger()) + sql """ ADMIN SET FRONTEND CONFIG ("aws_credentials_provider_version"="v2"); """ + + result = sql """ + SELECT count(1) FROM S3 ( + "uri"="${uri}", + "format" = "csv", + "s3.region" = "${region}", + "s3.credentials_provider_type"="ANONYMOUS", + "s3.endpoint" = "https://s3.${region}.amazonaws.com", + "column_separator" = "," ); + """ + + countValue = result[0][0] + assertTrue(countValue == expectDataCount.toInteger()) + + result = sql """ + SELECT count(1) FROM S3 ( + "uri"="${uri}", + "format" = "csv", + "s3.region" = "${region}", + "s3.endpoint" = "https://s3.${region}.amazonaws.com", + "s3.credentials_provider_type"="ANONYMOUS", + "column_separator" = "," ); + """ + + countValue = result[0][0] + assertTrue(countValue == expectDataCount.toInteger()) +} diff --git a/regression-test/suites/object_storage_iam_p0/test_tvf_iam.groovy b/regression-test/suites/object_storage_iam_p0/test_tvf_iam.groovy new file mode 100644 index 00000000000000..1a28289e55c035 --- /dev/null +++ b/regression-test/suites/object_storage_iam_p0/test_tvf_iam.groovy @@ -0,0 +1,66 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import org.apache.doris.regression.util.ObjectStorageIamTestUtils + +suite("test_tvf_iam") { + def config = ObjectStorageIamTestUtils.getConfig(context.config.otherConfigs) + if (config == null) { + logger.info("skip ${name} because objectStorageIamProvider is not configured") + return + } + + config.authCases.each { authCase -> + logger.info("run ${name} with ${authCase.name}") + def result = sql """ + SELECT COUNT(*) FROM s3( + "uri" = "${config.scheme}://${config.bucket}/${config.dataPath}", + ${authCase.storageSqlProperties}, + "format" = "csv", + "compress_type" = "gz", + "column_separator" = "|", + "use_path_style" = "false" + ) + """ + assertEquals(1500, result[0][0], authCase.name) + } + + if (config.provider == "GCP" && config.authCases.any { it.name == "gcp_default_omitted" }) { + // The preceding read succeeded with omitted properties. The same private object must not + // become readable with ANONYMOUS, even on a VM with a usable attached service account. + test { + sql """ + SELECT COUNT(*) FROM s3( + "uri" = "gs://${config.bucket}/${config.dataPath}", + "provider" = "GCP", + "gs.endpoint" = "${config.endpoint}", + "gs.credential_provider_type" = "ANONYMOUS", + "format" = "csv", + "compress_type" = "gz", + "column_separator" = "|" + ) + """ + check { result, exception, startTime, endTime -> + assertTrue(exception != null, "Anonymous access unexpectedly read the private GCS fixture") + def message = exception.toString().toLowerCase() + assertTrue(message.contains("403") || message.contains("access denied") + || message.contains("accessdenied") || message.contains("forbidden") + || message.contains("permission denied"), message) + } + } + } +} diff --git a/tools/be-java-plugins/check_plugin_layout.py b/tools/be-java-plugins/check_plugin_layout.py index e8576b9246d3ee..7d489a9152b59a 100755 --- a/tools/be-java-plugins/check_plugin_layout.py +++ b/tools/be-java-plugins/check_plugin_layout.py @@ -216,6 +216,58 @@ } +# gcs-connector 3.1.18 is a shaded uber-jar. Making its Hadoop filesystem a root +# also scans optional transports, test helpers and native-image substitutions. +# Match BOTH ends of these edges: a missing library referenced by Doris or the +# GCS filesystem/authentication implementation must still fail the check. +_GCS = "com.google.cloud.hadoop.repackaged.gcs." +_GCS_NETTY = _GCS + "io.grpc.netty.shaded.io.netty." +_GCS_CODECS = ("Optional Netty codecs bundled in the GCS jar. GCS uses HTTP/gRPC, not " + "SPDY, JBoss marshalling or protobuf-nano. Its gzip/deflate path uses " + "JDK zlib; Brotli and Zstd HTTP handlers probe isAvailable() before use.") +_GCS_ANNOTATIONS = ("Compile-time annotations in the shaded GCS jar; the generated " + "AutoValue implementations are already bundled.") +# Each tuple is (referencing class prefix, missing class prefix, reason). +GCS_CLOSURE_ALLOWLIST = [ + (_GCS + "com.google.api.client.extensions.appengine.http.", "com.google.appengine.api.urlfetch.", + "App Engine's UrlFetch transport; BE uses the normal JVM HTTP transport."), + ("com.google.cloud.hadoop.", "com.google.auto.value.AutoValue", _GCS_ANNOTATIONS), + (_GCS + "com.google.common.flogger.", "org.checkerframework.checker.nullness.compatqual.", + _GCS_ANNOTATIONS), + (_GCS_NETTY + "util.concurrent.", "org.jetbrains.annotations.Async", _GCS_ANNOTATIONS), + (_GCS_NETTY + "util.", "com.oracle.svm.core.annotate.", + "Graal native-image substitutions are not used by BE's JVM."), + (_GCS_NETTY + "util.internal.Hidden$NettyBlockHoundIntegration", "reactor.blockhound.", + "Optional BlockHound integration; BE does not install a BlockHound agent."), + (_GCS_NETTY + "handler.ssl.", "org.bouncycastle.", + "Optional PEM/self-signed certificate helpers. GCS uses trust-store TLS; " + "BouncyCastlePemReader checks availability and JDK PEM parsing is the fallback."), + (_GCS_NETTY + "handler.ssl.JettyAlpnSslEngine", "org.eclipse.jetty.alpn.", + "Legacy Jetty ALPN for old JDKs; BE requires JDK 17 and uses JDK ALPN."), + (_GCS_NETTY + "handler.ssl.JettyNpnSslEngine", "org.eclipse.jetty.npn.", + "Legacy NPN transport; GCS uses HTTP/2 ALPN on JDK 17."), + (_GCS + "io.grpc.testing.", "org.junit.", + "Upstream GrpcCleanupRule/GrpcServerRule test helpers bundled in the shaded runtime jar."), + (_GCS + "io.opentelemetry.sdk.", _GCS + "io.opentelemetry.api.incubator.", + "SdkTracer, SdkLogger and instrument builders probe incubator availability " + "with Class.forName and use the stable implementation when it is absent."), + ("com.google.cloud.hadoop.fs.gcs.auth.GcsDtFetcher", + "com.google.cloud.hadoop.gcsio.GoogleCloudStorageFileSystem", + "Unused pre-shading constant-pool entry in 3.1.18: javap shows no instruction " + "or signature using it. The actual filesystem implementation is relocated."), +] + [(_GCS_NETTY + "handler.codec.", target, _GCS_CODECS) for target in ( + "com.aayushatharva.brotli4j.", "com.github.luben.zstd.", "com.jcraft.jzlib.", + "com.ning.compress.", "lzma.sdk.", "net.jpountz.lz4.", "net.jpountz.xxhash.", + "org.jboss.marshalling.", _GCS + "com.google.protobuf.nano.", +)] + + +def _gcs_optional_reference(plugin, source, target): + return plugin in ("hudi", "iceberg", "paimon") and any( + source.startswith(src) and target.startswith(dst) + for src, dst, _reason in GCS_CLOSURE_ALLOWLIST) + + def _allowed(rules, path): for kind, pattern, _reason in rules: if kind == "basename" and path.rsplit("/", 1)[-1] == pattern: @@ -456,6 +508,8 @@ def _doris_owned(jars): # Entries are class file paths. A name absent from a plugin costs nothing: that plugin simply # bundles no such filesystem, and the root set is what it was. _NAMED_BY_DORIS = ( + # Native GCP Hadoop access in the isolated Hudi, Paimon and Iceberg scanners. + "com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystem.class", # fs.s3.impl / fs.s3a.impl / fs.cos.impl / fs.cosn.impl / fs.gs.impl, and the fallback # fs.obs.impl - one jar, hadoop-aws, and the S3A credential providers live in it too. "org/apache/hadoop/fs/s3a/S3AFileSystem.class", @@ -516,6 +570,8 @@ def check_closure_self_contained(plugin, jars, spi_jar, fail): continue if any(target.startswith(prefix) for prefix, _reason in rules): continue + if _gcs_optional_reference(plugin, src, target): + continue missing[target].add(src) for target in sorted(missing): fail("closure-self-contained", diff --git a/tools/be-java-plugins/test_check_plugin_layout.py b/tools/be-java-plugins/test_check_plugin_layout.py new file mode 100644 index 00000000000000..50a0199819a5d4 --- /dev/null +++ b/tools/be-java-plugins/test_check_plugin_layout.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Regression tests for source-scoped GCS dependency exceptions. + +Run with: python3 -m unittest discover -s tools/be-java-plugins -p 'test_*.py' +""" +import os +import subprocess +import tempfile +import unittest +from unittest import mock +import zipfile + +import check_plugin_layout as layout + + +class GcsClosureTest(unittest.TestCase): + def setUp(self): + self.directory = tempfile.TemporaryDirectory() + self.addCleanup(self.directory.cleanup) + self.doris_jar = self.jar("scanner.jar", ["org/apache/doris/Scanner.class"]) + self.gcs_jar = self.jar("gcs.jar", [ + "com/google/cloud/hadoop/fs/gcs/GoogleHadoopFileSystem.class"]) + self.spi_jar = self.jar("spi.jar", []) + + def jar(self, name, classes): + path = os.path.join(self.directory.name, name) + with zipfile.ZipFile(path, "w") as jar: + for entry in classes: + jar.writestr(entry, b"fixture") + return path + + def check_edges(self, plugin, edges): + output = "\n".join(" %s -> %s not found" % edge for edge in edges) + failures = [] + with mock.patch.object(layout.subprocess, "run", return_value=subprocess.CompletedProcess( + [], 0, stdout=output, stderr="")) as run: + layout.check_closure_self_contained( + plugin, [self.doris_jar, self.gcs_jar], self.spi_jar, + lambda check, message: failures.append((check, message))) + # The GCS jar remains a jdeps root; the fix must not hide the whole jar. + self.assertIn(self.gcs_jar, run.call_args.args[0]) + return failures + + def test_optional_gcs_edges(self): + edges = [ + (layout._GCS + "com.google.api.client.extensions.appengine.http.UrlFetchRequest", + "com.google.appengine.api.urlfetch.HTTPRequest"), + (layout._GCS_NETTY + "handler.codec.compression.BrotliDecoder", + "com.aayushatharva.brotli4j.decoder.DecoderJNI"), + (layout._GCS_NETTY + "handler.ssl.BouncyCastlePemReader", + "org.bouncycastle.openssl.PEMParser"), + (layout._GCS + "io.opentelemetry.sdk.trace.ExtendedSdkTracer", + layout._GCS + "io.opentelemetry.api.incubator.trace.ExtendedTracer"), + (layout._GCS + "io.grpc.testing.GrpcCleanupRule", "org.junit.runners.model.Statement"), + (layout._GCS + "io.grpc.testing.GrpcServerRule", "org.junit.rules.ExternalResource"), + ("com.google.cloud.hadoop.fs.gcs.auth.GcsDtFetcher", + "com.google.cloud.hadoop.gcsio.GoogleCloudStorageFileSystem"), + ] + for plugin in ("hudi", "iceberg", "paimon"): + with self.subTest(plugin=plugin): + self.assertEqual([], self.check_edges(plugin, edges)) + with mock.patch.object(layout, "GCS_CLOSURE_ALLOWLIST", []): + self.assertEqual(len(edges), len(self.check_edges(plugin, edges))) + + def test_same_target_from_runtime_code_still_fails(self): + target = "org.bouncycastle.openssl.PEMParser" + edges = [ + (layout._GCS_NETTY + "handler.ssl.BouncyCastlePemReader", target), + ("org.apache.doris.GcsReader", target), + ] + for plugin in ("hudi", "iceberg", "paimon"): + failures = self.check_edges(plugin, edges) + self.assertEqual(1, len(failures)) + self.assertIn("org.apache.doris.GcsReader", failures[0][1]) + self.assertIn("Either ship the jar", failures[0][1]) + + def test_required_gcs_dependencies_still_fail(self): + edges = [ + ("com.google.cloud.hadoop.fs.gcs.GoogleHadoopFileSystem", + "org.apache.hadoop.fs.FileSystem"), + (layout._GCS + "com.google.cloud.hadoop.gcsio.GoogleCloudStorageImpl", + layout._GCS + "com.google.auth.oauth2.GoogleCredentials"), + (layout._GCS + "com.google.auth.oauth2.ServiceAccountCredentials", + "org.bouncycastle.openssl.PEMParser"), + (layout._GCS_NETTY + "handler.ssl.JdkSslContext", "missing.RequiredTlsClass"), + ] + for plugin in ("hudi", "iceberg", "paimon"): + self.assertEqual(len(edges), len(self.check_edges(plugin, edges))) + + def test_exceptions_do_not_apply_to_other_plugins(self): + edges = [(layout._GCS + "io.grpc.testing.GrpcCleanupRule", "org.junit.rules.ExternalResource")] + self.assertEqual(1, len(self.check_edges("jdbc", edges))) + + def test_jdeps_failure_is_not_ignored(self): + failures = [] + with mock.patch.object(layout.subprocess, "run", return_value=subprocess.CompletedProcess( + [], 1, stdout="", stderr="broken classpath")): + layout.check_closure_self_contained( + "paimon", [self.doris_jar, self.gcs_jar], self.spi_jar, + lambda check, message: failures.append(message)) + self.assertEqual(1, len(failures)) + self.assertIn("jdeps exited 1", failures[0]) + + +if __name__ == "__main__": + unittest.main()