diff --git a/be/src/agent/task_worker_pool.cpp b/be/src/agent/task_worker_pool.cpp index cf124f67656132..013f34112f7beb 100644 --- a/be/src/agent/task_worker_pool.cpp +++ b/be/src/agent/task_worker_pool.cpp @@ -58,10 +58,10 @@ #include "common/logging.h" #include "common/metrics/doris_metrics.h" #include "common/status.h" +#include "cpp/obj-client/obj_storage_client.h" #include "io/fs/file_system.h" #include "io/fs/hdfs_file_system.h" #include "io/fs/local_file_system.h" -#include "io/fs/obj_storage_client.h" #include "io/fs/path.h" #include "io/fs/remote_file_system.h" #include "io/fs/s3_file_system.h" diff --git a/be/src/cloud/cloud_meta_mgr.cpp b/be/src/cloud/cloud_meta_mgr.cpp index f9cf4b31609e0d..039adaed690783 100644 --- a/be/src/cloud/cloud_meta_mgr.cpp +++ b/be/src/cloud/cloud_meta_mgr.cpp @@ -54,8 +54,8 @@ #include "common/config.h" #include "common/logging.h" #include "common/status.h" +#include "cpp/obj-client/obj_storage_client.h" #include "cpp/sync_point.h" -#include "io/fs/obj_storage_client.h" #include "load/stream_load/stream_load_context.h" #include "runtime/exec_env.h" #include "storage/olap_common.h" diff --git a/be/src/common/status.h b/be/src/common/status.h index 594303888bc092..001a3356ad04f8 100644 --- a/be/src/common/status.h +++ b/be/src/common/status.h @@ -25,14 +25,8 @@ namespace doris { -namespace io { -struct ObjectStorageStatus; -} - class Status; -extern io::ObjectStorageStatus convert_to_obj_response(Status st); - class PStatus; namespace ErrorCode { @@ -576,8 +570,6 @@ class [[nodiscard]] Status { std::pair retrieve_error_msg() { return {_code, std::move(_err_msg->_msg)}; } - friend io::ObjectStorageStatus convert_to_obj_response(Status st); - private: int _code; struct ErrMsg { diff --git a/be/src/exprs/function/ai/embed.h b/be/src/exprs/function/ai/embed.h index 2e7947df72b13c..a7e8332259ee46 100644 --- a/be/src/exprs/function/ai/embed.h +++ b/be/src/exprs/function/ai/embed.h @@ -454,18 +454,15 @@ class FunctionEmbed : public AIFunction { S3ClientConf s3_client_conf; RETURN_IF_ERROR(init_s3_client_conf_from_json(file_input, s3_client_conf)); - auto s3_client = S3ClientFactory::instance().create(s3_client_conf); - if (s3_client == nullptr) { - return Status::InternalError("Failed to create S3 client for EMBED file input"); - } + auto s3_client = DORIS_TRY(S3ClientFactory::instance().create(s3_client_conf)); S3URI s3_uri(uri); RETURN_IF_ERROR(s3_uri.parse()); std::string bucket = s3_uri.get_bucket(); std::string key = s3_uri.get_key(); DORIS_CHECK(!bucket.empty() && !key.empty()); - media_url = s3_client->generate_presigned_url({.bucket = bucket, .key = key}, ttl_seconds, - s3_client_conf); + media_url = s3_client->generate_presigned_url({.bucket = bucket, .key = key, .prefix = ""}, + ttl_seconds); return Status::OK(); } }; diff --git a/be/src/io/CMakeLists.txt b/be/src/io/CMakeLists.txt index 1ef99c64642a6a..027caa433d2c01 100644 --- a/be/src/io/CMakeLists.txt +++ b/be/src/io/CMakeLists.txt @@ -22,9 +22,6 @@ set(LIBRARY_OUTPUT_PATH "${BUILD_DIR}/src/io") set(EXECUTABLE_OUTPUT_PATH "${BUILD_DIR}/src/io") file(GLOB_RECURSE IO_FILES CONFIGURE_DEPENDS *.cpp) -if(BUILD_AZURE STREQUAL "OFF") - list(REMOVE_ITEM IO_FILES "${CMAKE_CURRENT_SOURCE_DIR}/fs/azure_obj_storage_client.cpp") -endif() if(ENABLE_TDE) list(REMOVE_ITEM IO_FILES "${CMAKE_CURRENT_SOURCE_DIR}/fs/encrypted_fs_factory.cpp") diff --git a/be/src/io/fs/azure_obj_storage_client.cpp b/be/src/io/fs/azure_obj_storage_client.cpp deleted file mode 100644 index 9702c87b3b304b..00000000000000 --- a/be/src/io/fs/azure_obj_storage_client.cpp +++ /dev/null @@ -1,430 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "io/fs/azure_obj_storage_client.h" - -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "common/exception.h" -#include "common/logging.h" -#include "common/status.h" -#include "cpp/obj_retry_strategy.h" -#include "io/fs/obj_storage_client.h" -#include "util/bvar_helper.h" -#include "util/coding.h" -#include "util/s3_util.h" - -using namespace Azure::Storage::Blobs; - -namespace { -std::string wrap_object_storage_path_msg(const doris::io::ObjectStoragePathOptions& opts) { - return fmt::format("bucket {}, key {}, prefix {}, path {}", opts.bucket, opts.key, opts.prefix, - opts.path.native()); -} - -std::string to_lower_ascii(std::string_view input) { - std::string lowered(input); - std::transform(lowered.begin(), lowered.end(), lowered.begin(), - [](unsigned char ch) { return static_cast(std::tolower(ch)); }); - return lowered; -} - -auto base64_encode_part_num(int part_num) { - uint8_t buf[4]; - doris::encode_fixed32_le(buf, static_cast(part_num)); - return Aws::Utils::HashingUtils::Base64Encode({buf, sizeof(buf)}); -} - -// Rate limiting is applied by RateLimitedObjStorageClient, the decorator that -// S3ClientFactory wraps around this client when the bucket is subject to limiting. - -constexpr char SAS_TOKEN_URL_TEMPLATE[] = "{}/{}/{}{}"; -constexpr char BlobNotFound[] = "BlobNotFound"; -} // namespace - -namespace doris::io { - -// As Azure's doc said, the batch size is 256 -// You can find out the num in https://learn.microsoft.com/en-us/rest/api/storageservices/blob-batch?tabs=microsoft-entra-id -// > Each batch request supports a maximum of 256 subrequests. -constexpr size_t BlobBatchMaxOperations = 256; - -bool is_azure_tls_ca_error_message(std::string_view message) { - std::string lower = to_lower_ascii(message); - return lower.find("ssl ca cert") != std::string::npos || - lower.find("peer failed verification") != std::string::npos || - lower.find("unable to get local issuer certificate") != std::string::npos || - lower.find("problem with the ssl ca cert") != std::string::npos; -} - -std::string build_azure_tls_debug_suffix(std::string_view error_message, - std::string_view tls_debug_context) { - if (tls_debug_context.empty() || !is_azure_tls_ca_error_message(error_message)) { - return ""; - } - return fmt::format(", {}", tls_debug_context); -} - -template -ObjectStorageResponse do_azure_client_call(Func f, const ObjectStoragePathOptions& opts, - std::string_view tls_debug_context) { - try { - f(); - } catch (Azure::Core::RequestFailedException& e) { - doris::record_object_request_failed(static_cast(e.StatusCode)); - auto tls_debug_suffix = build_azure_tls_debug_suffix( - fmt::format("{} {}", e.what(), e.Message), tls_debug_context); - auto msg = fmt::format( - "Azure request failed because {}, error msg {}, http code {}, path msg {}{}", - e.what(), e.Message, static_cast(e.StatusCode), - wrap_object_storage_path_msg(opts), tls_debug_suffix); - LOG_WARNING(msg); - return {.status = convert_to_obj_response(Status::InternalError(std::move(msg))), - .http_code = static_cast(e.StatusCode), - .request_id = std::move(e.RequestId)}; - } catch (std::exception& e) { - auto msg = fmt::format("Azure request failed because {}, path msg {}{}", e.what(), - wrap_object_storage_path_msg(opts), - build_azure_tls_debug_suffix(e.what(), tls_debug_context)); - LOG_WARNING(msg); - return {.status = convert_to_obj_response(Status::InternalError(std::move(msg)))}; - } - return ObjectStorageResponse::OK(); -} - -struct AzureBatchDeleter { - AzureBatchDeleter(BlobContainerClient* client, const ObjectStoragePathOptions& opts, - std::string_view tls_debug_context) - : _client(client), - _batch(client->CreateBatch()), - _opts(opts), - _tls_debug_context(tls_debug_context) {} - // Submit one blob to be deleted in `AzureBatchDeleter::execute` - void delete_blob(const std::string& blob_name) { - deferred_resps.emplace_back(_batch.DeleteBlob(blob_name)); - } - ObjectStorageResponse execute() { - if (deferred_resps.empty()) { - return ObjectStorageResponse::OK(); - } - auto resp = do_azure_client_call( - [&]() { - SCOPED_BVAR_LATENCY(s3_bvar::s3_delete_objects_latency); - _client->SubmitBatch(_batch); - }, - _opts, _tls_debug_context); - if (resp.status.code != ErrorCode::OK) { - return resp; - } - - for (auto&& defer_response : deferred_resps) { - try { - auto r = defer_response.GetResponse(); - if (!r.Value.Deleted) { - auto msg = fmt::format("Azure batch delete failed, path msg {}", - wrap_object_storage_path_msg(_opts)); - LOG_WARNING(msg); - return {.status = convert_to_obj_response( - Status::InternalError(std::move(msg)))}; - } - } catch (Azure::Core::RequestFailedException& e) { - if (Azure::Core::Http::HttpStatusCode::NotFound == e.StatusCode && - 0 == strcmp(e.ErrorCode.c_str(), BlobNotFound)) { - continue; - } - doris::record_object_request_failed(static_cast(e.StatusCode)); - auto msg = fmt::format( - "Azure request failed because {}, error msg {}, http code {}, path msg " - "{}{}", - e.what(), e.Message, static_cast(e.StatusCode), - wrap_object_storage_path_msg(_opts), - build_azure_tls_debug_suffix(fmt::format("{} {}", e.what(), e.Message), - _tls_debug_context)); - LOG_WARNING(msg); - return {.status = convert_to_obj_response( - Status::InternalError(std::move(msg))), - .http_code = static_cast(e.StatusCode), - .request_id = std::move(e.RequestId)}; - } - } - - return ObjectStorageResponse::OK(); - } - -private: - BlobContainerClient* _client; - BlobContainerBatch _batch; - const ObjectStoragePathOptions& _opts; - std::string_view _tls_debug_context; - std::vector> deferred_resps; -}; - -// Azure would do nothing -ObjectStorageUploadResponse AzureObjStorageClient::create_multipart_upload( - const ObjectStoragePathOptions& opts) { - return ObjectStorageUploadResponse { - .resp = ObjectStorageResponse::OK(), - }; -} - -ObjectStorageResponse AzureObjStorageClient::put_object(const ObjectStoragePathOptions& opts, - std::string_view stream) { - auto client = _client->GetBlockBlobClient(opts.key); - return do_azure_client_call( - [&]() { - SCOPED_BVAR_LATENCY(s3_bvar::s3_put_latency); - client.UploadFrom(reinterpret_cast(stream.data()), stream.size()); - }, - opts, _tls_debug_context); -} - -ObjectStorageUploadResponse AzureObjStorageClient::upload_part(const ObjectStoragePathOptions& opts, - std::string_view stream, - int part_num) { - auto client = _client->GetBlockBlobClient(opts.key); - auto resp = do_azure_client_call( - [&]() { - Azure::Core::IO::MemoryBodyStream memory_body( - reinterpret_cast(stream.data()), stream.size()); - // The blockId must be base64 encoded - SCOPED_BVAR_LATENCY(s3_bvar::s3_multi_part_upload_latency); - client.StageBlock(base64_encode_part_num(part_num), memory_body); - }, - opts, _tls_debug_context); - return ObjectStorageUploadResponse { - .resp = resp, - }; -} - -ObjectStorageResponse AzureObjStorageClient::complete_multipart_upload( - const ObjectStoragePathOptions& opts, - const std::vector& completed_parts) { - auto client = _client->GetBlockBlobClient(opts.key); - std::vector string_block_ids; - std::ranges::transform( - completed_parts, std::back_inserter(string_block_ids), - [](const ObjectCompleteMultiPart& i) { return base64_encode_part_num(i.part_num); }); - return do_azure_client_call( - [&]() { - SCOPED_BVAR_LATENCY(s3_bvar::s3_multi_part_upload_latency); - client.CommitBlockList(string_block_ids); - }, - opts, _tls_debug_context); -} - -ObjectStorageHeadResponse AzureObjStorageClient::head_object(const ObjectStoragePathOptions& opts) { - Models::BlobProperties properties {}; - auto resp = do_azure_client_call( - [&]() { - SCOPED_BVAR_LATENCY(s3_bvar::s3_head_latency); - properties = _client->GetBlockBlobClient(opts.key).GetProperties().Value; - }, - opts, _tls_debug_context); - if (resp.http_code == static_cast(Azure::Core::Http::HttpStatusCode::NotFound)) { - return ObjectStorageHeadResponse { - .resp = {.status = convert_to_obj_response( - Status::Error(""))}, - .file_size = properties.BlobSize, - }; - } - - return ObjectStorageHeadResponse { - .resp = resp, - .file_size = properties.BlobSize, - }; -} - -ObjectStorageResponse AzureObjStorageClient::get_object(const ObjectStoragePathOptions& opts, - void* buffer, size_t offset, - size_t bytes_read, size_t* size_return) { - auto client = _client->GetBlockBlobClient(opts.key); - return do_azure_client_call( - [&]() { - DownloadBlobToOptions download_opts; - Azure::Core::Http::HttpRange range {static_cast(offset), bytes_read}; - download_opts.Range = range; - SCOPED_BVAR_LATENCY(s3_bvar::s3_get_latency); - auto resp = client.DownloadTo(reinterpret_cast(buffer), bytes_read, - download_opts); - *size_return = resp.Value.ContentRange.Length.Value(); - }, - opts, _tls_debug_context); -} - -ObjectStorageResponse AzureObjStorageClient::list_objects(const ObjectStoragePathOptions& opts, - std::vector* files) { - auto get_file_file = [&](ListBlobsPagedResponse& resp) { - std::ranges::transform(resp.Blobs, std::back_inserter(*files), [](auto&& blob_item) { - return FileInfo { - .file_name = blob_item.Name, .file_size = blob_item.BlobSize, .is_file = true}; - }); - }; - return do_azure_client_call( - [&]() { - ListBlobsOptions list_opts; - list_opts.Prefix = opts.prefix; - ListBlobsPagedResponse resp; - { - SCOPED_BVAR_LATENCY(s3_bvar::s3_list_latency); - resp = _client->ListBlobs(list_opts); - } - get_file_file(resp); - while (resp.NextPageToken.HasValue()) { - list_opts.ContinuationToken = resp.NextPageToken; - { - SCOPED_BVAR_LATENCY(s3_bvar::s3_list_latency); - resp = _client->ListBlobs(list_opts); - } - get_file_file(resp); - } - }, - opts, _tls_debug_context); -} - -// As Azure's doc said, the batch size is 256 -// You can find out the num in https://learn.microsoft.com/en-us/rest/api/storageservices/blob-batch?tabs=microsoft-entra-id -// > Each batch request supports a maximum of 256 subrequests. -ObjectStorageResponse AzureObjStorageClient::delete_objects(const ObjectStoragePathOptions& opts, - std::vector objs) { - // TODO(ByteYue) : use range to adate this code when compiler is ready - // auto chunkedView = objs | std::views::chunk(BlobBatchMaxOperations); - auto begin = std::begin(objs); - auto end = std::end(objs); - - while (begin != end) { - auto deleter = AzureBatchDeleter(_client.get(), opts, _tls_debug_context); - auto chunk_end = begin; - std::advance(chunk_end, std::min(BlobBatchMaxOperations, - static_cast(std::distance(begin, end)))); - - std::ranges::for_each(std::ranges::subrange(begin, chunk_end), - [&](const std::string& obj) { deleter.delete_blob(obj); }); - begin = chunk_end; - if (auto resp = deleter.execute(); resp.status.code != ErrorCode::OK) { - return resp; - } - } - return ObjectStorageResponse::OK(); -} - -ObjectStorageResponse AzureObjStorageClient::delete_object(const ObjectStoragePathOptions& opts) { - return do_azure_client_call( - [&]() { - SCOPED_BVAR_LATENCY(s3_bvar::s3_delete_object_latency); - auto resp = _client->DeleteBlob(opts.key); - if (!resp.Value.Deleted) { - throw Exception(Status::IOError("Delete azure blob failed")); - } - }, - opts, _tls_debug_context); -} - -ObjectStorageResponse AzureObjStorageClient::delete_objects_recursively( - const ObjectStoragePathOptions& opts) { - ListBlobsOptions list_opts; - list_opts.Prefix = opts.prefix; - list_opts.PageSizeHint = BlobBatchMaxOperations; - auto delete_func = [&](const std::vector& blobs) -> ObjectStorageResponse { - auto deleter = AzureBatchDeleter(_client.get(), opts, _tls_debug_context); - auto batch = _client->CreateBatch(); - for (auto&& blob_item : blobs) { - deleter.delete_blob(blob_item.Name); - } - if (auto response = deleter.execute(); response.status.code != ErrorCode::OK) { - return response; - } - return ObjectStorageResponse::OK(); - }; - - ListBlobsPagedResponse resp; - auto list_resp = do_azure_client_call( - [&]() { - SCOPED_BVAR_LATENCY(s3_bvar::s3_list_latency); - resp = _client->ListBlobs(list_opts); - }, - opts, _tls_debug_context); - if (list_resp.status.code != ErrorCode::OK) { - return list_resp; - } - - if (auto response = delete_func(resp.Blobs); response.status.code != ErrorCode::OK) { - return response; - } - - while (resp.NextPageToken.HasValue()) { - list_opts.ContinuationToken = resp.NextPageToken; - list_resp = do_azure_client_call( - [&]() { - SCOPED_BVAR_LATENCY(s3_bvar::s3_list_latency); - resp = _client->ListBlobs(list_opts); - }, - opts, _tls_debug_context); - if (list_resp.status.code != ErrorCode::OK) { - return list_resp; - } - - if (auto response = delete_func(resp.Blobs); response.status.code != ErrorCode::OK) { - return response; - } - } - return ObjectStorageResponse::OK(); -} - -std::string AzureObjStorageClient::generate_presigned_url(const ObjectStoragePathOptions& opts, - int64_t expiration_secs, - const S3ClientConf& conf) { - Azure::Storage::Sas::BlobSasBuilder sas_builder; - sas_builder.ExpiresOn = - std::chrono::system_clock::now() + std::chrono::seconds(expiration_secs); - sas_builder.BlobContainerName = opts.bucket; - sas_builder.BlobName = opts.key; - sas_builder.Resource = Azure::Storage::Sas::BlobSasResource::Blob; - sas_builder.Protocol = Azure::Storage::Sas::SasProtocol::HttpsOnly; - sas_builder.SetPermissions(Azure::Storage::Sas::BlobSasPermissions::Read); - - std::string sasToken = sas_builder.GenerateSasToken( - Azure::Storage::StorageSharedKeyCredential(conf.ak, conf.sk)); - - std::string endpoint = conf.endpoint; - auto sasURL = fmt::format(SAS_TOKEN_URL_TEMPLATE, endpoint, conf.bucket, opts.key, sasToken); - if (sasURL.find("://") == std::string::npos) { - sasURL = "https://" + sasURL; - } - return sasURL; -} -} // namespace doris::io diff --git a/be/src/io/fs/azure_obj_storage_client.h b/be/src/io/fs/azure_obj_storage_client.h deleted file mode 100644 index 7d1cecc502e44d..00000000000000 --- a/be/src/io/fs/azure_obj_storage_client.h +++ /dev/null @@ -1,70 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include -#include - -#include "io/fs/obj_storage_client.h" - -namespace Azure::Storage::Blobs { -class BlobContainerClient; -} // namespace Azure::Storage::Blobs - -namespace doris::io { - -class ObjClientHolder; - -bool is_azure_tls_ca_error_message(std::string_view message); -std::string build_azure_tls_debug_suffix(std::string_view error_message, - std::string_view tls_debug_context); - -class AzureObjStorageClient final : public ObjStorageClient { -public: - AzureObjStorageClient(std::shared_ptr client, - std::string tls_debug_context = {}) - : _client(std::move(client)), _tls_debug_context(std::move(tls_debug_context)) {} - ~AzureObjStorageClient() override = default; - ObjectStorageUploadResponse create_multipart_upload( - const ObjectStoragePathOptions& opts) override; - ObjectStorageResponse put_object(const ObjectStoragePathOptions& opts, - std::string_view stream) override; - ObjectStorageUploadResponse upload_part(const ObjectStoragePathOptions& opts, std::string_view, - int partNum) override; - ObjectStorageResponse complete_multipart_upload( - const ObjectStoragePathOptions& opts, - const std::vector& completed_parts) override; - ObjectStorageHeadResponse head_object(const ObjectStoragePathOptions& opts) override; - ObjectStorageResponse get_object(const ObjectStoragePathOptions& opts, void* buffer, - size_t offset, size_t bytes_read, - size_t* size_return) override; - ObjectStorageResponse list_objects(const ObjectStoragePathOptions& opts, - std::vector* files) override; - ObjectStorageResponse delete_objects(const ObjectStoragePathOptions& opts, - std::vector objs) override; - ObjectStorageResponse delete_object(const ObjectStoragePathOptions& opts) override; - ObjectStorageResponse delete_objects_recursively(const ObjectStoragePathOptions& opts) override; - std::string generate_presigned_url(const ObjectStoragePathOptions& opts, - int64_t expiration_secs, const S3ClientConf& conf) override; - -private: - std::shared_ptr _client; - std::string _tls_debug_context; -}; - -} // namespace doris::io diff --git a/be/src/io/fs/connectivity/s3_connectivity_tester.cpp b/be/src/io/fs/connectivity/s3_connectivity_tester.cpp index 661e251b3eb4c6..5446ede9f82af7 100644 --- a/be/src/io/fs/connectivity/s3_connectivity_tester.cpp +++ b/be/src/io/fs/connectivity/s3_connectivity_tester.cpp @@ -36,10 +36,7 @@ Status S3ConnectivityTester::test(const std::map& prop S3Conf s3_conf; RETURN_IF_ERROR(S3ClientFactory::convert_properties_to_s3_conf(properties, s3_uri, &s3_conf)); - auto obj_client = S3ClientFactory::instance().create(s3_conf.client_conf); - if (!obj_client) { - return Status::InternalError("Failed to create S3 client"); - } + auto obj_client = DORIS_TRY(S3ClientFactory::instance().create(s3_conf.client_conf)); auto resp = obj_client->head_object({.bucket = bucket, .key = ""}); if (resp.resp.status.code != ErrorCode::OK && resp.resp.status.code != ErrorCode::NOT_FOUND) { diff --git a/be/src/io/fs/err_utils.cpp b/be/src/io/fs/err_utils.cpp index 3547f7b57f456c..74b8b0bc534815 100644 --- a/be/src/io/fs/err_utils.cpp +++ b/be/src/io/fs/err_utils.cpp @@ -18,7 +18,6 @@ #include "io/fs/err_utils.h" // IWYU pragma: no_include -#include #include // IWYU pragma: keep #include #include @@ -27,17 +26,10 @@ #include "common/status.h" #include "io/fs/hdfs.h" -#include "io/fs/obj_storage_client.h" namespace doris { using namespace ErrorCode; -io::ObjectStorageStatus convert_to_obj_response(Status st) { - int code = st._code; - std::string msg = st._err_msg == nullptr ? "" : std::move(st._err_msg->_msg); - return io::ObjectStorageStatus {.code = code, .msg = std::move(msg)}; -} - namespace io { std::string errno_to_str() { @@ -122,28 +114,5 @@ Status localfs_error(int posix_errno, std::string_view msg) { } } -Status s3fs_error(const Aws::S3::S3Error& err, std::string_view msg) { - using namespace Aws::Http; - // A failure raised by the client itself carries no request id, and a dangling - // `request_id=` has been read as a request id of the object storage. - std::string request_id = err.GetRequestId().empty() ? "" : err.GetRequestId().c_str(); - switch (err.GetResponseCode()) { - case HttpResponseCode::NOT_FOUND: - return Status::Error("{}: {} {} code=NOT_FOUND, type={}, request_id={}", - msg, err.GetExceptionName(), err.GetMessage(), - err.GetErrorType(), request_id); - case HttpResponseCode::UNAUTHORIZED: - case HttpResponseCode::FORBIDDEN: - return Status::Error( - "{}: {} {} code=PERMISSION_DENIED, http_code={}, type={}, request_id={}", msg, - err.GetExceptionName(), err.GetMessage(), err.GetResponseCode(), err.GetErrorType(), - request_id); - default: - return Status::Error( - "{}: {} {} code={} type={}, request_id={}", msg, err.GetExceptionName(), - err.GetMessage(), err.GetResponseCode(), err.GetErrorType(), request_id); - } -} - } // namespace io } // namespace doris diff --git a/be/src/io/fs/err_utils.h b/be/src/io/fs/err_utils.h index 4a7c597879a7ec..1eec6d4de4fd0a 100644 --- a/be/src/io/fs/err_utils.h +++ b/be/src/io/fs/err_utils.h @@ -22,10 +22,6 @@ #include "common/status.h" -namespace Aws::S3 { -class S3Error; -} // namespace Aws::S3 - namespace doris { namespace io { @@ -37,7 +33,6 @@ std::string glob_err_to_str(int code); Status localfs_error(const std::error_code& ec, std::string_view msg); Status localfs_error(int posix_errno, std::string_view msg); -Status s3fs_error(const Aws::S3::S3Error& err, std::string_view msg); } // namespace io } // namespace doris diff --git a/be/src/io/fs/obj_storage_client.h b/be/src/io/fs/obj_storage_client.h deleted file mode 100644 index fa239ca3282e2a..00000000000000 --- a/be/src/io/fs/obj_storage_client.h +++ /dev/null @@ -1,135 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include - -#include "io/fs/file_system.h" -#include "io/fs/path.h" -namespace doris { -class Status; -struct S3ClientConf; -namespace io { - -// Names are in lexico order. -enum class ObjStorageType : uint8_t { - UNKNOWN = 0, - AWS = 1, - AZURE, - BOS, - COS, - OSS, - OBS, - GCP, - TOS, -}; - -struct ObjectStoragePathOptions { - Path path = ""; - std::string bucket = std::string(); // blob container in azure - std::string key = std::string(); // blob name in azure - std::string prefix = std::string(); // for batch delete and recursive delete - std::optional upload_id = std::nullopt; // only used for S3 upload -}; - -struct ObjectCompleteMultiPart { - int part_num = 0; - std::string etag = std::string(); -}; - -struct ObjectStorageStatus { - int code = 0; - std::string msg = std::string(); -}; - -// We only store error code along with err_msg instead of Status to unify BE and recycler's error handle logic -struct ObjectStorageResponse { - ObjectStorageStatus status {}; - int http_code {200}; - std::string request_id = std::string(); - static ObjectStorageResponse OK() { - // clang-format off - return { - .status { .code = 0, }, - .http_code = 200, - }; - // clang-format on - } -}; - -struct ObjectStorageUploadResponse { - ObjectStorageResponse resp {}; - std::optional upload_id = std::nullopt; - std::optional etag = std::nullopt; -}; - -struct ObjectStorageHeadResponse : ObjectStorageResponse { - ObjectStorageResponse resp {}; - long long file_size {0}; -}; - -class ObjStorageClient { -public: - virtual ~ObjStorageClient() = default; - // Create a multi-part upload request. On AWS-compatible systems, it will return an upload ID, but not on Azure. - // The input parameters should include the bucket and key for the object storage. - virtual ObjectStorageUploadResponse create_multipart_upload( - const ObjectStoragePathOptions& opts) = 0; - // To directly upload a piece of data to object storage and generate a user-visible file. - // You need to clearly specify the bucket and key - virtual ObjectStorageResponse put_object(const ObjectStoragePathOptions& opts, - std::string_view stream) = 0; - // To upload a part of a large file to object storage as a temporary file, which is not visible to the user - // The temporary file's ID is the value of the part_num passed in - // You need to specify the bucket and key along with the upload_id if it's AWS-compatible system - // For the same bucket and key, as well as the same part_num, it will directly replace the original temporary file. - virtual ObjectStorageUploadResponse upload_part(const ObjectStoragePathOptions& opts, - std::string_view stream, int part_num) = 0; - // To combine the previously uploaded multiple file parts into a complete file, the file name is the name of the key passed in. - // If it is an AWS-compatible system, the upload_id needs to be included. - // After a successful execution, the large file can be accessed in the object storage - virtual ObjectStorageResponse complete_multipart_upload( - const ObjectStoragePathOptions& opts, - const std::vector& completed_parts) = 0; - // According to the passed bucket and key, it will access whether the corresponding file exists in the object storage. - // If it exists, it will return the corresponding file size - virtual ObjectStorageHeadResponse head_object(const ObjectStoragePathOptions& opts) = 0; - // According to the bucket and key, it finds the corresponding file in the object storage - // and starting from the offset, it reads bytes_read bytes into the buffer, with size_return recording the actual number of bytes read - virtual ObjectStorageResponse get_object(const ObjectStoragePathOptions& opts, void* buffer, - size_t offset, size_t bytes_read, - size_t* size_return) = 0; - // According to the passed bucket and prefix, it traverses and retrieves all files under the prefix, and returns the name and file size of all files. - // **Notice**: The files returned by this function contains the full key in object storage. - virtual ObjectStorageResponse list_objects(const ObjectStoragePathOptions& opts, - std::vector* files) = 0; - // According to the bucket and prefix specified by the user, it performs batch deletion based on the object names in the object array. - virtual ObjectStorageResponse delete_objects(const ObjectStoragePathOptions& opts, - std::vector objs) = 0; - // Delete the file named key in the object storage bucket. - virtual ObjectStorageResponse delete_object(const ObjectStoragePathOptions& opts) = 0; - // According to the prefix, recursively delete all files under the prefix. - virtual ObjectStorageResponse delete_objects_recursively( - const ObjectStoragePathOptions& opts) = 0; - // Return a presigned URL for users to access the object - virtual std::string generate_presigned_url(const ObjectStoragePathOptions& opts, - int64_t expiration_secs, - const S3ClientConf& conf) = 0; -}; -} // namespace io -} // namespace doris diff --git a/be/src/io/fs/rate_limited_obj_storage_client.cpp b/be/src/io/fs/rate_limited_obj_storage_client.cpp deleted file mode 100644 index 1b8730847162df..00000000000000 --- a/be/src/io/fs/rate_limited_obj_storage_client.cpp +++ /dev/null @@ -1,145 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "io/fs/rate_limited_obj_storage_client.h" - -#include "common/logging.h" -#include "common/status.h" -#include "util/s3_rate_limiter_manager.h" - -namespace doris::io { -namespace { - -ObjectStorageResponse rate_limited_response(S3RateLimitType type, S3RateLimitRejectReason reason) { - CHECK(reason != S3RateLimitRejectReason::NONE); - const auto* limit_type = reason == S3RateLimitRejectReason::QPS ? "QPS" : "bytes"; - return {.status = convert_to_obj_response(Status::Error( - "s3 {} request exceeds {} limit, rejected by BE rate limiter", to_string(type), - limit_type)), - // The BE rate limiter rejected the request before it reached the provider. - .http_code = 0}; -} - -} // namespace - -ObjectStorageUploadResponse RateLimitedObjStorageClient::create_multipart_upload( - const ObjectStoragePathOptions& opts) { - S3RateLimitGuard guard(S3RateLimitType::PUT, 0); - if (!guard.ok()) { - return {.resp = rate_limited_response(S3RateLimitType::PUT, guard.reject_reason())}; - } - return _inner->create_multipart_upload(opts); -} - -ObjectStorageResponse RateLimitedObjStorageClient::put_object(const ObjectStoragePathOptions& opts, - std::string_view stream) { - S3RateLimitGuard guard(S3RateLimitType::PUT, stream.size()); - if (!guard.ok()) { - return rate_limited_response(S3RateLimitType::PUT, guard.reject_reason()); - } - return _inner->put_object(opts, stream); -} - -ObjectStorageUploadResponse RateLimitedObjStorageClient::upload_part( - const ObjectStoragePathOptions& opts, std::string_view stream, int part_num) { - S3RateLimitGuard guard(S3RateLimitType::PUT, stream.size()); - if (!guard.ok()) { - return {.resp = rate_limited_response(S3RateLimitType::PUT, guard.reject_reason())}; - } - return _inner->upload_part(opts, stream, part_num); -} - -ObjectStorageResponse RateLimitedObjStorageClient::complete_multipart_upload( - const ObjectStoragePathOptions& opts, - const std::vector& completed_parts) { - S3RateLimitGuard guard(S3RateLimitType::PUT, 0); - if (!guard.ok()) { - return rate_limited_response(S3RateLimitType::PUT, guard.reject_reason()); - } - return _inner->complete_multipart_upload(opts, completed_parts); -} - -ObjectStorageHeadResponse RateLimitedObjStorageClient::head_object( - const ObjectStoragePathOptions& opts) { - S3RateLimitGuard guard(S3RateLimitType::GET, 0); - if (!guard.ok()) { - ObjectStorageHeadResponse response; - response.resp = rate_limited_response(S3RateLimitType::GET, guard.reject_reason()); - return response; - } - return _inner->head_object(opts); -} - -ObjectStorageResponse RateLimitedObjStorageClient::get_object(const ObjectStoragePathOptions& opts, - void* buffer, size_t offset, - size_t bytes_read, - size_t* size_return) { - S3RateLimitGuard guard(S3RateLimitType::GET, bytes_read); - if (!guard.ok()) { - return rate_limited_response(S3RateLimitType::GET, guard.reject_reason()); - } - auto resp = _inner->get_object(opts, buffer, offset, bytes_read, size_return); - if (resp.status.code == 0) { - // Refund the difference for short reads (e.g. requested range crosses EOF). - guard.settle(*size_return); - } - return resp; -} - -ObjectStorageResponse RateLimitedObjStorageClient::list_objects( - const ObjectStoragePathOptions& opts, std::vector* files) { - S3RateLimitGuard guard(S3RateLimitType::GET, 0); - if (!guard.ok()) { - return rate_limited_response(S3RateLimitType::GET, guard.reject_reason()); - } - return _inner->list_objects(opts, files); -} - -ObjectStorageResponse RateLimitedObjStorageClient::delete_objects( - const ObjectStoragePathOptions& opts, std::vector objs) { - S3RateLimitGuard guard(S3RateLimitType::PUT, 0); - if (!guard.ok()) { - return rate_limited_response(S3RateLimitType::PUT, guard.reject_reason()); - } - return _inner->delete_objects(opts, std::move(objs)); -} - -ObjectStorageResponse RateLimitedObjStorageClient::delete_object( - const ObjectStoragePathOptions& opts) { - S3RateLimitGuard guard(S3RateLimitType::PUT, 0); - if (!guard.ok()) { - return rate_limited_response(S3RateLimitType::PUT, guard.reject_reason()); - } - return _inner->delete_object(opts); -} - -ObjectStorageResponse RateLimitedObjStorageClient::delete_objects_recursively( - const ObjectStoragePathOptions& opts) { - S3RateLimitGuard guard(S3RateLimitType::PUT, 0); - if (!guard.ok()) { - return rate_limited_response(S3RateLimitType::PUT, guard.reject_reason()); - } - return _inner->delete_objects_recursively(opts); -} - -std::string RateLimitedObjStorageClient::generate_presigned_url( - const ObjectStoragePathOptions& opts, int64_t expiration_secs, const S3ClientConf& conf) { - // Generating a presigned URL is a local computation, no request goes out. - return _inner->generate_presigned_url(opts, expiration_secs, conf); -} - -} // namespace doris::io diff --git a/be/src/io/fs/rate_limited_obj_storage_client.h b/be/src/io/fs/rate_limited_obj_storage_client.h deleted file mode 100644 index 00725d7edcb299..00000000000000 --- a/be/src/io/fs/rate_limited_obj_storage_client.h +++ /dev/null @@ -1,70 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include - -#include "io/fs/obj_storage_client.h" - -namespace doris::io { - -// Decorator that applies the process-wide S3 GET/PUT QPS and bandwidth rate limiters -// in front of any ObjStorageClient. This is the single place where rate limiting is -// wired into the object storage path: provider clients (S3, Azure, future GCP, ...) -// contain no rate limiting code, and S3ClientFactory decides at construction time -// whether to wrap a client (internal storage-vault buckets) or return it bare -// (external buckets: S3 load, TVF, external catalogs in cloud mode). -// -// Each public API call is charged once against the QPS bucket, and data-carrying -// calls additionally reserve their payload size from the bytes bucket (reconciled -// with the actually transferred size for reads). Note that APIs which internally -// paginate (list_objects, delete_objects_recursively) are charged once per logical -// call, not once per underlying HTTP request. -class RateLimitedObjStorageClient final : public ObjStorageClient { -public: - explicit RateLimitedObjStorageClient(std::shared_ptr inner) - : _inner(std::move(inner)) {} - ~RateLimitedObjStorageClient() override = default; - - ObjectStorageUploadResponse create_multipart_upload( - const ObjectStoragePathOptions& opts) override; - ObjectStorageResponse put_object(const ObjectStoragePathOptions& opts, - std::string_view stream) override; - ObjectStorageUploadResponse upload_part(const ObjectStoragePathOptions& opts, - std::string_view stream, int part_num) override; - ObjectStorageResponse complete_multipart_upload( - const ObjectStoragePathOptions& opts, - const std::vector& completed_parts) override; - ObjectStorageHeadResponse head_object(const ObjectStoragePathOptions& opts) override; - ObjectStorageResponse get_object(const ObjectStoragePathOptions& opts, void* buffer, - size_t offset, size_t bytes_read, - size_t* size_return) override; - ObjectStorageResponse list_objects(const ObjectStoragePathOptions& opts, - std::vector* files) override; - ObjectStorageResponse delete_objects(const ObjectStoragePathOptions& opts, - std::vector objs) override; - ObjectStorageResponse delete_object(const ObjectStoragePathOptions& opts) override; - ObjectStorageResponse delete_objects_recursively(const ObjectStoragePathOptions& opts) override; - std::string generate_presigned_url(const ObjectStoragePathOptions& opts, - int64_t expiration_secs, const S3ClientConf& conf) override; - -private: - std::shared_ptr _inner; -}; - -} // namespace doris::io diff --git a/be/src/io/fs/s3_file_bufferpool.cpp b/be/src/io/fs/s3_file_bufferpool.cpp index bc0b313fda7862..51d1944a121849 100644 --- a/be/src/io/fs/s3_file_bufferpool.cpp +++ b/be/src/io/fs/s3_file_bufferpool.cpp @@ -28,10 +28,10 @@ #include "common/logging.h" #include "common/status.h" #include "core/arena.h" +#include "cpp/obj-client/s3_common.h" #include "cpp/sync_point.h" #include "io/cache/file_block.h" #include "io/cache/file_cache_common.h" -#include "io/fs/s3_common.h" #include "runtime/exec_env.h" #include "runtime/thread_context.h" #include "util/defer_op.h" diff --git a/be/src/io/fs/s3_file_reader.cpp b/be/src/io/fs/s3_file_reader.cpp index dbbfd8240e6321..04388a24010049 100644 --- a/be/src/io/fs/s3_file_reader.cpp +++ b/be/src/io/fs/s3_file_reader.cpp @@ -33,10 +33,10 @@ #include "common/compiler_util.h" // IWYU pragma: keep #include "common/metrics/doris_metrics.h" +#include "cpp/obj-client/obj_storage_client.h" +#include "cpp/obj-client/s3_common.h" #include "io/cache/block_file_cache.h" #include "io/fs/err_utils.h" -#include "io/fs/obj_storage_client.h" -#include "io/fs/s3_common.h" #include "runtime/file_scan_profile.h" #include "runtime/runtime_profile.h" #include "runtime/thread_context.h" diff --git a/be/src/io/fs/s3_file_system.cpp b/be/src/io/fs/s3_file_system.cpp index 62dea5f8c2a2fc..4797b573961a77 100644 --- a/be/src/io/fs/s3_file_system.cpp +++ b/be/src/io/fs/s3_file_system.cpp @@ -35,16 +35,16 @@ #include "common/config.h" #include "common/logging.h" #include "common/status.h" +#include "cpp/obj-client/obj_storage_client.h" +#include "cpp/obj-client/s3_common.h" #include "cpp/sync_point.h" #include "io/fs/err_utils.h" #include "io/fs/file_system.h" #include "io/fs/file_writer.h" #include "io/fs/local_file_system.h" #include "io/fs/remote_file_system.h" -#include "io/fs/s3_common.h" #include "io/fs/s3_file_reader.h" #include "io/fs/s3_file_writer.h" -#include "io/fs/s3_obj_storage_client.h" #include "runtime/exec_env.h" #include "runtime/thread_context.h" #include "util/s3_uri.h" @@ -76,11 +76,7 @@ ObjClientHolder::ObjClientHolder(S3ClientConf conf) : _conf(std::move(conf)) {} ObjClientHolder::~ObjClientHolder() = default; Status ObjClientHolder::init() { - _client = S3ClientFactory::instance().create(_conf); - if (!_client) { - return Status::InvalidArgument("failed to init s3 client with conf {}", _conf.to_string()); - } - + _client = DORIS_TRY(S3ClientFactory::instance().create(_conf)); return Status::OK(); } @@ -112,10 +108,7 @@ Status ObjClientHolder::reset(const S3ClientConf& conf) { } } - auto client = S3ClientFactory::instance().create(reset_conf); - if (!client) { - return Status::InvalidArgument("failed to init s3 client with conf {}", conf.to_string()); - } + auto client = DORIS_TRY(S3ClientFactory::instance().create(reset_conf)); LOG(WARNING) << "reset s3 client with new conf: " << conf.to_string(); @@ -234,11 +227,12 @@ Status S3FileSystem::delete_directory_impl(const Path& dir) { prefix.push_back('/'); } - auto resp = client->delete_objects_recursively({ + ObjStoragePath delete_path { .path = full_s3_path(prefix), .bucket = _bucket, .prefix = prefix, - }); + }; + auto resp = delete_objects_recursively(std::move(client), delete_path); return {resp.status.code, std::move(resp.status.msg)}; } @@ -310,16 +304,21 @@ Status S3FileSystem::list_impl(const Path& dir, bool only_file, std::vectorlist_objects( {.bucket = _bucket, .prefix = prefix,}, files); - // clang-format on - if (resp.status.code == ErrorCode::OK) { - for (auto&& file : *files) { - file.file_name.erase(0, prefix.size()); - } + std::vector objects; + auto resp = client->list_objects({.bucket = _bucket, .prefix = prefix}, &objects); + if (!resp.ok()) { + files->clear(); + return {resp.status.code, std::move(resp.status.msg)}; + } + files->reserve(files->size() + objects.size()); + for (auto& obj : objects) { + obj.key.erase(0, prefix.size()); + bool is_dir = obj.key.empty() || obj.key.back() == '/'; + files->emplace_back(FileInfo { + .file_name = std::move(obj.key), .file_size = obj.size, .is_file = !is_dir}); } - return {resp.status.code, std::move(resp.status.msg)}; + return Status::OK(); } Status S3FileSystem::rename_impl(const Path& orig_name, const Path& new_name) { @@ -454,12 +453,18 @@ std::string S3FileSystem::generate_presigned_url(const Path& path, int64_t expir new_s3_conf.endpoint.erase( _client->s3_client_conf().endpoint.size() - OSS_PRIVATE_ENDPOINT_SUFFIX.size(), LEN_OF_OSS_PRIVATE_SUFFIX); - client = S3ClientFactory::instance().create(new_s3_conf); + auto client_result = S3ClientFactory::instance().create(new_s3_conf); + if (!client_result) { + LOG(WARNING) << "failed to create S3 client for presigned URL: " + << client_result.error(); + return {}; + } + client = std::move(client_result).value(); } else { client = _client->get(); } - return client->generate_presigned_url({.bucket = _bucket, .key = key}, expiration_secs, - _client->s3_client_conf()); + return client->generate_presigned_url({.bucket = _bucket, .key = key, .prefix = ""}, + expiration_secs); } } // namespace doris::io diff --git a/be/src/io/fs/s3_file_system.h b/be/src/io/fs/s3_file_system.h index f6efa5053324ff..d1b969eedeca95 100644 --- a/be/src/io/fs/s3_file_system.h +++ b/be/src/io/fs/s3_file_system.h @@ -38,7 +38,6 @@ class PooledThreadExecutor; } // namespace Aws::Utils::Threading namespace doris::io { -class ObjStorageClient; // In runtime, AK and SK may be modified, and the original `S3Client` instance will be replaced. // The `S3FileReader` cached by the `Segment` must hold a shared `ObjClientHolder` in order to // access S3 data with latest AK SK. diff --git a/be/src/io/fs/s3_file_writer.cpp b/be/src/io/fs/s3_file_writer.cpp index ec8e54849f626f..f2f4bc8c60bcd9 100644 --- a/be/src/io/fs/s3_file_writer.cpp +++ b/be/src/io/fs/s3_file_writer.cpp @@ -30,6 +30,7 @@ #include "common/config.h" #include "common/status.h" +#include "cpp/obj-client/s3_obj_storage_client.h" #include "cpp/sync_point.h" #include "io/cache/block_file_cache.h" #include "io/cache/block_file_cache_factory.h" @@ -39,7 +40,6 @@ #include "io/fs/path.h" #include "io/fs/s3_file_bufferpool.h" #include "io/fs/s3_file_system.h" -#include "io/fs/s3_obj_storage_client.h" #include "runtime/exec_env.h" #include "util/debug_points.h" #include "util/s3_util.h" @@ -101,7 +101,7 @@ Status S3FileWriter::_create_multi_upload_request() { } auto resp = client->create_multipart_upload(_obj_storage_path_opts); if (resp.resp.status.code == ErrorCode::OK) { - _obj_storage_path_opts.upload_id = resp.upload_id; + _upload_id = resp.upload_id.value_or(""); } return {resp.resp.status.code, std::move(resp.resp.status.msg)}; } @@ -111,8 +111,7 @@ void S3FileWriter::_wait_until_finish(std::string_view task_name) { auto msg = fmt::format( "{} multipart upload already takes {} seconds, bucket={}, key={}, upload_id={}", task_name, timeout_duration, _obj_storage_path_opts.bucket, - _obj_storage_path_opts.path.native(), - _obj_storage_path_opts.upload_id.has_value() ? *_obj_storage_path_opts.upload_id : ""); + _obj_storage_path_opts.path.native(), _upload_id); timespec current_time; // We don't need high accuracy here, so we use time(nullptr) // since it's the fastest way to get current time(second) @@ -389,7 +388,8 @@ void S3FileWriter::_upload_one_part(int part_num, UploadFileBuffer& buf) { buf.set_status(Status::InternalError("invalid obj storage client")); return; } - auto resp = client->upload_part(_obj_storage_path_opts, buf.get_string_view_data(), part_num); + auto resp = client->upload_part(_obj_storage_path_opts, _upload_id, buf.get_string_view_data(), + part_num); if (resp.resp.status.code != ErrorCode::OK) { LOG_WARNING("failed to upload part, key={}, part_num={}, status={}", _obj_storage_path_opts.key, part_num, resp.resp.status.msg); @@ -398,7 +398,7 @@ void S3FileWriter::_upload_one_part(int part_num, UploadFileBuffer& buf) { } s3_bytes_written_total << buf.get_size(); - ObjectCompleteMultiPart completed_part { + ObjStorageCompletedPart completed_part { static_cast(part_num), resp.etag.has_value() ? std::move(resp.etag.value()) : ""}; std::unique_lock lck {_completed_lock}; @@ -408,8 +408,8 @@ void S3FileWriter::_upload_one_part(int part_num, UploadFileBuffer& buf) { // if enabled check // 1. issue a head object request for existence check // 2. check the file size -Status check_after_upload(ObjStorageClient* client, const ObjectStorageResponse& upload_res, - const ObjectStoragePathOptions& path_opt, int64_t bytes_appended, +Status check_after_upload(ObjStorageClient* client, const ObjStorageResponse& upload_res, + const ObjStoragePath& path_opt, int64_t bytes_appended, const std::string& put_or_comp) { if (!config::enable_s3_object_check_after_upload) return Status::OK(); @@ -503,7 +503,8 @@ Status S3FileWriter::_complete() { LOG(INFO) << "complete_multipart_upload " << _obj_storage_path_opts.path.native() << " size=" << _bytes_appended << " number_parts=" << _completed_parts.size() << " s3_write_buffer_size=" << config::s3_write_buffer_size; - auto resp = client->complete_multipart_upload(_obj_storage_path_opts, _completed_parts); + auto resp = + client->complete_multipart_upload(_obj_storage_path_opts, _upload_id, _completed_parts); if (resp.status.code != ErrorCode::OK) { LOG_WARNING("failed to complete multipart upload, err={}, file_path={}", resp.status.msg, _obj_storage_path_opts.path.native()); diff --git a/be/src/io/fs/s3_file_writer.h b/be/src/io/fs/s3_file_writer.h index 5a8075e03cf404..1f5d30b888a38f 100644 --- a/be/src/io/fs/s3_file_writer.h +++ b/be/src/io/fs/s3_file_writer.h @@ -26,9 +26,9 @@ #include #include "common/status.h" +#include "cpp/obj-client/obj_storage_client.h" #include "io/fs/file_system.h" #include "io/fs/file_writer.h" -#include "io/fs/obj_storage_client.h" #include "io/fs/path.h" #include "io/fs/s3_file_bufferpool.h" @@ -60,15 +60,11 @@ class S3FileWriter final : public FileWriter { size_t bytes_appended() const override { return _bytes_appended; } State state() const override { return _state; } - const std::vector& completed_parts() const { return _completed_parts; } + const std::vector& completed_parts() const { return _completed_parts; } const std::string& key() const { return _obj_storage_path_opts.key; } const std::string& bucket() const { return _obj_storage_path_opts.bucket; } - std::string upload_id() const { - return _obj_storage_path_opts.upload_id.has_value() - ? _obj_storage_path_opts.upload_id.value() - : std::string(); - } + std::string upload_id() const { return _upload_id; } Status close(bool non_block = false) override; Status try_finish_close() override; @@ -88,12 +84,13 @@ class S3FileWriter final : public FileWriter { Status _submit_upload_buffer(const std::shared_ptr& buf); void _record_close_latency(); - ObjectStoragePathOptions _obj_storage_path_opts; + ObjStoragePath _obj_storage_path_opts; + std::string _upload_id; // Current Part Num for CompletedPart int _cur_part_num = 1; std::mutex _completed_lock; - std::vector _completed_parts; + std::vector _completed_parts; // **Attention** call add_count() before submitting buf to async thread pool bthread::CountdownEvent _countdown_event {0}; diff --git a/be/src/io/fs/s3_obj_storage_client.cpp b/be/src/io/fs/s3_obj_storage_client.cpp deleted file mode 100644 index 92b56b7b2780c3..00000000000000 --- a/be/src/io/fs/s3_obj_storage_client.cpp +++ /dev/null @@ -1,513 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "io/fs/s3_obj_storage_client.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include - -#include "common/logging.h" -#include "common/status.h" -#include "cpp/obj_retry_strategy.h" -#include "cpp/sync_point.h" -#include "io/fs/err_utils.h" -#include "io/fs/gcs_signed_url_provider.h" -#include "io/fs/s3_common.h" -#include "util/bvar_helper.h" - -// Rate limiting is applied by RateLimitedObjStorageClient, the decorator that -// S3ClientFactory wraps around this client when the bucket is subject to limiting. -namespace { -void record_s3_request_failed(const Aws::S3::S3Error& error) { - doris::record_object_request_failed(static_cast(error.GetResponseCode())); -} -} // namespace - -namespace Aws::S3::Model { -class DeleteObjectRequest; -} // namespace Aws::S3::Model - -using Aws::S3::Model::CompletedPart; -using Aws::S3::Model::CompletedMultipartUpload; -using Aws::S3::Model::CompleteMultipartUploadRequest; -using Aws::S3::Model::CreateMultipartUploadRequest; -using Aws::S3::Model::UploadPartRequest; -using Aws::S3::Model::UploadPartOutcome; - -namespace doris::io { -using namespace Aws::S3::Model; - -static constexpr int S3_REQUEST_THRESHOLD_MS = 5000; - -ObjectStorageUploadResponse S3ObjStorageClient::create_multipart_upload( - const ObjectStoragePathOptions& opts) { - CreateMultipartUploadRequest request; - request.WithBucket(opts.bucket).WithKey(opts.key); - request.SetContentType("application/octet-stream"); - - MonotonicStopWatch watch; - watch.start(); - - auto outcome = SYNC_POINT_HOOK_RETURN_VALUE(_client->CreateMultipartUpload(request), - "s3_file_writer::create_multi_part_upload", - std::cref(request).get()); - SYNC_POINT_CALLBACK("s3_file_writer::_open", &outcome); - watch.stop(); - - s3_bvar::s3_multi_part_upload_latency << watch.elapsed_time_microseconds(); - const auto& request_id = outcome.IsSuccess() ? outcome.GetResult().GetRequestId() - : outcome.GetError().GetRequestId(); - - LOG_IF(INFO, watch.elapsed_time_milliseconds() > S3_REQUEST_THRESHOLD_MS) - << "CreateMultipartUpload cost=" << watch.elapsed_time_milliseconds() << "ms" - << ", request_id=" << request_id << ", bucket=" << opts.bucket << ", key=" << opts.key; - - if (!outcome.IsSuccess()) { - record_s3_request_failed(outcome.GetError()); - auto st = s3fs_error(outcome.GetError(), fmt::format("failed to CreateMultipartUpload: {} ", - opts.path.native())); - LOG(WARNING) << st << " request_id=" << request_id; - return ObjectStorageUploadResponse { - .resp = {convert_to_obj_response(std::move(st)), - static_cast(outcome.GetError().GetResponseCode()), - outcome.GetError().GetRequestId()}, - }; - } - - return ObjectStorageUploadResponse {.upload_id {outcome.GetResult().GetUploadId()}}; -} - -ObjectStorageResponse S3ObjStorageClient::put_object(const ObjectStoragePathOptions& opts, - std::string_view stream) { - Aws::S3::Model::PutObjectRequest request; - request.WithBucket(opts.bucket).WithKey(opts.key); - auto string_view_stream = std::make_shared(stream.data(), stream.size()); - Aws::Utils::ByteBuffer part_md5(Aws::Utils::HashingUtils::CalculateMD5(*string_view_stream)); - request.SetContentMD5(Aws::Utils::HashingUtils::Base64Encode(part_md5)); - request.SetBody(string_view_stream); - request.SetContentLength(stream.size()); - request.SetContentType("application/octet-stream"); - - MonotonicStopWatch watch; - watch.start(); - auto outcome = - SYNC_POINT_HOOK_RETURN_VALUE(_client->PutObject(request), "s3_file_writer::put_object", - std::cref(request).get(), &stream); - - watch.stop(); - - s3_bvar::s3_put_latency << watch.elapsed_time_microseconds(); - const auto& request_id = outcome.IsSuccess() ? outcome.GetResult().GetRequestId() - : outcome.GetError().GetRequestId(); - - if (!outcome.IsSuccess()) { - record_s3_request_failed(outcome.GetError()); - auto st = s3fs_error(outcome.GetError(), - fmt::format("failed to put object: {}", opts.path.native())); - LOG(WARNING) << st << ", request_id=" << request_id; - return ObjectStorageResponse {convert_to_obj_response(std::move(st)), - static_cast(outcome.GetError().GetResponseCode()), - request_id}; - } - - LOG_IF(INFO, watch.elapsed_time_milliseconds() > S3_REQUEST_THRESHOLD_MS) - << "PutObject cost=" << watch.elapsed_time_milliseconds() << "ms" - << ", request_id=" << request_id << ", bucket=" << opts.bucket << ", key=" << opts.key; - return ObjectStorageResponse::OK(); -} - -ObjectStorageUploadResponse S3ObjStorageClient::upload_part(const ObjectStoragePathOptions& opts, - std::string_view stream, int part_num) { - UploadPartRequest request; - request.WithBucket(opts.bucket) - .WithKey(opts.key) - .WithPartNumber(part_num) - .WithUploadId(*opts.upload_id); - auto string_view_stream = std::make_shared(stream.data(), stream.size()); - - request.SetBody(string_view_stream); - - Aws::Utils::ByteBuffer part_md5(Aws::Utils::HashingUtils::CalculateMD5(*string_view_stream)); - request.SetContentMD5(Aws::Utils::HashingUtils::Base64Encode(part_md5)); - - request.SetContentLength(stream.size()); - request.SetContentType("application/octet-stream"); - - MonotonicStopWatch watch; - watch.start(); - auto outcome = SYNC_POINT_HOOK_RETURN_VALUE(_client->UploadPart(request), - "s3_file_writer::upload_part", - std::cref(request).get(), &stream); - - watch.stop(); - - s3_bvar::s3_multi_part_upload_latency << watch.elapsed_time_microseconds(); - const auto& request_id = outcome.IsSuccess() ? outcome.GetResult().GetRequestId() - : outcome.GetError().GetRequestId(); - - TEST_SYNC_POINT_CALLBACK("S3FileWriter::_upload_one_part", &outcome); - if (!outcome.IsSuccess()) { - record_s3_request_failed(outcome.GetError()); - auto st = Status::IOError( - "failed to UploadPart bucket={}, key={}, part_num={}, upload_id={}, message={}, " - "exception_name={}, response_code={}, request_id={}", - opts.bucket, opts.path.native(), part_num, *opts.upload_id, - outcome.GetError().GetMessage(), outcome.GetError().GetExceptionName(), - outcome.GetError().GetResponseCode(), request_id); - - LOG(WARNING) << st << ", request_id=" << request_id; - return ObjectStorageUploadResponse { - .resp = {convert_to_obj_response(std::move(st)), - static_cast(outcome.GetError().GetResponseCode()), - outcome.GetError().GetRequestId()}}; - } - - LOG_IF(INFO, watch.elapsed_time_milliseconds() > S3_REQUEST_THRESHOLD_MS) - << "UploadPart cost=" << watch.elapsed_time_milliseconds() << "ms" - << ", request_id=" << request_id << ", bucket=" << opts.bucket << ", key=" << opts.key - << ", part_num=" << part_num << ", upload_id=" << *opts.upload_id; - return ObjectStorageUploadResponse {.etag = outcome.GetResult().GetETag()}; -} - -ObjectStorageResponse S3ObjStorageClient::complete_multipart_upload( - const ObjectStoragePathOptions& opts, - const std::vector& completed_parts) { - CompleteMultipartUploadRequest request; - request.WithBucket(opts.bucket).WithKey(opts.key).WithUploadId(*opts.upload_id); - - CompletedMultipartUpload completed_upload; - std::vector complete_parts; - std::ranges::transform(completed_parts, std::back_inserter(complete_parts), - [](const ObjectCompleteMultiPart& part_ptr) { - CompletedPart part; - part.SetPartNumber(part_ptr.part_num); - part.SetETag(part_ptr.etag); - return part; - }); - completed_upload.SetParts(std::move(complete_parts)); - request.WithMultipartUpload(completed_upload); - - TEST_SYNC_POINT_RETURN_WITH_VALUE("S3FileWriter::_complete:3", ObjectStorageResponse(), this); - - MonotonicStopWatch watch; - watch.start(); - auto outcome = SYNC_POINT_HOOK_RETURN_VALUE(_client->CompleteMultipartUpload(request), - "s3_file_writer::complete_multi_part", - std::cref(request).get()); - - watch.stop(); - s3_bvar::s3_multi_part_upload_latency << watch.elapsed_time_microseconds(); - const auto& request_id = outcome.IsSuccess() ? outcome.GetResult().GetRequestId() - : outcome.GetError().GetRequestId(); - - if (!outcome.IsSuccess()) { - record_s3_request_failed(outcome.GetError()); - auto st = s3fs_error(outcome.GetError(), - fmt::format("failed to CompleteMultipartUpload: {}, upload_id={}", - opts.path.native(), *opts.upload_id)); - LOG(WARNING) << st << ", request_id=" << request_id; - return {convert_to_obj_response(std::move(st)), - static_cast(outcome.GetError().GetResponseCode()), - outcome.GetError().GetRequestId()}; - } - - LOG_IF(INFO, watch.elapsed_time_milliseconds() > S3_REQUEST_THRESHOLD_MS) - << "CompleteMultipartUpload cost=" << watch.elapsed_time_milliseconds() << "ms" - << ", request_id=" << request_id << ", bucket=" << opts.bucket << ", key=" << opts.key - << ", upload_id=" << *opts.upload_id; - return ObjectStorageResponse::OK(); -} - -ObjectStorageHeadResponse S3ObjStorageClient::head_object(const ObjectStoragePathOptions& opts) { - Aws::S3::Model::HeadObjectRequest request; - request.WithBucket(opts.bucket).WithKey(opts.key); - - SCOPED_BVAR_LATENCY(s3_bvar::s3_head_latency); - auto outcome = SYNC_POINT_HOOK_RETURN_VALUE( - _client->HeadObject(request), "s3_file_system::head_object", std::ref(request).get()); - if (outcome.IsSuccess()) { - return {.resp = {convert_to_obj_response(Status::OK())}, - .file_size = outcome.GetResult().GetContentLength()}; - } else if (outcome.GetError().GetResponseCode() == Aws::Http::HttpResponseCode::NOT_FOUND) { - return {.resp = {convert_to_obj_response(Status::Error(""))}}; - } else { - record_s3_request_failed(outcome.GetError()); - return {.resp = {convert_to_obj_response( - s3fs_error(outcome.GetError(), - fmt::format("failed to check exists {}", opts.key))), - static_cast(outcome.GetError().GetResponseCode()), - outcome.GetError().GetRequestId()}}; - } -} - -ObjectStorageResponse S3ObjStorageClient::get_object(const ObjectStoragePathOptions& opts, - void* buffer, size_t offset, size_t bytes_read, - size_t* size_return) { - Aws::S3::Model::GetObjectRequest request; - request.WithBucket(opts.bucket).WithKey(opts.key); - request.SetRange(fmt::format("bytes={}-{}", offset, offset + bytes_read - 1)); - request.SetResponseStreamFactory(AwsWriteableStreamFactory(buffer, bytes_read)); - - SCOPED_BVAR_LATENCY(s3_bvar::s3_get_latency); - auto outcome = _client->GetObject(request); - if (!outcome.IsSuccess()) { - record_s3_request_failed(outcome.GetError()); - return {convert_to_obj_response(s3fs_error( - outcome.GetError(), - fmt::format("failed to get object: bucket={} object={} offset={} size={}", - opts.bucket, opts.key, offset, bytes_read))), - static_cast(outcome.GetError().GetResponseCode()), - outcome.GetError().GetRequestId()}; - } - *size_return = outcome.GetResult().GetContentLength(); - // Short read, or a server or a proxy answering a ranged read with the whole object. - SYNC_POINT_CALLBACK("s3_obj_storage_client::get_object", size_return); - if (*size_return != bytes_read) { - return {convert_to_obj_response( - Status::InternalError("incomplete read from bucket={} object={} offset={}, expect " - "{}, got {}, request_id={}", - opts.bucket, opts.key, offset, bytes_read, *size_return, - outcome.GetResult().GetRequestId()))}; - } - return ObjectStorageResponse::OK(); -} - -ObjectStorageResponse S3ObjStorageClient::list_objects(const ObjectStoragePathOptions& opts, - std::vector* files) { - Aws::S3::Model::ListObjectsV2Request request; - request.WithBucket(opts.bucket).WithPrefix(opts.prefix); - bool is_trucated = false; - do { - Aws::S3::Model::ListObjectsV2Outcome outcome; - { - SCOPED_BVAR_LATENCY(s3_bvar::s3_list_latency); - outcome = _client->ListObjectsV2(request); - } - if (!outcome.IsSuccess()) { - files->clear(); - // Treat NoSuchKey as empty response for compatibility with some S3-compatible storage providers - // e.g. TOS by ByteDance Cloud (Volcano Engine) - if (outcome.GetError().GetErrorType() == Aws::S3::S3Errors::NO_SUCH_KEY) { - LOG(INFO) << "NoSuchKey error when listing objects, treat as empty response" - << ", prefix=" << opts.prefix - << ", request_id=" << outcome.GetError().GetRequestId(); - return ObjectStorageResponse::OK(); - } - - record_s3_request_failed(outcome.GetError()); - return {convert_to_obj_response(s3fs_error( - outcome.GetError(), fmt::format("failed to list {}", opts.prefix))), - static_cast(outcome.GetError().GetResponseCode()), - outcome.GetError().GetRequestId()}; - } - for (const auto& obj : outcome.GetResult().GetContents()) { - std::string key = obj.GetKey(); - bool is_dir = (key.back() == '/'); - FileInfo file_info; - file_info.file_name = obj.GetKey(); - file_info.file_size = obj.GetSize(); - file_info.is_file = !is_dir; - files->push_back(std::move(file_info)); - } - is_trucated = outcome.GetResult().GetIsTruncated(); - if (is_trucated && outcome.GetResult().GetNextContinuationToken().empty()) { - return {convert_to_obj_response( - Status::InternalError("failed to list {}, is_trucated is true, but next " - "continuation token is empty, request_id={}", - opts.prefix, outcome.GetResult().GetRequestId()))}; - } - - request.SetContinuationToken(outcome.GetResult().GetNextContinuationToken()); - } while (is_trucated); - return ObjectStorageResponse::OK(); -} - -ObjectStorageResponse S3ObjStorageClient::delete_objects(const ObjectStoragePathOptions& opts, - std::vector objs) { - Aws::S3::Model::DeleteObjectsRequest delete_request; - delete_request.SetBucket(opts.bucket); - Aws::S3::Model::Delete del; - Aws::Vector objects; - std::ranges::transform(objs, std::back_inserter(objects), [](auto&& obj_key) { - Aws::S3::Model::ObjectIdentifier obj_identifier; - obj_identifier.SetKey(std::move(obj_key)); - return obj_identifier; - }); - del.WithObjects(std::move(objects)).SetQuiet(true); - delete_request.SetDelete(std::move(del)); - SCOPED_BVAR_LATENCY(s3_bvar::s3_delete_objects_latency); - auto delete_outcome = _client->DeleteObjects(delete_request); - if (!delete_outcome.IsSuccess()) { - record_s3_request_failed(delete_outcome.GetError()); - return {convert_to_obj_response( - s3fs_error(delete_outcome.GetError(), - fmt::format("failed to delete dir {}", opts.key))), - static_cast(delete_outcome.GetError().GetResponseCode()), - delete_outcome.GetError().GetRequestId()}; - } - // case for partial delete object failure - SYNC_POINT_CALLBACK("s3_obj_storage_client::delete_objects", &delete_outcome); - if (!delete_outcome.GetResult().GetErrors().empty()) { - const auto& e = delete_outcome.GetResult().GetErrors().front(); - return {convert_to_obj_response( - Status::InternalError("failed to delete object {}: {}, request_id={}", e.GetKey(), - e.GetMessage(), delete_outcome.GetResult().GetRequestId()))}; - } - return ObjectStorageResponse::OK(); -} - -ObjectStorageResponse S3ObjStorageClient::delete_object(const ObjectStoragePathOptions& opts) { - Aws::S3::Model::DeleteObjectRequest request; - request.WithBucket(opts.bucket).WithKey(opts.key); - - SCOPED_BVAR_LATENCY(s3_bvar::s3_delete_object_latency); - auto outcome = _client->DeleteObject(request); - if (outcome.IsSuccess() || - outcome.GetError().GetResponseCode() == Aws::Http::HttpResponseCode::NOT_FOUND) { - return ObjectStorageResponse::OK(); - } - record_s3_request_failed(outcome.GetError()); - return {convert_to_obj_response(s3fs_error(outcome.GetError(), - fmt::format("failed to delete file {}", opts.key))), - static_cast(outcome.GetError().GetResponseCode()), - outcome.GetError().GetRequestId()}; -} - -ObjectStorageResponse S3ObjStorageClient::delete_objects_recursively( - const ObjectStoragePathOptions& opts) { - Aws::S3::Model::ListObjectsV2Request request; - request.WithBucket(opts.bucket).WithPrefix(opts.prefix); - Aws::S3::Model::DeleteObjectsRequest delete_request; - delete_request.SetBucket(opts.bucket); - bool is_trucated = false; - do { - Aws::S3::Model::ListObjectsV2Outcome outcome; - { - SCOPED_BVAR_LATENCY(s3_bvar::s3_list_latency); - outcome = _client->ListObjectsV2(request); - } - if (!outcome.IsSuccess()) { - record_s3_request_failed(outcome.GetError()); - return {convert_to_obj_response(s3fs_error( - outcome.GetError(), - fmt::format("failed to list objects when delete dir {}", opts.prefix))), - static_cast(outcome.GetError().GetResponseCode()), - outcome.GetError().GetRequestId()}; - } - const auto& result = outcome.GetResult(); - Aws::Vector objects; - objects.reserve(result.GetContents().size()); - for (const auto& obj : result.GetContents()) { - objects.emplace_back().SetKey(obj.GetKey()); - } - if (!objects.empty()) { - Aws::S3::Model::Delete del; - del.WithObjects(std::move(objects)).SetQuiet(true); - delete_request.SetDelete(std::move(del)); - SCOPED_BVAR_LATENCY(s3_bvar::s3_delete_objects_latency); - auto delete_outcome = _client->DeleteObjects(delete_request); - if (!delete_outcome.IsSuccess()) { - record_s3_request_failed(delete_outcome.GetError()); - return {convert_to_obj_response( - s3fs_error(delete_outcome.GetError(), - fmt::format("failed to delete dir {}", opts.key))), - static_cast(delete_outcome.GetError().GetResponseCode()), - delete_outcome.GetError().GetRequestId()}; - } - // case for partial delete object failure - SYNC_POINT_CALLBACK("s3_obj_storage_client::delete_objects_recursively", - &delete_outcome); - if (!delete_outcome.GetResult().GetErrors().empty()) { - const auto& e = delete_outcome.GetResult().GetErrors().front(); - return {convert_to_obj_response(Status::InternalError( - "failed to delete object {}: {}, request_id={}", opts.key, e.GetMessage(), - delete_outcome.GetResult().GetRequestId()))}; - } - } - is_trucated = result.GetIsTruncated(); - request.SetContinuationToken(result.GetNextContinuationToken()); - } while (is_trucated); - return ObjectStorageResponse::OK(); -} - -std::string S3ObjStorageClient::generate_presigned_url(const ObjectStoragePathOptions& opts, - int64_t expiration_secs, - const S3ClientConf& conf) { - if (const auto* gcp_credential = std::get_if(&conf.credential)) { - std::string signed_url; - auto status = generate_gcs_v4_signed_url({.endpoint = conf.endpoint, - .bucket = opts.bucket, - .key = opts.key, - .expiration_secs = expiration_secs, - .request_timeout_ms = conf.request_timeout_ms}, - *gcp_credential, _gcs_signing_token_provider, - &signed_url); - if (!status.ok()) { - LOG(WARNING) << "failed to generate GCS V4 signed URL: " << status; - return {}; - } - return signed_url; - } - return _client->GeneratePresignedUrl(opts.bucket, opts.key, Aws::Http::HttpMethod::HTTP_GET, - expiration_secs); -} - -} // namespace doris::io diff --git a/be/src/io/fs/s3_obj_storage_client.h b/be/src/io/fs/s3_obj_storage_client.h deleted file mode 100644 index 532ef7058decff..00000000000000 --- a/be/src/io/fs/s3_obj_storage_client.h +++ /dev/null @@ -1,72 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include "io/fs/obj_storage_client.h" -#include "io/fs/s3_file_system.h" - -namespace Aws::S3 { -class S3Client; -namespace Model { -class CompletedPart; -} -} // namespace Aws::S3 - -namespace doris { -class GcpTokenProvider; -} - -namespace doris::io { -class ObjClientHolder; - -class S3ObjStorageClient final : public ObjStorageClient { -public: - explicit S3ObjStorageClient( - std::shared_ptr client, - std::shared_ptr gcs_signing_token_provider = nullptr) - : _client(std::move(client)), - _gcs_signing_token_provider(std::move(gcs_signing_token_provider)) {} - ~S3ObjStorageClient() override = default; - ObjectStorageUploadResponse create_multipart_upload( - const ObjectStoragePathOptions& opts) override; - ObjectStorageResponse put_object(const ObjectStoragePathOptions& opts, - std::string_view stream) override; - ObjectStorageUploadResponse upload_part(const ObjectStoragePathOptions& opts, std::string_view, - int partNum) override; - ObjectStorageResponse complete_multipart_upload( - const ObjectStoragePathOptions& opts, - const std::vector& completed_parts) override; - ObjectStorageHeadResponse head_object(const ObjectStoragePathOptions& opts) override; - ObjectStorageResponse get_object(const ObjectStoragePathOptions& opts, void* buffer, - size_t offset, size_t bytes_read, - size_t* size_return) override; - ObjectStorageResponse list_objects(const ObjectStoragePathOptions& opts, - std::vector* files) override; - ObjectStorageResponse delete_objects(const ObjectStoragePathOptions& opts, - std::vector objs) override; - ObjectStorageResponse delete_object(const ObjectStoragePathOptions& opts) override; - ObjectStorageResponse delete_objects_recursively(const ObjectStoragePathOptions& opts) override; - std::string generate_presigned_url(const ObjectStoragePathOptions& opts, - int64_t expiration_secs, const S3ClientConf&) override; - -private: - std::shared_ptr _client; - std::shared_ptr _gcs_signing_token_provider; -}; - -} // namespace doris::io diff --git a/be/src/util/s3_rate_limiter_manager.h b/be/src/util/s3_rate_limiter_manager.h index ed668b02f0db0f..0e548270638080 100644 --- a/be/src/util/s3_rate_limiter_manager.h +++ b/be/src/util/s3_rate_limiter_manager.h @@ -84,7 +84,8 @@ class S3RateLimiterManager { std::array, 2> _bytes_limiters; }; -// RAII admission for one logical object storage request. +// RAII admission for one logical object-storage call. RateLimitedObjStorageClient constructs the +// guard before forwarding the call to its inner client. // // The constructor charges the QPS bucket (may sleep when throttled; rejected only by // the legacy token_limit cumulative cap) and then reserves `estimated_bytes` from the diff --git a/be/src/util/s3_util.cpp b/be/src/util/s3_util.cpp index f069308ee78878..d011ed2488f61f 100644 --- a/be/src/util/s3_util.cpp +++ b/be/src/util/s3_util.cpp @@ -20,18 +20,11 @@ #include #include #include -#include #include #include #include #include -#include #include -#include -#include -#include - -#include #include "util/string_util.h" @@ -51,62 +44,51 @@ #include "common/config.h" #include "common/logging.h" #include "common/status.h" +#include "cpp/obj-client/auth/aws_credential_factory.h" +#ifdef USE_AZURE +#include "cpp/obj-client/auth/azure_auth_factory.h" +#include "cpp/obj-client/azure_obj_storage_client.h" +#endif +#include "cloud/config.h" #include "cpp/aws_logger.h" -#include "cpp/custom_aws_credentials_provider_chain.h" #include "cpp/obj-client/auth/gcp/gcp_token_provider.h" #include "cpp/obj-client/auth/obj_credential.h" #include "cpp/obj-client/auth/obj_s3_client_factory.h" +#include "cpp/obj-client/rate_limited_obj_storage_client.h" +#include "cpp/obj-client/s3_obj_storage_client.h" #include "cpp/obj_retry_strategy.h" #include "cpp/sync_point.h" #include "cpp/util.h" -#ifdef USE_AZURE -#include "io/fs/azure_obj_storage_client.h" -#endif -#include "cloud/config.h" #include "exec/scan/scanner_scheduler.h" -#include "io/fs/obj_storage_client.h" -#include "io/fs/rate_limited_obj_storage_client.h" -#include "io/fs/s3_obj_storage_client.h" +#include "io/fs/gcs_signed_url_provider.h" #include "runtime/exec_env.h" +#include "util/s3_rate_limiter_manager.h" #include "util/s3_uri.h" namespace doris { -namespace s3_bvar { -bvar::LatencyRecorder s3_get_latency("s3_get"); -bvar::LatencyRecorder s3_put_latency("s3_put"); -bvar::LatencyRecorder s3_delete_object_latency("s3_delete_object"); -bvar::LatencyRecorder s3_delete_objects_latency("s3_delete_objects"); -bvar::LatencyRecorder s3_head_latency("s3_head"); -bvar::LatencyRecorder s3_multi_part_upload_latency("s3_multi_part_upload"); -bvar::LatencyRecorder s3_list_latency("s3_list"); -bvar::LatencyRecorder s3_list_object_versions_latency("s3_list_object_versions"); -bvar::LatencyRecorder s3_get_bucket_version_latency("s3_get_bucket_version"); -bvar::LatencyRecorder s3_copy_object_latency("s3_copy_object"); -}; // namespace s3_bvar - namespace { // Adapter between the IO storage-domain enum and the credential layer. Keep it // local: neither layer should own or depend on the other's enum definition. -ObjCredentialProvider to_obj_credential_provider(io::ObjStorageType provider) { +ObjCredentialProvider to_obj_credential_provider(io::ObjStorageProvider provider) { switch (provider) { - case io::ObjStorageType::AWS: + case io::ObjStorageProvider::AWS: return ObjCredentialProvider::Aws; - case io::ObjStorageType::AZURE: + case io::ObjStorageProvider::AZURE: return ObjCredentialProvider::Azure; - case io::ObjStorageType::BOS: + case io::ObjStorageProvider::BOS: return ObjCredentialProvider::Bos; - case io::ObjStorageType::COS: + case io::ObjStorageProvider::COS: return ObjCredentialProvider::Cos; - case io::ObjStorageType::GCP: + case io::ObjStorageProvider::GCP: return ObjCredentialProvider::Gcp; - case io::ObjStorageType::OBS: + case io::ObjStorageProvider::OBS: return ObjCredentialProvider::Obs; - case io::ObjStorageType::OSS: + case io::ObjStorageProvider::OSS: return ObjCredentialProvider::Oss; - case io::ObjStorageType::TOS: + case io::ObjStorageProvider::TOS: return ObjCredentialProvider::Tos; - case io::ObjStorageType::UNKNOWN: + case io::ObjStorageProvider::UNKNOWN: return ObjCredentialProvider::Unknown; } return ObjCredentialProvider::Unknown; @@ -150,6 +132,33 @@ doris::Status is_s3_conf_valid(const S3ClientConf& conf) { return Status::OK(); } +ObjStorageResponse make_be_rate_limit_response(S3RateLimitType type, + S3RateLimitRejectReason reason) { + const auto* limit_type = reason == S3RateLimitRejectReason::QPS ? "QPS" : "bytes"; + // A local admission rejection is not an S3 HTTP 429. Keep the merged #65420 behavior so S3 + // readers do not retry it as provider throttling. + return ObjStorageResponse::rate_limit( + ErrorCode::EXCEEDED_LIMIT, 0, + fmt::format("s3 {} request exceeds {} limit, rejected by BE rate limiter", + to_string(type), limit_type)); +} + +class BeObjStorageRateLimitPolicy final : public ObjStorageRateLimitPolicy { +public: + ObjStorageAdmission acquire(S3RateLimitType type, size_t estimated_bytes) const override { + auto guard = std::make_shared(type, estimated_bytes); + if (!guard->ok()) { + return ObjStorageAdmission { + .resp = make_be_rate_limit_response(type, guard->reject_reason()), + }; + } + return ObjStorageAdmission { + .settle = [guard = std::move(guard)]( + size_t actual_bytes) { guard->settle(actual_bytes); }, + }; + } +}; + // Return true is convert `str` to int successfully bool to_int(std::string_view str, int& res) { auto [_, ec] = std::from_chars(str.data(), str.data() + str.size(), res); @@ -211,7 +220,7 @@ S3ClientFactory::S3ClientFactory() { return std::make_shared(logLevel); }; Aws::InitAPI(_aws_options); - _ca_cert_file_path = get_valid_ca_cert_path(doris::split(config::ca_cert_file_paths, ";")); + _get_ca_cert_file_path(); #ifdef USE_AZURE auto azureLogLevel = @@ -250,10 +259,8 @@ S3ClientFactory& S3ClientFactory::instance() { return ret; } -std::shared_ptr S3ClientFactory::create(const S3ClientConf& s3_conf) { - if (!is_s3_conf_valid(s3_conf).ok()) { - return nullptr; - } +Result> S3ClientFactory::create(const S3ClientConf& s3_conf) { + RETURN_IF_ERROR_RESULT(is_s3_conf_valid(s3_conf)); #ifdef BE_TEST { @@ -272,16 +279,16 @@ std::shared_ptr S3ClientFactory::create(const S3ClientConf } } - auto obj_client = (s3_conf.provider == io::ObjStorageType::AZURE) - ? _create_azure_client(s3_conf) - : _create_s3_client(s3_conf); - - // Rate limiting lives in one decorator, decided here at construction time: - // in cloud mode only internal storage-vault buckets are limited; external buckets - // (S3 load, TVF, external catalogs) get the bare client. In non-cloud mode every - // client is wrapped, preserving the legacy behavior. - if (obj_client != nullptr && (!config::is_cloud_mode() || s3_conf.is_internal_bucket)) { - obj_client = std::make_shared(std::move(obj_client)); + auto client_result = (s3_conf.provider == io::ObjStorageProvider::AZURE) + ? _create_azure_client(s3_conf) + : _create_s3_client(s3_conf); + if (!client_result.has_value()) { + return ResultError(std::move(client_result).error()); + } + auto obj_client = std::move(client_result).value(); + if (!config::is_cloud_mode() || s3_conf.is_internal_bucket) { + obj_client = std::make_shared( + std::move(obj_client), std::make_shared()); } { @@ -304,12 +311,9 @@ void S3ClientFactory::clear_client_creator_for_test() { } #endif -std::shared_ptr S3ClientFactory::_create_azure_client( +Result> S3ClientFactory::_create_azure_client( const S3ClientConf& s3_conf) { #ifdef USE_AZURE - auto cred = - std::make_shared(s3_conf.ak, s3_conf.sk); - const std::string container_name = s3_conf.bucket; std::string uri = fmt::format("{}/{}", s3_conf.endpoint, container_name); if (s3_conf.endpoint.find("://") == std::string::npos) { @@ -320,157 +324,89 @@ std::shared_ptr S3ClientFactory::_create_azure_client( options.Retry.StatusCodes.insert(Azure::Core::Http::HttpStatusCode::TooManyRequests); options.Retry.MaxRetries = config::max_s3_client_retry; options.PerRetryPolicies.emplace_back(std::make_unique()); - if (_ca_cert_file_path.empty()) { - _ca_cert_file_path = get_valid_ca_cert_path(doris::split(config::ca_cert_file_paths, ";")); - } - if (!_ca_cert_file_path.empty()) { + auto ca_cert_file_path = _get_ca_cert_file_path(); + if (!ca_cert_file_path.empty()) { Azure::Core::Http::CurlTransportOptions curl_options; - curl_options.CAInfo = _ca_cert_file_path; + curl_options.CAInfo = ca_cert_file_path; options.Transport.Transport = std::make_shared(std::move(curl_options)); } std::string normalized_uri = normalize_http_uri(uri); VLOG_DEBUG << "uri:" << uri << ", normalized_uri:" << normalized_uri; - std::string tls_debug_context = build_azure_tls_debug_context(_ca_cert_file_path); - - auto containerClient = std::make_shared( - uri, cred, std::move(options)); + std::string tls_debug_context = build_azure_tls_debug_context(ca_cert_file_path); + + auto built = AzureAuthFactory::create(uri, + { + .type = AzureCredentialType::SHARED_KEY, + .account_name = s3_conf.ak, + .account_key = s3_conf.sk, + }, + std::move(options)); + if (!built) { + return ResultError( + Status::InvalidArgument("failed to create Azure client: {}", built.error)); + } LOG_INFO("create one azure client with {}", s3_conf.to_string()); - return std::make_shared(std::move(containerClient), - std::move(tls_debug_context)); + return std::make_shared( + std::move(built.container_client), + ObjStorageEndpointInfo { + .endpoint = s3_conf.endpoint, + .ak = s3_conf.ak, + .sk = s3_conf.sk, + .tls_debug_context = std::move(tls_debug_context), + }, + std::move(built.shared_key_credential)); #else - LOG_FATAL("BE is not compiled with azure support, export BUILD_AZURE=ON before building"); - return nullptr; + return ResultError(Status::NotSupported( + "BE is not compiled with azure support, export BUILD_AZURE=ON before building")); #endif } -std::shared_ptr -S3ClientFactory::_get_aws_credentials_provider_v1(const S3ClientConf& s3_conf) { - if (!s3_conf.ak.empty() && !s3_conf.sk.empty()) { - Aws::Auth::AWSCredentials aws_cred(s3_conf.ak, s3_conf.sk); - DCHECK(!aws_cred.IsExpiredOrEmpty()); - if (!s3_conf.token.empty()) { - aws_cred.SetSessionToken(s3_conf.token); - } - return std::make_shared(std::move(aws_cred)); - } - - if (s3_conf.cred_provider_type == CredProviderType::InstanceProfile) { - if (s3_conf.role_arn.empty()) { - return std::make_shared(); - } - - Aws::Client::ClientConfiguration clientConfiguration = - S3ClientFactory::getClientConfiguration(); - - if (_ca_cert_file_path.empty()) { - _ca_cert_file_path = - get_valid_ca_cert_path(doris::split(config::ca_cert_file_paths, ";")); - } - if (!_ca_cert_file_path.empty()) { - clientConfiguration.caFile = _ca_cert_file_path; - } - - auto stsClient = std::make_shared( - std::make_shared(), - clientConfiguration); - - return std::make_shared( - s3_conf.role_arn, Aws::String(), s3_conf.external_id, - Aws::Auth::DEFAULT_CREDS_LOAD_FREQ_SECONDS, stsClient); - } - - // Support anonymous access for public datasets when no credentials are provided - if (s3_conf.ak.empty() && s3_conf.sk.empty()) { - return std::make_shared(); - } - - return std::make_shared(); -} - -std::shared_ptr S3ClientFactory::_create_credentials_provider( - CredProviderType type) { - switch (type) { - case CredProviderType::Env: - return std::make_shared(); - case CredProviderType::SystemProperties: - return std::make_shared(); - case CredProviderType::WebIdentity: - return std::make_shared(); - case CredProviderType::Container: - return create_container_credentials_provider(); - case CredProviderType::InstanceProfile: - return std::make_shared(); - case CredProviderType::Anonymous: - return std::make_shared(); - case CredProviderType::Default: - default: - return std::make_shared(); - } -} - -std::shared_ptr -S3ClientFactory::_get_aws_credentials_provider_v2(const S3ClientConf& s3_conf) { - if (!s3_conf.ak.empty() && !s3_conf.sk.empty()) { - Aws::Auth::AWSCredentials aws_cred(s3_conf.ak, s3_conf.sk); - DCHECK(!aws_cred.IsExpiredOrEmpty()); - if (!s3_conf.token.empty()) { - aws_cred.SetSessionToken(s3_conf.token); - } - return std::make_shared(std::move(aws_cred)); - } - - // Handle role_arn for assume role scenario - if (!s3_conf.role_arn.empty()) { - Aws::Client::ClientConfiguration clientConfiguration = - S3ClientFactory::getClientConfiguration(); - - if (_ca_cert_file_path.empty()) { - _ca_cert_file_path = - get_valid_ca_cert_path(doris::split(config::ca_cert_file_paths, ";")); - } - if (!_ca_cert_file_path.empty()) { - clientConfiguration.caFile = _ca_cert_file_path; - } - - auto baseProvider = _create_credentials_provider(s3_conf.cred_provider_type); - auto stsClient = std::make_shared(baseProvider, clientConfiguration); - - return std::make_shared( - s3_conf.role_arn, Aws::String(), s3_conf.external_id, - Aws::Auth::DEFAULT_CREDS_LOAD_FREQ_SECONDS, stsClient); +std::string S3ClientFactory::_get_ca_cert_file_path() { + std::lock_guard lock(_ca_cert_lock); + if (_ca_cert_file_path.empty()) { + _ca_cert_file_path = get_valid_ca_cert_path(doris::split(config::ca_cert_file_paths, ";")); } - - // Return provider based on cred_provider_type - return _create_credentials_provider(s3_conf.cred_provider_type); + return _ca_cert_file_path; } -std::shared_ptr S3ClientFactory::get_aws_credentials_provider( - const S3ClientConf& s3_conf) { - if (config::aws_credentials_provider_version == "v2") { - return _get_aws_credentials_provider_v2(s3_conf); - } - return _get_aws_credentials_provider_v1(s3_conf); +AwsCredentialResult S3ClientFactory::create_aws_credentials_provider(const S3ClientConf& s3_conf) { + auto sts_config = S3ClientFactory::getClientConfiguration(); + auto ca_cert_file_path = _get_ca_cert_file_path(); + if (!ca_cert_file_path.empty()) { + sts_config.caFile = ca_cert_file_path; + } + return AwsCredentialFactory::create({ + .version = config::aws_credentials_provider_version == "v2" + ? AwsCredentialProviderVersion::V2 + : AwsCredentialProviderVersion::V1, + .access_key = s3_conf.ak, + .secret_key = s3_conf.sk, + .session_token = s3_conf.token, + .provider_type = s3_conf.cred_provider_type, + .role_arn = s3_conf.role_arn, + .external_id = s3_conf.external_id, + .empty_credentials = EmptyCredentialsBehavior::ANONYMOUS, + .sts_client_config = std::move(sts_config), + }); } -std::shared_ptr S3ClientFactory::_create_s3_client( +Result> S3ClientFactory::_create_s3_client( const S3ClientConf& s3_conf) { TEST_SYNC_POINT_RETURN_WITH_VALUE( "s3_client_factory::create", - std::make_shared(std::make_shared())); + std::make_shared(std::make_shared(), + ObjStorageEndpointInfo {})); Aws::Client::ClientConfiguration aws_config = S3ClientFactory::getClientConfiguration(); if (s3_conf.need_override_endpoint) { aws_config.endpointOverride = s3_conf.endpoint; } aws_config.region = s3_conf.region; - if (_ca_cert_file_path.empty()) { - _ca_cert_file_path = get_valid_ca_cert_path(doris::split(config::ca_cert_file_paths, ";")); - } - - if (!_ca_cert_file_path.empty()) { - aws_config.caFile = _ca_cert_file_path; + auto ca_cert_file_path = _get_ca_cert_file_path(); + if (!ca_cert_file_path.empty()) { + aws_config.caFile = ca_cert_file_path; } if (s3_conf.max_connections > 0) { @@ -493,32 +429,53 @@ std::shared_ptr S3ClientFactory::_create_s3_client( aws_config.retryStrategy = std::make_shared( config::max_s3_client_retry /*scaleFactor = 25*/, /*retry_slow_down=*/true); + auto credentials = create_aws_credentials_provider(s3_conf); + if (!credentials) { + return ResultError(Status::InvalidArgument("failed to create AWS credential provider: {}", + credentials.error)); + } auto new_client = make_s3_client( s3_conf.credential, S3ClientBuildContext { - .fallback_provider = get_aws_credentials_provider(s3_conf), + .fallback_provider = std::move(credentials.provider), .config = std::move(aws_config), .payload_signing_policy = Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, .use_virtual_addressing = s3_conf.use_virtual_addressing, - .ca_cert_path = _ca_cert_file_path, + .ca_cert_path = ca_cert_file_path, }); - - std::shared_ptr gcs_signing_token_provider; + S3ObjStorageClient::PresignedUrlGenerator presigned_url_generator; if (const auto* gcp_credential = std::get_if(&s3_conf.credential)) { - // signBlob is authorized by the source identity, not by the optional - // impersonated target identity. Keep one refreshable cloud-platform - // token provider for the lifetime of the S3 client. auto signing_caller_credential = *gcp_credential; signing_caller_credential.impersonation_service_account.clear(); - gcs_signing_token_provider = std::make_shared( - signing_caller_credential, _ca_cert_file_path, GcpTokenScope::CloudPlatform); - } - - auto obj_client = std::make_shared( - std::move(new_client), std::move(gcs_signing_token_provider)); + auto token_provider = std::make_shared( + signing_caller_credential, ca_cert_file_path, GcpTokenScope::CloudPlatform); + presigned_url_generator = [credential = *gcp_credential, + token_provider = std::move(token_provider), + endpoint = s3_conf.endpoint, + timeout_ms = s3_conf.request_timeout_ms]( + const ObjStoragePath& path, int64_t expiration_secs) { + std::string signed_url; + auto status = io::generate_gcs_v4_signed_url({.endpoint = endpoint, + .bucket = path.bucket, + .key = path.key, + .expiration_secs = expiration_secs, + .request_timeout_ms = timeout_ms}, + credential, token_provider, &signed_url); + if (!status.ok()) { + LOG(WARNING) << "failed to generate GCS V4 signed URL: " << status; + return std::string {}; + } + return signed_url; + }; + } + auto provider_client = std::make_shared( + std::move(new_client), + ObjStorageEndpointInfo { + .endpoint = s3_conf.endpoint, .ak = s3_conf.ak, .sk = s3_conf.sk}, + std::move(presigned_url_generator)); LOG_INFO("create one s3 client with {}", s3_conf.to_string()); - return obj_client; + return provider_client; } Status S3ClientFactory::convert_properties_to_s3_conf( @@ -562,9 +519,9 @@ Status S3ClientFactory::convert_properties_to_s3_conf( if (auto it = properties.find(S3_PROVIDER); it != properties.end()) { // S3 Provider properties should be case insensitive. if (0 == strcasecmp(it->second.c_str(), AZURE_PROVIDER_STRING)) { - s3_conf->client_conf.provider = io::ObjStorageType::AZURE; + s3_conf->client_conf.provider = io::ObjStorageProvider::AZURE; } else if (0 == strcasecmp(it->second.c_str(), GCP_PROVIDER_STRING)) { - s3_conf->client_conf.provider = io::ObjStorageType::GCP; + s3_conf->client_conf.provider = io::ObjStorageProvider::GCP; } } @@ -643,9 +600,9 @@ S3Conf S3Conf::get_s3_conf(const cloud::ObjectStoreInfoPB& info) { .region = info.region(), .ak = info.ak(), .sk = info.sk(), - .token {}, + .token = {}, .bucket = info.bucket(), - .provider = io::ObjStorageType::AWS, + .provider = io::ObjStorageProvider::AWS, .use_virtual_addressing = info.has_use_path_style() ? !info.use_path_style() : true, @@ -663,31 +620,31 @@ S3Conf S3Conf::get_s3_conf(const cloud::ObjectStoreInfoPB& info) { } convert_obj_credential(info, &ret.client_conf.credential); - io::ObjStorageType type = io::ObjStorageType::AWS; + io::ObjStorageProvider type = io::ObjStorageProvider::AWS; switch (info.provider()) { case cloud::ObjectStoreInfoPB_Provider_OSS: - type = io::ObjStorageType::OSS; + type = io::ObjStorageProvider::OSS; break; case cloud::ObjectStoreInfoPB_Provider_S3: - type = io::ObjStorageType::AWS; + type = io::ObjStorageProvider::AWS; break; case cloud::ObjectStoreInfoPB_Provider_COS: - type = io::ObjStorageType::COS; + type = io::ObjStorageProvider::COS; break; case cloud::ObjectStoreInfoPB_Provider_OBS: - type = io::ObjStorageType::OBS; + type = io::ObjStorageProvider::OBS; break; case cloud::ObjectStoreInfoPB_Provider_BOS: - type = io::ObjStorageType::BOS; + type = io::ObjStorageProvider::BOS; break; case cloud::ObjectStoreInfoPB_Provider_GCP: - type = io::ObjStorageType::GCP; + type = io::ObjStorageProvider::GCP; break; case cloud::ObjectStoreInfoPB_Provider_AZURE: - type = io::ObjStorageType::AZURE; + type = io::ObjStorageProvider::AZURE; break; case cloud::ObjectStoreInfoPB_Provider_TOS: - type = io::ObjStorageType::TOS; + type = io::ObjStorageProvider::TOS; break; default: __builtin_unreachable(); @@ -708,7 +665,7 @@ S3Conf S3Conf::get_s3_conf(const TS3StorageParam& param) { .sk = param.sk, .token = param.token, .bucket = param.bucket, - .provider = io::ObjStorageType::AWS, + .provider = io::ObjStorageProvider::AWS, .max_connections = param.max_conn, .request_timeout_ms = param.request_timeout_ms, .connect_timeout_ms = param.conn_timeout_ms, @@ -727,36 +684,36 @@ S3Conf S3Conf::get_s3_conf(const TS3StorageParam& param) { convert_obj_credential(param.credential, &ret.client_conf.credential); } - io::ObjStorageType type = io::ObjStorageType::AWS; + io::ObjStorageProvider type = io::ObjStorageProvider::AWS; switch (param.provider) { case TObjStorageType::UNKNOWN: LOG_INFO("Receive one legal storage resource, set provider type to aws, param detail {}", ret.to_string()); - type = io::ObjStorageType::AWS; + type = io::ObjStorageProvider::AWS; break; case TObjStorageType::AWS: - type = io::ObjStorageType::AWS; + type = io::ObjStorageProvider::AWS; break; case TObjStorageType::AZURE: - type = io::ObjStorageType::AZURE; + type = io::ObjStorageProvider::AZURE; break; case TObjStorageType::BOS: - type = io::ObjStorageType::BOS; + type = io::ObjStorageProvider::BOS; break; case TObjStorageType::COS: - type = io::ObjStorageType::COS; + type = io::ObjStorageProvider::COS; break; case TObjStorageType::OBS: - type = io::ObjStorageType::OBS; + type = io::ObjStorageProvider::OBS; break; case TObjStorageType::OSS: - type = io::ObjStorageType::OSS; + type = io::ObjStorageProvider::OSS; break; case TObjStorageType::GCP: - type = io::ObjStorageType::GCP; + type = io::ObjStorageProvider::GCP; break; case TObjStorageType::TOS: - type = io::ObjStorageType::TOS; + type = io::ObjStorageProvider::TOS; break; default: LOG_FATAL("unknown provider type {}, info {}", param.provider, ret.to_string()); diff --git a/be/src/util/s3_util.h b/be/src/util/s3_util.h index ba836569c0121d..6856d205ff5241 100644 --- a/be/src/util/s3_util.h +++ b/be/src/util/s3_util.h @@ -24,6 +24,9 @@ #include #include #include +#ifdef BE_TEST +#include +#endif #include #include @@ -35,9 +38,9 @@ #include "common/status.h" #include "core/string_ref.h" #include "cpp/aws_common.h" +#include "cpp/obj-client/auth/aws_credential_factory.h" #include "cpp/obj-client/auth/obj_credential.h" -#include "cpp/token_bucket_rate_limiter.h" -#include "io/fs/obj_storage_client.h" +#include "cpp/obj-client/obj_storage_client.h" namespace Aws::S3 { class S3Client; @@ -50,19 +53,6 @@ class Adder; namespace doris { -namespace s3_bvar { -extern bvar::LatencyRecorder s3_get_latency; -extern bvar::LatencyRecorder s3_put_latency; -extern bvar::LatencyRecorder s3_delete_object_latency; -extern bvar::LatencyRecorder s3_delete_objects_latency; -extern bvar::LatencyRecorder s3_head_latency; -extern bvar::LatencyRecorder s3_multi_part_upload_latency; -extern bvar::LatencyRecorder s3_list_latency; -extern bvar::LatencyRecorder s3_list_object_versions_latency; -extern bvar::LatencyRecorder s3_get_bucket_version_latency; -extern bvar::LatencyRecorder s3_copy_object_latency; -}; // namespace s3_bvar - std::string hide_access_key(const std::string& ak); class S3URI; @@ -74,7 +64,7 @@ struct S3ClientConf { std::string token; // For azure we'd better support the bucket at the first time init azure blob container client std::string bucket; - io::ObjStorageType provider = io::ObjStorageType::AWS; + io::ObjStorageProvider provider = io::ObjStorageProvider::AWS; int max_connections = -1; int request_timeout_ms = -1; int connect_timeout_ms = -1; @@ -125,9 +115,10 @@ struct S3ClientConf { "request_timeout_ms={}, connect_timeout_ms={}, use_virtual_addressing={}, " "cred_provider_type={},role_arn={}, external_id={}, credential_type={}, " "is_internal_bucket={}", - hide_access_key(ak), token, endpoint, region, bucket, max_connections, - request_timeout_ms, connect_timeout_ms, use_virtual_addressing, cred_provider_type, - role_arn, external_id, credential_type_name(credential), is_internal_bucket); + hide_access_key(ak), token.empty() ? "" : "******", endpoint, region, bucket, + max_connections, request_timeout_ms, connect_timeout_ms, use_virtual_addressing, + cred_provider_type, role_arn, external_id, credential_type_name(credential), + is_internal_bucket); } }; @@ -158,7 +149,7 @@ class S3ClientFactory { static S3ClientFactory& instance(); - std::shared_ptr create(const S3ClientConf& s3_conf); + Result> create(const S3ClientConf& s3_conf); static Status convert_properties_to_s3_conf(const std::map& prop, const S3URI& s3_uri, S3Conf* s3_conf); @@ -174,6 +165,8 @@ class S3ClientFactory { return instance; } + AwsCredentialResult create_aws_credentials_provider(const S3ClientConf& s3_conf); + #ifdef BE_TEST void set_client_creator_for_test( std::function(const S3ClientConf&)> creator); @@ -182,23 +175,19 @@ class S3ClientFactory { #endif private: - std::shared_ptr _create_s3_client(const S3ClientConf& s3_conf); - std::shared_ptr _create_azure_client(const S3ClientConf& s3_conf); - std::shared_ptr _get_aws_credentials_provider_v1( - const S3ClientConf& s3_conf); - std::shared_ptr _get_aws_credentials_provider_v2( - const S3ClientConf& s3_conf); - std::shared_ptr _create_credentials_provider( - CredProviderType type); - std::shared_ptr get_aws_credentials_provider( - const S3ClientConf& s3_conf); - +#ifdef BE_TEST + FRIEND_TEST(S3ClientFactoryTest, RefreshCaCertForCredentialsProvider); +#endif + Result> _create_s3_client(const S3ClientConf& s3_conf); + Result> _create_azure_client(const S3ClientConf& s3_conf); + std::string _get_ca_cert_file_path(); S3ClientFactory(); Aws::SDKOptions _aws_options; std::mutex _lock; std::unordered_map, S3ClientConfHash> _cache; + std::mutex _ca_cert_lock; std::string _ca_cert_file_path; #ifdef BE_TEST std::function(const S3ClientConf&)> _test_client_creator; diff --git a/be/test/ai/embed_test.cpp b/be/test/ai/embed_test.cpp index e45082193afa2d..34378747720369 100644 --- a/be/test/ai/embed_test.cpp +++ b/be/test/ai/embed_test.cpp @@ -30,11 +30,12 @@ #include "core/data_type/data_type_jsonb.h" #include "core/data_type/data_type_number.h" #include "core/value/jsonb_value.h" +#include "cpp/obj-client/obj_storage_client.h" #include "exprs/function/ai/ai_adapter.h" #include "exprs/function/simple_function_factory.h" -#include "io/fs/obj_storage_client.h" #include "testutil/column_helper.h" #include "testutil/mock/mock_runtime_state.h" +#include "testutil/mock/obj_storage_client_test_stub.h" namespace doris { @@ -47,69 +48,60 @@ class MockHttpClient : public HttpClient { std::string _content_type; }; -class MockEmbedObjStorageClient : public io::ObjStorageClient { +class MockEmbedObjStorageClient : public io::ObjStorageClientTestStub { public: - io::ObjectStorageUploadResponse create_multipart_upload( - const io::ObjectStoragePathOptions& /*opts*/) override { + io::ObjStorageUploadResult create_multipart_upload( + const io::ObjStoragePath& /*opts*/) override { return {}; } - io::ObjectStorageResponse put_object(const io::ObjectStoragePathOptions& /*opts*/, - std::string_view /*stream*/) override { - return io::ObjectStorageResponse::OK(); + io::ObjStorageResponse put_object(const io::ObjStoragePath& /*opts*/, + std::string_view /*stream*/) override { + return io::ObjStorageResponse::OK(); } - io::ObjectStorageUploadResponse upload_part(const io::ObjectStoragePathOptions& /*opts*/, - std::string_view /*stream*/, - int /*part_num*/) override { + io::ObjStorageUploadResult upload_part(const io::ObjStoragePath& /*opts*/, + const std::string& /*upload_id*/, + std::string_view /*stream*/, int /*part_num*/) override { return {}; } - io::ObjectStorageResponse complete_multipart_upload( - const io::ObjectStoragePathOptions& /*opts*/, - const std::vector& /*completed_parts*/) override { - return io::ObjectStorageResponse::OK(); + io::ObjStorageResponse complete_multipart_upload( + const io::ObjStoragePath& /*opts*/, const std::string& /*upload_id*/, + const std::vector& /*completed_parts*/) override { + return io::ObjStorageResponse::OK(); } - io::ObjectStorageHeadResponse head_object( - const io::ObjectStoragePathOptions& /*opts*/) override { - return {}; - } - - io::ObjectStorageResponse get_object(const io::ObjectStoragePathOptions& /*opts*/, - void* /*buffer*/, size_t /*offset*/, size_t /*bytes_read*/, - size_t* /*size_return*/) override { - return io::ObjectStorageResponse::OK(); - } + io::ObjStorageHeadResult head_object(const io::ObjStoragePath& /*opts*/) override { return {}; } - io::ObjectStorageResponse list_objects(const io::ObjectStoragePathOptions& /*opts*/, - std::vector* /*files*/) override { - return io::ObjectStorageResponse::OK(); + io::ObjStorageResponse get_object(const io::ObjStoragePath& /*opts*/, void* /*buffer*/, + size_t /*offset*/, size_t /*bytes_read*/, + size_t* /*size_return*/) override { + return io::ObjStorageResponse::OK(); } - io::ObjectStorageResponse delete_objects(const io::ObjectStoragePathOptions& /*opts*/, - std::vector /*objs*/) override { - return io::ObjectStorageResponse::OK(); + io::ObjStorageListPageResult list_objects_page( + const io::ObjStoragePath& /*opts*/, std::string_view /*continuation_token*/) override { + return {.resp = io::ObjStorageResponse::OK()}; } - io::ObjectStorageResponse delete_object(const io::ObjectStoragePathOptions& /*opts*/) override { - return io::ObjectStorageResponse::OK(); + io::ObjStorageResponse delete_objects(const io::ObjStoragePath& /*opts*/, + std::vector /*objs*/) override { + return io::ObjStorageResponse::OK(); } - io::ObjectStorageResponse delete_objects_recursively( - const io::ObjectStoragePathOptions& /*opts*/) override { - return io::ObjectStorageResponse::OK(); + io::ObjStorageResponse delete_object(const io::ObjStoragePath& /*opts*/) override { + return io::ObjStorageResponse::OK(); } - std::string generate_presigned_url(const io::ObjectStoragePathOptions& opts, - int64_t expiration_secs, const S3ClientConf& conf) override { + std::string generate_presigned_url(const io::ObjStoragePath& opts, + int64_t expiration_secs) override { last_opts = opts; last_expiration_secs = expiration_secs; - last_conf = conf; return fmt::format("mock-s3://{}/{}?ttl={}", opts.bucket, opts.key, expiration_secs); } - io::ObjectStoragePathOptions last_opts; + io::ObjStoragePath last_opts; int64_t last_expiration_secs = 0; S3ClientConf last_conf; }; @@ -768,7 +760,10 @@ TEST(EMBED_TEST, embed_function_multimodal_s3_presigned_url) { auto mock_client = std::make_shared(); S3ClientFactory::instance().set_client_creator_for_test( - [mock_client](const S3ClientConf&) { return mock_client; }); + [mock_client](const S3ClientConf& conf) { + mock_client->last_conf = conf; + return mock_client; + }); std::vector resources = {"mock_resource"}; std::vector file_json_rows = {R"({ diff --git a/be/test/io/client/s3_file_system_test.cpp b/be/test/io/client/s3_file_system_test.cpp index 1cc11876bde4c8..a3a71e5b03b72a 100644 --- a/be/test/io/client/s3_file_system_test.cpp +++ b/be/test/io/client/s3_file_system_test.cpp @@ -29,10 +29,10 @@ #include #include "common/config.h" +#include "cpp/obj-client/obj_storage_client.h" #include "cpp/sync_point.h" #include "io/fs/file_system.h" #include "io/fs/file_writer.h" -#include "io/fs/obj_storage_client.h" #include "runtime/exec_env.h" #include "util/defer_op.h" #include "util/s3_rate_limiter_manager.h" @@ -176,11 +176,11 @@ class S3FileSystemTest : public ::testing::Test { EXPECT_TRUE(status.ok()) << "Failed to delete test file: " << status.to_string(); } - io::ObjStorageType convert_provider(const std::string& provider_str) { + io::ObjStorageProvider convert_provider(const std::string& provider_str) { if (provider_str == "AZURE") { - return io::ObjStorageType::AZURE; + return io::ObjStorageProvider::AZURE; } else { - return io::ObjStorageType::AWS; // Default to AWS S3 + return io::ObjStorageProvider::AWS; // Default to AWS S3 } } diff --git a/be/test/io/fs/azure_obj_storage_client_test.cpp b/be/test/io/fs/azure_obj_storage_client_test.cpp index 7591b4bf2ea997..fae07484196479 100644 --- a/be/test/io/fs/azure_obj_storage_client_test.cpp +++ b/be/test/io/fs/azure_obj_storage_client_test.cpp @@ -15,12 +15,20 @@ // specific language governing permissions and limitations // under the License. -#include "io/fs/azure_obj_storage_client.h" +#ifdef USE_AZURE +#include "cpp/obj-client/azure_obj_storage_client.h" + +#include "cpp/obj-client/auth/azure_auth_factory.h" +#endif #include +#include +#include +#include + +#include "cpp/obj-client/obj_storage_client.h" #include "io/fs/file_system.h" -#include "io/fs/obj_storage_client.h" #include "util/s3_util.h" #ifdef USE_AZURE @@ -34,8 +42,50 @@ namespace doris { #ifdef USE_AZURE +TEST(AzureObjStorageClientMultipartHelperTest, full_upload_uuid_isolates_writer_blocks) { + constexpr std::string_view first_upload = "09492e3d-e231-4ed9-bf84-b6fc772cda54"; + constexpr std::string_view second_upload = "06996d15-1c2e-4ddd-8853-43816ea84a07"; + auto first_block = io::azure_multipart_block_id(first_upload, 1); + auto second_block = io::azure_multipart_block_id(second_upload, 1); + + EXPECT_NE(first_block, second_block); + EXPECT_EQ(first_block.size(), io::azure_multipart_block_id(first_upload, 999).size()); + auto decoded = Aws::Utils::HashingUtils::Base64Decode(first_block); + ASSERT_EQ(first_upload.size() + sizeof(uint32_t), decoded.GetLength()); + EXPECT_EQ(first_upload, + std::string_view(reinterpret_cast(decoded.GetUnderlyingData()), + first_upload.size())); + EXPECT_EQ(1, decoded.GetUnderlyingData()[first_upload.size()]); + EXPECT_EQ(0, decoded.GetUnderlyingData()[first_upload.size() + 1]); + EXPECT_EQ(0, decoded.GetUnderlyingData()[first_upload.size() + 2]); + EXPECT_EQ(0, decoded.GetUnderlyingData()[first_upload.size() + 3]); +} + +TEST(AzureObjStorageClientMultipartHelperTest, create_upload_is_provider_free) { + io::AzureObjStorageClient client(std::shared_ptr {}, + {}); + + auto first = client.create_multipart_upload({}); + auto second = client.create_multipart_upload({}); + + ASSERT_EQ(ErrorCode::OK, first.resp.status.code); + ASSERT_EQ(ErrorCode::OK, second.resp.status.code); + ASSERT_TRUE(first.upload_id.has_value()); + ASSERT_TRUE(second.upload_id.has_value()); + EXPECT_EQ(36, first.upload_id->size()); + EXPECT_EQ(36, second.upload_id->size()); + EXPECT_NE(first.upload_id, second.upload_id); +} + using namespace Azure::Storage::Blobs; +TEST(AzureAuthFactoryTest, AllowsEmptySharedKeyCredentials) { + auto result = AzureAuthFactory::create("https://account.blob.core.windows.net/container", + {.account_name = "", .account_key = ""}, {}); + + EXPECT_TRUE(result); +} + TEST(AzureObjStorageClientTlsHelperTest, detects_tls_ca_error) { EXPECT_TRUE(io::is_azure_tls_ca_error_message( "Problem with the SSL CA cert (path? access rights?)")); @@ -57,9 +107,16 @@ TEST(AzureObjStorageClientTlsHelperTest, appends_debug_suffix_only_for_tls_ca_er ""); } +TEST(AzureObjStorageClientBatchDeleteTest, failure_message_preserves_object_key) { + EXPECT_EQ(io::build_azure_batch_delete_failure_message({.bucket = "container"}, + "directory/failed-blob"), + "Azure batch delete failed, path msg bucket container, key directory/failed-blob, " + "prefix , path "); +} + class AzureObjStorageClientTest : public testing::Test { protected: - static std::shared_ptr obj_storage_client; + static std::shared_ptr obj_storage_client; static void SetUpTestSuite() { if (!std::getenv("AZURE_ACCOUNT_NAME") || !std::getenv("AZURE_ACCOUNT_KEY") || @@ -74,16 +131,18 @@ class AzureObjStorageClientTest : public testing::Test { // Initialize Azure SDK [[maybe_unused]] auto& s3ClientFactory = S3ClientFactory::instance(); - AzureObjStorageClientTest::obj_storage_client = S3ClientFactory::instance().create( + auto client_result = S3ClientFactory::instance().create( {.endpoint = fmt::format("https://{}.blob.core.windows.net", accountName), .region = "dummy-region", .ak = accountName, .sk = accountKey, .token = "", .bucket = containerName, - .provider = io::ObjStorageType::AZURE, + .provider = ObjStorageProvider::AZURE, .role_arn = "", .external_id = ""}); + ASSERT_TRUE(client_result.has_value()) << client_result.error(); + AzureObjStorageClientTest::obj_storage_client = std::move(client_result).value(); } void SetUp() override { @@ -93,7 +152,7 @@ class AzureObjStorageClientTest : public testing::Test { } }; -std::shared_ptr AzureObjStorageClientTest::obj_storage_client = nullptr; +std::shared_ptr AzureObjStorageClientTest::obj_storage_client = nullptr; TEST_F(AzureObjStorageClientTest, put_list_delete_object) { LOG(INFO) << "AzureObjStorageClientTest::put_list_delete_object"; @@ -102,25 +161,23 @@ TEST_F(AzureObjStorageClientTest, put_list_delete_object) { {.key = "AzureObjStorageClientTest/put_list_delete_object"}, std::string("aaaa")); EXPECT_EQ(response.status.code, ErrorCode::OK); - std::vector files; - // clang-format off - response = AzureObjStorageClientTest::obj_storage_client->list_objects({.bucket = "dummy", - .prefix = "AzureObjStorageClientTest/put_list_delete_object",}, &files); - // clang-format on - EXPECT_EQ(response.status.code, ErrorCode::OK); - EXPECT_EQ(files.size(), 1); - files.clear(); + std::vector objects; + response = AzureObjStorageClientTest::obj_storage_client->list_objects( + {.bucket = "dummy", .prefix = "AzureObjStorageClientTest/put_list_delete_object"}, + &objects); + EXPECT_TRUE(response.ok()); + EXPECT_EQ(objects.size(), 1); + objects.clear(); response = AzureObjStorageClientTest::obj_storage_client->delete_object( {.key = "AzureObjStorageClientTest/put_list_delete_object"}); EXPECT_EQ(response.status.code, ErrorCode::OK); - // clang-format off - response = AzureObjStorageClientTest::obj_storage_client->list_objects({.bucket = "dummy", - .prefix = "AzureObjStorageClientTest/put_list_delete_object",}, &files); - // clang-format on - EXPECT_EQ(response.status.code, ErrorCode::OK); - EXPECT_EQ(files.size(), 0); + response = AzureObjStorageClientTest::obj_storage_client->list_objects( + {.bucket = "dummy", .prefix = "AzureObjStorageClientTest/put_list_delete_object"}, + &objects); + EXPECT_TRUE(response.ok()); + EXPECT_TRUE(objects.empty()); } TEST_F(AzureObjStorageClientTest, delete_objects_recursively) { @@ -136,25 +193,61 @@ TEST_F(AzureObjStorageClientTest, delete_objects_recursively) { LOG(INFO) << "put " << key << " OK"; } - std::vector files; - // clang-format off - auto response = AzureObjStorageClientTest::obj_storage_client->list_objects({.bucket = "dummy", - .prefix = "AzureObjStorageClientTest/delete_objects_recursively",}, &files); - // clang-format on - EXPECT_EQ(response.status.code, ErrorCode::OK); - EXPECT_EQ(files.size(), 22); - files.clear(); + std::vector objects; + auto response = AzureObjStorageClientTest::obj_storage_client->list_objects( + {.bucket = "dummy", .prefix = "AzureObjStorageClientTest/delete_objects_recursively"}, + &objects); + EXPECT_TRUE(response.ok()); + EXPECT_EQ(objects.size(), 22); + objects.clear(); - response = AzureObjStorageClientTest::obj_storage_client->delete_objects_recursively( + response = delete_objects_recursively( + AzureObjStorageClientTest::obj_storage_client, {.prefix = "AzureObjStorageClientTest/delete_objects_recursively"}); EXPECT_EQ(response.status.code, ErrorCode::OK); - // clang-format off - response = AzureObjStorageClientTest::obj_storage_client->list_objects({.bucket = "dummy", - .prefix = "AzureObjStorageClientTest/delete_objects_recursively",}, &files); - // clang-format on - EXPECT_EQ(response.status.code, ErrorCode::OK); - EXPECT_EQ(files.size(), 0); + response = AzureObjStorageClientTest::obj_storage_client->list_objects( + {.bucket = "dummy", .prefix = "AzureObjStorageClientTest/delete_objects_recursively"}, + &objects); + EXPECT_TRUE(response.ok()); + EXPECT_TRUE(objects.empty()); +} + +TEST_F(AzureObjStorageClientTest, concurrent_multipart_uploads_do_not_share_staged_blocks) { + ObjStoragePath first {.key = "AzureObjStorageClientTest/concurrent_multipart"}; + ObjStoragePath second = first; + auto first_create = obj_storage_client->create_multipart_upload(first); + auto second_create = obj_storage_client->create_multipart_upload(second); + ASSERT_EQ(first_create.resp.status.code, ErrorCode::OK); + ASSERT_EQ(second_create.resp.status.code, ErrorCode::OK); + ASSERT_TRUE(first_create.upload_id.has_value()); + ASSERT_TRUE(second_create.upload_id.has_value()); + ASSERT_NE(first_create.upload_id, second_create.upload_id); + + auto first_part = obj_storage_client->upload_part(first, *first_create.upload_id, "first", 1); + auto second_part = + obj_storage_client->upload_part(second, *second_create.upload_id, "second", 1); + ASSERT_EQ(first_part.resp.status.code, ErrorCode::OK); + ASSERT_EQ(second_part.resp.status.code, ErrorCode::OK); + ASSERT_NE(first_part.etag, second_part.etag); + ASSERT_EQ(obj_storage_client + ->complete_multipart_upload(first, *first_create.upload_id, {{.part_num = 1}}) + .status.code, + ErrorCode::OK); + ASSERT_NE( + obj_storage_client + ->complete_multipart_upload(second, *second_create.upload_id, {{.part_num = 1}}) + .status.code, + ErrorCode::OK); + + std::array contents {}; + size_t size_return = 0; + ASSERT_EQ(obj_storage_client + ->get_object(second, contents.data(), 0, contents.size(), &size_return) + .status.code, + ErrorCode::OK); + EXPECT_EQ(std::string_view(contents.data(), size_return), "first"); + EXPECT_EQ(obj_storage_client->delete_object(second).status.code, ErrorCode::OK); } #else diff --git a/be/test/io/fs/obj_storage_client_test.cpp b/be/test/io/fs/obj_storage_client_test.cpp new file mode 100644 index 00000000000000..7932f28058b5cb --- /dev/null +++ b/be/test/io/fs/obj_storage_client_test.cpp @@ -0,0 +1,174 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "cpp/obj-client/obj_storage_client.h" + +#include + +#include +#include +#include +#include + +namespace doris { +namespace { + +TEST(ObjStorageStatusTest, MapsProviderHttpErrors) { + auto expect_status = [](int http_code, ObjStorageStatus::Code expected) { + auto status = obj_storage_status_from_http_code(http_code, "provider error"); + EXPECT_EQ(status.code, expected); + EXPECT_EQ(status.msg, "provider error"); + }; + + expect_status(-1, ObjStorageStatus::NETWORK_ERROR); + expect_status(0, ObjStorageStatus::NETWORK_ERROR); + expect_status(401, ObjStorageStatus::PERMISSION_DENIED); + expect_status(403, ObjStorageStatus::PERMISSION_DENIED); + expect_status(404, ObjStorageStatus::NOT_FOUND); + expect_status(429, ObjStorageStatus::LIMIT_REACH); + expect_status(500, ObjStorageStatus::INTERNAL_ERROR); +} + +class FakeObjStorageClient final : public ObjStorageClient { +public: + ObjStorageUploadResult create_multipart_upload(const ObjStoragePath&) override { + ++calls; + return {}; + } + + ObjStorageResponse put_object(const ObjStoragePath&, std::string_view) override { + ++calls; + return ObjStorageResponse::OK(); + } + + ObjStorageUploadResult upload_part(const ObjStoragePath&, const std::string&, std::string_view, + int) override { + ++calls; + return {}; + } + + ObjStorageResponse complete_multipart_upload( + const ObjStoragePath&, const std::string&, + const std::vector&) override { + ++calls; + return ObjStorageResponse::OK(); + } + + ObjStorageHeadResult head_object(const ObjStoragePath&) override { + ++calls; + return {}; + } + + ObjStorageResponse get_object(const ObjStoragePath&, void*, size_t, size_t, + size_t* size_return) override { + ++calls; + *size_return = 4; + return ObjStorageResponse::OK(); + } + + ObjStorageListPageResult list_objects_page(const ObjStoragePath&, + std::string_view continuation_token) override { + ++calls; + ++list_page_calls; + if (list_pages.empty()) { + return {}; + } + const size_t index = + continuation_token.empty() + ? 0 + : static_cast(std::stoull(std::string(continuation_token))); + ObjStorageListPageResult page {.resp = ObjStorageResponse::OK()}; + page.objects = list_pages[index]; + page.has_more = index + 1 < list_pages.size(); + if (page.has_more) { + page.continuation_token = std::to_string(index + 1); + } + return page; + } + + ObjStorageResponse delete_objects(const ObjStoragePath&, std::vector) override { + ++calls; + return ObjStorageResponse::OK(); + } + + ObjStorageResponse delete_object(const ObjStoragePath&) override { + ++calls; + return ObjStorageResponse::OK(); + } + + ObjStorageCapabilities capabilities() const override { return {.max_delete_batch = 2}; } + + std::string generate_presigned_url(const ObjStoragePath&, int64_t) override { + ++presigned_url_calls; + return "url"; + } + + ObjStorageResponse get_lifecycle(const std::string&, int64_t*) override { + ++calls; + return ObjStorageResponse::OK(); + } + + ObjStorageResponse check_versioning(const std::string&) override { + ++calls; + return ObjStorageResponse::OK(); + } + + ObjStorageResponse abort_multipart_upload(const ObjStoragePath&, const std::string&) override { + ++calls; + return ObjStorageResponse::OK(); + } + + int calls = 0; + int presigned_url_calls = 0; + int list_page_calls = 0; + std::vector> list_pages; +}; + +TEST(ObjStorageClientTest, SupportsLazyAndEagerListing) { + auto client = std::make_shared(); + client->list_pages = { + {{.key = "first"}, {.key = "second"}}, + {{.key = "third"}}, + }; + ObjStoragePath opts {.bucket = "bucket", .prefix = "prefix"}; + + auto iter = client->list_objects(opts); + EXPECT_EQ(client->list_page_calls, 0); + auto first = iter->next(); + ASSERT_TRUE(first.object.has_value()); + EXPECT_EQ(first.object->key, "first"); + EXPECT_EQ(client->list_page_calls, 1); + auto second = iter->next(); + ASSERT_TRUE(second.object.has_value()); + EXPECT_EQ(second.object->key, "second"); + EXPECT_EQ(client->list_page_calls, 1); + auto third = iter->next(); + ASSERT_TRUE(third.object.has_value()); + EXPECT_EQ(third.object->key, "third"); + EXPECT_EQ(client->list_page_calls, 2); + + std::vector objects; + EXPECT_TRUE(client->list_objects(opts, &objects).ok()); + ASSERT_EQ(objects.size(), 3); + EXPECT_EQ(objects[0].key, "first"); + EXPECT_EQ(objects[1].key, "second"); + EXPECT_EQ(objects[2].key, "third"); + EXPECT_EQ(client->list_page_calls, 4); +} + +} // namespace +} // namespace doris diff --git a/be/test/io/fs/packed_file_concurrency_test.cpp b/be/test/io/fs/packed_file_concurrency_test.cpp index 14c41d53a7b442..d1bb80e148d77c 100644 --- a/be/test/io/fs/packed_file_concurrency_test.cpp +++ b/be/test/io/fs/packed_file_concurrency_test.cpp @@ -74,6 +74,7 @@ #include "io/io_common.h" #include "runtime/exec_env.h" #include "runtime/thread_context.h" +#include "testutil/mock/obj_storage_client_test_stub.h" #include "util/s3_util.h" #include "util/slice.h" #include "util/threadpool.h" @@ -202,14 +203,13 @@ void reset_mock_s3_store() { store.objects.clear(); } -class MockObjStorageClient : public ObjStorageClient { +class MockObjStorageClient : public ObjStorageClientTestStub { public: explicit MockObjStorageClient(MockS3Store* store) : _store(store) {} - ObjectStorageUploadResponse create_multipart_upload( - const ObjectStoragePathOptions& opts) override { - ObjectStorageUploadResponse resp; - resp.resp = ObjectStorageResponse::OK(); + ObjStorageUploadResult create_multipart_upload(const ObjStoragePath& opts) override { + ObjStorageUploadResult resp; + resp.resp = ObjStorageResponse::OK(); std::lock_guard lock(_store->mutex); auto upload_id = fmt::format("upload-{}", _store->next_upload_id++); resp.upload_id = upload_id; @@ -217,41 +217,40 @@ class MockObjStorageClient : public ObjStorageClient { return resp; } - ObjectStorageResponse put_object(const ObjectStoragePathOptions& opts, - std::string_view stream) override { + ObjStorageResponse put_object(const ObjStoragePath& opts, std::string_view stream) override { std::lock_guard lock(_store->mutex); _store->objects[_store->make_key(opts.bucket, opts.key)] = std::string(stream.data(), stream.size()); - return ObjectStorageResponse::OK(); + return ObjStorageResponse::OK(); } - ObjectStorageUploadResponse upload_part(const ObjectStoragePathOptions& opts, - std::string_view stream, int part_num) override { - ObjectStorageUploadResponse resp; - if (!opts.upload_id) { + ObjStorageUploadResult upload_part(const ObjStoragePath& opts, const std::string& upload_id, + std::string_view stream, int part_num) override { + ObjStorageUploadResult resp; + if (upload_id.empty()) { resp.resp = make_error("missing upload id"); return resp; } std::lock_guard lock(_store->mutex); - auto ctx_it = _store->uploads.find(*opts.upload_id); + auto ctx_it = _store->uploads.find(upload_id); if (ctx_it == _store->uploads.end()) { resp.resp = make_error("upload context not found"); return resp; } ctx_it->second.parts[part_num] = std::string(stream.data(), stream.size()); - resp.resp = ObjectStorageResponse::OK(); + resp.resp = ObjStorageResponse::OK(); resp.etag = fmt::format("\"mock-etag-{}\"", part_num); return resp; } - ObjectStorageResponse complete_multipart_upload( - const ObjectStoragePathOptions& opts, - const std::vector& completed_parts) override { - if (!opts.upload_id) { + ObjStorageResponse complete_multipart_upload( + const ObjStoragePath& opts, const std::string& upload_id, + const std::vector& completed_parts) override { + if (upload_id.empty()) { return make_error("missing upload id"); } std::lock_guard lock(_store->mutex); - auto ctx_it = _store->uploads.find(*opts.upload_id); + auto ctx_it = _store->uploads.find(upload_id); if (ctx_it == _store->uploads.end()) { return make_error("upload context not found"); } @@ -266,11 +265,11 @@ class MockObjStorageClient : public ObjStorageClient { _store->objects[_store->make_key(ctx_it->second.bucket, ctx_it->second.key)] = std::move(data); _store->uploads.erase(ctx_it); - return ObjectStorageResponse::OK(); + return ObjStorageResponse::OK(); } - ObjectStorageHeadResponse head_object(const ObjectStoragePathOptions& opts) override { - ObjectStorageHeadResponse resp; + ObjStorageHeadResult head_object(const ObjStoragePath& opts) override { + ObjStorageHeadResult resp; std::lock_guard lock(_store->mutex); auto key = _store->make_key(opts.bucket, opts.key); auto it = _store->objects.find(key); @@ -279,14 +278,13 @@ class MockObjStorageClient : public ObjStorageClient { static_cast(Aws::Http::HttpResponseCode::NOT_FOUND)); return resp; } - resp.resp = ObjectStorageResponse::OK(); + resp.resp = ObjStorageResponse::OK(); resp.file_size = it->second.size(); return resp; } - ObjectStorageResponse get_object(const ObjectStoragePathOptions& opts, void* buffer, - size_t offset, size_t bytes_read, - size_t* size_return) override { + ObjStorageResponse get_object(const ObjStoragePath& opts, void* buffer, size_t offset, + size_t bytes_read, size_t* size_return) override { std::lock_guard lock(_store->mutex); auto key = _store->make_key(opts.bucket, opts.key); auto it = _store->objects.find(key); @@ -299,64 +297,52 @@ class MockObjStorageClient : public ObjStorageClient { size_t to_copy = std::min(bytes_read, it->second.size() - offset); memcpy(buffer, it->second.data() + offset, to_copy); *size_return = to_copy; - return ObjectStorageResponse::OK(); + return ObjStorageResponse::OK(); } - ObjectStorageResponse list_objects(const ObjectStoragePathOptions& opts, - std::vector* files) override { + ObjStorageListPageResult list_objects_page(const ObjStoragePath& opts, + std::string_view /*continuation_token*/) override { std::lock_guard lock(_store->mutex); - std::string prefix = _store->make_key(opts.bucket, opts.prefix); + const auto& object_prefix = opts.prefix.empty() ? opts.key : opts.prefix; + std::string prefix = _store->make_key(opts.bucket, object_prefix); + ObjStorageListPageResult page {.resp = ObjStorageResponse::OK()}; for (const auto& [key, data] : _store->objects) { if (key.rfind(prefix, 0) == 0) { - FileInfo info; - info.file_name = key.substr(prefix.size()); - info.file_size = data.size(); - info.is_file = true; - files->push_back(std::move(info)); + page.objects.emplace_back(ObjectMeta { + .key = key.substr(opts.bucket.size() + 1), + .size = static_cast(data.size()), + }); } } - return ObjectStorageResponse::OK(); + return page; } - ObjectStorageResponse delete_objects(const ObjectStoragePathOptions& opts, - std::vector objs) override { + ObjStorageResponse delete_objects(const ObjStoragePath& opts, + std::vector objs) override { std::lock_guard lock(_store->mutex); for (const auto& obj : objs) { _store->objects.erase(_store->make_key(opts.bucket, obj)); } - return ObjectStorageResponse::OK(); + return ObjStorageResponse::OK(); } - ObjectStorageResponse delete_object(const ObjectStoragePathOptions& opts) override { + ObjStorageResponse delete_object(const ObjStoragePath& opts) override { std::lock_guard lock(_store->mutex); _store->objects.erase(_store->make_key(opts.bucket, opts.key)); - return ObjectStorageResponse::OK(); + return ObjStorageResponse::OK(); } - ObjectStorageResponse delete_objects_recursively( - const ObjectStoragePathOptions& opts) override { - std::lock_guard lock(_store->mutex); - std::string prefix = _store->make_key(opts.bucket, opts.prefix); - for (auto it = _store->objects.begin(); it != _store->objects.end();) { - if (it->first.rfind(prefix, 0) == 0) { - it = _store->objects.erase(it); - } else { - ++it; - } - } - return ObjectStorageResponse::OK(); - } - - std::string generate_presigned_url(const ObjectStoragePathOptions& opts, - int64_t /*expiration_secs*/, - const S3ClientConf& /*conf*/) override { + std::string generate_presigned_url(const ObjStoragePath& opts, + int64_t /*expiration_secs*/) override { return fmt::format("mock://{}/{}", opts.bucket, opts.key); } private: - static ObjectStorageResponse make_error(std::string msg, int http_code = 500) { - ObjectStorageResponse resp; - resp.status.code = static_cast(ErrorCode::INTERNAL_ERROR); + static ObjStorageResponse make_error(std::string msg, int http_code = 500) { + ObjStorageResponse resp; + resp.status.code = http_code == static_cast(Aws::Http::HttpResponseCode::NOT_FOUND) + ? ObjStorageStatus::NOT_FOUND + : static_cast(ErrorCode::INTERNAL_ERROR); resp.status.msg = std::move(msg); resp.http_code = http_code; return resp; @@ -365,11 +351,11 @@ class MockObjStorageClient : public ObjStorageClient { MockS3Store* _store; }; -std::shared_ptr g_mock_obj_client; +std::shared_ptr g_mock_obj_provider_client; void install_mock_environment() { - g_mock_obj_client = std::make_shared(&mock_s3_store()); - auto client = g_mock_obj_client; + g_mock_obj_provider_client = std::make_shared(&mock_s3_store()); + std::shared_ptr client = g_mock_obj_provider_client; S3ClientFactory::instance().set_client_creator_for_test( [client](const S3ClientConf&) { return client; }); @@ -384,7 +370,7 @@ void install_mock_environment() { void remove_mock_environment() { S3ClientFactory::instance().clear_client_creator_for_test(); - g_mock_obj_client.reset(); + g_mock_obj_provider_client.reset(); auto* sp = SyncPoint::get_instance(); sp->clear_call_back("PackedFileManager::update_meta_service"); diff --git a/be/test/io/fs/rate_limited_obj_storage_client_test.cpp b/be/test/io/fs/rate_limited_obj_storage_client_test.cpp index 657b139c0a4fcd..fce15d9ae75081 100644 --- a/be/test/io/fs/rate_limited_obj_storage_client_test.cpp +++ b/be/test/io/fs/rate_limited_obj_storage_client_test.cpp @@ -15,487 +15,255 @@ // specific language governing permissions and limitations // under the License. -#include "io/fs/rate_limited_obj_storage_client.h" +#include "cpp/obj-client/rate_limited_obj_storage_client.h" #include -#include "common/config.h" -#include "util/s3_rate_limiter_manager.h" -#include "util/s3_util.h" +#include +#include +#include +#include namespace doris { - -extern bvar::Adder s3_get_bytes_rate_limit_rejected_count; -extern bvar::Adder s3_put_bytes_rate_limit_rejected_count; - -} // namespace doris - -namespace doris::io { namespace { -constexpr size_t kNoThrottleBytesPerSecond = 1ULL << 40; - -// Provider-free fake: counts calls and reports a configurable read size. -class FakeObjStorageClient : public ObjStorageClient { +class FakeObjStorageClient final : public ObjStorageClient { public: - ObjectStorageUploadResponse create_multipart_upload( - const ObjectStoragePathOptions& opts) override { + ObjStorageUploadResult create_multipart_upload(const ObjStoragePath&) override { ++calls; ++create_multipart_upload_calls; create_multipart_upload_provider_calls += create_multipart_upload_provider_calls_per_logical_call; return {}; } - ObjectStorageResponse put_object(const ObjectStoragePathOptions& opts, - std::string_view stream) override { + + ObjStorageResponse put_object(const ObjStoragePath&, std::string_view) override { ++calls; - return ObjectStorageResponse::OK(); + return ObjStorageResponse::OK(); } - ObjectStorageUploadResponse upload_part(const ObjectStoragePathOptions& opts, - std::string_view stream, int part_num) override { + + ObjStorageUploadResult upload_part(const ObjStoragePath&, const std::string&, std::string_view, + int) override { ++calls; return {}; } - ObjectStorageResponse complete_multipart_upload( - const ObjectStoragePathOptions& opts, - const std::vector& completed_parts) override { + + ObjStorageResponse complete_multipart_upload( + const ObjStoragePath&, const std::string&, + const std::vector&) override { ++calls; - return ObjectStorageResponse::OK(); + return ObjStorageResponse::OK(); } - ObjectStorageHeadResponse head_object(const ObjectStoragePathOptions& opts) override { + + ObjStorageHeadResult head_object(const ObjStoragePath&) override { ++calls; return {}; } - ObjectStorageResponse get_object(const ObjectStoragePathOptions& opts, void* buffer, - size_t offset, size_t bytes_read, - size_t* size_return) override { + + ObjStorageResponse get_object(const ObjStoragePath&, void*, size_t, size_t, + size_t* size_return) override { ++calls; *size_return = actual_read_size; - return ObjectStorageResponse::OK(); + if (fail_get) { + return {.status = {TStatusCode::INTERNAL_ERROR, "simulated read failure"}}; + } + return ObjStorageResponse::OK(); } - ObjectStorageResponse list_objects(const ObjectStoragePathOptions& opts, - std::vector* files) override { + + ObjStorageListPageResult list_objects_page(const ObjStoragePath&, std::string_view) override { + ++calls; + return {}; + } + + ObjStorageResponse delete_objects(const ObjStoragePath&, std::vector) override { ++calls; - return ObjectStorageResponse::OK(); + return ObjStorageResponse::OK(); } - ObjectStorageResponse delete_objects(const ObjectStoragePathOptions& opts, - std::vector objs) override { + + ObjStorageResponse delete_object(const ObjStoragePath&) override { ++calls; - return ObjectStorageResponse::OK(); + return ObjStorageResponse::OK(); + } + + std::string generate_presigned_url(const ObjStoragePath&, int64_t) override { + ++presigned_url_calls; + return "url"; } - ObjectStorageResponse delete_object(const ObjectStoragePathOptions& opts) override { + + ObjStorageResponse get_lifecycle(const std::string&, int64_t*) override { ++calls; - return ObjectStorageResponse::OK(); + return ObjStorageResponse::OK(); } - ObjectStorageResponse delete_objects_recursively( - const ObjectStoragePathOptions& opts) override { + + ObjStorageResponse check_versioning(const std::string&) override { ++calls; - ++delete_objects_recursively_calls; - delete_objects_recursively_provider_calls += - delete_objects_recursively_provider_calls_per_logical_call; - return ObjectStorageResponse::OK(); + return ObjStorageResponse::OK(); } - std::string generate_presigned_url(const ObjectStoragePathOptions& opts, - int64_t expiration_secs, const S3ClientConf& conf) override { + + ObjStorageResponse abort_multipart_upload(const ObjStoragePath&, const std::string&) override { ++calls; - return "presigned"; + return ObjStorageResponse::OK(); + } + + ObjStorageCapabilities capabilities() const override { + return {.max_delete_batch = 17, .max_list_page = 23}; } int calls = 0; - size_t actual_read_size = 0; + int presigned_url_calls = 0; int create_multipart_upload_calls = 0; int create_multipart_upload_provider_calls = 0; int create_multipart_upload_provider_calls_per_logical_call = 1; - int delete_objects_recursively_calls = 0; - int delete_objects_recursively_provider_calls = 0; - int delete_objects_recursively_provider_calls_per_logical_call = 1; + size_t actual_read_size = 4; + bool fail_get = false; }; -struct RateLimiterConfigGuard { - bool enable = config::enable_s3_rate_limiter; - - ~RateLimiterConfigGuard() { - config::enable_s3_rate_limiter = enable; - S3RateLimiterManager::instance().refresh(); +class RecordingRateLimitPolicy final : public ObjStorageRateLimitPolicy { +public: + struct Request { + S3RateLimitType type; + size_t estimated_bytes; + }; + + ObjStorageAdmission acquire(S3RateLimitType type, size_t estimated_bytes) const override { + requests.push_back({type, estimated_bytes}); + if (reject) { + return {.resp = ObjStorageResponse::rate_limit(TStatusCode::LIMIT_REACH, 429, + "rejected by test policy")}; + } + return {.settle = [this](size_t actual_bytes) { settled_bytes.push_back(actual_bytes); }}; } -}; - -} // namespace - -TEST(RateLimitedObjStorageClientTest, forwards_all_calls_when_disabled) { - RateLimiterConfigGuard guard; - config::enable_s3_rate_limiter = false; - - auto fake = std::make_shared(); - RateLimitedObjStorageClient client(fake); - ObjectStoragePathOptions opts {.bucket = "b", .key = "k"}; - size_t size_return = 0; - EXPECT_EQ(0, client.create_multipart_upload(opts).resp.status.code); - EXPECT_EQ(0, client.put_object(opts, "data").status.code); - EXPECT_EQ(0, client.upload_part(opts, "data", 1).resp.status.code); - EXPECT_EQ(0, client.complete_multipart_upload(opts, {}).status.code); - EXPECT_EQ(0, client.head_object(opts).resp.status.code); - EXPECT_EQ(0, client.get_object(opts, nullptr, 0, 4, &size_return).status.code); - std::vector files; - EXPECT_EQ(0, client.list_objects(opts, &files).status.code); - EXPECT_EQ(0, client.delete_objects(opts, {}).status.code); - EXPECT_EQ(0, client.delete_object(opts).status.code); - EXPECT_EQ(0, client.delete_objects_recursively(opts).status.code); - EXPECT_EQ("presigned", client.generate_presigned_url(opts, 60, S3ClientConf {})); - EXPECT_EQ(11, fake->calls); -} - -TEST(RateLimitedObjStorageClientTest, get_rejected_by_count_limit_does_not_reach_inner) { - RateLimiterConfigGuard guard; - config::enable_s3_rate_limiter = true; - auto& manager = S3RateLimiterManager::instance(); - manager.qps_limiter(S3RateLimitType::GET)->reset(0, 0, 1); - manager.qps_limiter(S3RateLimitType::PUT)->reset(0, 0, 0); - manager.bytes_limiter(S3RateLimitType::GET)->reset(0, 0, 0); - manager.bytes_limiter(S3RateLimitType::PUT)->reset(0, 0, 0); - - auto fake = std::make_shared(); - RateLimitedObjStorageClient client(fake); - ObjectStoragePathOptions opts {.bucket = "b", .key = "k"}; - - EXPECT_EQ(0, client.head_object(opts).resp.status.code); - EXPECT_EQ(1, fake->calls); - - auto resp = client.head_object(opts); - EXPECT_EQ(ErrorCode::EXCEEDED_LIMIT, resp.resp.status.code); - EXPECT_EQ(0, resp.resp.http_code); - EXPECT_NE(std::string::npos, resp.resp.status.msg.find("exceeds QPS limit")); - EXPECT_EQ(1, fake->calls); // rejected before reaching the provider - - // PUT uses an independent bucket and is unaffected. - EXPECT_EQ(0, client.put_object(opts, "data").status.code); - EXPECT_EQ(2, fake->calls); -} - -TEST(RateLimitedObjStorageClientTest, put_rejected_by_count_limit_does_not_reach_inner) { - RateLimiterConfigGuard guard; - config::enable_s3_rate_limiter = true; - auto& manager = S3RateLimiterManager::instance(); - manager.qps_limiter(S3RateLimitType::GET)->reset(0, 0, 0); - manager.qps_limiter(S3RateLimitType::PUT)->reset(0, 0, 1); - manager.bytes_limiter(S3RateLimitType::GET)->reset(0, 0, 0); - manager.bytes_limiter(S3RateLimitType::PUT)->reset(0, 0, 0); - - auto fake = std::make_shared(); - RateLimitedObjStorageClient client(fake); - ObjectStoragePathOptions opts {.bucket = "b", .key = "k"}; - - EXPECT_EQ(0, client.put_object(opts, "data").status.code); - EXPECT_EQ(1, fake->calls); - - auto resp = client.put_object(opts, "data"); - EXPECT_EQ(ErrorCode::EXCEEDED_LIMIT, resp.status.code); - EXPECT_EQ(0, resp.http_code); - EXPECT_NE(std::string::npos, resp.status.msg.find("exceeds QPS limit")); - EXPECT_EQ(1, fake->calls); // rejected before reaching the provider - - // GET uses an independent bucket and is unaffected. - EXPECT_EQ(0, client.head_object(opts).resp.status.code); - EXPECT_EQ(2, fake->calls); -} + bool reject = false; + mutable std::vector requests; + mutable std::vector settled_bytes; +}; -TEST(RateLimitedObjStorageClientTest, head_and_list_map_to_get_qps_without_bytes) { - RateLimiterConfigGuard guard; - config::enable_s3_rate_limiter = true; - auto& manager = S3RateLimiterManager::instance(); - auto* get_bytes = manager.bytes_limiter(S3RateLimitType::GET); - manager.qps_limiter(S3RateLimitType::GET) - ->reset(kNoThrottleBytesPerSecond, kNoThrottleBytesPerSecond, 2); - get_bytes->reset(kNoThrottleBytesPerSecond, kNoThrottleBytesPerSecond, 1); - - auto fake = std::make_shared(); - RateLimitedObjStorageClient client(fake); - ObjectStoragePathOptions opts {.bucket = "b", .key = "k", .prefix = "p"}; - - EXPECT_EQ(0, client.head_object(opts).resp.status.code); - std::vector files; - EXPECT_EQ(0, client.list_objects(opts, &files).status.code); - - auto rejected = client.head_object(opts); - EXPECT_EQ(ErrorCode::EXCEEDED_LIMIT, rejected.resp.status.code); - EXPECT_EQ(0, rejected.resp.http_code); - EXPECT_NE(std::string::npos, rejected.resp.status.msg.find("exceeds QPS limit")); - EXPECT_EQ(2, fake->calls); - - // Neither HEAD nor LIST carries payload bytes. - EXPECT_EQ(0, get_bytes->add(1)); - EXPECT_EQ(-1, get_bytes->add(1)); +void expect_rejected(const ObjStorageResponse& response) { + EXPECT_EQ(response.status.code, static_cast(TStatusCode::LIMIT_REACH)); + EXPECT_EQ(response.http_code, 429); + EXPECT_EQ(response.status.msg, "rejected by test policy"); } -TEST(RateLimitedObjStorageClientTest, get_object_maps_to_get_qps_and_get_bytes) { - RateLimiterConfigGuard guard; - config::enable_s3_rate_limiter = true; - auto& manager = S3RateLimiterManager::instance(); - auto* get_bytes = manager.bytes_limiter(S3RateLimitType::GET); - manager.qps_limiter(S3RateLimitType::GET) - ->reset(kNoThrottleBytesPerSecond, kNoThrottleBytesPerSecond, 1); - get_bytes->reset(kNoThrottleBytesPerSecond, kNoThrottleBytesPerSecond, 4); - - auto fake = std::make_shared(); - fake->actual_read_size = 4; - RateLimitedObjStorageClient client(fake); - ObjectStoragePathOptions opts {.bucket = "b", .key = "k"}; - +TEST(RateLimitedObjStorageClientTest, AppliesAdmissionPolicyToEveryClientOperation) { + auto inner_client = std::make_shared(); + auto policy = std::make_shared(); + auto client = std::make_shared(inner_client, policy); + ObjStoragePath opts {.bucket = "bucket", .key = "key"}; + + EXPECT_TRUE(client->create_multipart_upload(opts).resp.ok()); + EXPECT_TRUE(client->put_object(opts, "abc").ok()); + EXPECT_TRUE(client->upload_part(opts, "upload-id", "part", 1).resp.ok()); + EXPECT_TRUE(client->complete_multipart_upload(opts, "upload-id", {}).ok()); + EXPECT_TRUE(client->head_object(opts).resp.ok()); + char buffer[10]; size_t size_return = 0; - EXPECT_EQ(0, client.get_object(opts, nullptr, 0, 4, &size_return).status.code); - EXPECT_EQ(4, size_return); - - auto rejected = client.get_object(opts, nullptr, 0, 4, &size_return); - EXPECT_EQ(ErrorCode::EXCEEDED_LIMIT, rejected.status.code); - EXPECT_EQ(0, rejected.http_code); - EXPECT_NE(std::string::npos, rejected.status.msg.find("exceeds QPS limit")); - EXPECT_EQ(1, fake->calls); - - // The admitted GET charged exactly its returned payload bytes. - EXPECT_EQ(-1, get_bytes->add(1)); + EXPECT_TRUE(client->get_object(opts, buffer, 0, sizeof(buffer), &size_return).ok()); + EXPECT_EQ(size_return, 4); + std::vector objects; + EXPECT_TRUE(client->list_objects(opts, &objects).ok()); + EXPECT_TRUE(client->delete_objects(opts, {"one", "two", "three"}).ok()); + EXPECT_TRUE(client->delete_object(opts).ok()); + EXPECT_TRUE(delete_objects_recursively(client, opts).ok()); + int64_t expiration_days = 0; + EXPECT_TRUE(client->get_lifecycle("bucket", &expiration_days).ok()); + EXPECT_TRUE(client->check_versioning("bucket").ok()); + EXPECT_TRUE(client->abort_multipart_upload(opts, "upload-id").ok()); + EXPECT_EQ(client->generate_presigned_url(opts, 60), "url"); + EXPECT_EQ(client->capabilities().max_delete_batch, 17); + EXPECT_EQ(client->capabilities().max_list_page, 23); + + const std::vector expected = { + {S3RateLimitType::PUT, 0}, {S3RateLimitType::PUT, 3}, {S3RateLimitType::PUT, 4}, + {S3RateLimitType::PUT, 0}, {S3RateLimitType::GET, 0}, {S3RateLimitType::GET, 10}, + {S3RateLimitType::GET, 0}, {S3RateLimitType::PUT, 0}, {S3RateLimitType::PUT, 0}, + {S3RateLimitType::GET, 0}, {S3RateLimitType::GET, 0}, {S3RateLimitType::GET, 0}, + {S3RateLimitType::PUT, 0}, + }; + ASSERT_EQ(policy->requests.size(), expected.size()); + for (size_t i = 0; i < expected.size(); ++i) { + EXPECT_EQ(policy->requests[i].type, expected[i].type) << "request index " << i; + EXPECT_EQ(policy->requests[i].estimated_bytes, expected[i].estimated_bytes) + << "request index " << i; + } + EXPECT_EQ(policy->settled_bytes, std::vector({4})); + EXPECT_EQ(inner_client->calls, 13); + EXPECT_EQ(inner_client->presigned_url_calls, 1); } -TEST(RateLimitedObjStorageClientTest, get_object_settles_short_read) { - RateLimiterConfigGuard guard; - config::enable_s3_rate_limiter = true; - auto& manager = S3RateLimiterManager::instance(); - manager.qps_limiter(S3RateLimitType::GET)->reset(0, 0, 0); - auto* bytes = manager.bytes_limiter(S3RateLimitType::GET); - bytes->reset(kNoThrottleBytesPerSecond, kNoThrottleBytesPerSecond, 1000); - - auto fake = std::make_shared(); - fake->actual_read_size = 100; // short read: 600 requested, 100 returned - RateLimitedObjStorageClient client(fake); - ObjectStoragePathOptions opts {.bucket = "b", .key = "k"}; - +TEST(RateLimitedObjStorageClientTest, RejectsEveryOperationBeforeDispatchingToInnerClient) { + auto inner_client = std::make_shared(); + auto policy = std::make_shared(); + policy->reject = true; + auto client = std::make_shared(inner_client, policy); + ObjStoragePath opts {.bucket = "bucket", .key = "key"}; + + expect_rejected(client->create_multipart_upload(opts).resp); + expect_rejected(client->put_object(opts, "abc")); + expect_rejected(client->upload_part(opts, "upload-id", "part", 1).resp); + expect_rejected(client->complete_multipart_upload(opts, "upload-id", {})); + expect_rejected(client->head_object(opts).resp); + char buffer[10]; size_t size_return = 0; - EXPECT_EQ(0, client.get_object(opts, nullptr, 0, 600, &size_return).status.code); - EXPECT_EQ(100, size_return); - - // Only 100 bytes remain cumulatively charged, so exactly 900 more are admitted. - EXPECT_EQ(0, bytes->add(900)); - EXPECT_EQ(-1, bytes->add(1)); + expect_rejected(client->get_object(opts, buffer, 0, sizeof(buffer), &size_return)); + std::vector objects; + expect_rejected(client->list_objects(opts, &objects)); + expect_rejected(client->delete_objects(opts, {"one", "two"})); + expect_rejected(client->delete_object(opts)); + expect_rejected(delete_objects_recursively(client, opts)); + int64_t expiration_days = 0; + expect_rejected(client->get_lifecycle("bucket", &expiration_days)); + expect_rejected(client->check_versioning("bucket")); + expect_rejected(client->abort_multipart_upload(opts, "upload-id")); + + EXPECT_EQ(inner_client->calls, 0); + EXPECT_EQ(policy->requests.size(), 13); + EXPECT_TRUE(policy->settled_bytes.empty()); } -TEST(RateLimitedObjStorageClientTest, put_object_charges_payload_bytes) { - RateLimiterConfigGuard guard; - config::enable_s3_rate_limiter = true; - auto& manager = S3RateLimiterManager::instance(); - manager.qps_limiter(S3RateLimitType::PUT)->reset(0, 0, 0); - auto* bytes = manager.bytes_limiter(S3RateLimitType::PUT); - bytes->reset(kNoThrottleBytesPerSecond, kNoThrottleBytesPerSecond, 1000); - - auto fake = std::make_shared(); - RateLimitedObjStorageClient client(fake); - ObjectStoragePathOptions opts {.bucket = "b", .key = "k"}; - - std::string payload(600, 'x'); - EXPECT_EQ(0, client.put_object(opts, payload).status.code); - EXPECT_EQ(0, bytes->add(400)); // exactly the cumulative count remainder - EXPECT_EQ(-1, bytes->add(1)); -} - -TEST(RateLimitedObjStorageClientTest, put_object_maps_to_put_qps_and_put_bytes) { - RateLimiterConfigGuard guard; - config::enable_s3_rate_limiter = true; - auto& manager = S3RateLimiterManager::instance(); - auto* put_bytes = manager.bytes_limiter(S3RateLimitType::PUT); - manager.qps_limiter(S3RateLimitType::PUT) - ->reset(kNoThrottleBytesPerSecond, kNoThrottleBytesPerSecond, 1); - put_bytes->reset(kNoThrottleBytesPerSecond, kNoThrottleBytesPerSecond, 4); - - auto fake = std::make_shared(); - RateLimitedObjStorageClient client(fake); - ObjectStoragePathOptions opts {.bucket = "b", .key = "k"}; - - EXPECT_EQ(0, client.put_object(opts, "data").status.code); - auto rejected = client.put_object(opts, "data"); - EXPECT_EQ(ErrorCode::EXCEEDED_LIMIT, rejected.status.code); - EXPECT_EQ(0, rejected.http_code); - EXPECT_NE(std::string::npos, rejected.status.msg.find("exceeds QPS limit")); - EXPECT_EQ(1, fake->calls); - - EXPECT_EQ(-1, put_bytes->add(1)); -} - -TEST(RateLimitedObjStorageClientTest, upload_part_maps_to_put_qps_and_put_bytes) { - RateLimiterConfigGuard guard; - config::enable_s3_rate_limiter = true; - auto& manager = S3RateLimiterManager::instance(); - auto* put_bytes = manager.bytes_limiter(S3RateLimitType::PUT); - manager.qps_limiter(S3RateLimitType::PUT) - ->reset(kNoThrottleBytesPerSecond, kNoThrottleBytesPerSecond, 1); - put_bytes->reset(kNoThrottleBytesPerSecond, kNoThrottleBytesPerSecond, 4); - - auto fake = std::make_shared(); - RateLimitedObjStorageClient client(fake); - ObjectStoragePathOptions opts {.bucket = "b", .key = "k", .upload_id = "upload"}; - - EXPECT_EQ(0, client.upload_part(opts, "data", 1).resp.status.code); - auto rejected = client.upload_part(opts, "data", 2); - EXPECT_EQ(ErrorCode::EXCEEDED_LIMIT, rejected.resp.status.code); - EXPECT_EQ(0, rejected.resp.http_code); - EXPECT_NE(std::string::npos, rejected.resp.status.msg.find("exceeds QPS limit")); - EXPECT_EQ(1, fake->calls); - - EXPECT_EQ(-1, put_bytes->add(1)); -} - -TEST(RateLimitedObjStorageClientTest, multipart_control_apis_map_to_put_qps_without_bytes) { - RateLimiterConfigGuard guard; - config::enable_s3_rate_limiter = true; - auto& manager = S3RateLimiterManager::instance(); - auto* put_bytes = manager.bytes_limiter(S3RateLimitType::PUT); - manager.qps_limiter(S3RateLimitType::PUT) - ->reset(kNoThrottleBytesPerSecond, kNoThrottleBytesPerSecond, 2); - put_bytes->reset(kNoThrottleBytesPerSecond, kNoThrottleBytesPerSecond, 1); - - auto fake = std::make_shared(); - RateLimitedObjStorageClient client(fake); - ObjectStoragePathOptions opts {.bucket = "b", .key = "k", .upload_id = "upload"}; - - EXPECT_EQ(0, client.create_multipart_upload(opts).resp.status.code); - EXPECT_EQ(0, client.complete_multipart_upload(opts, {}).status.code); - - auto rejected = client.create_multipart_upload(opts); - EXPECT_EQ(ErrorCode::EXCEEDED_LIMIT, rejected.resp.status.code); - EXPECT_EQ(0, rejected.resp.http_code); - EXPECT_NE(std::string::npos, rejected.resp.status.msg.find("exceeds QPS limit")); - EXPECT_EQ(2, fake->calls); - - EXPECT_EQ(0, put_bytes->add(1)); - EXPECT_EQ(-1, put_bytes->add(1)); -} - -TEST(RateLimitedObjStorageClientTest, delete_apis_map_to_put_qps_without_bytes) { - RateLimiterConfigGuard guard; - config::enable_s3_rate_limiter = true; - auto& manager = S3RateLimiterManager::instance(); - auto* put_bytes = manager.bytes_limiter(S3RateLimitType::PUT); - manager.qps_limiter(S3RateLimitType::PUT) - ->reset(kNoThrottleBytesPerSecond, kNoThrottleBytesPerSecond, 3); - put_bytes->reset(kNoThrottleBytesPerSecond, kNoThrottleBytesPerSecond, 1); - - auto fake = std::make_shared(); - RateLimitedObjStorageClient client(fake); - ObjectStoragePathOptions opts {.bucket = "b", .key = "k", .prefix = "p"}; - - EXPECT_EQ(0, client.delete_object(opts).status.code); - EXPECT_EQ(0, client.delete_objects(opts, {"a", "b"}).status.code); - EXPECT_EQ(0, client.delete_objects_recursively(opts).status.code); - - auto rejected = client.delete_object(opts); - EXPECT_EQ(ErrorCode::EXCEEDED_LIMIT, rejected.status.code); - EXPECT_EQ(0, rejected.http_code); - EXPECT_NE(std::string::npos, rejected.status.msg.find("exceeds QPS limit")); - EXPECT_EQ(3, fake->calls); - - EXPECT_EQ(0, put_bytes->add(1)); - EXPECT_EQ(-1, put_bytes->add(1)); -} - -TEST(RateLimitedObjStorageClientTest, bytes_rejections_have_distinct_text_and_metrics) { - RateLimiterConfigGuard guard; - config::enable_s3_rate_limiter = true; - auto& manager = S3RateLimiterManager::instance(); - manager.qps_limiter(S3RateLimitType::GET)->reset(0, 0, 0); - manager.qps_limiter(S3RateLimitType::PUT)->reset(0, 0, 0); - manager.bytes_limiter(S3RateLimitType::GET) - ->reset(kNoThrottleBytesPerSecond, kNoThrottleBytesPerSecond, 1); - manager.bytes_limiter(S3RateLimitType::PUT) - ->reset(kNoThrottleBytesPerSecond, kNoThrottleBytesPerSecond, 1); - - const int64_t get_rejected_before = s3_get_bytes_rate_limit_rejected_count.get_value(); - const int64_t put_rejected_before = s3_put_bytes_rate_limit_rejected_count.get_value(); - - auto fake = std::make_shared(); - RateLimitedObjStorageClient client(fake); - ObjectStoragePathOptions opts {.bucket = "b", .key = "k"}; +TEST(RateLimitedObjStorageClientTest, DoesNotSettleFailedReads) { + auto inner_client = std::make_shared(); + inner_client->fail_get = true; + auto policy = std::make_shared(); + RateLimitedObjStorageClient client(inner_client, policy); + char buffer[10]; size_t size_return = 0; - auto get_resp = client.get_object(opts, nullptr, 0, 2, &size_return); - EXPECT_EQ(ErrorCode::EXCEEDED_LIMIT, get_resp.status.code); - EXPECT_EQ(0, get_resp.http_code); - EXPECT_NE(std::string::npos, get_resp.status.msg.find("exceeds bytes limit")); - EXPECT_EQ(get_rejected_before + 1, s3_get_bytes_rate_limit_rejected_count.get_value()); - - auto put_resp = client.put_object(opts, "xx"); - EXPECT_EQ(ErrorCode::EXCEEDED_LIMIT, put_resp.status.code); - EXPECT_EQ(0, put_resp.http_code); - EXPECT_NE(std::string::npos, put_resp.status.msg.find("exceeds bytes limit")); - EXPECT_EQ(put_rejected_before + 1, s3_put_bytes_rate_limit_rejected_count.get_value()); - - EXPECT_EQ(0, fake->calls); + auto response = client.get_object({.bucket = "bucket", .key = "key"}, buffer, 0, sizeof(buffer), + &size_return); + + EXPECT_FALSE(response.ok()); + EXPECT_EQ(inner_client->calls, 1); + ASSERT_EQ(policy->requests.size(), 1); + EXPECT_EQ(policy->requests[0].type, S3RateLimitType::GET); + EXPECT_EQ(policy->requests[0].estimated_bytes, sizeof(buffer)); + EXPECT_TRUE(policy->settled_bytes.empty()); } -TEST(RateLimitedObjStorageClientTest, recursive_delete_charges_one_put_qps) { - RateLimiterConfigGuard guard; - config::enable_s3_rate_limiter = true; - auto& manager = S3RateLimiterManager::instance(); - manager.qps_limiter(S3RateLimitType::GET)->reset(0, 0, 1); - manager.qps_limiter(S3RateLimitType::PUT)->reset(0, 0, 1); - manager.bytes_limiter(S3RateLimitType::PUT)->reset(0, 0, 0); - - auto fake = std::make_shared(); - fake->delete_objects_recursively_provider_calls_per_logical_call = 4; - RateLimitedObjStorageClient client(fake); - ObjectStoragePathOptions opts {.bucket = "b", .prefix = "p"}; - - // Exhaust GET first. Recursive delete still succeeds because the logical API is PUT. - EXPECT_EQ(0, client.head_object(opts).resp.status.code); - EXPECT_EQ(0, client.delete_objects_recursively(opts).status.code); - EXPECT_EQ(1, fake->delete_objects_recursively_calls); - EXPECT_EQ(4, fake->delete_objects_recursively_provider_calls); - - auto resp = client.delete_objects_recursively(opts); - EXPECT_EQ(ErrorCode::EXCEEDED_LIMIT, resp.status.code); - EXPECT_EQ(0, resp.http_code); - EXPECT_EQ(1, fake->delete_objects_recursively_calls); - EXPECT_EQ(4, fake->delete_objects_recursively_provider_calls); -} +TEST(RateLimitedObjStorageClientTest, PresignedUrlBypassesRejectedPolicy) { + auto inner_client = std::make_shared(); + auto policy = std::make_shared(); + policy->reject = true; + RateLimitedObjStorageClient client(inner_client, policy); -TEST(RateLimitedObjStorageClientTest, azure_noop_multipart_create_charges_one_put_qps) { - RateLimiterConfigGuard guard; - config::enable_s3_rate_limiter = true; - auto& manager = S3RateLimiterManager::instance(); - manager.qps_limiter(S3RateLimitType::PUT)->reset(0, 0, 1); - manager.bytes_limiter(S3RateLimitType::PUT)->reset(0, 0, 0); - - auto fake = std::make_shared(); - // Azure implements create_multipart_upload as a provider-side no-op. - fake->create_multipart_upload_provider_calls_per_logical_call = 0; - RateLimitedObjStorageClient client(fake); - ObjectStoragePathOptions opts {.bucket = "b", .key = "k"}; - - EXPECT_EQ(0, client.create_multipart_upload(opts).resp.status.code); - EXPECT_EQ(1, fake->create_multipart_upload_calls); - EXPECT_EQ(0, fake->create_multipart_upload_provider_calls); - - auto resp = client.create_multipart_upload(opts); - EXPECT_EQ(ErrorCode::EXCEEDED_LIMIT, resp.resp.status.code); - EXPECT_EQ(0, resp.resp.http_code); - EXPECT_EQ(1, fake->create_multipart_upload_calls); - EXPECT_EQ(0, fake->create_multipart_upload_provider_calls); + EXPECT_EQ(client.generate_presigned_url({.bucket = "bucket", .key = "key"}, 60), "url"); + EXPECT_EQ(inner_client->presigned_url_calls, 1); + EXPECT_TRUE(policy->requests.empty()); } -TEST(RateLimitedObjStorageClientTest, presigned_url_bypasses_rate_limiters) { - RateLimiterConfigGuard guard; - config::enable_s3_rate_limiter = true; - auto& manager = S3RateLimiterManager::instance(); - auto* get_qps = manager.qps_limiter(S3RateLimitType::GET); - auto* put_qps = manager.qps_limiter(S3RateLimitType::PUT); - get_qps->reset(0, 0, 1); - put_qps->reset(0, 0, 1); - EXPECT_EQ(0, get_qps->add(1)); - EXPECT_EQ(0, put_qps->add(1)); - - auto fake = std::make_shared(); - RateLimitedObjStorageClient client(fake); - ObjectStoragePathOptions opts {.bucket = "b", .key = "k"}; - - EXPECT_EQ("presigned", client.generate_presigned_url(opts, 60, S3ClientConf {})); - EXPECT_EQ(1, fake->calls); +TEST(RateLimitedObjStorageClientTest, MultipartCreateUsesOneLogicalAdmission) { + auto inner_client = std::make_shared(); + inner_client->create_multipart_upload_provider_calls_per_logical_call = 2; + auto policy = std::make_shared(); + RateLimitedObjStorageClient client(inner_client, policy); + + EXPECT_TRUE(client.create_multipart_upload({.bucket = "bucket", .key = "key"}).resp.ok()); + ASSERT_EQ(policy->requests.size(), 1); + EXPECT_EQ(policy->requests[0].type, S3RateLimitType::PUT); + EXPECT_EQ(policy->requests[0].estimated_bytes, 0); + EXPECT_EQ(inner_client->create_multipart_upload_calls, 1); + EXPECT_EQ(inner_client->create_multipart_upload_provider_calls, 2); } -} // namespace doris::io +} // namespace +} // namespace doris diff --git a/be/test/io/fs/s3_file_writer_test.cpp b/be/test/io/fs/s3_file_writer_test.cpp index 9f97e335422cbf..1293c6fc36d465 100644 --- a/be/test/io/fs/s3_file_writer_test.cpp +++ b/be/test/io/fs/s3_file_writer_test.cpp @@ -48,6 +48,7 @@ #include "common/config.h" #include "common/status.h" +#include "cpp/obj-client/s3_obj_storage_client.h" #include "cpp/sync_point.h" #include "io/fs/file_reader.h" #include "io/fs/file_system.h" @@ -55,11 +56,11 @@ #include "io/fs/local_file_system.h" #include "io/fs/s3_file_bufferpool.h" #include "io/fs/s3_file_system.h" -#include "io/fs/s3_obj_storage_client.h" #include "io/io_common.h" #include "runtime/exec_env.h" #include "storage/index/index_file_writer.h" #include "storage/index/index_writer.h" +#include "testutil/mock/obj_storage_client_test_stub.h" #include "util/slice.h" #include "util/threadpool.h" #include "util/uuid_generator.h" @@ -988,7 +989,7 @@ TEST_F(S3FileWriterTest, multi_part_complete_error_2) { sp->set_call_back("S3FileWriter::_complete:2", [](auto&& outcome) { // Deliberately make one upload one part task fail to test if s3 file writer could // handle io error - auto* parts = try_any_cast*>(outcome.back()); + auto* parts = try_any_cast*>(outcome.back()); size_t size = parts->size(); parts->back().part_num = (size + 2); }); @@ -1036,7 +1037,7 @@ TEST_F(S3FileWriterTest, multi_part_complete_error_1) { // Deliberately make one upload one part task fail to test if s3 file writer could // handle io error const auto& points = try_any_cast< - const std::pair*>&>( + const std::pair*>&>( outcome.back()); (*points.first) = false; points.second->pop_back(); @@ -1082,10 +1083,10 @@ TEST_F(S3FileWriterTest, multi_part_complete_error_3) { auto sp = SyncPoint::get_instance(); sp->set_call_back("S3FileWriter::_complete:3", [](auto&& outcome) { - auto pair = try_any_cast_ret(outcome); + auto pair = try_any_cast_ret(outcome); pair->second = true; - pair->first = io::ObjectStorageResponse { - .status = convert_to_obj_response(Status::IOError("inject error"))}; + pair->first = + io::ObjStorageResponse {.status = {TStatusCode::INTERNAL_ERROR, "inject error"}}; }); Defer defer {[&]() { sp->clear_call_back("S3FileWriter::_complete:3"); }}; auto client = s3_fs->client_holder(); @@ -1125,21 +1126,19 @@ namespace io { /** * This class is for boundary test */ -class SimpleMockObjStorageClient : public io::ObjStorageClient { +class SimpleMockObjStorageClient : public io::ObjStorageClientTestStub { public: SimpleMockObjStorageClient() = default; ~SimpleMockObjStorageClient() override = default; - ObjectStorageResponse default_response {ObjectStorageResponse::OK()}; - ObjectStorageUploadResponse default_upload_response {.resp = ObjectStorageResponse::OK(), - .upload_id = "mock-upload-id", - .etag = "mock-etag"}; - ObjectStorageHeadResponse default_head_response {.resp = ObjectStorageResponse::OK(), - .file_size = 1024}; + ObjStorageResponse default_response {ObjStorageResponse::OK()}; + ObjStorageUploadResult default_upload_response { + .resp = ObjStorageResponse::OK(), .upload_id = "mock-upload-id", .etag = "mock-etag"}; + ObjStorageHeadResult default_head_response {.resp = ObjStorageResponse::OK(), + .file_size = 1024}; std::string default_presigned_url = "https://mock-presigned-url.com"; - ObjectStorageUploadResponse create_multipart_upload( - const ObjectStoragePathOptions& opts) override { + ObjStorageUploadResult create_multipart_upload(const ObjStoragePath& opts) override { std::lock_guard lock(_mutex); create_multipart_count++; create_multipart_params.push_back(opts); @@ -1147,8 +1146,7 @@ class SimpleMockObjStorageClient : public io::ObjStorageClient { return default_upload_response; } - ObjectStorageResponse put_object(const ObjectStoragePathOptions& opts, - std::string_view stream) override { + ObjStorageResponse put_object(const ObjStoragePath& opts, std::string_view stream) override { std::lock_guard lock(_mutex); put_object_count++; put_object_params.emplace_back(opts, std::string(stream)); @@ -1159,12 +1157,13 @@ class SimpleMockObjStorageClient : public io::ObjStorageClient { return default_response; } - ObjectStorageUploadResponse upload_part(const ObjectStoragePathOptions& opts, - std::string_view stream, int part_num) override { + ObjStorageUploadResult upload_part(const ObjStoragePath& opts, const std::string& upload_id, + std::string_view stream, int part_num) override { std::lock_guard lock(_mutex); upload_part_count++; // upload_part_params.push_back({opts, std::string(stream), part_num}); last_opts = opts; + last_upload_id = upload_id; last_stream = std::string(stream); last_part_num = part_num; parts[_part_key(opts.path.native(), part_num)] = std::string(stream); @@ -1172,13 +1171,14 @@ class SimpleMockObjStorageClient : public io::ObjStorageClient { return default_upload_response; } - ObjectStorageResponse complete_multipart_upload( - const ObjectStoragePathOptions& opts, - const std::vector& completed_parts) override { + ObjStorageResponse complete_multipart_upload( + const ObjStoragePath& opts, const std::string& upload_id, + const std::vector& completed_parts) override { std::lock_guard lock(_mutex); complete_multipart_count++; complete_multipart_params.push_back({opts, completed_parts}); last_opts = opts; + last_upload_id = upload_id; last_completed_parts = completed_parts; std::string final_obj; final_obj.reserve(uploaded_bytes); @@ -1190,15 +1190,14 @@ class SimpleMockObjStorageClient : public io::ObjStorageClient { return default_response; } - ObjectStorageHeadResponse head_object(const ObjectStoragePathOptions& opts) override { + ObjStorageHeadResult head_object(const ObjStoragePath& opts) override { std::lock_guard lock(_mutex); - return {.resp = ObjectStorageResponse::OK(), + return {.resp = ObjStorageResponse::OK(), .file_size = static_cast(objects[opts.path.native()].size())}; } - ObjectStorageResponse get_object(const ObjectStoragePathOptions& opts, void* buffer, - size_t offset, size_t bytes_read, - size_t* size_return) override { + ObjStorageResponse get_object(const ObjStoragePath& opts, void* buffer, size_t offset, + size_t bytes_read, size_t* size_return) override { std::lock_guard lock(_mutex); last_opts = opts; last_offset = offset; @@ -1209,39 +1208,29 @@ class SimpleMockObjStorageClient : public io::ObjStorageClient { return default_response; } - ObjectStorageResponse list_objects(const ObjectStoragePathOptions& opts, - std::vector* files) override { + ObjStorageListPageResult list_objects_page(const ObjStoragePath& opts, + std::string_view /*continuation_token*/) override { std::lock_guard lock(_mutex); last_opts = opts; - if (files) { - *files = default_file_list; - } - return default_response; + return {.resp = default_response}; } - ObjectStorageResponse delete_objects(const ObjectStoragePathOptions& opts, - std::vector objs) override { + ObjStorageResponse delete_objects(const ObjStoragePath& opts, + std::vector objs) override { std::lock_guard lock(_mutex); last_opts = opts; last_deleted_objects = std::move(objs); return default_response; } - ObjectStorageResponse delete_object(const ObjectStoragePathOptions& opts) override { - std::lock_guard lock(_mutex); - last_opts = opts; - return default_response; - } - - ObjectStorageResponse delete_objects_recursively( - const ObjectStoragePathOptions& opts) override { + ObjStorageResponse delete_object(const ObjStoragePath& opts) override { std::lock_guard lock(_mutex); last_opts = opts; return default_response; } - std::string generate_presigned_url(const ObjectStoragePathOptions& opts, - int64_t expiration_secs, const S3ClientConf& conf) override { + std::string generate_presigned_url(const ObjStoragePath& opts, + int64_t expiration_secs) override { std::lock_guard lock(_mutex); last_opts = opts; last_expiration_secs = expiration_secs; @@ -1249,10 +1238,11 @@ class SimpleMockObjStorageClient : public io::ObjStorageClient { } // Variables to store the last call - ObjectStoragePathOptions last_opts; + ObjStoragePath last_opts; + std::string last_upload_id; std::string last_stream; int last_part_num = 0; - std::vector last_completed_parts; + std::vector last_completed_parts; size_t last_offset = 0; size_t last_bytes_read = 0; std::vector last_deleted_objects; @@ -1267,19 +1257,19 @@ class SimpleMockObjStorageClient : public io::ObjStorageClient { // Structures to store input parameters for each call struct UploadPartParams { - ObjectStoragePathOptions opts; + ObjStoragePath opts; std::string stream; int part_num; }; struct CompleteMultipartParams { - ObjectStoragePathOptions opts; - std::vector parts; + ObjStoragePath opts; + std::vector parts; }; // Vectors to store parameters from each call - std::vector create_multipart_params; - std::vector> put_object_params; + std::vector create_multipart_params; + std::vector> put_object_params; // std::vector upload_part_params; std::vector complete_multipart_params; std::map objects; @@ -1289,7 +1279,8 @@ class SimpleMockObjStorageClient : public io::ObjStorageClient { void reset() { std::lock_guard lock(_mutex); - last_opts = ObjectStoragePathOptions {}; + last_opts = ObjStoragePath {}; + last_upload_id.clear(); last_stream.clear(); last_part_num = 0; last_completed_parts.clear(); diff --git a/be/test/io/fs/s3_obj_storage_client_role_test.cpp b/be/test/io/fs/s3_obj_storage_client_role_test.cpp index 4646de690df047..86705c06730a18 100644 --- a/be/test/io/fs/s3_obj_storage_client_role_test.cpp +++ b/be/test/io/fs/s3_obj_storage_client_role_test.cpp @@ -17,14 +17,15 @@ #include -#include "io/fs/obj_storage_client.h" +#include "cpp/obj-client/obj_storage_client.h" +#include "io/fs/file_system.h" #include "util/s3_util.h" namespace doris { class S3ObjStorageClientRoleTest : public testing::Test { protected: - static std::shared_ptr obj_storage_client; + static std::shared_ptr obj_storage_client; static std::string bucket; static std::string prefix; @@ -48,20 +49,20 @@ class S3ObjStorageClientRoleTest : public testing::Test { S3ObjStorageClientRoleTest::prefix = std::getenv("AWS_PREFIX"); } - S3ObjStorageClientRoleTest::obj_storage_client = S3ClientFactory::instance().create( + auto client_result = S3ClientFactory::instance().create( {.endpoint = endpoint, .region = region, .ak = "", .sk = "", .token = "", .bucket = bucket, - .provider = io::ObjStorageType::AWS, + .provider = ObjStorageProvider::AWS, .use_virtual_addressing = false, .cred_provider_type = CredProviderType::InstanceProfile, .role_arn = role_arn, .external_id = external_id}); - - ASSERT_TRUE(S3ObjStorageClientRoleTest::obj_storage_client != nullptr); + ASSERT_TRUE(client_result.has_value()) << client_result.error(); + S3ObjStorageClientRoleTest::obj_storage_client = std::move(client_result).value(); } void SetUp() override { @@ -71,7 +72,7 @@ class S3ObjStorageClientRoleTest : public testing::Test { } }; -std::shared_ptr S3ObjStorageClientRoleTest::obj_storage_client = nullptr; +std::shared_ptr S3ObjStorageClientRoleTest::obj_storage_client = nullptr; std::string S3ObjStorageClientRoleTest::bucket; std::string S3ObjStorageClientRoleTest::prefix; @@ -83,26 +84,24 @@ TEST_F(S3ObjStorageClientRoleTest, put_list_delete_object) { std::string("aaaa")); EXPECT_EQ(response.status.code, ErrorCode::OK); - std::vector files; - // clang-format off - response = S3ObjStorageClientRoleTest::obj_storage_client->list_objects({.bucket = bucket, - .prefix = prefix + "S3ObjStorageClientRoleTest/put_list_delete_object",}, &files); - // clang-format on + std::vector objects; + response = S3ObjStorageClientRoleTest::obj_storage_client->list_objects( + {.bucket = bucket, .key = prefix + "S3ObjStorageClientRoleTest/put_list_delete_object"}, + &objects); EXPECT_EQ(response.status.code, ErrorCode::OK); - EXPECT_EQ(files.size(), 1); - files.clear(); + EXPECT_EQ(objects.size(), 1); + objects.clear(); response = S3ObjStorageClientRoleTest::obj_storage_client->delete_object( {.bucket = bucket, .key = prefix + "S3ObjStorageClientRoleTest/put_list_delete_object"}); EXPECT_EQ(response.status.code, ErrorCode::OK); - // clang-format off - response = S3ObjStorageClientRoleTest::obj_storage_client->list_objects({.bucket = bucket, - .prefix = prefix + "S3ObjStorageClientRoleTest/put_list_delete_object",}, &files); - // clang-format on + response = S3ObjStorageClientRoleTest::obj_storage_client->list_objects( + {.bucket = bucket, .key = prefix + "S3ObjStorageClientRoleTest/put_list_delete_object"}, + &objects); EXPECT_EQ(response.status.code, ErrorCode::OK); - EXPECT_EQ(files.size(), 0); + EXPECT_TRUE(objects.empty()); } TEST_F(S3ObjStorageClientRoleTest, delete_objects_recursively) { @@ -118,26 +117,27 @@ TEST_F(S3ObjStorageClientRoleTest, delete_objects_recursively) { LOG(INFO) << "put " << key << " OK"; } - std::vector files; - // clang-format off - auto response = S3ObjStorageClientRoleTest::obj_storage_client->list_objects({.bucket = bucket, - .prefix = prefix + "S3ObjStorageClientRoleTest/delete_objects_recursively",}, &files); - // clang-format on + std::vector objects; + auto response = S3ObjStorageClientRoleTest::obj_storage_client->list_objects( + {.bucket = bucket, + .key = prefix + "S3ObjStorageClientRoleTest/delete_objects_recursively"}, + &objects); EXPECT_EQ(response.status.code, ErrorCode::OK); - EXPECT_EQ(files.size(), 22); - files.clear(); + EXPECT_EQ(objects.size(), 22); + objects.clear(); - response = S3ObjStorageClientRoleTest::obj_storage_client->delete_objects_recursively( + response = delete_objects_recursively( + S3ObjStorageClientRoleTest::obj_storage_client, {.bucket = bucket, .prefix = prefix + "S3ObjStorageClientRoleTest/delete_objects_recursively"}); EXPECT_EQ(response.status.code, ErrorCode::OK); - // clang-format off - response = S3ObjStorageClientRoleTest::obj_storage_client->list_objects({.bucket = bucket, - .prefix = prefix + "S3ObjStorageClientRoleTest/delete_objects_recursively",}, &files); - // clang-format on + response = S3ObjStorageClientRoleTest::obj_storage_client->list_objects( + {.bucket = bucket, + .key = prefix + "S3ObjStorageClientRoleTest/delete_objects_recursively"}, + &objects); EXPECT_EQ(response.status.code, ErrorCode::OK); - EXPECT_EQ(files.size(), 0); + EXPECT_TRUE(objects.empty()); } TEST_F(S3ObjStorageClientRoleTest, multipart_upload) { @@ -146,43 +146,40 @@ TEST_F(S3ObjStorageClientRoleTest, multipart_upload) { auto response = S3ObjStorageClientRoleTest::obj_storage_client->create_multipart_upload( {.bucket = bucket, .key = prefix + "S3ObjStorageClientRoleTest/multipart_upload"}); EXPECT_EQ(response.resp.status.code, ErrorCode::OK); - auto upload_id = response.upload_id; + ASSERT_TRUE(response.upload_id.has_value()); + const auto& upload_id = *response.upload_id; std::string body = "S3ObjStorageClientRoleTest::multipart_upload"; body.resize(5 * 1024 * 1024); - std::vector completed_parts; + std::vector completed_parts; response = S3ObjStorageClientRoleTest::obj_storage_client->upload_part( - {.bucket = bucket, - .key = prefix + "S3ObjStorageClientRoleTest/multipart_upload", - .upload_id = upload_id}, - body, 1); + {.bucket = bucket, .key = prefix + "S3ObjStorageClientRoleTest/multipart_upload"}, + upload_id, body, 1); EXPECT_EQ(response.resp.status.code, ErrorCode::OK); - doris::io::ObjectCompleteMultiPart completed_part { - 1, response.etag.has_value() ? std::move(response.etag.value()) : ""}; + ObjStorageCompletedPart completed_part { + .part_num = 1, + .etag = response.etag.has_value() ? std::move(response.etag.value()) : ""}; completed_parts.emplace_back(std::move(completed_part)); response = S3ObjStorageClientRoleTest::obj_storage_client->upload_part( - {.bucket = bucket, - .key = prefix + "S3ObjStorageClientRoleTest/multipart_upload", - .upload_id = upload_id}, - body, 2); + {.bucket = bucket, .key = prefix + "S3ObjStorageClientRoleTest/multipart_upload"}, + upload_id, body, 2); EXPECT_EQ(response.resp.status.code, ErrorCode::OK); - doris::io::ObjectCompleteMultiPart completed_part2 { - 2, response.etag.has_value() ? std::move(response.etag.value()) : ""}; + ObjStorageCompletedPart completed_part2 { + .part_num = 2, + .etag = response.etag.has_value() ? std::move(response.etag.value()) : ""}; completed_parts.emplace_back(std::move(completed_part2)); auto response2 = S3ObjStorageClientRoleTest::obj_storage_client->complete_multipart_upload( - {.bucket = bucket, - .key = prefix + "S3ObjStorageClientRoleTest/multipart_upload", - .upload_id = upload_id}, - completed_parts); + {.bucket = bucket, .key = prefix + "S3ObjStorageClientRoleTest/multipart_upload"}, + upload_id, completed_parts); EXPECT_EQ(response2.status.code, ErrorCode::OK); } -} // namespace doris \ No newline at end of file +} // namespace doris diff --git a/be/test/io/fs/s3_obj_storage_client_test.cpp b/be/test/io/fs/s3_obj_storage_client_test.cpp index 97b44b8fa0cba4..4bafbc4dde71f7 100644 --- a/be/test/io/fs/s3_obj_storage_client_test.cpp +++ b/be/test/io/fs/s3_obj_storage_client_test.cpp @@ -15,16 +15,19 @@ // specific language governing permissions and limitations // under the License. +#include "cpp/obj-client/s3_obj_storage_client.h" + #include -#include "io/fs/obj_storage_client.h" +#include "cpp/obj-client/obj_storage_client.h" +#include "io/fs/file_system.h" #include "util/s3_util.h" namespace doris { class S3ObjStorageClientTest : public testing::Test { protected: - static std::shared_ptr obj_storage_client; + static std::shared_ptr obj_storage_client; static std::string bucket; static void SetUpTestSuite() { @@ -39,20 +42,20 @@ class S3ObjStorageClientTest : public testing::Test { S3ObjStorageClientTest::bucket = std::getenv("AWS_BUCKET"); - S3ObjStorageClientTest::obj_storage_client = S3ClientFactory::instance().create({ + auto client_result = S3ClientFactory::instance().create({ .endpoint = endpoint, .region = "dummy-region", .ak = access_key, .sk = secret_key, .token = "", .bucket = bucket, - .provider = io::ObjStorageType::AWS, + .provider = ObjStorageProvider::AWS, .use_virtual_addressing = false, .role_arn = "", .external_id = "", }); - - ASSERT_TRUE(S3ObjStorageClientTest::obj_storage_client != nullptr); + ASSERT_TRUE(client_result.has_value()) << client_result.error(); + S3ObjStorageClientTest::obj_storage_client = std::move(client_result).value(); } void SetUp() override { @@ -62,9 +65,23 @@ class S3ObjStorageClientTest : public testing::Test { } }; -std::shared_ptr S3ObjStorageClientTest::obj_storage_client = nullptr; +std::shared_ptr S3ObjStorageClientTest::obj_storage_client = nullptr; std::string S3ObjStorageClientTest::bucket; +TEST(S3ObjStorageClientErrorTest, MapsProviderResponseCodes) { + Aws::S3::S3Error error; + error.SetResponseCode(Aws::Http::HttpResponseCode::FORBIDDEN); + + EXPECT_EQ(s3fs_error(error, "access denied").code, ErrorCode::PERMISSION_DENIED); + static_assert(ObjStorageStatus::PERMISSION_DENIED == ErrorCode::PERMISSION_DENIED); + + error.SetResponseCode(Aws::Http::HttpResponseCode::REQUEST_NOT_MADE); + EXPECT_EQ(s3fs_error(error, "network failure").code, ObjStorageStatus::NETWORK_ERROR); + + error.SetResponseCode(Aws::Http::HttpResponseCode::TOO_MANY_REQUESTS); + EXPECT_EQ(s3fs_error(error, "throttled").code, ObjStorageStatus::LIMIT_REACH); +} + TEST_F(S3ObjStorageClientTest, put_list_delete_object) { LOG(INFO) << "S3ObjStorageClientTest::put_list_delete_object"; @@ -73,25 +90,21 @@ TEST_F(S3ObjStorageClientTest, put_list_delete_object) { std::string("aaaa")); EXPECT_EQ(response.status.code, ErrorCode::OK); - std::vector files; - // clang-format off - response = S3ObjStorageClientTest::obj_storage_client->list_objects({.bucket = bucket, - .prefix = "S3ObjStorageClientTest/put_list_delete_object",}, &files); - // clang-format on + std::vector objects; + response = S3ObjStorageClientTest::obj_storage_client->list_objects( + {.bucket = bucket, .key = "S3ObjStorageClientTest/put_list_delete_object"}, &objects); EXPECT_EQ(response.status.code, ErrorCode::OK); - EXPECT_EQ(files.size(), 1); - files.clear(); + EXPECT_EQ(objects.size(), 1); + objects.clear(); response = S3ObjStorageClientTest::obj_storage_client->delete_object( {.bucket = bucket, .key = "S3ObjStorageClientTest/put_list_delete_object"}); EXPECT_EQ(response.status.code, ErrorCode::OK); - // clang-format off - response = S3ObjStorageClientTest::obj_storage_client->list_objects({.bucket = bucket, - .prefix = "S3ObjStorageClientTest/put_list_delete_object",}, &files); - // clang-format on + response = S3ObjStorageClientTest::obj_storage_client->list_objects( + {.bucket = bucket, .key = "S3ObjStorageClientTest/put_list_delete_object"}, &objects); EXPECT_EQ(response.status.code, ErrorCode::OK); - EXPECT_EQ(files.size(), 0); + EXPECT_TRUE(objects.empty()); } TEST_F(S3ObjStorageClientTest, delete_objects_recursively) { @@ -106,25 +119,24 @@ TEST_F(S3ObjStorageClientTest, delete_objects_recursively) { LOG(INFO) << "put " << key << " OK"; } - std::vector files; - // clang-format off - auto response = S3ObjStorageClientTest::obj_storage_client->list_objects({.bucket = bucket, - .prefix = "S3ObjStorageClientTest/delete_objects_recursively",}, &files); - // clang-format on + std::vector objects; + auto response = S3ObjStorageClientTest::obj_storage_client->list_objects( + {.bucket = bucket, .key = "S3ObjStorageClientTest/delete_objects_recursively"}, + &objects); EXPECT_EQ(response.status.code, ErrorCode::OK); - EXPECT_EQ(files.size(), 22); - files.clear(); + EXPECT_EQ(objects.size(), 22); + objects.clear(); - response = S3ObjStorageClientTest::obj_storage_client->delete_objects_recursively( + response = delete_objects_recursively( + S3ObjStorageClientTest::obj_storage_client, {.bucket = bucket, .prefix = "S3ObjStorageClientTest/delete_objects_recursively"}); EXPECT_EQ(response.status.code, ErrorCode::OK); - // clang-format off - response = S3ObjStorageClientTest::obj_storage_client->list_objects({.bucket = bucket, - .prefix = "S3ObjStorageClientTest/delete_objects_recursively",}, &files); - // clang-format on + response = S3ObjStorageClientTest::obj_storage_client->list_objects( + {.bucket = bucket, .key = "S3ObjStorageClientTest/delete_objects_recursively"}, + &objects); EXPECT_EQ(response.status.code, ErrorCode::OK); - EXPECT_EQ(files.size(), 0); + EXPECT_TRUE(objects.empty()); } TEST_F(S3ObjStorageClientTest, multipart_upload) { @@ -133,43 +145,40 @@ TEST_F(S3ObjStorageClientTest, multipart_upload) { auto response = S3ObjStorageClientTest::obj_storage_client->create_multipart_upload( {.bucket = bucket, .key = "S3ObjStorageClientTest/multipart_upload"}); EXPECT_EQ(response.resp.status.code, ErrorCode::OK); - auto upload_id = response.upload_id; + ASSERT_TRUE(response.upload_id.has_value()); + const auto& upload_id = *response.upload_id; std::string body = "S3ObjStorageClientTest::multipart_upload"; body.resize(5 * 1024 * 1024); - std::vector completed_parts; + std::vector completed_parts; response = S3ObjStorageClientTest::obj_storage_client->upload_part( - {.bucket = bucket, - .key = "S3ObjStorageClientTest/multipart_upload", - .upload_id = upload_id}, - body, 1); + {.bucket = bucket, .key = "S3ObjStorageClientTest/multipart_upload"}, upload_id, body, + 1); EXPECT_EQ(response.resp.status.code, ErrorCode::OK); - doris::io::ObjectCompleteMultiPart completed_part { - 1, response.etag.has_value() ? std::move(response.etag.value()) : ""}; + ObjStorageCompletedPart completed_part { + .part_num = 1, + .etag = response.etag.has_value() ? std::move(response.etag.value()) : ""}; completed_parts.emplace_back(std::move(completed_part)); response = S3ObjStorageClientTest::obj_storage_client->upload_part( - {.bucket = bucket, - .key = "S3ObjStorageClientTest/multipart_upload", - .upload_id = upload_id}, - body, 2); + {.bucket = bucket, .key = "S3ObjStorageClientTest/multipart_upload"}, upload_id, body, + 2); EXPECT_EQ(response.resp.status.code, ErrorCode::OK); - doris::io::ObjectCompleteMultiPart completed_part2 { - 2, response.etag.has_value() ? std::move(response.etag.value()) : ""}; + ObjStorageCompletedPart completed_part2 { + .part_num = 2, + .etag = response.etag.has_value() ? std::move(response.etag.value()) : ""}; completed_parts.emplace_back(std::move(completed_part2)); auto response2 = S3ObjStorageClientTest::obj_storage_client->complete_multipart_upload( - {.bucket = bucket, - .key = "S3ObjStorageClientTest/multipart_upload", - .upload_id = upload_id}, + {.bucket = bucket, .key = "S3ObjStorageClientTest/multipart_upload"}, upload_id, completed_parts); EXPECT_EQ(response2.status.code, ErrorCode::OK); } -} // namespace doris \ No newline at end of file +} // namespace doris diff --git a/be/test/io/fs/s3_obj_stroage_client_mock_test.cpp b/be/test/io/fs/s3_obj_stroage_client_mock_test.cpp index 1a00bcb8642f0c..fb45933ac2ab3e 100644 --- a/be/test/io/fs/s3_obj_stroage_client_mock_test.cpp +++ b/be/test/io/fs/s3_obj_stroage_client_mock_test.cpp @@ -21,35 +21,41 @@ #include #include +#include "cpp/obj-client/obj_storage_client.h" +#include "cpp/obj-client/rate_limited_obj_storage_client.h" +#include "cpp/obj-client/s3_obj_storage_client.h" #include "gmock/gmock.h" -#include "io/fs/rate_limited_obj_storage_client.h" -#include "io/fs/s3_obj_storage_client.h" -#include "util/s3_rate_limiter_manager.h" +#include "io/fs/file_system.h" #include "util/s3_util.h" #include "util/string_util.h" using namespace Aws::S3::Model; namespace doris::io { -namespace { - -struct RateLimiterConfigGuard { - bool enable = config::enable_s3_rate_limiter; - - ~RateLimiterConfigGuard() { - config::enable_s3_rate_limiter = enable; - S3RateLimiterManager::instance().refresh(); - } -}; - -} // namespace - class MockS3Client : public Aws::S3::S3Client { public: MockS3Client() {}; MOCK_METHOD(Aws::S3::Model::ListObjectsV2Outcome, ListObjectsV2, (const Aws::S3::Model::ListObjectsV2Request& request), (const, override)); + MOCK_METHOD(Aws::S3::Model::DeleteObjectOutcome, DeleteObject, + (const Aws::S3::Model::DeleteObjectRequest& request), (const, override)); + MOCK_METHOD(Aws::S3::Model::DeleteObjectsOutcome, DeleteObjects, + (const Aws::S3::Model::DeleteObjectsRequest& request), (const, override)); +}; + +class CountingGetRateLimitPolicy final : public ObjStorageRateLimitPolicy { +public: + explicit CountingGetRateLimitPolicy(size_t* request_count) : request_count_(request_count) {} + + ObjStorageAdmission acquire(S3RateLimitType type, size_t) const override { + EXPECT_EQ(type, S3RateLimitType::GET); + ++*request_count_; + return {}; + } + +private: + size_t* request_count_; }; class S3ObjStorageClientMockTest : public testing::Test { @@ -66,21 +72,20 @@ TEST_F(S3ObjStorageClientMockTest, list_objects_compatibility) { // If storage only supports ListObjectsV1, s3_obj_storage_client.list_objects // should return an error. auto mock_s3_client = std::make_shared(); - S3ObjStorageClient s3_obj_storage_client(mock_s3_client); - - std::vector files; + auto s3_obj_storage_client = std::make_shared(mock_s3_client); ListObjectsV2Result result; result.SetIsTruncated(true); EXPECT_CALL(*mock_s3_client, ListObjectsV2(testing::_)) .WillOnce(testing::Return(ListObjectsV2Outcome(result))); - auto response = s3_obj_storage_client.list_objects( - {.bucket = "dummy-bucket", .prefix = "S3ObjStorageClientMockTest/list_objects_test"}, - &files); + std::vector objects; + auto response = s3_obj_storage_client->list_objects( + {.bucket = "dummy-bucket", .key = "S3ObjStorageClientMockTest/list_objects_test"}, + &objects); + EXPECT_TRUE(objects.empty()); EXPECT_EQ(response.status.code, ErrorCode::INTERNAL_ERROR); - files.clear(); } ListObjectsV2Result CreatePageResult(const std::string& nextToken, @@ -96,16 +101,14 @@ ListObjectsV2Result CreatePageResult(const std::string& nextToken, return result; } -TEST_F(S3ObjStorageClientMockTest, list_objects_pagination_charges_one_get_qps) { - RateLimiterConfigGuard guard; - config::enable_s3_rate_limiter = true; - auto& manager = S3RateLimiterManager::instance(); - manager.qps_limiter(S3RateLimitType::GET)->reset(0, 0, 1); - manager.bytes_limiter(S3RateLimitType::GET)->reset(0, 0, 0); - +TEST_F(S3ObjStorageClientMockTest, list_objects_with_pagination) { auto mock_s3_client = std::make_shared(); - auto s3_obj_storage_client = std::make_shared(mock_s3_client); - RateLimitedObjStorageClient rate_limited_client(s3_obj_storage_client); + size_t get_request_count = 0; + auto inner_client = std::make_shared(mock_s3_client); + auto obj_storage_client = std::make_shared( + std::move(inner_client), + std::make_shared(&get_request_count)); + std::string prefix = "S3ObjStorageClientMockTest/list_objects_with_pagination/"; std::vector> pages = { {"key1", "key2"}, // page1 @@ -113,6 +116,12 @@ TEST_F(S3ObjStorageClientMockTest, list_objects_pagination_charges_one_get_qps) {"key5"} // page3 }; + for (auto& page : pages) { + for (auto& key : page) { + key = prefix + key; + } + } + EXPECT_CALL(*mock_s3_client, ListObjectsV2(testing::_)) .WillOnce([&](const ListObjectsV2Request& req) { // page1：no ContinuationToken @@ -131,20 +140,46 @@ TEST_F(S3ObjStorageClientMockTest, list_objects_pagination_charges_one_get_qps) return ListObjectsV2Outcome(CreatePageResult("", pages[2], false)); }); - std::vector files; - const ObjectStoragePathOptions opts { - .bucket = "dummy-bucket", - .prefix = "S3ObjStorageClientMockTest/list_objects_with_pagination"}; - auto response = rate_limited_client.list_objects(opts, &files); + std::vector objects; + auto response = obj_storage_client->list_objects( + {.bucket = "dummy-bucket", + .key = "S3ObjStorageClientMockTest/list_objects_with_pagination"}, + &objects); EXPECT_EQ(response.status.code, ErrorCode::OK); - EXPECT_EQ(files.size(), 5); + EXPECT_EQ(objects.size(), 5); + EXPECT_EQ(get_request_count, pages.size()); +} + +TEST_F(S3ObjStorageClientMockTest, + delete_object_preserves_not_found_and_batch_uses_delete_objects) { + auto mock_s3_client = std::make_shared(); + auto s3_obj_storage_client = std::make_shared(mock_s3_client); + auto not_found = [](const DeleteObjectRequest&) { + Aws::S3::S3Error error; + error.SetResponseCode(Aws::Http::HttpResponseCode::NOT_FOUND); + error.SetMessage("object not found"); + error.SetRequestId("request-id"); + return DeleteObjectOutcome(std::move(error)); + }; + EXPECT_CALL(*mock_s3_client, DeleteObject(testing::_)).WillOnce(not_found); + EXPECT_CALL(*mock_s3_client, DeleteObjects(testing::_)) + .WillOnce([](const DeleteObjectsRequest& request) { + const auto& objects = request.GetDelete().GetObjects(); + EXPECT_EQ(objects.size(), 1); + EXPECT_EQ(objects.front().GetKey(), "missing-object"); + return DeleteObjectsOutcome(DeleteObjectsResult {}); + }); + + auto response = s3_obj_storage_client->delete_object( + {.bucket = "dummy-bucket", .key = "missing-object"}); + EXPECT_EQ(response.status.code, ObjStorageStatus::NOT_FOUND); + EXPECT_EQ(response.http_code, static_cast(Aws::Http::HttpResponseCode::NOT_FOUND)); + EXPECT_EQ(response.request_id, "request-id"); - // The first logical list used one GET token despite issuing three provider requests. - // A second logical list is rejected before it reaches the provider. - response = rate_limited_client.list_objects(opts, &files); - EXPECT_EQ(ErrorCode::EXCEEDED_LIMIT, response.status.code); - EXPECT_EQ(0, response.http_code); + response = + s3_obj_storage_client->delete_objects({.bucket = "dummy-bucket"}, {"missing-object"}); + EXPECT_TRUE(response.ok()); } TEST_F(S3ObjStorageClientMockTest, test_ca_cert) { diff --git a/be/test/io/fs/s3_response_stream_test.cpp b/be/test/io/fs/s3_response_stream_test.cpp index 65b246f7c1e2a6..a874cdc1c09347 100644 --- a/be/test/io/fs/s3_response_stream_test.cpp +++ b/be/test/io/fs/s3_response_stream_test.cpp @@ -21,7 +21,7 @@ #include #include -#include "io/fs/s3_common.h" +#include "cpp/obj-client/s3_common.h" namespace doris { diff --git a/be/test/io/s3_client_factory_test.cpp b/be/test/io/s3_client_factory_test.cpp index 74299eea30a411..0a87ec394bff5c 100644 --- a/be/test/io/s3_client_factory_test.cpp +++ b/be/test/io/s3_client_factory_test.cpp @@ -24,10 +24,14 @@ #include #include #include +#include #include +#include #include #include +#include +#include #include #include #include @@ -35,14 +39,17 @@ #include #include "cloud/config.h" +#include "common/config.h" #include "cpp/aws_common.h" #include "cpp/custom_aws_credentials_provider_chain.h" #include "cpp/obj-client/auth/gcp/gcp_auth.h" #include "cpp/obj-client/auth/gcp/gcp_s3_client.h" -#include "io/fs/err_utils.h" -#include "io/fs/rate_limited_obj_storage_client.h" -#include "io/fs/s3_obj_storage_client.h" +#include "cpp/obj-client/rate_limited_obj_storage_client.h" +#include "cpp/obj-client/s3_obj_storage_client.h" +#include "cpp/sync_point.h" +#include "io/fs/s3_file_system.h" #include "testutil/container_credentials_endpoint.h" +#include "util/s3_rate_limiter_manager.h" #include "util/s3_uri.h" #include "util/s3_util.h" @@ -57,6 +64,23 @@ class S3ClientFactoryTest : public testing::Test { namespace { +constexpr size_t kNoThrottleBytesPerSecond = 1ULL << 40; + +S3ClientConf make_factory_conf(std::string endpoint, bool is_internal_bucket) { + S3ClientConf conf; + conf.endpoint = std::move(endpoint); + conf.region = "us-east-1"; + conf.cred_provider_type = CredProviderType::Anonymous; + conf.is_internal_bucket = is_internal_bucket; + return conf; +} + +S3ClientConf make_hash_collision_conf(std::string endpoint, bool is_internal_bucket) { + auto conf = make_factory_conf(std::move(endpoint), is_internal_bucket); + conf.use_virtual_addressing = !is_internal_bucket; + return conf; +} + class CloudModeConfigGuard { public: explicit CloudModeConfigGuard(bool cloud_mode) @@ -75,24 +99,58 @@ class CloudModeConfigGuard { std::string _cloud_unique_id; }; -S3ClientConf make_factory_conf(std::string endpoint, bool is_internal_bucket) { - S3ClientConf conf; - conf.endpoint = std::move(endpoint); - conf.region = "us-east-1"; - conf.cred_provider_type = CredProviderType::Anonymous; - conf.is_internal_bucket = is_internal_bucket; - return conf; -} +class RateLimiterConfigGuard { +public: + RateLimiterConfigGuard() + : _enabled(config::enable_s3_rate_limiter), + _qps_max_speed(_qps()->get_max_speed()), + _qps_max_burst(_qps()->get_max_burst()), + _qps_limit(_qps()->get_limit()), + _bytes_max_speed(_bytes()->get_max_speed()), + _bytes_max_burst(_bytes()->get_max_burst()), + _bytes_limit(_bytes()->get_limit()) {} + + ~RateLimiterConfigGuard() { + config::enable_s3_rate_limiter = _enabled; + _qps()->reset(_qps_max_speed, _qps_max_burst, _qps_limit); + _bytes()->reset(_bytes_max_speed, _bytes_max_burst, _bytes_limit); + } -S3ClientConf make_hash_collision_conf(std::string endpoint, bool is_internal_bucket) { - auto conf = make_factory_conf(std::move(endpoint), is_internal_bucket); - conf.use_virtual_addressing = !is_internal_bucket; - return conf; -} +private: + static S3RateLimiterHolder* _qps() { + return S3RateLimiterManager::instance().qps_limiter(S3RateLimitType::GET); + } + static S3RateLimiterHolder* _bytes() { + return S3RateLimiterManager::instance().bytes_limiter(S3RateLimitType::GET); + } + + bool _enabled; + size_t _qps_max_speed; + size_t _qps_max_burst; + size_t _qps_limit; + size_t _bytes_max_speed; + size_t _bytes_max_burst; + size_t _bytes_limit; +}; + +class SyncPointProcessingGuard { +public: + SyncPointProcessingGuard() : _was_enabled(SyncPoint::get_instance()->get_enable()) { + SyncPoint::get_instance()->enable_processing(); + } + ~SyncPointProcessingGuard() { + if (!_was_enabled) { + SyncPoint::get_instance()->disable_processing(); + } + } + +private: + bool _was_enabled; +}; S3ClientConf make_gcp_native_conf(std::string impersonation_service_account = {}) { auto conf = make_factory_conf("storage.googleapis.com", true); - conf.provider = io::ObjStorageType::GCP; + conf.provider = io::ObjStorageProvider::GCP; conf.region = "us-east1"; conf.cred_provider_type = CredProviderType::Default; conf.credential = GcpCredentialConfig { @@ -116,23 +174,7 @@ class FixedTokenGcpS3Client final : public GcpS3Client { } // namespace -TEST_F(S3ClientFactoryTest, WrapsAllClientsInNonCloudMode) { - CloudModeConfigGuard guard(false); - auto& factory = S3ClientFactory::instance(); - - auto external_client = - factory.create(make_factory_conf("non-cloud-external-rate-limit.example.com", false)); - auto internal_client = - factory.create(make_factory_conf("non-cloud-internal-rate-limit.example.com", true)); - - ASSERT_NE(external_client, nullptr); - ASSERT_NE(internal_client, nullptr); - EXPECT_NE(std::dynamic_pointer_cast(external_client), nullptr); - EXPECT_NE(std::dynamic_pointer_cast(internal_client), nullptr); -} - -TEST_F(S3ClientFactoryTest, WrapsOnlyInternalClientsInCloudModeAndDistinguishesHashCollisions) { - CloudModeConfigGuard guard(true); +TEST_F(S3ClientFactoryTest, DistinguishesHashCollisions) { auto external_conf = make_hash_collision_conf("cloud-rate-limit-hash-collision.example.com", false); auto internal_conf = @@ -141,16 +183,20 @@ TEST_F(S3ClientFactoryTest, WrapsOnlyInternalClientsInCloudModeAndDistinguishesH ASSERT_NE(external_conf, internal_conf); auto& factory = S3ClientFactory::instance(); - auto external_client = factory.create(external_conf); - auto internal_client = factory.create(internal_conf); + auto external_result = factory.create(external_conf); + auto internal_result = factory.create(internal_conf); + ASSERT_TRUE(external_result.has_value()) << external_result.error(); + ASSERT_TRUE(internal_result.has_value()) << internal_result.error(); + auto external_client = std::move(external_result).value(); + auto internal_client = std::move(internal_result).value(); - ASSERT_NE(external_client, nullptr); - ASSERT_NE(internal_client, nullptr); - EXPECT_EQ(std::dynamic_pointer_cast(external_client), nullptr); - EXPECT_NE(std::dynamic_pointer_cast(internal_client), nullptr); EXPECT_NE(external_client, internal_client); - EXPECT_EQ(factory.create(external_conf), external_client); - EXPECT_EQ(factory.create(internal_conf), internal_client); + auto cached_external = factory.create(external_conf); + auto cached_internal = factory.create(internal_conf); + ASSERT_TRUE(cached_external.has_value()) << cached_external.error(); + ASSERT_TRUE(cached_internal.has_value()) << cached_internal.error(); + EXPECT_EQ(cached_external.value(), external_client); + EXPECT_EQ(cached_internal.value(), internal_client); } TEST_F(S3ClientFactoryTest, ObjClientHolderResetDistinguishesHashCollisions) { @@ -266,8 +312,8 @@ TEST_F(S3ClientFactoryTest, UnauthorizedObjectStoreResponseIsPermissionDenied) { error.SetExceptionName("InvalidAuthenticationCredentials"); error.SetMessage("missing or invalid bearer token"); - auto status = io::s3fs_error(error, "GCS request failed"); - EXPECT_EQ(status.code(), ErrorCode::PERMISSION_DENIED); + auto status = s3fs_error(error, "GCS request failed"); + EXPECT_EQ(status.code, ErrorCode::PERMISSION_DENIED); } TEST_F(S3ClientFactoryTest, GcpServiceAccountEmailValidation) { @@ -295,6 +341,91 @@ TEST_F(S3ClientFactoryTest, GcpServiceAccountEmailValidation) { } } +TEST_F(S3ClientFactoryTest, SelectsRateLimiterByDeploymentAndBucketType) { + RateLimiterConfigGuard rate_limiter_guard; + SyncPointProcessingGuard sync_point_guard; + auto* sync_point = SyncPoint::get_instance(); + SyncPoint::CallbackGuard create_client_callback; + sync_point->set_call_back( + "s3_client_factory::create", + [](auto&& args) { + auto result = try_any_cast_ret>(args); + result->second = true; + }, + &create_client_callback); + SyncPoint::CallbackGuard head_object_callback; + sync_point->set_call_back( + "s3_file_system::head_object", + [](auto&& args) { + auto result = try_any_cast_ret(args); + result->first = + Aws::S3::Model::HeadObjectOutcome(Aws::S3::Model::HeadObjectResult {}); + result->second = true; + }, + &head_object_callback); + + config::enable_s3_rate_limiter = true; + auto check_selection = [&](bool cloud_mode, bool internal_bucket, bool expect_limited, + std::string endpoint) { + CloudModeConfigGuard cloud_mode_guard(cloud_mode); + auto& manager = S3RateLimiterManager::instance(); + manager.qps_limiter(S3RateLimitType::GET)->reset(0, 0, 1); + manager.bytes_limiter(S3RateLimitType::GET)->reset(0, 0, 0); + + auto result = S3ClientFactory::instance().create( + make_factory_conf(std::move(endpoint), internal_bucket)); + ASSERT_TRUE(result.has_value()) << result.error(); + auto client = std::move(result).value(); + EXPECT_TRUE(client->head_object({.bucket = "bucket", .key = "key"}).resp.ok()); + auto second = client->head_object({.bucket = "bucket", .key = "key"}); + if (expect_limited) { + EXPECT_EQ(second.resp.status.code, static_cast(ErrorCode::EXCEEDED_LIMIT)); + EXPECT_EQ(second.resp.http_code, 0); + EXPECT_NE(second.resp.status.msg.find("s3 get request exceeds QPS limit"), + std::string::npos); + } else { + EXPECT_TRUE(second.resp.ok()); + } + }; + + check_selection(false, false, true, "non-cloud-external-rate-limit.example.com"); + check_selection(true, true, true, "cloud-internal-rate-limit.example.com"); + check_selection(true, false, false, "cloud-external-no-rate-limit.example.com"); +} + +TEST_F(S3ClientFactoryTest, RateLimitResponseDistinguishesBytesFromProviderThrottling) { + RateLimiterConfigGuard rate_limiter_guard; + SyncPointProcessingGuard sync_point_guard; + SyncPoint::CallbackGuard create_client_callback; + SyncPoint::get_instance()->set_call_back( + "s3_client_factory::create", + [](auto&& args) { + auto result = try_any_cast_ret>(args); + result->second = true; + }, + &create_client_callback); + + CloudModeConfigGuard cloud_mode_guard(false); + config::enable_s3_rate_limiter = true; + auto& manager = S3RateLimiterManager::instance(); + manager.qps_limiter(S3RateLimitType::GET)->reset(0, 0, 0); + manager.bytes_limiter(S3RateLimitType::GET) + ->reset(kNoThrottleBytesPerSecond, kNoThrottleBytesPerSecond, 1); + + auto result = S3ClientFactory::instance().create( + make_factory_conf("be-bytes-rate-limit-response.example.com", false)); + ASSERT_TRUE(result.has_value()) << result.error(); + auto client = std::move(result).value(); + char buffer[2]; + size_t size_return = 0; + auto response = client->get_object({.bucket = "bucket", .key = "key"}, buffer, 0, + sizeof(buffer), &size_return); + + EXPECT_EQ(response.status.code, static_cast(ErrorCode::EXCEEDED_LIMIT)); + EXPECT_EQ(response.http_code, 0); + EXPECT_NE(response.status.msg.find("s3 get request exceeds bytes limit"), std::string::npos); +} + TEST_F(S3ClientFactoryTest, AwsCredentialsProvider) { S3ClientFactory& factory = S3ClientFactory::instance(); S3ClientConf anonymous_conf; @@ -315,20 +446,20 @@ TEST_F(S3ClientFactoryTest, AwsCredentialsProvider) { config::aws_credentials_provider_version = "v2"; { - auto provider_v2 = factory.get_aws_credentials_provider(anonymous_conf); + auto provider_v2 = factory.create_aws_credentials_provider(anonymous_conf).provider; auto custom_chain_v2 = std::dynamic_pointer_cast(provider_v2); ASSERT_NE(custom_chain_v2, nullptr); } { - auto provider_v2 = factory.get_aws_credentials_provider(ak_sk_conf); + auto provider_v2 = factory.create_aws_credentials_provider(ak_sk_conf).provider; auto custom_chain_v2 = std::dynamic_pointer_cast(provider_v2); ASSERT_NE(custom_chain_v2, nullptr); } { - auto provider_v2 = factory.get_aws_credentials_provider(role_conf1); + auto provider_v2 = factory.create_aws_credentials_provider(role_conf1).provider; auto instance_profile_v2 = std::dynamic_pointer_cast( provider_v2); @@ -336,14 +467,14 @@ TEST_F(S3ClientFactoryTest, AwsCredentialsProvider) { } { - auto provider_v2 = factory.get_aws_credentials_provider(role_conf2); + auto provider_v2 = factory.create_aws_credentials_provider(role_conf2).provider; auto custom_chain_v2 = std::dynamic_pointer_cast(provider_v2); ASSERT_NE(custom_chain_v2, nullptr); } { - auto provider_v2 = factory.get_aws_credentials_provider(web_identity_conf); + auto provider_v2 = factory.create_aws_credentials_provider(web_identity_conf).provider; auto web_identity_v2 = std::dynamic_pointer_cast( provider_v2); @@ -352,21 +483,21 @@ TEST_F(S3ClientFactoryTest, AwsCredentialsProvider) { config::aws_credentials_provider_version = "v1"; { - auto provider_v1 = factory.get_aws_credentials_provider(anonymous_conf); + auto provider_v1 = factory.create_aws_credentials_provider(anonymous_conf).provider; auto default_chain_v1 = std::dynamic_pointer_cast(provider_v1); ASSERT_NE(default_chain_v1, nullptr); } { - auto provider_v1 = factory.get_aws_credentials_provider(ak_sk_conf); + auto provider_v1 = factory.create_aws_credentials_provider(ak_sk_conf).provider; auto default_chain_v1 = std::dynamic_pointer_cast(provider_v1); ASSERT_NE(default_chain_v1, nullptr); } { - auto provider_v1 = factory.get_aws_credentials_provider(role_conf1); + auto provider_v1 = factory.create_aws_credentials_provider(role_conf1).provider; auto default_chain_v1 = std::dynamic_pointer_cast( provider_v1); @@ -374,7 +505,7 @@ TEST_F(S3ClientFactoryTest, AwsCredentialsProvider) { } { - auto provider_v1 = factory.get_aws_credentials_provider(role_conf2); + auto provider_v1 = factory.create_aws_credentials_provider(role_conf2).provider; auto default_chain_v1 = std::dynamic_pointer_cast(provider_v1); ASSERT_NE(default_chain_v1, nullptr); @@ -383,6 +514,45 @@ TEST_F(S3ClientFactoryTest, AwsCredentialsProvider) { config::aws_credentials_provider_version = "v2"; } +TEST_F(S3ClientFactoryTest, RefreshCaCertForCredentialsProvider) { + auto& factory = S3ClientFactory::instance(); + auto old_ca_cert_file_paths = config::ca_cert_file_paths; + std::string old_ca_cert_file_path; + { + std::lock_guard lock(factory._ca_cert_lock); + old_ca_cert_file_path = std::exchange(factory._ca_cert_file_path, ""); + } + + auto ca_cert_file_path = std::filesystem::temp_directory_path() / + ("doris_s3_client_factory_ca_" + std::to_string(getpid()) + ".pem"); + std::filesystem::remove(ca_cert_file_path); + config::ca_cert_file_paths = ca_cert_file_path.string(); + + S3ClientConf role_conf; + role_conf.cred_provider_type = CredProviderType::InstanceProfile; + role_conf.role_arn = "role_arn"; + auto provider_without_ca = factory.create_aws_credentials_provider(role_conf).provider; + + { + std::ofstream ca_cert_file(ca_cert_file_path); + ca_cert_file << "test CA bundle"; + } + auto provider_with_ca = factory.create_aws_credentials_provider(role_conf).provider; + + std::string refreshed_ca_cert_file_path; + { + std::lock_guard lock(factory._ca_cert_lock); + refreshed_ca_cert_file_path = factory._ca_cert_file_path; + factory._ca_cert_file_path = std::move(old_ca_cert_file_path); + } + config::ca_cert_file_paths = std::move(old_ca_cert_file_paths); + std::filesystem::remove(ca_cert_file_path); + + EXPECT_NE(provider_without_ca, nullptr); + EXPECT_NE(provider_with_ca, nullptr); + EXPECT_EQ(refreshed_ca_cert_file_path, ca_cert_file_path.string()); +} + TEST_F(S3ClientFactoryTest, SetS3ClientDefaultHttpScheme) { S3ClientFactory::instance(); Aws::Client::ClientConfiguration client_config; @@ -506,7 +676,7 @@ TEST_F(S3ClientFactoryTest, ConvertPropertiesToS3ConfGcpCredentialModes) { ASSERT_TRUE( S3ClientFactory::convert_properties_to_s3_conf(properties, s3_uri, &s3_conf).ok()) << "provider_type=" << test_case.provider_type; - ASSERT_EQ(s3_conf.client_conf.provider, io::ObjStorageType::GCP); + ASSERT_EQ(s3_conf.client_conf.provider, io::ObjStorageProvider::GCP); ASSERT_TRUE(std::holds_alternative(s3_conf.client_conf.credential)); const auto& credential = std::get(s3_conf.client_conf.credential); EXPECT_EQ(credential.provider_type, test_case.expected_provider_type); @@ -679,25 +849,25 @@ TEST_F(S3ClientFactoryTest, AwsCredentialsProviderV2ProviderTypeWithoutRoleArn) S3ClientConf default_conf; default_conf.cred_provider_type = CredProviderType::Default; - auto provider = factory.get_aws_credentials_provider(default_conf); + auto provider = factory.create_aws_credentials_provider(default_conf).provider; ASSERT_NE(std::dynamic_pointer_cast(provider), nullptr); S3ClientConf env_conf; env_conf.cred_provider_type = CredProviderType::Env; - provider = factory.get_aws_credentials_provider(env_conf); + provider = factory.create_aws_credentials_provider(env_conf).provider; ASSERT_NE(std::dynamic_pointer_cast(provider), nullptr); S3ClientConf sys_conf; sys_conf.cred_provider_type = CredProviderType::SystemProperties; - provider = factory.get_aws_credentials_provider(sys_conf); + provider = factory.create_aws_credentials_provider(sys_conf).provider; ASSERT_NE( std::dynamic_pointer_cast(provider), nullptr); S3ClientConf web_identity_conf; web_identity_conf.cred_provider_type = CredProviderType::WebIdentity; - provider = factory.get_aws_credentials_provider(web_identity_conf); + provider = factory.create_aws_credentials_provider(web_identity_conf).provider; ASSERT_NE(std::dynamic_pointer_cast( provider), nullptr); @@ -709,19 +879,19 @@ TEST_F(S3ClientFactoryTest, AwsCredentialsProviderV2ProviderTypeWithoutRoleArn) env.set_ecs_task_role("/v2/credentials/mock"); S3ClientConf container_conf; container_conf.cred_provider_type = CredProviderType::Container; - provider = factory.get_aws_credentials_provider(container_conf); + provider = factory.create_aws_credentials_provider(container_conf).provider; ASSERT_NE(as_valid_http_provider(provider), nullptr); } S3ClientConf instance_profile_conf; instance_profile_conf.cred_provider_type = CredProviderType::InstanceProfile; - provider = factory.get_aws_credentials_provider(instance_profile_conf); + provider = factory.create_aws_credentials_provider(instance_profile_conf).provider; ASSERT_NE(std::dynamic_pointer_cast(provider), nullptr); S3ClientConf anonymous_conf; anonymous_conf.cred_provider_type = CredProviderType::Anonymous; - provider = factory.get_aws_credentials_provider(anonymous_conf); + provider = factory.create_aws_credentials_provider(anonymous_conf).provider; ASSERT_NE(std::dynamic_pointer_cast(provider), nullptr); } @@ -734,7 +904,7 @@ TEST_F(S3ClientFactoryTest, AwsCredentialsProviderV2WithRoleArnAlwaysAssumeRole) CredProviderType::Default, CredProviderType::Env, CredProviderType::SystemProperties, CredProviderType::WebIdentity, CredProviderType::Container, CredProviderType::InstanceProfile, - CredProviderType::Anonymous, + CredProviderType::Anonymous, CredProviderType::Simple, }; for (auto provider_type : provider_types) { @@ -742,12 +912,25 @@ TEST_F(S3ClientFactoryTest, AwsCredentialsProviderV2WithRoleArnAlwaysAssumeRole) conf.cred_provider_type = provider_type; conf.role_arn = "arn:aws:iam::123456789012:role/test-role"; conf.external_id = "external-id"; - auto provider = factory.get_aws_credentials_provider(conf); + auto provider = factory.create_aws_credentials_provider(conf).provider; ASSERT_NE(std::dynamic_pointer_cast(provider), nullptr); } } +TEST_F(S3ClientFactoryTest, AwsCredentialsProviderV2SimpleWithoutAkSkUsesDefaultChain) { + S3ClientFactory& factory = S3ClientFactory::instance(); + config::aws_credentials_provider_version = "v2"; + + S3ClientConf conf; + conf.cred_provider_type = CredProviderType::Simple; + auto result = factory.create_aws_credentials_provider(conf); + + ASSERT_TRUE(result); + EXPECT_NE(std::dynamic_pointer_cast(result.provider), + nullptr); +} + TEST_F(S3ClientFactoryTest, AwsCredentialsProviderAkSkTakePrecedenceOverRoleArn) { S3ClientFactory& factory = S3ClientFactory::instance(); S3ClientConf conf; @@ -758,12 +941,12 @@ TEST_F(S3ClientFactoryTest, AwsCredentialsProviderAkSkTakePrecedenceOverRoleArn) conf.cred_provider_type = CredProviderType::InstanceProfile; config::aws_credentials_provider_version = "v2"; - auto provider_v2 = factory.get_aws_credentials_provider(conf); + auto provider_v2 = factory.create_aws_credentials_provider(conf).provider; ASSERT_NE(std::dynamic_pointer_cast(provider_v2), nullptr); config::aws_credentials_provider_version = "v1"; - auto provider_v1 = factory.get_aws_credentials_provider(conf); + auto provider_v1 = factory.create_aws_credentials_provider(conf).provider; ASSERT_NE(std::dynamic_pointer_cast(provider_v1), nullptr); @@ -777,7 +960,7 @@ TEST_F(S3ClientFactoryTest, AwsCredentialsProviderV1RoleArnDefaultFallback) { S3ClientConf conf; conf.cred_provider_type = CredProviderType::Default; conf.role_arn = "arn:aws:iam::123456789012:role/test-role"; - auto provider = factory.get_aws_credentials_provider(conf); + auto provider = factory.create_aws_credentials_provider(conf).provider; ASSERT_NE(std::dynamic_pointer_cast(provider), nullptr); @@ -862,7 +1045,7 @@ TEST_F(S3ClientFactoryTest, ContainerProviderTypeReadsRotatedTokenFileForPodIden S3ClientConf conf; conf.cred_provider_type = CredProviderType::Container; - auto provider = as_valid_http_provider(factory.get_aws_credentials_provider(conf)); + auto provider = as_valid_http_provider(factory.create_aws_credentials_provider(conf).provider); ASSERT_NE(provider, nullptr) << "CONTAINER did not yield a usable container credentials " "provider for AWS_CONTAINER_CREDENTIALS_FULL_URI"; @@ -891,7 +1074,7 @@ TEST_F(S3ClientFactoryTest, ContainerProviderTypeForwardsInlineAuthorizationToke S3ClientConf conf; conf.cred_provider_type = CredProviderType::Container; - auto provider = as_valid_http_provider(factory.get_aws_credentials_provider(conf)); + auto provider = as_valid_http_provider(factory.create_aws_credentials_provider(conf).provider); ASSERT_NE(provider, nullptr) << "CONTAINER did not yield a usable container credentials " "provider for an inline authorization token"; @@ -911,7 +1094,8 @@ TEST_F(S3ClientFactoryTest, ContainerProviderTypeStillHonoursEcsRelativeUri) { S3ClientConf conf; conf.cred_provider_type = CredProviderType::Container; - EXPECT_NE(as_valid_http_provider(factory.get_aws_credentials_provider(conf)), nullptr) + EXPECT_NE(as_valid_http_provider(factory.create_aws_credentials_provider(conf).provider), + nullptr) << "CONTAINER did not yield a usable container credentials provider for " "AWS_CONTAINER_CREDENTIALS_RELATIVE_URI"; } @@ -929,10 +1113,30 @@ TEST_F(S3ClientFactoryTest, ContainerProviderTypeIsUnusableWithoutAnyUri) { S3ClientConf conf; conf.cred_provider_type = CredProviderType::Container; - auto provider = factory.get_aws_credentials_provider(conf); + auto provider = factory.create_aws_credentials_provider(conf).provider; ASSERT_NE(std::dynamic_pointer_cast(provider), nullptr); EXPECT_EQ(as_valid_http_provider(provider), nullptr); } +TEST_F(S3ClientFactoryTest, AwsCredentialsProviderV1PartialCredentialsUseDefaultChain) { + S3ClientFactory& factory = S3ClientFactory::instance(); + config::aws_credentials_provider_version = "v1"; + + for (bool provide_access_key : {false, true}) { + S3ClientConf conf; + conf.cred_provider_type = CredProviderType::Default; + conf.role_arn = "arn:aws:iam::123456789012:role/test-role"; + conf.ak = provide_access_key ? "ak" : ""; + conf.sk = provide_access_key ? "" : "sk"; + + auto provider = factory.create_aws_credentials_provider(conf).provider; + EXPECT_NE( + std::dynamic_pointer_cast(provider), + nullptr); + } + + config::aws_credentials_provider_version = "v2"; +} + } // namespace doris diff --git a/be/test/storage/cloud_file_cache_write_index_only_test.cpp b/be/test/storage/cloud_file_cache_write_index_only_test.cpp index d4a62a01b99c96..1ee187c1a51f84 100644 --- a/be/test/storage/cloud_file_cache_write_index_only_test.cpp +++ b/be/test/storage/cloud_file_cache_write_index_only_test.cpp @@ -29,13 +29,13 @@ #include "cloud/config.h" #include "common/config.h" #include "core/block/block.h" +#include "cpp/obj-client/s3_obj_storage_client.h" #include "cpp/sync_point.h" #include "io/cache/block_file_cache_factory.h" #include "io/fs/file_writer.h" #include "io/fs/local_file_system.h" #include "io/fs/s3_file_system.h" #include "io/fs/s3_file_writer.h" -#include "io/fs/s3_obj_storage_client.h" #include "io/io_common.h" #include "runtime/exec_env.h" #include "storage/index/inverted/clucene/inverted_index_writer.h" diff --git a/be/test/storage/rowset/beta_rowset_test.cpp b/be/test/storage/rowset/beta_rowset_test.cpp index ab9fe46d07e9b8..6402c5eccc69e9 100644 --- a/be/test/storage/rowset/beta_rowset_test.cpp +++ b/be/test/storage/rowset/beta_rowset_test.cpp @@ -40,12 +40,12 @@ #include "common/config.h" #include "common/status.h" +#include "cpp/obj-client/s3_obj_storage_client.h" #include "cpp/sync_point.h" #include "gtest/gtest_pred_impl.h" #include "io/fs/file_system.h" #include "io/fs/local_file_system.h" #include "io/fs/s3_file_system.h" -#include "io/fs/s3_obj_storage_client.h" #include "json2pb/json_to_pb.h" #include "runtime/exec_env.h" #include "storage/data_dir.h" @@ -319,7 +319,7 @@ TEST_F(BetaRowsetTest, ReadTest) { aws_cred, aws_config, Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, true); - client.reset(new io::S3ObjStorageClient(std::move(s3_client))); + client = std::make_shared(std::move(s3_client)); rowset.rowset_meta()->set_num_segments(1); rowset.rowset_meta()->set_remote_storage_resource(storage_resource); @@ -333,8 +333,8 @@ TEST_F(BetaRowsetTest, ReadTest) { { Aws::Auth::AWSCredentials aws_cred("ak", "sk"); Aws::Client::ClientConfiguration aws_config; - client.reset(new io::S3ObjStorageClient( - std::make_shared(S3ClientMockGetError()))); + client = std::make_shared( + std::make_shared(S3ClientMockGetError())); rowset.rowset_meta()->set_num_segments(1); rowset.rowset_meta()->set_remote_storage_resource(storage_resource); @@ -348,8 +348,8 @@ TEST_F(BetaRowsetTest, ReadTest) { { Aws::Auth::AWSCredentials aws_cred("ak", "sk"); Aws::Client::ClientConfiguration aws_config; - client.reset(new io::S3ObjStorageClient( - std::make_shared(S3ClientMockGetErrorData()))); + client = std::make_shared( + std::make_shared(S3ClientMockGetErrorData())); rowset.rowset_meta()->set_num_segments(1); rowset.rowset_meta()->set_remote_storage_resource(storage_resource); diff --git a/be/test/testutil/mock/obj_storage_client_test_stub.h b/be/test/testutil/mock/obj_storage_client_test_stub.h new file mode 100644 index 00000000000000..6cec949121aa16 --- /dev/null +++ b/be/test/testutil/mock/obj_storage_client_test_stub.h @@ -0,0 +1,50 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include "cpp/obj-client/obj_storage_client.h" + +namespace doris::io { + +// Supplies explicit unsupported implementations for tests that only exercise a subset of the +// object storage interface. Production providers must implement every method themselves. +class ObjStorageClientTestStub : public ObjStorageClient { +public: + ObjStorageCapabilities capabilities() const override { return {.max_delete_batch = 1}; } + + ObjStorageResponse get_lifecycle(const std::string&, int64_t*) override { + return not_supported(); + } + + ObjStorageResponse check_versioning(const std::string&) override { return not_supported(); } + + ObjStorageResponse abort_multipart_upload(const ObjStoragePath&, const std::string&) override { + return not_supported(); + } + +private: + static ObjStorageResponse not_supported() { + return { + .status = {TStatusCode::NOT_IMPLEMENTED_ERROR, + "operation is not supported by the object storage test stub"}, + .http_code = 0, + }; + } +}; + +} // namespace doris::io diff --git a/cloud/src/recycler/CMakeLists.txt b/cloud/src/recycler/CMakeLists.txt index 3721e23faabdf7..d8206878fcbc8a 100644 --- a/cloud/src/recycler/CMakeLists.txt +++ b/cloud/src/recycler/CMakeLists.txt @@ -29,10 +29,6 @@ if (NOT ENABLE_HDFS_STORAGE_VAULT) list(REMOVE_ITEM SRC_LIST ${CMAKE_CURRENT_SOURCE_DIR}/hdfs_accessor.cpp) endif() -if(BUILD_AZURE STREQUAL "OFF") - list(REMOVE_ITEM SRC_LIST "${CMAKE_CURRENT_SOURCE_DIR}/azure_obj_client.cpp") -endif() - if(BUILD_CHECK_META STREQUAL "OFF") list(REMOVE_ITEM SRC_LIST ${CMAKE_CURRENT_SOURCE_DIR}/meta_checker.cpp) endif () diff --git a/cloud/src/recycler/azure_obj_client.cpp b/cloud/src/recycler/azure_obj_client.cpp deleted file mode 100644 index e7e6abd66edfd2..00000000000000 --- a/cloud/src/recycler/azure_obj_client.cpp +++ /dev/null @@ -1,335 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "recycler/azure_obj_client.h" - -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "common/config.h" -#include "common/logging.h" -#include "common/stopwatch.h" -#include "cpp/obj_retry_strategy.h" -#include "cpp/sync_point.h" -#include "cpp/token_bucket_rate_limiter.h" -#include "recycler/s3_accessor.h" -#include "recycler/util.h" - -using namespace Azure::Storage::Blobs; - -namespace doris::cloud { - -template -auto s3_rate_limit(S3RateLimitType op, Func callback) -> decltype(callback()) { - using T = decltype(callback()); - if (!config::enable_s3_rate_limiter) { - return callback(); - } - auto sleep_duration = - doris::apply_s3_rate_limit(op, AccessorRateLimiter::instance().rate_limiter(op), - config::s3_rate_limiter_log_interval); - if (sleep_duration < 0) { - throw std::runtime_error("Azure exceeds request limit"); - } - return callback(); -} - -template -auto s3_get_rate_limit(Func callback) -> decltype(callback()) { - return s3_rate_limit(S3RateLimitType::GET, std::move(callback)); -} - -template -auto s3_put_rate_limit(Func callback) -> decltype(callback()) { - return s3_rate_limit(S3RateLimitType::PUT, std::move(callback)); -} - -static constexpr size_t BlobBatchMaxOperations = 256; -static constexpr char BlobNotFound[] = "BlobNotFound"; - -template -ObjectStorageResponse do_azure_client_call(Func f, std::string_view url, std::string_view key) { - try { - f(); - } catch (Azure::Core::RequestFailedException& e) { - doris::record_object_request_failed(static_cast(e.StatusCode)); - auto msg = fmt::format( - "Azure request failed because {}, http_code: {}, request_id: {}, url: {}, " - "key: {}", - e.Message, static_cast(e.StatusCode), e.RequestId, url, key); - LOG_WARNING(msg); - return {-1, std::move(msg)}; - } catch (std::exception& e) { - auto msg = fmt::format("Azure request failed because {}, url: {}, key: {}", e.what(), url, - key); - LOG_WARNING(msg); - return {-1, std::move(msg)}; - } - return {}; -} - -static const Azure::DateTime SystemClockEpoch {1970, 1, 1}; - -class AzureListIterator final : public ObjectListIterator { -public: - AzureListIterator(std::shared_ptr client, std::string prefix) - : client_(std::move(client)), req_({.Prefix = std::move(prefix)}) { - TEST_SYNC_POINT_CALLBACK("AzureListIterator", &req_); - } - - ~AzureListIterator() override = default; - - bool is_valid() override { return is_valid_; } - - bool has_next() override { - if (!is_valid_) { - return false; - } - - if (!results_.empty()) { - return true; - } - - if (!has_more_) { - return false; - } - - ListBlobsPagedResponse resp; - auto obj_resp = do_azure_client_call( - [&]() { - resp = s3_get_rate_limit([&]() { - SCOPED_BVAR_LATENCY(s3_bvar::s3_list_latency); - return client_->ListBlobs(req_); - }); - }, - client_->GetUrl(), req_.Prefix.Value()); - if (obj_resp.ret != 0) { - is_valid_ = false; - return false; - } - - has_more_ = resp.NextPageToken.HasValue(); - DCHECK(!(has_more_ && resp.Blobs.empty())) << has_more_ << ' ' << resp.Blobs.empty(); - req_.ContinuationToken = std::move(resp.NextPageToken); - results_.reserve(resp.Blobs.size()); - for (auto&& item : std::ranges::reverse_view(resp.Blobs)) { - DCHECK(item.Name.starts_with(*req_.Prefix)) << item.Name << ' ' << *req_.Prefix; - results_.emplace_back(ObjectMeta { - .key = std::move(item.Name), - .size = item.BlobSize, - // `Azure::DateTime` adds the offset of `SystemClockEpoch` to the given Unix timestamp, - // so here we need to subtract this offset to obtain the Unix timestamp of the mtime. - // https://github.com/Azure/azure-sdk-for-cpp/blob/azure-core_1.12.0/sdk/core/azure-core/inc/azure/core/datetime.hpp#L129 - .mtime_s = duration_cast(item.Details.LastModified - - SystemClockEpoch) - .count()}); - } - - return !results_.empty(); - } - - std::optional next() override { - std::optional res; - if (!has_next()) { - return res; - } - - res = std::move(results_.back()); - results_.pop_back(); - return res; - } - -private: - std::shared_ptr client_; - ListBlobsOptions req_; - std::vector results_; - bool is_valid_ {true}; - bool has_more_ {true}; -}; - -AzureObjClient::~AzureObjClient() = default; - -ObjectStorageResponse AzureObjClient::put_object(ObjectStoragePathRef path, - std::string_view stream) { - auto client = client_->GetBlockBlobClient(path.key); - return do_azure_client_call( - [&]() { - s3_put_rate_limit([&]() { - SCOPED_BVAR_LATENCY(s3_bvar::s3_put_latency); - return client.UploadFrom(reinterpret_cast(stream.data()), - stream.size()); - }); - }, - client_->GetUrl(), path.key); -} - -ObjectStorageResponse AzureObjClient::head_object(ObjectStoragePathRef path, ObjectMeta* res) { - Models::BlobProperties properties {}; - bool not_found = false; - - auto resp = do_azure_client_call( - [&]() { - try { - properties = s3_get_rate_limit([&]() { - SCOPED_BVAR_LATENCY(s3_bvar::s3_head_latency); - return client_->GetBlockBlobClient(path.key).GetProperties().Value; - }); - } catch (Azure::Storage::StorageException& e) { - if (e.StatusCode == Azure::Core::Http::HttpStatusCode::NotFound) { - not_found = true; - return; - } - throw; - } - }, - client_->GetUrl(), path.key); - - if (not_found) { - return {1}; - } - - if (resp.ret != 0) { - return resp; - } - - res->key = path.key; - res->mtime_s = properties.LastModified.time_since_epoch().count(); - res->size = properties.BlobSize; - return {0}; -} - -std::unique_ptr AzureObjClient::list_objects(ObjectStoragePathRef path) { - return std::make_unique(client_, path.key); -} - -// As Azure's doc said, the batch size is 256 -// You can find out the num in https://learn.microsoft.com/en-us/rest/api/storageservices/blob-batch?tabs=microsoft-entra-id -// > Each batch request supports a maximum of 256 subrequests. -ObjectStorageResponse AzureObjClient::delete_objects(const std::string& bucket, - std::vector keys, - ObjClientOptions option) { - if (keys.empty()) { - return {0}; - } - - // TODO(ByteYue) : use range to adate this code when compiler is ready - // auto chunkedView = objs | std::views::chunk(BlobBatchMaxOperations); - auto begin = std::begin(keys); - auto end = std::end(keys); - - while (begin != end) { - auto batch = client_->CreateBatch(); - auto chunk_end = begin; - size_t batch_size = BlobBatchMaxOperations; - TEST_SYNC_POINT_CALLBACK("AzureObjClient::delete_objects", &batch_size); - std::advance(chunk_end, - std::min(batch_size, static_cast(std::distance(begin, end)))); - std::vector> deferred_resps; - deferred_resps.reserve(std::distance(begin, chunk_end)); - for (auto it = begin; it != chunk_end; ++it) { - deferred_resps.emplace_back(batch.DeleteBlob(*it)); - } - auto resp = do_azure_client_call( - [&]() { - s3_put_rate_limit([&]() { - SCOPED_BVAR_LATENCY(s3_bvar::s3_delete_objects_latency); - return client_->SubmitBatch(batch); - }); - }, - client_->GetUrl(), *begin); - if (resp.ret != 0) { - return resp; - } - for (auto&& defer : deferred_resps) { - try { - auto r = defer.GetResponse(); - if (!r.Value.Deleted) { - LOG_INFO("Azure batch delete failed, url {}", client_->GetUrl()); - return {-1}; - } - } catch (Azure::Storage::StorageException& e) { - if (Azure::Core::Http::HttpStatusCode::NotFound == e.StatusCode && - 0 == strcmp(e.ErrorCode.c_str(), BlobNotFound)) { - continue; - } - doris::record_object_request_failed(static_cast(e.StatusCode)); - auto msg = fmt::format( - "Azure request failed because {}, http code {}, request id {}, url {}", - e.Message, static_cast(e.StatusCode), e.RequestId, client_->GetUrl()); - LOG_WARNING(msg); - return {-1, std::move(msg)}; - } - } - - begin = chunk_end; - } - - return {0}; -} - -ObjectStorageResponse AzureObjClient::delete_object(ObjectStoragePathRef path) { - return do_azure_client_call( - [&]() { - if (auto r = s3_put_rate_limit([&]() { - SCOPED_BVAR_LATENCY(s3_bvar::s3_delete_object_latency); - return client_->DeleteBlob(path.key); - }); - !r.Value.Deleted) { - throw std::runtime_error("Delete azure blob failed"); - } - }, - client_->GetUrl(), path.key); -} - -ObjectStorageResponse AzureObjClient::delete_objects_recursively(ObjectStoragePathRef path, - ObjClientOptions option, - int64_t expiration_time) { - return delete_objects_recursively_(path, option, expiration_time, BlobBatchMaxOperations); -} - -ObjectStorageResponse AzureObjClient::get_life_cycle(const std::string& bucket, - int64_t* expiration_days) { - // TODO(plat1ko) - *expiration_days = INT64_MAX; - return {0}; -} - -ObjectStorageResponse AzureObjClient::check_versioning(const std::string& bucket) { - // TODO(plat1ko) - return {0}; -} - -ObjectStorageResponse AzureObjClient::abort_multipart_upload(ObjectStoragePathRef path, - const std::string& upload_id) { - // delete uncommitted blobs - // https://learn.microsoft.com/en-us/rest/api/storageservices/delete-blob?tabs=microsoft-entra-id#remarks - return delete_object(path); -} - -} // namespace doris::cloud diff --git a/cloud/src/recycler/azure_obj_client.h b/cloud/src/recycler/azure_obj_client.h deleted file mode 100644 index bdf9b4eda752cc..00000000000000 --- a/cloud/src/recycler/azure_obj_client.h +++ /dev/null @@ -1,62 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include - -#include "recycler/obj_storage_client.h" - -namespace Azure::Storage::Blobs { -class BlobContainerClient; -} // namespace Azure::Storage::Blobs - -namespace doris::cloud { -class AzureObjClient final : public ObjStorageClient { -public: - AzureObjClient(std::shared_ptr client) - : client_(std::move(client)) {} - ~AzureObjClient() override; - - ObjectStorageResponse put_object(ObjectStoragePathRef path, std::string_view stream) override; - - ObjectStorageResponse head_object(ObjectStoragePathRef path, ObjectMeta* res) override; - - std::unique_ptr list_objects(ObjectStoragePathRef path) override; - - ObjectStorageResponse delete_objects(const std::string& bucket, std::vector keys, - ObjClientOptions option) override; - - ObjectStorageResponse delete_object(ObjectStoragePathRef path) override; - - ObjectStorageResponse delete_objects_recursively(ObjectStoragePathRef path, - ObjClientOptions option, - int64_t expiration_time = 0) override; - - ObjectStorageResponse get_life_cycle(const std::string& bucket, - int64_t* expiration_days) override; - - ObjectStorageResponse check_versioning(const std::string& bucket) override; - - ObjectStorageResponse abort_multipart_upload(ObjectStoragePathRef path, - const std::string& upload_id) override; - -private: - std::shared_ptr client_; -}; - -} // namespace doris::cloud diff --git a/cloud/src/recycler/checker.cpp b/cloud/src/recycler/checker.cpp index 182b6461fe5931..0e7cff8ea6648d 100644 --- a/cloud/src/recycler/checker.cpp +++ b/cloud/src/recycler/checker.cpp @@ -797,7 +797,7 @@ int InstanceChecker::do_check() { int InstanceChecker::get_bucket_lifecycle(int64_t* lifecycle_days) { // If there are multiple buckets, return the minimum lifecycle. int64_t min_lifecycle_days = INT64_MAX; - int64_t tmp_liefcycle_days = 0; + int64_t tmp_lifecycle_days = 0; for (const auto& [id, accessor] : accessor_map_) { if (accessor->type() != AccessorType::S3) { continue; @@ -809,12 +809,12 @@ int InstanceChecker::get_bucket_lifecycle(int64_t* lifecycle_days) { return -1; } - if (s3_accessor->get_life_cycle(&tmp_liefcycle_days) != 0) { + if (s3_accessor->get_lifecycle(&tmp_lifecycle_days) != 0) { return -1; } - if (tmp_liefcycle_days < min_lifecycle_days) { - min_lifecycle_days = tmp_liefcycle_days; + if (tmp_lifecycle_days < min_lifecycle_days) { + min_lifecycle_days = tmp_lifecycle_days; } } *lifecycle_days = min_lifecycle_days; diff --git a/cloud/src/recycler/obj_storage_client.cpp b/cloud/src/recycler/obj_storage_client.cpp deleted file mode 100644 index f1fc52f2c281be..00000000000000 --- a/cloud/src/recycler/obj_storage_client.cpp +++ /dev/null @@ -1,154 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "recycler/obj_storage_client.h" - -#include - -#include "common/config.h" -#include "cpp/sync_point.h" -#include "recycler/sync_executor.h" - -using namespace std::chrono; - -namespace doris::cloud { - -ObjectStorageResponse ObjStorageClient::delete_objects_recursively_(ObjectStoragePathRef path, - const ObjClientOptions& option, - int64_t expired_time, - size_t batch_size) { - TEST_SYNC_POINT_CALLBACK("ObjStorageClient::delete_objects_recursively_", &batch_size); - auto list_iter = list_objects(path); - ObjectStorageResponse ret; - size_t num_deleted = 0; - int error_count = 0; - size_t batch_count = 0; - auto start_time = steady_clock::now(); - - // Read max tasks per batch from config, validate to prevent overflow - int32_t config_val = config::recycler_max_tasks_per_batch; - size_t max_tasks_per_batch = 1000; // default value - if (config_val > 0) { - max_tasks_per_batch = static_cast(config_val); - } else { - LOG(WARNING) << "recycler_max_tasks_per_batch=" << config_val - << " is not positive, using default 1000"; - } - - while (true) { - // Create a new SyncExecutor for each batch - // Note: cancel lambda only takes effect within the current batch - SyncExecutor batch_executor( - option.executor, fmt::format("delete batch under {}/{}", path.bucket, path.key), - [](const int& r) { return r != 0; }); - - std::vector keys; - size_t tasks_in_batch = 0; - bool has_more = true; - - // Collect tasks until reaching batch limit or no more files - while (tasks_in_batch < max_tasks_per_batch && has_more) { - auto obj = list_iter->next(); - if (!obj.has_value()) { - has_more = false; - break; - } - if (expired_time > 0 && obj->mtime_s > expired_time) { - continue; - } - - num_deleted++; - keys.emplace_back(std::move(obj->key)); - - // Submit a delete task when we have batch_size keys - if (keys.size() >= batch_size) { - batch_executor.add([this, &path, k = std::move(keys), option]() mutable { - return delete_objects(path.bucket, std::move(k), option).ret; - }); - keys.clear(); - tasks_in_batch++; - } - } - - // Handle remaining keys (less than batch_size) - if (!keys.empty()) { - batch_executor.add([this, &path, k = std::move(keys), option]() mutable { - return delete_objects(path.bucket, std::move(k), option).ret; - }); - tasks_in_batch++; - } - - // Before exiting on empty batch, check if listing is valid - // Avoid silently treating listing failure as success - if (tasks_in_batch == 0) { - if (!list_iter->is_valid()) { - LOG(WARNING) << "list_iter invalid with no tasks collected"; - ret = {-1}; - } - break; - } - - // Wait for current batch to complete - bool finished = true; - std::vector rets = batch_executor.when_all(&finished); - batch_count++; - - for (int r : rets) { - if (r != 0) { - error_count++; - } - } - - // Log batch progress for monitoring long-running delete tasks - auto batch_elapsed = duration_cast(steady_clock::now() - start_time).count(); - LOG(INFO) << "delete objects under " << path.bucket << "/" << path.key << " batch " - << batch_count << " completed" - << ", tasks_in_batch=" << tasks_in_batch << ", total_deleted=" << num_deleted - << ", elapsed=" << batch_elapsed << " ms"; - - // Check finished status: false means stop_token triggered, task timeout, or task invalid - if (!finished) { - LOG(WARNING) << "batch execution did not finish normally, stopping"; - ret = {-1}; - break; - } - - // Check if list_iter is still valid (network errors, etc.) - if (!list_iter->is_valid()) { - LOG(WARNING) << "list_iter became invalid during iteration"; - ret = {-1}; - break; - } - - // batch_executor goes out of scope, resources are automatically released - } - - if (error_count > 0) { - LOG(WARNING) << "delete_objects_recursively completed with " << error_count << " errors"; - ret = {-1}; - } - - auto elapsed = duration_cast(steady_clock::now() - start_time).count(); - LOG(INFO) << "delete objects under " << path.bucket << "/" << path.key - << " finished, ret=" << ret.ret << ", total_batches=" << batch_count - << ", num_deleted=" << num_deleted << ", error_count=" << error_count - << ", cost=" << elapsed << " ms"; - - return ret; -} - -} // namespace doris::cloud diff --git a/cloud/src/recycler/obj_storage_client.h b/cloud/src/recycler/obj_storage_client.h deleted file mode 100644 index 358dfba4943fd7..00000000000000 --- a/cloud/src/recycler/obj_storage_client.h +++ /dev/null @@ -1,115 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include -#include -#include -#include - -namespace doris::cloud { - -struct ObjectStoragePathRef { - const std::string& bucket; - const std::string& key; -}; - -struct ObjectStorageResponse { - enum Code : int { - UNDEFINED = -1, - OK = 0, - NOT_FOUND = 1, - }; - - ObjectStorageResponse(int r = OK, std::string msg = "") : ret(r), error_msg(std::move(msg)) {} - // clang-format off - int ret {OK}; // To unify the error handle logic with BE, we'd better use the same error code as BE - // clang-format on - std::string error_msg; -}; - -struct ObjectMeta { - std::string key; - int64_t size {0}; - int64_t mtime_s {0}; -}; - -class ObjectListIterator { -public: - virtual ~ObjectListIterator() = default; - virtual bool is_valid() = 0; - virtual bool has_next() = 0; - virtual std::optional next() = 0; -}; - -class SimpleThreadPool; -struct ObjClientOptions { - bool prefetch {true}; - std::shared_ptr executor; -}; - -class ObjStorageClient { -public: - ObjStorageClient() = default; - virtual ~ObjStorageClient() = default; - - ObjStorageClient(const ObjStorageClient&) = delete; - ObjStorageClient& operator=(const ObjStorageClient&) = delete; - - virtual ObjectStorageResponse put_object(ObjectStoragePathRef path, - std::string_view stream) = 0; - - // If it exists, it will return the corresponding object meta - virtual ObjectStorageResponse head_object(ObjectStoragePathRef path, ObjectMeta* res) = 0; - - // According to the passed bucket and prefix, it traverses and retrieves all files under the prefix, and returns the name and file size of all files. - // **Attention**: The ObjectMeta contains the full key in object storage - virtual std::unique_ptr list_objects(ObjectStoragePathRef path) = 0; - - // According to the bucket and prefix specified by the user, it performs batch deletion based on the object names in the object array. - virtual ObjectStorageResponse delete_objects(const std::string& bucket, - std::vector keys, - ObjClientOptions option) = 0; - - // Delete the file named key in the object storage bucket. - virtual ObjectStorageResponse delete_object(ObjectStoragePathRef path) = 0; - - // According to the prefix, recursively delete all objects under the prefix. - // If `expiration_time` > 0, only delete objects with mtime earlier than `expiration_time`. - virtual ObjectStorageResponse delete_objects_recursively(ObjectStoragePathRef path, - ObjClientOptions option, - int64_t expiration_time = 0) = 0; - - // Get the objects' expiration time on the bucket - virtual ObjectStorageResponse get_life_cycle(const std::string& bucket, - int64_t* expiration_days) = 0; - - // Check if the objects' versioning is on or off - // returns 0 when versioning is on, otherwise versioning is off or check failed - virtual ObjectStorageResponse check_versioning(const std::string& bucket) = 0; - - virtual ObjectStorageResponse abort_multipart_upload(ObjectStoragePathRef path, - const std::string& upload_id) = 0; - -protected: - ObjectStorageResponse delete_objects_recursively_(ObjectStoragePathRef path, - const ObjClientOptions& option, - int64_t expiration_time, size_t batch_size); -}; - -} // namespace doris::cloud diff --git a/cloud/src/recycler/recycler.cpp b/cloud/src/recycler/recycler.cpp index e0f7c2ccffa135..b37833b8059398 100644 --- a/cloud/src/recycler/recycler.cpp +++ b/cloud/src/recycler/recycler.cpp @@ -770,6 +770,7 @@ int InstanceRecycler::init_storage_vault_accessors() { << "but HDFS storage vaults were detected"; #endif } else if (vault.has_obj_info()) { + // TODO: Propagate object storage session tokens to Recycler in a follow-up PR. auto s3_conf = S3Conf::from_obj_store_info(vault.obj_info()); if (!s3_conf) { LOG(WARNING) << "failed to init object accessor, invalid conf, instance_id=" diff --git a/cloud/src/recycler/s3_accessor.cpp b/cloud/src/recycler/s3_accessor.cpp index 56b2338cd84b14..016204bcb5966a 100644 --- a/cloud/src/recycler/s3_accessor.cpp +++ b/cloud/src/recycler/s3_accessor.cpp @@ -20,15 +20,12 @@ #include #include #include -#include #include -#include #include -#include -#include #include #include +#include #ifdef USE_AZURE #include @@ -45,34 +42,24 @@ #include "common/simple_thread_pool.h" #include "common/string_util.h" #include "common/util.h" +#include "cpp/obj-client/auth/aws_credential_factory.h" +#ifdef USE_AZURE +#include "cpp/obj-client/auth/azure_auth_factory.h" +#include "cpp/obj-client/azure_obj_storage_client.h" +#endif #include "cpp/aws_logger.h" -#include "cpp/custom_aws_credentials_provider_chain.h" #include "cpp/obj-client/auth/obj_credential.h" #include "cpp/obj-client/auth/obj_s3_client_factory.h" +#include "cpp/obj-client/rate_limited_obj_storage_client.h" +#include "cpp/obj-client/s3_obj_storage_client.h" #include "cpp/obj_retry_strategy.h" #include "cpp/sync_point.h" #include "cpp/token_bucket_rate_limiter.h" #include "cpp/util.h" -#ifdef USE_AZURE -#include "recycler/azure_obj_client.h" -#endif -#include "recycler/obj_storage_client.h" -#include "recycler/s3_obj_client.h" #include "recycler/storage_vault_accessor.h" +#include "recycler/sync_executor.h" namespace doris::cloud { -namespace s3_bvar { -bvar::LatencyRecorder s3_get_latency("s3_get"); -bvar::LatencyRecorder s3_put_latency("s3_put"); -bvar::LatencyRecorder s3_delete_object_latency("s3_delete_object"); -bvar::LatencyRecorder s3_delete_objects_latency("s3_delete_objects"); -bvar::LatencyRecorder s3_head_latency("s3_head"); -bvar::LatencyRecorder s3_multi_part_upload_latency("s3_multi_part_upload"); -bvar::LatencyRecorder s3_list_latency("s3_list"); -bvar::LatencyRecorder s3_list_object_versions_latency("s3_list_object_versions"); -bvar::LatencyRecorder s3_get_bucket_version_latency("s3_get_bucket_version"); -bvar::LatencyRecorder s3_copy_object_latency("s3_copy_object"); -}; // namespace s3_bvar AccessorRateLimiter::AccessorRateLimiter() : _rate_limiters({std::make_unique( @@ -110,6 +97,70 @@ int reset_s3_rate_limiter(S3RateLimitType type, size_t max_speed, size_t max_bur return AccessorRateLimiter::instance().rate_limiter(type)->reset(max_speed, max_burst, limit); } +namespace { + +class RecyclerObjStorageRateLimitPolicy final : public ObjStorageRateLimitPolicy { +public: + ObjStorageAdmission acquire(S3RateLimitType type, size_t) const override { + if (config::enable_s3_rate_limiter && + doris::apply_s3_rate_limit(type, AccessorRateLimiter::instance().rate_limiter(type), + config::s3_rate_limiter_log_interval) < 0) { + return ObjStorageAdmission { + .resp = ObjStorageResponse::rate_limit( + TStatusCode::LIMIT_REACH, 429, + fmt::format( + "object storage {} request rejected by recycler rate limiter", + to_string(type))), + }; + } + return ObjStorageAdmission {}; + } +}; + +class RecyclerObjStorageDeleteExecutor final : public ObjStorageDeleteExecutor { +public: + explicit RecyclerObjStorageDeleteExecutor(std::shared_ptr pool) + : pool_(std::move(pool)) { + reset(); + } + + ObjStorageResponse submit(ObjStorageDeleteTask task) override { + executor_->add(std::move(task)); + return ObjStorageResponse::OK(); + } + + ObjStorageResponse wait() override { + bool finished = false; + auto responses = executor_->when_all(&finished); + reset(); + if (!finished) { + return { + .status = {TStatusCode::INTERNAL_ERROR, + "object storage batch deletion did not finish"}, + .http_code = 0, + }; + } + for (auto& response : responses) { + if (!response.ok()) { + return response; + } + } + return ObjStorageResponse::OK(); + } + +private: + void reset() { + executor_ = std::make_unique>( + pool_, "delete object storage batches", + [](const ObjStorageResponse& response) { return !response.ok(); }); + } + + std::shared_ptr pool_; + std::unique_ptr> executor_; +}; + +} // namespace + S3Environment::S3Environment() { LOG(INFO) << "Initializing S3 environment"; aws_options_ = Aws::SDKOptions {}; @@ -159,25 +210,26 @@ S3Environment::~S3Environment() { class S3ListIterator final : public ListIterator { public: - S3ListIterator(std::unique_ptr iter, size_t prefix_length) + S3ListIterator(std::unique_ptr iter, size_t prefix_length) : iter_(std::move(iter)), prefix_length_(prefix_length) {} ~S3ListIterator() override = default; bool is_valid() override { return iter_->is_valid(); } - bool has_next() override { return iter_->has_next(); } + bool has_next() override { return iter_->has_next().ok(); } std::optional next() override { - std::optional ret; - if (auto obj = iter_->next(); obj.has_value()) { - ret = FileMeta { - .path = get_relative_path(obj->key), - .size = obj->size, - .mtime_s = obj->mtime_s, - }; + auto result = iter_->next(); + if (!result.object.has_value()) { + return std::nullopt; } - return ret; + auto& obj = *result.object; + return FileMeta { + .path = get_relative_path(obj.key), + .size = obj.size, + .mtime_s = obj.mtime_s, + }; } private: @@ -185,7 +237,7 @@ class S3ListIterator final : public ListIterator { return key.substr(prefix_length_); } - std::unique_ptr iter_; + std::unique_ptr iter_; size_t prefix_length_; }; @@ -230,7 +282,6 @@ std::optional S3Conf::from_obj_store_info(const ObjectStoreInfoPB& obj_i s3_conf.sk = obj_info.sk(); } } - if (obj_info.has_cred_provider_type()) { s3_conf.cred_provider_type = cred_provider_type_from_pb(obj_info.cred_provider_type()); } @@ -284,96 +335,41 @@ int S3Accessor::create(S3Conf conf, std::shared_ptr* accessor) { static std::shared_ptr worker_pool; -std::shared_ptr S3Accessor::_get_aws_credentials_provider_v1( - const S3Conf& s3_conf) { - if (!s3_conf.ak.empty() && !s3_conf.sk.empty()) { - Aws::Auth::AWSCredentials aws_cred(s3_conf.ak, s3_conf.sk); - DCHECK(!aws_cred.IsExpiredOrEmpty()); - return std::make_shared(std::move(aws_cred)); - } - - if (s3_conf.cred_provider_type == CredProviderType::InstanceProfile) { - if (s3_conf.role_arn.empty()) { - return std::make_shared(); - } - - Aws::Client::ClientConfiguration clientConfiguration = - S3Environment::getClientConfiguration(); - if (_ca_cert_file_path.empty()) { - _ca_cert_file_path = - get_valid_ca_cert_path(doris::cloud::split(config::ca_cert_file_paths, ';')); - } - if (!_ca_cert_file_path.empty()) { - clientConfiguration.caFile = _ca_cert_file_path; - } - - auto stsClient = std::make_shared( - std::make_shared(), - clientConfiguration); - - return std::make_shared( - s3_conf.role_arn, Aws::String(), s3_conf.external_id, - Aws::Auth::DEFAULT_CREDS_LOAD_FREQ_SECONDS, stsClient); - } - return std::make_shared(); -} - -std::shared_ptr S3Accessor::_create_credentials_provider( - CredProviderType type) { - switch (type) { - case CredProviderType::Env: - return std::make_shared(); - case CredProviderType::SystemProperties: - return std::make_shared(); - case CredProviderType::WebIdentity: - return std::make_shared(); - case CredProviderType::Container: - return create_container_credentials_provider(); - case CredProviderType::InstanceProfile: - return std::make_shared(); - case CredProviderType::Anonymous: - return std::make_shared(); - case CredProviderType::Default: - default: - return std::make_shared(); - } -} - -std::shared_ptr S3Accessor::_get_aws_credentials_provider_v2( - const S3Conf& s3_conf) { - if (!s3_conf.ak.empty() && !s3_conf.sk.empty()) { - Aws::Auth::AWSCredentials aws_cred(s3_conf.ak, s3_conf.sk); - DCHECK(!aws_cred.IsExpiredOrEmpty()); - return std::make_shared(std::move(aws_cred)); - } - - if (!s3_conf.role_arn.empty()) { - Aws::Client::ClientConfiguration clientConfiguration = - S3Environment::getClientConfiguration(); - if (_ca_cert_file_path.empty()) { - _ca_cert_file_path = - get_valid_ca_cert_path(doris::cloud::split(config::ca_cert_file_paths, ';')); - } - if (!_ca_cert_file_path.empty()) { - clientConfiguration.caFile = _ca_cert_file_path; - } - - auto stsClient = std::make_shared( - _create_credentials_provider(s3_conf.cred_provider_type), clientConfiguration); - - return std::make_shared( - s3_conf.role_arn, Aws::String(), s3_conf.external_id, - Aws::Auth::DEFAULT_CREDS_LOAD_FREQ_SECONDS, stsClient); +ObjStorageRecursiveDeleteOptions S3Accessor::make_recursive_delete_options( + int64_t expiration_time, std::shared_ptr pool) { + const auto configured_max_tasks = config::recycler_max_tasks_per_batch; + size_t max_tasks_per_batch = 1000; + if (configured_max_tasks > 0) { + max_tasks_per_batch = static_cast(configured_max_tasks); + } else { + LOG(WARNING) << "recycler_max_tasks_per_batch=" << configured_max_tasks + << " is not positive, using default 1000"; } - return _create_credentials_provider(s3_conf.cred_provider_type); + ObjStorageRecursiveDeleteOptions options { + .expiration_time = expiration_time, + .max_tasks_per_batch = max_tasks_per_batch, + .executor = std::make_shared(std::move(pool)), + }; + return options; } -std::shared_ptr S3Accessor::get_aws_credentials_provider( - const S3Conf& s3_conf) { - if (config::aws_credentials_provider_version == "v2") { - return _get_aws_credentials_provider_v2(s3_conf); +AwsCredentialResult S3Accessor::create_aws_credentials_provider(const S3Conf& s3_conf) { + auto sts_config = S3Environment::getClientConfiguration(); + if (!_ca_cert_file_path.empty()) { + sts_config.caFile = _ca_cert_file_path; } - return _get_aws_credentials_provider_v1(s3_conf); + return AwsCredentialFactory::create({ + .version = config::aws_credentials_provider_version == "v2" + ? AwsCredentialProviderVersion::V2 + : AwsCredentialProviderVersion::V1, + .access_key = s3_conf.ak, + .secret_key = s3_conf.sk, + .provider_type = s3_conf.cred_provider_type, + .role_arn = s3_conf.role_arn, + .external_id = s3_conf.external_id, + .empty_credentials = EmptyCredentialsBehavior::DEFAULT_CHAIN, + .sts_client_config = std::move(sts_config), + }); } int S3Accessor::init() { @@ -390,8 +386,6 @@ int S3Accessor::init() { #ifdef USE_AZURE Azure::Storage::Blobs::BlobClientOptions options; options.Retry.MaxRetries = config::max_s3_client_retry; - auto cred = - std::make_shared(conf_.ak, conf_.sk); uri_ = fmt::format("{}/{}", conf_.endpoint, conf_.bucket); if (uri_.find("://") == std::string::npos) { uri_ = "https://" + uri_; @@ -402,11 +396,29 @@ int S3Accessor::init() { // All policies in the PerRetryPolicies are downstream of the RetryPolicy. // Therefore, the policy can record retries after the RetryPolicy has handled the response. options.PerRetryPolicies.emplace_back(std::make_unique()); - auto container_client = std::make_shared( - uri_, cred, std::move(options)); + auto built = AzureAuthFactory::create(uri_, + { + .type = AzureCredentialType::SHARED_KEY, + .account_name = conf_.ak, + .account_key = conf_.sk, + }, + std::move(options)); + if (!built) { + LOG_WARNING("failed to create Azure client").tag("error", built.error); + return -1; + } // uri format for debug: ${scheme}://${ak}.blob.core.windows.net/${bucket}/${prefix} uri_ = normalize_http_uri(uri_ + '/' + conf_.prefix); - obj_client_ = std::make_shared(std::move(container_client)); + auto client = + std::make_shared(std::move(built.container_client), + ObjStorageEndpointInfo { + .endpoint = conf_.endpoint, + .ak = conf_.ak, + .sk = conf_.sk, + }, + std::move(built.shared_key_credential)); + obj_client_ = std::make_shared( + std::move(client), std::make_shared()); return 0; #else LOG_FATAL("BE is not compiled with azure support, export BUILD_AZURE=ON before building"); @@ -447,17 +459,29 @@ int S3Accessor::init() { // curl error 28 on slow/large S3 DeleteObjects (OVH cold vault). aws_config.requestTimeoutMs = 30000; aws_config.connectTimeoutMs = 5000; + auto credentials = create_aws_credentials_provider(conf_); + if (!credentials) { + LOG(WARNING) << "failed to create AWS credential provider: " << credentials.error; + return -1; + } auto s3_client = doris::make_s3_client( conf_.credential, doris::S3ClientBuildContext { - .fallback_provider = get_aws_credentials_provider(conf_), + .fallback_provider = std::move(credentials.provider), .config = std::move(aws_config), .payload_signing_policy = Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, .use_virtual_addressing = conf_.use_virtual_addressing, .ca_cert_path = _ca_cert_file_path, }); - obj_client_ = std::make_shared(std::move(s3_client), conf_.endpoint); + auto client = std::make_shared(std::move(s3_client), + ObjStorageEndpointInfo { + .endpoint = conf_.endpoint, + .ak = conf_.ak, + .sk = conf_.sk, + }); + obj_client_ = std::make_shared( + std::move(client), std::make_shared()); return 0; } } @@ -465,10 +489,13 @@ int S3Accessor::init() { int S3Accessor::delete_prefix_impl(const std::string& path_prefix, int64_t expiration_time) { LOG_INFO("delete prefix").tag("uri", to_uri(path_prefix)); - return obj_client_ - ->delete_objects_recursively({.bucket = conf_.bucket, .key = get_key(path_prefix)}, - {.executor = worker_pool}, expiration_time) - .ret; + ObjStoragePath delete_path { + .bucket = conf_.bucket, + .prefix = get_key(path_prefix), + }; + return delete_objects_recursively(obj_client_, delete_path, + make_recursive_delete_options(expiration_time, worker_pool)) + .status.code; } int S3Accessor::delete_prefix(const std::string& path_prefix, int64_t expiration_time) { @@ -509,29 +536,29 @@ int S3Accessor::delete_files(const std::vector& paths) { keys.emplace_back(get_key(path)); } - return obj_client_->delete_objects(conf_.bucket, std::move(keys), {.executor = worker_pool}) - .ret; + return obj_client_->delete_objects({.bucket = conf_.bucket}, std::move(keys)).status.code; } int S3Accessor::delete_file(const std::string& path) { LOG_INFO("delete file").tag("uri", to_uri(path)); - int ret = obj_client_->delete_object({.bucket = conf_.bucket, .key = get_key(path)}).ret; - static_assert(ObjectStorageResponse::OK == 0); - if (ret == ObjectStorageResponse::OK || ret == ObjectStorageResponse::NOT_FOUND) { + int ret = + obj_client_->delete_object({.bucket = conf_.bucket, .key = get_key(path)}).status.code; + static_assert(ObjStorageStatus::OK == 0); + if (ret == ObjStorageStatus::OK || ret == ObjStorageStatus::NOT_FOUND) { return 0; } return ret; } int S3Accessor::put_file(const std::string& path, const std::string& content) { - return obj_client_->put_object({.bucket = conf_.bucket, .key = get_key(path)}, content).ret; + return obj_client_->put_object({.bucket = conf_.bucket, .key = get_key(path)}, content) + .status.code; } int S3Accessor::list_prefix(const std::string& path_prefix, std::unique_ptr* res) { - size_t prefix_length = conf_.prefix.empty() ? 0 : conf_.prefix.length() + 1; *res = std::make_unique( - obj_client_->list_objects({.bucket = conf_.bucket, .key = get_key(path_prefix)}), - prefix_length); + obj_client_->list_objects({.bucket = conf_.bucket, .prefix = get_key(path_prefix)}), + conf_.prefix.empty() ? 0 : conf_.prefix.length() + 1); return 0; } @@ -551,8 +578,14 @@ int S3Accessor::list_all(std::unique_ptr* res) { } int S3Accessor::exists(const std::string& path) { - ObjectMeta obj_meta; - return obj_client_->head_object({.bucket = conf_.bucket, .key = get_key(path)}, &obj_meta).ret; + auto response = obj_client_->head_object({.bucket = conf_.bucket, .key = get_key(path)}).resp; + if (response.ok()) { + return 0; + } + if (response.status.code == ObjStorageStatus::NOT_FOUND) { + return 1; + } + return -1; } int S3Accessor::abort_multipart_upload(const std::string& path, const std::string& upload_id) { @@ -560,9 +593,9 @@ int S3Accessor::abort_multipart_upload(const std::string& path, const std::strin int ret = obj_client_ ->abort_multipart_upload({.bucket = conf_.bucket, .key = get_key(path)}, upload_id) - .ret; - static_assert(ObjectStorageResponse::OK == 0); - if (ret == ObjectStorageResponse::OK || ret == ObjectStorageResponse::NOT_FOUND) { + .status.code; + static_assert(ObjStorageStatus::OK == 0); + if (ret == ObjStorageStatus::OK || ret == ObjStorageStatus::NOT_FOUND) { return 0; } LOG_WARNING("fail abort multipart upload") @@ -572,12 +605,12 @@ int S3Accessor::abort_multipart_upload(const std::string& path, const std::strin return ret; } -int S3Accessor::get_life_cycle(int64_t* expiration_days) { - return obj_client_->get_life_cycle(conf_.bucket, expiration_days).ret; +int S3Accessor::get_lifecycle(int64_t* expiration_days) { + return obj_client_->get_lifecycle(conf_.bucket, expiration_days).status.code; } int S3Accessor::check_versioning() { - return obj_client_->check_versioning(conf_.bucket).ret; + return obj_client_->check_versioning(conf_.bucket).status.code; } int GcsAccessor::delete_prefix_impl(const std::string& path_prefix, int64_t expiration_time) { @@ -588,8 +621,17 @@ int GcsAccessor::delete_prefix_impl(const std::string& path_prefix, int64_t expi int skip = 0; int64_t del_nonexisted = 0; int del = 0; - auto iter = obj_client_->list_objects({conf_.bucket, get_key(path_prefix)}); - for (auto obj = iter->next(); obj.has_value(); obj = iter->next()) { + auto iter = obj_client_->list_objects({.bucket = conf_.bucket, .prefix = get_key(path_prefix)}); + for (;;) { + auto result = iter->next(); + if (!result.object.has_value()) { + if (result.resp.status.code != ObjStorageStatus::NOT_FOUND && + result.resp.status.code != ObjStorageStatus::OK) { + ret = result.resp.status.code; + } + break; + } + auto& obj = *result.object; if (!(++cnt % 100)) { LOG_INFO("loop delete prefix") .tag("uri", to_uri(path_prefix)) @@ -598,17 +640,18 @@ int GcsAccessor::delete_prefix_impl(const std::string& path_prefix, int64_t expi .tag("del_nonexisted", del_nonexisted) .tag("skipped", skip); } - if (expiration_time > 0 && obj->mtime_s > expiration_time) { + if (expiration_time > 0 && obj.mtime_s > expiration_time) { skip++; continue; } del++; // FIXME(plat1ko): Delete objects by batch with genuine GCS client - int del_ret = obj_client_->delete_object({conf_.bucket, obj->key}).ret; - del_nonexisted += (del_ret == ObjectStorageResponse::NOT_FOUND); - static_assert(ObjectStorageResponse::OK == 0); - if (del_ret != ObjectStorageResponse::OK && del_ret != ObjectStorageResponse::NOT_FOUND) { + int del_ret = + obj_client_->delete_object({.bucket = conf_.bucket, .key = obj.key}).status.code; + del_nonexisted += (del_ret == ObjStorageStatus::NOT_FOUND); + static_assert(ObjStorageStatus::OK == 0); + if (del_ret != ObjStorageStatus::OK && del_ret != ObjStorageStatus::NOT_FOUND) { ret = del_ret; } } diff --git a/cloud/src/recycler/s3_accessor.h b/cloud/src/recycler/s3_accessor.h index 3e1d57e3e2a02f..adee55e2195b0c 100644 --- a/cloud/src/recycler/s3_accessor.h +++ b/cloud/src/recycler/s3_accessor.h @@ -18,7 +18,6 @@ #pragma once #include -#include #include #include @@ -26,8 +25,9 @@ #include #include "cpp/aws_common.h" +#include "cpp/obj-client/auth/aws_credential_factory.h" #include "cpp/obj-client/auth/obj_credential.h" -#include "recycler/obj_storage_client.h" +#include "cpp/obj-client/obj_storage_client.h" #include "recycler/storage_vault_accessor.h" namespace Aws::S3 { @@ -43,19 +43,6 @@ namespace cloud { class ObjectStoreInfoPB; class SimpleThreadPool; -namespace s3_bvar { -extern bvar::LatencyRecorder s3_get_latency; -extern bvar::LatencyRecorder s3_put_latency; -extern bvar::LatencyRecorder s3_delete_object_latency; -extern bvar::LatencyRecorder s3_delete_objects_latency; -extern bvar::LatencyRecorder s3_head_latency; -extern bvar::LatencyRecorder s3_multi_part_upload_latency; -extern bvar::LatencyRecorder s3_list_latency; -extern bvar::LatencyRecorder s3_list_object_versions_latency; -extern bvar::LatencyRecorder s3_get_bucket_version_latency; -extern bvar::LatencyRecorder s3_copy_object_latency; -}; // namespace s3_bvar - class S3Environment { public: S3Environment(const S3Environment&) = delete; @@ -149,28 +136,21 @@ class S3Accessor : public StorageVaultAccessor { // Get the objects' expiration time on the conf.bucket // returns 0 for success otherwise error - int get_life_cycle(int64_t* expiration_days); + int get_lifecycle(int64_t* expiration_days); // Check if the objects' versioning is on or off // returns 0 when versioning is on, otherwise versioning is off or check failed int check_versioning(); protected: + static ObjStorageRecursiveDeleteOptions make_recursive_delete_options( + int64_t expiration_time, std::shared_ptr pool); + int list_prefix(const std::string& path_prefix, std::unique_ptr* res); virtual int delete_prefix_impl(const std::string& path_prefix, int64_t expiration_time = 0); - std::shared_ptr _get_aws_credentials_provider_v1( - const S3Conf& s3_conf); - - std::shared_ptr _get_aws_credentials_provider_v2( - const S3Conf& s3_conf); - - std::shared_ptr _create_credentials_provider( - CredProviderType type); - - std::shared_ptr get_aws_credentials_provider( - const S3Conf& s3_conf); + AwsCredentialResult create_aws_credentials_provider(const S3Conf& s3_conf); std::string get_key(const std::string& relative_path) const; std::string to_uri(const std::string& relative_path) const; diff --git a/cloud/src/recycler/s3_obj_client.cpp b/cloud/src/recycler/s3_obj_client.cpp deleted file mode 100644 index 3437a879d649bd..00000000000000 --- a/cloud/src/recycler/s3_obj_client.cpp +++ /dev/null @@ -1,443 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#include "recycler/s3_obj_client.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include - -#include "common/config.h" -#include "common/logging.h" -#include "common/stopwatch.h" -#include "cpp/obj_retry_strategy.h" -#include "cpp/sync_point.h" -#include "cpp/token_bucket_rate_limiter.h" -#include "recycler/s3_accessor.h" -#include "recycler/util.h" - -namespace doris::cloud { - -[[maybe_unused]] static Aws::Client::AWSError s3_error_factory() { - return {Aws::S3::S3Errors::INTERNAL_FAILURE, "exceeds limit", "exceeds limit", false}; -} - -void record_s3_request_failed(const Aws::S3::S3Error& error) { - doris::record_object_request_failed(static_cast(error.GetResponseCode())); -} - -template -auto s3_rate_limit(S3RateLimitType op, Func callback) -> decltype(callback()) { - using T = decltype(callback()); - if (!config::enable_s3_rate_limiter) { - return callback(); - } - auto sleep_duration = - doris::apply_s3_rate_limit(op, AccessorRateLimiter::instance().rate_limiter(op), - config::s3_rate_limiter_log_interval); - if (sleep_duration < 0) { - return T(s3_error_factory()); - } - return callback(); -} - -template -auto s3_get_rate_limit(Func callback) -> decltype(callback()) { - return s3_rate_limit(S3RateLimitType::GET, std::move(callback)); -} - -template -auto s3_put_rate_limit(Func callback) -> decltype(callback()) { - return s3_rate_limit(S3RateLimitType::PUT, std::move(callback)); -} - -class S3ObjListIterator final : public ObjectListIterator { -public: - S3ObjListIterator(std::shared_ptr client, std::string bucket, - std::string prefix, std::string endpoint) - : client_(std::move(client)), endpoint_(std::move(endpoint)) { - req_.WithBucket(std::move(bucket)).WithPrefix(std::move(prefix)); - TEST_SYNC_POINT_CALLBACK("S3ObjListIterator", &req_); - } - - ~S3ObjListIterator() override = default; - - bool is_valid() override { return is_valid_; } - - bool has_next() override { - if (!is_valid_) { - return false; - } - - if (!results_.empty()) { - return true; - } - - if (!has_more_) { - return false; - } - - auto outcome = s3_get_rate_limit([&]() { - SCOPED_BVAR_LATENCY(s3_bvar::s3_list_latency); - return client_->ListObjectsV2(req_); - }); - - const auto& request_id = outcome.IsSuccess() ? outcome.GetResult().GetRequestId() - : outcome.GetError().GetRequestId(); - if (!outcome.IsSuccess()) { - // Treat NoSuchKey as empty response for compatibility with some S3-compatible storage providers - // e.g. TOS by ByteDance Cloud (Volcano Engine) - if (outcome.GetError().GetErrorType() == Aws::S3::S3Errors::NO_SUCH_KEY) { - LOG_INFO("NoSuchKey error when listing objects, treat as empty response") - .tag("endpoint", endpoint_) - .tag("bucket", req_.GetBucket()) - .tag("prefix", req_.GetPrefix()) - .tag("request_id", request_id); - has_more_ = false; - return false; - } - - record_s3_request_failed(outcome.GetError()); - LOG_WARNING("failed to list objects") - .tag("endpoint", endpoint_) - .tag("bucket", req_.GetBucket()) - .tag("prefix", req_.GetPrefix()) - .tag("responseCode", static_cast(outcome.GetError().GetResponseCode())) - .tag("error", outcome.GetError().GetMessage()) - .tag("request_id", request_id); - is_valid_ = false; - return false; - } - - if (outcome.GetResult().GetIsTruncated() && - outcome.GetResult().GetNextContinuationToken().empty()) { - LOG_WARNING("failed to list objects, isTruncated but no continuation token") - .tag("endpoint", endpoint_) - .tag("bucket", req_.GetBucket()) - .tag("prefix", req_.GetPrefix()) - .tag("request_id", request_id); - - is_valid_ = false; - return false; - } - - has_more_ = outcome.GetResult().GetIsTruncated(); - req_.SetContinuationToken(std::move( - const_cast(outcome.GetResult().GetNextContinuationToken()))); - - auto&& content = outcome.GetResult().GetContents(); - // clang-format off - DCHECK(!(has_more_ && req_.GetContinuationToken().empty())) - << "has_more=" << has_more_ - << " token=" << req_.GetContinuationToken() - << " request_id=" << request_id; - // clang-format on - if (has_more_ && req_.GetContinuationToken().empty()) { - LOG(ERROR) << "it is impossible to have more results but no continuation token"; - has_more_ = false; - } - if (has_more_ && content.empty()) { - LOG(INFO) << "Empty page with more results (possible concurrent deletion), continuing" - << " request_id=" << request_id; - return has_next(); - } - - results_.reserve(content.size()); - for (auto&& obj : std::ranges::reverse_view(content)) { - DCHECK(obj.GetKey().starts_with(req_.GetPrefix())) - << obj.GetKey() << ' ' << req_.GetPrefix(); - results_.emplace_back( - ObjectMeta {.key = std::move(const_cast(obj.GetKey())), - .size = obj.GetSize(), - .mtime_s = obj.GetLastModified().Seconds()}); - } - - return !results_.empty(); - } - - std::optional next() override { - std::optional res; - if (!has_next()) { - return res; - } - - res = std::move(results_.back()); - results_.pop_back(); - return res; - } - -private: - std::shared_ptr client_; - Aws::S3::Model::ListObjectsV2Request req_; - std::vector results_; - std::string endpoint_; - bool is_valid_ {true}; - bool has_more_ {true}; -}; - -static constexpr size_t MaxDeleteBatch = 1000; - -S3ObjClient::~S3ObjClient() = default; - -ObjectStorageResponse S3ObjClient::put_object(ObjectStoragePathRef path, std::string_view stream) { - Aws::S3::Model::PutObjectRequest request; - request.WithBucket(path.bucket).WithKey(path.key); - auto input = Aws::MakeShared("S3Accessor"); - *input << stream; - request.SetBody(input); - auto outcome = s3_put_rate_limit([&]() { - SCOPED_BVAR_LATENCY(s3_bvar::s3_put_latency); - return s3_client_->PutObject(request); - }); - if (!outcome.IsSuccess()) { - record_s3_request_failed(outcome.GetError()); - LOG_WARNING("failed to put object") - .tag("endpoint", endpoint_) - .tag("bucket", path.bucket) - .tag("key", path.key) - .tag("responseCode", static_cast(outcome.GetError().GetResponseCode())) - .tag("error", outcome.GetError().GetMessage()) - .tag("request_id", outcome.GetError().GetRequestId()); - return -1; - } - return 0; -} - -ObjectStorageResponse S3ObjClient::head_object(ObjectStoragePathRef path, ObjectMeta* res) { - Aws::S3::Model::HeadObjectRequest request; - request.WithBucket(path.bucket).WithKey(path.key); - auto outcome = s3_get_rate_limit([&]() { - SCOPED_BVAR_LATENCY(s3_bvar::s3_head_latency); - return s3_client_->HeadObject(request); - }); - if (outcome.IsSuccess()) { - res->key = path.key; - res->size = outcome.GetResult().GetContentLength(); - res->mtime_s = outcome.GetResult().GetLastModified().Seconds(); - return 0; - } else if (outcome.GetError().GetResponseCode() == Aws::Http::HttpResponseCode::NOT_FOUND) { - return 1; - } else { - record_s3_request_failed(outcome.GetError()); - LOG_WARNING("failed to head object") - .tag("endpoint", endpoint_) - .tag("bucket", path.bucket) - .tag("key", path.key) - .tag("responseCode", static_cast(outcome.GetError().GetResponseCode())) - .tag("error", outcome.GetError().GetMessage()) - .tag("request_id", outcome.GetError().GetRequestId()); - return -1; - } -} - -std::unique_ptr S3ObjClient::list_objects(ObjectStoragePathRef path) { - return std::make_unique(s3_client_, path.bucket, path.key, endpoint_); -} - -ObjectStorageResponse S3ObjClient::delete_objects(const std::string& bucket, - std::vector keys, - ObjClientOptions option) { - if (keys.empty()) { - return {0}; - } - - Aws::S3::Model::DeleteObjectsRequest delete_request; - delete_request.SetBucket(bucket); - - auto issue_delete = [&bucket, &delete_request, - this](std::vector objects) -> int { - if (objects.size() == 1) { - return delete_object({.bucket = bucket, .key = objects[0].GetKey()}).ret; - } - - Aws::S3::Model::Delete del; - del.WithObjects(std::move(objects)).SetQuiet(true); - delete_request.SetDelete(std::move(del)); - auto delete_outcome = s3_put_rate_limit([&]() { - SCOPED_BVAR_LATENCY(s3_bvar::s3_delete_objects_latency); - return s3_client_->DeleteObjects(delete_request); - }); - if (!delete_outcome.IsSuccess()) { - record_s3_request_failed(delete_outcome.GetError()); - LOG_WARNING("failed to delete objects") - .tag("endpoint", endpoint_) - .tag("bucket", bucket) - .tag("key[0]", delete_request.GetDelete().GetObjects().front().GetKey()) - .tag("responseCode", - static_cast(delete_outcome.GetError().GetResponseCode())) - .tag("error", delete_outcome.GetError().GetMessage()) - .tag("request_id", delete_outcome.GetError().GetRequestId()); - return -1; - } - - return 0; - }; - - int ret = 0; - // `DeleteObjectsRequest` can only contain 1000 keys at most. - std::vector objects; - - size_t delete_batch_size = MaxDeleteBatch; - TEST_INJECTION_POINT_CALLBACK("S3ObjClient::delete_objects", &delete_batch_size); - - // std::views::chunk(1000) - for (auto&& key : keys) { - objects.emplace_back().SetKey(std::move(key)); - if (objects.size() < delete_batch_size) { - continue; - } - - ret = issue_delete(std::move(objects)); - if (ret != 0) { - return {ret}; - } - } - - if (!objects.empty()) { - ret = issue_delete(std::move(objects)); - } - - return {ret}; -} - -ObjectStorageResponse S3ObjClient::delete_object(ObjectStoragePathRef path) { - Aws::S3::Model::DeleteObjectRequest request; - request.WithBucket(path.bucket).WithKey(path.key); - auto outcome = s3_put_rate_limit([&]() { - SCOPED_BVAR_LATENCY(s3_bvar::s3_delete_object_latency); - return s3_client_->DeleteObject(request); - }); - TEST_SYNC_POINT_CALLBACK("S3ObjClient::delete_object", &outcome); - if (!outcome.IsSuccess()) { - if (outcome.GetError().GetResponseCode() == Aws::Http::HttpResponseCode::NOT_FOUND) { - return {ObjectStorageResponse::NOT_FOUND, outcome.GetError().GetMessage()}; - } - record_s3_request_failed(outcome.GetError()); - LOG_WARNING("failed to delete object") - .tag("endpoint", endpoint_) - .tag("bucket", path.bucket) - .tag("key", path.key) - .tag("responseCode", static_cast(outcome.GetError().GetResponseCode())) - .tag("error", outcome.GetError().GetMessage()) - .tag("exception", outcome.GetError().GetExceptionName()) - .tag("request_id", outcome.GetError().GetRequestId()); - return {ObjectStorageResponse::UNDEFINED, outcome.GetError().GetMessage()}; - } - return {ObjectStorageResponse::OK}; -} - -ObjectStorageResponse S3ObjClient::delete_objects_recursively(ObjectStoragePathRef path, - ObjClientOptions option, - int64_t expiration_time) { - return delete_objects_recursively_(path, option, expiration_time, MaxDeleteBatch); -} - -ObjectStorageResponse S3ObjClient::get_life_cycle(const std::string& bucket, - int64_t* expiration_days) { - Aws::S3::Model::GetBucketLifecycleConfigurationRequest request; - request.SetBucket(bucket); - - auto outcome = s3_get_rate_limit( - [&]() { return s3_client_->GetBucketLifecycleConfiguration(request); }); - bool has_lifecycle = false; - if (outcome.IsSuccess()) { - const auto& rules = outcome.GetResult().GetRules(); - for (const auto& rule : rules) { - if (rule.NoncurrentVersionExpirationHasBeenSet()) { - has_lifecycle = true; - *expiration_days = rule.GetNoncurrentVersionExpiration().GetNoncurrentDays(); - } - } - } else { - record_s3_request_failed(outcome.GetError()); - LOG_WARNING("Err for check interval: failed to get bucket lifecycle") - .tag("endpoint", endpoint_) - .tag("bucket", bucket) - .tag("responseCode", static_cast(outcome.GetError().GetResponseCode())) - .tag("error", outcome.GetError().GetMessage()) - .tag("request_id", outcome.GetError().GetRequestId()); - return -1; - } - - if (!has_lifecycle) { - LOG_WARNING("Err for check interval: bucket doesn't have lifecycle configuration") - .tag("endpoint", endpoint_) - .tag("bucket", bucket); - return -1; - } - return 0; -} - -ObjectStorageResponse S3ObjClient::check_versioning(const std::string& bucket) { - Aws::S3::Model::GetBucketVersioningRequest request; - request.SetBucket(bucket); - auto outcome = s3_get_rate_limit([&]() { return s3_client_->GetBucketVersioning(request); }); - - if (outcome.IsSuccess()) { - const auto& versioning_configuration = outcome.GetResult().GetStatus(); - if (versioning_configuration != Aws::S3::Model::BucketVersioningStatus::Enabled) { - LOG_WARNING("Err for check interval: bucket doesn't enable bucket versioning") - .tag("endpoint", endpoint_) - .tag("bucket", bucket); - return -1; - } - } else { - record_s3_request_failed(outcome.GetError()); - LOG_WARNING("Err for check interval: failed to get status of bucket versioning") - .tag("endpoint", endpoint_) - .tag("bucket", bucket) - .tag("responseCode", static_cast(outcome.GetError().GetResponseCode())) - .tag("error", outcome.GetError().GetMessage()) - .tag("request_id", outcome.GetError().GetRequestId()); - return -1; - } - return 0; -} - -ObjectStorageResponse S3ObjClient::abort_multipart_upload(ObjectStoragePathRef path, - const std::string& upload_id) { - Aws::S3::Model::AbortMultipartUploadRequest request; - request.WithBucket(path.bucket).WithKey(path.key).WithUploadId(upload_id); - auto outcome = s3_put_rate_limit([&]() { return s3_client_->AbortMultipartUpload(request); }); - if (!outcome.IsSuccess()) { - if (outcome.GetError().GetResponseCode() == Aws::Http::HttpResponseCode::NOT_FOUND) { - return {ObjectStorageResponse::OK}; - } - record_s3_request_failed(outcome.GetError()); - LOG_WARNING("failed to abort multipart upload") - .tag("endpoint", endpoint_) - .tag("bucket", path.bucket) - .tag("key", path.key) - .tag("upload_id", upload_id) - .tag("responseCode", static_cast(outcome.GetError().GetResponseCode())) - .tag("error", outcome.GetError().GetMessage()) - .tag("exception", outcome.GetError().GetExceptionName()) - .tag("request_id", outcome.GetError().GetRequestId()); - return {ObjectStorageResponse::UNDEFINED, outcome.GetError().GetMessage()}; - } - return {ObjectStorageResponse::OK}; -} - -} // namespace doris::cloud diff --git a/cloud/src/recycler/s3_obj_client.h b/cloud/src/recycler/s3_obj_client.h deleted file mode 100644 index e53564b6c9f708..00000000000000 --- a/cloud/src/recycler/s3_obj_client.h +++ /dev/null @@ -1,64 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -#pragma once - -#include - -#include "recycler/obj_storage_client.h" - -namespace Aws::S3 { -class S3Client; -} // namespace Aws::S3 - -namespace doris::cloud { - -class S3ObjClient final : public ObjStorageClient { -public: - S3ObjClient(std::shared_ptr client, std::string endpoint) - : s3_client_(std::move(client)), endpoint_(std::move(endpoint)) {} - ~S3ObjClient() override; - - ObjectStorageResponse put_object(ObjectStoragePathRef path, std::string_view stream) override; - - ObjectStorageResponse head_object(ObjectStoragePathRef path, ObjectMeta* res) override; - - std::unique_ptr list_objects(ObjectStoragePathRef path) override; - - ObjectStorageResponse delete_objects(const std::string& bucket, std::vector keys, - ObjClientOptions option) override; - - ObjectStorageResponse delete_object(ObjectStoragePathRef path) override; - - ObjectStorageResponse delete_objects_recursively(ObjectStoragePathRef path, - ObjClientOptions option, - int64_t expiration_time = 0) override; - - ObjectStorageResponse get_life_cycle(const std::string& bucket, - int64_t* expiration_days) override; - - ObjectStorageResponse check_versioning(const std::string& bucket) override; - - ObjectStorageResponse abort_multipart_upload(ObjectStoragePathRef path, - const std::string& upload_id) override; - -private: - std::shared_ptr s3_client_; - std::string endpoint_; -}; - -} // namespace doris::cloud \ No newline at end of file diff --git a/cloud/test/recycler_batch_delete_test.cpp b/cloud/test/recycler_batch_delete_test.cpp index ffb47c8745c859..fb151e37bd2318 100644 --- a/cloud/test/recycler_batch_delete_test.cpp +++ b/cloud/test/recycler_batch_delete_test.cpp @@ -15,389 +15,413 @@ // specific language governing permissions and limitations // under the License. +#include #include #include #include -#include +#include #include +#include #include #include "common/config.h" -#include "common/logging.h" #include "common/simple_thread_pool.h" -#include "recycler/obj_storage_client.h" +#include "cpp/obj-client/obj_storage_client.h" +#include "cpp/obj-client/rate_limited_obj_storage_client.h" +#include "recycler/s3_accessor.h" -using namespace doris; +namespace doris { +namespace { -namespace doris::cloud { - -// Mock ObjectListIterator for testing -class MockObjectListIterator : public ObjectListIterator { +class MockObjStorageClient final : public ObjStorageClient { public: - MockObjectListIterator(std::vector objects, int fail_after = -1) - : objects_(std::move(objects)), fail_after_(fail_after) {} - - bool is_valid() override { return is_valid_; } - - bool has_next() override { - if (!is_valid_) return false; - return current_index_ < objects_.size(); + MockObjStorageClient(std::vector objects, size_t batch_size, + int iterator_fail_after = -1) + : objects_(std::move(objects)), + batch_size_(batch_size), + iterator_fail_after_(iterator_fail_after) {} + + ObjStorageUploadResult create_multipart_upload(const ObjStoragePath&) override { + return {.resp = ObjStorageResponse::OK()}; } - - std::optional next() override { - if (!is_valid_ || current_index_ >= objects_.size()) { - return std::nullopt; + ObjStorageResponse put_object(const ObjStoragePath&, std::string_view) override { + return ObjStorageResponse::OK(); + } + ObjStorageUploadResult upload_part(const ObjStoragePath&, const std::string&, std::string_view, + int) override { + return {.resp = ObjStorageResponse::OK()}; + } + ObjStorageResponse complete_multipart_upload( + const ObjStoragePath&, const std::string&, + const std::vector&) override { + return ObjStorageResponse::OK(); + } + ObjStorageHeadResult head_object(const ObjStoragePath&) override { + return {.resp = ObjStorageResponse::OK()}; + } + ObjStorageResponse get_object(const ObjStoragePath&, void*, size_t, size_t, size_t*) override { + return ObjStorageResponse::OK(); + } + ObjStorageListPageResult list_objects_page(const ObjStoragePath&, + std::string_view continuation_token) override { + ++list_calls_; + const size_t index = + continuation_token.empty() + ? 0 + : static_cast(std::stoull(std::string(continuation_token))); + if (iterator_fail_after_ >= 0 && index >= static_cast(iterator_fail_after_)) { + return {.resp = {.status = {TStatusCode::INTERNAL_ERROR, "simulated list failure"}}}; } - - // Simulate iterator becoming invalid after certain number of calls - if (fail_after_ >= 0 && static_cast(current_index_) >= fail_after_) { - is_valid_ = false; - return std::nullopt; + ObjStorageListPageResult page {.resp = ObjStorageResponse::OK()}; + if (index < objects_.size()) { + page.objects.emplace_back(objects_[index]); + page.has_more = index + 1 < objects_.size(); + if (page.has_more) { + page.continuation_token = std::to_string(index + 1); + } } - - return objects_[current_index_++]; + return page; + } + ObjStorageResponse delete_objects(const ObjStoragePath&, + std::vector keys) override { + const int call = delete_calls_.fetch_add(1); + if (fail_delete_after_.load() >= 0 && call >= fail_delete_after_.load()) { + return {.status = {TStatusCode::INTERNAL_ERROR, "simulated delete failure"}}; + } + std::lock_guard lock(deleted_keys_mutex_); + deleted_keys_.insert(deleted_keys_.end(), keys.begin(), keys.end()); + return ObjStorageResponse::OK(); + } + ObjStorageResponse delete_object(const ObjStoragePath&) override { + return ObjStorageResponse::OK(); + } + std::string generate_presigned_url(const ObjStoragePath&, int64_t) override { return {}; } + ObjStorageCapabilities capabilities() const override { + return {.max_delete_batch = batch_size_}; + } + ObjStorageResponse get_lifecycle(const std::string&, int64_t*) override { + return not_supported(); + } + ObjStorageResponse check_versioning(const std::string&) override { return not_supported(); } + ObjStorageResponse abort_multipart_upload(const ObjStoragePath&, const std::string&) override { + return not_supported(); } - void set_invalid() { is_valid_ = false; } + int delete_calls() const { return delete_calls_.load(); } + int list_calls() const { return list_calls_.load(); } + const std::vector& deleted_keys() const { return deleted_keys_; } + void fail_delete() { fail_delete_after_ = 0; } private: + static ObjStorageResponse not_supported() { + return { + .status = {TStatusCode::NOT_IMPLEMENTED_ERROR, + "operation is not supported by the recycler test client"}, + .http_code = 0, + }; + } + std::vector objects_; - size_t current_index_ = 0; - bool is_valid_ = true; - int fail_after_ = -1; // -1 means never fail + size_t batch_size_; + int iterator_fail_after_; + std::atomic list_calls_ {0}; + std::atomic delete_calls_ {0}; + std::atomic fail_delete_after_ {-1}; + std::mutex deleted_keys_mutex_; + std::vector deleted_keys_; }; -// Mock ObjStorageClient for testing delete_objects_recursively_ -class MockObjStorageClient : public ObjStorageClient { +class CountingDeleteExecutor final : public ObjStorageDeleteExecutor { public: - MockObjStorageClient(std::vector objects, int iterator_fail_after = -1) - : objects_(std::move(objects)), iterator_fail_after_(iterator_fail_after) {} - - ObjectStorageResponse put_object(ObjectStoragePathRef path, std::string_view stream) override { - return {0}; - } + CountingDeleteExecutor(std::shared_ptr inner, size_t* wait_calls) + : inner_(std::move(inner)), wait_calls_(wait_calls) {} - ObjectStorageResponse head_object(ObjectStoragePathRef path, ObjectMeta* res) override { - return {0}; + ObjStorageResponse submit(ObjStorageDeleteTask task) override { + return inner_->submit(std::move(task)); } - std::unique_ptr list_objects(ObjectStoragePathRef path) override { - return std::make_unique(objects_, iterator_fail_after_); + ObjStorageResponse wait() override { + ++*wait_calls_; + return inner_->wait(); } - ObjectStorageResponse delete_objects(const std::string& bucket, std::vector keys, - ObjClientOptions option) override { - delete_calls_++; - total_keys_deleted_ += keys.size(); +private: + std::shared_ptr inner_; + size_t* wait_calls_; +}; - // Simulate delete failure if configured - if (fail_delete_after_ >= 0 && delete_calls_ > fail_delete_after_) { - return {-1, "simulated delete failure"}; - } +class StreamingDeleteExecutor final : public ObjStorageDeleteExecutor { +public: + StreamingDeleteExecutor(std::shared_ptr client, + std::vector* list_calls_at_submit) + : client_(std::move(client)), list_calls_at_submit_(list_calls_at_submit) {} - return {0}; + ObjStorageResponse submit(ObjStorageDeleteTask task) override { + list_calls_at_submit_->push_back(client_->list_calls()); + return task(); } - ObjectStorageResponse delete_object(ObjectStoragePathRef path) override { return {0}; } - - ObjectStorageResponse delete_objects_recursively(ObjectStoragePathRef path, - ObjClientOptions option, - int64_t expiration_time = 0) override { - return delete_objects_recursively_(path, option, expiration_time, 1000); - } + ObjStorageResponse wait() override { return ObjStorageResponse::OK(); } - ObjectStorageResponse get_life_cycle(const std::string& bucket, - int64_t* expiration_days) override { - return {0}; - } +private: + std::shared_ptr client_; + std::vector* list_calls_at_submit_; +}; - ObjectStorageResponse check_versioning(const std::string& bucket) override { return {0}; } +class TestS3Accessor final : public cloud::S3Accessor { +public: + using cloud::S3Accessor::make_recursive_delete_options; +}; - ObjectStorageResponse abort_multipart_upload(ObjectStoragePathRef path, - const std::string& upload_id) override { - return {0}; +class ScopedMaxTasksPerBatch { +public: + explicit ScopedMaxTasksPerBatch(int32_t value) + : original_(cloud::config::recycler_max_tasks_per_batch) { + cloud::config::recycler_max_tasks_per_batch = value; } - - // Test helper methods - int get_delete_calls() const { return delete_calls_; } - size_t get_total_keys_deleted() const { return total_keys_deleted_; } - void set_fail_delete_after(int n) { fail_delete_after_ = n; } + ~ScopedMaxTasksPerBatch() { cloud::config::recycler_max_tasks_per_batch = original_; } private: - std::vector objects_; - int iterator_fail_after_ = -1; - std::atomic delete_calls_ {0}; - std::atomic total_keys_deleted_ {0}; - int fail_delete_after_ = -1; // -1 means never fail + int32_t original_; }; -class RecyclerBatchDeleteTest : public testing::Test { -protected: - void SetUp() override { - thread_pool_ = std::make_shared(4); - thread_pool_->start(); - } - - void TearDown() override { - if (thread_pool_) { - thread_pool_->stop(); - } +class CapturingLogSink final : public google::LogSink { +public: + void send(google::LogSeverity, const char*, const char*, int, const google::LogMessageTime&, + const char* message, std::size_t message_len) override { + std::lock_guard lock(mutex_); + messages_.emplace_back(message, message_len); } - std::vector generate_objects(size_t count) { - std::vector objects; - objects.reserve(count); - for (size_t i = 0; i < count; ++i) { - objects.push_back(ObjectMeta { - .key = "test_key_" + std::to_string(i), - .size = 100, - .mtime_s = 0, - }); + bool contains(std::string_view text) const { + std::lock_guard lock(mutex_); + for (const auto& message : messages_) { + if (message.find(text) != std::string::npos) { + return true; + } } - return objects; + return false; } - std::shared_ptr thread_pool_; +private: + mutable std::mutex mutex_; + std::vector messages_; }; -// Test 1: Basic batch processing with multiple batches -TEST_F(RecyclerBatchDeleteTest, MultipleBatches) { - // Save original config and set small batch size for testing - int32_t original_config = config::recycler_max_tasks_per_batch; - config::recycler_max_tasks_per_batch = 3; // 3 tasks per batch - - // Create 10 objects, with batch_size=2 (keys per task), max_tasks_per_batch=3 - // Expected: 10 objects / 2 keys per task = 5 tasks - // 5 tasks / 3 tasks per batch = 2 batches (3 tasks + 2 tasks) - auto objects = generate_objects(10); - MockObjStorageClient client(objects); +class ScopedLogSink final { +public: + explicit ScopedLogSink(google::LogSink* sink) : sink_(sink) { google::AddLogSink(sink_); } + ~ScopedLogSink() { google::RemoveLogSink(sink_); } - ObjClientOptions options; - options.executor = thread_pool_; +private: + google::LogSink* sink_; +}; - // Use batch_size=2 to create more tasks - auto response = client.delete_objects_recursively_( - {.bucket = "test_bucket", .key = "test_prefix"}, options, 0, 2); +class CountingRateLimitPolicy final : public ObjStorageRateLimitPolicy { +public: + CountingRateLimitPolicy(size_t* get_requests, size_t* put_requests) + : get_requests_(get_requests), put_requests_(put_requests) {} + + ObjStorageAdmission acquire(S3RateLimitType type, size_t) const override { + if (type == S3RateLimitType::GET) { + ++*get_requests_; + } else { + ++*put_requests_; + } + return {}; + } - EXPECT_EQ(response.ret, 0); - EXPECT_EQ(client.get_delete_calls(), 5); // 10 objects / 2 = 5 delete calls - EXPECT_EQ(client.get_total_keys_deleted(), 10); // All 10 keys deleted +private: + size_t* get_requests_; + size_t* put_requests_; +}; - // Restore config - config::recycler_max_tasks_per_batch = original_config; +std::vector make_objects(size_t count) { + std::vector objects; + for (size_t i = 0; i < count; ++i) { + objects.push_back({ + .key = "test_key_" + std::to_string(i), + .size = 100, + .mtime_s = static_cast(i), + }); + } + return objects; } -// Test 2: Iterator becomes invalid during iteration -TEST_F(RecyclerBatchDeleteTest, IteratorInvalidMidway) { - int32_t original_config = config::recycler_max_tasks_per_batch; - config::recycler_max_tasks_per_batch = 100; - - // Create 20 objects but iterator fails after 10 - auto objects = generate_objects(20); - MockObjStorageClient client(objects, 10); // fail_after=10 - - ObjClientOptions options; - options.executor = thread_pool_; +TEST(RecyclerBatchDeleteTest, UsesProviderBatchCapability) { + auto provider_client = std::make_shared(make_objects(10), 3); + auto response = delete_objects_recursively(provider_client, + {.bucket = "bucket", .prefix = "test_key_"}); - auto response = client.delete_objects_recursively_( - {.bucket = "test_bucket", .key = "test_prefix"}, options, 0, 5); - - // Should return error because iterator became invalid - EXPECT_EQ(response.ret, -1); - // Should have processed some objects before failure - EXPECT_GT(client.get_total_keys_deleted(), 0); - EXPECT_LT(client.get_total_keys_deleted(), 20); - - config::recycler_max_tasks_per_batch = original_config; + EXPECT_TRUE(response.ok()); + EXPECT_EQ(provider_client->delete_calls(), 4); + EXPECT_EQ(provider_client->deleted_keys().size(), 10); } -// Test 3: Delete operation fails (triggers cancel) -TEST_F(RecyclerBatchDeleteTest, DeleteFailureTriggersCancel) { - int32_t original_config = config::recycler_max_tasks_per_batch; - config::recycler_max_tasks_per_batch = 10; - - auto objects = generate_objects(30); - MockObjStorageClient client(objects); - client.set_fail_delete_after(2); // Fail after 2 successful deletes +TEST(RecyclerBatchDeleteTest, CountsDeleteObjectsOperationOnce) { + auto inner_client = std::make_shared(std::vector {}, 3); + size_t get_requests = 0; + size_t put_requests = 0; + RateLimitedObjStorageClient client( + inner_client, std::make_shared(&get_requests, &put_requests)); - ObjClientOptions options; - options.executor = thread_pool_; + std::vector keys(7, "key"); + auto response = client.delete_objects({.bucket = "bucket"}, std::move(keys)); - auto response = client.delete_objects_recursively_( - {.bucket = "test_bucket", .key = "test_prefix"}, options, 0, 5); - - // Should return error because delete failed - EXPECT_EQ(response.ret, -1); - - config::recycler_max_tasks_per_batch = original_config; + EXPECT_TRUE(response.ok()); + EXPECT_EQ(get_requests, 0); + EXPECT_EQ(put_requests, 1); + EXPECT_EQ(inner_client->delete_calls(), 1); } -// Test 4: Empty object list -TEST_F(RecyclerBatchDeleteTest, EmptyObjectList) { - int32_t original_config = config::recycler_max_tasks_per_batch; - config::recycler_max_tasks_per_batch = 100; - - std::vector empty_objects; - MockObjStorageClient client(empty_objects); +TEST(RecyclerBatchDeleteTest, RateLimitsEveryRecursiveDeleteRequest) { + auto inner_client = std::make_shared(make_objects(5), 2); + size_t get_requests = 0; + size_t put_requests = 0; + auto client = std::make_shared( + inner_client, std::make_shared(&get_requests, &put_requests)); - ObjClientOptions options; - options.executor = thread_pool_; + auto response = delete_objects_recursively(client, {.bucket = "bucket", .prefix = "test_key_"}); - auto response = client.delete_objects_recursively_( - {.bucket = "test_bucket", .key = "test_prefix"}, options, 0, 1000); - - EXPECT_EQ(response.ret, 0); - EXPECT_EQ(client.get_delete_calls(), 0); - EXPECT_EQ(client.get_total_keys_deleted(), 0); - - config::recycler_max_tasks_per_batch = original_config; + EXPECT_TRUE(response.ok()); + EXPECT_EQ(get_requests, 5); + EXPECT_EQ(put_requests, 3); + EXPECT_EQ(inner_client->list_calls(), 5); + EXPECT_EQ(inner_client->delete_calls(), 3); } -// Test 5: Objects less than batch_size -TEST_F(RecyclerBatchDeleteTest, ObjectsLessThanBatchSize) { - int32_t original_config = config::recycler_max_tasks_per_batch; - config::recycler_max_tasks_per_batch = 100; - - auto objects = generate_objects(5); - MockObjStorageClient client(objects); - - ObjClientOptions options; - options.executor = thread_pool_; - - // batch_size=1000, but only 5 objects - auto response = client.delete_objects_recursively_( - {.bucket = "test_bucket", .key = "test_prefix"}, options, 0, 1000); - - EXPECT_EQ(response.ret, 0); - EXPECT_EQ(client.get_delete_calls(), 1); // All 5 keys in one delete call - EXPECT_EQ(client.get_total_keys_deleted(), 5); - - config::recycler_max_tasks_per_batch = original_config; +TEST(RecyclerBatchDeleteTest, ProductionExecutorRunsMultipleTaskBatches) { + ScopedMaxTasksPerBatch max_tasks_per_batch(2); + auto pool = std::make_shared(4, "recursive_delete_test"); + ASSERT_EQ(pool->start(), 0); + + auto options = TestS3Accessor::make_recursive_delete_options(0, pool); + size_t executor_batches = 0; + options.executor = + std::make_shared(options.executor, &executor_batches); + + auto provider_client = std::make_shared(make_objects(10), 2); + auto response = delete_objects_recursively( + provider_client, {.bucket = "bucket", .prefix = "test_key_"}, options); + + EXPECT_TRUE(response.ok()); + EXPECT_EQ(executor_batches, 3); + EXPECT_EQ(provider_client->delete_calls(), 5); + EXPECT_EQ(provider_client->deleted_keys().size(), 10); + EXPECT_EQ(pool->stop(), 0); } -// Test 6: Exact batch boundary -TEST_F(RecyclerBatchDeleteTest, ExactBatchBoundary) { - int32_t original_config = config::recycler_max_tasks_per_batch; - config::recycler_max_tasks_per_batch = 2; // 2 tasks per batch - - // 8 objects with batch_size=2 = 4 tasks - // 4 tasks with max_tasks_per_batch=2 = exactly 2 batches - auto objects = generate_objects(8); - MockObjStorageClient client(objects); - - ObjClientOptions options; - options.executor = thread_pool_; - - auto response = client.delete_objects_recursively_( - {.bucket = "test_bucket", .key = "test_prefix"}, options, 0, 2); - - EXPECT_EQ(response.ret, 0); - EXPECT_EQ(client.get_delete_calls(), 4); // 8 / 2 = 4 tasks - EXPECT_EQ(client.get_total_keys_deleted(), 8); - - config::recycler_max_tasks_per_batch = original_config; +TEST(RecyclerBatchDeleteTest, LogsBatchProgressAndFinalSummary) { + CapturingLogSink logs; + ScopedLogSink scoped_log_sink(&logs); + auto provider_client = std::make_shared(make_objects(5), 2); + + auto response = + delete_objects_recursively(provider_client, {.bucket = "bucket", .prefix = "test_key_"}, + {.max_tasks_per_batch = 2}); + + EXPECT_TRUE(response.ok()); + EXPECT_TRUE( + logs.contains("delete objects under bucket/test_key_ batch 1 completed, " + "tasks_in_batch=2, total_deleted=4")); + EXPECT_TRUE( + logs.contains("delete objects under bucket/test_key_ batch 2 completed, " + "tasks_in_batch=1, total_deleted=5")); + EXPECT_TRUE( + logs.contains("delete objects under bucket/test_key_ finished, ret=0, " + "total_batches=2, num_deleted=5, error_count=0")); } -// Test 7: Invalid config value (negative) -TEST_F(RecyclerBatchDeleteTest, InvalidConfigNegative) { - int32_t original_config = config::recycler_max_tasks_per_batch; - config::recycler_max_tasks_per_batch = -1; // Invalid negative value - - auto objects = generate_objects(10); - MockObjStorageClient client(objects); - - ObjClientOptions options; - options.executor = thread_pool_; - - // Should use default value 1000 and still work - auto response = client.delete_objects_recursively_( - {.bucket = "test_bucket", .key = "test_prefix"}, options, 0, 5); - - EXPECT_EQ(response.ret, 0); - EXPECT_EQ(client.get_total_keys_deleted(), 10); - - config::recycler_max_tasks_per_batch = original_config; +TEST(RecyclerBatchDeleteTest, StreamsDeleteTasksWhileListing) { + auto provider_client = std::make_shared(make_objects(5), 1); + std::vector list_calls_at_submit; + ObjStorageRecursiveDeleteOptions options { + .max_tasks_per_batch = 1000, + .executor = std::make_shared(provider_client, + &list_calls_at_submit), + }; + + auto response = delete_objects_recursively( + provider_client, {.bucket = "bucket", .prefix = "test_key_"}, options); + + EXPECT_TRUE(response.ok()); + ASSERT_EQ(list_calls_at_submit.size(), 5); + EXPECT_EQ(list_calls_at_submit.front(), 1); + EXPECT_EQ(provider_client->list_calls(), 5); + EXPECT_EQ(provider_client->deleted_keys().size(), 5); } -// Test 8: Invalid config value (zero) -TEST_F(RecyclerBatchDeleteTest, InvalidConfigZero) { - int32_t original_config = config::recycler_max_tasks_per_batch; - config::recycler_max_tasks_per_batch = 0; // Invalid zero value - - auto objects = generate_objects(10); - MockObjStorageClient client(objects); - - ObjClientOptions options; - options.executor = thread_pool_; - - // Should use default value 1000 and still work - auto response = client.delete_objects_recursively_( - {.bucket = "test_bucket", .key = "test_prefix"}, options, 0, 5); - - EXPECT_EQ(response.ret, 0); - EXPECT_EQ(client.get_total_keys_deleted(), 10); - - config::recycler_max_tasks_per_batch = original_config; +TEST(RecyclerBatchDeleteTest, ProductionExecutorStopsListingAfterCancellation) { + ScopedMaxTasksPerBatch max_tasks_per_batch(3); + auto pool = std::make_shared(1, "recursive_delete_failure_test"); + ASSERT_EQ(pool->start(), 0); + + auto provider_client = std::make_shared(make_objects(6), 1); + provider_client->fail_delete(); + auto response = + delete_objects_recursively(provider_client, {.bucket = "bucket", .prefix = "test_key_"}, + TestS3Accessor::make_recursive_delete_options(0, pool)); + + EXPECT_FALSE(response.ok()); + EXPECT_EQ(response.status.msg, "object storage batch deletion did not finish"); + EXPECT_EQ(provider_client->list_calls(), 3); + EXPECT_EQ(provider_client->delete_calls(), 1); + EXPECT_EQ(pool->stop(), 0); } -// Test 9: Expiration time filtering -TEST_F(RecyclerBatchDeleteTest, ExpirationTimeFiltering) { - int32_t original_config = config::recycler_max_tasks_per_batch; - config::recycler_max_tasks_per_batch = 100; - - std::vector objects; - // Create 10 objects: 5 with old mtime (should be deleted), 5 with new mtime (should be kept) - for (int i = 0; i < 5; ++i) { - objects.push_back(ObjectMeta { - .key = "old_key_" + std::to_string(i), - .size = 100, - .mtime_s = 100, // Old timestamp - }); +TEST(RecyclerBatchDeleteTest, InvalidMaxTasksPerBatchUsesDefault) { + CapturingLogSink logs; + ScopedLogSink scoped_log_sink(&logs); + auto pool = std::make_shared(1, "recursive_delete_config_test"); + { + ScopedMaxTasksPerBatch max_tasks_per_batch(0); + auto options = TestS3Accessor::make_recursive_delete_options(0, pool); + EXPECT_EQ(options.max_tasks_per_batch, 1000); } - for (int i = 0; i < 5; ++i) { - objects.push_back(ObjectMeta { - .key = "new_key_" + std::to_string(i), - .size = 100, - .mtime_s = 1000, // New timestamp - }); + { + ScopedMaxTasksPerBatch max_tasks_per_batch(-1); + auto options = TestS3Accessor::make_recursive_delete_options(0, pool); + EXPECT_EQ(options.max_tasks_per_batch, 1000); } - - MockObjStorageClient client(objects); - - ObjClientOptions options; - options.executor = thread_pool_; - - // Set expiration_time=500, so only objects with mtime_s <= 500 should be deleted - auto response = client.delete_objects_recursively_( - {.bucket = "test_bucket", .key = "test_prefix"}, options, 500, 1000); - - EXPECT_EQ(response.ret, 0); - EXPECT_EQ(client.get_total_keys_deleted(), 5); // Only old objects deleted - - config::recycler_max_tasks_per_batch = original_config; + EXPECT_TRUE( + logs.contains("recycler_max_tasks_per_batch=0 is not positive, using default 1000")); + EXPECT_TRUE( + logs.contains("recycler_max_tasks_per_batch=-1 is not positive, using default 1000")); } -// Test 10: Iterator invalid at start (empty batch scenario) -TEST_F(RecyclerBatchDeleteTest, IteratorInvalidAtStart) { - int32_t original_config = config::recycler_max_tasks_per_batch; - config::recycler_max_tasks_per_batch = 100; - - // Iterator fails immediately (fail_after=0) - auto objects = generate_objects(10); - MockObjStorageClient client(objects, 0); +TEST(RecyclerBatchDeleteTest, FiltersByExpirationTime) { + auto provider_client = std::make_shared(make_objects(10), 1000); + auto response = delete_objects_recursively( + provider_client, {.bucket = "bucket", .prefix = "test_key_"}, {.expiration_time = 4}); - ObjClientOptions options; - options.executor = thread_pool_; - - auto response = client.delete_objects_recursively_( - {.bucket = "test_bucket", .key = "test_prefix"}, options, 0, 5); - - // Should return error because iterator was invalid from the start - EXPECT_EQ(response.ret, -1); - EXPECT_EQ(client.get_delete_calls(), 0); + EXPECT_TRUE(response.ok()); + ASSERT_EQ(provider_client->deleted_keys().size(), 5); + EXPECT_EQ(provider_client->deleted_keys().back(), "test_key_4"); +} - config::recycler_max_tasks_per_batch = original_config; +TEST(RecyclerBatchDeleteTest, PropagatesListAndDeleteFailures) { + CapturingLogSink logs; + ScopedLogSink scoped_log_sink(&logs); + auto list_failure_provider_client = + std::make_shared(make_objects(10), 3, 2); + EXPECT_FALSE(delete_objects_recursively(list_failure_provider_client, + {.bucket = "bucket", .prefix = "list_failure"}) + .ok()); + + auto delete_failure_provider_client = + std::make_shared(make_objects(7), 3); + delete_failure_provider_client->fail_delete(); + auto delete_response = delete_objects_recursively( + delete_failure_provider_client, {.bucket = "bucket", .prefix = "delete_failure"}); + EXPECT_FALSE(delete_response.ok()); + EXPECT_EQ(delete_response.status.msg, "simulated delete failure"); + EXPECT_EQ(delete_failure_provider_client->list_calls(), 3); + EXPECT_EQ(delete_failure_provider_client->delete_calls(), 1); + EXPECT_TRUE(logs.contains("delete objects under bucket/list_failure finished, ret=")); + EXPECT_TRUE(logs.contains("delete objects under bucket/delete_failure finished, ret=")); + EXPECT_TRUE(logs.contains("total_batches=1, num_deleted=3, error_count=1")); } -} // namespace doris::cloud +} // namespace +} // namespace doris diff --git a/cloud/test/s3_accessor_mock_test.cpp b/cloud/test/s3_accessor_mock_test.cpp index 36ceb1da1cc0d2..26f4dbe816e927 100644 --- a/cloud/test/s3_accessor_mock_test.cpp +++ b/cloud/test/s3_accessor_mock_test.cpp @@ -31,10 +31,10 @@ #include "common/config.h" #include "common/logging.h" #include "cpp/aws_common.h" +#include "cpp/obj-client/s3_obj_storage_client.h" #include "cpp/sync_point.h" #include "cpp/test/container_credentials_test_util.h" #include "recycler/s3_accessor.h" -#include "recycler/s3_obj_client.h" using namespace doris; using namespace Aws::S3::Model; @@ -77,18 +77,21 @@ TEST_F(S3AccessorMockTest, list_objects_compatibility) { // If storage only supports ListObjectsV1, s3_obj_storage_client.list_objects // should return an error. auto mock_s3_client = std::make_shared(); - S3ObjClient s3_obj_client(mock_s3_client, "dummy-endpoint"); + auto s3_obj_client = std::make_shared( + mock_s3_client, ObjStorageEndpointInfo {.endpoint = "dummy-endpoint"}); ListObjectsV2Result result; result.SetIsTruncated(true); EXPECT_CALL(*mock_s3_client, ListObjectsV2(testing::_)) .WillOnce(testing::Return(ListObjectsV2Outcome(result))); - auto response = s3_obj_client.list_objects( - {.bucket = "dummy-bucket", .key = "S3AccessorMockTest/list_objects_compatibility"}); + std::vector objects; + auto response = s3_obj_client->list_objects( + {.bucket = "dummy-bucket", .key = "S3AccessorMockTest/list_objects_compatibility"}, + &objects); - EXPECT_FALSE(response->has_next()); - EXPECT_FALSE(response->is_valid()); + EXPECT_FALSE(response.ok()); + EXPECT_TRUE(objects.empty()); } namespace { @@ -97,14 +100,14 @@ class ProviderProbe : public S3Accessor { public: ProviderProbe() : S3Accessor(S3Conf {}) {} - // The recycler builds its provider through _get_aws_credentials_provider_v2(), so the probe + // The recycler builds its provider through create_aws_credentials_provider(), so the probe // goes through the same call rather than reaching past it. An S3Conf with no ak/sk and no // role_arn is what a vault configured for container credentials looks like, and it makes the // factory return the CONTAINER base provider unwrapped. std::shared_ptr container_provider() { S3Conf conf; conf.cred_provider_type = CredProviderType::Container; - return _get_aws_credentials_provider_v2(conf); + return create_aws_credentials_provider(conf).provider; } }; diff --git a/cloud/test/s3_accessor_test.cpp b/cloud/test/s3_accessor_test.cpp index 62a606d768ed72..ef69a0d45ab647 100644 --- a/cloud/test/s3_accessor_test.cpp +++ b/cloud/test/s3_accessor_test.cpp @@ -220,9 +220,10 @@ TEST_F(S3AccessorTest, s3) { auto* sp = SyncPoint::get_instance(); std::vector guards; sp->set_call_back( - "S3ObjListIterator", + "S3ObjStorageClient::list_objects", [](auto&& args) { auto* req = try_any_cast(args[0]); + EXPECT_EQ(req->GetMaxKeys(), 1000); req->SetMaxKeys(7); }, &guards.emplace_back()); @@ -262,9 +263,11 @@ TEST_F(S3AccessorTest, azure) { auto* sp = SyncPoint::get_instance(); std::vector guards; sp->set_call_back( - "AzureListIterator", + "AzureObjStorageClient::list_objects", [](auto&& args) { auto* req = try_any_cast(args[0]); + ASSERT_TRUE(req->PageSizeHint.HasValue()); + EXPECT_EQ(req->PageSizeHint.Value(), 5000); req->PageSizeHint = 7; }, &guards.emplace_back()); @@ -304,7 +307,7 @@ TEST_F(S3AccessorTest, gcs) { auto* sp = SyncPoint::get_instance(); std::vector guards; sp->set_call_back( - "S3ObjListIterator", + "S3ObjStorageClient::list_objects", [](auto&& args) { auto* req = try_any_cast(args[0]); req->SetMaxKeys(7); @@ -488,7 +491,7 @@ TEST_F(S3AccessorRoleTest, s3) { auto* sp = SyncPoint::get_instance(); std::vector guards; sp->set_call_back( - "S3ObjListIterator", + "S3ObjStorageClient::list_objects", [](auto&& args) { auto* req = try_any_cast(args[0]); req->SetMaxKeys(7); diff --git a/common/cpp/CMakeLists.txt b/common/cpp/CMakeLists.txt index 81a600e57bf84d..6a24df1bcfaf4b 100644 --- a/common/cpp/CMakeLists.txt +++ b/common/cpp/CMakeLists.txt @@ -19,4 +19,9 @@ set(LIBRARY_OUTPUT_PATH "${BUILD_DIR}/src/common_cpp") file(GLOB_RECURSE SRC_FILES CONFIGURE_DEPENDS *.cpp) +if(NOT BUILD_AZURE STREQUAL "ON") + list(REMOVE_ITEM SRC_FILES + "${CMAKE_CURRENT_SOURCE_DIR}/obj-client/azure_obj_storage_client.cpp" + "${CMAKE_CURRENT_SOURCE_DIR}/obj-client/auth/azure_auth_factory.cpp") +endif() add_library(CommonCPP STATIC ${SRC_FILES}) diff --git a/common/cpp/obj-client/auth/aws_credential_factory.cpp b/common/cpp/obj-client/auth/aws_credential_factory.cpp new file mode 100644 index 00000000000000..6e8c3b8ce5302d --- /dev/null +++ b/common/cpp/obj-client/auth/aws_credential_factory.cpp @@ -0,0 +1,106 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "aws_credential_factory.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "cpp/custom_aws_credentials_provider_chain.h" + +namespace doris { +namespace { + +using Provider = Aws::Auth::AWSCredentialsProvider; + +std::shared_ptr create_v2_base_provider(CredProviderType type) { + switch (type) { + case CredProviderType::Env: + return std::make_shared(); + case CredProviderType::SystemProperties: + return std::make_shared(); + case CredProviderType::WebIdentity: + return std::make_shared(); + case CredProviderType::Container: + return create_container_credentials_provider(); + case CredProviderType::Anonymous: + return std::make_shared(); + case CredProviderType::Default: + case CredProviderType::Simple: + return std::make_shared(); + case CredProviderType::InstanceProfile: + return std::make_shared(); + } + __builtin_unreachable(); +} + +AwsCredentialResult assume_role(const AwsCredentialOptions& options, + std::shared_ptr base_provider) { + auto sts_client = + std::make_shared(base_provider, options.sts_client_config); + return { + .provider = std::make_shared( + options.role_arn, Aws::String(), options.external_id, + Aws::Auth::DEFAULT_CREDS_LOAD_FREQ_SECONDS, std::move(sts_client)), + }; +} + +} // namespace + +AwsCredentialResult AwsCredentialFactory::create(const AwsCredentialOptions& options) { + const bool has_access_key = !options.access_key.empty(); + const bool has_secret_key = !options.secret_key.empty(); + + if (has_access_key && has_secret_key) { + Aws::Auth::AWSCredentials credentials(options.access_key, options.secret_key); + if (!options.session_token.empty()) { + credentials.SetSessionToken(options.session_token); + } + return { + .provider = std::make_shared( + std::move(credentials)), + }; + } + + if (options.version == AwsCredentialProviderVersion::V1) { + if (options.provider_type == CredProviderType::InstanceProfile) { + auto base = std::make_shared(); + return options.role_arn.empty() ? AwsCredentialResult {.provider = std::move(base)} + : assume_role(options, std::move(base)); + } + if (!has_access_key && !has_secret_key && + options.empty_credentials == EmptyCredentialsBehavior::ANONYMOUS) { + return { + .provider = std::make_shared(), + }; + } + return { + .provider = std::make_shared(), + }; + } + + auto base = create_v2_base_provider(options.provider_type); + return options.role_arn.empty() ? AwsCredentialResult {.provider = std::move(base)} + : assume_role(options, std::move(base)); +} + +} // namespace doris diff --git a/common/cpp/obj-client/auth/aws_credential_factory.h b/common/cpp/obj-client/auth/aws_credential_factory.h new file mode 100644 index 00000000000000..f5d528fcfd0e51 --- /dev/null +++ b/common/cpp/obj-client/auth/aws_credential_factory.h @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include + +#include "cpp/aws_common.h" + +namespace Aws::Auth { +class AWSCredentialsProvider; +} + +namespace doris { + +enum class AwsCredentialProviderVersion { + V1, + V2, +}; + +enum class EmptyCredentialsBehavior { + ANONYMOUS, + DEFAULT_CHAIN, +}; + +struct AwsCredentialOptions { + AwsCredentialProviderVersion version = AwsCredentialProviderVersion::V1; + std::string access_key; + std::string secret_key; + std::string session_token; + CredProviderType provider_type = CredProviderType::Default; + std::string role_arn; + std::string external_id; + EmptyCredentialsBehavior empty_credentials = EmptyCredentialsBehavior::DEFAULT_CHAIN; + Aws::Client::ClientConfiguration sts_client_config; +}; + +struct AwsCredentialResult { + std::shared_ptr provider {}; + std::string error {}; + + explicit operator bool() const { return provider != nullptr; } +}; + +class AwsCredentialFactory { +public: + static AwsCredentialResult create(const AwsCredentialOptions& options); +}; + +} // namespace doris diff --git a/common/cpp/obj-client/auth/azure_auth_factory.cpp b/common/cpp/obj-client/auth/azure_auth_factory.cpp new file mode 100644 index 00000000000000..3e30546a5529db --- /dev/null +++ b/common/cpp/obj-client/auth/azure_auth_factory.cpp @@ -0,0 +1,41 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "azure_auth_factory.h" + +#include + +namespace doris { + +AzureClientBuildResult AzureAuthFactory::create( + std::string_view container_url, const AzureCredentialOptions& credential, + Azure::Storage::Blobs::BlobClientOptions client_options) { + if (credential.type != AzureCredentialType::SHARED_KEY) { + return {.error = "unsupported Azure credential type"}; + } + + auto shared_key = std::make_shared( + credential.account_name, credential.account_key); + auto client = std::make_shared( + std::string(container_url), shared_key, std::move(client_options)); + return { + .container_client = std::move(client), + .shared_key_credential = std::move(shared_key), + }; +} + +} // namespace doris diff --git a/common/cpp/obj-client/auth/azure_auth_factory.h b/common/cpp/obj-client/auth/azure_auth_factory.h new file mode 100644 index 00000000000000..f41e6b8aefc198 --- /dev/null +++ b/common/cpp/obj-client/auth/azure_auth_factory.h @@ -0,0 +1,56 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include + +namespace Azure::Storage { +class StorageSharedKeyCredential; +} + +namespace doris { + +enum class AzureCredentialType { + SHARED_KEY, +}; + +struct AzureCredentialOptions { + AzureCredentialType type = AzureCredentialType::SHARED_KEY; + std::string account_name; + std::string account_key; +}; + +struct AzureClientBuildResult { + std::shared_ptr container_client {}; + std::shared_ptr shared_key_credential {}; + std::string error {}; + + explicit operator bool() const { return container_client != nullptr; } +}; + +class AzureAuthFactory { +public: + static AzureClientBuildResult create(std::string_view container_url, + const AzureCredentialOptions& credential, + Azure::Storage::Blobs::BlobClientOptions client_options); +}; + +} // namespace doris diff --git a/common/cpp/obj-client/azure_obj_storage_client.cpp b/common/cpp/obj-client/azure_obj_storage_client.cpp new file mode 100644 index 00000000000000..7e2f02f243bd3a --- /dev/null +++ b/common/cpp/obj-client/azure_obj_storage_client.cpp @@ -0,0 +1,541 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "azure_obj_storage_client.h" + +#include + +#include +#include +#include +#include + +#include "cpp/obj_retry_strategy.h" + +using namespace Azure::Storage::Blobs; + +namespace { +std::string wrap_object_storage_path_msg(const doris::ObjStoragePath& opts) { + return fmt::format("bucket {}, key {}, prefix {}, path {}", opts.bucket, opts.key, opts.prefix, + opts.path.native()); +} + +doris::ObjStoragePath with_object_key(const doris::ObjStoragePath& opts, std::string_view key) { + auto path = opts; + path.key = key; + return path; +} + +std::string to_lower_ascii(std::string_view input) { + std::string lowered(input); + std::transform(lowered.begin(), lowered.end(), lowered.begin(), + [](unsigned char ch) { return static_cast(std::tolower(ch)); }); + return lowered; +} + +std::string encode_azure_block_id(std::string_view upload_id, int part_num) { + // Azure has no multipart upload namespace. Include the writer UUID in every block ID so + // concurrent writers for the same key cannot stage interchangeable blocks. + std::vector raw_id(upload_id.begin(), upload_id.end()); + auto part = static_cast(part_num); + for (size_t i = 0; i < sizeof(part); ++i) { + raw_id.push_back(static_cast(part >> (i * 8))); + } + Aws::Utils::ByteBuffer bytes(raw_id.data(), raw_id.size()); + return Aws::Utils::HashingUtils::Base64Encode(bytes); +} + +constexpr char SAS_TOKEN_URL_TEMPLATE[] = "{}/{}/{}{}"; +constexpr char BlobNotFound[] = "BlobNotFound"; +} // namespace + +namespace doris { + +std::string azure_multipart_block_id(std::string_view upload_id, int part_num) { + return encode_azure_block_id(upload_id, part_num); +} + +std::string build_azure_batch_delete_failure_message(const ObjStoragePath& opts, + std::string_view key) { + return fmt::format("Azure batch delete failed, path msg {}", + wrap_object_storage_path_msg(with_object_key(opts, key))); +} + +// As Azure's doc said, the batch size is 256 +// You can find out the num in https://learn.microsoft.com/en-us/rest/api/storageservices/blob-batch?tabs=microsoft-entra-id +// > Each batch request supports a maximum of 256 subrequests. +constexpr size_t BlobBatchMaxOperations = 256; + +bool is_azure_tls_ca_error_message(std::string_view message) { + std::string lower = to_lower_ascii(message); + return lower.find("ssl ca cert") != std::string::npos || + lower.find("peer failed verification") != std::string::npos || + lower.find("unable to get local issuer certificate") != std::string::npos || + lower.find("problem with the ssl ca cert") != std::string::npos; +} + +std::string build_azure_tls_debug_suffix(std::string_view error_message, + std::string_view tls_debug_context) { + if (tls_debug_context.empty() || !is_azure_tls_ca_error_message(error_message)) { + return ""; + } + return fmt::format(", {}", tls_debug_context); +} + +static ObjStorageResponse make_azure_std_exception_response(const std::exception& e, + const ObjStoragePath& opts, + std::string_view tls_debug_context) { + auto msg = fmt::format("Azure request failed because {}, path msg {}{}", e.what(), + wrap_object_storage_path_msg(opts), + build_azure_tls_debug_suffix(e.what(), tls_debug_context)); + LOG(WARNING) << msg; + return {.status = ObjStorageStatus {ObjStorageStatus::INTERNAL_ERROR, std::move(msg)}, + .http_code = 0, + .request_id = ""}; +} + +template +ObjStorageResponse do_azure_client_call(Func f, const ObjStoragePath& opts, + std::string_view tls_debug_context) { + try { + f(); + } catch (Azure::Core::RequestFailedException& e) { + doris::record_object_request_failed(static_cast(e.StatusCode)); + auto msg = fmt::format( + "Azure request failed because {}, error msg {}, http code {}, path msg {}{}", + e.what(), e.Message, static_cast(e.StatusCode), + wrap_object_storage_path_msg(opts), + build_azure_tls_debug_suffix(fmt::format("{} {}", e.what(), e.Message), + tls_debug_context)); + LOG(WARNING) << msg; + return {.status = obj_storage_status_from_http_code(static_cast(e.StatusCode), + std::move(msg)), + .http_code = static_cast(e.StatusCode), + .request_id = std::move(e.RequestId)}; + } catch (const std::exception& e) { + return make_azure_std_exception_response(e, opts, tls_debug_context); + } + return ObjStorageResponse::OK(); +} + +struct AzureBatchDeleter { + AzureBatchDeleter(BlobContainerClient* client, const ObjStoragePath& opts, + std::string_view representative_key, std::string_view tls_debug_context) + : _client(client), + _batch(client->CreateBatch()), + _opts(with_object_key(opts, representative_key)), + _tls_debug_context(tls_debug_context) {} + // Submit one blob to be deleted in `AzureBatchDeleter::execute` + void delete_blob(const std::string& blob_name) { + deferred_resps.emplace_back(DeferredDeleteResponse { + .key = blob_name, .response = _batch.DeleteBlob(blob_name)}); + } + ObjStorageResponse execute() { + if (deferred_resps.empty()) { + return ObjStorageResponse::OK(); + } + auto resp = do_azure_client_call( + [&]() { + client_bvar::ScopedLatency scoped_latency( + client_bvar::s3_delete_objects_latency); + _client->SubmitBatch(_batch); + }, + _opts, _tls_debug_context); + if (resp.status.code != ObjStorageStatus::OK) { + return resp; + } + + for (auto&& deferred : deferred_resps) { + try { + auto r = deferred.response.GetResponse(); + if (!r.Value.Deleted) { + auto msg = build_azure_batch_delete_failure_message(_opts, deferred.key); + LOG(WARNING) << msg; + return {.status = ObjStorageStatus {ObjStorageStatus::INTERNAL_ERROR, + std::move(msg)}, + .http_code = 0, + .request_id = ""}; + } + } catch (Azure::Core::RequestFailedException& e) { + if (Azure::Core::Http::HttpStatusCode::NotFound == e.StatusCode && + 0 == strcmp(e.ErrorCode.c_str(), BlobNotFound)) { + continue; + } + doris::record_object_request_failed(static_cast(e.StatusCode)); + auto msg = fmt::format( + "Azure request failed because {}, error msg {}, http code {}, path msg " + "{}{}", + e.what(), e.Message, static_cast(e.StatusCode), + wrap_object_storage_path_msg(with_object_key(_opts, deferred.key)), + build_azure_tls_debug_suffix(fmt::format("{} {}", e.what(), e.Message), + _tls_debug_context)); + LOG(WARNING) << msg; + return {.status = obj_storage_status_from_http_code(static_cast(e.StatusCode), + std::move(msg)), + .http_code = static_cast(e.StatusCode), + .request_id = std::move(e.RequestId)}; + } + } + + return ObjStorageResponse::OK(); + } + +private: + struct DeferredDeleteResponse { + std::string key; + Azure::Storage::DeferredResponse response; + }; + + BlobContainerClient* _client; + BlobContainerBatch _batch; + ObjStoragePath _opts; + std::string_view _tls_debug_context; + std::vector deferred_resps; +}; + +ObjStorageUploadResult AzureObjStorageClient::create_multipart_upload(const ObjStoragePath&) { + // Azure has no provider-side multipart session. This local UUID namespaces the writer's + // staged block IDs and is carried through the same interface as an S3 upload ID. + auto upload_id = butil::GenerateGUID(); + if (upload_id.empty()) { + return {.resp = {.status = {ObjStorageStatus::INTERNAL_ERROR, + "failed to generate Azure multipart upload ID"}}}; + } + return ObjStorageUploadResult { + .resp = ObjStorageResponse::OK(), + .upload_id = std::move(upload_id), + }; +} + +ObjStorageResponse AzureObjStorageClient::put_object(const ObjStoragePath& opts, + std::string_view stream) { + auto client = _client->GetBlockBlobClient(opts.key); + return do_azure_client_call( + [&]() { + client_bvar::ScopedLatency scoped_latency(client_bvar::s3_put_latency); + client.UploadFrom(reinterpret_cast(stream.data()), stream.size()); + }, + opts, _config.tls_debug_context); +} + +ObjStorageUploadResult AzureObjStorageClient::upload_part(const ObjStoragePath& opts, + const std::string& upload_id, + std::string_view stream, int part_num) { + DCHECK(!upload_id.empty()); + auto client = _client->GetBlockBlobClient(opts.key); + std::string block_id = azure_multipart_block_id(upload_id, part_num); + try { + Azure::Core::IO::MemoryBodyStream memory_body( + reinterpret_cast(stream.data()), stream.size()); + // The blockId must be base64 encoded + client_bvar::ScopedLatency scoped_latency(client_bvar::s3_multi_part_upload_latency); + client.StageBlock(block_id, memory_body); + } catch (Azure::Core::RequestFailedException& e) { + record_object_request_failed(static_cast(e.StatusCode)); + auto tls_debug_suffix = build_azure_tls_debug_suffix( + fmt::format("{} {}", e.what(), e.Message), _config.tls_debug_context); + auto msg = fmt::format( + "Azure request failed because {}, error msg {}, http code {}, path msg {}{}", + e.what(), e.Message, static_cast(e.StatusCode), + wrap_object_storage_path_msg(opts), tls_debug_suffix); + LOG(WARNING) << msg; + // clang-format off + return { + .resp = { + .status = obj_storage_status_from_http_code(static_cast(e.StatusCode), + std::move(msg)), + .http_code = static_cast(e.StatusCode), + .request_id = std::move(e.RequestId), + }, + }; + // clang-format on + } catch (const std::exception& e) { + return {.resp = make_azure_std_exception_response(e, opts, _config.tls_debug_context)}; + } + return ObjStorageUploadResult {.resp = ObjStorageResponse::OK(), .etag = std::move(block_id)}; +} + +ObjStorageResponse AzureObjStorageClient::complete_multipart_upload( + const ObjStoragePath& opts, const std::string& upload_id, + const std::vector& completed_parts) { + DCHECK(!upload_id.empty()); + auto client = _client->GetBlockBlobClient(opts.key); + std::vector string_block_ids; + std::ranges::transform(completed_parts, std::back_inserter(string_block_ids), + [&upload_id](const ObjStorageCompletedPart& i) { + return azure_multipart_block_id(upload_id, i.part_num); + }); + return do_azure_client_call( + [&]() { + client_bvar::ScopedLatency scoped_latency( + client_bvar::s3_multi_part_upload_latency); + client.CommitBlockList(string_block_ids); + }, + opts, _config.tls_debug_context); +} + +ObjStorageHeadResult AzureObjStorageClient::head_object(const ObjStoragePath& opts) { + try { + Models::BlobProperties properties = [&]() { + client_bvar::ScopedLatency scoped_latency(client_bvar::s3_head_latency); + return _client->GetBlockBlobClient(opts.key).GetProperties().Value; + }(); + return {.resp = ObjStorageResponse::OK(), .file_size = properties.BlobSize}; + } catch (Azure::Core::RequestFailedException& e) { + if (e.StatusCode == Azure::Core::Http::HttpStatusCode::NotFound) { + return ObjStorageHeadResult { + .resp = {.status = obj_storage_status_from_http_code( + static_cast(e.StatusCode), ""), + .http_code = static_cast(e.StatusCode), + .request_id = std::move(e.RequestId)}, + }; + } + record_object_request_failed(static_cast(e.StatusCode)); + auto tls_debug_suffix = build_azure_tls_debug_suffix( + fmt::format("{} {}", e.what(), e.Message), _config.tls_debug_context); + auto msg = fmt::format( + "Azure request failed because {}, error msg {}, http code {}, path msg {}{}", + e.what(), e.Message, static_cast(e.StatusCode), + wrap_object_storage_path_msg(opts), tls_debug_suffix); + LOG(WARNING) << msg << ", request_id=" << e.RequestId; + return ObjStorageHeadResult { + .resp = {.status = obj_storage_status_from_http_code(static_cast(e.StatusCode), + std::move(msg)), + .http_code = static_cast(e.StatusCode), + .request_id = std::move(e.RequestId)}, + }; + } catch (const std::exception& e) { + return {.resp = make_azure_std_exception_response(e, opts, _config.tls_debug_context)}; + } +} + +ObjStorageResponse AzureObjStorageClient::get_object(const ObjStoragePath& opts, void* buffer, + size_t offset, size_t bytes_read, + size_t* size_return) { + auto client = _client->GetBlockBlobClient(opts.key); + auto response = do_azure_client_call( + [&]() { + DownloadBlobToOptions download_opts; + Azure::Core::Http::HttpRange range {.Offset = static_cast(offset), + .Length = bytes_read}; + download_opts.Range = range; + client_bvar::ScopedLatency scoped_latency(client_bvar::s3_get_latency); + auto resp = client.DownloadTo(reinterpret_cast(buffer), bytes_read, + download_opts); + *size_return = resp.Value.ContentRange.Length.Value(); + }, + opts, _config.tls_debug_context); + if (!response.ok() || *size_return == bytes_read) { + return response; + } + return { + .status = {ObjStorageStatus::INTERNAL_ERROR, + fmt::format("incomplete read from {}, expect {}, got {}", + wrap_object_storage_path_msg(opts), bytes_read, *size_return)}, + .http_code = response.http_code, + .request_id = std::move(response.request_id), + }; +} + +ObjStorageListPageResult AzureObjStorageClient::list_objects_page( + const ObjStoragePath& opts, std::string_view continuation_token) { + const auto& prefix = opts.prefix.empty() ? opts.key : opts.prefix; + ListBlobsOptions request; + request.Prefix = prefix; + request.PageSizeHint = static_cast(capabilities().max_list_page); + if (!continuation_token.empty()) { + request.ContinuationToken = std::string(continuation_token); + } + TEST_SYNC_POINT_CALLBACK("AzureObjStorageClient::list_objects", &request); + + try { + auto response = [&]() { + client_bvar::ScopedLatency scoped_latency(client_bvar::s3_list_latency); + return _client->ListBlobs(request); + }(); + const bool has_more = response.NextPageToken.HasValue(); + auto next_token = has_more ? response.NextPageToken.Value() : std::string {}; + if (has_more && next_token.empty()) { + return { + .resp = {.status = {ObjStorageStatus::INTERNAL_ERROR, + "Azure list response has an empty continuation token"}, + .http_code = 0}, + }; + } + ObjStorageListPageResult page {.resp = ObjStorageResponse::OK(), + .continuation_token = std::move(next_token), + .has_more = has_more}; + page.objects.reserve(response.Blobs.size()); + for (auto&& item : response.Blobs) { + DCHECK(item.Name.starts_with(*request.Prefix)) << item.Name << ' ' << *request.Prefix; + page.objects.emplace_back(ObjectMeta { + .key = std::move(item.Name), + .size = item.BlobSize, + // `Azure::DateTime` adds the offset of `SystemClockEpoch` to the given Unix timestamp, + // so here we need to subtract this offset to obtain the Unix timestamp of the mtime. + // https://github.com/Azure/azure-sdk-for-cpp/blob/azure-core_1.12.0/sdk/core/azure-core/inc/azure/core/datetime.hpp#L129 + .mtime_s = duration_cast(item.Details.LastModified - + SystemClockEpoch) + .count()}); + } + return page; + } catch (Azure::Core::RequestFailedException& e) { + record_object_request_failed(static_cast(e.StatusCode)); + auto tls_debug_suffix = build_azure_tls_debug_suffix( + fmt::format("{} {}", e.what(), e.Message), _config.tls_debug_context); + LOG(WARNING) << fmt::format("Azure request failed because {}, url: {}, prefix: {}{}", + e.what(), _client->GetUrl(), request.Prefix.Value(), + tls_debug_suffix); + return { + .resp = {.status = obj_storage_status_from_http_code(static_cast(e.StatusCode), + e.Message + tls_debug_suffix), + .http_code = static_cast(e.StatusCode), + .request_id = std::move(e.RequestId)}, + }; + } catch (std::exception& e) { + LOG(WARNING) << fmt::format("Azure request failed because {}, url: {}, prefix: {}", + e.what(), _client->GetUrl(), request.Prefix.Value()); + return { + .resp = {.status = {ObjStorageStatus::INTERNAL_ERROR, e.what()}, + .http_code = 0, + .request_id = ""}, + }; + } +} + +// As Azure's doc said, the batch size is 256 +// You can find out the num in https://learn.microsoft.com/en-us/rest/api/storageservices/blob-batch?tabs=microsoft-entra-id +// > Each batch request supports a maximum of 256 subrequests. +ObjStorageResponse AzureObjStorageClient::delete_objects(const ObjStoragePath& opts, + std::vector objs) { + // TODO(ByteYue) : use range to adate this code when compiler is ready + // auto chunkedView = objs | std::views::chunk(BlobBatchMaxOperations); + auto begin = std::begin(objs); + auto end = std::end(objs); + + while (begin != end) { + auto chunk_end = begin; + size_t batch_size = BlobBatchMaxOperations; + TEST_SYNC_POINT_CALLBACK("AzureObjClient::delete_objects", &batch_size); + TEST_SYNC_POINT_CALLBACK("AzureObjStorageClient::delete_objects", &batch_size); + batch_size = std::max(1, batch_size); + std::advance(chunk_end, + std::min(batch_size, static_cast(std::distance(begin, end)))); + + auto deleter = AzureBatchDeleter(_client.get(), opts, *begin, _config.tls_debug_context); + std::ranges::for_each(std::ranges::subrange(begin, chunk_end), + [&](const std::string& obj) { deleter.delete_blob(obj); }); + begin = chunk_end; + if (auto resp = deleter.execute(); resp.status.code != ObjStorageStatus::OK) { + return resp; + } + } + return ObjStorageResponse::OK(); +} + +ObjStorageResponse AzureObjStorageClient::delete_object(const ObjStoragePath& opts) { + try { + auto resp = [&]() { + client_bvar::ScopedLatency scoped_latency(client_bvar::s3_delete_object_latency); + return _client->DeleteBlob(opts.key); + }(); + if (!resp.Value.Deleted) { + return { + .status = ObjStorageStatus {ObjStorageStatus::IO_ERROR, + "Delete azure blob failed"}, + .http_code = 0, + .request_id = "", + }; + } + return ObjStorageResponse::OK(); + } catch (Azure::Core::RequestFailedException& e) { + if (e.StatusCode == Azure::Core::Http::HttpStatusCode::NotFound && + e.ErrorCode == BlobNotFound) { + return ObjStorageResponse::OK(); + } + record_object_request_failed(static_cast(e.StatusCode)); + auto tls_debug_suffix = build_azure_tls_debug_suffix( + fmt::format("{} {}", e.what(), e.Message), _config.tls_debug_context); + auto msg = fmt::format( + "Azure request failed because {}, error msg {}, http code {}, path msg {}{}", + e.what(), e.Message, static_cast(e.StatusCode), + wrap_object_storage_path_msg(opts), tls_debug_suffix); + LOG(WARNING) << msg; + return { + .status = obj_storage_status_from_http_code(static_cast(e.StatusCode), + std::move(msg)), + .http_code = static_cast(e.StatusCode), + .request_id = std::move(e.RequestId), + }; + } catch (std::exception& e) { + auto msg = fmt::format("Azure request failed because {}, path msg {}{}", e.what(), + wrap_object_storage_path_msg(opts), + build_azure_tls_debug_suffix(e.what(), _config.tls_debug_context)); + LOG(WARNING) << msg; + return { + .status = ObjStorageStatus {ObjStorageStatus::INTERNAL_ERROR, std::move(msg)}, + .http_code = 0, + .request_id = "", + }; + } +} + +std::string AzureObjStorageClient::generate_presigned_url(const ObjStoragePath& opts, + int64_t expiration_secs) { + Azure::Storage::Sas::BlobSasBuilder sas_builder; + sas_builder.ExpiresOn = + std::chrono::system_clock::now() + std::chrono::seconds(expiration_secs); + sas_builder.BlobContainerName = opts.bucket; + sas_builder.BlobName = opts.key; + sas_builder.Resource = Azure::Storage::Sas::BlobSasResource::Blob; + sas_builder.Protocol = Azure::Storage::Sas::SasProtocol::HttpsOnly; + sas_builder.SetPermissions(Azure::Storage::Sas::BlobSasPermissions::Read); + + auto credential = _credential; + if (credential == nullptr) { + credential = std::make_shared(_config.ak, + _config.sk); + } + std::string sasToken = sas_builder.GenerateSasToken(*credential); + + auto sasURL = + fmt::format(SAS_TOKEN_URL_TEMPLATE, _config.endpoint, opts.bucket, opts.key, sasToken); + if (sasURL.find("://") == std::string::npos) { + sasURL = "https://" + sasURL; + } + return sasURL; +} + +ObjStorageResponse AzureObjStorageClient::get_lifecycle(const std::string& /*bucket*/, + int64_t* expiration_days) { + // TODO(plat1ko) + *expiration_days = INT64_MAX; + return ObjStorageResponse::OK(); +} + +ObjStorageResponse AzureObjStorageClient::check_versioning(const std::string& /*bucket*/) { + // TODO(plat1ko) + return ObjStorageResponse::OK(); +} + +ObjStorageResponse AzureObjStorageClient::abort_multipart_upload(const ObjStoragePath& opts, + const std::string& upload_id) { + // delete uncommitted blobs + // https://learn.microsoft.com/en-us/rest/api/storageservices/delete-blob?tabs=microsoft-entra-id#remarks + return delete_object(opts); +} +} // namespace doris diff --git a/common/cpp/obj-client/azure_obj_storage_client.h b/common/cpp/obj-client/azure_obj_storage_client.h new file mode 100644 index 00000000000000..29d22b81e08e22 --- /dev/null +++ b/common/cpp/obj-client/azure_obj_storage_client.h @@ -0,0 +1,126 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#ifdef USE_AZURE +#include +#include +#endif +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "client_bvar.h" +#include "cpp/obj_retry_strategy.h" +#include "cpp/util.h" +#include "obj_storage_client.h" +#include "s3_common.h" + +namespace Azure::Storage::Blobs { +class BlobContainerClient; +} // namespace Azure::Storage::Blobs + +namespace doris { + +using namespace Azure::Storage::Blobs; + +static const Azure::DateTime SystemClockEpoch {1970, 1, 1}; + +bool is_azure_tls_ca_error_message(std::string_view message); +std::string build_azure_tls_debug_suffix(std::string_view error_message, + std::string_view tls_debug_context); +std::string azure_multipart_block_id(std::string_view upload_id, int part_num); +std::string build_azure_batch_delete_failure_message(const ObjStoragePath& opts, + std::string_view key); + +class ObjClientHolder; + +class AzureObjStorageClient final : public ObjStorageClient { +public: + AzureObjStorageClient( + std::shared_ptr client, + ObjStorageEndpointInfo config, + std::shared_ptr credential = nullptr) + : _config(std::move(config)), + _client(std::move(client)), + _credential(std::move(credential)) {} + ~AzureObjStorageClient() override = default; + ObjStorageUploadResult create_multipart_upload(const ObjStoragePath& opts) override; + ObjStorageResponse put_object(const ObjStoragePath& opts, std::string_view stream) override; + ObjStorageUploadResult upload_part(const ObjStoragePath& opts, const std::string& upload_id, + std::string_view, int partNum) override; + ObjStorageResponse complete_multipart_upload( + const ObjStoragePath& opts, const std::string& upload_id, + const std::vector& completed_parts) override; + ObjStorageHeadResult head_object(const ObjStoragePath& opts) override; + ObjStorageResponse get_object(const ObjStoragePath& opts, void* buffer, size_t offset, + size_t bytes_read, size_t* size_return) override; + ObjStorageResponse delete_objects(const ObjStoragePath& opts, + std::vector objs) override; + ObjStorageResponse delete_object(const ObjStoragePath& opts) override; + std::string generate_presigned_url(const ObjStoragePath& opts, + int64_t expiration_secs) override; + ObjStorageResponse get_lifecycle(const std::string& bucket, int64_t* expiration_days) override; + + ObjStorageResponse check_versioning(const std::string& bucket) override; + + ObjStorageResponse abort_multipart_upload(const ObjStoragePath& opts, + const std::string& upload_id) override; + ObjStorageCapabilities capabilities() const override { + return {.max_delete_batch = 256, .max_list_page = 5000}; + } + +protected: + ObjStorageListPageResult list_objects_page(const ObjStoragePath& path, + std::string_view continuation_token) override; + +private: + ObjStorageEndpointInfo _config; + std::shared_ptr _client; + std::shared_ptr _credential; +}; + +} // namespace doris + +namespace doris::io { +using ::doris::AzureObjStorageClient; +using ::doris::azure_multipart_block_id; +using ::doris::build_azure_batch_delete_failure_message; +using ::doris::build_azure_tls_debug_suffix; +using ::doris::is_azure_tls_ca_error_message; +} // namespace doris::io diff --git a/common/cpp/obj-client/client_bvar.cpp b/common/cpp/obj-client/client_bvar.cpp new file mode 100644 index 00000000000000..d98ecbf9baee1e --- /dev/null +++ b/common/cpp/obj-client/client_bvar.cpp @@ -0,0 +1,37 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "client_bvar.h" + +#include + +namespace doris { + +namespace client_bvar { +bvar::LatencyRecorder s3_get_latency("s3_get"); +bvar::LatencyRecorder s3_put_latency("s3_put"); +bvar::LatencyRecorder s3_delete_object_latency("s3_delete_object"); +bvar::LatencyRecorder s3_delete_objects_latency("s3_delete_objects"); +bvar::LatencyRecorder s3_head_latency("s3_head"); +bvar::LatencyRecorder s3_multi_part_upload_latency("s3_multi_part_upload"); +bvar::LatencyRecorder s3_list_latency("s3_list"); +bvar::LatencyRecorder s3_list_object_versions_latency("s3_list_object_versions"); +bvar::LatencyRecorder s3_get_bucket_version_latency("s3_get_bucket_version"); +bvar::LatencyRecorder s3_copy_object_latency("s3_copy_object"); + +} // namespace client_bvar +} // namespace doris diff --git a/common/cpp/obj-client/client_bvar.h b/common/cpp/obj-client/client_bvar.h new file mode 100644 index 00000000000000..dfbcae8f692972 --- /dev/null +++ b/common/cpp/obj-client/client_bvar.h @@ -0,0 +1,58 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include + +namespace doris { + +namespace client_bvar { + +class ScopedLatency { +public: + explicit ScopedLatency(bvar::LatencyRecorder& recorder) + : recorder_(recorder), start_(std::chrono::steady_clock::now()) {} + ~ScopedLatency() { + recorder_ << std::chrono::duration_cast( + std::chrono::steady_clock::now() - start_) + .count(); + } + + ScopedLatency(const ScopedLatency&) = delete; + ScopedLatency& operator=(const ScopedLatency&) = delete; + +private: + bvar::LatencyRecorder& recorder_; + std::chrono::steady_clock::time_point start_; +}; + +extern bvar::LatencyRecorder s3_get_latency; +extern bvar::LatencyRecorder s3_put_latency; +extern bvar::LatencyRecorder s3_delete_object_latency; +extern bvar::LatencyRecorder s3_delete_objects_latency; +extern bvar::LatencyRecorder s3_head_latency; +extern bvar::LatencyRecorder s3_multi_part_upload_latency; +extern bvar::LatencyRecorder s3_list_latency; +extern bvar::LatencyRecorder s3_list_object_versions_latency; +extern bvar::LatencyRecorder s3_get_bucket_version_latency; +extern bvar::LatencyRecorder s3_copy_object_latency; +} // namespace client_bvar + +} // namespace doris diff --git a/common/cpp/obj-client/obj_storage_client.cpp b/common/cpp/obj-client/obj_storage_client.cpp new file mode 100644 index 00000000000000..d573c17e208501 --- /dev/null +++ b/common/cpp/obj-client/obj_storage_client.cpp @@ -0,0 +1,221 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "obj_storage_client.h" + +#include +#include + +#include +#include + +namespace doris { +ObjStorageStatus obj_storage_status_from_http_code(int http_code, std::string message) { + switch (http_code) { + case 401: + case 403: + return {ObjStorageStatus::PERMISSION_DENIED, std::move(message)}; + case 404: + return {ObjStorageStatus::NOT_FOUND, std::move(message)}; + case 429: + return {ObjStorageStatus::LIMIT_REACH, std::move(message)}; + default: + return {http_code <= 0 ? ObjStorageStatus::NETWORK_ERROR : ObjStorageStatus::INTERNAL_ERROR, + std::move(message)}; + } +} + +std::unique_ptr ObjStorageClient::list_objects(const ObjStoragePath& opts) { + return std::make_unique(shared_from_this(), opts); +} + +ObjStorageResponse ObjStorageClient::list_objects(const ObjStoragePath& opts, + std::vector* objects) { + objects->clear(); + auto iter = list_objects(opts); + for (;;) { + auto result = iter->next(); + if (!result.object.has_value()) { + if (!result.resp.ok()) { + objects->clear(); + } + return result.resp; + } + objects->emplace_back(std::move(*result.object)); + } +} + +ObjStorageResponse ObjStorageListIterator::has_next() { + if (!is_valid_) { + return { + .status = {ObjStorageStatus::INTERNAL_ERROR, "Iterator is invalid"}, + .http_code = 0, + }; + } + while (next_index_ == objects_.size()) { + if (!has_more_) { + return { + .status = {ObjStorageStatus::END_OF_FILE, "No more results"}, + .http_code = 200, + }; + } + auto page = client_->list_objects_page(opts_, continuation_token_); + if (!page.resp.ok()) { + is_valid_ = false; + return page.resp; + } + objects_ = std::move(page.objects); + next_index_ = 0; + continuation_token_ = std::move(page.continuation_token); + has_more_ = page.has_more; + } + return ObjStorageResponse::OK(); +} + +ObjStorageListResult ObjStorageListIterator::next() { + auto response = has_next(); + if (response.status.code == ObjStorageStatus::END_OF_FILE) { + return {.resp = ObjStorageResponse::OK(), .object = {}}; + } + if (!response.ok()) { + return {.resp = std::move(response), .object = {}}; + } + return { + .resp = ObjStorageResponse::OK(), + .object = std::move(objects_[next_index_++]), + }; +} + +ObjStorageResponse delete_objects_recursively( + std::shared_ptr client, const ObjStoragePath& path, + const ObjStorageRecursiveDeleteOptions& delete_options) { + const auto start_time = std::chrono::steady_clock::now(); + auto list_path = path; + if (list_path.prefix.empty()) { + list_path.prefix = list_path.key; + } + auto delete_batch_size = std::max(1, client->capabilities().max_delete_batch); + TEST_SYNC_POINT_CALLBACK("ObjStorageClient::delete_objects_recursively_", &delete_batch_size); + delete_batch_size = std::max(1, delete_batch_size); + const auto max_tasks_per_batch = std::max(1, delete_options.max_tasks_per_batch); + std::vector keys; + keys.reserve(delete_batch_size); + size_t pending_tasks = 0; + size_t total_batches = 0; + size_t num_deleted = 0; + size_t error_count = 0; + auto first_error = ObjStorageResponse::OK(); + + auto elapsed_milliseconds = [&]() { + return std::chrono::duration_cast( + std::chrono::steady_clock::now() - start_time) + .count(); + }; + auto finish = [&](ObjStorageResponse response) { + LOG(INFO) << "delete objects under " << list_path.bucket << "/" << list_path.prefix + << " finished, ret=" << response.status.code + << ", total_batches=" << total_batches << ", num_deleted=" << num_deleted + << ", error_count=" << error_count << ", cost=" << elapsed_milliseconds() + << " ms"; + return response; + }; + auto record_error = [&](ObjStorageResponse response) { + if (response.ok()) { + return; + } + ++error_count; + if (first_error.ok()) { + first_error = std::move(response); + } + }; + + auto wait_for_tasks = [&]() { + if (pending_tasks == 0) { + return ObjStorageResponse::OK(); + } + const auto tasks_in_batch = pending_tasks; + pending_tasks = 0; + auto response = delete_options.executor ? delete_options.executor->wait() + : ObjStorageResponse::OK(); + ++total_batches; + LOG(INFO) << "delete objects under " << list_path.bucket << "/" << list_path.prefix + << " batch " << total_batches << " completed" + << ", tasks_in_batch=" << tasks_in_batch << ", total_deleted=" << num_deleted + << ", elapsed=" << elapsed_milliseconds() << " ms"; + return response; + }; + auto submit_delete_task = [&]() -> bool { + ObjStorageDeleteTask task = [client, bucket = path.bucket, + batch = std::move(keys)]() mutable { + return client->delete_objects(ObjStoragePath {.bucket = std::move(bucket)}, + std::move(batch)); + }; + keys.clear(); + keys.reserve(delete_batch_size); + + ObjStorageResponse response; + if (delete_options.executor) { + response = delete_options.executor->submit(std::move(task)); + } else { + response = task(); + } + ++pending_tasks; + if (!response.ok()) { + record_error(std::move(response)); + record_error(wait_for_tasks()); + return false; + } + if (pending_tasks == max_tasks_per_batch) { + record_error(wait_for_tasks()); + } + // Match the pre-refactor Recycler behavior: do not scan the next task batch after the + // current batch reports a submit, delete, or wait failure. + return first_error.ok(); + }; + + auto iter = client->list_objects(list_path); + for (;;) { + auto result = iter->next(); + if (!result.object.has_value()) { + if (result.resp.ok()) { + break; + } + if (!keys.empty()) { + submit_delete_task(); + } + record_error(wait_for_tasks()); + record_error(std::move(result.resp)); + return finish(std::move(first_error)); + } + auto& object = *result.object; + if (delete_options.expiration_time > 0 && object.mtime_s > delete_options.expiration_time) { + continue; + } + ++num_deleted; + keys.emplace_back(std::move(object.key)); + if (keys.size() == delete_batch_size && !submit_delete_task()) { + return finish(std::move(first_error)); + } + } + if (!keys.empty() && !submit_delete_task()) { + return finish(std::move(first_error)); + } + record_error(wait_for_tasks()); + return finish(std::move(first_error)); +} + +} // namespace doris diff --git a/common/cpp/obj-client/obj_storage_client.h b/common/cpp/obj-client/obj_storage_client.h new file mode 100644 index 00000000000000..5e93dd1e83980d --- /dev/null +++ b/common/cpp/obj-client/obj_storage_client.h @@ -0,0 +1,295 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace doris { +// Names are in lexico order. +enum class ObjStorageProvider : uint8_t { + UNKNOWN = 0, + AWS = 1, + AZURE = 2, + BOS = 3, + COS = 4, + OSS = 5, + OBS = 6, + GCP = 7, + TOS = 8, +}; + +/// eg: +/// s3://bucket1/path/to/file.txt +/// path: s3://bucket1/path/to/file.txt +/// bucket: bucket1 +/// key: path/to/file.txt +struct ObjStoragePath { + std::filesystem::path path = ""; + std::string bucket {}; // blob container in azure + std::string key {}; // blob name in azure + std::string prefix {}; // for list and recursive delete +}; + +struct ObjStorageEndpointInfo { + std::string endpoint {}; + std::string ak {}; + std::string sk {}; + std::string tls_debug_context {}; +}; + +struct ObjectMeta { + std::string key {}; + int64_t size {0}; + int64_t mtime_s {0}; +}; + +struct ObjStorageCompletedPart { + int part_num = 0; + std::string etag {}; +}; + +struct ObjStorageStatus { + enum Code : int { + OK = TStatusCode::OK, + INTERNAL_ERROR = TStatusCode::INTERNAL_ERROR, + LIMIT_REACH = TStatusCode::LIMIT_REACH, + NOT_FOUND = TStatusCode::NOT_FOUND, + END_OF_FILE = TStatusCode::END_OF_FILE, + IO_ERROR = TStatusCode::IO_ERROR, + NETWORK_ERROR = TStatusCode::NETWORK_ERROR, + // Keep the legacy BE error code for access-denied object storage responses. + PERMISSION_DENIED = -256, + }; + + ObjStorageStatus(int r = OK, std::string msg = "") : code(r), msg(std::move(msg)) {} + // clang-format off + int code {OK}; // To unify the error handle logic with BE, we'd better use the same error code as BE + // clang-format on + std::string msg; +}; + +// We only store error code along with err_msg instead of Status to unify BE and recycler's error handle logic +struct ObjStorageResponse { + ObjStorageStatus status {ObjStorageStatus::OK, ""}; + int http_code {200}; + std::string request_id {}; + static ObjStorageResponse OK() { + // clang-format off + return { + .status = ObjStorageStatus{ObjStorageStatus::OK, ""}, + .http_code = 200, + .request_id = "" + }; + // clang-format on + } + + static ObjStorageResponse rate_limit(int status_code, int http_code, std::string message) { + return { + .status = ObjStorageStatus {status_code, std::move(message)}, + .http_code = http_code, + }; + } + + bool ok() const { return status.code == ObjStorageStatus::OK; } +}; + +// Convert a provider HTTP response code into the object-storage status domain. A non-positive +// response code means that no HTTP response was received. +ObjStorageStatus obj_storage_status_from_http_code(int http_code, std::string message); + +struct ObjStorageUploadResult { + ObjStorageResponse resp = ObjStorageResponse::OK(); + std::optional upload_id = std::nullopt; + std::optional etag = std::nullopt; +}; + +struct ObjStorageHeadResult { + ObjStorageResponse resp = ObjStorageResponse::OK(); + long long file_size {0}; +}; + +struct ObjStorageListResult { + ObjStorageResponse resp = ObjStorageResponse::OK(); + std::optional object = std::nullopt; +}; + +struct ObjStorageListPageResult { + ObjStorageResponse resp = ObjStorageResponse::OK(); + std::vector objects {}; + std::string continuation_token {}; + bool has_more = false; +}; + +struct ObjStorageCapabilities { + size_t max_delete_batch = 1; + size_t max_list_page = 1000; +}; + +using ObjStorageDeleteTask = std::function; + +// Implementations must enqueue tasks immediately and remain reusable after wait() completes. +// Recycler implements this in cloud/src/recycler/s3_accessor.cpp; BE uses the synchronous fallback. +class ObjStorageDeleteExecutor { +public: + virtual ~ObjStorageDeleteExecutor() = default; + virtual ObjStorageResponse submit(ObjStorageDeleteTask task) = 0; + virtual ObjStorageResponse wait() = 0; +}; + +struct ObjStorageRecursiveDeleteOptions { + int64_t expiration_time = 0; + size_t max_tasks_per_batch = 1; + std::shared_ptr executor {}; +}; + +class ObjStorageListIterator; +class RateLimitedObjStorageClient; + +// Provider implementations are in common/cpp/obj-client/s3_obj_storage_client.cpp and +// common/cpp/obj-client/azure_obj_storage_client.cpp. +// Clients are shared-owned so a lazy list iterator can keep the complete decorator chain alive. +class ObjStorageClient : public std::enable_shared_from_this { +public: + ObjStorageClient() = default; + virtual ~ObjStorageClient() = default; + + ObjStorageClient(const ObjStorageClient&) = delete; + ObjStorageClient& operator=(const ObjStorageClient&) = delete; + // Create a multipart upload request. The returned token may be provider-issued or local and + // identifies this writer's parts. + // The input parameters should include the bucket and key for the object storage. + virtual ObjStorageUploadResult create_multipart_upload(const ObjStoragePath& opts) = 0; + // To directly upload a piece of data to object storage and generate a user-visible file. + // You need to clearly specify the bucket and key + virtual ObjStorageResponse put_object(const ObjStoragePath& opts, std::string_view stream) = 0; + // Upload one part of a large object without making the object visible to users. upload_id is + // the provider-issued or local writer token returned by create_multipart_upload. + // Reusing the same bucket, key, upload_id, and part_num replaces that writer's staged part. + virtual ObjStorageUploadResult upload_part(const ObjStoragePath& opts, + const std::string& upload_id, + std::string_view stream, int part_num) = 0; + // Combine the parts belonging to upload_id into the key passed in opts. After this succeeds, + // the complete object is visible in object storage. + virtual ObjStorageResponse complete_multipart_upload( + const ObjStoragePath& opts, const std::string& upload_id, + const std::vector& completed_parts) = 0; + // According to the passed bucket and key, it will access whether the corresponding file exists in the object storage. + // If it exists, it will return the corresponding file size + virtual ObjStorageHeadResult head_object(const ObjStoragePath& opts) = 0; + // According to the bucket and key, it finds the corresponding file in the object storage + // and starting from the offset, it reads bytes_read bytes into the buffer, with size_return recording the actual number of bytes read + virtual ObjStorageResponse get_object(const ObjStoragePath& opts, void* buffer, size_t offset, + size_t bytes_read, size_t* size_return) = 0; + // Return a lazy iterator that fetches one provider page at a time. + virtual std::unique_ptr list_objects(const ObjStoragePath& opts); + // Collect all objects by consuming the lazy iterator. This preserves the eager BE API. + // **Notice**: The files returned by this function contain the full key in object storage. + virtual ObjStorageResponse list_objects(const ObjStoragePath& opts, + std::vector* objects); + + // According to the bucket and prefix specified by the user, it performs batch deletion based on the object names in the object array. + virtual ObjStorageResponse delete_objects(const ObjStoragePath& opts, + std::vector objs) = 0; + // Delete the file named key in the object storage bucket. + virtual ObjStorageResponse delete_object(const ObjStoragePath& opts) = 0; + virtual ObjStorageCapabilities capabilities() const = 0; + // Return a presigned URL for users to access the object + virtual std::string generate_presigned_url(const ObjStoragePath& opts, + int64_t expiration_secs) = 0; + + // Get the objects' expiration time on the bucket + virtual ObjStorageResponse get_lifecycle(const std::string& bucket, + int64_t* expiration_days) = 0; + + // Check if the objects' versioning is on or off + // returns 0 when versioning is on, otherwise versioning is off or check failed + virtual ObjStorageResponse check_versioning(const std::string& bucket) = 0; + + virtual ObjStorageResponse abort_multipart_upload(const ObjStoragePath& path, + const std::string& upload_id) = 0; + +protected: + // Fetch at most one page. One call corresponds to exactly one provider request. + virtual ObjStorageListPageResult list_objects_page(const ObjStoragePath& opts, + std::string_view continuation_token) = 0; + +private: + friend class ObjStorageListIterator; + friend class RateLimitedObjStorageClient; +}; + +// A client-side iterator returned by ObjStorageClient. It requests one fixed-size page at a time. +class ObjStorageListIterator { +public: + ObjStorageListIterator(std::shared_ptr client, ObjStoragePath opts) + : client_(std::move(client)), opts_(std::move(opts)) {} + + bool is_valid() const { return is_valid_; } + ObjStorageResponse has_next(); + ObjStorageListResult next(); + +private: + std::shared_ptr client_; + ObjStoragePath opts_; + std::vector objects_; + size_t next_index_ = 0; + std::string continuation_token_; + bool has_more_ = true; + bool is_valid_ = true; +}; + +// Provider-independent recursive deletion shared by concrete clients. Passing the complete client +// decorator chain keeps it alive for asynchronous delete tasks and applies policy per list page and +// delete batch. +ObjStorageResponse delete_objects_recursively(std::shared_ptr client, + const ObjStoragePath& path, + const ObjStorageRecursiveDeleteOptions& options = {}); +} // namespace doris + +// Keep the BE namespace spelling source-compatible while the implementation is +// shared with Recycler in `doris`. +namespace doris::io { +using ::doris::ObjStorageCapabilities; +using ::doris::ObjStorageClient; +using ::doris::ObjStorageCompletedPart; +using ::doris::ObjStorageDeleteExecutor; +using ::doris::ObjStorageDeleteTask; +using ::doris::ObjStorageEndpointInfo; +using ::doris::ObjStorageHeadResult; +using ::doris::ObjStorageListIterator; +using ::doris::ObjStorageListPageResult; +using ::doris::ObjStorageListResult; +using ::doris::ObjStoragePath; +using ::doris::ObjStorageProvider; +using ::doris::ObjStorageRecursiveDeleteOptions; +using ::doris::ObjStorageResponse; +using ::doris::ObjStorageStatus; +using ::doris::ObjStorageUploadResult; +using ::doris::ObjectMeta; +using ::doris::delete_objects_recursively; +} // namespace doris::io diff --git a/common/cpp/obj-client/rate_limited_obj_storage_client.cpp b/common/cpp/obj-client/rate_limited_obj_storage_client.cpp new file mode 100644 index 00000000000000..1d1ae5be2b8ff7 --- /dev/null +++ b/common/cpp/obj-client/rate_limited_obj_storage_client.cpp @@ -0,0 +1,145 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "rate_limited_obj_storage_client.h" + +namespace doris { + +ObjStorageAdmission RateLimitedObjStorageClient::acquire(S3RateLimitType type, + size_t estimated_bytes) const { + return rate_limit_policy_->acquire(type, estimated_bytes); +} + +ObjStorageUploadResult RateLimitedObjStorageClient::create_multipart_upload( + const ObjStoragePath& opts) { + auto rate_limit = acquire(S3RateLimitType::PUT); + if (!rate_limit.resp.ok()) { + return {.resp = std::move(rate_limit.resp)}; + } + return inner_->create_multipart_upload(opts); +} + +ObjStorageResponse RateLimitedObjStorageClient::put_object(const ObjStoragePath& opts, + std::string_view stream) { + auto rate_limit = acquire(S3RateLimitType::PUT, stream.size()); + if (!rate_limit.resp.ok()) { + return rate_limit.resp; + } + return inner_->put_object(opts, stream); +} + +ObjStorageUploadResult RateLimitedObjStorageClient::upload_part(const ObjStoragePath& opts, + const std::string& upload_id, + std::string_view stream, + int part_num) { + auto rate_limit = acquire(S3RateLimitType::PUT, stream.size()); + if (!rate_limit.resp.ok()) { + return {.resp = std::move(rate_limit.resp)}; + } + return inner_->upload_part(opts, upload_id, stream, part_num); +} + +ObjStorageResponse RateLimitedObjStorageClient::complete_multipart_upload( + const ObjStoragePath& opts, const std::string& upload_id, + const std::vector& completed_parts) { + auto rate_limit = acquire(S3RateLimitType::PUT); + if (!rate_limit.resp.ok()) { + return rate_limit.resp; + } + return inner_->complete_multipart_upload(opts, upload_id, completed_parts); +} + +ObjStorageHeadResult RateLimitedObjStorageClient::head_object(const ObjStoragePath& opts) { + auto rate_limit = acquire(S3RateLimitType::GET); + if (!rate_limit.resp.ok()) { + return {.resp = std::move(rate_limit.resp)}; + } + return inner_->head_object(opts); +} + +ObjStorageResponse RateLimitedObjStorageClient::get_object(const ObjStoragePath& opts, void* buffer, + size_t offset, size_t bytes_read, + size_t* size_return) { + auto rate_limit = acquire(S3RateLimitType::GET, bytes_read); + if (!rate_limit.resp.ok()) { + return rate_limit.resp; + } + auto response = inner_->get_object(opts, buffer, offset, bytes_read, size_return); + if (response.ok()) { + rate_limit.settle_bytes(*size_return); + } + return response; +} + +ObjStorageListPageResult RateLimitedObjStorageClient::list_objects_page( + const ObjStoragePath& opts, std::string_view continuation_token) { + auto rate_limit = acquire(S3RateLimitType::GET); + if (!rate_limit.resp.ok()) { + return {.resp = std::move(rate_limit.resp)}; + } + return inner_->list_objects_page(opts, continuation_token); +} + +ObjStorageResponse RateLimitedObjStorageClient::delete_objects(const ObjStoragePath& opts, + std::vector objs) { + auto rate_limit = acquire(S3RateLimitType::PUT); + if (!rate_limit.resp.ok()) { + return rate_limit.resp; + } + return inner_->delete_objects(opts, std::move(objs)); +} + +ObjStorageResponse RateLimitedObjStorageClient::delete_object(const ObjStoragePath& opts) { + auto rate_limit = acquire(S3RateLimitType::PUT); + if (!rate_limit.resp.ok()) { + return rate_limit.resp; + } + return inner_->delete_object(opts); +} + +std::string RateLimitedObjStorageClient::generate_presigned_url(const ObjStoragePath& opts, + int64_t expiration_secs) { + return inner_->generate_presigned_url(opts, expiration_secs); +} + +ObjStorageResponse RateLimitedObjStorageClient::get_lifecycle(const std::string& bucket, + int64_t* expiration_days) { + auto rate_limit = acquire(S3RateLimitType::GET); + if (!rate_limit.resp.ok()) { + return rate_limit.resp; + } + return inner_->get_lifecycle(bucket, expiration_days); +} + +ObjStorageResponse RateLimitedObjStorageClient::check_versioning(const std::string& bucket) { + auto rate_limit = acquire(S3RateLimitType::GET); + if (!rate_limit.resp.ok()) { + return rate_limit.resp; + } + return inner_->check_versioning(bucket); +} + +ObjStorageResponse RateLimitedObjStorageClient::abort_multipart_upload( + const ObjStoragePath& opts, const std::string& upload_id) { + auto rate_limit = acquire(S3RateLimitType::PUT); + if (!rate_limit.resp.ok()) { + return rate_limit.resp; + } + return inner_->abort_multipart_upload(opts, upload_id); +} + +} // namespace doris diff --git a/common/cpp/obj-client/rate_limited_obj_storage_client.h b/common/cpp/obj-client/rate_limited_obj_storage_client.h new file mode 100644 index 00000000000000..4c582245756317 --- /dev/null +++ b/common/cpp/obj-client/rate_limited_obj_storage_client.h @@ -0,0 +1,93 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include + +#include "cpp/token_bucket_rate_limiter.h" +#include "obj_storage_client.h" + +namespace doris { + +struct ObjStorageAdmission { + ObjStorageResponse resp = ObjStorageResponse::OK(); + std::function settle {}; + + void settle_bytes(size_t actual_bytes) const { + if (settle) { + settle(actual_bytes); + } + } +}; + +// Implementations are in be/src/util/s3_util.cpp and cloud/src/recycler/s3_accessor.cpp. +class ObjStorageRateLimitPolicy { +public: + virtual ~ObjStorageRateLimitPolicy() = default; + virtual ObjStorageAdmission acquire(S3RateLimitType type, size_t estimated_bytes) const = 0; +}; + +class RateLimitedObjStorageClient final : public ObjStorageClient { +public: + RateLimitedObjStorageClient(std::shared_ptr inner, + std::shared_ptr rate_limit_policy) + : inner_(std::move(inner)), rate_limit_policy_(std::move(rate_limit_policy)) {} + ~RateLimitedObjStorageClient() override = default; + + ObjStorageUploadResult create_multipart_upload(const ObjStoragePath& opts) override; + ObjStorageResponse put_object(const ObjStoragePath& opts, std::string_view stream) override; + ObjStorageUploadResult upload_part(const ObjStoragePath& opts, const std::string& upload_id, + std::string_view stream, int part_num) override; + ObjStorageResponse complete_multipart_upload( + const ObjStoragePath& opts, const std::string& upload_id, + const std::vector& completed_parts) override; + ObjStorageHeadResult head_object(const ObjStoragePath& opts) override; + ObjStorageResponse get_object(const ObjStoragePath& opts, void* buffer, size_t offset, + size_t bytes_read, size_t* size_return) override; + ObjStorageResponse delete_objects(const ObjStoragePath& opts, + std::vector objs) override; + ObjStorageResponse delete_object(const ObjStoragePath& opts) override; + ObjStorageCapabilities capabilities() const override { return inner_->capabilities(); } + std::string generate_presigned_url(const ObjStoragePath& opts, + int64_t expiration_secs) override; + ObjStorageResponse get_lifecycle(const std::string& bucket, int64_t* expiration_days) override; + ObjStorageResponse check_versioning(const std::string& bucket) override; + ObjStorageResponse abort_multipart_upload(const ObjStoragePath& opts, + const std::string& upload_id) override; + +protected: + ObjStorageListPageResult list_objects_page(const ObjStoragePath& opts, + std::string_view continuation_token) override; + +private: + ObjStorageAdmission acquire(S3RateLimitType type, size_t estimated_bytes = 0) const; + + std::shared_ptr inner_; + std::shared_ptr rate_limit_policy_; +}; + +} // namespace doris + +namespace doris::io { +using ::doris::ObjStorageAdmission; +using ::doris::ObjStorageRateLimitPolicy; +using ::doris::RateLimitedObjStorageClient; +using ::doris::S3RateLimitType; +} // namespace doris::io diff --git a/be/src/io/fs/s3_common.h b/common/cpp/obj-client/s3_common.h similarity index 100% rename from be/src/io/fs/s3_common.h rename to common/cpp/obj-client/s3_common.h diff --git a/common/cpp/obj-client/s3_obj_storage_client.cpp b/common/cpp/obj-client/s3_obj_storage_client.cpp new file mode 100644 index 00000000000000..3b9c67b6f3ee20 --- /dev/null +++ b/common/cpp/obj-client/s3_obj_storage_client.cpp @@ -0,0 +1,606 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#include "s3_obj_storage_client.h" + +#include +#include + +#include +#include + +#include "client_bvar.h" +#include "cpp/obj_retry_strategy.h" + +namespace Aws::S3::Model { +class DeleteObjectRequest; +} // namespace Aws::S3::Model + +using Aws::S3::Model::CompletedPart; +using Aws::S3::Model::CompletedMultipartUpload; +using Aws::S3::Model::CompleteMultipartUploadRequest; +using Aws::S3::Model::CreateMultipartUploadRequest; +using Aws::S3::Model::UploadPartRequest; +using Aws::S3::Model::UploadPartOutcome; + +namespace doris { +using namespace Aws::S3::Model; +namespace { + +constexpr int64_t S3_REQUEST_THRESHOLD_MS = 5000; + +int64_t elapsed_time_milliseconds(std::chrono::steady_clock::time_point start) { + return std::chrono::duration_cast(std::chrono::steady_clock::now() - + start) + .count(); +} + +void record_s3_request_failed(const Aws::S3::S3Error& error) { + record_object_request_failed(static_cast(error.GetResponseCode())); +} + +std::string object_identity(const ObjStoragePath& opts) { + return opts.path.empty() ? opts.key : opts.path.native(); +} + +std::string s3_error_message(const Aws::S3::S3Error& error, std::string_view message) { + return fmt::format("{}: {} {} code={}, type={}, request_id={}", message, + error.GetExceptionName(), error.GetMessage(), + static_cast(error.GetResponseCode()), + static_cast(error.GetErrorType()), + error.GetRequestId().empty() ? "" : error.GetRequestId().c_str()); +} + +} // namespace + +ObjStorageStatus s3fs_error(const Aws::S3::S3Error& err, std::string_view msg) { + return obj_storage_status_from_http_code(static_cast(err.GetResponseCode()), + s3_error_message(err, msg)); +} + +ObjStorageUploadResult S3ObjStorageClient::create_multipart_upload(const ObjStoragePath& opts) { + CreateMultipartUploadRequest request; + request.WithBucket(opts.bucket).WithKey(opts.key); + request.SetContentType("application/octet-stream"); + + const auto start = std::chrono::steady_clock::now(); + auto outcome = SYNC_POINT_HOOK_RETURN_VALUE( + [&]() { + client_bvar::ScopedLatency scoped_latency( + client_bvar::s3_multi_part_upload_latency); + return _client->CreateMultipartUpload(request); + }(), + "s3_file_writer::create_multi_part_upload", std::cref(request).get()); + SYNC_POINT_CALLBACK("s3_file_writer::_open", &outcome); + const auto elapsed_ms = elapsed_time_milliseconds(start); + + const auto& request_id = outcome.IsSuccess() ? outcome.GetResult().GetRequestId() + : outcome.GetError().GetRequestId(); + + LOG_IF(INFO, elapsed_ms > S3_REQUEST_THRESHOLD_MS) + << "CreateMultipartUpload cost=" << elapsed_ms << "ms" + << ", request_id=" << request_id << ", bucket=" << opts.bucket << ", key=" << opts.key; + + if (!outcome.IsSuccess()) { + record_s3_request_failed(outcome.GetError()); + auto st = s3fs_error(outcome.GetError(), fmt::format("failed to CreateMultipartUpload: {} ", + opts.path.native())); + LOG(WARNING) << st.code << " request_id=" << request_id; + return ObjStorageUploadResult { + .resp = {.status = st, + .http_code = static_cast(outcome.GetError().GetResponseCode()), + .request_id = outcome.GetError().GetRequestId()}, + }; + } + + return ObjStorageUploadResult {.resp = ObjStorageResponse::OK(), + .upload_id {outcome.GetResult().GetUploadId()}}; +} + +ObjStorageResponse S3ObjStorageClient::put_object(const ObjStoragePath& opts, + std::string_view stream) { + Aws::S3::Model::PutObjectRequest request; + request.WithBucket(opts.bucket).WithKey(opts.key); + auto string_view_stream = std::make_shared(stream.data(), stream.size()); + Aws::Utils::ByteBuffer part_md5(Aws::Utils::HashingUtils::CalculateMD5(*string_view_stream)); + request.SetContentMD5(Aws::Utils::HashingUtils::Base64Encode(part_md5)); + request.SetBody(string_view_stream); + request.SetContentLength(stream.size()); + request.SetContentType("application/octet-stream"); + + const auto start = std::chrono::steady_clock::now(); + auto outcome = SYNC_POINT_HOOK_RETURN_VALUE( + [&]() { + client_bvar::ScopedLatency scoped_latency(client_bvar::s3_put_latency); + return _client->PutObject(request); + }(), + "s3_file_writer::put_object", std::cref(request).get(), &stream); + const auto elapsed_ms = elapsed_time_milliseconds(start); + + const auto& request_id = outcome.IsSuccess() ? outcome.GetResult().GetRequestId() + : outcome.GetError().GetRequestId(); + + if (!outcome.IsSuccess()) { + record_s3_request_failed(outcome.GetError()); + auto st = s3fs_error(outcome.GetError(), + fmt::format("failed to put object: {}", opts.path.native())); + LOG(WARNING) << st.code << ", request_id=" << request_id; + return ObjStorageResponse { + .status = st, + .http_code = static_cast(outcome.GetError().GetResponseCode()), + .request_id = outcome.GetError().GetRequestId()}; + } + + LOG_IF(INFO, elapsed_ms > S3_REQUEST_THRESHOLD_MS) + << "PutObject cost=" << elapsed_ms << "ms" + << ", request_id=" << request_id << ", bucket=" << opts.bucket << ", key=" << opts.key; + return ObjStorageResponse::OK(); +} + +ObjStorageUploadResult S3ObjStorageClient::upload_part(const ObjStoragePath& opts, + const std::string& upload_id, + std::string_view stream, int part_num) { + UploadPartRequest request; + request.WithBucket(opts.bucket) + .WithKey(opts.key) + .WithPartNumber(part_num) + .WithUploadId(upload_id); + auto string_view_stream = std::make_shared(stream.data(), stream.size()); + + request.SetBody(string_view_stream); + + Aws::Utils::ByteBuffer part_md5(Aws::Utils::HashingUtils::CalculateMD5(*string_view_stream)); + request.SetContentMD5(Aws::Utils::HashingUtils::Base64Encode(part_md5)); + + request.SetContentLength(stream.size()); + request.SetContentType("application/octet-stream"); + + const auto start = std::chrono::steady_clock::now(); + auto outcome = SYNC_POINT_HOOK_RETURN_VALUE( + [&]() { + client_bvar::ScopedLatency scoped_latency( + client_bvar::s3_multi_part_upload_latency); + + return _client->UploadPart(request); + }(), + "s3_file_writer::upload_part", std::cref(request).get(), &stream); + const auto elapsed_ms = elapsed_time_milliseconds(start); + + const auto& request_id = outcome.IsSuccess() ? outcome.GetResult().GetRequestId() + : outcome.GetError().GetRequestId(); + + TEST_SYNC_POINT_CALLBACK("S3FileWriter::_upload_one_part", &outcome); + if (!outcome.IsSuccess()) { + record_s3_request_failed(outcome.GetError()); + auto st = s3fs_error(outcome.GetError(), + fmt::format("failed to UploadPart: {}, part_num {}, upload_id={}", + opts.path.native(), part_num, upload_id)); + + LOG(WARNING) << st.code << ", request_id=" << request_id; + return ObjStorageUploadResult { + .resp = {.status = st, + .http_code = static_cast(outcome.GetError().GetResponseCode()), + .request_id = outcome.GetError().GetRequestId()}}; + } + LOG_IF(INFO, elapsed_ms > S3_REQUEST_THRESHOLD_MS) + << "UploadPart cost=" << elapsed_ms << "ms" + << ", request_id=" << request_id << ", bucket=" << opts.bucket << ", key=" << opts.key + << ", part_num=" << part_num << ", upload_id=" << upload_id; + return ObjStorageUploadResult {.resp = ObjStorageResponse::OK(), + .etag = outcome.GetResult().GetETag()}; +} + +ObjStorageResponse S3ObjStorageClient::complete_multipart_upload( + const ObjStoragePath& opts, const std::string& upload_id, + const std::vector& completed_parts) { + CompleteMultipartUploadRequest request; + request.WithBucket(opts.bucket).WithKey(opts.key).WithUploadId(upload_id); + + CompletedMultipartUpload completed_upload; + std::vector complete_parts; + std::ranges::transform(completed_parts, std::back_inserter(complete_parts), + [](const ObjStorageCompletedPart& part_ptr) { + CompletedPart part; + part.SetPartNumber(part_ptr.part_num); + part.SetETag(part_ptr.etag); + return part; + }); + completed_upload.SetParts(std::move(complete_parts)); + request.WithMultipartUpload(completed_upload); + + TEST_SYNC_POINT_RETURN_WITH_VALUE("S3FileWriter::_complete:3", ObjStorageResponse(), this); + + const auto start = std::chrono::steady_clock::now(); + auto outcome = SYNC_POINT_HOOK_RETURN_VALUE( + [&]() { + client_bvar::ScopedLatency scoped_latency( + client_bvar::s3_multi_part_upload_latency); + return _client->CompleteMultipartUpload(request); + }(), + "s3_file_writer::complete_multi_part", std::cref(request).get()); + const auto elapsed_ms = elapsed_time_milliseconds(start); + + const auto& request_id = outcome.IsSuccess() ? outcome.GetResult().GetRequestId() + : outcome.GetError().GetRequestId(); + + if (!outcome.IsSuccess()) { + record_s3_request_failed(outcome.GetError()); + auto st = s3fs_error(outcome.GetError(), + fmt::format("failed to CompleteMultipartUpload: {}, upload_id={}", + opts.path.native(), upload_id)); + LOG(WARNING) << st.code << ", request_id=" << request_id; + return {.status = st, + .http_code = static_cast(outcome.GetError().GetResponseCode()), + .request_id = outcome.GetError().GetRequestId()}; + } + + LOG_IF(INFO, elapsed_ms > S3_REQUEST_THRESHOLD_MS) + << "CompleteMultipartUpload cost=" << elapsed_ms << "ms" + << ", request_id=" << request_id << ", bucket=" << opts.bucket << ", key=" << opts.key + << ", upload_id=" << upload_id; + return ObjStorageResponse::OK(); +} + +ObjStorageHeadResult S3ObjStorageClient::head_object(const ObjStoragePath& opts) { + Aws::S3::Model::HeadObjectRequest request; + request.WithBucket(opts.bucket).WithKey(opts.key); + + auto outcome = SYNC_POINT_HOOK_RETURN_VALUE( + [&]() { + client_bvar::ScopedLatency scoped_latency(client_bvar::s3_head_latency); + return _client->HeadObject(request); + }(), + "s3_file_system::head_object", std::ref(request).get()); + + if (outcome.IsSuccess()) { + return {.resp = ObjStorageResponse::OK(), + .file_size = outcome.GetResult().GetContentLength()}; + } else if (outcome.GetError().GetResponseCode() == Aws::Http::HttpResponseCode::NOT_FOUND) { + return {.resp = {.status = ObjStorageStatus::NOT_FOUND}, .file_size = 0}; + } else { + record_s3_request_failed(outcome.GetError()); + LOG(WARNING) << "failed to head object" + << "bucket " << opts.bucket << " key " << opts.key << " responseCode " + << outcome.GetError() << " error " << outcome.GetError().GetMessage() + << " request_id " << outcome.GetError().GetRequestId(); + return {.resp = {.status = s3fs_error( + outcome.GetError(), + fmt::format("failed to head object: {}", object_identity(opts))), + .http_code = static_cast(outcome.GetError().GetResponseCode()), + .request_id = outcome.GetError().GetRequestId()}, + .file_size = -1}; + } +} + +ObjStorageResponse S3ObjStorageClient::get_object(const ObjStoragePath& opts, void* buffer, + size_t offset, size_t bytes_read, + size_t* size_return) { + Aws::S3::Model::GetObjectRequest request; + request.WithBucket(opts.bucket).WithKey(opts.key); + request.SetRange(fmt::format("bytes={}-{}", offset, offset + bytes_read - 1)); + request.SetResponseStreamFactory(AwsWriteableStreamFactory(buffer, bytes_read)); + + auto outcome = [&]() { + client_bvar::ScopedLatency scoped_latency(client_bvar::s3_get_latency); + return _client->GetObject(request); + }(); + if (!outcome.IsSuccess()) { + record_s3_request_failed(outcome.GetError()); + return ObjStorageResponse { + .status = s3fs_error( + outcome.GetError(), + fmt::format("failed to get object: bucket={} object={} offset={} size={}", + opts.bucket, opts.key, offset, bytes_read)), + .http_code = static_cast(outcome.GetError().GetResponseCode()), + .request_id = outcome.GetError().GetRequestId(), + }; + } + *size_return = outcome.GetResult().GetContentLength(); + SYNC_POINT_CALLBACK("s3_obj_storage_client::get_object", size_return); + if (*size_return != bytes_read) { + const auto& request_id = outcome.GetResult().GetRequestId(); + return ObjStorageResponse { + .status = {ObjStorageStatus::INTERNAL_ERROR, + fmt::format("incomplete read from bucket={} object={} offset={}, expect " + "{}, got {}, request_id={}", + opts.bucket, opts.key, offset, bytes_read, *size_return, + request_id)}, + .request_id = request_id}; + } + return ObjStorageResponse::OK(); +} + +ObjStorageListPageResult S3ObjStorageClient::list_objects_page( + const ObjStoragePath& opts, std::string_view continuation_token) { + const auto& prefix = opts.prefix.empty() ? opts.key : opts.prefix; + Aws::S3::Model::ListObjectsV2Request request; + request.WithBucket(opts.bucket) + .WithPrefix(prefix) + .WithMaxKeys(static_cast(capabilities().max_list_page)); + if (!continuation_token.empty()) { + request.SetContinuationToken(std::string(continuation_token)); + } + TEST_SYNC_POINT_CALLBACK("S3ObjStorageClient::list_objects", &request); + + auto outcome = [&]() { + client_bvar::ScopedLatency scoped_latency(client_bvar::s3_list_latency); + return _client->ListObjectsV2(request); + }(); + + const auto& request_id = outcome.IsSuccess() ? outcome.GetResult().GetRequestId() + : outcome.GetError().GetRequestId(); + if (!outcome.IsSuccess()) { + // Some S3-compatible providers (for example TOS) return NoSuchKey instead of an empty page + // when a prefix does not exist. + if (outcome.GetError().GetErrorType() == Aws::S3::S3Errors::NO_SUCH_KEY) { + LOG(INFO) << fmt::format( + "NoSuchKey when listing objects, treat as empty response, endpoint: {}, " + "bucket: {}, prefix: {}, request_id: {}", + _config.endpoint, request.GetBucket(), request.GetPrefix(), request_id); + return {.resp = ObjStorageResponse::OK()}; + } + record_object_request_failed(static_cast(outcome.GetError().GetResponseCode())); + const auto status = s3fs_error(outcome.GetError(), + fmt::format("failed to list objects: {}, prefix: {}", + request.GetBucket(), request.GetPrefix())); + LOG(WARNING) << fmt::format( + "failed to list objects, endpoint: {}, bucket: {}, prefix: {}, responseCode: {}, " + "error: {}, request_id: {}", + _config.endpoint, request.GetBucket(), request.GetPrefix(), + static_cast(outcome.GetError().GetResponseCode()), + outcome.GetError().GetMessage(), request_id); + return { + .resp = {.status = status, + .http_code = static_cast(outcome.GetError().GetResponseCode()), + .request_id = request_id}, + }; + } + + const auto& result = outcome.GetResult(); + if (result.GetIsTruncated() && result.GetNextContinuationToken().empty()) { + LOG(WARNING) << fmt::format( + "failed to list objects, isTruncated but no continuation token, endpoint: {}, " + "bucket: {}, prefix: {}, request_id: {}", + _config.endpoint, request.GetBucket(), request.GetPrefix(), request_id); + return { + .resp = {.status = {ObjStorageStatus::INTERNAL_ERROR, + fmt::format("failed to list objects: {}, prefix: {}", + request.GetBucket(), request.GetPrefix())}, + .http_code = 0, + .request_id = request_id}, + }; + } + + ObjStorageListPageResult page { + .resp = ObjStorageResponse::OK(), + .continuation_token = result.GetNextContinuationToken(), + .has_more = result.GetIsTruncated(), + }; + const auto& content = result.GetContents(); + page.objects.reserve(content.size()); + for (const auto& obj : content) { + DCHECK(obj.GetKey().starts_with(request.GetPrefix())) + << obj.GetKey() << ' ' << request.GetPrefix(); + page.objects.emplace_back(ObjectMeta {.key = obj.GetKey(), + .size = obj.GetSize(), + .mtime_s = obj.GetLastModified().Seconds()}); + } + return page; +} + +ObjStorageResponse S3ObjStorageClient::delete_objects(const ObjStoragePath& opts, + std::vector objs) { + size_t max_delete_batch = 1000; + TEST_SYNC_POINT_CALLBACK("S3ObjClient::delete_objects", &max_delete_batch); + TEST_SYNC_POINT_CALLBACK("S3ObjStorageClient::delete_objects", &max_delete_batch); + max_delete_batch = std::max(1, max_delete_batch); + for (size_t begin = 0; begin < objs.size(); begin += max_delete_batch) { + const size_t end = std::min(begin + max_delete_batch, objs.size()); + Aws::S3::Model::DeleteObjectsRequest delete_request; + delete_request.SetBucket(opts.bucket); + Aws::S3::Model::Delete del; + Aws::Vector objects; + objects.reserve(end - begin); + for (size_t i = begin; i < end; ++i) { + Aws::S3::Model::ObjectIdentifier object; + object.SetKey(std::move(objs[i])); + objects.emplace_back(std::move(object)); + } + del.WithObjects(std::move(objects)).SetQuiet(true); + delete_request.SetDelete(std::move(del)); + + auto delete_outcome = [&]() { + client_bvar::ScopedLatency scoped_latency(client_bvar::s3_delete_objects_latency); + return _client->DeleteObjects(delete_request); + }(); + SYNC_POINT_CALLBACK("s3_obj_storage_client::delete_objects", &delete_outcome); + SYNC_POINT_CALLBACK("s3_obj_storage_client::delete_objects_recursively", &delete_outcome); + if (!delete_outcome.IsSuccess()) { + record_s3_request_failed(delete_outcome.GetError()); + LOG(WARNING) << fmt::format( + "failed to delete objects, endpoint: {}, bucket: {}, key: {}, responseCode: " + "{}, error: {}, request_id: {}", + _config.endpoint, opts.bucket, + delete_request.GetDelete().GetObjects().front().GetKey(), + static_cast(delete_outcome.GetError().GetResponseCode()), + delete_outcome.GetError().GetMessage(), + delete_outcome.GetError().GetRequestId()); + return ObjStorageResponse { + .status = s3fs_error(delete_outcome.GetError(), + fmt::format("failed to delete dir {}", opts.key)), + .http_code = static_cast(delete_outcome.GetError().GetResponseCode()), + .request_id = delete_outcome.GetError().GetRequestId()}; + } + if (!delete_outcome.GetResult().GetErrors().empty()) { + const auto& error = delete_outcome.GetResult().GetErrors().front(); + LOG(WARNING) << fmt::format( + "failed to delete object in batch, endpoint: {}, bucket: {}, key: {}, error " + "code: {}, error: {}, request_id: {}", + _config.endpoint, opts.bucket, error.GetKey(), error.GetCode(), + error.GetMessage(), delete_outcome.GetResult().GetRequestId()); + return ObjStorageResponse { + .status = {ObjStorageStatus::INTERNAL_ERROR, + fmt::format("failed to delete object {}: {}, request_id={}", + error.GetKey(), error.GetMessage(), + delete_outcome.GetResult().GetRequestId())}, + .request_id = delete_outcome.GetResult().GetRequestId()}; + } + } + return ObjStorageResponse::OK(); +} + +ObjStorageResponse S3ObjStorageClient::delete_object(const ObjStoragePath& opts) { + Aws::S3::Model::DeleteObjectRequest request; + request.WithBucket(opts.bucket).WithKey(opts.key); + + auto outcome = [&]() { + client_bvar::ScopedLatency scoped_latency(client_bvar::s3_delete_object_latency); + + return _client->DeleteObject(request); + }(); + TEST_SYNC_POINT_CALLBACK("S3ObjClient::delete_object", &outcome); + TEST_SYNC_POINT_CALLBACK("S3ObjStorageClient::delete_object", &outcome); + if (outcome.IsSuccess()) { + return ObjStorageResponse::OK(); + } + ObjStorageResponse response { + .status = s3fs_error(outcome.GetError(), + fmt::format("failed to delete object {}", opts.key)), + .http_code = static_cast(outcome.GetError().GetResponseCode()), + .request_id = outcome.GetError().GetRequestId()}; + if (response.status.code == ObjStorageStatus::NOT_FOUND) { + return response; + } + record_s3_request_failed(outcome.GetError()); + LOG(WARNING) << fmt::format( + "failed to delete object, endpoint: {}, bucket: {}, key: {}, responseCode: {}, " + "error: {}, request_id: {}", + _config.endpoint, opts.bucket, opts.key, + static_cast(outcome.GetError().GetResponseCode()), outcome.GetError().GetMessage(), + outcome.GetError().GetRequestId()); + return response; +} + +std::string S3ObjStorageClient::generate_presigned_url(const ObjStoragePath& opts, + int64_t expiration_secs) { + if (_presigned_url_generator) { + return _presigned_url_generator(opts, expiration_secs); + } + return _client->GeneratePresignedUrl(opts.bucket, opts.key, Aws::Http::HttpMethod::HTTP_GET, + expiration_secs); +} + +ObjStorageResponse S3ObjStorageClient::check_versioning(const std::string& bucket) { + Aws::S3::Model::GetBucketVersioningRequest request; + request.SetBucket(bucket); + + auto outcome = _client->GetBucketVersioning(request); + + if (outcome.IsSuccess()) { + const auto& versioning_configuration = outcome.GetResult().GetStatus(); + if (versioning_configuration != Aws::S3::Model::BucketVersioningStatus::Enabled) { + LOG(WARNING) << "Err for check interval: bucket doesn't enable bucket versioning" + << " endpoint=" << _config.endpoint << " bucket=" << bucket; + return ObjStorageResponse { + .status = {ObjStorageStatus::INTERNAL_ERROR, + fmt::format("bucket versioning is not enabled: {}", bucket)}}; + } + } else { + record_s3_request_failed(outcome.GetError()); + LOG(WARNING) << "Err for check interval: failed to get status of bucket versioning" + << " endpoint=" << _config.endpoint << " bucket=" << bucket + << " responseCode=" << static_cast(outcome.GetError().GetResponseCode()) + << " error=" << outcome.GetError().GetMessage() + << " request_id=" << outcome.GetError().GetRequestId(); + return ObjStorageResponse { + .status = s3fs_error(outcome.GetError(), + fmt::format("failed to get bucket versioning: {}", bucket)), + .http_code = static_cast(outcome.GetError().GetResponseCode()), + .request_id = outcome.GetError().GetRequestId()}; + } + return ObjStorageResponse::OK(); +} + +ObjStorageResponse S3ObjStorageClient::abort_multipart_upload(const ObjStoragePath& opts, + const std::string& upload_id) { + Aws::S3::Model::AbortMultipartUploadRequest request; + request.WithBucket(opts.bucket).WithKey(opts.key).WithUploadId(upload_id); + + auto outcome = _client->AbortMultipartUpload(request); + if (!outcome.IsSuccess()) { + LOG(WARNING) << "failed to abort multipart upload" + << " endpoint=" << _config.endpoint << " bucket=" << opts.bucket + << " key=" << opts.key << " upload_id=" << upload_id + << " responseCode=" << static_cast(outcome.GetError().GetResponseCode()) + << " error=" << outcome.GetError().GetMessage() + << " request_id=" << outcome.GetError().GetRequestId(); + if (outcome.GetError().GetResponseCode() == Aws::Http::HttpResponseCode::NOT_FOUND) { + return ObjStorageResponse::OK(); + } + record_s3_request_failed(outcome.GetError()); + return ObjStorageResponse { + .status = + s3fs_error(outcome.GetError(), + fmt::format("failed to abort multipart upload: {}, upload_id={}", + opts.path.native(), upload_id)), + .http_code = static_cast(outcome.GetError().GetResponseCode()), + .request_id = outcome.GetError().GetRequestId(), + }; + } + return ObjStorageResponse::OK(); +} + +ObjStorageResponse S3ObjStorageClient::get_lifecycle(const std::string& bucket, + int64_t* expiration_days) { + Aws::S3::Model::GetBucketLifecycleConfigurationRequest request; + request.SetBucket(bucket); + + auto outcome = _client->GetBucketLifecycleConfiguration(request); + bool has_lifecycle = false; + if (outcome.IsSuccess()) { + const auto& rules = outcome.GetResult().GetRules(); + for (const auto& rule : rules) { + if (rule.NoncurrentVersionExpirationHasBeenSet()) { + has_lifecycle = true; + *expiration_days = rule.GetNoncurrentVersionExpiration().GetNoncurrentDays(); + } + } + } else { + record_s3_request_failed(outcome.GetError()); + LOG(WARNING) << "Err for check interval: failed to get bucket lifecycle" + << " endpoint=" << _config.endpoint << " bucket=" << bucket + << " responseCode=" << static_cast(outcome.GetError().GetResponseCode()) + << " error=" << outcome.GetError().GetMessage() + << " request_id=" << outcome.GetError().GetRequestId(); + return ObjStorageResponse { + .status = s3fs_error(outcome.GetError(), + fmt::format("failed to get lift cycle: {}", bucket)), + .http_code = static_cast(outcome.GetError().GetResponseCode()), + .request_id = outcome.GetError().GetRequestId()}; + } + + if (!has_lifecycle) { + LOG(WARNING) << "Err for check interval: bucket doesn't have lifecycle configuration" + << " endpoint=" << _config.endpoint << " bucket=" << bucket; + return ObjStorageResponse { + .status = {ObjStorageStatus::NOT_FOUND, + fmt::format("bucket has no lifecycle configuration: {}", bucket)}}; + } + return ObjStorageResponse::OK(); +} + +} // namespace doris diff --git a/common/cpp/obj-client/s3_obj_storage_client.h b/common/cpp/obj-client/s3_obj_storage_client.h new file mode 100644 index 00000000000000..7dd64358f030ed --- /dev/null +++ b/common/cpp/obj-client/s3_obj_storage_client.h @@ -0,0 +1,141 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include "client_bvar.h" +#include "cpp/obj_retry_strategy.h" +#include "cpp/sync_point.h" +#include "obj_storage_client.h" +#include "s3_common.h" + +namespace Aws::S3 { +class S3Client; +namespace Model { +class CompletedPart; +} +} // namespace Aws::S3 + +namespace doris { + +ObjStorageStatus s3fs_error(const Aws::S3::S3Error& err, std::string_view msg); + +class S3ObjStorageClient final : public ObjStorageClient { +public: + using PresignedUrlGenerator = std::function; + + S3ObjStorageClient(std::shared_ptr client, + ObjStorageEndpointInfo config = {}, + PresignedUrlGenerator presigned_url_generator = {}) + : _config(std::move(config)), + _client(std::move(client)), + _presigned_url_generator(std::move(presigned_url_generator)) {} + ~S3ObjStorageClient() override = default; + ObjStorageUploadResult create_multipart_upload(const ObjStoragePath& opts) override; + ObjStorageResponse put_object(const ObjStoragePath& opts, std::string_view stream) override; + ObjStorageUploadResult upload_part(const ObjStoragePath& opts, const std::string& upload_id, + std::string_view, int partNum) override; + ObjStorageResponse complete_multipart_upload( + const ObjStoragePath& opts, const std::string& upload_id, + const std::vector& completed_parts) override; + ObjStorageHeadResult head_object(const ObjStoragePath& opts) override; + ObjStorageResponse get_object(const ObjStoragePath& opts, void* buffer, size_t offset, + size_t bytes_read, size_t* size_return) override; + ObjStorageResponse delete_objects(const ObjStoragePath& opts, + std::vector objs) override; + ObjStorageResponse delete_object(const ObjStoragePath& opts) override; + std::string generate_presigned_url(const ObjStoragePath& opts, + int64_t expiration_secs) override; + ObjStorageResponse get_lifecycle(const std::string& bucket, int64_t* expiration_days) override; + + ObjStorageResponse check_versioning(const std::string& bucket) override; + + ObjStorageResponse abort_multipart_upload(const ObjStoragePath& opts, + const std::string& upload_id) override; + ObjStorageCapabilities capabilities() const override { + return {.max_delete_batch = 1000, .max_list_page = 1000}; + } + +protected: + ObjStorageListPageResult list_objects_page(const ObjStoragePath& opts, + std::string_view continuation_token) override; + +private: + ObjStorageEndpointInfo _config; + std::shared_ptr _client; + PresignedUrlGenerator _presigned_url_generator; +}; + +} // namespace doris + +namespace doris::io { +using ::doris::S3ObjStorageClient; +} // namespace doris::io