diff --git a/docs/source/user_guide/catalog.rst b/docs/source/user_guide/catalog.rst index d881968c4..e311b52b0 100644 --- a/docs/source/user_guide/catalog.rst +++ b/docs/source/user_guide/catalog.rst @@ -67,8 +67,11 @@ registered on the REST server. The catalog is configured through the the role from the metadata endpoint when it is omitted. * ``dlf.signing-algorithm``: ``default`` selects DLF4-HMAC-SHA256 for DLF VPC endpoints and ``openapi`` selects ROA HMAC-SHA1 for DlfNext OpenAPI endpoints. - When omitted, an endpoint containing ``dlfnext`` selects ``openapi`` and other - endpoints select ``default``. + ``openapi-v4`` selects ACS4-HMAC-SHA256 for DlfNext OpenAPI endpoints. When a + request matches a registered DlfNext operation, it also names that operation's + API in the ``x-acs-action`` header; other requests are signed without it. + Without this option, an endpoint containing ``dlfnext`` selects ``openapi`` and + other endpoints select ``default``; ``openapi-v4`` is only used when configured. * ``table-default.``: table option defaults applied when a created table left ```` unset. * ``header.``: sent as the ```` http header on every request to the @@ -103,6 +106,8 @@ are reloaded when less than one hour of validity remains. {"dlf.access-key-secret", ""}, // Optional for temporary credentials: {"dlf.security-token", ""}, + // Optional for endpoints that require ACS4-HMAC-SHA256 signing: + {"dlf.signing-algorithm", "openapi-v4"}, }; On creation the catalog queries the server's ``/v1/config`` endpoint and merges diff --git a/include/paimon/catalog_options.h b/include/paimon/catalog_options.h index f781fa189..c1862ad9d 100644 --- a/include/paimon/catalog_options.h +++ b/include/paimon/catalog_options.h @@ -62,7 +62,7 @@ struct PAIMON_EXPORT CatalogOptions { /// "dlf.token-ecs-role-name" - Optional ECS RAM role name. static const char DLF_TOKEN_ECS_ROLE_NAME[]; - /// "dlf.signing-algorithm" - DLF signer ("default" or "openapi"). + /// "dlf.signing-algorithm" - DLF signer ("default", "openapi" or "openapi-v4"). static const char DLF_SIGNING_ALGORITHM[]; /// "dlf.oss-endpoint" - OSS endpoint that overrides the "fs.oss.endpoint" of a data diff --git a/src/paimon/rest/dlf_auth.cpp b/src/paimon/rest/dlf_auth.cpp index 6a4906279..ae5b6f0c9 100644 --- a/src/paimon/rest/dlf_auth.cpp +++ b/src/paimon/rest/dlf_auth.cpp @@ -21,6 +21,7 @@ #include +#include #include #include #include @@ -72,6 +73,13 @@ constexpr const char kAcsSignatureNonceHeader[] = "x-acs-signature-nonce"; constexpr const char kAcsSignatureVersionHeader[] = "x-acs-signature-version"; constexpr const char kAcsVersionHeader[] = "x-acs-version"; constexpr const char kAcsSecurityTokenHeader[] = "x-acs-security-token"; +constexpr const char kAcsDateHeader[] = "x-acs-date"; +constexpr const char kAcsContentSha256Header[] = "x-acs-content-sha256"; +constexpr const char kAcsActionHeader[] = "x-acs-action"; +constexpr const char kAcs4Algorithm[] = "ACS4-HMAC-SHA256"; +// SHA-256 of the empty string, the payload hash of a request without a body. +constexpr const char kEmptyBodySha256[] = + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; Result RequiredNonEmptyOption(const std::map& options, const std::string& key) { @@ -119,6 +127,15 @@ Result FormatDlfTime(std::chrono::system_clock::time_point time) { return std::string(buffer.data()); } +Result FormatAcsTime(std::chrono::system_clock::time_point time) { + PAIMON_ASSIGN_OR_RAISE(std::tm utc, ToUtc(time)); + std::array buffer{}; + if (std::strftime(buffer.data(), buffer.size(), "%Y-%m-%dT%H:%M:%SZ", &utc) == 0) { + return Status::Invalid("failed to format DLF OpenAPI V4 signing time"); + } + return std::string(buffer.data()); +} + Result FormatRfc1123Time(std::chrono::system_clock::time_point time) { PAIMON_ASSIGN_OR_RAISE(std::tm utc, ToUtc(time)); static constexpr std::array kWeekdays = {"Sun", "Mon", "Tue", "Wed", @@ -226,6 +243,17 @@ std::string HexEncode(const Bytes& value) { return encoded; } +// The signing key shared by the DLF4 and ACS4 schemes, derived from the secret through the +// date, the region and the product. +Result DeriveSigningKey(const std::string& access_key_secret, const std::string& date, + const std::string& region) { + PAIMON_ASSIGN_OR_RAISE(Bytes date_key, + Hmac(EVP_sha256(), ToBytes("aliyun_v4" + access_key_secret), date)); + PAIMON_ASSIGN_OR_RAISE(Bytes region_key, Hmac(EVP_sha256(), date_key, region)); + PAIMON_ASSIGN_OR_RAISE(Bytes product_key, Hmac(EVP_sha256(), region_key, "DlfNext")); + return Hmac(EVP_sha256(), product_key, "aliyun_v4_request"); +} + Result Base64Encode(const Bytes& value) { if (value.size() > static_cast(INT_MAX)) { return Status::Invalid("DLF digest is too large to encode"); @@ -343,6 +371,85 @@ Result GenerateNonce(std::chrono::system_clock::time_point now) { return fmt::format("{}{}{}", uuid, millis, thread_id.str()); } +struct Acs4CanonicalHeaders { + std::string canonical_headers; + std::string signed_headers; +}; + +// The headers the ACS4 scheme signs: "host", "content-type" and every "x-acs-*" header, named +// in lower case with trimmed values and sorted by name. +Acs4CanonicalHeaders Acs4CanonicalizeHeaders(const DlfRequestSigner::Headers& headers) { + std::map sorted; + for (const auto& [key, value] : headers) { + std::string lower_key = StringUtils::ToLowerCase(key); + if (StringUtils::StartsWith(lower_key, "x-acs-") || lower_key == "host" || + lower_key == "content-type") { + sorted[lower_key] = Trimmed(value); + } + } + Acs4CanonicalHeaders result; + for (const auto& [key, value] : sorted) { + result.canonical_headers += key + ":" + value + "\n"; + if (!result.signed_headers.empty()) { + result.signed_headers += ";"; + } + result.signed_headers += key; + } + return result; +} + +// The query parameters sorted by name and RFC 3986 encoded. `RestAuthParameter` holds the +// values url-encoded once already, so they are decoded first: encoding them again would sign a +// form the gateway cannot rebuild from the wire. +std::string Acs4CanonicalQueryString(const RestAuthParameter& parameter) { + std::string canonical; + bool first = true; + for (const auto& [key, value] : parameter.parameters) { + if (!first) { + canonical += "&"; + } + canonical += UrlUtils::PercentEncode(key) + "="; + if (!value.empty()) { + canonical += UrlUtils::PercentEncode(UrlUtils::DecodeString(value)); + } + first = false; + } + return canonical; +} + +struct SplitOperation { + std::vector template_segments; + std::string action; +}; + +// The registered operations grouped by method, with their path templates split once. +const std::map>& OperationsByMethod() { + static const std::map> kOperationsByMethod = [] { + std::map> operations_by_method; + for (const DlfOpenApiActions::Operation& operation : DlfOpenApiActions::Operations()) { + operations_by_method[operation.method].push_back( + {StringUtils::Split(operation.path_template, "/", /*ignore_empty=*/false), + operation.action}); + } + return operations_by_method; + }(); + return kOperationsByMethod; +} + +bool MatchesPathTemplate(const std::vector& template_segments, + const std::vector& segments) { + if (segments.size() != template_segments.size()) { + return false; + } + for (size_t i = 0; i < template_segments.size(); ++i) { + bool variable = StringUtils::StartsWith(template_segments[i], "{"); + if (variable ? segments[i].empty() : template_segments[i] != segments[i]) { + return false; + } + } + return true; +} + Result> CreateSigner(const std::string& algorithm, const std::string& region) { if (algorithm == DlfDefaultSigner::kIdentifier) { @@ -351,9 +458,13 @@ Result> CreateSigner(const std::string& algori if (algorithm == DlfOpenApiSigner::kIdentifier) { return std::make_unique(); } - return Status::Invalid(fmt::format( - "unsupported DLF signing algorithm '{}', supported values are 'default' and 'openapi'", - algorithm)); + if (algorithm == DlfOpenApiV4Signer::kIdentifier) { + return std::make_unique(region); + } + return Status::Invalid( + fmt::format("unsupported DLF signing algorithm '{}', supported values are 'default', " + "'openapi' and 'openapi-v4'", + algorithm)); } } // namespace @@ -531,24 +642,21 @@ Result DlfDefaultSigner::SignHeaders( Result DlfDefaultSigner::Authorization(const RestAuthParameter& parameter, const DlfToken& token, const std::string& host, - const Headers& sign_headers) const { - PAIMON_ASSIGN_OR_RAISE(std::string date_time, RequiredHeader(sign_headers, kDlfDateHeader)); + Headers* sign_headers) const { + const Headers& headers = *sign_headers; + PAIMON_ASSIGN_OR_RAISE(std::string date_time, RequiredHeader(headers, kDlfDateHeader)); if (date_time.size() < 8) { return Status::Invalid("DLF signing date is invalid"); } std::string date = date_time.substr(0, 8); std::string scope = fmt::format("{}/{}/DlfNext/aliyun_v4_request", date, region_); - std::string canonical_request = DefaultCanonicalRequest(parameter, sign_headers); + std::string canonical_request = DefaultCanonicalRequest(parameter, headers); PAIMON_ASSIGN_OR_RAISE(Bytes canonical_hash, Digest(EVP_sha256(), canonical_request)); std::string string_to_sign = fmt::format("DLF4-HMAC-SHA256\n{}\n{}\n{}", date_time, scope, HexEncode(canonical_hash)); - PAIMON_ASSIGN_OR_RAISE( - Bytes date_key, - Hmac(EVP_sha256(), ToBytes("aliyun_v4" + token.GetAccessKeySecret()), date)); - PAIMON_ASSIGN_OR_RAISE(Bytes region_key, Hmac(EVP_sha256(), date_key, region_)); - PAIMON_ASSIGN_OR_RAISE(Bytes service_key, Hmac(EVP_sha256(), region_key, "DlfNext")); - PAIMON_ASSIGN_OR_RAISE(Bytes signing_key, Hmac(EVP_sha256(), service_key, "aliyun_v4_request")); + PAIMON_ASSIGN_OR_RAISE(Bytes signing_key, + DeriveSigningKey(token.GetAccessKeySecret(), date, region_)); PAIMON_ASSIGN_OR_RAISE(Bytes signature, Hmac(EVP_sha256(), signing_key, string_to_sign)); return fmt::format("DLF4-HMAC-SHA256 Credential={}/{},Signature={}", token.GetAccessKeyId(), scope, HexEncode(signature)); @@ -579,28 +687,212 @@ Result DlfOpenApiSigner::SignHeaders( Result DlfOpenApiSigner::Authorization(const RestAuthParameter& parameter, const DlfToken& token, const std::string& host, - const Headers& sign_headers) const { - PAIMON_ASSIGN_OR_RAISE(std::string accept, RequiredHeader(sign_headers, kOpenApiAcceptHeader)); - PAIMON_ASSIGN_OR_RAISE(std::string date, RequiredHeader(sign_headers, kOpenApiDateHeader)); + Headers* sign_headers) const { + const Headers& headers = *sign_headers; + PAIMON_ASSIGN_OR_RAISE(std::string accept, RequiredHeader(headers, kOpenApiAcceptHeader)); + PAIMON_ASSIGN_OR_RAISE(std::string date, RequiredHeader(headers, kOpenApiDateHeader)); std::string content_md5; - auto md5_iter = sign_headers.find(kContentMd5Header); - if (md5_iter != sign_headers.end()) { + auto md5_iter = headers.find(kContentMd5Header); + if (md5_iter != headers.end()) { content_md5 = md5_iter->second; } std::string content_type; - auto type_iter = sign_headers.find(kContentTypeHeader); - if (type_iter != sign_headers.end()) { + auto type_iter = headers.find(kContentTypeHeader); + if (type_iter != headers.end()) { content_type = type_iter->second; } std::string string_to_sign = parameter.method + "\n" + accept + "\n" + content_md5 + "\n" + content_type + "\n" + date + - "\n" + OpenApiCanonicalizedHeaders(sign_headers) + OpenApiCanonicalizedResource(parameter); + "\n" + OpenApiCanonicalizedHeaders(headers) + OpenApiCanonicalizedResource(parameter); PAIMON_ASSIGN_OR_RAISE(Bytes signature, Hmac(EVP_sha1(), ToBytes(token.GetAccessKeySecret()), string_to_sign)); PAIMON_ASSIGN_OR_RAISE(std::string encoded_signature, Base64Encode(signature)); return fmt::format("acs {}:{}", token.GetAccessKeyId(), encoded_signature); } +DlfOpenApiV4Signer::DlfOpenApiV4Signer(const std::string& region) : region_(region) {} + +Result DlfOpenApiV4Signer::SignHeaders( + const std::string& body, std::chrono::system_clock::time_point now, + const std::optional& security_token, const std::string& host) const { + if (host.empty()) { + return Status::Invalid("DLF OpenAPI signing host is empty"); + } + PAIMON_ASSIGN_OR_RAISE(std::string date, FormatAcsTime(now)); + PAIMON_ASSIGN_OR_RAISE(std::string nonce, GenerateNonce(now)); + // "Host" and "Content-Type" keep the spelling `RestApi` and the other signers use, so that + // merging these headers replaces a header of the same name rather than sending a second + // copy of it; the signature names every header in lower case either way. + Headers headers = {{kAcsDateHeader, date}, + {kOpenApiHostHeader, host}, + {kAcsContentSha256Header, kEmptyBodySha256}, + {kAcsSignatureNonceHeader, nonce}, + {kAcsVersionHeader, "2026-01-18"}}; + if (!body.empty()) { + PAIMON_ASSIGN_OR_RAISE(Bytes body_hash, Digest(EVP_sha256(), body)); + headers[kAcsContentSha256Header] = HexEncode(body_hash); + headers[kContentTypeHeader] = kJsonMediaType; + } + if (security_token) { + headers[kAcsSecurityTokenHeader] = security_token.value(); + } + return headers; +} + +Result DlfOpenApiV4Signer::Authorization(const RestAuthParameter& parameter, + const DlfToken& token, + const std::string& host, + Headers* sign_headers) const { + PAIMON_ASSIGN_OR_RAISE(std::string date_time, RequiredHeader(*sign_headers, kAcsDateHeader)); + if (date_time.size() < 10) { + return Status::Invalid("DLF signing date is invalid"); + } + std::string date = date_time.substr(0, 10); + date.erase(std::remove(date.begin(), date.end(), '-'), date.end()); + + // POP gateways may route by the action, so it is signed and sent with the other headers. + std::optional action = + DlfOpenApiActions::Resolve(parameter.method, parameter.resource_path); + if (action) { + (*sign_headers)[kAcsActionHeader] = action.value(); + } + + Acs4CanonicalHeaders canonical_headers = Acs4CanonicalizeHeaders(*sign_headers); + std::string canonical_uri = StringUtils::IsNullOrWhitespaceOnly(parameter.resource_path) + ? std::string("/") + : parameter.resource_path; + auto payload_iter = sign_headers->find(kAcsContentSha256Header); + std::string hashed_payload = + payload_iter == sign_headers->end() ? std::string(kEmptyBodySha256) : payload_iter->second; + std::string canonical_request = parameter.method + "\n" + canonical_uri + "\n" + + Acs4CanonicalQueryString(parameter) + "\n" + + canonical_headers.canonical_headers + "\n" + + canonical_headers.signed_headers + "\n" + hashed_payload; + PAIMON_ASSIGN_OR_RAISE(Bytes canonical_hash, Digest(EVP_sha256(), canonical_request)); + std::string string_to_sign = std::string(kAcs4Algorithm) + "\n" + HexEncode(canonical_hash); + + PAIMON_ASSIGN_OR_RAISE(Bytes signing_key, + DeriveSigningKey(token.GetAccessKeySecret(), date, region_)); + PAIMON_ASSIGN_OR_RAISE(Bytes signature, Hmac(EVP_sha256(), signing_key, string_to_sign)); + return fmt::format( + "{} Credential={}/{}/{}/DlfNext/aliyun_v4_request,SignedHeaders={},Signature={}", + kAcs4Algorithm, token.GetAccessKeyId(), date, region_, canonical_headers.signed_headers, + HexEncode(signature)); +} + +const std::vector& DlfOpenApiActions::Operations() { + // Mirrors the Java `DLFOpenApiActions`, so both clients name every request the same way. + static const std::vector kOperations = { + {"GET", "/v1/config", "GetConfig"}, + {"GET", "/v1/{prefix}/databases", "ListDatabases"}, + {"POST", "/v1/{prefix}/databases", "CreateDatabase"}, + {"GET", "/v1/{prefix}/databases/{database}", "GetDatabase"}, + {"POST", "/v1/{prefix}/databases/{database}", "AlterDatabase"}, + {"DELETE", "/v1/{prefix}/databases/{database}", "DropDatabase"}, + {"POST", "/v1/{prefix}/databases/{database}/register", "RegisterTable"}, + {"GET", "/v1/{prefix}/databases/{database}/tables", "ListTables"}, + {"POST", "/v1/{prefix}/databases/{database}/tables", "CreateTable"}, + {"GET", "/v1/{prefix}/databases/{database}/table-details", "ListTableDetails"}, + {"GET", "/v1/{prefix}/tables", "ListTablesGlobally"}, + {"GET", "/v1/{prefix}/databases/{database}/tables/{table}", "GetTable"}, + {"POST", "/v1/{prefix}/databases/{database}/tables/{table}", "AlterTable"}, + {"DELETE", "/v1/{prefix}/databases/{database}/tables/{table}", "DropTable"}, + {"GET", "/v1/{prefix}/tables/id/{tableId}", "GetTableById"}, + {"POST", "/v1/{prefix}/tables/rename", "RenameTable"}, + {"POST", "/v1/{prefix}/databases/{database}/tables/{table}/auth", "AuthTableQuery"}, + {"POST", "/v1/{prefix}/databases/{database}/tables/{table}/commit", "CommitTable"}, + {"POST", "/v1/{prefix}/databases/{database}/tables/{table}/commit-lock", + "AcquireCommitLock"}, + {"POST", "/v1/{prefix}/databases/{database}/tables/{table}/commit-lock/renew", + "RenewCommitLock"}, + {"POST", "/v1/{prefix}/databases/{database}/tables/{table}/replace", "ReplaceTable"}, + {"GET", "/v1/{prefix}/databases/{database}/tables/{table}/token", "GetTableToken"}, + {"POST", "/v1/{prefix}/databases/{database}/tables/{table}/rollback", "RollbackToSnapshot"}, + {"POST", "/v1/{prefix}/databases/{database}/tables/{table}/rollback-schema", + "RollbackSchema"}, + {"GET", "/v1/{prefix}/databases/{database}/tables/{table}/snapshot", "GetTableSnapshot"}, + {"GET", "/v1/{prefix}/databases/{database}/tables/{table}/snapshots", "ListSnapshots"}, + {"GET", "/v1/{prefix}/databases/{database}/tables/{table}/snapshots/{version}", + "GetVersionSnapshot"}, + {"GET", "/v1/{prefix}/databases/{database}/tables/{table}/schemas", "ListSchemas"}, + {"GET", "/v1/{prefix}/databases/{database}/tables/{table}/schemas/{version}", "GetSchema"}, + {"GET", "/v1/{prefix}/databases/{database}/tables/{table}/partitions", "ListPartitions"}, + {"POST", "/v1/{prefix}/databases/{database}/tables/{table}/partitions", "CreatePartitions"}, + {"POST", "/v1/{prefix}/databases/{database}/tables/{table}/partitions/drop", + "DropPartitions"}, + {"POST", "/v1/{prefix}/databases/{database}/tables/{table}/partitions/mark", + "MarkDonePartitions"}, + {"POST", "/v1/{prefix}/databases/{database}/tables/{table}/partitions/list-by-names", + "ListPartitionsByNames"}, + {"POST", "/v1/{prefix}/databases/{database}/tables/{table}/partitions/list-by-filter", + "ListPartitionsByFilter"}, + {"GET", "/v1/{prefix}/databases/{database}/tables/{table}/branches", "ListBranches"}, + {"POST", "/v1/{prefix}/databases/{database}/tables/{table}/branches", "CreateBranch"}, + {"DELETE", "/v1/{prefix}/databases/{database}/tables/{table}/branches/{branch}", + "DropBranch"}, + {"POST", "/v1/{prefix}/databases/{database}/tables/{table}/branches/{branch}/forward", + "FastForwardBranch"}, + {"GET", "/v1/{prefix}/databases/{database}/tables/{table}/tags", "ListTags"}, + {"POST", "/v1/{prefix}/databases/{database}/tables/{table}/tags", "CreateTag"}, + {"GET", "/v1/{prefix}/databases/{database}/tables/{table}/tags/{tag}", "GetTag"}, + {"DELETE", "/v1/{prefix}/databases/{database}/tables/{table}/tags/{tag}", "DropTag"}, + {"GET", "/v1/{prefix}/databases/{database}/tables/{table}/consumers", "ListConsumers"}, + {"POST", "/v1/{prefix}/databases/{database}/tables/{table}/consumers/reset", + "ResetConsumer"}, + {"GET", "/v1/{prefix}/databases/{database}/tables/{table}/policies", "ListPolicies"}, + {"POST", "/v1/{prefix}/databases/{database}/tables/{table}/policies", "CreatePolicy"}, + {"POST", "/v1/{prefix}/databases/{database}/tables/{table}/policies/drop", "DropPolicy"}, + {"GET", "/v1/{prefix}/databases/{database}/views", "ListViews"}, + {"POST", "/v1/{prefix}/databases/{database}/views", "CreateView"}, + {"GET", "/v1/{prefix}/databases/{database}/view-details", "ListViewDetails"}, + {"GET", "/v1/{prefix}/views", "ListViewsGlobally"}, + {"GET", "/v1/{prefix}/databases/{database}/views/{view}", "GetView"}, + {"POST", "/v1/{prefix}/databases/{database}/views/{view}", "AlterView"}, + {"DELETE", "/v1/{prefix}/databases/{database}/views/{view}", "DropView"}, + {"POST", "/v1/{prefix}/views/rename", "RenameView"}, + {"GET", "/v1/{prefix}/databases/{database}/semantic-views", "ListSemanticViews"}, + {"GET", "/v1/{prefix}/databases/{database}/semantic-views/{semanticView}", + "GetSemanticView"}, + {"POST", "/v1/{prefix}/databases/{database}/semantic-views/{semanticView}", + "UpsertSemanticView"}, + {"DELETE", "/v1/{prefix}/databases/{database}/semantic-views/{semanticView}", + "DeleteSemanticView"}, + {"GET", "/v1/{prefix}/databases/{database}/functions", "ListFunctions"}, + {"POST", "/v1/{prefix}/databases/{database}/functions", "CreateFunction"}, + {"GET", "/v1/{prefix}/databases/{database}/function-details", "ListFunctionDetails"}, + {"GET", "/v1/{prefix}/functions", "ListFunctionsGlobally"}, + {"GET", "/v1/{prefix}/databases/{database}/functions/{function}", "GetFunction"}, + {"POST", "/v1/{prefix}/databases/{database}/functions/{function}", "AlterFunction"}, + {"DELETE", "/v1/{prefix}/databases/{database}/functions/{function}", "DropFunction"}, + {"GET", "/v1/{prefix}/permissions", "ListPermissionAssignments"}, + {"POST", "/v1/{prefix}/permissions/grant", "GrantPermissionAssignment"}, + {"POST", "/v1/{prefix}/permissions/revoke", "RevokePermissionAssignment"}, + {"GET", "/v1/{prefix}/labels", "ListLabels"}, + {"POST", "/v1/{prefix}/labels/get", "GetLabel"}, + {"POST", "/v1/{prefix}/labels/upsert", "UpsertLabel"}, + {"POST", "/v1/{prefix}/labels/delete", "DeleteLabel"}, + }; + return kOperations; +} + +std::optional DlfOpenApiActions::Resolve(const std::string& method, + const std::string& resource_path) { + const std::map>& operations_by_method = + OperationsByMethod(); + auto candidates = operations_by_method.find(StringUtils::ToUpperCase(method)); + if (candidates == operations_by_method.end()) { + return std::nullopt; + } + std::vector segments = + StringUtils::Split(resource_path, "/", /*ignore_empty=*/false); + for (const SplitOperation& operation : candidates->second) { + if (MatchesPathTemplate(operation.template_segments, segments)) { + return operation.action; + } + } + return std::nullopt; +} + DlfAuthProvider::DlfAuthProvider(std::unique_ptr token_loader, const std::optional& token, const std::string& host, std::unique_ptr signer, Clock clock) @@ -744,7 +1036,7 @@ Result> DlfAuthProvider::MergeAuthHeader( DlfRequestSigner::Headers sign_headers, signer_->SignHeaders(parameter.data, signing_time, token.GetSecurityToken(), host_)); PAIMON_ASSIGN_OR_RAISE(std::string authorization, - signer_->Authorization(parameter, token, host_, sign_headers)); + signer_->Authorization(parameter, token, host_, &sign_headers)); std::map headers = base_header; for (const auto& [key, value] : sign_headers) { headers[key] = value; diff --git a/src/paimon/rest/dlf_auth.h b/src/paimon/rest/dlf_auth.h index c10b655b7..2f8238f0f 100644 --- a/src/paimon/rest/dlf_auth.h +++ b/src/paimon/rest/dlf_auth.h @@ -124,9 +124,11 @@ class DlfRequestSigner { const std::optional& security_token, const std::string& host) const = 0; + /// Signs the request. `sign_headers` holds the headers `SignHeaders` returned and must not + /// be null; a signer may add headers to it, which it signs and the request must send too. virtual Result Authorization(const RestAuthParameter& parameter, const DlfToken& token, const std::string& host, - const Headers& sign_headers) const = 0; + Headers* sign_headers) const = 0; }; /// DLF4-HMAC-SHA256 signer used by the default DLF VPC endpoint. @@ -142,7 +144,7 @@ class DlfDefaultSigner : public DlfRequestSigner { Result Authorization(const RestAuthParameter& parameter, const DlfToken& token, const std::string& host, - const Headers& sign_headers) const override; + Headers* sign_headers) const override; private: std::string region_; @@ -159,7 +161,50 @@ class DlfOpenApiSigner : public DlfRequestSigner { Result Authorization(const RestAuthParameter& parameter, const DlfToken& token, const std::string& host, - const Headers& sign_headers) const override; + Headers* sign_headers) const override; +}; + +/// ACS4-HMAC-SHA256 signer for DlfNext OpenAPI endpoints, selected only when configured. +/// Unlike the other signers, it adds an "x-acs-action" header to a request that matches a +/// registered operation, naming that operation's API, and signs it with the other headers. +class DlfOpenApiV4Signer : public DlfRequestSigner { + public: + static constexpr const char* kIdentifier = "openapi-v4"; + + explicit DlfOpenApiV4Signer(const std::string& region); + + Result SignHeaders(const std::string& body, std::chrono::system_clock::time_point now, + const std::optional& security_token, + const std::string& host) const override; + + Result Authorization(const RestAuthParameter& parameter, const DlfToken& token, + const std::string& host, + Headers* sign_headers) const override; + + private: + std::string region_; +}; + +/// Maps a REST request to the POP action it is registered under in DlfNext 2026-01-18. +class DlfOpenApiActions { + public: + /// One registered operation. A "{name}" segment of `path_template` matches any one + /// non-empty path segment. + struct Operation { + const char* method; + const char* path_template; + const char* action; + }; + + DlfOpenApiActions() = delete; + + /// Every registered operation, in the order requests are matched against them. + static const std::vector& Operations(); + + /// Returns the action of the request, or `std::nullopt` when no operation is registered + /// for it. The method is matched ignoring case, the path segment by segment. + static std::optional Resolve(const std::string& method, + const std::string& resource_path); }; /// Generates DLF authentication headers and refreshes expiring credentials. diff --git a/src/paimon/rest/dlf_auth_test.cpp b/src/paimon/rest/dlf_auth_test.cpp index 5e82ff506..a6e3a3883 100644 --- a/src/paimon/rest/dlf_auth_test.cpp +++ b/src/paimon/rest/dlf_auth_test.cpp @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -36,7 +37,9 @@ #include "gtest/gtest.h" #include "paimon/catalog_options.h" +#include "paimon/rest/resource_paths.h" #include "paimon/testing/utils/testharness.h" +#include "paimon/testing/utils/timezone_guard.h" namespace paimon::test { @@ -48,6 +51,14 @@ std::chrono::system_clock::time_point FixedTime() { return std::chrono::system_clock::from_time_t(1744775086); } +constexpr const char kEmptyBodySha256[] = + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"; + +void AssertAction(const std::optional& expected, const std::string& method, + const std::string& path) { + ASSERT_EQ(expected, DlfOpenApiActions::Resolve(method, path)) << method << " " << path; +} + class SequenceTokenLoader : public DlfTokenLoader { public: explicit SequenceTokenLoader( @@ -159,7 +170,7 @@ TEST(DlfDefaultSignerTest, SignsJavaCompatibleRequest) { ASSERT_EQ("securityToken", headers.at("x-dlf-security-token")); ASSERT_OK_AND_ASSIGN(std::string authorization, - signer.Authorization(parameter, token, "unused", headers)); + signer.Authorization(parameter, token, "unused", &headers)); ASSERT_EQ( "DLF4-HMAC-SHA256 Credential=YourAccessKeyId/20250416/cn-beijing/" "DlfNext/aliyun_v4_request,Signature=" @@ -180,7 +191,7 @@ TEST(DlfDefaultSignerTest, MatchesJavaGoldenAuthorization) { ASSERT_OK_AND_ASSIGN(DlfRequestSigner::Headers headers, signer.SignHeaders(body, signing_time, token.GetSecurityToken(), "host")); ASSERT_OK_AND_ASSIGN(std::string authorization, - signer.Authorization(parameter, token, "host", headers)); + signer.Authorization(parameter, token, "host", &headers)); ASSERT_EQ( "DLF4-HMAC-SHA256 Credential=access-key-id/20231203/cn-hangzhou/" "DlfNext/aliyun_v4_request,Signature=" @@ -220,7 +231,7 @@ TEST(DlfOpenApiSignerTest, SignsJavaCompatibleRequest) { ASSERT_EQ("securityToken", headers.at("x-acs-security-token")); ASSERT_OK_AND_ASSIGN(std::string authorization, - signer.Authorization(parameter, token, host, headers)); + signer.Authorization(parameter, token, host, &headers)); ASSERT_EQ("acs YourAccessKeyId:wX4CDPSCtfgYkxdK9tJIO3ez5VI=", authorization); } @@ -234,7 +245,7 @@ TEST(DlfOpenApiSignerTest, DecodesPathAndQueryValuesBeforeSigning) { headers["x-acs-signature-nonce"] = "fixed-nonce"; ASSERT_OK_AND_ASSIGN(std::string authorization, - signer.Authorization(parameter, token, "host", headers)); + signer.Authorization(parameter, token, "host", &headers)); ASSERT_EQ("acs ak:vD+M7291KKoOTvt2gQuD6jPDlw8=", authorization); ASSERT_EQ(0, headers.count("Content-MD5")); ASSERT_EQ(0, headers.count("Content-Type")); @@ -487,7 +498,7 @@ TEST(DlfDefaultSignerTest, SignsSpecialCharQueryLikeJava) { ASSERT_EQ("20231203T121212Z", headers.at("x-dlf-date")); ASSERT_EQ("F2vHiexkcYvt4XOY5o4tmQ==", headers.at("Content-MD5")); ASSERT_OK_AND_ASSIGN(std::string authorization, - signer.Authorization(parameter, token, "host", headers)); + signer.Authorization(parameter, token, "host", &headers)); ASSERT_EQ( "DLF4-HMAC-SHA256 Credential=access-key-id/20231203/cn-hangzhou/" "DlfNext/aliyun_v4_request,Signature=" @@ -579,6 +590,7 @@ TEST(DlfOpenApiSignerTest, ConcurrentNonceGenerationStaysUnique) { TEST(DlfSignerTest, ExposesJavaCompatibleIdentifiers) { ASSERT_EQ(std::string("default"), std::string(DlfDefaultSigner::kIdentifier)); ASSERT_EQ(std::string("openapi"), std::string(DlfOpenApiSigner::kIdentifier)); + ASSERT_EQ(std::string("openapi-v4"), std::string(DlfOpenApiV4Signer::kIdentifier)); } TEST(DlfAuthProviderTest, ParsesSigningAlgorithmFromUriBoundaries) { @@ -618,4 +630,253 @@ TEST(DlfAuthProviderTest, ParsesRegionFromUriVariants) { "could not determine DLF region"); } +TEST(DlfOpenApiV4SignerTest, SignsHeadersWithBody) { + DlfOpenApiV4Signer signer("cn-beijing"); + const std::string body = R"({"CategoryName":"test","CategoryType":"UNSTRUCTURED"})"; + const std::string host = "dlfnext.cn-beijing.aliyuncs.com"; + ASSERT_OK_AND_ASSIGN(DlfRequestSigner::Headers headers, + signer.SignHeaders(body, FixedTime(), std::nullopt, host)); + ASSERT_EQ("2025-04-16T03:44:46Z", headers.at("x-acs-date")); + ASSERT_EQ(host, headers.at("Host")); + ASSERT_EQ("d8171130c3edc0f7d4e2827c5dab801a603c9a376b214a9ab6d5d12665c8c5df", + headers.at("x-acs-content-sha256")); + ASSERT_EQ("application/json", headers.at("Content-Type")); + ASSERT_FALSE(headers.at("x-acs-signature-nonce").empty()); + ASSERT_EQ("2026-01-18", headers.at("x-acs-version")); + ASSERT_EQ(6, headers.size()); + + ASSERT_NOK_WITH_MSG(signer.SignHeaders(body, FixedTime(), std::nullopt, "").status(), + "host is empty"); +} + +TEST(DlfOpenApiV4SignerTest, OmitsBodyAndSecurityTokenHeadersWhenAbsent) { + DlfOpenApiV4Signer signer("cn-beijing"); + ASSERT_OK_AND_ASSIGN(DlfRequestSigner::Headers headers, + signer.SignHeaders("", FixedTime(), std::nullopt, "host")); + ASSERT_EQ(kEmptyBodySha256, headers.at("x-acs-content-sha256")); + ASSERT_EQ(0, headers.count("Content-Type")); + ASSERT_EQ(0, headers.count("x-acs-security-token")); + ASSERT_EQ(5, headers.size()); +} + +TEST(DlfOpenApiV4SignerTest, SignsUtcDateInNonUtcTimezone) { + TimezoneGuard timezone_guard("Asia/Shanghai"); + DlfOpenApiV4Signer signer("cn-beijing"); + DlfToken token("TestAKId", "TestAKSecret", std::nullopt, std::nullopt); + RestAuthParameter parameter = RestAuthParameter::Create("GET", "/test/path", {}, ""); + ASSERT_OK_AND_ASSIGN(DlfRequestSigner::Headers headers, + signer.SignHeaders("", std::chrono::system_clock::from_time_t(1744747200), + std::nullopt, "host")); + ASSERT_EQ("2025-04-15T20:00:00Z", headers.at("x-acs-date")); + + ASSERT_OK_AND_ASSIGN(std::string authorization, + signer.Authorization(parameter, token, "host", &headers)); + ASSERT_EQ(0, authorization.find("ACS4-HMAC-SHA256 Credential=TestAKId/20250415/cn-beijing/")); +} + +TEST(DlfOpenApiV4SignerTest, MatchesJavaGoldenSignatureForEncodedQueryValues) { + DlfOpenApiV4Signer signer("cn-hangzhou"); + DlfToken token("TestAKId", "TestAKSecret", std::nullopt, std::nullopt); + const std::string host = "dlfnext.cn-hangzhou.aliyuncs.com"; + RestAuthParameter parameter = + RestAuthParameter::Create("GET", "/v1/prefix/functions", + {{"pageToken", "a b/c"}, {"functionNamePattern", "func%"}}, ""); + ASSERT_OK_AND_ASSIGN(DlfRequestSigner::Headers headers, + signer.SignHeaders("", FixedTime(), std::nullopt, host)); + headers["x-acs-signature-nonce"] = "fixed-nonce-for-test"; + + ASSERT_OK_AND_ASSIGN(std::string authorization, + signer.Authorization(parameter, token, host, &headers)); + ASSERT_EQ( + "ACS4-HMAC-SHA256 Credential=TestAKId/20250416/cn-hangzhou/DlfNext/aliyun_v4_request," + "SignedHeaders=host;x-acs-action;x-acs-content-sha256;x-acs-date;" + "x-acs-signature-nonce;x-acs-version," + "Signature=2d5fdb5201901a5cdceee97318366666a88504eeae0b1ce534dbf86889cc00fb", + authorization); + ASSERT_EQ("ListFunctionsGlobally", headers.at("x-acs-action")); +} + +TEST(DlfOpenApiV4SignerTest, MatchesJavaGoldenSignatureWithAction) { + DlfOpenApiV4Signer signer("cn-hangzhou"); + DlfToken token("TestAKId", "TestAKSecret", std::nullopt, std::nullopt); + const std::string host = "dlfnext.cn-hangzhou.aliyuncs.com"; + const std::string body = R"({"identifier":{"database":"db","object":"t"}})"; + RestAuthParameter parameter = + RestAuthParameter::Create("POST", "/v1/clg-paimon-1/databases/db/tables", {}, body); + ASSERT_OK_AND_ASSIGN(DlfRequestSigner::Headers headers, + signer.SignHeaders(body, FixedTime(), std::nullopt, host)); + headers["x-acs-signature-nonce"] = "fixed-nonce-for-test"; + + ASSERT_OK_AND_ASSIGN(std::string authorization, + signer.Authorization(parameter, token, host, &headers)); + ASSERT_EQ("CreateTable", headers.at("x-acs-action")); + ASSERT_EQ( + "ACS4-HMAC-SHA256 Credential=TestAKId/20250416/cn-hangzhou/DlfNext/aliyun_v4_request," + "SignedHeaders=content-type;host;x-acs-action;x-acs-content-sha256;" + "x-acs-date;x-acs-signature-nonce;x-acs-version," + "Signature=420206b5263536e6bc271a7a32582b4a820e23c8c58ae692b708c69f9d19e51d", + authorization); +} + +TEST(DlfOpenApiV4SignerTest, SignatureDependsOnRegion) { + DlfToken token("TestAKId", "TestAKSecret", std::nullopt, std::nullopt); + RestAuthParameter parameter = RestAuthParameter::Create("GET", "/test/path", {}, ""); + DlfOpenApiV4Signer beijing("cn-beijing"); + ASSERT_OK_AND_ASSIGN(DlfRequestSigner::Headers headers, + beijing.SignHeaders("", FixedTime(), std::nullopt, "host")); + headers["x-acs-signature-nonce"] = "fixed-nonce-for-test"; + ASSERT_OK_AND_ASSIGN(std::string beijing_authorization, + beijing.Authorization(parameter, token, "host", &headers)); + + DlfOpenApiV4Signer hangzhou("cn-hangzhou"); + ASSERT_OK_AND_ASSIGN(std::string hangzhou_authorization, + hangzhou.Authorization(parameter, token, "host", &headers)); + size_t beijing_signature_start = beijing_authorization.find(",Signature="); + size_t hangzhou_signature_start = hangzhou_authorization.find(",Signature="); + ASSERT_NE(std::string::npos, beijing_signature_start); + ASSERT_NE(std::string::npos, hangzhou_signature_start); + ASSERT_NE(beijing_authorization.substr(beijing_signature_start), + hangzhou_authorization.substr(hangzhou_signature_start)); + ASSERT_NE(std::string::npos, beijing_authorization.find("/cn-beijing/DlfNext/")); + ASSERT_NE(std::string::npos, hangzhou_authorization.find("/cn-hangzhou/DlfNext/")); +} + +TEST(DlfOpenApiV4SignerTest, SignsHeaderNamesInLowerCaseAndValuesTrimmed) { + DlfOpenApiV4Signer signer("cn-beijing"); + DlfToken token("TestAKId", "TestAKSecret", std::nullopt, std::nullopt); + RestAuthParameter parameter = RestAuthParameter::Create("GET", "/test/path", {}, ""); + ASSERT_OK_AND_ASSIGN(DlfRequestSigner::Headers headers, + signer.SignHeaders("", FixedTime(), std::nullopt, "host")); + headers["x-acs-signature-nonce"] = "fixed-nonce-for-test"; + DlfRequestSigner::Headers renamed_headers = headers; + renamed_headers.erase("x-acs-signature-nonce"); + renamed_headers["X-ACS-SIGNATURE-NONCE"] = " fixed-nonce-for-test "; + + ASSERT_OK_AND_ASSIGN(std::string expected, + signer.Authorization(parameter, token, "host", &headers)); + ASSERT_OK_AND_ASSIGN(std::string actual, + signer.Authorization(parameter, token, "host", &renamed_headers)); + ASSERT_EQ(expected, actual); +} + +TEST(DlfOpenApiV4SignerTest, SignsUnregisteredPathWithoutAction) { + DlfOpenApiV4Signer signer("cn-beijing"); + DlfToken token("TestAKId", "TestAKSecret", std::nullopt, std::nullopt); + RestAuthParameter parameter = + RestAuthParameter::Create("GET", "/v1/clg-paimon-1/databases/db/tables/t/unknown", {}, ""); + ASSERT_OK_AND_ASSIGN(DlfRequestSigner::Headers headers, + signer.SignHeaders("", FixedTime(), std::nullopt, "host")); + + ASSERT_OK_AND_ASSIGN(std::string authorization, + signer.Authorization(parameter, token, "host", &headers)); + ASSERT_EQ(0, headers.count("x-acs-action")); + ASSERT_EQ(std::string::npos, authorization.find("x-acs-action")); +} + +TEST(DlfOpenApiV4SignerTest, RequiresSigningDate) { + DlfOpenApiV4Signer signer("cn-beijing"); + DlfToken token("TestAKId", "TestAKSecret", std::nullopt, std::nullopt); + RestAuthParameter parameter = RestAuthParameter::Create("GET", "/test/path", {}, ""); + ASSERT_OK_AND_ASSIGN(DlfRequestSigner::Headers headers, + signer.SignHeaders("", FixedTime(), std::nullopt, "host")); + headers["x-acs-date"] = "2025"; + ASSERT_NOK_WITH_MSG(signer.Authorization(parameter, token, "host", &headers).status(), + "signing date is invalid"); + headers.erase("x-acs-date"); + ASSERT_NOK_WITH_MSG(signer.Authorization(parameter, token, "host", &headers).status(), + "x-acs-date"); +} + +TEST(DlfAuthProviderTest, SignsWithOpenApiV4OnlyWhenConfigured) { + std::map options = { + {CatalogOptions::URI, "https://dlfnext.cn-hangzhou.aliyuncs.com"}, + {CatalogOptions::TOKEN_PROVIDER, "dlf"}, + {CatalogOptions::DLF_ACCESS_KEY_ID, "ak"}, + {CatalogOptions::DLF_ACCESS_KEY_SECRET, "sk"}, + {CatalogOptions::DLF_SECURITY_TOKEN, "sts"}}; + ASSERT_EQ("openapi", + DlfAuthProvider::ParseSigningAlgorithmFromUri(options.at(CatalogOptions::URI))); + const std::string body = R"({"name":"db"})"; + RestAuthParameter parameter = + RestAuthParameter::Create("POST", "/v1/clg-paimon-1/databases", {}, body); + const StringMap base_headers = {{"Content-Type", "application/json"}}; + for (const char* algorithm : {DlfDefaultSigner::kIdentifier, DlfOpenApiSigner::kIdentifier}) { + options[CatalogOptions::DLF_SIGNING_ALGORITHM] = algorithm; + ASSERT_OK_AND_ASSIGN(std::unique_ptr provider, AuthProvider::Create(options)); + ASSERT_OK_AND_ASSIGN(StringMap headers, provider->MergeAuthHeader(base_headers, parameter)); + ASSERT_EQ(0, headers.count("x-acs-action")) << algorithm; + } + + options[CatalogOptions::DLF_SIGNING_ALGORITHM] = DlfOpenApiV4Signer::kIdentifier; + ASSERT_OK_AND_ASSIGN(std::unique_ptr provider, AuthProvider::Create(options)); + ASSERT_OK_AND_ASSIGN(StringMap headers, provider->MergeAuthHeader(base_headers, parameter)); + const std::string& authorization = headers.at("Authorization"); + ASSERT_EQ(0, authorization.find("ACS4-HMAC-SHA256 Credential=ak/")); + ASSERT_NE(std::string::npos, + authorization.find("/cn-hangzhou/DlfNext/aliyun_v4_request,SignedHeaders=" + "content-type;host;x-acs-action;x-acs-content-sha256;x-acs-date;" + "x-acs-security-token;x-acs-signature-nonce;x-acs-version,")); + ASSERT_EQ("CreateDatabase", headers.at("x-acs-action")); + ASSERT_EQ("sts", headers.at("x-acs-security-token")); + ASSERT_EQ("application/json", headers.at("Content-Type")); + ASSERT_EQ(0, headers.count("content-type")); + ASSERT_EQ("dlfnext.cn-hangzhou.aliyuncs.com", headers.at("Host")); + ASSERT_EQ(0, headers.count("host")); +} + +TEST(DlfOpenApiActionsTest, EveryOperationResolvesToItself) { + static const std::regex kVariable("\\{[^}]+\\}"); + std::set actions; + for (const DlfOpenApiActions::Operation& operation : DlfOpenApiActions::Operations()) { + AssertAction(operation.action, operation.method, + std::regex_replace(operation.path_template, kVariable, "x1")); + actions.insert(operation.action); + } + ASSERT_EQ(DlfOpenApiActions::Operations().size(), actions.size()); + ASSERT_EQ(74, actions.size()); +} + +TEST(DlfOpenApiActionsTest, ClientPathsResolveToRegisteredActions) { + ResourcePaths paths("clg-paimon-1"); + AssertAction("GetConfig", "GET", ResourcePaths::Config()); + AssertAction("ListDatabases", "GET", paths.Databases()); + AssertAction("CreateDatabase", "POST", paths.Databases()); + AssertAction("GetDatabase", "GET", paths.Database("db")); + AssertAction("DropDatabase", "DELETE", paths.Database("db")); + AssertAction("ListTables", "GET", paths.Tables("db")); + AssertAction("CreateTable", "POST", paths.Tables("db")); + AssertAction("GetTable", "GET", paths.Table("db", "t")); + AssertAction("DropTable", "DELETE", paths.Table("db", "t")); + AssertAction("RenameTable", "POST", paths.RenameTable()); + AssertAction("ListSnapshots", "GET", paths.Snapshots("db", "t")); + AssertAction("CommitTable", "POST", paths.CommitTable("db", "t")); + AssertAction("GetTableSnapshot", "GET", paths.TableSnapshot("db", "t")); + AssertAction("GetTableToken", "GET", paths.TableToken("db", "t")); +} + +TEST(DlfOpenApiActionsTest, MatchesWholeSegmentsAndMethodIgnoringCase) { + ResourcePaths paths("clg-paimon-1"); + AssertAction("GetConfig", "get", ResourcePaths::Config()); + AssertAction("GetDatabase", "GET", paths.Database("tables")); + AssertAction("GetTable", "GET", paths.Table("db", "token")); + AssertAction("GetTableToken", "GET", paths.TableToken("config", "t$snapshots")); + AssertAction("GetTable", "GET", paths.Table("a/b", "c d")); + AssertAction("ListDatabases", "GET", ResourcePaths("rename").Databases()); + AssertAction("DropBranch", "DELETE", "/v1/clg-paimon-1/databases/db/tables/t/branches/forward"); + AssertAction("GetTag", "GET", "/v1/clg-paimon-1/databases/db/tables/t/tags/tags"); +} + +TEST(DlfOpenApiActionsTest, UnregisteredRequestsHaveNoAction) { + ResourcePaths paths("clg-paimon-1"); + AssertAction(std::nullopt, "GET", paths.RenameTable()); + AssertAction(std::nullopt, "DELETE", "/v1/clg-paimon-1/tables/id/tbl-1"); + AssertAction(std::nullopt, "GET", paths.CommitTable("db", "t")); + AssertAction(std::nullopt, "PUT", paths.Table("db", "t")); + AssertAction(std::nullopt, "GET", ResourcePaths::Config() + "/"); + AssertAction(std::nullopt, "GET", "/v1//databases"); + AssertAction(std::nullopt, "GET", ResourcePaths("").Databases()); + AssertAction(std::nullopt, "", ResourcePaths::Config()); + AssertAction(std::nullopt, "GET", ""); +} + } // namespace paimon::test