From da615de9cac733b69721e52ce603cedf239d2578 Mon Sep 17 00:00:00 2001 From: "Sung Yun (CODE SIGNING KEY)" Date: Thu, 24 Sep 2026 21:02:09 -0400 Subject: [PATCH 1/2] introduce batch signing endpoint --- open-api/rest-catalog-open-api.py | 74 ++++++++++++ open-api/rest-catalog-open-api.yaml | 168 ++++++++++++++++++++++++++++ 2 files changed, 242 insertions(+) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index 5e3580ae3825..059043e48ddc 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -1175,6 +1175,64 @@ class RemoteSignResult(BaseModel): headers: MultiValuedMap +class RemoteSignBatchElement(BaseModel): + """ + The fields of one request in a batch that are not repeated with the other requests. + """ + + uri: str + headers: MultiValuedMap + body: str | None = Field( + None, + description='Optional body of the request to send to the signing API, as in `RemoteSignRequest`.', + ) + + +class SignStatus(RootModel[Literal['completed', 'failed']]): + root: Literal['completed', 'failed'] = Field( + ..., description='Status of signing one element of a batch' + ) + + +class CompletedSignResult(RemoteSignResult): + """ + A successfully signed batch element + """ + + status: Literal['completed'] = Field( + ..., description='Status of signing one element of a batch' + ) + + +class FailedSignResult(BaseModel): + """ + A batch element that could not be signed + """ + + status: Literal['failed'] = Field( + ..., description='Status of signing one element of a batch' + ) + error: ErrorModel + + +class RemoteSignBatchResult(RootModel[CompletedSignResult | FailedSignResult]): + root: CompletedSignResult | FailedSignResult = Field( + ..., + description='The result of signing one element of a batch', + discriminator='status', + ) + + +class RemoteSignBatchResponse(BaseModel): + """ + The result of a batch remote request signing operation. + """ + + results: list[RemoteSignBatchResult] = Field( + ..., description='Results for each element of the request, in request order' + ) + + class RemoteSigningConfig(BaseModel): """ Configuration for the remote signer client. @@ -1370,6 +1428,22 @@ class FetchScanTasksRequest(BaseModel): plan_task: PlanTask = Field(..., alias='plan-task') +class RemoteSignBatchRequest(BaseModel): + """ + A batch of requests to be signed remotely. `region`, `method`, `provider` and `properties` apply to every element of `requests`. + + """ + + region: str + method: Literal['PUT', 'GET', 'HEAD', 'POST', 'DELETE', 'PATCH', 'OPTIONS'] + provider: str | None = Field( + None, + description='The storage provider for every element of the batch. If this is not specified, the provider is assumed to be `s3`, as in `RemoteSignRequest`.', + ) + properties: dict[str, str] | None = None + requests: list[RemoteSignBatchElement] = Field(..., min_length=1) + + class Term(RootModel[TermReference | TransformTerm]): root: TermReference | TransformTerm = Field( ..., diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index e9a530a12804..29c0e5830ab0 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -1429,6 +1429,53 @@ paths: 5XX: $ref: '#/components/responses/ServerErrorResponse' + /v1/{prefix}/namespaces/{namespace}/tables/{table}/sign/batch: + parameters: + - $ref: '#/components/parameters/prefix' + - $ref: '#/components/parameters/namespace' + - $ref: '#/components/parameters/table' + - $ref: '#/components/parameters/data-access' + + post: + tags: + - Catalog API + summary: Remotely signs a batch of requests to object storage + operationId: batchSignRequests + description: > + Signs a batch of requests to object storage on the client's behalf. + + A server MAY + advertise its batch size limit as `sign-batch-max-size` in `CatalogConfig`. If not the value is not advertised, the server + SHOULD use a default batch size of 100 elements. A batch above the server's limit is rejected with 413. + + `X-Iceberg-Access-Delegation` applies to the batch as a whole. + requestBody: + description: The batch of requests to be signed + content: + application/json: + schema: + $ref: '#/components/schemas/RemoteSignBatchRequest' + required: true + responses: + 200: + $ref: '#/components/responses/RemoteSignBatchResponse' + 400: + $ref: '#/components/responses/BadRequestErrorResponse' + 401: + $ref: '#/components/responses/UnauthorizedResponse' + 403: + $ref: '#/components/responses/ForbiddenResponse' + 406: + $ref: '#/components/responses/UnsupportedOperationResponse' + 413: + $ref: '#/components/responses/ContentTooLargeResponse' + 419: + $ref: '#/components/responses/AuthenticationTimeoutResponse' + 503: + $ref: '#/components/responses/ServiceUnavailableResponse' + 5XX: + $ref: '#/components/responses/ServerErrorResponse' + /v1/{prefix}/tables/rename: parameters: - $ref: '#/components/parameters/prefix' @@ -5466,6 +5513,104 @@ components: headers: $ref: '#/components/schemas/MultiValuedMap' + RemoteSignBatchRequest: + description: > + A batch of requests to be signed remotely. `region`, `method`, `provider` and + `properties` apply to every element of `requests`. + type: object + required: + - region + - method + - requests + properties: + region: + type: string + method: + type: string + enum: ["PUT", "GET", "HEAD", "POST", "DELETE", "PATCH", "OPTIONS"] + provider: + type: string + description: The storage provider for every element of the batch. If this is not specified, the + provider is assumed to be `s3`, as in `RemoteSignRequest`. + properties: + type: object + additionalProperties: + type: string + requests: + type: array + minItems: 1 + items: + $ref: '#/components/schemas/RemoteSignBatchElement' + + RemoteSignBatchElement: + description: The fields of one request in a batch that are not repeated with the other requests. + type: object + required: + - uri + - headers + properties: + uri: + type: string + headers: + $ref: '#/components/schemas/MultiValuedMap' + body: + type: string + description: Optional body of the request to send to the signing API, as in `RemoteSignRequest`. + + SignStatus: + description: Status of signing one element of a batch + type: string + enum: ["completed", "failed"] + + CompletedSignResult: + description: A successfully signed batch element + allOf: + - $ref: '#/components/schemas/RemoteSignResult' + - type: object + required: + - status + properties: + status: + $ref: '#/components/schemas/SignStatus' + const: "completed" + + FailedSignResult: + description: A batch element that could not be signed + type: object + required: + - status + - error + properties: + status: + $ref: '#/components/schemas/SignStatus' + const: "failed" + error: + $ref: '#/components/schemas/ErrorModel' + + RemoteSignBatchResult: + type: object + description: The result of signing one element of a batch + discriminator: + propertyName: status + mapping: + completed: '#/components/schemas/CompletedSignResult' + failed: '#/components/schemas/FailedSignResult' + oneOf: + - $ref: '#/components/schemas/CompletedSignResult' + - $ref: '#/components/schemas/FailedSignResult' + + RemoteSignBatchResponse: + description: The result of a batch remote request signing operation. + type: object + required: + - results + properties: + results: + description: Results for each element of the request, in request order + type: array + items: + $ref: '#/components/schemas/RemoteSignBatchResult' + RemoteSigningConfig: description: Configuration for the remote signer client. type: object @@ -5662,6 +5807,22 @@ components: } } + ContentTooLargeResponse: + description: + The request body exceeds a limit the server enforces, such as the maximum number of + elements in a batch. + content: + application/json: + schema: + $ref: '#/components/schemas/IcebergErrorResponse' + example: { + "error": { + "message": "Batch exceeds the maximum of 100 elements", + "type": "ContentTooLargeException", + "code": 413 + } + } + ServiceUnavailableResponse: description: The service is not ready to handle the request, request could have been partially processed. @@ -5800,6 +5961,13 @@ components: schema: $ref: '#/components/schemas/RemoteSignResult' + RemoteSignBatchResponse: + description: The results of signing each element of a batch, in request order. + content: + application/json: + schema: + $ref: '#/components/schemas/RemoteSignBatchResponse' + ####################################### # Common examples of different values # ####################################### From 3207a0847d6c29062b1dc40697a20d602d23e117 Mon Sep 17 00:00:00 2001 From: "Sung Yun (CODE SIGNING KEY)" Date: Tue, 20 Oct 2026 10:21:15 -0400 Subject: [PATCH 2/2] adopt feedback from dan --- open-api/rest-catalog-open-api.py | 41 +++++++-------- open-api/rest-catalog-open-api.yaml | 81 +++++++++++++++-------------- 2 files changed, 59 insertions(+), 63 deletions(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index 059043e48ddc..91a52b8d993f 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -1188,19 +1188,13 @@ class RemoteSignBatchElement(BaseModel): ) -class SignStatus(RootModel[Literal['completed', 'failed']]): - root: Literal['completed', 'failed'] = Field( - ..., description='Status of signing one element of a batch' - ) - - class CompletedSignResult(RemoteSignResult): """ A successfully signed batch element """ - status: Literal['completed'] = Field( - ..., description='Status of signing one element of a batch' + index: int = Field( + ..., description='The 0-based position of the element in `requests`', ge=0 ) @@ -1209,27 +1203,22 @@ class FailedSignResult(BaseModel): A batch element that could not be signed """ - status: Literal['failed'] = Field( - ..., description='Status of signing one element of a batch' + index: int = Field( + ..., description='The 0-based position of the element in `requests`', ge=0 ) error: ErrorModel -class RemoteSignBatchResult(RootModel[CompletedSignResult | FailedSignResult]): - root: CompletedSignResult | FailedSignResult = Field( - ..., - description='The result of signing one element of a batch', - discriminator='status', - ) - - class RemoteSignBatchResponse(BaseModel): """ The result of a batch remote request signing operation. """ - results: list[RemoteSignBatchResult] = Field( - ..., description='Results for each element of the request, in request order' + completed: list[CompletedSignResult] = Field( + ..., description='Elements that were signed' + ) + failed: list[FailedSignResult] = Field( + ..., description='Elements that could not be signed' ) @@ -1246,6 +1235,12 @@ class RemoteSigningConfig(BaseModel): None, description='Static headers the signer client MUST include unchanged in every request to the signing endpoint.\n', ) + sign_batch_max_size: int | None = Field( + None, + alias='sign-batch-max-size', + description='The maximum number of elements in one `batchSignRequests` request for this table.', + ge=1, + ) class CreateNamespaceRequest(BaseModel): @@ -1436,9 +1431,9 @@ class RemoteSignBatchRequest(BaseModel): region: str method: Literal['PUT', 'GET', 'HEAD', 'POST', 'DELETE', 'PATCH', 'OPTIONS'] - provider: str | None = Field( - None, - description='The storage provider for every element of the batch. If this is not specified, the provider is assumed to be `s3`, as in `RemoteSignRequest`.', + provider: str = Field( + ..., + description='The storage provider for every element of the batch. The provider should correspond to the scheme used for a storage native URI. For example `s3` for AWS S3 paths.', ) properties: dict[str, str] | None = None requests: list[RemoteSignBatchElement] = Field(..., min_length=1) diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index 29c0e5830ab0..19f8448922d2 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -1441,12 +1441,18 @@ paths: - Catalog API summary: Remotely signs a batch of requests to object storage operationId: batchSignRequests - description: > + description: Signs a batch of requests to object storage on the client's behalf. - A server MAY - advertise its batch size limit as `sign-batch-max-size` in `CatalogConfig`. If not the value is not advertised, the server - SHOULD use a default batch size of 100 elements. A batch above the server's limit is rejected with 413. + + A server that supports this operation for a table MUST set `sign-batch-max-size` in the + table's `RemoteSigningConfig`. A client MUST NOT send more elements than that value. A + batch above the server's limit is rejected with 413. + + + A 200 response means the batch was processed. Each element of `requests` is reported once, + by its 0-based index, in either `completed` or `failed`. + `X-Iceberg-Access-Delegation` applies to the batch as a whole. requestBody: @@ -1468,7 +1474,7 @@ paths: 406: $ref: '#/components/responses/UnsupportedOperationResponse' 413: - $ref: '#/components/responses/ContentTooLargeResponse' + $ref: '#/components/responses/BatchContentTooLargeResponse' 419: $ref: '#/components/responses/AuthenticationTimeoutResponse' 503: @@ -5521,6 +5527,7 @@ components: required: - region - method + - provider - requests properties: region: @@ -5530,8 +5537,8 @@ components: enum: ["PUT", "GET", "HEAD", "POST", "DELETE", "PATCH", "OPTIONS"] provider: type: string - description: The storage provider for every element of the batch. If this is not specified, the - provider is assumed to be `s3`, as in `RemoteSignRequest`. + description: The storage provider for every element of the batch. The provider should correspond + to the scheme used for a storage native URI. For example `s3` for AWS S3 paths. properties: type: object additionalProperties: @@ -5557,59 +5564,50 @@ components: type: string description: Optional body of the request to send to the signing API, as in `RemoteSignRequest`. - SignStatus: - description: Status of signing one element of a batch - type: string - enum: ["completed", "failed"] - CompletedSignResult: description: A successfully signed batch element allOf: - $ref: '#/components/schemas/RemoteSignResult' - type: object required: - - status + - index properties: - status: - $ref: '#/components/schemas/SignStatus' - const: "completed" + index: + type: integer + minimum: 0 + description: The 0-based position of the element in `requests` FailedSignResult: description: A batch element that could not be signed type: object required: - - status + - index - error properties: - status: - $ref: '#/components/schemas/SignStatus' - const: "failed" + index: + type: integer + minimum: 0 + description: The 0-based position of the element in `requests` error: $ref: '#/components/schemas/ErrorModel' - RemoteSignBatchResult: - type: object - description: The result of signing one element of a batch - discriminator: - propertyName: status - mapping: - completed: '#/components/schemas/CompletedSignResult' - failed: '#/components/schemas/FailedSignResult' - oneOf: - - $ref: '#/components/schemas/CompletedSignResult' - - $ref: '#/components/schemas/FailedSignResult' - RemoteSignBatchResponse: description: The result of a batch remote request signing operation. type: object required: - - results + - completed + - failed properties: - results: - description: Results for each element of the request, in request order + completed: + description: Elements that were signed + type: array + items: + $ref: '#/components/schemas/CompletedSignResult' + failed: + description: Elements that could not be signed type: array items: - $ref: '#/components/schemas/RemoteSignBatchResult' + $ref: '#/components/schemas/FailedSignResult' RemoteSigningConfig: description: Configuration for the remote signer client. @@ -5628,6 +5626,10 @@ components: description: > Static headers the signer client MUST include unchanged in every request to the signing endpoint. + sign-batch-max-size: + type: integer + minimum: 1 + description: The maximum number of elements in one `batchSignRequests` request for this table. ############################# # Reusable Response Objects # @@ -5807,10 +5809,9 @@ components: } } - ContentTooLargeResponse: + BatchContentTooLargeResponse: description: - The request body exceeds a limit the server enforces, such as the maximum number of - elements in a batch. + The batch has more elements than the server accepts, as set in `sign-batch-max-size`. content: application/json: schema: @@ -5962,7 +5963,7 @@ components: $ref: '#/components/schemas/RemoteSignResult' RemoteSignBatchResponse: - description: The results of signing each element of a batch, in request order. + description: The outcome of signing each element of a batch. content: application/json: schema: