From 3f7b0e5c26d9add06f277753e55b26aa81e27600 Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Wed, 20 Aug 2025 12:57:54 -0700 Subject: [PATCH 01/51] [SPEC] Add FGAC enforcement instructions as part of loadTable --- open-api/rest-catalog-open-api.py | 22 ++++++++++++++++++++++ open-api/rest-catalog-open-api.yaml | 24 ++++++++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index 5e3580ae3825..fff7c9bea3f0 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -1703,6 +1703,25 @@ class AddSchemaUpdate(BaseUpdate): ) +class FineGrainedDataProtectionRules(BaseModel): + """ + Fine-grained data protection rules for a table as result of fine grained policy evaluation at the catalog end based on the clients access rights. + The client SHOULD use these rules to enforce fine-grained data protection like column and row level access when reading data from the table. + + """ + + projections: Optional[List[Term]] = Field( + None, + description='This field contains a list of columns or column transforms to be projected.\nNote: That each column must have only a single projection, meaning a column can be projected as-is or as a transformed value, but not both.\n', + ) + row_filter: Optional[Expression] = Field( + None, + alias='row-filter', + description='An expression that filters rows. Only rows for which the expression evaluates to true should be allowed to read. If the catalog supports multiple row access filter against the table, its the catalogs responsibility to combine them with the appropriate logic (e.g., AND, OR).\n', + ) + + + class LoadTableResult(BaseModel): """ Result used when a table is successfully loaded. @@ -1766,6 +1785,9 @@ class LoadTableResult(BaseModel): remote_signing_config: RemoteSigningConfig | None = Field( None, alias='remote-signing-config' ) + fine_grained_data_protection_instructions: Optional[ + FineGrainedDataProtectionRules + ] = Field(None, alias='fine-grained-data-protection-instructions') class ScanTasks(BaseModel): diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index e9a530a12804..4eb8bfdd1a4a 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3839,6 +3839,28 @@ components: additionalProperties: type: string + FineGrainedDataProtectionRules: + type: object + description: > + Fine-grained data protection rules for a table as result of fine grained policy evaluation at the catalog end based on the clients access rights. + + The client SHOULD use these rules to enforce fine-grained data protection like column and row level access when reading data from the table. + properties: + projections: + description: > + This field contains a list of columns or column transforms to be projected. + + Note: That each column must have only a single projection, meaning a column can be projected as-is or as a transformed value, but not both. + type: array + items: + $ref: '#/components/schemas/Term' + row-filter: + description: > + An expression that filters rows. Only rows for which the expression evaluates to true should be allowed to read. + If the catalog supports multiple row access filter against the table, its the catalogs responsibility to combine them with + the appropriate logic (e.g., AND, OR). + $ref: '#/components/schemas/Expression' + LoadCredentialsResponse: type: object required: @@ -3917,6 +3939,8 @@ components: $ref: '#/components/schemas/StorageCredential' remote-signing-config: $ref: '#/components/schemas/RemoteSigningConfig' + fine-grained-data-protection-instructions: + $ref: '#/components/schemas/FineGrainedDataProtectionRules' ScanTasks: type: object From 42f56c36acc95e239e294e571fb6b8f1983af26c Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Wed, 20 Aug 2025 19:08:34 -0700 Subject: [PATCH 02/51] Address review feedbacks --- open-api/rest-catalog-open-api.py | 22 +++++++++++----------- open-api/rest-catalog-open-api.yaml | 22 +++++++++++----------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index fff7c9bea3f0..fb0067f4184b 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -1703,21 +1703,21 @@ class AddSchemaUpdate(BaseUpdate): ) -class FineGrainedDataProtectionRules(BaseModel): +class ReadRestrictions(BaseModel): """ - Fine-grained data protection rules for a table as result of fine grained policy evaluation at the catalog end based on the clients access rights. - The client SHOULD use these rules to enforce fine-grained data protection like column and row level access when reading data from the table. + Read Restrictions for a table including projection and row filter expressions. The client MUST enforce these rules to read data from the table. """ - projections: Optional[List[Term]] = Field( + required_projection: Optional[List[Term]] = Field( None, - description='This field contains a list of columns or column transforms to be projected.\nNote: That each column must have only a single projection, meaning a column can be projected as-is or as a transformed value, but not both.\n', + alias='required-projection', + description='A list of projections that must be applied before query projections. If the term is a transform, it must replace the column referenced by the term. Readers are not allowed to project columns that are not listed and must apply transforms.\nNote: That each column must have only a single projection, meaning a column can be projected as-is or as a transformed value, but not both.\n', ) - row_filter: Optional[Expression] = Field( + required_row_filter: Optional[Expression] = Field( None, - alias='row-filter', - description='An expression that filters rows. Only rows for which the expression evaluates to true should be allowed to read. If the catalog supports multiple row access filter against the table, its the catalogs responsibility to combine them with the appropriate logic (e.g., AND, OR).\n', + alias='required-row-filter', + description='An expression that filters rows. Rows for which the filter evaluates to false must be discarded and no information derived from the filtered rows may be included in the query result. If the catalog supports multiple row access filter against the table, it is the catalogs responsibility to combine them with the appropriate logic (e.g., AND, OR).\n', ) @@ -1785,9 +1785,9 @@ class LoadTableResult(BaseModel): remote_signing_config: RemoteSigningConfig | None = Field( None, alias='remote-signing-config' ) - fine_grained_data_protection_instructions: Optional[ - FineGrainedDataProtectionRules - ] = Field(None, alias='fine-grained-data-protection-instructions') + read_restrictions: Optional[ReadRestrictions] = Field( + None, alias='read-restrictions' + ) class ScanTasks(BaseModel): diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index 4eb8bfdd1a4a..c046ec9a77ef 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3839,25 +3839,25 @@ components: additionalProperties: type: string - FineGrainedDataProtectionRules: + ReadRestrictions: type: object description: > - Fine-grained data protection rules for a table as result of fine grained policy evaluation at the catalog end based on the clients access rights. - - The client SHOULD use these rules to enforce fine-grained data protection like column and row level access when reading data from the table. + Read Restrictions for a table including projection and row filter expressions. + The client MUST enforce these rules to read data from the table. properties: - projections: + required-projection: description: > - This field contains a list of columns or column transforms to be projected. + A list of projections that must be applied before query projections. If the term is a transform, it must replace the column referenced by the term. + Readers are not allowed to project columns that are not listed and must apply transforms. Note: That each column must have only a single projection, meaning a column can be projected as-is or as a transformed value, but not both. type: array items: $ref: '#/components/schemas/Term' - row-filter: + required-row-filter: description: > - An expression that filters rows. Only rows for which the expression evaluates to true should be allowed to read. - If the catalog supports multiple row access filter against the table, its the catalogs responsibility to combine them with + An expression that filters rows. Rows for which the filter evaluates to false must be discarded and no information derived from the filtered rows may be included in the query result. + If the catalog supports multiple row access filter against the table, it is the catalogs responsibility to combine them with the appropriate logic (e.g., AND, OR). $ref: '#/components/schemas/Expression' @@ -3939,8 +3939,8 @@ components: $ref: '#/components/schemas/StorageCredential' remote-signing-config: $ref: '#/components/schemas/RemoteSigningConfig' - fine-grained-data-protection-instructions: - $ref: '#/components/schemas/FineGrainedDataProtectionRules' + read-restrictions: + $ref: '#/components/schemas/ReadRestrictions' ScanTasks: type: object From cdcc2b6959e08fa8b30af00a167b9007cd4f9f8c Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Thu, 21 Aug 2025 14:32:37 -0700 Subject: [PATCH 03/51] Address review feedbacks --- open-api/rest-catalog-open-api.py | 6 +++--- open-api/rest-catalog-open-api.yaml | 7 ++++++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index fb0067f4184b..300718b1569c 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -1705,19 +1705,19 @@ class AddSchemaUpdate(BaseUpdate): class ReadRestrictions(BaseModel): """ - Read Restrictions for a table including projection and row filter expressions. The client MUST enforce these rules to read data from the table. + Read Restrictions for a table including projection and row filter expressions. The client MUST enforce these rules to read data from the table. If the read-restrictions section is not present or is empty, clients MUST treat it as equivalent to having no restrictions. """ required_projection: Optional[List[Term]] = Field( None, alias='required-projection', - description='A list of projections that must be applied before query projections. If the term is a transform, it must replace the column referenced by the term. Readers are not allowed to project columns that are not listed and must apply transforms.\nNote: That each column must have only a single projection, meaning a column can be projected as-is or as a transformed value, but not both.\n', + description='A list of projections that must be applied before query projections. If the term is a transform, it must replace the column referenced by the term. For example, if the term is mask(cc, 0, 4) i.e mask transform on column cc, it must replace the column cc in the query with the masked value, essentially projecting it as mask(cc, 0, 4) AS cc. Readers are NOT allowed to project columns that are not listed and must apply transforms. If the required-projection is not present or is empty, it means that no projection is required and all columns can be read as-is.\nNote: That each column must have only a single projection, meaning a column can be projected as-is or as a transformed value, but not both.\n', ) required_row_filter: Optional[Expression] = Field( None, alias='required-row-filter', - description='An expression that filters rows. Rows for which the filter evaluates to false must be discarded and no information derived from the filtered rows may be included in the query result. If the catalog supports multiple row access filter against the table, it is the catalogs responsibility to combine them with the appropriate logic (e.g., AND, OR).\n', + description='An expression that filters rows. Rows for which the filter evaluates to false must be discarded and no information derived from the filtered rows may be included in the query result. If the catalog supports multiple row access filter against the table, it is the catalogs responsibility to combine them with the appropriate logic (e.g., AND, OR). If the required-row-filter is not present or is empty, it means that no row filtering is required and all rows can be read.\n', ) diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index c046ec9a77ef..714386f1781e 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3844,11 +3844,15 @@ components: description: > Read Restrictions for a table including projection and row filter expressions. The client MUST enforce these rules to read data from the table. + If the read-restrictions section is not present or is empty, clients MUST treat it as equivalent to having no restrictions. properties: required-projection: description: > A list of projections that must be applied before query projections. If the term is a transform, it must replace the column referenced by the term. - Readers are not allowed to project columns that are not listed and must apply transforms. + For example, if the term is mask(cc, 0, 4) i.e mask transform on column cc, it must replace the column cc in the query with the masked value, + essentially projecting it as mask(cc, 0, 4) AS cc. + Readers are NOT allowed to project columns that are not listed and must apply transforms. + If the required-projection is not present or is empty, it means that no projection is required and all columns can be read as-is. Note: That each column must have only a single projection, meaning a column can be projected as-is or as a transformed value, but not both. type: array @@ -3859,6 +3863,7 @@ components: An expression that filters rows. Rows for which the filter evaluates to false must be discarded and no information derived from the filtered rows may be included in the query result. If the catalog supports multiple row access filter against the table, it is the catalogs responsibility to combine them with the appropriate logic (e.g., AND, OR). + If the required-row-filter is not present or is empty, it means that no row filtering is required and all rows can be read. $ref: '#/components/schemas/Expression' LoadCredentialsResponse: From 150dcafc8328314d2f76c756ae3716e883256204 Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Fri, 22 Aug 2025 15:55:50 -0700 Subject: [PATCH 04/51] Address review feedback --- open-api/rest-catalog-open-api.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index 714386f1781e..1e57cbe2ad55 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3849,8 +3849,8 @@ components: required-projection: description: > A list of projections that must be applied before query projections. If the term is a transform, it must replace the column referenced by the term. - For example, if the term is mask(cc, 0, 4) i.e mask transform on column cc, it must replace the column cc in the query with the masked value, - essentially projecting it as mask(cc, 0, 4) AS cc. + For example, if the term is truncate(4, cc) i.e truncate transform on column cc, it must replace the column cc in the query with the transformed value, + essentially projecting it as truncate(4, cc) AS cc. Readers are NOT allowed to project columns that are not listed and must apply transforms. If the required-projection is not present or is empty, it means that no projection is required and all columns can be read as-is. From 6f0a217fe6173487e2ee316e9e55671a919df0a8 Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Mon, 25 Aug 2025 16:12:20 -0700 Subject: [PATCH 05/51] Address review feedbacks and align to RFC-2119 --- open-api/rest-catalog-open-api.py | 6 ++-- open-api/rest-catalog-open-api.yaml | 53 ++++++++++++++++++++++------- 2 files changed, 44 insertions(+), 15 deletions(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index 300718b1569c..dffc046f4510 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -1705,19 +1705,19 @@ class AddSchemaUpdate(BaseUpdate): class ReadRestrictions(BaseModel): """ - Read Restrictions for a table including projection and row filter expressions. The client MUST enforce these rules to read data from the table. If the read-restrictions section is not present or is empty, clients MUST treat it as equivalent to having no restrictions. + Read Restrictions for a table including projection and row filter expressions. The client MUST enforce these rules to read data from the table. """ required_projection: Optional[List[Term]] = Field( None, alias='required-projection', - description='A list of projections that must be applied before query projections. If the term is a transform, it must replace the column referenced by the term. For example, if the term is mask(cc, 0, 4) i.e mask transform on column cc, it must replace the column cc in the query with the masked value, essentially projecting it as mask(cc, 0, 4) AS cc. Readers are NOT allowed to project columns that are not listed and must apply transforms. If the required-projection is not present or is empty, it means that no projection is required and all columns can be read as-is.\nNote: That each column must have only a single projection, meaning a column can be projected as-is or as a transformed value, but not both.\n', + description='A list of projections that MUST be applied prior to any query-specified projections. If the required-projection property is absent or empty, no mandatory projection applies, and a reader MAY project any subset of columns of the table, including all columns.\n1. A reader MUST project only columns listed in the required-projection.\n - If a listed column has a transform, the reader MUST apply it and replace\n all references to the underlying column with the transformed value\n (for example, truncate(4, cc) MUST be projected as truncate(4, cc) AS cc,\n and all references to cc during query evaluation MUST resolve to this alias).\n - If a listed column has no transform, the reader MUST read it as-is.\n - Columns not listed in the required-projection MUST NOT be read.\n\n2. A column MUST appear at most once in the required-projection.\n3. A projection entry MUST reference either the column itself or exactly one\n transformed version of the column, but not both.\n\n4. Multiple transformed versions of the same column (e.g., truncate(5, col)\n and truncate(3, col)) MUST NOT appear in the required-projection.\n\n5. If a projection entry includes a transform that the reader cannot evaluate,\n the reader MUST fail rather than ignore the transform.\n\n6. An identity transform is equivalent to projecting the column directly.\n A reader MAY represent it in either form.\n', ) required_row_filter: Optional[Expression] = Field( None, alias='required-row-filter', - description='An expression that filters rows. Rows for which the filter evaluates to false must be discarded and no information derived from the filtered rows may be included in the query result. If the catalog supports multiple row access filter against the table, it is the catalogs responsibility to combine them with the appropriate logic (e.g., AND, OR). If the required-row-filter is not present or is empty, it means that no row filtering is required and all rows can be read.\n', + description="An expression that filters rows in the table.\n1. A reader MUST discard any row for which the filter evaluates to false, and\n no information derived from discarded rows MAY be included in the query result.\n\n2. If the catalog supports multiple row access filters for the table, it is\n the catalog's responsibility to combine them using the appropriate logic (e.g., AND, OR).\n\n3. If a client cannot interpret or evaluate a provided filter expression, it\n MUST NOT return partially filtered results and MUST fail.\n\n4. If the required-row-filter property is absent or empty, no mandatory\n filtering is imposed, and a reader MAY return any subset of rows,\n including all rows.\n", ) diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index 1e57cbe2ad55..980ac8b1c608 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3844,26 +3844,55 @@ components: description: > Read Restrictions for a table including projection and row filter expressions. The client MUST enforce these rules to read data from the table. - If the read-restrictions section is not present or is empty, clients MUST treat it as equivalent to having no restrictions. properties: required-projection: description: > - A list of projections that must be applied before query projections. If the term is a transform, it must replace the column referenced by the term. - For example, if the term is truncate(4, cc) i.e truncate transform on column cc, it must replace the column cc in the query with the transformed value, - essentially projecting it as truncate(4, cc) AS cc. - Readers are NOT allowed to project columns that are not listed and must apply transforms. - If the required-projection is not present or is empty, it means that no projection is required and all columns can be read as-is. - - Note: That each column must have only a single projection, meaning a column can be projected as-is or as a transformed value, but not both. + A list of projections that MUST be applied prior to any query-specified + projections. + If the required-projection property is absent or empty, no mandatory projection applies, + and a reader MAY project any subset of columns of the table, including all columns. + + 1. A reader MUST project only columns listed in the required-projection. + - If a listed column has a transform, the reader MUST apply it and replace + all references to the underlying column with the transformed value + (for example, truncate(4, cc) MUST be projected as truncate(4, cc) AS cc, + and all references to cc during query evaluation MUST resolve to this alias). + - If a listed column has no transform, the reader MUST read it as-is. + - Columns not listed in the required-projection MUST NOT be read. + + 2. A column MUST appear at most once in the required-projection. + + 3. A projection entry MUST reference either the column itself or exactly one + transformed version of the column, but not both. + + 4. Multiple transformed versions of the same column (e.g., truncate(5, col) + and truncate(3, col)) MUST NOT appear in the required-projection. + + 5. If a projection entry includes a transform that the reader cannot evaluate, + the reader MUST fail rather than ignore the transform. + + 6. An identity transform is equivalent to projecting the column directly. + A reader MAY represent it in either form. + type: array items: $ref: '#/components/schemas/Term' required-row-filter: description: > - An expression that filters rows. Rows for which the filter evaluates to false must be discarded and no information derived from the filtered rows may be included in the query result. - If the catalog supports multiple row access filter against the table, it is the catalogs responsibility to combine them with - the appropriate logic (e.g., AND, OR). - If the required-row-filter is not present or is empty, it means that no row filtering is required and all rows can be read. + An expression that filters rows in the table. + + 1. A reader MUST discard any row for which the filter evaluates to false, and + no information derived from discarded rows MAY be included in the query result. + + 2. If the catalog supports multiple row access filters for the table, it is + the catalog's responsibility to combine them using the appropriate logic (e.g., AND, OR). + + 3. If a client cannot interpret or evaluate a provided filter expression, it + MUST NOT return partially filtered results and MUST fail. + + 4. If the required-row-filter property is absent or empty, no mandatory + filtering is imposed, and a reader MAY return any subset of rows, + including all rows. $ref: '#/components/schemas/Expression' LoadCredentialsResponse: From 596e41cd1d15ba749f48d5dafe942097cefa4f98 Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Tue, 26 Aug 2025 08:36:57 -0700 Subject: [PATCH 06/51] Make who the evaluated policy is applicable to more verbose --- open-api/rest-catalog-open-api.py | 4 +++- open-api/rest-catalog-open-api.yaml | 11 +++++++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index dffc046f4510..9728bc4e994b 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -1705,7 +1705,9 @@ class AddSchemaUpdate(BaseUpdate): class ReadRestrictions(BaseModel): """ - Read Restrictions for a table including projection and row filter expressions. The client MUST enforce these rules to read data from the table. + Read restrictions for a table, including projection and row filter expressions. + A client MUST enforce the restrictions defined in this object when reading data from the table. + These restrictions apply only to the authenticated principal, user, or account associated with the client. They MUST NOT be interpreted as global policy and MUST NOT be applied beyond the entity identified by the Authentication header (or other applicable authentication mechanism). """ diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index 980ac8b1c608..ef63fbd88ade 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3842,8 +3842,15 @@ components: ReadRestrictions: type: object description: > - Read Restrictions for a table including projection and row filter expressions. - The client MUST enforce these rules to read data from the table. + Read restrictions for a table, including projection and row filter expressions. + + A client MUST enforce the restrictions defined in this object when reading data + from the table. + + These restrictions apply only to the authenticated principal, user, or account + associated with the client. They MUST NOT be interpreted as global policy and + MUST NOT be applied beyond the entity identified by the Authentication header + (or other applicable authentication mechanism). properties: required-projection: description: > From 1563cff48fc41d24d4c303c60acd516e086d80ee Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Wed, 27 Aug 2025 11:49:50 -0700 Subject: [PATCH 07/51] Add projection must be applied after filtering --- open-api/rest-catalog-open-api.py | 2 +- open-api/rest-catalog-open-api.yaml | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index 9728bc4e994b..a4afe0eac1bf 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -1719,7 +1719,7 @@ class ReadRestrictions(BaseModel): required_row_filter: Optional[Expression] = Field( None, alias='required-row-filter', - description="An expression that filters rows in the table.\n1. A reader MUST discard any row for which the filter evaluates to false, and\n no information derived from discarded rows MAY be included in the query result.\n\n2. If the catalog supports multiple row access filters for the table, it is\n the catalog's responsibility to combine them using the appropriate logic (e.g., AND, OR).\n\n3. If a client cannot interpret or evaluate a provided filter expression, it\n MUST NOT return partially filtered results and MUST fail.\n\n4. If the required-row-filter property is absent or empty, no mandatory\n filtering is imposed, and a reader MAY return any subset of rows,\n including all rows.\n", + description="An expression that filters rows in the table.\n1. A reader MUST discard any row for which the filter evaluates to false, and\n no information derived from discarded rows MAY be included in the query result.\n\n2. Row filters MUST be evaluated against the original, untransformed column values.\n Required projections MUST be applied only after row filters are applied.\n\n2. If the catalog supports multiple row access filters for the table, it is\n the catalog's responsibility to combine them using the appropriate logic (e.g., AND, OR).\n\n3. If a client cannot interpret or evaluate a provided filter expression, it\n MUST NOT return partially filtered results and MUST fail.\n\n4. If the required-row-filter property is absent or empty, no mandatory\n filtering is imposed, and a reader MAY return any subset of rows,\n including all rows.\n", ) diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index ef63fbd88ade..e51deb9dd44e 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3890,6 +3890,9 @@ components: 1. A reader MUST discard any row for which the filter evaluates to false, and no information derived from discarded rows MAY be included in the query result. + + 2. Row filters MUST be evaluated against the original, untransformed column values. + Required projections MUST be applied only after row filters are applied. 2. If the catalog supports multiple row access filters for the table, it is the catalog's responsibility to combine them using the appropriate logic (e.g., AND, OR). From d8447a1759a4a4fa58d800d51b66dc9d7c3d0315 Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Wed, 27 Aug 2025 18:31:08 -0700 Subject: [PATCH 08/51] Add restriction for data type match --- open-api/rest-catalog-open-api.py | 2 +- open-api/rest-catalog-open-api.yaml | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index a4afe0eac1bf..fbcc7e23a8ea 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -1714,7 +1714,7 @@ class ReadRestrictions(BaseModel): required_projection: Optional[List[Term]] = Field( None, alias='required-projection', - description='A list of projections that MUST be applied prior to any query-specified projections. If the required-projection property is absent or empty, no mandatory projection applies, and a reader MAY project any subset of columns of the table, including all columns.\n1. A reader MUST project only columns listed in the required-projection.\n - If a listed column has a transform, the reader MUST apply it and replace\n all references to the underlying column with the transformed value\n (for example, truncate(4, cc) MUST be projected as truncate(4, cc) AS cc,\n and all references to cc during query evaluation MUST resolve to this alias).\n - If a listed column has no transform, the reader MUST read it as-is.\n - Columns not listed in the required-projection MUST NOT be read.\n\n2. A column MUST appear at most once in the required-projection.\n3. A projection entry MUST reference either the column itself or exactly one\n transformed version of the column, but not both.\n\n4. Multiple transformed versions of the same column (e.g., truncate(5, col)\n and truncate(3, col)) MUST NOT appear in the required-projection.\n\n5. If a projection entry includes a transform that the reader cannot evaluate,\n the reader MUST fail rather than ignore the transform.\n\n6. An identity transform is equivalent to projecting the column directly.\n A reader MAY represent it in either form.\n', + description='A list of projections that MUST be applied prior to any query-specified projections. If the required-projection property is absent or empty, no mandatory projection applies, and a reader MAY project any subset of columns of the table, including all columns.\n1. A reader MUST project only columns listed in the required-projection.\n - If a listed column has a transform, the reader MUST apply it and replace\n all references to the underlying column with the transformed value\n (for example, truncate(4, cc) MUST be projected as truncate(4, cc) AS cc,\n and all references to cc during query evaluation MUST resolve to this alias).\n - If a listed column has no transform, the reader MUST read it as-is.\n - Columns not listed in the required-projection MUST NOT be read.\n\n2. A column MUST appear at most once in the required-projection.\n3. A projection entry MUST reference either the column itself or exactly one\n transformed version of the column, but not both.\n\n4. Multiple transformed versions of the same column (e.g., truncate(5, col)\n and truncate(3, col)) MUST NOT appear in the required-projection.\n\n5. If a projection entry includes a transform that the reader cannot evaluate,\n the reader MUST fail rather than ignore the transform.\n\n6. An identity transform is equivalent to projecting the column directly.\n A reader MAY represent it in either form.\n\n7. The data type of the projected column MUST match the data type defined for the transform result.\n', ) required_row_filter: Optional[Expression] = Field( None, diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index e51deb9dd44e..f1d87030b933 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3880,6 +3880,8 @@ components: 6. An identity transform is equivalent to projecting the column directly. A reader MAY represent it in either form. + + 7. The data type of the projected column MUST match the data type defined for the transform result. type: array items: From 647b4ae40270d8338eb8a6f6bcf175f0f0ad1bfd Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Sun, 7 Sep 2025 23:25:09 -0700 Subject: [PATCH 09/51] [Ongoing Discussion] Add projections and filters are per current schema for now --- open-api/rest-catalog-open-api.py | 2 +- open-api/rest-catalog-open-api.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index fbcc7e23a8ea..ab8546ee08ca 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -1705,7 +1705,7 @@ class AddSchemaUpdate(BaseUpdate): class ReadRestrictions(BaseModel): """ - Read restrictions for a table, including projection and row filter expressions. + Read restrictions for a table, including projection and row filter expressions, according to the current schema. A client MUST enforce the restrictions defined in this object when reading data from the table. These restrictions apply only to the authenticated principal, user, or account associated with the client. They MUST NOT be interpreted as global policy and MUST NOT be applied beyond the entity identified by the Authentication header (or other applicable authentication mechanism). diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index f1d87030b933..33f158e89b50 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3842,7 +3842,7 @@ components: ReadRestrictions: type: object description: > - Read restrictions for a table, including projection and row filter expressions. + Read restrictions for a table, including projection and row filter expressions, according to the current schema. A client MUST enforce the restrictions defined in this object when reading data from the table. From b8a27aa326f6bd6b3bd8423ef037e88a955b1c53 Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Wed, 24 Sep 2025 18:56:32 -0700 Subject: [PATCH 10/51] Address review feedbacks --- open-api/rest-catalog-open-api.py | 2 +- open-api/rest-catalog-open-api.yaml | 9 +++------ 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index ab8546ee08ca..bf1f1aed9526 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -1719,7 +1719,7 @@ class ReadRestrictions(BaseModel): required_row_filter: Optional[Expression] = Field( None, alias='required-row-filter', - description="An expression that filters rows in the table.\n1. A reader MUST discard any row for which the filter evaluates to false, and\n no information derived from discarded rows MAY be included in the query result.\n\n2. Row filters MUST be evaluated against the original, untransformed column values.\n Required projections MUST be applied only after row filters are applied.\n\n2. If the catalog supports multiple row access filters for the table, it is\n the catalog's responsibility to combine them using the appropriate logic (e.g., AND, OR).\n\n3. If a client cannot interpret or evaluate a provided filter expression, it\n MUST NOT return partially filtered results and MUST fail.\n\n4. If the required-row-filter property is absent or empty, no mandatory\n filtering is imposed, and a reader MAY return any subset of rows,\n including all rows.\n", + description="An expression that filters rows in the table.\n1. A reader MUST discard any row for which the filter evaluates to false, and\n no information derived from discarded rows MAY be included in the query result.\n\n2. Row filters MUST be evaluated against the original, untransformed column values.\n Required projections MUST be applied only after row filters are applied.\n\n3. If the catalog supports multiple row access filters for the table, it is\n the catalog's responsibility to combine them using the appropriate logic (e.g., AND, OR).\n\n4. If a client cannot interpret or evaluate a provided filter expression, it MUST fail.\n5. If the required-row-filter property is absent or empty, no mandatory filtering is imposed.\n", ) diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index 33f158e89b50..435d8a51fb21 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3896,15 +3896,12 @@ components: 2. Row filters MUST be evaluated against the original, untransformed column values. Required projections MUST be applied only after row filters are applied. - 2. If the catalog supports multiple row access filters for the table, it is + 3. If the catalog supports multiple row access filters for the table, it is the catalog's responsibility to combine them using the appropriate logic (e.g., AND, OR). - 3. If a client cannot interpret or evaluate a provided filter expression, it - MUST NOT return partially filtered results and MUST fail. + 4. If a client cannot interpret or evaluate a provided filter expression, it MUST fail. - 4. If the required-row-filter property is absent or empty, no mandatory - filtering is imposed, and a reader MAY return any subset of rows, - including all rows. + 5. If the required-row-filter property is absent or empty, no mandatory filtering is imposed. $ref: '#/components/schemas/Expression' LoadCredentialsResponse: From 2c41a430a9aa0e36584e155ee97139d4bbfc2b1e Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Thu, 30 Oct 2025 06:00:36 -0700 Subject: [PATCH 11/51] Address review feedback propose a new structure` --- open-api/rest-catalog-open-api.py | 382 ++++++++++------------------ open-api/rest-catalog-open-api.yaml | 99 +++++-- 2 files changed, 213 insertions(+), 268 deletions(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index bf1f1aed9526..40abba115bf8 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -156,11 +156,6 @@ class ExpressionType(RootModel[str]): class TrueExpression(BaseModel): - """ - Deprecated. Use the bare boolean literal `true` as a predicate instead. - - """ - type: Literal['true'] = Field( ..., examples=[ @@ -190,11 +185,6 @@ class TrueExpression(BaseModel): class FalseExpression(BaseModel): - """ - Deprecated. Use the bare boolean literal `false` as a predicate instead. - - """ - type: Literal['false'] = Field( ..., examples=[ @@ -223,49 +213,8 @@ class FalseExpression(BaseModel): ) -class IdReference(BaseModel): - """ - A bound reference to a field by field ID. - """ - - model_config = ConfigDict( - extra='forbid', - ) - type: Literal['reference'] - id: int - - -class NamedReference(BaseModel): - """ - An unbound reference to a field by name. - """ - - model_config = ConfigDict( - extra='forbid', - ) - type: Literal['reference'] - name: str - - -class Function(RootModel[str]): - root: str = Field(..., min_length=1) - - -class Function1(BaseModel): - model_config = ConfigDict( - extra='forbid', - ) - catalog: str | None = None - identifier: CatalogObjectIdentifier - - -class TermReference(RootModel[str]): - root: str = Field( - ..., - deprecated=True, - description='Deprecated string-form field reference used in older REST predicates. Use Reference (IdReference or NamedReference) instead.\n', - examples=[['column-name']], - ) +class Reference(RootModel[str]): + root: str = Field(..., examples=[['column-name']]) class Transform(RootModel[str]): @@ -643,6 +592,42 @@ class StorageCredential(BaseModel): config: dict[str, str] +class MaskHashSha256(BaseModel): + """ + Mask the data of the column by apply SHA256 hash algorithm. Engines are free to use their own implementation of SHA256. + """ + + type: Literal['mask_hash_sha256'] + + +class MaskReplaceWithNull(BaseModel): + """ + Masks data by replacing it with a NULL value. + """ + + type: Literal['mask_replace_with_null'] + + +class MaskAlphanumeric(BaseModel): + """ + mask all alphabetic characters with 'x' and numeric characters with 'n' + """ + + type: Literal['mask_alphanumeric'] + + +class ApplyTransform(BaseModel): + """ + Applies the transform + """ + + type: Literal['apply_transform'] + transform: Transform + source_ids: List[int] = Field( + ..., alias='source-ids', description='args of transform' + ) + + class LoadCredentialsResponse(BaseModel): storage_credentials: list[StorageCredential] = Field( ..., alias='storage-credentials' @@ -1204,64 +1189,10 @@ class RenameTableRequest(BaseModel): destination: TableIdentifier -class Literal1(BaseModel): - """ - A literal is either a bare value, an untyped literal object, or a typed literal object. - - """ - - model_config = ConfigDict( - extra='forbid', - ) - type: Literal['literal'] - value: PrimitiveTypeValue - data_type: PrimitiveType | None = Field(None, alias='data-type') - - -class LiteralModel(RootModel[PrimitiveTypeValue | Literal1]): - root: PrimitiveTypeValue | Literal1 = Field( - ..., - description='A literal is either a bare value, an untyped literal object, or a typed literal object.\n', - ) - - -class Literals1(BaseModel): - """ - Literals is either a bare array of literals or a typed object with an explicit data-type applied to all values. - - """ - - model_config = ConfigDict( - extra='forbid', - ) - type: Literal['literals'] - values: list[PrimitiveTypeValue] - data_type: PrimitiveType = Field(..., alias='data-type') - - -class Literals(RootModel[list[LiteralModel] | Literals1]): - root: list[LiteralModel] | Literals1 = Field( - ..., - description='Literals is either a bare array of literals or a typed object with an explicit data-type applied to all values.\n', - ) - - -class Reference(RootModel[IdReference | NamedReference]): - root: IdReference | NamedReference = Field( - ..., - description='A reference to a field. Either a bound reference (by field ID) or an unbound reference (by name). The context in which an expression is used determines which form is valid.\n', - ) - - class TransformTerm(BaseModel): - """ - Deprecated. Legacy transform-applied-to-a-term form. Use Apply with a Reference argument instead. - - """ - type: Literal['transform'] transform: Transform - term: TermReference + term: Reference class SetPartitionStatisticsUpdate(BaseUpdate): @@ -1275,6 +1206,15 @@ class ViewRequirement(RootModel[AssertViewUUID]): root: AssertViewUUID = Field(..., discriminator='type') +class Action(BaseModel): + __root__: Union[ + MaskHashSha256, MaskReplaceWithNull, MaskAlphanumeric, ApplyTransform + ] = Field( + ..., + description='Defines the specific action to be executed for computing the projection.', + ) + + class FailedPlanningResult(IcebergErrorResponse): """ Failed server-side planning result @@ -1370,12 +1310,8 @@ class FetchScanTasksRequest(BaseModel): plan_task: PlanTask = Field(..., alias='plan-task') -class Term(RootModel[TermReference | TransformTerm]): - root: TermReference | TransformTerm = Field( - ..., - deprecated=True, - description='Deprecated. Legacy term form used by older REST predicates. Use Reference or Apply instead.\n', - ) +class Term(RootModel[Reference | TransformTerm]): + root: Reference | TransformTerm class SetStatisticsUpdate(BaseUpdate): @@ -1413,39 +1349,19 @@ class FunctionDefinitionVersion(BaseModel): ) -class StructField(BaseModel): - id: int - name: str - type: Type - required: bool - doc: str | None = None - initial_default: PrimitiveTypeValue | None = Field(None, alias='initial-default') - write_default: PrimitiveTypeValue | None = Field(None, alias='write-default') - - -class StructType(BaseModel): - type: Literal['struct'] - fields: list[StructField] - - -class ListType(BaseModel): - type: Literal['list'] - element_id: int = Field(..., alias='element-id') - element: Type - element_required: bool = Field(..., alias='element-required') - +class Projection(BaseModel): + """ + Defines a projection for a column. + """ -class MapType(BaseModel): - type: Literal['map'] - key_id: int = Field(..., alias='key-id') - key: Type - value_id: int = Field(..., alias='value-id') - value: Type - value_required: bool = Field(..., alias='value-required') + source_id: Any = Field( + ..., alias='source-id', description='field id of the column being projected.' + ) + action: Action -class AndOrPredicate(BaseModel): - type: Literal['and', 'or'] = Field( +class UnaryExpression(BaseModel): + type: Literal['is-null', 'not-null', 'is-nan', 'not-nan'] = Field( ..., examples=[ [ @@ -1471,12 +1387,13 @@ class AndOrPredicate(BaseModel): ] ], ) - left: Predicate - right: Predicate + term: Term -class NotPredicate(BaseModel): - type: Literal['not'] = Field( +class LiteralExpression(BaseModel): + type: Literal[ + 'lt', 'lt-eq', 'gt', 'gt-eq', 'eq', 'not-eq', 'starts-with', 'not-starts-with' + ] = Field( ..., examples=[ [ @@ -1502,19 +1419,12 @@ class NotPredicate(BaseModel): ] ], ) - child: Predicate - - -class UnaryPredicate(BaseModel): - """ - A predicate that tests a single value expression. Accepts either 'child' (preferred) or 'term' (deprecated) to identify the operand. + term: Term + value: PrimitiveTypeValue - """ - model_config = ConfigDict( - extra='forbid', - ) - type: Literal['is-null', 'not-null', 'is-nan', 'not-nan'] = Field( +class SetExpression(BaseModel): + type: Literal['in', 'not-in'] = Field( ..., examples=[ [ @@ -1540,24 +1450,43 @@ class UnaryPredicate(BaseModel): ] ], ) - child: ValueExpression | None = None - term: Term | None = Field( - None, deprecated=True, description="Deprecated. Use 'child' instead." - ) + term: Term + values: list[PrimitiveTypeValue] -class ComparisonPredicate(BaseModel): - """ - A predicate that compares two value expressions. Accepts either 'left'/'right' (preferred) or 'term'/'value' (deprecated). +class StructField(BaseModel): + id: int + name: str + type: Type + required: bool + doc: str | None = None + initial_default: PrimitiveTypeValue | None = Field(None, alias='initial-default') + write_default: PrimitiveTypeValue | None = Field(None, alias='write-default') - """ - model_config = ConfigDict( - extra='forbid', - ) - type: Literal[ - 'lt', 'lt-eq', 'gt', 'gt-eq', 'eq', 'not-eq', 'starts-with', 'not-starts-with' - ] = Field( +class StructType(BaseModel): + type: Literal['struct'] + fields: list[StructField] + + +class ListType(BaseModel): + type: Literal['list'] + element_id: int = Field(..., alias='element-id') + element: Type + element_required: bool = Field(..., alias='element-required') + + +class MapType(BaseModel): + type: Literal['map'] + key_id: int = Field(..., alias='key-id') + key: Type + value_id: int = Field(..., alias='value-id') + value: Type + value_required: bool = Field(..., alias='value-required') + + +class AndOrExpression(BaseModel): + type: Literal['and', 'or'] = Field( ..., examples=[ [ @@ -1583,26 +1512,12 @@ class ComparisonPredicate(BaseModel): ] ], ) - left: ValueExpression | None = None - right: ValueExpression | None = None - term: Term | None = Field( - None, deprecated=True, description="Deprecated. Use 'left' instead." - ) - value: LiteralModel | None = Field( - None, deprecated=True, description="Deprecated. Use 'right' instead." - ) + left: Expression + right: Expression -class SetPredicate(BaseModel): - """ - A predicate that tests whether a value is in a set of literals. Accepts either 'child' (preferred) or 'term' (deprecated) to identify the operand. - - """ - - model_config = ConfigDict( - extra='forbid', - ) - type: Literal['in', 'not-in'] = Field( +class NotExpression(BaseModel): + type: Literal['not'] = Field( ..., examples=[ [ @@ -1628,25 +1543,7 @@ class SetPredicate(BaseModel): ] ], ) - child: ValueExpression | None = None - term: Term | None = Field( - None, deprecated=True, description="Deprecated. Use 'child' instead." - ) - values: Literals - - -class Apply(BaseModel): - """ - A function application on zero or more value expressions or predicates. - - """ - - model_config = ConfigDict( - extra='forbid', - ) - type: Literal['apply'] - function: Function | CatalogObjectIdentifier | Function1 - arguments: list[FunctionArgument] + child: Expression class TableMetadata(BaseModel): @@ -1705,21 +1602,21 @@ class AddSchemaUpdate(BaseUpdate): class ReadRestrictions(BaseModel): """ - Read restrictions for a table, including projection and row filter expressions, according to the current schema. + Read restrictions for a table, including projections and row filter expressions, according to the current schema. A client MUST enforce the restrictions defined in this object when reading data from the table. These restrictions apply only to the authenticated principal, user, or account associated with the client. They MUST NOT be interpreted as global policy and MUST NOT be applied beyond the entity identified by the Authentication header (or other applicable authentication mechanism). """ - required_projection: Optional[List[Term]] = Field( + required_projections: Optional[List[Projection]] = Field( None, - alias='required-projection', - description='A list of projections that MUST be applied prior to any query-specified projections. If the required-projection property is absent or empty, no mandatory projection applies, and a reader MAY project any subset of columns of the table, including all columns.\n1. A reader MUST project only columns listed in the required-projection.\n - If a listed column has a transform, the reader MUST apply it and replace\n all references to the underlying column with the transformed value\n (for example, truncate(4, cc) MUST be projected as truncate(4, cc) AS cc,\n and all references to cc during query evaluation MUST resolve to this alias).\n - If a listed column has no transform, the reader MUST read it as-is.\n - Columns not listed in the required-projection MUST NOT be read.\n\n2. A column MUST appear at most once in the required-projection.\n3. A projection entry MUST reference either the column itself or exactly one\n transformed version of the column, but not both.\n\n4. Multiple transformed versions of the same column (e.g., truncate(5, col)\n and truncate(3, col)) MUST NOT appear in the required-projection.\n\n5. If a projection entry includes a transform that the reader cannot evaluate,\n the reader MUST fail rather than ignore the transform.\n\n6. An identity transform is equivalent to projecting the column directly.\n A reader MAY represent it in either form.\n\n7. The data type of the projected column MUST match the data type defined for the transform result.\n', + alias='required-projections', + description='A list of projections that MUST be applied prior to any query-specified projections. If the required-projection property is absent, no mandatory projection applies, and a reader MAY project any subset of columns of the table, including all columns.\n1. A reader MUST project only columns listed in the required-projection.\n - If a listed column has a transform, the reader MUST apply it and replace\n all references to the underlying column with the transformed value\n (for example, truncate[4](cc) MUST be projected as truncate[4](cc) AS cc,\n and all references to cc during query evaluation post applying required-row-filter MUST resolve to this alias).\n - Columns not listed in the required-projection MUST NOT be read.\n\n2. A column MUST appear at most once in the required-projection.\n3. Multiple transformed versions of the same column (e.g., truncate[5](col)\n and truncate[3](col) MUST NOT appear in the required-projection.\n\n4. If a projection entry includes an action that the reader cannot evaluate,\n the reader MUST fail rather than ignore the transform.\n\n5. An identity transform is equivalent to projecting the column directly.\n8. The data type of the projected column MUST match the data type defined for the transform result.\n', ) required_row_filter: Optional[Expression] = Field( None, alias='required-row-filter', - description="An expression that filters rows in the table.\n1. A reader MUST discard any row for which the filter evaluates to false, and\n no information derived from discarded rows MAY be included in the query result.\n\n2. Row filters MUST be evaluated against the original, untransformed column values.\n Required projections MUST be applied only after row filters are applied.\n\n3. If the catalog supports multiple row access filters for the table, it is\n the catalog's responsibility to combine them using the appropriate logic (e.g., AND, OR).\n\n4. If a client cannot interpret or evaluate a provided filter expression, it MUST fail.\n5. If the required-row-filter property is absent or empty, no mandatory filtering is imposed.\n", + description="An expression that filters rows in the table.\n1. A reader MUST discard any row for which the filter evaluates to false or null, and\n no information derived from discarded rows MAY be included in the query result.\n\n2. Row filters MUST be evaluated against the original, untransformed column values.\n Required projections MUST be applied only after row filters are applied.\n\n3. If the catalog supports multiple row access filters for the table, it is\n the catalog's responsibility to combine them using the appropriate logic (e.g., AND, OR).\n\n4. If a client cannot interpret or evaluate a provided filter expression, it MUST fail.\n5. If the required-row-filter property is absent or empty, no mandatory filtering is imposed.\n", ) @@ -1906,7 +1803,7 @@ class LoadViewResult(BaseModel): class ScanReport(BaseModel): table_name: str = Field(..., alias='table-name') snapshot_id: int = Field(..., alias='snapshot-id') - filter: Predicate + filter: Expression schema_id: int = Field(..., alias='schema-id') projected_field_ids: list[int] = Field(..., alias='projected-field-ids') projected_field_names: list[str] = Field(..., alias='projected-field-names') @@ -2063,8 +1960,8 @@ class PlanTableScanRequest(BaseModel): select: list[FieldName] | None = Field( None, description='List of selected schema fields' ) - filter: Predicate | None = Field( - None, description='Predicate used to filter the table data' + filter: Expression | None = Field( + None, description='Expression used to filter the table data' ) min_rows_requested: int | None = Field( None, @@ -2105,7 +2002,7 @@ class FileScanTask(BaseModel): alias='delete-file-references', description='A list of indices in the delete files array (0-based)', ) - residual_filter: Predicate | None = Field( + residual_filter: Expression | None = Field( None, alias='residual-filter', description='An optional filter to be applied to rows in this file scan task.\nIf the residual is not present, the client must produce the residual or use the original filter.', @@ -2121,34 +2018,25 @@ class Type(RootModel[VariantType | PrimitiveType | StructType | ListType | MapTy root: VariantType | PrimitiveType | StructType | ListType | MapType -class Predicate( +class Expression( RootModel[ - bool - | TrueExpression + TrueExpression | FalseExpression - | AndOrPredicate - | NotPredicate - | UnaryPredicate - | ComparisonPredicate - | SetPredicate + | AndOrExpression + | NotExpression + | SetExpression + | LiteralExpression + | UnaryExpression ] ): root: ( - bool - | TrueExpression + TrueExpression | FalseExpression - | AndOrPredicate - | NotPredicate - | UnaryPredicate - | ComparisonPredicate - | SetPredicate - ) - - -class ValueExpression(RootModel[LiteralModel | Reference | Apply]): - root: LiteralModel | Reference | Apply = Field( - ..., - description='A value expression: a literal, a field reference, or a function application.\n', + | AndOrExpression + | NotExpression + | SetExpression + | LiteralExpression + | UnaryExpression ) @@ -2264,10 +2152,6 @@ class FunctionDataType( ) -class FunctionArgument(RootModel[ValueExpression | Predicate]): - root: ValueExpression | Predicate - - class CompletedPlanningWithIDResult(CompletedPlanningResult): plan_id: str = Field( ..., alias='plan-id', description='ID used to track a planning request' @@ -2312,12 +2196,8 @@ class PlanTableScanResult( StructField.model_rebuild() ListType.model_rebuild() MapType.model_rebuild() -AndOrPredicate.model_rebuild() -NotPredicate.model_rebuild() -UnaryPredicate.model_rebuild() -ComparisonPredicate.model_rebuild() -SetPredicate.model_rebuild() -Apply.model_rebuild() +AndOrExpression.model_rebuild() +NotExpression.model_rebuild() TableMetadata.model_rebuild() ViewMetadata.model_rebuild() AddSchemaUpdate.model_rebuild() diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index 435d8a51fb21..3262422635d6 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3842,7 +3842,7 @@ components: ReadRestrictions: type: object description: > - Read restrictions for a table, including projection and row filter expressions, according to the current schema. + Read restrictions for a table, including projections and row filter expressions, according to the current schema. A client MUST enforce the restrictions defined in this object when reading data from the table. @@ -3852,45 +3852,40 @@ components: MUST NOT be applied beyond the entity identified by the Authentication header (or other applicable authentication mechanism). properties: - required-projection: + required-projections: description: > A list of projections that MUST be applied prior to any query-specified projections. - If the required-projection property is absent or empty, no mandatory projection applies, + If the required-projection property is absent, no mandatory projection applies, and a reader MAY project any subset of columns of the table, including all columns. 1. A reader MUST project only columns listed in the required-projection. - If a listed column has a transform, the reader MUST apply it and replace all references to the underlying column with the transformed value - (for example, truncate(4, cc) MUST be projected as truncate(4, cc) AS cc, - and all references to cc during query evaluation MUST resolve to this alias). - - If a listed column has no transform, the reader MUST read it as-is. + (for example, truncate[4](cc) MUST be projected as truncate[4](cc) AS cc, + and all references to cc during query evaluation post applying required-row-filter MUST resolve to this alias). - Columns not listed in the required-projection MUST NOT be read. 2. A column MUST appear at most once in the required-projection. - 3. A projection entry MUST reference either the column itself or exactly one - transformed version of the column, but not both. + 3. Multiple transformed versions of the same column (e.g., truncate[5](col) + and truncate[3](col) MUST NOT appear in the required-projection. - 4. Multiple transformed versions of the same column (e.g., truncate(5, col) - and truncate(3, col)) MUST NOT appear in the required-projection. - - 5. If a projection entry includes a transform that the reader cannot evaluate, + 4. If a projection entry includes an action that the reader cannot evaluate, the reader MUST fail rather than ignore the transform. - 6. An identity transform is equivalent to projecting the column directly. - A reader MAY represent it in either form. + 5. An identity transform is equivalent to projecting the column directly. - 7. The data type of the projected column MUST match the data type defined for the transform result. + 8. The data type of the projected column MUST match the data type defined for the transform result. type: array items: - $ref: '#/components/schemas/Term' + $ref: '#/components/schemas/Projection' required-row-filter: description: > An expression that filters rows in the table. - 1. A reader MUST discard any row for which the filter evaluates to false, and + 1. A reader MUST discard any row for which the filter evaluates to false or null, and no information derived from discarded rows MAY be included in the query result. 2. Row filters MUST be evaluated against the original, untransformed column values. @@ -3904,6 +3899,76 @@ components: 5. If the required-row-filter property is absent or empty, no mandatory filtering is imposed. $ref: '#/components/schemas/Expression' + Projection: + type: object + description: Defines a projection for a column. + properties: + source-id: + type: int + description: field id of the column being projected. + action: + $ref: '#/components/schemas/Action' + required: + - source-id + - action + + Action: + description: Defines the specific action to be executed for computing the projection. + oneOf: + - $ref: '#/components/schemas/MaskHashSha256' + - $ref: '#/components/schemas/MaskReplaceWithNull' + - $ref: '#/components/schemas/MaskAlphanumeric' + - $ref: '#/components/schemas/ApplyTransform' + + MaskHashSha256: + type: object + description: Mask the data of the column by apply SHA256 hash algorithm. Engines are free to use their own implementation of SHA256. + properties: + type: + type: string + enum: [ mask_hash_sha256 ] + required: + - type + + MaskReplaceWithNull: + type: object + description: Masks data by replacing it with a NULL value. + properties: + type: + type: string + enum: [ mask_replace_with_null ] + required: + - type + + MaskAlphanumeric: + type: object + description: mask all alphabetic characters with 'x' and numeric characters with 'n' + properties: + type: + type: string + enum: [ mask_alphanumeric ] + required: + - type + + ApplyTransform: + type: object + description: Applies the transform + properties: + type: + type: string + enum: [ apply_transform ] + transform: + $ref: '#/components/schemas/Transform' + source-ids: + description: args of transform + type: array + items: + type: integer + required: + - type + - transform + - source-ids + LoadCredentialsResponse: type: object required: From 251fa809d35e06c2126ffb9cd75b089e1015fd7e Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Tue, 4 Nov 2025 19:43:11 -0800 Subject: [PATCH 12/51] Address review feedback --- open-api/rest-catalog-open-api.py | 12 ++++++------ open-api/rest-catalog-open-api.yaml | 23 ++++++++++------------- 2 files changed, 16 insertions(+), 19 deletions(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index 40abba115bf8..8497e32ffa04 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -1354,8 +1354,8 @@ class Projection(BaseModel): Defines a projection for a column. """ - source_id: Any = Field( - ..., alias='source-id', description='field id of the column being projected.' + field_id: int = Field( + ..., alias='field-id', description='field id of the column being projected.' ) action: Action @@ -1602,16 +1602,16 @@ class AddSchemaUpdate(BaseUpdate): class ReadRestrictions(BaseModel): """ - Read restrictions for a table, including projections and row filter expressions, according to the current schema. + Read restrictions for a table, including column projections and row filter expressions, according to the current schema. A client MUST enforce the restrictions defined in this object when reading data from the table. These restrictions apply only to the authenticated principal, user, or account associated with the client. They MUST NOT be interpreted as global policy and MUST NOT be applied beyond the entity identified by the Authentication header (or other applicable authentication mechanism). """ - required_projections: Optional[List[Projection]] = Field( + required_column_projections: Optional[List[Projection]] = Field( None, - alias='required-projections', - description='A list of projections that MUST be applied prior to any query-specified projections. If the required-projection property is absent, no mandatory projection applies, and a reader MAY project any subset of columns of the table, including all columns.\n1. A reader MUST project only columns listed in the required-projection.\n - If a listed column has a transform, the reader MUST apply it and replace\n all references to the underlying column with the transformed value\n (for example, truncate[4](cc) MUST be projected as truncate[4](cc) AS cc,\n and all references to cc during query evaluation post applying required-row-filter MUST resolve to this alias).\n - Columns not listed in the required-projection MUST NOT be read.\n\n2. A column MUST appear at most once in the required-projection.\n3. Multiple transformed versions of the same column (e.g., truncate[5](col)\n and truncate[3](col) MUST NOT appear in the required-projection.\n\n4. If a projection entry includes an action that the reader cannot evaluate,\n the reader MUST fail rather than ignore the transform.\n\n5. An identity transform is equivalent to projecting the column directly.\n8. The data type of the projected column MUST match the data type defined for the transform result.\n', + alias='required-column-projections', + description='A list of projections that MUST be applied prior to any query-specified projections. If the required-colum-projections property is absent, no mandatory projection applies, and a reader MAY project any subset of columns of the table, including all columns.\n1. A reader MUST project only columns listed in the required-colum-projections.\n - If a listed column has a transform, the reader MUST apply it and replace\n all references to the underlying column with the transformed value\n (for example, truncate[4](cc) MUST be projected as truncate[4](cc) AS cc,\n and all references to cc during query evaluation post applying required-row-filter MUST resolve to this alias).\n - Columns not listed in the required-colum-projections MUST NOT be read.\n\n2. A column MUST appear at most once in the required-colum-projections.\n3. If a projection entry includes an action that the reader cannot evaluate,\n the reader MUST fail rather than ignore the transform.\n\n5. An identity transform is equivalent to projecting the column directly.\n8. The data type of the projected column MUST match the data type defined for the transform result.\n', ) required_row_filter: Optional[Expression] = Field( None, diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index 3262422635d6..bf947e04d480 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3842,7 +3842,7 @@ components: ReadRestrictions: type: object description: > - Read restrictions for a table, including projections and row filter expressions, according to the current schema. + Read restrictions for a table, including column projections and row filter expressions, according to the current schema. A client MUST enforce the restrictions defined in this object when reading data from the table. @@ -3852,26 +3852,23 @@ components: MUST NOT be applied beyond the entity identified by the Authentication header (or other applicable authentication mechanism). properties: - required-projections: + required-column-projections: description: > A list of projections that MUST be applied prior to any query-specified projections. - If the required-projection property is absent, no mandatory projection applies, + If the required-colum-projections property is absent, no mandatory projection applies, and a reader MAY project any subset of columns of the table, including all columns. - 1. A reader MUST project only columns listed in the required-projection. + 1. A reader MUST project only columns listed in the required-colum-projections. - If a listed column has a transform, the reader MUST apply it and replace all references to the underlying column with the transformed value (for example, truncate[4](cc) MUST be projected as truncate[4](cc) AS cc, and all references to cc during query evaluation post applying required-row-filter MUST resolve to this alias). - - Columns not listed in the required-projection MUST NOT be read. + - Columns not listed in the required-colum-projections MUST NOT be read. - 2. A column MUST appear at most once in the required-projection. + 2. A column MUST appear at most once in the required-colum-projections. - 3. Multiple transformed versions of the same column (e.g., truncate[5](col) - and truncate[3](col) MUST NOT appear in the required-projection. - - 4. If a projection entry includes an action that the reader cannot evaluate, + 3. If a projection entry includes an action that the reader cannot evaluate, the reader MUST fail rather than ignore the transform. 5. An identity transform is equivalent to projecting the column directly. @@ -3903,13 +3900,13 @@ components: type: object description: Defines a projection for a column. properties: - source-id: - type: int + field-id: + type: integer description: field id of the column being projected. action: $ref: '#/components/schemas/Action' required: - - source-id + - field-id - action Action: From a65bfa3b793a2c93b2e7a282c9fa1081f72619f1 Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Mon, 10 Nov 2025 09:16:55 -0800 Subject: [PATCH 13/51] Address review feedback --- open-api/rest-catalog-open-api.py | 105 ++++++++++++++++------------ open-api/rest-catalog-open-api.yaml | 81 +++++++-------------- 2 files changed, 85 insertions(+), 101 deletions(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index 8497e32ffa04..d75141e25d79 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -593,38 +593,22 @@ class StorageCredential(BaseModel): class MaskHashSha256(BaseModel): - """ - Mask the data of the column by apply SHA256 hash algorithm. Engines are free to use their own implementation of SHA256. - """ - - type: Literal['mask_hash_sha256'] - + __root__: Any = Field( + ..., + description='Mask the data of the column by applying SHA-256. \nThe input must be UTF-8 encoded bytes of the column value. \nThe SHA-256 digest is represented as a lowercase hexadecimal string. \nEngines must follow this procedure to ensure consistency:\n1. Convert the column value to a UTF-8 byte array.\n2. Apply the SHA-256 algorithm as specified in NIST FIPS 180-4.\n3. Convert the resulting 32-byte digest to a 64-character lowercase hexadecimal string.\n', + ) -class MaskReplaceWithNull(BaseModel): - """ - Masks data by replacing it with a NULL value. - """ - type: Literal['mask_replace_with_null'] +class ReplaceWithNull(BaseModel): + __root__: Any = Field( + ..., description='Masks data by replacing it with a NULL value.' + ) class MaskAlphanumeric(BaseModel): - """ - mask all alphabetic characters with 'x' and numeric characters with 'n' - """ - - type: Literal['mask_alphanumeric'] - - -class ApplyTransform(BaseModel): - """ - Applies the transform - """ - - type: Literal['apply_transform'] - transform: Transform - source_ids: List[int] = Field( - ..., alias='source-ids', description='args of transform' + __root__: Any = Field( + ..., + description="mask all alphabetic characters with 'x' and numeric characters with 'n'", ) @@ -1206,15 +1190,6 @@ class ViewRequirement(RootModel[AssertViewUUID]): root: AssertViewUUID = Field(..., discriminator='type') -class Action(BaseModel): - __root__: Union[ - MaskHashSha256, MaskReplaceWithNull, MaskAlphanumeric, ApplyTransform - ] = Field( - ..., - description='Defines the specific action to be executed for computing the projection.', - ) - - class FailedPlanningResult(IcebergErrorResponse): """ Failed server-side planning result @@ -1349,15 +1324,12 @@ class FunctionDefinitionVersion(BaseModel): ) -class Projection(BaseModel): +class ApplyTransform(BaseModel): """ - Defines a projection for a column. + Replace the field with the result of a transform expression. Produce the original field name with the transformed values. """ - field_id: int = Field( - ..., alias='field-id', description='field id of the column being projected.' - ) - action: Action + term: Optional[Term] = None class UnaryExpression(BaseModel): @@ -1454,6 +1426,47 @@ class SetExpression(BaseModel): values: list[PrimitiveTypeValue] +class ResidualFilter6(SetExpression, ResidualFilter1): + """ + An optional filter to be applied to rows in this file scan task. + If the residual is not present, the client must produce the residual or use the original filter. + """ + + +class ResidualFilter7(LiteralExpression, ResidualFilter1): + """ + An optional filter to be applied to rows in this file scan task. + If the residual is not present, the client must produce the residual or use the original filter. + """ + + +class ResidualFilter8(UnaryExpression, ResidualFilter1): + """ + An optional filter to be applied to rows in this file scan task. + If the residual is not present, the client must produce the residual or use the original filter. + """ + + +class Action(BaseModel): + __root__: Union[ + MaskHashSha256, ReplaceWithNull, MaskAlphanumeric, ApplyTransform + ] = Field( + ..., + description='Defines the specific action to be executed for computing the projection.', + ) + + +class Projection(BaseModel): + """ + Defines a projection for a column. + """ + + field_id: int = Field( + ..., alias='field-id', description='field id of the column being projected.' + ) + action: Action + + class StructField(BaseModel): id: int name: str @@ -1602,21 +1615,21 @@ class AddSchemaUpdate(BaseUpdate): class ReadRestrictions(BaseModel): """ - Read restrictions for a table, including column projections and row filter expressions, according to the current schema. + Read restrictions for a table, including column projections and row filter expressions. A client MUST enforce the restrictions defined in this object when reading data from the table. - These restrictions apply only to the authenticated principal, user, or account associated with the client. They MUST NOT be interpreted as global policy and MUST NOT be applied beyond the entity identified by the Authentication header (or other applicable authentication mechanism). + These restrictions apply only to the authenticated principal, user, or account associated with the request. They MUST NOT be interpreted as global policy and MUST NOT be applied beyond the entity identified by the Authentication header (or other applicable authentication mechanism). """ required_column_projections: Optional[List[Projection]] = Field( None, alias='required-column-projections', - description='A list of projections that MUST be applied prior to any query-specified projections. If the required-colum-projections property is absent, no mandatory projection applies, and a reader MAY project any subset of columns of the table, including all columns.\n1. A reader MUST project only columns listed in the required-colum-projections.\n - If a listed column has a transform, the reader MUST apply it and replace\n all references to the underlying column with the transformed value\n (for example, truncate[4](cc) MUST be projected as truncate[4](cc) AS cc,\n and all references to cc during query evaluation post applying required-row-filter MUST resolve to this alias).\n - Columns not listed in the required-colum-projections MUST NOT be read.\n\n2. A column MUST appear at most once in the required-colum-projections.\n3. If a projection entry includes an action that the reader cannot evaluate,\n the reader MUST fail rather than ignore the transform.\n\n5. An identity transform is equivalent to projecting the column directly.\n8. The data type of the projected column MUST match the data type defined for the transform result.\n', + description="A list of projections that MUST be applied prior to any query-specified projections. If this property is absent, no mandatory projection applies, and a reader MAY project any subset of columns of the table, including all columns.\n1. A reader MUST project only columns listed in the required-column-projections.\n - If a listed column has a transform, the reader MUST apply it and replace\n all references to the underlying column with the transformed value\n (for example, truncate[4](cc) MUST be projected as truncate[4](cc) AS cc,\n and all references to cc during query evaluation post applying required-row-filter MUST resolve to this alias).\n - Columns not listed in the required-column-projections MUST NOT be read.\n\n2. A column MUST appear at most once in the required-column-projections.\n3. If a projected column's corresponding entry includes an action that the reader cannot evaluate,\n the reader MUST fail rather than ignore the transform.\n\n4. An identity transform is equivalent to projecting the column directly.\n5. The data type of the projected column MUST match the data type defined for the transform result.\n", ) required_row_filter: Optional[Expression] = Field( None, alias='required-row-filter', - description="An expression that filters rows in the table.\n1. A reader MUST discard any row for which the filter evaluates to false or null, and\n no information derived from discarded rows MAY be included in the query result.\n\n2. Row filters MUST be evaluated against the original, untransformed column values.\n Required projections MUST be applied only after row filters are applied.\n\n3. If the catalog supports multiple row access filters for the table, it is\n the catalog's responsibility to combine them using the appropriate logic (e.g., AND, OR).\n\n4. If a client cannot interpret or evaluate a provided filter expression, it MUST fail.\n5. If the required-row-filter property is absent or empty, no mandatory filtering is imposed.\n", + description='An expression that filters rows in the table that the authenticated principal does not have access to.\n1. A reader MUST discard any row for which the filter evaluates to false or null, and\n no information derived from discarded rows MAY be included in the query result.\n\n2. Row filters MUST be evaluated against the original, untransformed column values.\n Required projections MUST be applied only after row filters are applied.\n\n3. If a client cannot interpret or evaluate a provided filter expression, it MUST fail.\n4. If this property is absent, null, or always true then no mandatory filtering is required.\n', ) diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index bf947e04d480..cca5b89d252f 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3842,13 +3842,13 @@ components: ReadRestrictions: type: object description: > - Read restrictions for a table, including column projections and row filter expressions, according to the current schema. + Read restrictions for a table, including column projections and row filter expressions. A client MUST enforce the restrictions defined in this object when reading data from the table. These restrictions apply only to the authenticated principal, user, or account - associated with the client. They MUST NOT be interpreted as global policy and + associated with the request. They MUST NOT be interpreted as global policy and MUST NOT be applied beyond the entity identified by the Authentication header (or other applicable authentication mechanism). properties: @@ -3856,31 +3856,31 @@ components: description: > A list of projections that MUST be applied prior to any query-specified projections. - If the required-colum-projections property is absent, no mandatory projection applies, + If this property is absent, no mandatory projection applies, and a reader MAY project any subset of columns of the table, including all columns. - 1. A reader MUST project only columns listed in the required-colum-projections. + 1. A reader MUST project only columns listed in the required-column-projections. - If a listed column has a transform, the reader MUST apply it and replace all references to the underlying column with the transformed value (for example, truncate[4](cc) MUST be projected as truncate[4](cc) AS cc, and all references to cc during query evaluation post applying required-row-filter MUST resolve to this alias). - - Columns not listed in the required-colum-projections MUST NOT be read. + - Columns not listed in the required-column-projections MUST NOT be read. - 2. A column MUST appear at most once in the required-colum-projections. + 2. A column MUST appear at most once in the required-column-projections. - 3. If a projection entry includes an action that the reader cannot evaluate, + 3. If a projected column's corresponding entry includes an action that the reader cannot evaluate, the reader MUST fail rather than ignore the transform. - 5. An identity transform is equivalent to projecting the column directly. + 4. An identity transform is equivalent to projecting the column directly. - 8. The data type of the projected column MUST match the data type defined for the transform result. + 5. The data type of the projected column MUST match the data type defined for the transform result. type: array items: $ref: '#/components/schemas/Projection' required-row-filter: description: > - An expression that filters rows in the table. + An expression that filters rows in the table that the authenticated principal does not have access to. 1. A reader MUST discard any row for which the filter evaluates to false or null, and no information derived from discarded rows MAY be included in the query result. @@ -3888,12 +3888,9 @@ components: 2. Row filters MUST be evaluated against the original, untransformed column values. Required projections MUST be applied only after row filters are applied. - 3. If the catalog supports multiple row access filters for the table, it is - the catalog's responsibility to combine them using the appropriate logic (e.g., AND, OR). + 3. If a client cannot interpret or evaluate a provided filter expression, it MUST fail. - 4. If a client cannot interpret or evaluate a provided filter expression, it MUST fail. - - 5. If the required-row-filter property is absent or empty, no mandatory filtering is imposed. + 4. If this property is absent, null, or always true then no mandatory filtering is required. $ref: '#/components/schemas/Expression' Projection: @@ -3913,58 +3910,32 @@ components: description: Defines the specific action to be executed for computing the projection. oneOf: - $ref: '#/components/schemas/MaskHashSha256' - - $ref: '#/components/schemas/MaskReplaceWithNull' + - $ref: '#/components/schemas/ReplaceWithNull' - $ref: '#/components/schemas/MaskAlphanumeric' - $ref: '#/components/schemas/ApplyTransform' MaskHashSha256: - type: object - description: Mask the data of the column by apply SHA256 hash algorithm. Engines are free to use their own implementation of SHA256. - properties: - type: - type: string - enum: [ mask_hash_sha256 ] - required: - - type - - MaskReplaceWithNull: - type: object + description: | + Mask the data of the column by applying SHA-256. + The input must be UTF-8 encoded bytes of the column value. + The SHA-256 digest is represented as a lowercase hexadecimal string. + Engines must follow this procedure to ensure consistency: + 1. Convert the column value to a UTF-8 byte array. + 2. Apply the SHA-256 algorithm as specified in NIST FIPS 180-4. + 3. Convert the resulting 32-byte digest to a 64-character lowercase hexadecimal string. + + ReplaceWithNull: description: Masks data by replacing it with a NULL value. - properties: - type: - type: string - enum: [ mask_replace_with_null ] - required: - - type MaskAlphanumeric: - type: object description: mask all alphabetic characters with 'x' and numeric characters with 'n' - properties: - type: - type: string - enum: [ mask_alphanumeric ] - required: - - type ApplyTransform: type: object - description: Applies the transform + description: Replace the field with the result of a transform expression. Produce the original field name with the transformed values. properties: - type: - type: string - enum: [ apply_transform ] - transform: - $ref: '#/components/schemas/Transform' - source-ids: - description: args of transform - type: array - items: - type: integer - required: - - type - - transform - - source-ids + term: + $ref: '#/components/schemas/Term' LoadCredentialsResponse: type: object From 7fde07ce2532fc111beeeacd244ec3fa52014572 Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Sun, 14 Dec 2025 09:26:23 -0800 Subject: [PATCH 14/51] pass lint --- open-api/rest-catalog-open-api.py | 86 +++++++++++++++-------------- open-api/rest-catalog-open-api.yaml | 20 +++---- 2 files changed, 54 insertions(+), 52 deletions(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index d75141e25d79..e22fcd8f1778 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -18,7 +18,7 @@ from __future__ import annotations from datetime import date, timedelta -from typing import Dict, Literal +from typing import Any, Dict, List, Literal, Optional, Union from uuid import UUID from pydantic import Base64Str, BaseModel, ConfigDict, Field, RootModel @@ -371,17 +371,17 @@ class AssignUUIDUpdate(BaseUpdate): Assigning a UUID to a table/view should only be done when creating the table/view. It is not safe to re-assign the UUID if a table/view already has a UUID assigned """ - action: Literal['assign-uuid'] + action: Literal['assign-uuid'] = 'assign-uuid' uuid: str class UpgradeFormatVersionUpdate(BaseUpdate): - action: Literal['upgrade-format-version'] + action: Literal['upgrade-format-version'] = 'upgrade-format-version' format_version: int = Field(..., alias='format-version') class SetCurrentSchemaUpdate(BaseUpdate): - action: Literal['set-current-schema'] + action: Literal['set-current-schema'] = 'set-current-schema' schema_id: int = Field( ..., alias='schema-id', @@ -390,12 +390,12 @@ class SetCurrentSchemaUpdate(BaseUpdate): class AddPartitionSpecUpdate(BaseUpdate): - action: Literal['add-spec'] + action: Literal['add-spec'] = 'add-spec' spec: PartitionSpec class SetDefaultSpecUpdate(BaseUpdate): - action: Literal['set-default-spec'] + action: Literal['set-default-spec'] = 'set-default-spec' spec_id: int = Field( ..., alias='spec-id', @@ -404,12 +404,12 @@ class SetDefaultSpecUpdate(BaseUpdate): class AddSortOrderUpdate(BaseUpdate): - action: Literal['add-sort-order'] + action: Literal['add-sort-order'] = 'add-sort-order' sort_order: SortOrder = Field(..., alias='sort-order') class SetDefaultSortOrderUpdate(BaseUpdate): - action: Literal['set-default-sort-order'] + action: Literal['set-default-sort-order'] = 'set-default-sort-order' sort_order_id: int = Field( ..., alias='sort-order-id', @@ -418,47 +418,47 @@ class SetDefaultSortOrderUpdate(BaseUpdate): class AddSnapshotUpdate(BaseUpdate): - action: Literal['add-snapshot'] + action: Literal['add-snapshot'] = 'add-snapshot' snapshot: Snapshot class SetSnapshotRefUpdate(BaseUpdate, SnapshotReference): - action: Literal['set-snapshot-ref'] + action: Literal['set-snapshot-ref'] = 'set-snapshot-ref' ref_name: str = Field(..., alias='ref-name') class RemoveSnapshotsUpdate(BaseUpdate): - action: Literal['remove-snapshots'] + action: Literal['remove-snapshots'] = 'remove-snapshots' snapshot_ids: list[int] = Field(..., alias='snapshot-ids') class RemoveSnapshotRefUpdate(BaseUpdate): - action: Literal['remove-snapshot-ref'] + action: Literal['remove-snapshot-ref'] = 'remove-snapshot-ref' ref_name: str = Field(..., alias='ref-name') class SetLocationUpdate(BaseUpdate): - action: Literal['set-location'] + action: Literal['set-location'] = 'set-location' location: str class SetPropertiesUpdate(BaseUpdate): - action: Literal['set-properties'] + action: Literal['set-properties'] = 'set-properties' updates: dict[str, str] class RemovePropertiesUpdate(BaseUpdate): - action: Literal['remove-properties'] + action: Literal['remove-properties'] = 'remove-properties' removals: list[str] class AddViewVersionUpdate(BaseUpdate): - action: Literal['add-view-version'] + action: Literal['add-view-version'] = 'add-view-version' view_version: ViewVersion = Field(..., alias='view-version') class SetCurrentViewVersionUpdate(BaseUpdate): - action: Literal['set-current-view-version'] + action: Literal['set-current-view-version'] = 'set-current-view-version' view_version_id: int = Field( ..., alias='view-version-id', @@ -467,32 +467,32 @@ class SetCurrentViewVersionUpdate(BaseUpdate): class RemoveStatisticsUpdate(BaseUpdate): - action: Literal['remove-statistics'] + action: Literal['remove-statistics'] = 'remove-statistics' snapshot_id: int = Field(..., alias='snapshot-id') class RemovePartitionStatisticsUpdate(BaseUpdate): - action: Literal['remove-partition-statistics'] + action: Literal['remove-partition-statistics'] = 'remove-partition-statistics' snapshot_id: int = Field(..., alias='snapshot-id') class RemovePartitionSpecsUpdate(BaseUpdate): - action: Literal['remove-partition-specs'] + action: Literal['remove-partition-specs'] = 'remove-partition-specs' spec_ids: list[int] = Field(..., alias='spec-ids') class RemoveSchemasUpdate(BaseUpdate): - action: Literal['remove-schemas'] + action: Literal['remove-schemas'] = 'remove-schemas' schema_ids: list[int] = Field(..., alias='schema-ids') class AddEncryptionKeyUpdate(BaseUpdate): - action: Literal['add-encryption-key'] + action: Literal['add-encryption-key'] = 'add-encryption-key' encryption_key: EncryptedKey = Field(..., alias='encryption-key') class RemoveEncryptionKeyUpdate(BaseUpdate): - action: Literal['remove-encryption-key'] + action: Literal['remove-encryption-key'] = 'remove-encryption-key' key_id: str = Field(..., alias='key-id') @@ -525,7 +525,7 @@ class AssertRefSnapshotId(TableRequirement): """ - type: Literal['assert-ref-snapshot-id'] + type: Literal['assert-ref-snapshot-id'] = 'assert-ref-snapshot-id' ref: str snapshot_id: int = Field(..., alias='snapshot-id') @@ -535,7 +535,7 @@ class AssertLastAssignedFieldId(TableRequirement): The table's last assigned column id must match the requirement's `last-assigned-field-id` """ - type: Literal['assert-last-assigned-field-id'] + type: Literal['assert-last-assigned-field-id'] = 'assert-last-assigned-field-id' last_assigned_field_id: int = Field(..., alias='last-assigned-field-id') @@ -544,7 +544,7 @@ class AssertCurrentSchemaId(TableRequirement): The table's current schema id must match the requirement's `current-schema-id` """ - type: Literal['assert-current-schema-id'] + type: Literal['assert-current-schema-id'] = 'assert-current-schema-id' current_schema_id: int = Field(..., alias='current-schema-id') @@ -553,7 +553,9 @@ class AssertLastAssignedPartitionId(TableRequirement): The table's last assigned partition id must match the requirement's `last-assigned-partition-id` """ - type: Literal['assert-last-assigned-partition-id'] + type: Literal['assert-last-assigned-partition-id'] = ( + 'assert-last-assigned-partition-id' + ) last_assigned_partition_id: int = Field(..., alias='last-assigned-partition-id') @@ -562,7 +564,7 @@ class AssertDefaultSpecId(TableRequirement): The table's default spec id must match the requirement's `default-spec-id` """ - type: Literal['assert-default-spec-id'] + type: Literal['assert-default-spec-id'] = 'assert-default-spec-id' default_spec_id: int = Field(..., alias='default-spec-id') @@ -571,7 +573,7 @@ class AssertDefaultSortOrderId(TableRequirement): The table's default sort order id must match the requirement's `default-sort-order-id` """ - type: Literal['assert-default-sort-order-id'] + type: Literal['assert-default-sort-order-id'] = 'assert-default-sort-order-id' default_sort_order_id: int = Field(..., alias='default-sort-order-id') @@ -595,7 +597,7 @@ class StorageCredential(BaseModel): class MaskHashSha256(BaseModel): __root__: Any = Field( ..., - description='Mask the data of the column by applying SHA-256. \nThe input must be UTF-8 encoded bytes of the column value. \nThe SHA-256 digest is represented as a lowercase hexadecimal string. \nEngines must follow this procedure to ensure consistency:\n1. Convert the column value to a UTF-8 byte array.\n2. Apply the SHA-256 algorithm as specified in NIST FIPS 180-4.\n3. Convert the resulting 32-byte digest to a 64-character lowercase hexadecimal string.\n', + description='Mask the data of the column by applying SHA-256.\nThe input must be UTF-8 encoded bytes of the column value.\nThe SHA-256 digest is represented as a lowercase hexadecimal string.\nEngines must follow this procedure to ensure consistency:\n1. Convert the column value to a UTF-8 byte array.\n2. Apply the SHA-256 algorithm as specified in NIST FIPS 180-4.\n3. Convert the resulting 32-byte digest to a 64-character lowercase hexadecimal string.\n', ) @@ -1180,7 +1182,7 @@ class TransformTerm(BaseModel): class SetPartitionStatisticsUpdate(BaseUpdate): - action: Literal['set-partition-statistics'] + action: Literal['set-partition-statistics'] = 'set-partition-statistics' partition_statistics: PartitionStatisticsFile = Field( ..., alias='partition-statistics' ) @@ -1290,7 +1292,7 @@ class Term(RootModel[Reference | TransformTerm]): class SetStatisticsUpdate(BaseUpdate): - action: Literal['set-statistics'] + action: Literal['set-statistics'] = 'set-statistics' snapshot_id: int | None = Field( None, alias='snapshot-id', @@ -1426,6 +1428,15 @@ class SetExpression(BaseModel): values: list[PrimitiveTypeValue] +class Action(BaseModel): + __root__: Union[ + MaskHashSha256, ReplaceWithNull, MaskAlphanumeric, ApplyTransform + ] = Field( + ..., + description='Defines the specific action to be executed for computing the projection.', + ) + + class ResidualFilter6(SetExpression, ResidualFilter1): """ An optional filter to be applied to rows in this file scan task. @@ -1447,15 +1458,6 @@ class ResidualFilter8(UnaryExpression, ResidualFilter1): """ -class Action(BaseModel): - __root__: Union[ - MaskHashSha256, ReplaceWithNull, MaskAlphanumeric, ApplyTransform - ] = Field( - ..., - description='Defines the specific action to be executed for computing the projection.', - ) - - class Projection(BaseModel): """ Defines a projection for a column. @@ -1603,7 +1605,7 @@ class ViewMetadata(BaseModel): class AddSchemaUpdate(BaseUpdate): - action: Literal['add-schema'] + action: Literal['add-schema'] = 'add-schema' schema_: Schema = Field(..., alias='schema') last_column_id: int | None = Field( None, diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index cca5b89d252f..5f725dcc18a4 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3867,12 +3867,12 @@ components: - Columns not listed in the required-column-projections MUST NOT be read. 2. A column MUST appear at most once in the required-column-projections. - + 3. If a projected column's corresponding entry includes an action that the reader cannot evaluate, the reader MUST fail rather than ignore the transform. - + 4. An identity transform is equivalent to projecting the column directly. - + 5. The data type of the projected column MUST match the data type defined for the transform result. type: array @@ -3881,15 +3881,15 @@ components: required-row-filter: description: > An expression that filters rows in the table that the authenticated principal does not have access to. - + 1. A reader MUST discard any row for which the filter evaluates to false or null, and no information derived from discarded rows MAY be included in the query result. - + 2. Row filters MUST be evaluated against the original, untransformed column values. Required projections MUST be applied only after row filters are applied. - + 3. If a client cannot interpret or evaluate a provided filter expression, it MUST fail. - + 4. If this property is absent, null, or always true then no mandatory filtering is required. $ref: '#/components/schemas/Expression' @@ -3916,9 +3916,9 @@ components: MaskHashSha256: description: | - Mask the data of the column by applying SHA-256. - The input must be UTF-8 encoded bytes of the column value. - The SHA-256 digest is represented as a lowercase hexadecimal string. + Mask the data of the column by applying SHA-256. + The input must be UTF-8 encoded bytes of the column value. + The SHA-256 digest is represented as a lowercase hexadecimal string. Engines must follow this procedure to ensure consistency: 1. Convert the column value to a UTF-8 byte array. 2. Apply the SHA-256 algorithm as specified in NIST FIPS 180-4. From 2c00be85ece887956463e5e3adad97dc40fb4329 Mon Sep 17 00:00:00 2001 From: Prashant Kumar Singh Date: Sun, 11 Jan 2026 05:15:42 +0000 Subject: [PATCH 15/51] Address review feedback from steven --- open-api/rest-catalog-open-api.py | 48 ++++++++--------------------- open-api/rest-catalog-open-api.yaml | 5 +-- 2 files changed, 16 insertions(+), 37 deletions(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index e22fcd8f1778..77c7638edb23 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -18,7 +18,7 @@ from __future__ import annotations from datetime import date, timedelta -from typing import Any, Dict, List, Literal, Optional, Union +from typing import Any, Literal from uuid import UUID from pydantic import Base64Str, BaseModel, ConfigDict, Field, RootModel @@ -1331,7 +1331,7 @@ class ApplyTransform(BaseModel): Replace the field with the result of a transform expression. Produce the original field name with the transformed values. """ - term: Optional[Term] = None + term: Term | None = None class UnaryExpression(BaseModel): @@ -1429,44 +1429,24 @@ class SetExpression(BaseModel): class Action(BaseModel): - __root__: Union[ - MaskHashSha256, ReplaceWithNull, MaskAlphanumeric, ApplyTransform - ] = Field( - ..., - description='Defines the specific action to be executed for computing the projection.', + __root__: MaskHashSha256 | ReplaceWithNull | MaskAlphanumeric | ApplyTransform = ( + Field( + ..., + description='Defines the specific action to be executed for computing the projection.', + ) ) -class ResidualFilter6(SetExpression, ResidualFilter1): - """ - An optional filter to be applied to rows in this file scan task. - If the residual is not present, the client must produce the residual or use the original filter. - """ - - -class ResidualFilter7(LiteralExpression, ResidualFilter1): - """ - An optional filter to be applied to rows in this file scan task. - If the residual is not present, the client must produce the residual or use the original filter. - """ - - -class ResidualFilter8(UnaryExpression, ResidualFilter1): - """ - An optional filter to be applied to rows in this file scan task. - If the residual is not present, the client must produce the residual or use the original filter. - """ - - class Projection(BaseModel): """ - Defines a projection for a column. + Defines a projection for a column. If action is not specified, the column is projected as-is. + """ field_id: int = Field( ..., alias='field-id', description='field id of the column being projected.' ) - action: Action + action: Action | None = None class StructField(BaseModel): @@ -1623,12 +1603,12 @@ class ReadRestrictions(BaseModel): """ - required_column_projections: Optional[List[Projection]] = Field( + required_column_projections: list[Projection] | None = Field( None, alias='required-column-projections', description="A list of projections that MUST be applied prior to any query-specified projections. If this property is absent, no mandatory projection applies, and a reader MAY project any subset of columns of the table, including all columns.\n1. A reader MUST project only columns listed in the required-column-projections.\n - If a listed column has a transform, the reader MUST apply it and replace\n all references to the underlying column with the transformed value\n (for example, truncate[4](cc) MUST be projected as truncate[4](cc) AS cc,\n and all references to cc during query evaluation post applying required-row-filter MUST resolve to this alias).\n - Columns not listed in the required-column-projections MUST NOT be read.\n\n2. A column MUST appear at most once in the required-column-projections.\n3. If a projected column's corresponding entry includes an action that the reader cannot evaluate,\n the reader MUST fail rather than ignore the transform.\n\n4. An identity transform is equivalent to projecting the column directly.\n5. The data type of the projected column MUST match the data type defined for the transform result.\n", ) - required_row_filter: Optional[Expression] = Field( + required_row_filter: Expression | None = Field( None, alias='required-row-filter', description='An expression that filters rows in the table that the authenticated principal does not have access to.\n1. A reader MUST discard any row for which the filter evaluates to false or null, and\n no information derived from discarded rows MAY be included in the query result.\n\n2. Row filters MUST be evaluated against the original, untransformed column values.\n Required projections MUST be applied only after row filters are applied.\n\n3. If a client cannot interpret or evaluate a provided filter expression, it MUST fail.\n4. If this property is absent, null, or always true then no mandatory filtering is required.\n', @@ -1699,9 +1679,7 @@ class LoadTableResult(BaseModel): remote_signing_config: RemoteSigningConfig | None = Field( None, alias='remote-signing-config' ) - read_restrictions: Optional[ReadRestrictions] = Field( - None, alias='read-restrictions' - ) + read_restrictions: ReadRestrictions | None = Field(None, alias='read-restrictions') class ScanTasks(BaseModel): diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index 5f725dcc18a4..58810d9c38a3 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3895,7 +3895,9 @@ components: Projection: type: object - description: Defines a projection for a column. + description: > + Defines a projection for a column. + If action is not specified, the column is projected as-is. properties: field-id: type: integer @@ -3904,7 +3906,6 @@ components: $ref: '#/components/schemas/Action' required: - field-id - - action Action: description: Defines the specific action to be executed for computing the projection. From 4d65ae34934eebc957ff5316f23cacbe9c357711 Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Tue, 17 Feb 2026 07:09:18 -0800 Subject: [PATCH 16/51] Address feedbacks per sync 02/03/2026 --- open-api/rest-catalog-open-api.py | 2 +- open-api/rest-catalog-open-api.yaml | 43 +++++++++++++++++++---------- 2 files changed, 29 insertions(+), 16 deletions(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index 77c7638edb23..96bda6b9bc61 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -1606,7 +1606,7 @@ class ReadRestrictions(BaseModel): required_column_projections: list[Projection] | None = Field( None, alias='required-column-projections', - description="A list of projections that MUST be applied prior to any query-specified projections. If this property is absent, no mandatory projection applies, and a reader MAY project any subset of columns of the table, including all columns.\n1. A reader MUST project only columns listed in the required-column-projections.\n - If a listed column has a transform, the reader MUST apply it and replace\n all references to the underlying column with the transformed value\n (for example, truncate[4](cc) MUST be projected as truncate[4](cc) AS cc,\n and all references to cc during query evaluation post applying required-row-filter MUST resolve to this alias).\n - Columns not listed in the required-column-projections MUST NOT be read.\n\n2. A column MUST appear at most once in the required-column-projections.\n3. If a projected column's corresponding entry includes an action that the reader cannot evaluate,\n the reader MUST fail rather than ignore the transform.\n\n4. An identity transform is equivalent to projecting the column directly.\n5. The data type of the projected column MUST match the data type defined for the transform result.\n", + description="A list of columns that require specific projections or transforms to be applied.\nIf this property is absent, a reader MAY access all columns of the table as-is without any mandatory transformations.\nIf this property is present, each listed column MUST have its specified projection or action applied. Columns not listed in required-column-projections MAY be read as-is without any restrictions.\nWhen this list is present:\n1. For each column listed in required-column-projections, the reader MUST apply\n the specified action or transform.\n\n2. If a listed column has an action, the reader MUST apply it and replace\n all references to the underlying column with the transformed value.\n For example, if the action specifies truncate[4](cc), the reader MUST project it\n as truncate[4](cc) AS cc, and all references to cc during query evaluation\n (including in required-row-filter) MUST resolve to this transformed alias.\n\n3. If a listed column has no action specified, the reader MUST project the column as-is\n (equivalent to an identity transform).\n\n4. Columns not listed in required-column-projections MAY be projected normally\n by the reader without any mandatory transformations.\n\n5. A column MUST appear at most once in required-column-projections.\n6. If a projected column's action cannot be evaluated by the reader,\n the reader MUST fail rather than ignore or skip the transform.\n\n7. The data type of a projected column MUST match the data type defined for\n the transform result or the original column type if no transform is specified.\n", ) required_row_filter: Expression | None = Field( None, diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index 58810d9c38a3..c63383b27737 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3854,26 +3854,39 @@ components: properties: required-column-projections: description: > - A list of projections that MUST be applied prior to any query-specified - projections. - If this property is absent, no mandatory projection applies, - and a reader MAY project any subset of columns of the table, including all columns. + A list of columns that require specific projections or transforms to be applied. - 1. A reader MUST project only columns listed in the required-column-projections. - - If a listed column has a transform, the reader MUST apply it and replace - all references to the underlying column with the transformed value - (for example, truncate[4](cc) MUST be projected as truncate[4](cc) AS cc, - and all references to cc during query evaluation post applying required-row-filter MUST resolve to this alias). - - Columns not listed in the required-column-projections MUST NOT be read. + If this property is absent, a reader MAY access all columns of the table as-is + without any mandatory transformations. - 2. A column MUST appear at most once in the required-column-projections. + If this property is present, each listed column MUST have its specified + projection or action applied. Columns not listed in required-column-projections + MAY be read as-is without any restrictions. - 3. If a projected column's corresponding entry includes an action that the reader cannot evaluate, - the reader MUST fail rather than ignore the transform. + When this list is present: - 4. An identity transform is equivalent to projecting the column directly. + 1. For each column listed in required-column-projections, the reader MUST apply + the specified action or transform. - 5. The data type of the projected column MUST match the data type defined for the transform result. + 2. If a listed column has an action, the reader MUST apply it and replace + all references to the underlying column with the transformed value. + For example, if the action specifies truncate[4](cc), the reader MUST project it + as truncate[4](cc) AS cc, and all references to cc during query evaluation + (including in required-row-filter) MUST resolve to this transformed alias. + + 3. If a listed column has no action specified, the reader MUST project the column as-is + (equivalent to an identity transform). + + 4. Columns not listed in required-column-projections MAY be projected normally + by the reader without any mandatory transformations. + + 5. A column MUST appear at most once in required-column-projections. + + 6. If a projected column's action cannot be evaluated by the reader, + the reader MUST fail rather than ignore or skip the transform. + + 7. The data type of a projected column MUST match the data type defined for + the transform result or the original column type if no transform is specified. type: array items: From 8f59e2a9e3351333e8d860f5a2f7db00d1ec1029 Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Sun, 8 Mar 2026 21:41:59 -0700 Subject: [PATCH 17/51] Address comment: clarify unlisted columns wording --- open-api/rest-catalog-open-api.py | 2 +- open-api/rest-catalog-open-api.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index 96bda6b9bc61..a5b41064d8d4 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -1606,7 +1606,7 @@ class ReadRestrictions(BaseModel): required_column_projections: list[Projection] | None = Field( None, alias='required-column-projections', - description="A list of columns that require specific projections or transforms to be applied.\nIf this property is absent, a reader MAY access all columns of the table as-is without any mandatory transformations.\nIf this property is present, each listed column MUST have its specified projection or action applied. Columns not listed in required-column-projections MAY be read as-is without any restrictions.\nWhen this list is present:\n1. For each column listed in required-column-projections, the reader MUST apply\n the specified action or transform.\n\n2. If a listed column has an action, the reader MUST apply it and replace\n all references to the underlying column with the transformed value.\n For example, if the action specifies truncate[4](cc), the reader MUST project it\n as truncate[4](cc) AS cc, and all references to cc during query evaluation\n (including in required-row-filter) MUST resolve to this transformed alias.\n\n3. If a listed column has no action specified, the reader MUST project the column as-is\n (equivalent to an identity transform).\n\n4. Columns not listed in required-column-projections MAY be projected normally\n by the reader without any mandatory transformations.\n\n5. A column MUST appear at most once in required-column-projections.\n6. If a projected column's action cannot be evaluated by the reader,\n the reader MUST fail rather than ignore or skip the transform.\n\n7. The data type of a projected column MUST match the data type defined for\n the transform result or the original column type if no transform is specified.\n", + description="A list of columns that require specific projections or transforms to be applied.\nIf this property is absent, a reader MAY access all columns of the table as-is without any mandatory transformations.\nIf this property is present, each listed column MUST have its specified projection or action applied. Columns not listed in required-column-projections are not subject to any read restrictions.\nWhen this list is present:\n1. For each column listed in required-column-projections, the reader MUST apply\n the specified action or transform.\n\n2. If a listed column has an action, the reader MUST apply it and replace\n all references to the underlying column with the transformed value.\n For example, if the action specifies truncate[4](cc), the reader MUST project it\n as truncate[4](cc) AS cc, and all references to cc during query evaluation\n (including in required-row-filter) MUST resolve to this transformed alias.\n\n3. If a listed column has no action specified, the reader MUST project the column as-is\n (equivalent to an identity transform).\n\n4. Columns not listed in required-column-projections MAY be projected normally\n by the reader without any mandatory transformations.\n\n5. A column MUST appear at most once in required-column-projections.\n6. If a projected column's action cannot be evaluated by the reader,\n the reader MUST fail rather than ignore or skip the transform.\n\n7. The data type of a projected column MUST match the data type defined for\n the transform result or the original column type if no transform is specified.\n", ) required_row_filter: Expression | None = Field( None, diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index c63383b27737..ca1a2c46a1cd 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3861,7 +3861,7 @@ components: If this property is present, each listed column MUST have its specified projection or action applied. Columns not listed in required-column-projections - MAY be read as-is without any restrictions. + are not subject to any read restrictions. When this list is present: From 5fce286a05e26e86f220813168c60005f213872a Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Sun, 15 Mar 2026 19:31:24 -0700 Subject: [PATCH 18/51] Add discriminator for Action and remove Projection wrapper Flatten Projection into Action with an "action" discriminator property, matching the BaseUpdate pattern. Each Action subtype now uses allOf to extend Action with a const action value, making them distinguishable in JSON. --- open-api/rest-catalog-open-api.py | 77 ++++++++++++++-------------- open-api/rest-catalog-open-api.yaml | 78 +++++++++++++++++------------ 2 files changed, 86 insertions(+), 69 deletions(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index a5b41064d8d4..7bd8c08f81b5 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -18,7 +18,7 @@ from __future__ import annotations from datetime import date, timedelta -from typing import Any, Literal +from typing import Literal from uuid import UUID from pydantic import Base64Str, BaseModel, ConfigDict, Field, RootModel @@ -594,24 +594,42 @@ class StorageCredential(BaseModel): config: dict[str, str] -class MaskHashSha256(BaseModel): - __root__: Any = Field( - ..., - description='Mask the data of the column by applying SHA-256.\nThe input must be UTF-8 encoded bytes of the column value.\nThe SHA-256 digest is represented as a lowercase hexadecimal string.\nEngines must follow this procedure to ensure consistency:\n1. Convert the column value to a UTF-8 byte array.\n2. Apply the SHA-256 algorithm as specified in NIST FIPS 180-4.\n3. Convert the resulting 32-byte digest to a 64-character lowercase hexadecimal string.\n', +class Action(BaseModel): + action: str + field_id: int = Field( + ..., alias='field-id', description='field id of the column being projected.' ) -class ReplaceWithNull(BaseModel): - __root__: Any = Field( - ..., description='Masks data by replacing it with a NULL value.' - ) +class MaskHashSha256(Action): + """ + Mask the data of the column by applying SHA-256. + The input must be UTF-8 encoded bytes of the column value. + The SHA-256 digest is represented as a lowercase hexadecimal string. + Engines must follow this procedure to ensure consistency: + 1. Convert the column value to a UTF-8 byte array. + 2. Apply the SHA-256 algorithm as specified in NIST FIPS 180-4. + 3. Convert the resulting 32-byte digest to a 64-character lowercase hexadecimal string. + """ -class MaskAlphanumeric(BaseModel): - __root__: Any = Field( - ..., - description="mask all alphabetic characters with 'x' and numeric characters with 'n'", - ) + action: Literal['mask-hash-sha256'] = Field('mask-hash-sha256', const=True) + + +class ReplaceWithNull(Action): + """ + Masks data by replacing it with a NULL value. + """ + + action: Literal['replace-with-null'] = Field('replace-with-null', const=True) + + +class MaskAlphanumeric(Action): + """ + mask all alphabetic characters with 'x' and numeric characters with 'n' + """ + + action: Literal['mask-alphanumeric'] = Field('mask-alphanumeric', const=True) class LoadCredentialsResponse(BaseModel): @@ -1326,11 +1344,12 @@ class FunctionDefinitionVersion(BaseModel): ) -class ApplyTransform(BaseModel): +class ApplyTransform(Action): """ Replace the field with the result of a transform expression. Produce the original field name with the transformed values. """ + action: Literal['apply-transform'] = Field('apply-transform', const=True) term: Term | None = None @@ -1428,27 +1447,6 @@ class SetExpression(BaseModel): values: list[PrimitiveTypeValue] -class Action(BaseModel): - __root__: MaskHashSha256 | ReplaceWithNull | MaskAlphanumeric | ApplyTransform = ( - Field( - ..., - description='Defines the specific action to be executed for computing the projection.', - ) - ) - - -class Projection(BaseModel): - """ - Defines a projection for a column. If action is not specified, the column is projected as-is. - - """ - - field_id: int = Field( - ..., alias='field-id', description='field id of the column being projected.' - ) - action: Action | None = None - - class StructField(BaseModel): id: int name: str @@ -1603,10 +1601,13 @@ class ReadRestrictions(BaseModel): """ - required_column_projections: list[Projection] | None = Field( + required_column_projections: ( + list[MaskHashSha256 | ReplaceWithNull | MaskAlphanumeric | ApplyTransform] + | None + ) = Field( None, alias='required-column-projections', - description="A list of columns that require specific projections or transforms to be applied.\nIf this property is absent, a reader MAY access all columns of the table as-is without any mandatory transformations.\nIf this property is present, each listed column MUST have its specified projection or action applied. Columns not listed in required-column-projections are not subject to any read restrictions.\nWhen this list is present:\n1. For each column listed in required-column-projections, the reader MUST apply\n the specified action or transform.\n\n2. If a listed column has an action, the reader MUST apply it and replace\n all references to the underlying column with the transformed value.\n For example, if the action specifies truncate[4](cc), the reader MUST project it\n as truncate[4](cc) AS cc, and all references to cc during query evaluation\n (including in required-row-filter) MUST resolve to this transformed alias.\n\n3. If a listed column has no action specified, the reader MUST project the column as-is\n (equivalent to an identity transform).\n\n4. Columns not listed in required-column-projections MAY be projected normally\n by the reader without any mandatory transformations.\n\n5. A column MUST appear at most once in required-column-projections.\n6. If a projected column's action cannot be evaluated by the reader,\n the reader MUST fail rather than ignore or skip the transform.\n\n7. The data type of a projected column MUST match the data type defined for\n the transform result or the original column type if no transform is specified.\n", + description="A list of columns that require specific projections or transforms to be applied.\nIf this property is absent, a reader MAY access all columns of the table as-is without any mandatory transformations.\nIf this property is present, each listed column MUST have its specified projection or action applied. Columns not listed in required-column-projections are not subject to any read restrictions.\nWhen this list is present:\n1. For each column listed in required-column-projections, the reader MUST apply\n the specified action or transform.\n\n2. The reader MUST apply the action and replace all references to the underlying\n column with the transformed value. For example, if the action specifies\n truncate[4](cc), the reader MUST project it as truncate[4](cc) AS cc, and all\n references to cc during query evaluation (including in required-row-filter)\n MUST resolve to this transformed alias.\n\n3. Columns not listed in required-column-projections MAY be projected normally\n by the reader without any mandatory transformations.\n\n4. A column MUST appear at most once in required-column-projections.\n5. If a projected column's action cannot be evaluated by the reader,\n the reader MUST fail rather than ignore or skip the transform.\n\n6. The data type of a projected column MUST match the data type defined for\n the transform result.\n", ) required_row_filter: Expression | None = Field( None, diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index ca1a2c46a1cd..d811ff0ebfb4 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3868,29 +3868,26 @@ components: 1. For each column listed in required-column-projections, the reader MUST apply the specified action or transform. - 2. If a listed column has an action, the reader MUST apply it and replace - all references to the underlying column with the transformed value. - For example, if the action specifies truncate[4](cc), the reader MUST project it - as truncate[4](cc) AS cc, and all references to cc during query evaluation - (including in required-row-filter) MUST resolve to this transformed alias. + 2. The reader MUST apply the action and replace all references to the underlying + column with the transformed value. For example, if the action specifies + truncate[4](cc), the reader MUST project it as truncate[4](cc) AS cc, and all + references to cc during query evaluation (including in required-row-filter) + MUST resolve to this transformed alias. - 3. If a listed column has no action specified, the reader MUST project the column as-is - (equivalent to an identity transform). - - 4. Columns not listed in required-column-projections MAY be projected normally + 3. Columns not listed in required-column-projections MAY be projected normally by the reader without any mandatory transformations. - 5. A column MUST appear at most once in required-column-projections. + 4. A column MUST appear at most once in required-column-projections. - 6. If a projected column's action cannot be evaluated by the reader, + 5. If a projected column's action cannot be evaluated by the reader, the reader MUST fail rather than ignore or skip the transform. - 7. The data type of a projected column MUST match the data type defined for - the transform result or the original column type if no transform is specified. + 6. The data type of a projected column MUST match the data type defined for + the transform result. type: array items: - $ref: '#/components/schemas/Projection' + $ref: '#/components/schemas/Action' required-row-filter: description: > An expression that filters rows in the table that the authenticated principal does not have access to. @@ -3906,27 +3903,24 @@ components: 4. If this property is absent, null, or always true then no mandatory filtering is required. $ref: '#/components/schemas/Expression' - Projection: + Action: + discriminator: + propertyName: action + mapping: + mask-hash-sha256: '#/components/schemas/MaskHashSha256' + replace-with-null: '#/components/schemas/ReplaceWithNull' + mask-alphanumeric: '#/components/schemas/MaskAlphanumeric' + apply-transform: '#/components/schemas/ApplyTransform' type: object - description: > - Defines a projection for a column. - If action is not specified, the column is projected as-is. + required: + - action + - field-id properties: + action: + type: string field-id: type: integer description: field id of the column being projected. - action: - $ref: '#/components/schemas/Action' - required: - - field-id - - Action: - description: Defines the specific action to be executed for computing the projection. - oneOf: - - $ref: '#/components/schemas/MaskHashSha256' - - $ref: '#/components/schemas/ReplaceWithNull' - - $ref: '#/components/schemas/MaskAlphanumeric' - - $ref: '#/components/schemas/ApplyTransform' MaskHashSha256: description: | @@ -3937,17 +3931,39 @@ components: 1. Convert the column value to a UTF-8 byte array. 2. Apply the SHA-256 algorithm as specified in NIST FIPS 180-4. 3. Convert the resulting 32-byte digest to a 64-character lowercase hexadecimal string. + allOf: + - $ref: '#/components/schemas/Action' + properties: + action: + type: string + const: "mask-hash-sha256" ReplaceWithNull: description: Masks data by replacing it with a NULL value. + allOf: + - $ref: '#/components/schemas/Action' + properties: + action: + type: string + const: "replace-with-null" MaskAlphanumeric: description: mask all alphabetic characters with 'x' and numeric characters with 'n' + allOf: + - $ref: '#/components/schemas/Action' + properties: + action: + type: string + const: "mask-alphanumeric" ApplyTransform: - type: object description: Replace the field with the result of a transform expression. Produce the original field name with the transformed values. + allOf: + - $ref: '#/components/schemas/Action' properties: + action: + type: string + const: "apply-transform" term: $ref: '#/components/schemas/Term' From 07278d79fdca20c8728dcd733bfafa6b6472e6f1 Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Fri, 17 Apr 2026 15:36:40 -0700 Subject: [PATCH 19/51] Model predefined masks as Actions and replace ApplyTransform with ApplyExpression - Replace 4 original actions (mask-hash-sha256, replace-with-null, mask-alphanumeric, apply-transform) with 10 predefined action types: mask-alphanum, mask-to-default, replace-with-null, show-first-4, show-last-4, truncate-to-year, truncate-to-month, sha-256-global, sha-256-query-local, apply-expression - Add discriminator mapping for all action subtypes - Add detailed descriptions for each action including type applicability, encoding rules for SHA-256 variants, and type-specific defaults - Replace Term-based ApplyTransform with Expression-based ApplyExpression - Update required-column-projections rules for action model - Regenerate Python from updated YAML spec --- open-api/rest-catalog-open-api.py | 161 ++++++++++++++++---- open-api/rest-catalog-open-api.yaml | 221 +++++++++++++++++++++++----- 2 files changed, 324 insertions(+), 58 deletions(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index 7bd8c08f81b5..49593a2d7738 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -601,35 +601,134 @@ class Action(BaseModel): ) -class MaskHashSha256(Action): +class MaskAlphanum(Action): """ - Mask the data of the column by applying SHA-256. - The input must be UTF-8 encoded bytes of the column value. - The SHA-256 digest is represented as a lowercase hexadecimal string. - Engines must follow this procedure to ensure consistency: - 1. Convert the column value to a UTF-8 byte array. - 2. Apply the SHA-256 algorithm as specified in NIST FIPS 180-4. - 3. Convert the resulting 32-byte digest to a 64-character lowercase hexadecimal string. + Redacts the column value character by character using the following rules: + - Digits (U+0030–U+0039, 0-9) are replaced with 'n' - The following punctuation characters are kept as-is: + U+0028 '(' LEFT PARENTHESIS + U+0029 ')' RIGHT PARENTHESIS + U+002C ',' COMMA + U+002E '.' FULL STOP + U+002D '-' HYPHEN-MINUS + U+0040 '@' COMMERCIAL AT + - All other Unicode characters (including letters, whitespace, and any punctuation + not listed above) are replaced with 'x' + + For example: "prashant010696@gmail.com" → "xxxxxxxxnnnnnn@xxxxx.xxx" + Applicable to: string + + """ + + action: Literal['mask-alphanum'] = 'mask-alphanum' + + +class MaskToDefault(Action): + """ + Replaces the column value with a predefined type-specific default that conceals the original data while preserving type compatibility. Engines MUST use exactly the values listed below to ensure consistency across implementations. + Default values by type: - boolean: false - int: 999999999 - long: 999999999 - float: 0.0 - double: 0.0 - decimal(p, s): 0 (zero with s digits after the decimal point, e.g. 0.00 for decimal(p,2)) - string: "XXXXXXXX" - date: 9999-12-31 - time: 00:00:00 - timestamp: 9999-12-31T00:00:00 - timestamptz: 9999-12-31T00:00:00+00:00 - timestamp_ns: 9999-12-31T00:00:00.000000000 - timestamptz_ns: 9999-12-31T00:00:00.000000000+00:00 - uuid: 00000000-0000-0000-0000-000000000000 - fixed(n): n zero bytes - binary: empty byte sequence - variant: {"masked": true} - geometry: POINT EMPTY - geography: POINT EMPTY - list: empty list [] - map: empty map {} - struct: struct with each field set to its type-specific default (applied recursively) + Applicable to: all data types """ - action: Literal['mask-hash-sha256'] = Field('mask-hash-sha256', const=True) + action: Literal['mask-to-default'] = 'mask-to-default' class ReplaceWithNull(Action): """ - Masks data by replacing it with a NULL value. + Replaces the entire column value with NULL. + Applicable to: all nullable types + + """ + + action: Literal['replace-with-null'] = 'replace-with-null' + + +class ShowFirst4(Action): + """ + Preserves the first 4 characters of the column value and redacts the remainder using mask-alphanum rules (see MaskAlphanum for the exact character rules). Values with 4 or fewer characters are returned unchanged. + For example: "prashant010696@gmail.com" → "prasxxxxnnnnnn@xxxxx.xxx" + Applicable to: string + + """ + + action: Literal['show-first-4'] = 'show-first-4' + + +class ShowLast4(Action): + """ + Redacts all characters except the last 4 using mask-alphanum rules (see MaskAlphanum for the exact character rules). Values with 4 or fewer characters are returned unchanged. + For example: "4111-1111-1111-4444" → "nnnn-nnnn-nnnn-4444" + Applicable to: string + + """ + + action: Literal['show-last-4'] = 'show-last-4' + + +class TruncateToYear(Action): """ + Truncates the column value to year precision, setting month, day, and time components to their minimum values. The output type matches the input type. + For example: 2024-07-15 → 2024-01-01 + Applicable to: date, timestamp, timestamptz, timestamp_ns, timestamptz_ns - action: Literal['replace-with-null'] = Field('replace-with-null', const=True) + """ + action: Literal['truncate-to-year'] = 'truncate-to-year' -class MaskAlphanumeric(Action): + +class TruncateToMonth(Action): """ - mask all alphabetic characters with 'x' and numeric characters with 'n' + Truncates the column value to year and month precision, setting day and time components to their minimum values. The output type matches the input type. + For example: 2024-07-15 → 2024-07-01 + Applicable to: date, timestamp, timestamptz, timestamp_ns, timestamptz_ns + """ - action: Literal['mask-alphanumeric'] = Field('mask-alphanumeric', const=True) + action: Literal['truncate-to-month'] = 'truncate-to-month' + + +class Sha256Global(Action): + """ + Applies SHA-256 as specified in NIST FIPS 180-4. Deterministic across all queries + and engines — the same input always produces the same output. + + Input-to-bytes encoding by type: + - string: UTF-8 encoded bytes + - int: 4 bytes, little-endian + - long: 8 bytes, little-endian + - uuid: 16 bytes, big-endian (RFC 4122 byte order) + - binary: raw bytes as-is + + Output encoding by type: + - string: 64-character lowercase hexadecimal string + - int: first 4 bytes of the digest, read as a little-endian int + - long: first 8 bytes of the digest, read as a little-endian long + - uuid: all 16 bytes of the digest, interpreted as a UUID (RFC 4122 byte order) + - binary: the full 32-byte raw SHA-256 digest, output type is binary(32) + + Applicable to: string, int, long, uuid, binary + + """ + + action: Literal['sha-256-global'] = 'sha-256-global' + + +class Sha256QueryLocal(Action): + """ + Applies SHA-256 with a per-query random salt, making the output non-deterministic + across queries while remaining consistent within a single query. + + The engine MUST generate a cryptographically random salt for each query and apply it as: + SHA-256(salt_bytes || canonical_bytes) + where canonical_bytes follows the same encoding rules as sha-256-global. + + Output encoding follows the same rules as sha-256-global. + + Applicable to: string, int, long, uuid, binary + + """ + + action: Literal['sha-256-query-local'] = 'sha-256-query-local' class LoadCredentialsResponse(BaseModel): @@ -1344,15 +1443,6 @@ class FunctionDefinitionVersion(BaseModel): ) -class ApplyTransform(Action): - """ - Replace the field with the result of a transform expression. Produce the original field name with the transformed values. - """ - - action: Literal['apply-transform'] = Field('apply-transform', const=True) - term: Term | None = None - - class UnaryExpression(BaseModel): type: Literal['is-null', 'not-null', 'is-nan', 'not-nan'] = Field( ..., @@ -1602,12 +1692,23 @@ class ReadRestrictions(BaseModel): """ required_column_projections: ( - list[MaskHashSha256 | ReplaceWithNull | MaskAlphanumeric | ApplyTransform] + list[ + MaskAlphanum + | MaskToDefault + | ReplaceWithNull + | ShowFirst4 + | ShowLast4 + | TruncateToYear + | TruncateToMonth + | Sha256Global + | Sha256QueryLocal + | ApplyExpression + ] | None ) = Field( None, alias='required-column-projections', - description="A list of columns that require specific projections or transforms to be applied.\nIf this property is absent, a reader MAY access all columns of the table as-is without any mandatory transformations.\nIf this property is present, each listed column MUST have its specified projection or action applied. Columns not listed in required-column-projections are not subject to any read restrictions.\nWhen this list is present:\n1. For each column listed in required-column-projections, the reader MUST apply\n the specified action or transform.\n\n2. The reader MUST apply the action and replace all references to the underlying\n column with the transformed value. For example, if the action specifies\n truncate[4](cc), the reader MUST project it as truncate[4](cc) AS cc, and all\n references to cc during query evaluation (including in required-row-filter)\n MUST resolve to this transformed alias.\n\n3. Columns not listed in required-column-projections MAY be projected normally\n by the reader without any mandatory transformations.\n\n4. A column MUST appear at most once in required-column-projections.\n5. If a projected column's action cannot be evaluated by the reader,\n the reader MUST fail rather than ignore or skip the transform.\n\n6. The data type of a projected column MUST match the data type defined for\n the transform result.\n", + description="A list of columns that require specific actions to be applied when reading.\nIf this property is absent, a reader MAY access all columns of the table as-is without any mandatory transformations.\nIf this property is present, each listed column MUST have its specified action applied. Columns not listed in required-column-projections are not subject to any read restrictions.\nWhen this list is present:\n1. For each column listed in required-column-projections, the reader MUST apply\n the specified action before returning values for that column.\n\n2. The reader MUST replace all output references to the column with the result\n of the action, presenting the result under the original column name. For\n example, if the action for column cc is mask-alphanum, the reader MUST\n return the masked value as cc in the query output.\n\n3. Columns not listed in required-column-projections MAY be projected normally\n by the reader without any mandatory transformations.\n\n4. A column MUST appear at most once in required-column-projections.\n5. If a projected column's action cannot be evaluated by the reader,\n the reader MUST fail rather than ignore or skip the action.\n\n6. Each action defines the output type for its column. For all predefined\n actions except apply-expression, the output type matches the input column\n type. For apply-expression, the output type is determined by the expression.\n", ) required_row_filter: Expression | None = Field( None, @@ -1616,6 +1717,14 @@ class ReadRestrictions(BaseModel): ) +class ApplyExpression(Action): + """ + Replace the field with the result of an expression. Produce the original field name with the expression result. + """ + + action: Literal['apply-expression'] = 'apply-expression' + expression: Expression | None = None + class LoadTableResult(BaseModel): """ @@ -2195,6 +2304,8 @@ class PlanTableScanResult( TableMetadata.model_rebuild() ViewMetadata.model_rebuild() AddSchemaUpdate.model_rebuild() +ReadRestrictions.model_rebuild() +ApplyExpression.model_rebuild() ScanTasks.model_rebuild() CommitTableRequest.model_rebuild() CommitViewRequest.model_rebuild() diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index d811ff0ebfb4..97d15c05f5e9 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3854,25 +3854,24 @@ components: properties: required-column-projections: description: > - A list of columns that require specific projections or transforms to be applied. + A list of columns that require specific actions to be applied when reading. If this property is absent, a reader MAY access all columns of the table as-is without any mandatory transformations. If this property is present, each listed column MUST have its specified - projection or action applied. Columns not listed in required-column-projections + action applied. Columns not listed in required-column-projections are not subject to any read restrictions. When this list is present: 1. For each column listed in required-column-projections, the reader MUST apply - the specified action or transform. + the specified action before returning values for that column. - 2. The reader MUST apply the action and replace all references to the underlying - column with the transformed value. For example, if the action specifies - truncate[4](cc), the reader MUST project it as truncate[4](cc) AS cc, and all - references to cc during query evaluation (including in required-row-filter) - MUST resolve to this transformed alias. + 2. The reader MUST replace all output references to the column with the result + of the action, presenting the result under the original column name. For + example, if the action for column cc is mask-alphanum, the reader MUST + return the masked value as cc in the query output. 3. Columns not listed in required-column-projections MAY be projected normally by the reader without any mandatory transformations. @@ -3880,10 +3879,11 @@ components: 4. A column MUST appear at most once in required-column-projections. 5. If a projected column's action cannot be evaluated by the reader, - the reader MUST fail rather than ignore or skip the transform. + the reader MUST fail rather than ignore or skip the action. - 6. The data type of a projected column MUST match the data type defined for - the transform result. + 6. Each action defines the output type for its column. For all predefined + actions except apply-expression, the output type matches the input column + type. For apply-expression, the output type is determined by the expression. type: array items: @@ -3907,10 +3907,16 @@ components: discriminator: propertyName: action mapping: - mask-hash-sha256: '#/components/schemas/MaskHashSha256' + mask-alphanum: '#/components/schemas/MaskAlphanum' + mask-to-default: '#/components/schemas/MaskToDefault' replace-with-null: '#/components/schemas/ReplaceWithNull' - mask-alphanumeric: '#/components/schemas/MaskAlphanumeric' - apply-transform: '#/components/schemas/ApplyTransform' + show-first-4: '#/components/schemas/ShowFirst4' + show-last-4: '#/components/schemas/ShowLast4' + truncate-to-year: '#/components/schemas/TruncateToYear' + truncate-to-month: '#/components/schemas/TruncateToMonth' + sha-256-global: '#/components/schemas/Sha256Global' + sha-256-query-local: '#/components/schemas/Sha256QueryLocal' + apply-expression: '#/components/schemas/ApplyExpression' type: object required: - action @@ -3922,24 +3928,74 @@ components: type: integer description: field id of the column being projected. - MaskHashSha256: - description: | - Mask the data of the column by applying SHA-256. - The input must be UTF-8 encoded bytes of the column value. - The SHA-256 digest is represented as a lowercase hexadecimal string. - Engines must follow this procedure to ensure consistency: - 1. Convert the column value to a UTF-8 byte array. - 2. Apply the SHA-256 algorithm as specified in NIST FIPS 180-4. - 3. Convert the resulting 32-byte digest to a 64-character lowercase hexadecimal string. + MaskAlphanum: + description: > + Redacts the column value character by character using the following rules: + + - Digits (U+0030–U+0039, 0-9) are replaced with 'n' + - The following punctuation characters are kept as-is: + U+0028 '(' LEFT PARENTHESIS + U+0029 ')' RIGHT PARENTHESIS + U+002C ',' COMMA + U+002E '.' FULL STOP + U+002D '-' HYPHEN-MINUS + U+0040 '@' COMMERCIAL AT + - All other Unicode characters (including letters, whitespace, and any punctuation + not listed above) are replaced with 'x' + + For example: "prashant010696@gmail.com" → "xxxxxxxxnnnnnn@xxxxx.xxx" + + Applicable to: string + allOf: + - $ref: '#/components/schemas/Action' + properties: + action: + type: string + const: "mask-alphanum" + + MaskToDefault: + description: > + Replaces the column value with a predefined type-specific default that conceals + the original data while preserving type compatibility. Engines MUST use exactly + the values listed below to ensure consistency across implementations. + + Default values by type: + - boolean: false + - int: 999999999 + - long: 999999999 + - float: 0.0 + - double: 0.0 + - decimal(p, s): 0 (zero with s digits after the decimal point, e.g. 0.00 for decimal(p,2)) + - string: "XXXXXXXX" + - date: 9999-12-31 + - time: 00:00:00 + - timestamp: 9999-12-31T00:00:00 + - timestamptz: 9999-12-31T00:00:00+00:00 + - timestamp_ns: 9999-12-31T00:00:00.000000000 + - timestamptz_ns: 9999-12-31T00:00:00.000000000+00:00 + - uuid: 00000000-0000-0000-0000-000000000000 + - fixed(n): n zero bytes + - binary: empty byte sequence + - variant: {"masked": true} + - geometry: POINT EMPTY + - geography: POINT EMPTY + - list: empty list [] + - map: empty map {} + - struct: struct with each field set to its type-specific default (applied recursively) + + Applicable to: all data types allOf: - $ref: '#/components/schemas/Action' properties: action: type: string - const: "mask-hash-sha256" + const: "mask-to-default" ReplaceWithNull: - description: Masks data by replacing it with a NULL value. + description: > + Replaces the entire column value with NULL. + + Applicable to: all nullable types allOf: - $ref: '#/components/schemas/Action' properties: @@ -3947,25 +4003,124 @@ components: type: string const: "replace-with-null" - MaskAlphanumeric: - description: mask all alphabetic characters with 'x' and numeric characters with 'n' + ShowFirst4: + description: > + Preserves the first 4 characters of the column value and redacts the remainder + using mask-alphanum rules (see MaskAlphanum for the exact character rules). + Values with 4 or fewer characters are returned unchanged. + + For example: "prashant010696@gmail.com" → "prasxxxxnnnnnn@xxxxx.xxx" + + Applicable to: string allOf: - $ref: '#/components/schemas/Action' properties: action: type: string - const: "mask-alphanumeric" + const: "show-first-4" + + ShowLast4: + description: > + Redacts all characters except the last 4 using mask-alphanum rules + (see MaskAlphanum for the exact character rules). + Values with 4 or fewer characters are returned unchanged. - ApplyTransform: - description: Replace the field with the result of a transform expression. Produce the original field name with the transformed values. + For example: "4111-1111-1111-4444" → "nnnn-nnnn-nnnn-4444" + + Applicable to: string allOf: - $ref: '#/components/schemas/Action' properties: action: type: string - const: "apply-transform" - term: - $ref: '#/components/schemas/Term' + const: "show-last-4" + + TruncateToYear: + description: > + Truncates the column value to year precision, setting month, day, and time components + to their minimum values. The output type matches the input type. + + For example: 2024-07-15 → 2024-01-01 + + Applicable to: date, timestamp, timestamptz, timestamp_ns, timestamptz_ns + allOf: + - $ref: '#/components/schemas/Action' + properties: + action: + type: string + const: "truncate-to-year" + + TruncateToMonth: + description: > + Truncates the column value to year and month precision, setting day and time components + to their minimum values. The output type matches the input type. + + For example: 2024-07-15 → 2024-07-01 + + Applicable to: date, timestamp, timestamptz, timestamp_ns, timestamptz_ns + allOf: + - $ref: '#/components/schemas/Action' + properties: + action: + type: string + const: "truncate-to-month" + + Sha256Global: + description: | + Applies SHA-256 as specified in NIST FIPS 180-4. Deterministic across all queries + and engines — the same input always produces the same output. + + Input-to-bytes encoding by type: + - string: UTF-8 encoded bytes + - int: 4 bytes, little-endian + - long: 8 bytes, little-endian + - uuid: 16 bytes, big-endian (RFC 4122 byte order) + - binary: raw bytes as-is + + Output encoding by type: + - string: 64-character lowercase hexadecimal string + - int: first 4 bytes of the digest, read as a little-endian int + - long: first 8 bytes of the digest, read as a little-endian long + - uuid: all 16 bytes of the digest, interpreted as a UUID (RFC 4122 byte order) + - binary: the full 32-byte raw SHA-256 digest, output type is binary(32) + + Applicable to: string, int, long, uuid, binary + allOf: + - $ref: '#/components/schemas/Action' + properties: + action: + type: string + const: "sha-256-global" + + Sha256QueryLocal: + description: | + Applies SHA-256 with a per-query random salt, making the output non-deterministic + across queries while remaining consistent within a single query. + + The engine MUST generate a cryptographically random salt for each query and apply it as: + SHA-256(salt_bytes || canonical_bytes) + where canonical_bytes follows the same encoding rules as sha-256-global. + + Output encoding follows the same rules as sha-256-global. + + Applicable to: string, int, long, uuid, binary + allOf: + - $ref: '#/components/schemas/Action' + properties: + action: + type: string + const: "sha-256-query-local" + + ApplyExpression: + description: Replace the field with the result of an expression. Produce the original field name with the expression result. + allOf: + - $ref: '#/components/schemas/Action' + properties: + action: + type: string + const: "apply-expression" + expression: + $ref: '#/components/schemas/Expression' LoadCredentialsResponse: type: object From f9fd2a813c28bc3cfe3418a9e4811c871cfbe01a Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Sat, 18 Apr 2026 00:24:20 -0700 Subject: [PATCH 20/51] Tighten spec precision for catalog and engine implementers - Mark ApplyExpression.expression as required - Fix SHA-256 binary output type from binary(32) to binary - Add NULL-in/NULL-out global rule for all actions - Specify Unicode code point semantics for masking actions - Add signed two's complement for SHA-256 int/long output - Add UTC truncation rule for timestamptz types - Clarify empty ReadRestrictions equivalence and server type validation - Specify minimum 16-byte salt for sha-256-query-local - Clarify unrecognized action types trigger MUST-fail - Add "Applicable to: all data types" for ApplyExpression - Regenerate Python from updated YAML spec --- open-api/rest-catalog-open-api.py | 27 ++++++++++--------- open-api/rest-catalog-open-api.yaml | 40 ++++++++++++++++++++--------- 2 files changed, 43 insertions(+), 24 deletions(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index 49593a2d7738..56ab519d91c9 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -603,7 +603,7 @@ class Action(BaseModel): class MaskAlphanum(Action): """ - Redacts the column value character by character using the following rules: + Redacts the column value Unicode code point by code point using the following rules: - Digits (U+0030–U+0039, 0-9) are replaced with 'n' - The following punctuation characters are kept as-is: U+0028 '(' LEFT PARENTHESIS U+0029 ')' RIGHT PARENTHESIS @@ -645,7 +645,7 @@ class ReplaceWithNull(Action): class ShowFirst4(Action): """ - Preserves the first 4 characters of the column value and redacts the remainder using mask-alphanum rules (see MaskAlphanum for the exact character rules). Values with 4 or fewer characters are returned unchanged. + Preserves the first 4 Unicode code points of the column value and redacts the remainder using mask-alphanum rules (see MaskAlphanum for the exact character rules). Values with 4 or fewer Unicode code points are returned unchanged. For example: "prashant010696@gmail.com" → "prasxxxxnnnnnn@xxxxx.xxx" Applicable to: string @@ -656,7 +656,7 @@ class ShowFirst4(Action): class ShowLast4(Action): """ - Redacts all characters except the last 4 using mask-alphanum rules (see MaskAlphanum for the exact character rules). Values with 4 or fewer characters are returned unchanged. + Redacts all Unicode code points except the last 4 using mask-alphanum rules (see MaskAlphanum for the exact character rules). Values with 4 or fewer Unicode code points are returned unchanged. For example: "4111-1111-1111-4444" → "nnnn-nnnn-nnnn-4444" Applicable to: string @@ -668,7 +668,7 @@ class ShowLast4(Action): class TruncateToYear(Action): """ Truncates the column value to year precision, setting month, day, and time components to their minimum values. The output type matches the input type. - For example: 2024-07-15 → 2024-01-01 + For example: 2024-07-15 → 2024-01-01 For timestamptz and timestamptz_ns, truncation is performed in UTC. Applicable to: date, timestamp, timestamptz, timestamp_ns, timestamptz_ns """ @@ -679,7 +679,7 @@ class TruncateToYear(Action): class TruncateToMonth(Action): """ Truncates the column value to year and month precision, setting day and time components to their minimum values. The output type matches the input type. - For example: 2024-07-15 → 2024-07-01 + For example: 2024-07-15 → 2024-07-01 For timestamptz and timestamptz_ns, truncation is performed in UTC. Applicable to: date, timestamp, timestamptz, timestamp_ns, timestamptz_ns """ @@ -701,10 +701,10 @@ class Sha256Global(Action): Output encoding by type: - string: 64-character lowercase hexadecimal string - - int: first 4 bytes of the digest, read as a little-endian int - - long: first 8 bytes of the digest, read as a little-endian long + - int: first 4 bytes of the digest, read as a signed two's complement little-endian int + - long: first 8 bytes of the digest, read as a signed two's complement little-endian long - uuid: all 16 bytes of the digest, interpreted as a UUID (RFC 4122 byte order) - - binary: the full 32-byte raw SHA-256 digest, output type is binary(32) + - binary: the full 32-byte raw SHA-256 digest Applicable to: string, int, long, uuid, binary @@ -718,7 +718,7 @@ class Sha256QueryLocal(Action): Applies SHA-256 with a per-query random salt, making the output non-deterministic across queries while remaining consistent within a single query. - The engine MUST generate a cryptographically random salt for each query and apply it as: + The engine MUST generate a cryptographically random salt of at least 16 bytes for each query and apply it as: SHA-256(salt_bytes || canonical_bytes) where canonical_bytes follows the same encoding rules as sha-256-global. @@ -1688,6 +1688,7 @@ class ReadRestrictions(BaseModel): Read restrictions for a table, including column projections and row filter expressions. A client MUST enforce the restrictions defined in this object when reading data from the table. These restrictions apply only to the authenticated principal, user, or account associated with the request. They MUST NOT be interpreted as global policy and MUST NOT be applied beyond the entity identified by the Authentication header (or other applicable authentication mechanism). + If both properties are absent or empty, the ReadRestrictions object imposes no restrictions and is equivalent to the field being absent from the response. A server MUST NOT return an action for a column whose type is not listed in that action's "Applicable to" set. For all actions, if the input column value is NULL, the output MUST be NULL. """ @@ -1708,7 +1709,7 @@ class ReadRestrictions(BaseModel): ) = Field( None, alias='required-column-projections', - description="A list of columns that require specific actions to be applied when reading.\nIf this property is absent, a reader MAY access all columns of the table as-is without any mandatory transformations.\nIf this property is present, each listed column MUST have its specified action applied. Columns not listed in required-column-projections are not subject to any read restrictions.\nWhen this list is present:\n1. For each column listed in required-column-projections, the reader MUST apply\n the specified action before returning values for that column.\n\n2. The reader MUST replace all output references to the column with the result\n of the action, presenting the result under the original column name. For\n example, if the action for column cc is mask-alphanum, the reader MUST\n return the masked value as cc in the query output.\n\n3. Columns not listed in required-column-projections MAY be projected normally\n by the reader without any mandatory transformations.\n\n4. A column MUST appear at most once in required-column-projections.\n5. If a projected column's action cannot be evaluated by the reader,\n the reader MUST fail rather than ignore or skip the action.\n\n6. Each action defines the output type for its column. For all predefined\n actions except apply-expression, the output type matches the input column\n type. For apply-expression, the output type is determined by the expression.\n", + description="A list of columns that require specific actions to be applied when reading.\nIf this property is absent, a reader MAY access all columns of the table as-is without any mandatory transformations.\nIf this property is present, each listed column MUST have its specified action applied. Columns not listed in required-column-projections are not subject to any read restrictions.\nWhen this list is present:\n1. For each column listed in required-column-projections, the reader MUST apply\n the specified action before returning values for that column.\n\n2. The reader MUST replace all output references to the column with the result\n of the action, presenting the result under the original column name. For\n example, if the action for column cc is mask-alphanum, the reader MUST\n return the masked value as cc in the query output.\n\n3. Columns not listed in required-column-projections MAY be projected normally\n by the reader without any mandatory transformations.\n\n4. A column MUST appear at most once in required-column-projections.\n5. If a projected column's action cannot be evaluated by the reader\n (including unrecognized action types), the reader MUST fail rather than\n ignore or skip the action.\n\n6. Each action defines the output type for its column. For all predefined\n actions except apply-expression, the output type matches the input column\n type. For apply-expression, the output type is determined by the expression.\n", ) required_row_filter: Expression | None = Field( None, @@ -1720,10 +1721,12 @@ class ReadRestrictions(BaseModel): class ApplyExpression(Action): """ Replace the field with the result of an expression. Produce the original field name with the expression result. + Applicable to: all data types + """ - action: Literal['apply-expression'] = 'apply-expression' - expression: Expression | None = None + action: Literal['apply-expression'] + expression: Expression class LoadTableResult(BaseModel): diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index 97d15c05f5e9..5fcc9f79206e 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3851,6 +3851,12 @@ components: associated with the request. They MUST NOT be interpreted as global policy and MUST NOT be applied beyond the entity identified by the Authentication header (or other applicable authentication mechanism). + + If both properties are absent or empty, the ReadRestrictions object imposes no + restrictions and is equivalent to the field being absent from the response. + A server MUST NOT return an action for a column whose type is not listed in + that action's "Applicable to" set. + For all actions, if the input column value is NULL, the output MUST be NULL. properties: required-column-projections: description: > @@ -3878,8 +3884,9 @@ components: 4. A column MUST appear at most once in required-column-projections. - 5. If a projected column's action cannot be evaluated by the reader, - the reader MUST fail rather than ignore or skip the action. + 5. If a projected column's action cannot be evaluated by the reader + (including unrecognized action types), the reader MUST fail rather than + ignore or skip the action. 6. Each action defines the output type for its column. For all predefined actions except apply-expression, the output type matches the input column @@ -3930,7 +3937,7 @@ components: MaskAlphanum: description: > - Redacts the column value character by character using the following rules: + Redacts the column value Unicode code point by code point using the following rules: - Digits (U+0030–U+0039, 0-9) are replaced with 'n' - The following punctuation characters are kept as-is: @@ -4005,9 +4012,9 @@ components: ShowFirst4: description: > - Preserves the first 4 characters of the column value and redacts the remainder + Preserves the first 4 Unicode code points of the column value and redacts the remainder using mask-alphanum rules (see MaskAlphanum for the exact character rules). - Values with 4 or fewer characters are returned unchanged. + Values with 4 or fewer Unicode code points are returned unchanged. For example: "prashant010696@gmail.com" → "prasxxxxnnnnnn@xxxxx.xxx" @@ -4021,9 +4028,9 @@ components: ShowLast4: description: > - Redacts all characters except the last 4 using mask-alphanum rules + Redacts all Unicode code points except the last 4 using mask-alphanum rules (see MaskAlphanum for the exact character rules). - Values with 4 or fewer characters are returned unchanged. + Values with 4 or fewer Unicode code points are returned unchanged. For example: "4111-1111-1111-4444" → "nnnn-nnnn-nnnn-4444" @@ -4041,6 +4048,7 @@ components: to their minimum values. The output type matches the input type. For example: 2024-07-15 → 2024-01-01 + For timestamptz and timestamptz_ns, truncation is performed in UTC. Applicable to: date, timestamp, timestamptz, timestamp_ns, timestamptz_ns allOf: @@ -4056,6 +4064,7 @@ components: to their minimum values. The output type matches the input type. For example: 2024-07-15 → 2024-07-01 + For timestamptz and timestamptz_ns, truncation is performed in UTC. Applicable to: date, timestamp, timestamptz, timestamp_ns, timestamptz_ns allOf: @@ -4079,10 +4088,10 @@ components: Output encoding by type: - string: 64-character lowercase hexadecimal string - - int: first 4 bytes of the digest, read as a little-endian int - - long: first 8 bytes of the digest, read as a little-endian long + - int: first 4 bytes of the digest, read as a signed two's complement little-endian int + - long: first 8 bytes of the digest, read as a signed two's complement little-endian long - uuid: all 16 bytes of the digest, interpreted as a UUID (RFC 4122 byte order) - - binary: the full 32-byte raw SHA-256 digest, output type is binary(32) + - binary: the full 32-byte raw SHA-256 digest Applicable to: string, int, long, uuid, binary allOf: @@ -4097,7 +4106,7 @@ components: Applies SHA-256 with a per-query random salt, making the output non-deterministic across queries while remaining consistent within a single query. - The engine MUST generate a cryptographically random salt for each query and apply it as: + The engine MUST generate a cryptographically random salt of at least 16 bytes for each query and apply it as: SHA-256(salt_bytes || canonical_bytes) where canonical_bytes follows the same encoding rules as sha-256-global. @@ -4112,9 +4121,16 @@ components: const: "sha-256-query-local" ApplyExpression: - description: Replace the field with the result of an expression. Produce the original field name with the expression result. + description: > + Replace the field with the result of an expression. Produce the original field name + with the expression result. + + Applicable to: all data types allOf: - $ref: '#/components/schemas/Action' + required: + - action + - expression properties: action: type: string From 2b85d808e3f099f1b7590d85268b2ff374a37f2d Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Sat, 18 Apr 2026 01:12:34 -0700 Subject: [PATCH 21/51] Remove uuid from SHA-256 action applicable types SHA-256 digest truncated to 16 bytes does not produce valid RFC 4122 UUIDs (version and variant bits are not set), so uuid is removed from the input encoding, output encoding, and applicable types for both sha-256-global and sha-256-query-local actions. --- open-api/rest-catalog-open-api.py | 6 ++---- open-api/rest-catalog-open-api.yaml | 6 ++---- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index 56ab519d91c9..ca080f0a7232 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -696,17 +696,15 @@ class Sha256Global(Action): - string: UTF-8 encoded bytes - int: 4 bytes, little-endian - long: 8 bytes, little-endian - - uuid: 16 bytes, big-endian (RFC 4122 byte order) - binary: raw bytes as-is Output encoding by type: - string: 64-character lowercase hexadecimal string - int: first 4 bytes of the digest, read as a signed two's complement little-endian int - long: first 8 bytes of the digest, read as a signed two's complement little-endian long - - uuid: all 16 bytes of the digest, interpreted as a UUID (RFC 4122 byte order) - binary: the full 32-byte raw SHA-256 digest - Applicable to: string, int, long, uuid, binary + Applicable to: string, int, long, binary """ @@ -724,7 +722,7 @@ class Sha256QueryLocal(Action): Output encoding follows the same rules as sha-256-global. - Applicable to: string, int, long, uuid, binary + Applicable to: string, int, long, binary """ diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index 5fcc9f79206e..422d407a0db5 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -4083,17 +4083,15 @@ components: - string: UTF-8 encoded bytes - int: 4 bytes, little-endian - long: 8 bytes, little-endian - - uuid: 16 bytes, big-endian (RFC 4122 byte order) - binary: raw bytes as-is Output encoding by type: - string: 64-character lowercase hexadecimal string - int: first 4 bytes of the digest, read as a signed two's complement little-endian int - long: first 8 bytes of the digest, read as a signed two's complement little-endian long - - uuid: all 16 bytes of the digest, interpreted as a UUID (RFC 4122 byte order) - binary: the full 32-byte raw SHA-256 digest - Applicable to: string, int, long, uuid, binary + Applicable to: string, int, long, binary allOf: - $ref: '#/components/schemas/Action' properties: @@ -4112,7 +4110,7 @@ components: Output encoding follows the same rules as sha-256-global. - Applicable to: string, int, long, uuid, binary + Applicable to: string, int, long, binary allOf: - $ref: '#/components/schemas/Action' properties: From 061de26ebb2ab87763945edd5fa5fc5ad15ecc65 Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Tue, 21 Apr 2026 17:58:54 -0700 Subject: [PATCH 22/51] Fix timestamp_ns mask-to-default overflow in spec The spec mandated 9999-12-31T00:00:00 as the mask-to-default sentinel for timestamp_ns and timestamptz_ns, but that value overflows a 64-bit signed integer as nanoseconds from epoch (max representable is ~2262-04-11). Engines MUST use exactly the listed default, so specifying an unrepresentable value broke cross-engine consistency. Use 2261-12-31T00:00:00 as the closest clean far-future sentinel that fits. Add a note explaining the overflow so implementers don't mistake it for a typo. --- open-api/rest-catalog-open-api.py | 3 ++- open-api/rest-catalog-open-api.yaml | 8 ++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index ca080f0a7232..b3686c38d6db 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -625,7 +625,8 @@ class MaskAlphanum(Action): class MaskToDefault(Action): """ Replaces the column value with a predefined type-specific default that conceals the original data while preserving type compatibility. Engines MUST use exactly the values listed below to ensure consistency across implementations. - Default values by type: - boolean: false - int: 999999999 - long: 999999999 - float: 0.0 - double: 0.0 - decimal(p, s): 0 (zero with s digits after the decimal point, e.g. 0.00 for decimal(p,2)) - string: "XXXXXXXX" - date: 9999-12-31 - time: 00:00:00 - timestamp: 9999-12-31T00:00:00 - timestamptz: 9999-12-31T00:00:00+00:00 - timestamp_ns: 9999-12-31T00:00:00.000000000 - timestamptz_ns: 9999-12-31T00:00:00.000000000+00:00 - uuid: 00000000-0000-0000-0000-000000000000 - fixed(n): n zero bytes - binary: empty byte sequence - variant: {"masked": true} - geometry: POINT EMPTY - geography: POINT EMPTY - list: empty list [] - map: empty map {} - struct: struct with each field set to its type-specific default (applied recursively) + Default values by type: - boolean: false - int: 999999999 - long: 999999999 - float: 0.0 - double: 0.0 - decimal(p, s): 0 (zero with s digits after the decimal point, e.g. 0.00 for decimal(p,2)) - string: "XXXXXXXX" - date: 9999-12-31 - time: 00:00:00 - timestamp: 9999-12-31T00:00:00 - timestamptz: 9999-12-31T00:00:00+00:00 - timestamp_ns: 2261-12-31T00:00:00.000000000 - timestamptz_ns: 2261-12-31T00:00:00.000000000+00:00 - uuid: 00000000-0000-0000-0000-000000000000 - fixed(n): n zero bytes - binary: empty byte sequence - variant: {"masked": true} - geometry: POINT EMPTY - geography: POINT EMPTY - list: empty list [] - map: empty map {} - struct: struct with each field set to its type-specific default (applied recursively) + Note: nanosecond-precision timestamps (timestamp_ns, timestamptz_ns) cannot represent 9999-12-31 because nanoseconds from epoch overflows a 64-bit signed integer past approximately 2262-04-11; 2261-12-31 is used as the closest clean far-future sentinel. Applicable to: all data types """ diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index 422d407a0db5..f5fe2afeb182 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3978,8 +3978,8 @@ components: - time: 00:00:00 - timestamp: 9999-12-31T00:00:00 - timestamptz: 9999-12-31T00:00:00+00:00 - - timestamp_ns: 9999-12-31T00:00:00.000000000 - - timestamptz_ns: 9999-12-31T00:00:00.000000000+00:00 + - timestamp_ns: 2261-12-31T00:00:00.000000000 + - timestamptz_ns: 2261-12-31T00:00:00.000000000+00:00 - uuid: 00000000-0000-0000-0000-000000000000 - fixed(n): n zero bytes - binary: empty byte sequence @@ -3990,6 +3990,10 @@ components: - map: empty map {} - struct: struct with each field set to its type-specific default (applied recursively) + Note: nanosecond-precision timestamps (timestamp_ns, timestamptz_ns) cannot represent + 9999-12-31 because nanoseconds from epoch overflows a 64-bit signed integer past + approximately 2262-04-11; 2261-12-31 is used as the closest clean far-future sentinel. + Applicable to: all data types allOf: - $ref: '#/components/schemas/Action' From 9eb3477d3f9b130ba140a21b37e357cf9ca0c1c1 Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Sat, 2 May 2026 16:25:17 -0700 Subject: [PATCH 23/51] Open API: Address FGAC spec review feedback - Row filter: require boolean evaluation (resolves @nicholasdoyleg) - Add struct coexistence rule: no subfield projections when struct is projected - Rename mask-to-default to mask-to-fixed-value - Adopt zero-of-type defaults (epoch dates/timestamps, int/long=0, variant={}) while keeping string="XXXXXXXX" for observability - Drop nanos-overflow note (epoch fits int64 nanos) --- open-api/rest-catalog-open-api.py | 14 ++++----- open-api/rest-catalog-open-api.yaml | 44 +++++++++++++++-------------- 2 files changed, 30 insertions(+), 28 deletions(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index b3686c38d6db..5dd530894b90 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -622,16 +622,15 @@ class MaskAlphanum(Action): action: Literal['mask-alphanum'] = 'mask-alphanum' -class MaskToDefault(Action): +class MaskToFixedValue(Action): """ - Replaces the column value with a predefined type-specific default that conceals the original data while preserving type compatibility. Engines MUST use exactly the values listed below to ensure consistency across implementations. - Default values by type: - boolean: false - int: 999999999 - long: 999999999 - float: 0.0 - double: 0.0 - decimal(p, s): 0 (zero with s digits after the decimal point, e.g. 0.00 for decimal(p,2)) - string: "XXXXXXXX" - date: 9999-12-31 - time: 00:00:00 - timestamp: 9999-12-31T00:00:00 - timestamptz: 9999-12-31T00:00:00+00:00 - timestamp_ns: 2261-12-31T00:00:00.000000000 - timestamptz_ns: 2261-12-31T00:00:00.000000000+00:00 - uuid: 00000000-0000-0000-0000-000000000000 - fixed(n): n zero bytes - binary: empty byte sequence - variant: {"masked": true} - geometry: POINT EMPTY - geography: POINT EMPTY - list: empty list [] - map: empty map {} - struct: struct with each field set to its type-specific default (applied recursively) - Note: nanosecond-precision timestamps (timestamp_ns, timestamptz_ns) cannot represent 9999-12-31 because nanoseconds from epoch overflows a 64-bit signed integer past approximately 2262-04-11; 2261-12-31 is used as the closest clean far-future sentinel. + Replaces the column value with a predefined type-specific fixed value. Engines MUST use exactly the values listed below to ensure consistency across implementations. + Fixed values by type: - boolean: false - int: 0 - long: 0 - float: 0.0 - double: 0.0 - decimal(p, s): 0 (zero with s digits after the decimal point, e.g. 0.00 for decimal(p,2)) - string: "XXXXXXXX" - date: 1970-01-01 - time: 00:00:00 - timestamp: 1970-01-01T00:00:00 - timestamptz: 1970-01-01T00:00:00+00:00 - timestamp_ns: 1970-01-01T00:00:00.000000000 - timestamptz_ns: 1970-01-01T00:00:00.000000000+00:00 - uuid: 00000000-0000-0000-0000-000000000000 - fixed(n): n zero bytes - binary: empty byte sequence - variant: {} - geometry: POINT EMPTY - geography: POINT EMPTY - list: empty list [] - map: empty map {} - struct: struct with each field set to its type-specific default (applied recursively) Applicable to: all data types """ - action: Literal['mask-to-default'] = 'mask-to-default' + action: Literal['mask-to-fixed-value'] = 'mask-to-fixed-value' class ReplaceWithNull(Action): @@ -1688,13 +1687,14 @@ class ReadRestrictions(BaseModel): A client MUST enforce the restrictions defined in this object when reading data from the table. These restrictions apply only to the authenticated principal, user, or account associated with the request. They MUST NOT be interpreted as global policy and MUST NOT be applied beyond the entity identified by the Authentication header (or other applicable authentication mechanism). If both properties are absent or empty, the ReadRestrictions object imposes no restrictions and is equivalent to the field being absent from the response. A server MUST NOT return an action for a column whose type is not listed in that action's "Applicable to" set. For all actions, if the input column value is NULL, the output MUST be NULL. + If a column projection targets a struct-typed field, other column projections in the same ReadRestrictions MUST NOT target any of that struct's subfields (at any depth). This avoids ambiguity about which action governs a given leaf value. """ required_column_projections: ( list[ MaskAlphanum - | MaskToDefault + | MaskToFixedValue | ReplaceWithNull | ShowFirst4 | ShowLast4 @@ -1713,7 +1713,7 @@ class ReadRestrictions(BaseModel): required_row_filter: Expression | None = Field( None, alias='required-row-filter', - description='An expression that filters rows in the table that the authenticated principal does not have access to.\n1. A reader MUST discard any row for which the filter evaluates to false or null, and\n no information derived from discarded rows MAY be included in the query result.\n\n2. Row filters MUST be evaluated against the original, untransformed column values.\n Required projections MUST be applied only after row filters are applied.\n\n3. If a client cannot interpret or evaluate a provided filter expression, it MUST fail.\n4. If this property is absent, null, or always true then no mandatory filtering is required.\n', + description='An expression that filters rows in the table that the authenticated principal does not have access to.\n1. The expression MUST evaluate to a boolean. A reader MUST discard any row for which\n the filter evaluates to FALSE, and no information derived from discarded rows\n MAY be included in the query result.\n\n2. Row filters MUST be evaluated against the original, untransformed column values.\n Required projections MUST be applied only after row filters are applied.\n\n3. If a client cannot interpret or evaluate a provided filter expression, it MUST fail.\n4. If this property is absent, null, or always true then no mandatory filtering is required.\n', ) diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index f5fe2afeb182..379e4a767ba2 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3857,6 +3857,11 @@ components: A server MUST NOT return an action for a column whose type is not listed in that action's "Applicable to" set. For all actions, if the input column value is NULL, the output MUST be NULL. + + If a column projection targets a struct-typed field, other column projections + in the same ReadRestrictions MUST NOT target any of that struct's subfields + (at any depth). This avoids ambiguity about which action governs a given + leaf value. properties: required-column-projections: description: > @@ -3899,8 +3904,9 @@ components: description: > An expression that filters rows in the table that the authenticated principal does not have access to. - 1. A reader MUST discard any row for which the filter evaluates to false or null, and - no information derived from discarded rows MAY be included in the query result. + 1. The expression MUST evaluate to a boolean. A reader MUST discard any row for which + the filter evaluates to FALSE, and no information derived from discarded rows + MAY be included in the query result. 2. Row filters MUST be evaluated against the original, untransformed column values. Required projections MUST be applied only after row filters are applied. @@ -3915,7 +3921,7 @@ components: propertyName: action mapping: mask-alphanum: '#/components/schemas/MaskAlphanum' - mask-to-default: '#/components/schemas/MaskToDefault' + mask-to-fixed-value: '#/components/schemas/MaskToFixedValue' replace-with-null: '#/components/schemas/ReplaceWithNull' show-first-4: '#/components/schemas/ShowFirst4' show-last-4: '#/components/schemas/ShowLast4' @@ -3960,47 +3966,43 @@ components: type: string const: "mask-alphanum" - MaskToDefault: + MaskToFixedValue: description: > - Replaces the column value with a predefined type-specific default that conceals - the original data while preserving type compatibility. Engines MUST use exactly - the values listed below to ensure consistency across implementations. + Replaces the column value with a predefined type-specific fixed value. + Engines MUST use exactly the values listed below to ensure consistency + across implementations. - Default values by type: + Fixed values by type: - boolean: false - - int: 999999999 - - long: 999999999 + - int: 0 + - long: 0 - float: 0.0 - double: 0.0 - decimal(p, s): 0 (zero with s digits after the decimal point, e.g. 0.00 for decimal(p,2)) - string: "XXXXXXXX" - - date: 9999-12-31 + - date: 1970-01-01 - time: 00:00:00 - - timestamp: 9999-12-31T00:00:00 - - timestamptz: 9999-12-31T00:00:00+00:00 - - timestamp_ns: 2261-12-31T00:00:00.000000000 - - timestamptz_ns: 2261-12-31T00:00:00.000000000+00:00 + - timestamp: 1970-01-01T00:00:00 + - timestamptz: 1970-01-01T00:00:00+00:00 + - timestamp_ns: 1970-01-01T00:00:00.000000000 + - timestamptz_ns: 1970-01-01T00:00:00.000000000+00:00 - uuid: 00000000-0000-0000-0000-000000000000 - fixed(n): n zero bytes - binary: empty byte sequence - - variant: {"masked": true} + - variant: {} - geometry: POINT EMPTY - geography: POINT EMPTY - list: empty list [] - map: empty map {} - struct: struct with each field set to its type-specific default (applied recursively) - Note: nanosecond-precision timestamps (timestamp_ns, timestamptz_ns) cannot represent - 9999-12-31 because nanoseconds from epoch overflows a 64-bit signed integer past - approximately 2262-04-11; 2261-12-31 is used as the closest clean far-future sentinel. - Applicable to: all data types allOf: - $ref: '#/components/schemas/Action' properties: action: type: string - const: "mask-to-default" + const: "mask-to-fixed-value" ReplaceWithNull: description: > From 973b28f82122248e23579ac64b76695c1644e7d4 Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Sun, 17 May 2026 11:45:24 -0700 Subject: [PATCH 24/51] Open API: Refine FGAC read restrictions spec Address review feedback and sync conclusions on the ReadRestrictions schema and its actions: - Remove apply-expression action (deferred until expression-language extension lands). - Drop the global "NULL -> NULL" rule and write per-action NULL behavior into all 9 actions. mask-to-fixed-value emits the type-specific default for NULL inputs (NULL not preserved); the other 8 actions preserve NULL. - ReplaceWithNull: server MUST NOT return for non-nullable columns. - Tighten fail-mode wording to fail-closed semantics. Reader MUST fail the query with an error and MUST NOT silently return raw, partial, or empty results to mask the failure (projection rule 5 and row-filter rule 3). - Unify the normative actor as "reader" for per-row behavior, keep "engine" for query-scoped behavior (salt generation), and define both terms inline. - Add a downstream-applicability sentence: restrictions apply to every read operation that uses metadata from the load response, including planTableScan calls. - Add a worked JSON example for ReadRestrictions. - Clarify row-filter rule 1: Iceberg expressions are 2-valued (TRUE or FALSE; never NULL). - Sha256QueryLocal: replace cryptographic concat notation (SHA-256(salt_bytes || canonical_bytes)) with plain prose so the spec is dialect-agnostic. - Wrap required-row-filter $ref in allOf so the description is no longer silently dropped by OpenAPI tooling. - Style/wording cleanup: capitalize "Field ID", clarify "two properties" wording. --- open-api/rest-catalog-open-api.py | 63 ++++++++++------ open-api/rest-catalog-open-api.yaml | 108 ++++++++++++++++++---------- 2 files changed, 110 insertions(+), 61 deletions(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index 5dd530894b90..8159f1e4864d 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -597,7 +597,7 @@ class StorageCredential(BaseModel): class Action(BaseModel): action: str field_id: int = Field( - ..., alias='field-id', description='field id of the column being projected.' + ..., alias='field-id', description='Field ID of the column being projected.' ) @@ -615,6 +615,7 @@ class MaskAlphanum(Action): not listed above) are replaced with 'x' For example: "prashant010696@gmail.com" → "xxxxxxxxnnnnnn@xxxxx.xxx" + NULL input is preserved (NULL → NULL). Applicable to: string """ @@ -624,8 +625,9 @@ class MaskAlphanum(Action): class MaskToFixedValue(Action): """ - Replaces the column value with a predefined type-specific fixed value. Engines MUST use exactly the values listed below to ensure consistency across implementations. + Replaces the column value with a predefined type-specific fixed value. Readers MUST use exactly the values listed below to ensure consistency across implementations. Fixed values by type: - boolean: false - int: 0 - long: 0 - float: 0.0 - double: 0.0 - decimal(p, s): 0 (zero with s digits after the decimal point, e.g. 0.00 for decimal(p,2)) - string: "XXXXXXXX" - date: 1970-01-01 - time: 00:00:00 - timestamp: 1970-01-01T00:00:00 - timestamptz: 1970-01-01T00:00:00+00:00 - timestamp_ns: 1970-01-01T00:00:00.000000000 - timestamptz_ns: 1970-01-01T00:00:00.000000000+00:00 - uuid: 00000000-0000-0000-0000-000000000000 - fixed(n): n zero bytes - binary: empty byte sequence - variant: {} - geometry: POINT EMPTY - geography: POINT EMPTY - list: empty list [] - map: empty map {} - struct: struct with each field set to its type-specific default (applied recursively) + NULL input is also replaced with the type-specific fixed value; NULL is not preserved. Applicable to: all data types """ @@ -635,7 +637,7 @@ class MaskToFixedValue(Action): class ReplaceWithNull(Action): """ - Replaces the entire column value with NULL. + Replaces the entire column value with NULL. NULL input is preserved (NULL → NULL). A server MUST NOT return this action for a non-nullable (required) column. Applicable to: all nullable types """ @@ -647,6 +649,7 @@ class ShowFirst4(Action): """ Preserves the first 4 Unicode code points of the column value and redacts the remainder using mask-alphanum rules (see MaskAlphanum for the exact character rules). Values with 4 or fewer Unicode code points are returned unchanged. For example: "prashant010696@gmail.com" → "prasxxxxnnnnnn@xxxxx.xxx" + NULL input is preserved (NULL → NULL). Applicable to: string """ @@ -658,6 +661,7 @@ class ShowLast4(Action): """ Redacts all Unicode code points except the last 4 using mask-alphanum rules (see MaskAlphanum for the exact character rules). Values with 4 or fewer Unicode code points are returned unchanged. For example: "4111-1111-1111-4444" → "nnnn-nnnn-nnnn-4444" + NULL input is preserved (NULL → NULL). Applicable to: string """ @@ -669,6 +673,7 @@ class TruncateToYear(Action): """ Truncates the column value to year precision, setting month, day, and time components to their minimum values. The output type matches the input type. For example: 2024-07-15 → 2024-01-01 For timestamptz and timestamptz_ns, truncation is performed in UTC. + NULL input is preserved (NULL → NULL). Applicable to: date, timestamp, timestamptz, timestamp_ns, timestamptz_ns """ @@ -680,6 +685,7 @@ class TruncateToMonth(Action): """ Truncates the column value to year and month precision, setting day and time components to their minimum values. The output type matches the input type. For example: 2024-07-15 → 2024-07-01 For timestamptz and timestamptz_ns, truncation is performed in UTC. + NULL input is preserved (NULL → NULL). Applicable to: date, timestamp, timestamptz, timestamp_ns, timestamptz_ns """ @@ -704,6 +710,8 @@ class Sha256Global(Action): - long: first 8 bytes of the digest, read as a signed two's complement little-endian long - binary: the full 32-byte raw SHA-256 digest + NULL input is preserved (NULL → NULL). + Applicable to: string, int, long, binary """ @@ -716,12 +724,16 @@ class Sha256QueryLocal(Action): Applies SHA-256 with a per-query random salt, making the output non-deterministic across queries while remaining consistent within a single query. - The engine MUST generate a cryptographically random salt of at least 16 bytes for each query and apply it as: - SHA-256(salt_bytes || canonical_bytes) - where canonical_bytes follows the same encoding rules as sha-256-global. + The engine MUST generate a cryptographically random salt of at least 16 bytes for each query. + + For each column value, the engine MUST encode the value to bytes using + sha-256-global's input rules, prepend the per-query salt, and compute + SHA-256 over the result. Output encoding follows the same rules as sha-256-global. + NULL input is preserved (NULL → NULL). + Applicable to: string, int, long, binary """ @@ -1684,10 +1696,28 @@ class AddSchemaUpdate(BaseUpdate): class ReadRestrictions(BaseModel): """ Read restrictions for a table, including column projections and row filter expressions. - A client MUST enforce the restrictions defined in this object when reading data from the table. + A reader MUST enforce the restrictions defined in this object when reading data from the table. Read restrictions returned in a loadTable response apply to every read operation on the loaded table performed using this response, including subsequent planTableScan and fetchScanTasks calls. + In this section, "reader" refers to the read-side actor that applies restrictions per row or per column. "Engine" refers to the broader query-execution context that defines query lifetime and scope (e.g. a SQL session, a single PyIceberg scan), and is the actor responsible for query-scoped behavior such as salt generation in sha-256-query-local. These restrictions apply only to the authenticated principal, user, or account associated with the request. They MUST NOT be interpreted as global policy and MUST NOT be applied beyond the entity identified by the Authentication header (or other applicable authentication mechanism). - If both properties are absent or empty, the ReadRestrictions object imposes no restrictions and is equivalent to the field being absent from the response. A server MUST NOT return an action for a column whose type is not listed in that action's "Applicable to" set. For all actions, if the input column value is NULL, the output MUST be NULL. + An empty ReadRestrictions object (no required-column-projections and no required-row-filter) imposes no restrictions and is equivalent to the field being absent from the response. A server MUST NOT return an action for a column whose type is not listed in that action's "Applicable to" set. + NULL handling is action-specific. Each action's description specifies its behavior on NULL input. If a column projection targets a struct-typed field, other column projections in the same ReadRestrictions MUST NOT target any of that struct's subfields (at any depth). This avoids ambiguity about which action governs a given leaf value. + Example: + + { + "required-column-projections": [ + { "field-id": 4, "action": "show-last-4" }, + { "field-id": 6, "action": "replace-with-null" }, + { "field-id": 8, "action": "truncate-to-year" }, + { "field-id": 10, "action": "sha-256-global" }, + { "field-id": 12, "action": "mask-alphanum" } + ], + "required-row-filter": { + "type": "eq", + "term": "region", + "value": "US" + } + } """ @@ -1702,32 +1732,20 @@ class ReadRestrictions(BaseModel): | TruncateToMonth | Sha256Global | Sha256QueryLocal - | ApplyExpression ] | None ) = Field( None, alias='required-column-projections', - description="A list of columns that require specific actions to be applied when reading.\nIf this property is absent, a reader MAY access all columns of the table as-is without any mandatory transformations.\nIf this property is present, each listed column MUST have its specified action applied. Columns not listed in required-column-projections are not subject to any read restrictions.\nWhen this list is present:\n1. For each column listed in required-column-projections, the reader MUST apply\n the specified action before returning values for that column.\n\n2. The reader MUST replace all output references to the column with the result\n of the action, presenting the result under the original column name. For\n example, if the action for column cc is mask-alphanum, the reader MUST\n return the masked value as cc in the query output.\n\n3. Columns not listed in required-column-projections MAY be projected normally\n by the reader without any mandatory transformations.\n\n4. A column MUST appear at most once in required-column-projections.\n5. If a projected column's action cannot be evaluated by the reader\n (including unrecognized action types), the reader MUST fail rather than\n ignore or skip the action.\n\n6. Each action defines the output type for its column. For all predefined\n actions except apply-expression, the output type matches the input column\n type. For apply-expression, the output type is determined by the expression.\n", + description="A list of columns that require specific actions to be applied when reading.\nIf this property is absent, a reader MAY access all columns of the table as-is without any mandatory transformations.\nIf this property is present, each listed column MUST have its specified action applied. Columns not listed in required-column-projections are not subject to any read restrictions.\nWhen this list is present:\n1. For each column listed in required-column-projections, the reader MUST apply\n the specified action before returning values for that column.\n\n2. The reader MUST replace all output references to the column with the result\n of the action, presenting the result under the original column name. For\n example, if the action for column cc is mask-alphanum, the reader MUST\n return the masked value as cc in the query output.\n\n3. Columns not listed in required-column-projections MAY be projected normally\n by the reader without any mandatory transformations.\n\n4. A column MUST appear at most once in required-column-projections.\n5. If a projected column's action cannot be evaluated by the reader (including\n unrecognized action types), the reader MUST fail the query with an error to\n the caller. The reader MUST NOT silently return raw, partial, or empty\n results to mask the failure.\n\n6. Each action defines the output type for its column. For all predefined\n actions, the output type matches the input column type.\n", ) required_row_filter: Expression | None = Field( None, alias='required-row-filter', - description='An expression that filters rows in the table that the authenticated principal does not have access to.\n1. The expression MUST evaluate to a boolean. A reader MUST discard any row for which\n the filter evaluates to FALSE, and no information derived from discarded rows\n MAY be included in the query result.\n\n2. Row filters MUST be evaluated against the original, untransformed column values.\n Required projections MUST be applied only after row filters are applied.\n\n3. If a client cannot interpret or evaluate a provided filter expression, it MUST fail.\n4. If this property is absent, null, or always true then no mandatory filtering is required.\n', + description='An expression that filters rows in the table that the authenticated principal does not have access to.\n1. The expression MUST evaluate to a boolean (TRUE or FALSE; Iceberg expressions\n never produce NULL). A reader MUST discard any row for which\n the filter evaluates to FALSE, and no information derived from discarded rows\n MAY be included in the query result.\n\n2. Row filters MUST be evaluated against the original, untransformed column values.\n Required projections MUST be applied only after row filters are applied.\n\n3. If a reader cannot interpret or evaluate a provided filter expression, it MUST\n fail the query with an error to the caller. The reader MUST NOT silently return\n partial, raw, or empty results to mask the failure.\n\n4. If this property is absent, null, or always true then no mandatory filtering is required.\n', ) -class ApplyExpression(Action): - """ - Replace the field with the result of an expression. Produce the original field name with the expression result. - Applicable to: all data types - - """ - - action: Literal['apply-expression'] - expression: Expression - - class LoadTableResult(BaseModel): """ Result used when a table is successfully loaded. @@ -2307,7 +2325,6 @@ class PlanTableScanResult( ViewMetadata.model_rebuild() AddSchemaUpdate.model_rebuild() ReadRestrictions.model_rebuild() -ApplyExpression.model_rebuild() ScanTasks.model_rebuild() CommitTableRequest.model_rebuild() CommitViewRequest.model_rebuild() diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index 379e4a767ba2..6321882283b4 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3844,24 +3844,52 @@ components: description: > Read restrictions for a table, including column projections and row filter expressions. - A client MUST enforce the restrictions defined in this object when reading data - from the table. + A reader MUST enforce the restrictions defined in this object when reading data + from the table. Read restrictions returned in a loadTable response apply to every + read operation on the loaded table performed using this response, including + subsequent planTableScan and fetchScanTasks calls. + + In this section, "reader" refers to the read-side actor that applies restrictions + per row or per column. "Engine" refers to the broader query-execution context + that defines query lifetime and scope (e.g. a SQL session, a single PyIceberg + scan), and is the actor responsible for query-scoped behavior such as salt + generation in sha-256-query-local. These restrictions apply only to the authenticated principal, user, or account associated with the request. They MUST NOT be interpreted as global policy and MUST NOT be applied beyond the entity identified by the Authentication header (or other applicable authentication mechanism). - If both properties are absent or empty, the ReadRestrictions object imposes no - restrictions and is equivalent to the field being absent from the response. + An empty ReadRestrictions object (no required-column-projections and no + required-row-filter) imposes no restrictions and is equivalent to the field + being absent from the response. A server MUST NOT return an action for a column whose type is not listed in that action's "Applicable to" set. - For all actions, if the input column value is NULL, the output MUST be NULL. + + NULL handling is action-specific. Each action's description specifies its + behavior on NULL input. If a column projection targets a struct-typed field, other column projections in the same ReadRestrictions MUST NOT target any of that struct's subfields (at any depth). This avoids ambiguity about which action governs a given leaf value. + + Example: + + { + "required-column-projections": [ + { "field-id": 4, "action": "show-last-4" }, + { "field-id": 6, "action": "replace-with-null" }, + { "field-id": 8, "action": "truncate-to-year" }, + { "field-id": 10, "action": "sha-256-global" }, + { "field-id": 12, "action": "mask-alphanum" } + ], + "required-row-filter": { + "type": "eq", + "term": "region", + "value": "US" + } + } properties: required-column-projections: description: > @@ -3889,13 +3917,13 @@ components: 4. A column MUST appear at most once in required-column-projections. - 5. If a projected column's action cannot be evaluated by the reader - (including unrecognized action types), the reader MUST fail rather than - ignore or skip the action. + 5. If a projected column's action cannot be evaluated by the reader (including + unrecognized action types), the reader MUST fail the query with an error to + the caller. The reader MUST NOT silently return raw, partial, or empty + results to mask the failure. 6. Each action defines the output type for its column. For all predefined - actions except apply-expression, the output type matches the input column - type. For apply-expression, the output type is determined by the expression. + actions, the output type matches the input column type. type: array items: @@ -3904,17 +3932,21 @@ components: description: > An expression that filters rows in the table that the authenticated principal does not have access to. - 1. The expression MUST evaluate to a boolean. A reader MUST discard any row for which + 1. The expression MUST evaluate to a boolean (TRUE or FALSE; Iceberg expressions + never produce NULL). A reader MUST discard any row for which the filter evaluates to FALSE, and no information derived from discarded rows MAY be included in the query result. 2. Row filters MUST be evaluated against the original, untransformed column values. Required projections MUST be applied only after row filters are applied. - 3. If a client cannot interpret or evaluate a provided filter expression, it MUST fail. + 3. If a reader cannot interpret or evaluate a provided filter expression, it MUST + fail the query with an error to the caller. The reader MUST NOT silently return + partial, raw, or empty results to mask the failure. 4. If this property is absent, null, or always true then no mandatory filtering is required. - $ref: '#/components/schemas/Expression' + allOf: + - $ref: '#/components/schemas/Expression' Action: discriminator: @@ -3929,7 +3961,6 @@ components: truncate-to-month: '#/components/schemas/TruncateToMonth' sha-256-global: '#/components/schemas/Sha256Global' sha-256-query-local: '#/components/schemas/Sha256QueryLocal' - apply-expression: '#/components/schemas/ApplyExpression' type: object required: - action @@ -3939,7 +3970,7 @@ components: type: string field-id: type: integer - description: field id of the column being projected. + description: Field ID of the column being projected. MaskAlphanum: description: > @@ -3958,6 +3989,8 @@ components: For example: "prashant010696@gmail.com" → "xxxxxxxxnnnnnn@xxxxx.xxx" + NULL input is preserved (NULL → NULL). + Applicable to: string allOf: - $ref: '#/components/schemas/Action' @@ -3969,7 +4002,7 @@ components: MaskToFixedValue: description: > Replaces the column value with a predefined type-specific fixed value. - Engines MUST use exactly the values listed below to ensure consistency + Readers MUST use exactly the values listed below to ensure consistency across implementations. Fixed values by type: @@ -3996,6 +4029,8 @@ components: - map: empty map {} - struct: struct with each field set to its type-specific default (applied recursively) + NULL input is also replaced with the type-specific fixed value; NULL is not preserved. + Applicable to: all data types allOf: - $ref: '#/components/schemas/Action' @@ -4006,7 +4041,8 @@ components: ReplaceWithNull: description: > - Replaces the entire column value with NULL. + Replaces the entire column value with NULL. NULL input is preserved (NULL → NULL). + A server MUST NOT return this action for a non-nullable (required) column. Applicable to: all nullable types allOf: @@ -4024,6 +4060,8 @@ components: For example: "prashant010696@gmail.com" → "prasxxxxnnnnnn@xxxxx.xxx" + NULL input is preserved (NULL → NULL). + Applicable to: string allOf: - $ref: '#/components/schemas/Action' @@ -4040,6 +4078,8 @@ components: For example: "4111-1111-1111-4444" → "nnnn-nnnn-nnnn-4444" + NULL input is preserved (NULL → NULL). + Applicable to: string allOf: - $ref: '#/components/schemas/Action' @@ -4056,6 +4096,8 @@ components: For example: 2024-07-15 → 2024-01-01 For timestamptz and timestamptz_ns, truncation is performed in UTC. + NULL input is preserved (NULL → NULL). + Applicable to: date, timestamp, timestamptz, timestamp_ns, timestamptz_ns allOf: - $ref: '#/components/schemas/Action' @@ -4072,6 +4114,8 @@ components: For example: 2024-07-15 → 2024-07-01 For timestamptz and timestamptz_ns, truncation is performed in UTC. + NULL input is preserved (NULL → NULL). + Applicable to: date, timestamp, timestamptz, timestamp_ns, timestamptz_ns allOf: - $ref: '#/components/schemas/Action' @@ -4097,6 +4141,8 @@ components: - long: first 8 bytes of the digest, read as a signed two's complement little-endian long - binary: the full 32-byte raw SHA-256 digest + NULL input is preserved (NULL → NULL). + Applicable to: string, int, long, binary allOf: - $ref: '#/components/schemas/Action' @@ -4110,12 +4156,16 @@ components: Applies SHA-256 with a per-query random salt, making the output non-deterministic across queries while remaining consistent within a single query. - The engine MUST generate a cryptographically random salt of at least 16 bytes for each query and apply it as: - SHA-256(salt_bytes || canonical_bytes) - where canonical_bytes follows the same encoding rules as sha-256-global. + The engine MUST generate a cryptographically random salt of at least 16 bytes for each query. + + For each column value, the engine MUST encode the value to bytes using + sha-256-global's input rules, prepend the per-query salt, and compute + SHA-256 over the result. Output encoding follows the same rules as sha-256-global. + NULL input is preserved (NULL → NULL). + Applicable to: string, int, long, binary allOf: - $ref: '#/components/schemas/Action' @@ -4124,24 +4174,6 @@ components: type: string const: "sha-256-query-local" - ApplyExpression: - description: > - Replace the field with the result of an expression. Produce the original field name - with the expression result. - - Applicable to: all data types - allOf: - - $ref: '#/components/schemas/Action' - required: - - action - - expression - properties: - action: - type: string - const: "apply-expression" - expression: - $ref: '#/components/schemas/Expression' - LoadCredentialsResponse: type: object required: From 6b67e4059863ba83209ec45e6a544e228b0ba97b Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Tue, 9 Jun 2026 14:34:46 -0700 Subject: [PATCH 25/51] Spec: V1 conditional wording for read-restrictions Replace unconditional MUST with SHOULD support + conditional MUST fail-closed. Clients that support read-restrictions MUST fail if they cannot apply any returned restriction (including unrecognized action or expression types). This leaves room for V2 to tighten to unconditional. --- open-api/rest-catalog-open-api.yaml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index 6321882283b4..99a9bd233fd9 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3844,10 +3844,12 @@ components: description: > Read restrictions for a table, including column projections and row filter expressions. - A reader MUST enforce the restrictions defined in this object when reading data - from the table. Read restrictions returned in a loadTable response apply to every - read operation on the loaded table performed using this response, including - subsequent planTableScan and fetchScanTasks calls. + A client SHOULD support the read-restrictions field. If a client supports + read-restrictions, it MUST fail if it cannot apply any returned restriction + (including unrecognized action or expression types). Read restrictions returned + in a loadTable response apply to every read operation on the loaded table + performed using this response, including subsequent planTableScan and + fetchScanTasks calls. In this section, "reader" refers to the read-side actor that applies restrictions per row or per column. "Engine" refers to the broader query-execution context From eda96631e1a34549c8e65afbdf5e294c23b499ba Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Tue, 9 Jun 2026 14:35:13 -0700 Subject: [PATCH 26/51] Spec: Update Python model with V1 read-restrictions wording --- open-api/rest-catalog-open-api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index 8159f1e4864d..633613c1652a 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -1696,7 +1696,7 @@ class AddSchemaUpdate(BaseUpdate): class ReadRestrictions(BaseModel): """ Read restrictions for a table, including column projections and row filter expressions. - A reader MUST enforce the restrictions defined in this object when reading data from the table. Read restrictions returned in a loadTable response apply to every read operation on the loaded table performed using this response, including subsequent planTableScan and fetchScanTasks calls. + A client SHOULD support the read-restrictions field. If a client supports read-restrictions, it MUST fail if it cannot apply any returned restriction (including unrecognized action or expression types). Read restrictions returned in a loadTable response apply to every read operation on the loaded table performed using this response, including subsequent planTableScan and fetchScanTasks calls. In this section, "reader" refers to the read-side actor that applies restrictions per row or per column. "Engine" refers to the broader query-execution context that defines query lifetime and scope (e.g. a SQL session, a single PyIceberg scan), and is the actor responsible for query-scoped behavior such as salt generation in sha-256-query-local. These restrictions apply only to the authenticated principal, user, or account associated with the request. They MUST NOT be interpreted as global policy and MUST NOT be applied beyond the entity identified by the Authentication header (or other applicable authentication mechanism). An empty ReadRestrictions object (no required-column-projections and no required-row-filter) imposes no restrictions and is equivalent to the field being absent from the response. A server MUST NOT return an action for a column whose type is not listed in that action's "Applicable to" set. From e8d97f2651a2dac3a1dbb006cb62b1daa445894a Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Sun, 14 Jun 2026 15:41:40 -0700 Subject: [PATCH 27/51] Spec: Lowercase RFC2119 keywords and drop redundant MAYs Address rdblue review on PR #13879: - L3635: lowercase MUST/SHOULD/MAY across the ReadRestrictions section. - L3711: drop the duplicated "MAY be projected normally" rule from required-column-projections; renumber rules 4-6 to 3-5. - L3698 (partial): drop the "If absent, a reader MAY access all columns" sentence; absence is implied by the columns-not-listed semantics. --- open-api/rest-catalog-open-api.py | 20 +++++----- open-api/rest-catalog-open-api.yaml | 58 +++++++++++++---------------- 2 files changed, 36 insertions(+), 42 deletions(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index 633613c1652a..196c980ddb6b 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -625,7 +625,7 @@ class MaskAlphanum(Action): class MaskToFixedValue(Action): """ - Replaces the column value with a predefined type-specific fixed value. Readers MUST use exactly the values listed below to ensure consistency across implementations. + Replaces the column value with a predefined type-specific fixed value. Readers must use exactly the values listed below to ensure consistency across implementations. Fixed values by type: - boolean: false - int: 0 - long: 0 - float: 0.0 - double: 0.0 - decimal(p, s): 0 (zero with s digits after the decimal point, e.g. 0.00 for decimal(p,2)) - string: "XXXXXXXX" - date: 1970-01-01 - time: 00:00:00 - timestamp: 1970-01-01T00:00:00 - timestamptz: 1970-01-01T00:00:00+00:00 - timestamp_ns: 1970-01-01T00:00:00.000000000 - timestamptz_ns: 1970-01-01T00:00:00.000000000+00:00 - uuid: 00000000-0000-0000-0000-000000000000 - fixed(n): n zero bytes - binary: empty byte sequence - variant: {} - geometry: POINT EMPTY - geography: POINT EMPTY - list: empty list [] - map: empty map {} - struct: struct with each field set to its type-specific default (applied recursively) NULL input is also replaced with the type-specific fixed value; NULL is not preserved. Applicable to: all data types @@ -637,7 +637,7 @@ class MaskToFixedValue(Action): class ReplaceWithNull(Action): """ - Replaces the entire column value with NULL. NULL input is preserved (NULL → NULL). A server MUST NOT return this action for a non-nullable (required) column. + Replaces the entire column value with NULL. NULL input is preserved (NULL → NULL). A server must not return this action for a non-nullable (required) column. Applicable to: all nullable types """ @@ -724,9 +724,9 @@ class Sha256QueryLocal(Action): Applies SHA-256 with a per-query random salt, making the output non-deterministic across queries while remaining consistent within a single query. - The engine MUST generate a cryptographically random salt of at least 16 bytes for each query. + The engine must generate a cryptographically random salt of at least 16 bytes for each query. - For each column value, the engine MUST encode the value to bytes using + For each column value, the engine must encode the value to bytes using sha-256-global's input rules, prepend the per-query salt, and compute SHA-256 over the result. @@ -1696,12 +1696,12 @@ class AddSchemaUpdate(BaseUpdate): class ReadRestrictions(BaseModel): """ Read restrictions for a table, including column projections and row filter expressions. - A client SHOULD support the read-restrictions field. If a client supports read-restrictions, it MUST fail if it cannot apply any returned restriction (including unrecognized action or expression types). Read restrictions returned in a loadTable response apply to every read operation on the loaded table performed using this response, including subsequent planTableScan and fetchScanTasks calls. + A client should support the read-restrictions field. If a client supports read-restrictions, it must fail if it cannot apply any returned restriction (including unrecognized action or expression types). Read restrictions returned in a loadTable response apply to every read operation on the loaded table performed using this response, including subsequent planTableScan and fetchScanTasks calls. In this section, "reader" refers to the read-side actor that applies restrictions per row or per column. "Engine" refers to the broader query-execution context that defines query lifetime and scope (e.g. a SQL session, a single PyIceberg scan), and is the actor responsible for query-scoped behavior such as salt generation in sha-256-query-local. - These restrictions apply only to the authenticated principal, user, or account associated with the request. They MUST NOT be interpreted as global policy and MUST NOT be applied beyond the entity identified by the Authentication header (or other applicable authentication mechanism). - An empty ReadRestrictions object (no required-column-projections and no required-row-filter) imposes no restrictions and is equivalent to the field being absent from the response. A server MUST NOT return an action for a column whose type is not listed in that action's "Applicable to" set. + These restrictions apply only to the authenticated principal, user, or account associated with the request. They must not be interpreted as global policy and must not be applied beyond the entity identified by the Authentication header (or other applicable authentication mechanism). + An empty ReadRestrictions object (no required-column-projections and no required-row-filter) imposes no restrictions and is equivalent to the field being absent from the response. A server must not return an action for a column whose type is not listed in that action's "Applicable to" set. NULL handling is action-specific. Each action's description specifies its behavior on NULL input. - If a column projection targets a struct-typed field, other column projections in the same ReadRestrictions MUST NOT target any of that struct's subfields (at any depth). This avoids ambiguity about which action governs a given leaf value. + If a column projection targets a struct-typed field, other column projections in the same ReadRestrictions must not target any of that struct's subfields (at any depth). This avoids ambiguity about which action governs a given leaf value. Example: { @@ -1737,12 +1737,12 @@ class ReadRestrictions(BaseModel): ) = Field( None, alias='required-column-projections', - description="A list of columns that require specific actions to be applied when reading.\nIf this property is absent, a reader MAY access all columns of the table as-is without any mandatory transformations.\nIf this property is present, each listed column MUST have its specified action applied. Columns not listed in required-column-projections are not subject to any read restrictions.\nWhen this list is present:\n1. For each column listed in required-column-projections, the reader MUST apply\n the specified action before returning values for that column.\n\n2. The reader MUST replace all output references to the column with the result\n of the action, presenting the result under the original column name. For\n example, if the action for column cc is mask-alphanum, the reader MUST\n return the masked value as cc in the query output.\n\n3. Columns not listed in required-column-projections MAY be projected normally\n by the reader without any mandatory transformations.\n\n4. A column MUST appear at most once in required-column-projections.\n5. If a projected column's action cannot be evaluated by the reader (including\n unrecognized action types), the reader MUST fail the query with an error to\n the caller. The reader MUST NOT silently return raw, partial, or empty\n results to mask the failure.\n\n6. Each action defines the output type for its column. For all predefined\n actions, the output type matches the input column type.\n", + description="A list of columns that require specific actions to be applied when reading.\nIf this property is present, each listed column must have its specified action applied. Columns not listed in required-column-projections are not subject to any read restrictions.\nWhen this list is present:\n1. For each column listed in required-column-projections, the reader must apply\n the specified action before returning values for that column.\n\n2. The reader must replace all output references to the column with the result\n of the action, presenting the result under the original column name. For\n example, if the action for column cc is mask-alphanum, the reader must\n return the masked value as cc in the query output.\n\n3. A column must appear at most once in required-column-projections.\n4. If a projected column's action cannot be evaluated by the reader (including\n unrecognized action types), the reader must fail the query with an error to\n the caller. The reader must not silently return raw, partial, or empty\n results to mask the failure.\n\n5. Each action defines the output type for its column. For all predefined\n actions, the output type matches the input column type.\n", ) required_row_filter: Expression | None = Field( None, alias='required-row-filter', - description='An expression that filters rows in the table that the authenticated principal does not have access to.\n1. The expression MUST evaluate to a boolean (TRUE or FALSE; Iceberg expressions\n never produce NULL). A reader MUST discard any row for which\n the filter evaluates to FALSE, and no information derived from discarded rows\n MAY be included in the query result.\n\n2. Row filters MUST be evaluated against the original, untransformed column values.\n Required projections MUST be applied only after row filters are applied.\n\n3. If a reader cannot interpret or evaluate a provided filter expression, it MUST\n fail the query with an error to the caller. The reader MUST NOT silently return\n partial, raw, or empty results to mask the failure.\n\n4. If this property is absent, null, or always true then no mandatory filtering is required.\n', + description='An expression that filters rows in the table that the authenticated principal does not have access to.\n1. The expression must evaluate to a boolean (TRUE or FALSE; Iceberg expressions\n never produce NULL). A reader must discard any row for which\n the filter evaluates to FALSE, and no information derived from discarded rows\n may be included in the query result.\n\n2. Row filters must be evaluated against the original, untransformed column values.\n Required projections must be applied only after row filters are applied.\n\n3. If a reader cannot interpret or evaluate a provided filter expression, it must\n fail the query with an error to the caller. The reader must not silently return\n partial, raw, or empty results to mask the failure.\n\n4. If this property is absent, null, or always true then no mandatory filtering is required.\n', ) diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index 99a9bd233fd9..644bb71a8445 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3844,8 +3844,8 @@ components: description: > Read restrictions for a table, including column projections and row filter expressions. - A client SHOULD support the read-restrictions field. If a client supports - read-restrictions, it MUST fail if it cannot apply any returned restriction + A client should support the read-restrictions field. If a client supports + read-restrictions, it must fail if it cannot apply any returned restriction (including unrecognized action or expression types). Read restrictions returned in a loadTable response apply to every read operation on the loaded table performed using this response, including subsequent planTableScan and @@ -3858,21 +3858,21 @@ components: generation in sha-256-query-local. These restrictions apply only to the authenticated principal, user, or account - associated with the request. They MUST NOT be interpreted as global policy and - MUST NOT be applied beyond the entity identified by the Authentication header + associated with the request. They must not be interpreted as global policy and + must not be applied beyond the entity identified by the Authentication header (or other applicable authentication mechanism). An empty ReadRestrictions object (no required-column-projections and no required-row-filter) imposes no restrictions and is equivalent to the field being absent from the response. - A server MUST NOT return an action for a column whose type is not listed in + A server must not return an action for a column whose type is not listed in that action's "Applicable to" set. NULL handling is action-specific. Each action's description specifies its behavior on NULL input. If a column projection targets a struct-typed field, other column projections - in the same ReadRestrictions MUST NOT target any of that struct's subfields + in the same ReadRestrictions must not target any of that struct's subfields (at any depth). This avoids ambiguity about which action governs a given leaf value. @@ -3897,34 +3897,28 @@ components: description: > A list of columns that require specific actions to be applied when reading. - If this property is absent, a reader MAY access all columns of the table as-is - without any mandatory transformations. - - If this property is present, each listed column MUST have its specified + If this property is present, each listed column must have its specified action applied. Columns not listed in required-column-projections are not subject to any read restrictions. When this list is present: - 1. For each column listed in required-column-projections, the reader MUST apply + 1. For each column listed in required-column-projections, the reader must apply the specified action before returning values for that column. - 2. The reader MUST replace all output references to the column with the result + 2. The reader must replace all output references to the column with the result of the action, presenting the result under the original column name. For - example, if the action for column cc is mask-alphanum, the reader MUST + example, if the action for column cc is mask-alphanum, the reader must return the masked value as cc in the query output. - 3. Columns not listed in required-column-projections MAY be projected normally - by the reader without any mandatory transformations. - - 4. A column MUST appear at most once in required-column-projections. + 3. A column must appear at most once in required-column-projections. - 5. If a projected column's action cannot be evaluated by the reader (including - unrecognized action types), the reader MUST fail the query with an error to - the caller. The reader MUST NOT silently return raw, partial, or empty + 4. If a projected column's action cannot be evaluated by the reader (including + unrecognized action types), the reader must fail the query with an error to + the caller. The reader must not silently return raw, partial, or empty results to mask the failure. - 6. Each action defines the output type for its column. For all predefined + 5. Each action defines the output type for its column. For all predefined actions, the output type matches the input column type. type: array @@ -3934,16 +3928,16 @@ components: description: > An expression that filters rows in the table that the authenticated principal does not have access to. - 1. The expression MUST evaluate to a boolean (TRUE or FALSE; Iceberg expressions - never produce NULL). A reader MUST discard any row for which + 1. The expression must evaluate to a boolean (TRUE or FALSE; Iceberg expressions + never produce NULL). A reader must discard any row for which the filter evaluates to FALSE, and no information derived from discarded rows - MAY be included in the query result. + may be included in the query result. - 2. Row filters MUST be evaluated against the original, untransformed column values. - Required projections MUST be applied only after row filters are applied. + 2. Row filters must be evaluated against the original, untransformed column values. + Required projections must be applied only after row filters are applied. - 3. If a reader cannot interpret or evaluate a provided filter expression, it MUST - fail the query with an error to the caller. The reader MUST NOT silently return + 3. If a reader cannot interpret or evaluate a provided filter expression, it must + fail the query with an error to the caller. The reader must not silently return partial, raw, or empty results to mask the failure. 4. If this property is absent, null, or always true then no mandatory filtering is required. @@ -4004,7 +3998,7 @@ components: MaskToFixedValue: description: > Replaces the column value with a predefined type-specific fixed value. - Readers MUST use exactly the values listed below to ensure consistency + Readers must use exactly the values listed below to ensure consistency across implementations. Fixed values by type: @@ -4044,7 +4038,7 @@ components: ReplaceWithNull: description: > Replaces the entire column value with NULL. NULL input is preserved (NULL → NULL). - A server MUST NOT return this action for a non-nullable (required) column. + A server must not return this action for a non-nullable (required) column. Applicable to: all nullable types allOf: @@ -4158,9 +4152,9 @@ components: Applies SHA-256 with a per-query random salt, making the output non-deterministic across queries while remaining consistent within a single query. - The engine MUST generate a cryptographically random salt of at least 16 bytes for each query. + The engine must generate a cryptographically random salt of at least 16 bytes for each query. - For each column value, the engine MUST encode the value to bytes using + For each column value, the engine must encode the value to bytes using sha-256-global's input rules, prepend the per-query salt, and compute SHA-256 over the result. From f7cac5dbc4dfc73c41f37ddcd63d5db4233d5d0b Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Sun, 14 Jun 2026 15:54:36 -0700 Subject: [PATCH 28/51] Spec: Restructure ReadRestrictions per rdblue review - Move client-support, response-scope, and principal-scoping prose to loadTable.description (L3640, L3645, L3656). - Delete the reader/engine glossary paragraph; engine is self-defined in sha-256-query-local (L3651). - Move the type-applicability rule into required-column-projections (L3661). - Replace the struct-subfield rule with an outermost-wins precedence rule that covers struct, list, and map containers (L3665, L3670). - Replace the inline JSON example with a yaml example: block (L3672). - Move the type invariant, evaluation order, and reader failure semantics up to the parent description (L3721, L3736, L3740). --- open-api/rest-catalog-open-api.py | 31 ++------- open-api/rest-catalog-open-api.yaml | 101 ++++++++++++---------------- 2 files changed, 48 insertions(+), 84 deletions(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index 196c980ddb6b..d21c56841ce4 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -1695,29 +1695,10 @@ class AddSchemaUpdate(BaseUpdate): class ReadRestrictions(BaseModel): """ - Read restrictions for a table, including column projections and row filter expressions. - A client should support the read-restrictions field. If a client supports read-restrictions, it must fail if it cannot apply any returned restriction (including unrecognized action or expression types). Read restrictions returned in a loadTable response apply to every read operation on the loaded table performed using this response, including subsequent planTableScan and fetchScanTasks calls. - In this section, "reader" refers to the read-side actor that applies restrictions per row or per column. "Engine" refers to the broader query-execution context that defines query lifetime and scope (e.g. a SQL session, a single PyIceberg scan), and is the actor responsible for query-scoped behavior such as salt generation in sha-256-query-local. - These restrictions apply only to the authenticated principal, user, or account associated with the request. They must not be interpreted as global policy and must not be applied beyond the entity identified by the Authentication header (or other applicable authentication mechanism). - An empty ReadRestrictions object (no required-column-projections and no required-row-filter) imposes no restrictions and is equivalent to the field being absent from the response. A server must not return an action for a column whose type is not listed in that action's "Applicable to" set. - NULL handling is action-specific. Each action's description specifies its behavior on NULL input. - If a column projection targets a struct-typed field, other column projections in the same ReadRestrictions must not target any of that struct's subfields (at any depth). This avoids ambiguity about which action governs a given leaf value. - Example: - - { - "required-column-projections": [ - { "field-id": 4, "action": "show-last-4" }, - { "field-id": 6, "action": "replace-with-null" }, - { "field-id": 8, "action": "truncate-to-year" }, - { "field-id": 10, "action": "sha-256-global" }, - { "field-id": 12, "action": "mask-alphanum" } - ], - "required-row-filter": { - "type": "eq", - "term": "region", - "value": "US" - } - } + Read restrictions for a table. + A reader evaluates the row filter against original, untransformed column values, then applies required-column-projections to the surviving rows. Each action must produce a value of the same type as the input column. If a reader cannot apply any returned restriction (a filter expression or an action), it must fail the query and must not silently return raw, partial, or empty results. + If a projection targets a field, that action governs every value reachable through it; other projections in the same object that target a descendant of that field have no effect. + An empty ReadRestrictions object (no required-column-projections and no required-row-filter) imposes no restrictions and is equivalent to the field being absent from the response. """ @@ -1737,12 +1718,12 @@ class ReadRestrictions(BaseModel): ) = Field( None, alias='required-column-projections', - description="A list of columns that require specific actions to be applied when reading.\nIf this property is present, each listed column must have its specified action applied. Columns not listed in required-column-projections are not subject to any read restrictions.\nWhen this list is present:\n1. For each column listed in required-column-projections, the reader must apply\n the specified action before returning values for that column.\n\n2. The reader must replace all output references to the column with the result\n of the action, presenting the result under the original column name. For\n example, if the action for column cc is mask-alphanum, the reader must\n return the masked value as cc in the query output.\n\n3. A column must appear at most once in required-column-projections.\n4. If a projected column's action cannot be evaluated by the reader (including\n unrecognized action types), the reader must fail the query with an error to\n the caller. The reader must not silently return raw, partial, or empty\n results to mask the failure.\n\n5. Each action defines the output type for its column. For all predefined\n actions, the output type matches the input column type.\n", + description='A list of columns that require specific actions to be applied when reading. A server must not return an action for a column whose type is not listed in that action\'s "Applicable to" set.\nIf this property is present, each listed column must have its specified action applied. Columns not listed in required-column-projections are not subject to any read restrictions.\nWhen this list is present:\n1. For each column listed in required-column-projections, the reader must apply\n the specified action before returning values for that column.\n\n2. The reader must replace all output references to the column with the result\n of the action, presenting the result under the original column name. For\n example, if the action for column cc is mask-alphanum, the reader must\n return the masked value as cc in the query output.\n\n3. A column must appear at most once in required-column-projections.\n4. If a projected column\'s action cannot be evaluated by the reader (including\n unrecognized action types), the reader must fail the query with an error to\n the caller. The reader must not silently return raw, partial, or empty\n results to mask the failure.\n', ) required_row_filter: Expression | None = Field( None, alias='required-row-filter', - description='An expression that filters rows in the table that the authenticated principal does not have access to.\n1. The expression must evaluate to a boolean (TRUE or FALSE; Iceberg expressions\n never produce NULL). A reader must discard any row for which\n the filter evaluates to FALSE, and no information derived from discarded rows\n may be included in the query result.\n\n2. Row filters must be evaluated against the original, untransformed column values.\n Required projections must be applied only after row filters are applied.\n\n3. If a reader cannot interpret or evaluate a provided filter expression, it must\n fail the query with an error to the caller. The reader must not silently return\n partial, raw, or empty results to mask the failure.\n\n4. If this property is absent, null, or always true then no mandatory filtering is required.\n', + description='An expression that filters rows in the table that the authenticated principal does not have access to.\n1. The expression must evaluate to a boolean (TRUE or FALSE; Iceberg expressions\n never produce NULL). A reader must discard any row for which\n the filter evaluates to FALSE, and no information derived from discarded rows\n may be included in the query result.\n\n2. If this property is absent, null, or always true then no mandatory filtering is required.\n', ) diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index 644bb71a8445..ecef522de382 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -1051,6 +1051,15 @@ paths: table. The configuration key "token" is used to pass an access token to be used as a bearer token for table requests. Otherwise, a token may be passed using a RFC 8693 token type as a configuration key. For example, "urn:ietf:params:oauth:token-type:jwt=". + + + The response may include a read-restrictions field. A client should support this field; + if it does, it must fail any read against the loaded table when it cannot apply a returned + restriction. This includes unrecognized action or expression types, and actions whose + definition cannot be loaded, parsed, or evaluated. These restrictions apply to every read + performed using this response (including subsequent planTableScan and fetchScanTasks + calls), only to the authenticated principal associated with the request, and must not be + interpreted as global policy. parameters: - $ref: '#/components/parameters/data-access' - name: If-None-Match @@ -3842,60 +3851,44 @@ components: ReadRestrictions: type: object description: > - Read restrictions for a table, including column projections and row filter expressions. - - A client should support the read-restrictions field. If a client supports - read-restrictions, it must fail if it cannot apply any returned restriction - (including unrecognized action or expression types). Read restrictions returned - in a loadTable response apply to every read operation on the loaded table - performed using this response, including subsequent planTableScan and - fetchScanTasks calls. - - In this section, "reader" refers to the read-side actor that applies restrictions - per row or per column. "Engine" refers to the broader query-execution context - that defines query lifetime and scope (e.g. a SQL session, a single PyIceberg - scan), and is the actor responsible for query-scoped behavior such as salt - generation in sha-256-query-local. - - These restrictions apply only to the authenticated principal, user, or account - associated with the request. They must not be interpreted as global policy and - must not be applied beyond the entity identified by the Authentication header - (or other applicable authentication mechanism). + Read restrictions for a table. + + A reader evaluates the row filter against original, untransformed column + values, then applies required-column-projections to the surviving rows. + Each action must produce a value of the same type as the input column. + If a reader cannot apply any returned restriction (a filter expression + or an action), it must fail the query and must not silently return raw, + partial, or empty results. + + If a projection targets a field, that action governs every value reachable + through it; other projections in the same object that target a descendant + of that field have no effect. An empty ReadRestrictions object (no required-column-projections and no - required-row-filter) imposes no restrictions and is equivalent to the field - being absent from the response. - A server must not return an action for a column whose type is not listed in - that action's "Applicable to" set. - - NULL handling is action-specific. Each action's description specifies its - behavior on NULL input. - - If a column projection targets a struct-typed field, other column projections - in the same ReadRestrictions must not target any of that struct's subfields - (at any depth). This avoids ambiguity about which action governs a given - leaf value. - - Example: - - { - "required-column-projections": [ - { "field-id": 4, "action": "show-last-4" }, - { "field-id": 6, "action": "replace-with-null" }, - { "field-id": 8, "action": "truncate-to-year" }, - { "field-id": 10, "action": "sha-256-global" }, - { "field-id": 12, "action": "mask-alphanum" } - ], - "required-row-filter": { - "type": "eq", - "term": "region", - "value": "US" - } - } + required-row-filter) imposes no restrictions and is equivalent to the + field being absent from the response. + example: + required-column-projections: + - field-id: 4 + action: show-last-4 + - field-id: 6 + action: replace-with-null + - field-id: 8 + action: truncate-to-year + - field-id: 10 + action: sha-256-global + - field-id: 12 + action: mask-alphanum + required-row-filter: + type: eq + term: region + value: US properties: required-column-projections: description: > A list of columns that require specific actions to be applied when reading. + A server must not return an action for a column whose type is not listed in + that action's "Applicable to" set. If this property is present, each listed column must have its specified action applied. Columns not listed in required-column-projections @@ -3918,9 +3911,6 @@ components: the caller. The reader must not silently return raw, partial, or empty results to mask the failure. - 5. Each action defines the output type for its column. For all predefined - actions, the output type matches the input column type. - type: array items: $ref: '#/components/schemas/Action' @@ -3933,14 +3923,7 @@ components: the filter evaluates to FALSE, and no information derived from discarded rows may be included in the query result. - 2. Row filters must be evaluated against the original, untransformed column values. - Required projections must be applied only after row filters are applied. - - 3. If a reader cannot interpret or evaluate a provided filter expression, it must - fail the query with an error to the caller. The reader must not silently return - partial, raw, or empty results to mask the failure. - - 4. If this property is absent, null, or always true then no mandatory filtering is required. + 2. If this property is absent, null, or always true then no mandatory filtering is required. allOf: - $ref: '#/components/schemas/Expression' From 024c511251d1d16d3e0f1c017c02085e82fb59c5 Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Sun, 14 Jun 2026 16:01:59 -0700 Subject: [PATCH 29/51] Spec: Rewrite ReadRestrictions property descriptions per rdblue review - required-column-projections: collapse the numbered rule list into declarative prose; consolidate absence semantics into the lead; drop rule 4 (failure semantics now stated once on the parent description); quote 'cc' in the example sentence (L3698, L3700, L3707, L3716). - required-row-filter: collapse to a single declarative paragraph parallel with required-column-projections. The "Iceberg expressions never produce NULL" phrase from L3731 is preserved. --- open-api/rest-catalog-open-api.py | 4 +-- open-api/rest-catalog-open-api.yaml | 45 +++++++++-------------------- 2 files changed, 16 insertions(+), 33 deletions(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index d21c56841ce4..7ba543eb7e2a 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -1718,12 +1718,12 @@ class ReadRestrictions(BaseModel): ) = Field( None, alias='required-column-projections', - description='A list of columns that require specific actions to be applied when reading. A server must not return an action for a column whose type is not listed in that action\'s "Applicable to" set.\nIf this property is present, each listed column must have its specified action applied. Columns not listed in required-column-projections are not subject to any read restrictions.\nWhen this list is present:\n1. For each column listed in required-column-projections, the reader must apply\n the specified action before returning values for that column.\n\n2. The reader must replace all output references to the column with the result\n of the action, presenting the result under the original column name. For\n example, if the action for column cc is mask-alphanum, the reader must\n return the masked value as cc in the query output.\n\n3. A column must appear at most once in required-column-projections.\n4. If a projected column\'s action cannot be evaluated by the reader (including\n unrecognized action types), the reader must fail the query with an error to\n the caller. The reader must not silently return raw, partial, or empty\n results to mask the failure.\n', + description='A list of columns that require specific actions to be applied when reading. A server must not return an action for a column whose type is not listed in that action\'s "Applicable to" set. If absent or empty, no required actions apply.\nEach entry binds an action to a column by field-id. The reader applies the entry\'s action to that column and presents the result under the original column name. For example, if the action for column \'cc\' is mask-alphanum, the reader returns the masked value as \'cc\' in the query output. Columns not listed are read as-is. A field-id must appear at most once.\n', ) required_row_filter: Expression | None = Field( None, alias='required-row-filter', - description='An expression that filters rows in the table that the authenticated principal does not have access to.\n1. The expression must evaluate to a boolean (TRUE or FALSE; Iceberg expressions\n never produce NULL). A reader must discard any row for which\n the filter evaluates to FALSE, and no information derived from discarded rows\n may be included in the query result.\n\n2. If this property is absent, null, or always true then no mandatory filtering is required.\n', + description='An expression that filters out rows the authenticated principal does not have access to. The expression must evaluate to a boolean (TRUE or FALSE; Iceberg expressions never produce NULL). A reader must discard any row for which the filter evaluates to FALSE, and no information derived from discarded rows may be included in the query result. If absent, null, or always true, no mandatory filtering applies.\n', ) diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index ecef522de382..e4fae1537e70 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3888,42 +3888,25 @@ components: description: > A list of columns that require specific actions to be applied when reading. A server must not return an action for a column whose type is not listed in - that action's "Applicable to" set. - - If this property is present, each listed column must have its specified - action applied. Columns not listed in required-column-projections - are not subject to any read restrictions. - - When this list is present: - - 1. For each column listed in required-column-projections, the reader must apply - the specified action before returning values for that column. - - 2. The reader must replace all output references to the column with the result - of the action, presenting the result under the original column name. For - example, if the action for column cc is mask-alphanum, the reader must - return the masked value as cc in the query output. - - 3. A column must appear at most once in required-column-projections. - - 4. If a projected column's action cannot be evaluated by the reader (including - unrecognized action types), the reader must fail the query with an error to - the caller. The reader must not silently return raw, partial, or empty - results to mask the failure. - + that action's "Applicable to" set. If absent or empty, no required actions + apply. + + Each entry binds an action to a column by field-id. The reader applies the + entry's action to that column and presents the result under the original + column name. For example, if the action for column 'cc' is mask-alphanum, + the reader returns the masked value as 'cc' in the query output. Columns + not listed are read as-is. A field-id must appear at most once. type: array items: $ref: '#/components/schemas/Action' required-row-filter: description: > - An expression that filters rows in the table that the authenticated principal does not have access to. - - 1. The expression must evaluate to a boolean (TRUE or FALSE; Iceberg expressions - never produce NULL). A reader must discard any row for which - the filter evaluates to FALSE, and no information derived from discarded rows - may be included in the query result. - - 2. If this property is absent, null, or always true then no mandatory filtering is required. + An expression that filters out rows the authenticated principal does not have + access to. The expression must evaluate to a boolean (TRUE or FALSE; Iceberg + expressions never produce NULL). A reader must discard any row for which the + filter evaluates to FALSE, and no information derived from discarded rows may + be included in the query result. If absent, null, or always true, no mandatory + filtering applies. allOf: - $ref: '#/components/schemas/Expression' From b7f5279aec28201fa0f4bc4d92ad2e6bbded3ade Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Sun, 14 Jun 2026 16:14:40 -0700 Subject: [PATCH 30/51] Spec: Restore numbered rule lists in ReadRestrictions properties Commit 35c8cba08 flattened the numbered rules in required-column-projections and required-row-filter into prose. That restructuring was not asked for by rdblue; only the wording fixes were. Restore the numbered lists while preserving the wording fixes: - required-column-projections: lead consolidates absence and not-listed semantics; rules 1-3 retained; quote 'cc' in rule 2; rule 4 (failure semantics) deleted because the parent description already covers it for both filters and actions (L3698, L3700, L3707, L3716). - required-row-filter: 2 numbered rules retained; "Iceberg expressions never produce NULL" preserved in rule 1 (L3731). --- open-api/rest-catalog-open-api.py | 4 ++-- open-api/rest-catalog-open-api.yaml | 30 +++++++++++++++++------------ 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index 7ba543eb7e2a..48ebc32a77a4 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -1718,12 +1718,12 @@ class ReadRestrictions(BaseModel): ) = Field( None, alias='required-column-projections', - description='A list of columns that require specific actions to be applied when reading. A server must not return an action for a column whose type is not listed in that action\'s "Applicable to" set. If absent or empty, no required actions apply.\nEach entry binds an action to a column by field-id. The reader applies the entry\'s action to that column and presents the result under the original column name. For example, if the action for column \'cc\' is mask-alphanum, the reader returns the masked value as \'cc\' in the query output. Columns not listed are read as-is. A field-id must appear at most once.\n', + description='A list of columns that require specific actions to be applied when reading. A server must not return an action for a column whose type is not listed in that action\'s "Applicable to" set. If absent or empty, no required actions apply; columns not listed are not subject to any required action.\n1. For each column listed, the reader must apply the specified action before\n returning values for that column.\n\n2. The reader must replace all output references to the column with the result\n of the action, presenting the result under the original column name. For\n example, if the action for column \'cc\' is mask-alphanum, the reader must\n return the masked value as \'cc\' in the query output.\n\n3. A column must appear at most once in required-column-projections.\n', ) required_row_filter: Expression | None = Field( None, alias='required-row-filter', - description='An expression that filters out rows the authenticated principal does not have access to. The expression must evaluate to a boolean (TRUE or FALSE; Iceberg expressions never produce NULL). A reader must discard any row for which the filter evaluates to FALSE, and no information derived from discarded rows may be included in the query result. If absent, null, or always true, no mandatory filtering applies.\n', + description='An expression that filters rows in the table that the authenticated principal does not have access to.\n1. The expression must evaluate to a boolean (TRUE or FALSE; Iceberg expressions\n never produce NULL). A reader must discard any row for which the filter\n evaluates to FALSE, and no information derived from discarded rows may be\n included in the query result.\n\n2. If this property is absent, null, or always true then no mandatory filtering is required.\n', ) diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index e4fae1537e70..b32932eece95 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3889,24 +3889,30 @@ components: A list of columns that require specific actions to be applied when reading. A server must not return an action for a column whose type is not listed in that action's "Applicable to" set. If absent or empty, no required actions - apply. + apply; columns not listed are not subject to any required action. - Each entry binds an action to a column by field-id. The reader applies the - entry's action to that column and presents the result under the original - column name. For example, if the action for column 'cc' is mask-alphanum, - the reader returns the masked value as 'cc' in the query output. Columns - not listed are read as-is. A field-id must appear at most once. + 1. For each column listed, the reader must apply the specified action before + returning values for that column. + + 2. The reader must replace all output references to the column with the result + of the action, presenting the result under the original column name. For + example, if the action for column 'cc' is mask-alphanum, the reader must + return the masked value as 'cc' in the query output. + + 3. A column must appear at most once in required-column-projections. type: array items: $ref: '#/components/schemas/Action' required-row-filter: description: > - An expression that filters out rows the authenticated principal does not have - access to. The expression must evaluate to a boolean (TRUE or FALSE; Iceberg - expressions never produce NULL). A reader must discard any row for which the - filter evaluates to FALSE, and no information derived from discarded rows may - be included in the query result. If absent, null, or always true, no mandatory - filtering applies. + An expression that filters rows in the table that the authenticated principal does not have access to. + + 1. The expression must evaluate to a boolean (TRUE or FALSE; Iceberg expressions + never produce NULL). A reader must discard any row for which the filter + evaluates to FALSE, and no information derived from discarded rows may be + included in the query result. + + 2. If this property is absent, null, or always true then no mandatory filtering is required. allOf: - $ref: '#/components/schemas/Expression' From 30f9f4881a02c2fd72b594c8063103e764999b2a Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Sun, 14 Jun 2026 16:49:05 -0700 Subject: [PATCH 31/51] Spec: Per-action wording fixes per rdblue review - MaskAlphanum: rephrase lead so the rules transform code points rather than redacting "code point by code point" (L3772). - MaskToFixedValue: drop redundant "predefined" from lead (L3799); state decimal fixed value as "the unscaled value is 0" (L3809). - ReplaceWithNull: write the negation as "required (non-nullable)" and use "optional" instead of "nullable" in Applicable-to (L3840, L3842). - Replace 13 hard-to-type Unicode arrows with ASCII -> across action examples and NULL-input lines (L3891). - Sha256Global: drop NIST FIPS citation (L3924); simplify int and long output spec by dropping "signed two's complement" (L3935). - Sha256QueryLocal: state that the definition of a query is left to the implementation (L3952). L3825 (nested types) resolved without spec change. L3821 (geometry well-definedness) and L3931 (fixed input support) held for follow-up. --- open-api/rest-catalog-open-api.py | 43 +++++++++++++-------------- open-api/rest-catalog-open-api.yaml | 45 +++++++++++++++-------------- 2 files changed, 45 insertions(+), 43 deletions(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index 48ebc32a77a4..d9724103b5f9 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -603,7 +603,7 @@ class Action(BaseModel): class MaskAlphanum(Action): """ - Redacts the column value Unicode code point by code point using the following rules: + Redacts the column value using the following rules to transform Unicode code points: - Digits (U+0030–U+0039, 0-9) are replaced with 'n' - The following punctuation characters are kept as-is: U+0028 '(' LEFT PARENTHESIS U+0029 ')' RIGHT PARENTHESIS @@ -614,8 +614,8 @@ class MaskAlphanum(Action): - All other Unicode characters (including letters, whitespace, and any punctuation not listed above) are replaced with 'x' - For example: "prashant010696@gmail.com" → "xxxxxxxxnnnnnn@xxxxx.xxx" - NULL input is preserved (NULL → NULL). + For example: "prashant010696@gmail.com" -> "xxxxxxxxnnnnnn@xxxxx.xxx" + NULL input is preserved (NULL -> NULL). Applicable to: string """ @@ -625,8 +625,8 @@ class MaskAlphanum(Action): class MaskToFixedValue(Action): """ - Replaces the column value with a predefined type-specific fixed value. Readers must use exactly the values listed below to ensure consistency across implementations. - Fixed values by type: - boolean: false - int: 0 - long: 0 - float: 0.0 - double: 0.0 - decimal(p, s): 0 (zero with s digits after the decimal point, e.g. 0.00 for decimal(p,2)) - string: "XXXXXXXX" - date: 1970-01-01 - time: 00:00:00 - timestamp: 1970-01-01T00:00:00 - timestamptz: 1970-01-01T00:00:00+00:00 - timestamp_ns: 1970-01-01T00:00:00.000000000 - timestamptz_ns: 1970-01-01T00:00:00.000000000+00:00 - uuid: 00000000-0000-0000-0000-000000000000 - fixed(n): n zero bytes - binary: empty byte sequence - variant: {} - geometry: POINT EMPTY - geography: POINT EMPTY - list: empty list [] - map: empty map {} - struct: struct with each field set to its type-specific default (applied recursively) + Replaces the column value with a type-specific fixed value. Readers must use exactly the values listed below to ensure consistency across implementations. + Fixed values by type: - boolean: false - int: 0 - long: 0 - float: 0.0 - double: 0.0 - decimal(p, s): 0 (the unscaled value is 0) - string: "XXXXXXXX" - date: 1970-01-01 - time: 00:00:00 - timestamp: 1970-01-01T00:00:00 - timestamptz: 1970-01-01T00:00:00+00:00 - timestamp_ns: 1970-01-01T00:00:00.000000000 - timestamptz_ns: 1970-01-01T00:00:00.000000000+00:00 - uuid: 00000000-0000-0000-0000-000000000000 - fixed(n): n zero bytes - binary: empty byte sequence - variant: {} - geometry: POINT EMPTY - geography: POINT EMPTY - list: empty list [] - map: empty map {} - struct: struct with each field set to its type-specific default (applied recursively) NULL input is also replaced with the type-specific fixed value; NULL is not preserved. Applicable to: all data types @@ -637,8 +637,8 @@ class MaskToFixedValue(Action): class ReplaceWithNull(Action): """ - Replaces the entire column value with NULL. NULL input is preserved (NULL → NULL). A server must not return this action for a non-nullable (required) column. - Applicable to: all nullable types + Replaces the entire column value with NULL. NULL input is preserved (NULL -> NULL). A server must not return this action for a required (non-nullable) column. + Applicable to: all optional types """ @@ -648,8 +648,8 @@ class ReplaceWithNull(Action): class ShowFirst4(Action): """ Preserves the first 4 Unicode code points of the column value and redacts the remainder using mask-alphanum rules (see MaskAlphanum for the exact character rules). Values with 4 or fewer Unicode code points are returned unchanged. - For example: "prashant010696@gmail.com" → "prasxxxxnnnnnn@xxxxx.xxx" - NULL input is preserved (NULL → NULL). + For example: "prashant010696@gmail.com" -> "prasxxxxnnnnnn@xxxxx.xxx" + NULL input is preserved (NULL -> NULL). Applicable to: string """ @@ -660,8 +660,8 @@ class ShowFirst4(Action): class ShowLast4(Action): """ Redacts all Unicode code points except the last 4 using mask-alphanum rules (see MaskAlphanum for the exact character rules). Values with 4 or fewer Unicode code points are returned unchanged. - For example: "4111-1111-1111-4444" → "nnnn-nnnn-nnnn-4444" - NULL input is preserved (NULL → NULL). + For example: "4111-1111-1111-4444" -> "nnnn-nnnn-nnnn-4444" + NULL input is preserved (NULL -> NULL). Applicable to: string """ @@ -672,8 +672,8 @@ class ShowLast4(Action): class TruncateToYear(Action): """ Truncates the column value to year precision, setting month, day, and time components to their minimum values. The output type matches the input type. - For example: 2024-07-15 → 2024-01-01 For timestamptz and timestamptz_ns, truncation is performed in UTC. - NULL input is preserved (NULL → NULL). + For example: 2024-07-15 -> 2024-01-01 For timestamptz and timestamptz_ns, truncation is performed in UTC. + NULL input is preserved (NULL -> NULL). Applicable to: date, timestamp, timestamptz, timestamp_ns, timestamptz_ns """ @@ -684,8 +684,8 @@ class TruncateToYear(Action): class TruncateToMonth(Action): """ Truncates the column value to year and month precision, setting day and time components to their minimum values. The output type matches the input type. - For example: 2024-07-15 → 2024-07-01 For timestamptz and timestamptz_ns, truncation is performed in UTC. - NULL input is preserved (NULL → NULL). + For example: 2024-07-15 -> 2024-07-01 For timestamptz and timestamptz_ns, truncation is performed in UTC. + NULL input is preserved (NULL -> NULL). Applicable to: date, timestamp, timestamptz, timestamp_ns, timestamptz_ns """ @@ -695,7 +695,7 @@ class TruncateToMonth(Action): class Sha256Global(Action): """ - Applies SHA-256 as specified in NIST FIPS 180-4. Deterministic across all queries + Applies SHA-256. Deterministic across all queries and engines — the same input always produces the same output. Input-to-bytes encoding by type: @@ -706,11 +706,11 @@ class Sha256Global(Action): Output encoding by type: - string: 64-character lowercase hexadecimal string - - int: first 4 bytes of the digest, read as a signed two's complement little-endian int - - long: first 8 bytes of the digest, read as a signed two's complement little-endian long + - int: first 4 bytes of the digest, read as a little-endian int + - long: first 8 bytes of the digest, read as a little-endian long - binary: the full 32-byte raw SHA-256 digest - NULL input is preserved (NULL → NULL). + NULL input is preserved (NULL -> NULL). Applicable to: string, int, long, binary @@ -722,7 +722,8 @@ class Sha256Global(Action): class Sha256QueryLocal(Action): """ Applies SHA-256 with a per-query random salt, making the output non-deterministic - across queries while remaining consistent within a single query. + across queries while remaining consistent within a single query. The definition + of a query is left to the implementation. The engine must generate a cryptographically random salt of at least 16 bytes for each query. @@ -732,7 +733,7 @@ class Sha256QueryLocal(Action): Output encoding follows the same rules as sha-256-global. - NULL input is preserved (NULL → NULL). + NULL input is preserved (NULL -> NULL). Applicable to: string, int, long, binary diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index b32932eece95..63d5e75f512f 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3942,7 +3942,7 @@ components: MaskAlphanum: description: > - Redacts the column value Unicode code point by code point using the following rules: + Redacts the column value using the following rules to transform Unicode code points: - Digits (U+0030–U+0039, 0-9) are replaced with 'n' - The following punctuation characters are kept as-is: @@ -3955,9 +3955,9 @@ components: - All other Unicode characters (including letters, whitespace, and any punctuation not listed above) are replaced with 'x' - For example: "prashant010696@gmail.com" → "xxxxxxxxnnnnnn@xxxxx.xxx" + For example: "prashant010696@gmail.com" -> "xxxxxxxxnnnnnn@xxxxx.xxx" - NULL input is preserved (NULL → NULL). + NULL input is preserved (NULL -> NULL). Applicable to: string allOf: @@ -3969,7 +3969,7 @@ components: MaskToFixedValue: description: > - Replaces the column value with a predefined type-specific fixed value. + Replaces the column value with a type-specific fixed value. Readers must use exactly the values listed below to ensure consistency across implementations. @@ -3979,7 +3979,7 @@ components: - long: 0 - float: 0.0 - double: 0.0 - - decimal(p, s): 0 (zero with s digits after the decimal point, e.g. 0.00 for decimal(p,2)) + - decimal(p, s): 0 (the unscaled value is 0) - string: "XXXXXXXX" - date: 1970-01-01 - time: 00:00:00 @@ -4009,10 +4009,10 @@ components: ReplaceWithNull: description: > - Replaces the entire column value with NULL. NULL input is preserved (NULL → NULL). - A server must not return this action for a non-nullable (required) column. + Replaces the entire column value with NULL. NULL input is preserved (NULL -> NULL). + A server must not return this action for a required (non-nullable) column. - Applicable to: all nullable types + Applicable to: all optional types allOf: - $ref: '#/components/schemas/Action' properties: @@ -4026,9 +4026,9 @@ components: using mask-alphanum rules (see MaskAlphanum for the exact character rules). Values with 4 or fewer Unicode code points are returned unchanged. - For example: "prashant010696@gmail.com" → "prasxxxxnnnnnn@xxxxx.xxx" + For example: "prashant010696@gmail.com" -> "prasxxxxnnnnnn@xxxxx.xxx" - NULL input is preserved (NULL → NULL). + NULL input is preserved (NULL -> NULL). Applicable to: string allOf: @@ -4044,9 +4044,9 @@ components: (see MaskAlphanum for the exact character rules). Values with 4 or fewer Unicode code points are returned unchanged. - For example: "4111-1111-1111-4444" → "nnnn-nnnn-nnnn-4444" + For example: "4111-1111-1111-4444" -> "nnnn-nnnn-nnnn-4444" - NULL input is preserved (NULL → NULL). + NULL input is preserved (NULL -> NULL). Applicable to: string allOf: @@ -4061,10 +4061,10 @@ components: Truncates the column value to year precision, setting month, day, and time components to their minimum values. The output type matches the input type. - For example: 2024-07-15 → 2024-01-01 + For example: 2024-07-15 -> 2024-01-01 For timestamptz and timestamptz_ns, truncation is performed in UTC. - NULL input is preserved (NULL → NULL). + NULL input is preserved (NULL -> NULL). Applicable to: date, timestamp, timestamptz, timestamp_ns, timestamptz_ns allOf: @@ -4079,10 +4079,10 @@ components: Truncates the column value to year and month precision, setting day and time components to their minimum values. The output type matches the input type. - For example: 2024-07-15 → 2024-07-01 + For example: 2024-07-15 -> 2024-07-01 For timestamptz and timestamptz_ns, truncation is performed in UTC. - NULL input is preserved (NULL → NULL). + NULL input is preserved (NULL -> NULL). Applicable to: date, timestamp, timestamptz, timestamp_ns, timestamptz_ns allOf: @@ -4094,7 +4094,7 @@ components: Sha256Global: description: | - Applies SHA-256 as specified in NIST FIPS 180-4. Deterministic across all queries + Applies SHA-256. Deterministic across all queries and engines — the same input always produces the same output. Input-to-bytes encoding by type: @@ -4105,11 +4105,11 @@ components: Output encoding by type: - string: 64-character lowercase hexadecimal string - - int: first 4 bytes of the digest, read as a signed two's complement little-endian int - - long: first 8 bytes of the digest, read as a signed two's complement little-endian long + - int: first 4 bytes of the digest, read as a little-endian int + - long: first 8 bytes of the digest, read as a little-endian long - binary: the full 32-byte raw SHA-256 digest - NULL input is preserved (NULL → NULL). + NULL input is preserved (NULL -> NULL). Applicable to: string, int, long, binary allOf: @@ -4122,7 +4122,8 @@ components: Sha256QueryLocal: description: | Applies SHA-256 with a per-query random salt, making the output non-deterministic - across queries while remaining consistent within a single query. + across queries while remaining consistent within a single query. The definition + of a query is left to the implementation. The engine must generate a cryptographically random salt of at least 16 bytes for each query. @@ -4132,7 +4133,7 @@ components: Output encoding follows the same rules as sha-256-global. - NULL input is preserved (NULL → NULL). + NULL input is preserved (NULL -> NULL). Applicable to: string, int, long, binary allOf: From 98d7f8f3a68a464a4f58e5ed5674cb00ce61d0a4 Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Mon, 22 Jun 2026 16:56:37 -0700 Subject: [PATCH 32/51] Spec: Replace em-dash with colon in Sha256Global description --- open-api/rest-catalog-open-api.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index 63d5e75f512f..e5fc07300d68 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -4095,7 +4095,7 @@ components: Sha256Global: description: | Applies SHA-256. Deterministic across all queries - and engines — the same input always produces the same output. + and engines: the same input always produces the same output. Input-to-bytes encoding by type: - string: UTF-8 encoded bytes From 7cae2623bd8af4141f586dfb7ecd8fabf6271f10 Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Mon, 22 Jun 2026 18:13:58 -0700 Subject: [PATCH 33/51] Spec: Regenerate rest-catalog-open-api.py --- open-api/rest-catalog-open-api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index d9724103b5f9..35448ba05800 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -696,7 +696,7 @@ class TruncateToMonth(Action): class Sha256Global(Action): """ Applies SHA-256. Deterministic across all queries - and engines — the same input always produces the same output. + and engines: the same input always produces the same output. Input-to-bytes encoding by type: - string: UTF-8 encoded bytes From 768a1c44213da16a2c004740c9cf5af34ec74835 Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Thu, 25 Jun 2026 18:31:46 -0700 Subject: [PATCH 34/51] Spec: Address review nits and restore nested-projection prohibition - Sha256Global / Sha256QueryLocal: replace "engine" with "reader" for consistent normative actor terminology (Steven, Sung) - required-column-projections example: refer to field-id instead of column name to match the field-id-based targeting model (Sung) - ReadRestrictions: restore the explicit prohibition on overlapping projections in the same object, extended to all nested types (struct, list, map). The earlier "outermost takes precedence" rewrite implied inheritance semantics that were not intended and left list/map overlap unaddressed. --- open-api/rest-catalog-open-api.py | 10 +++++----- open-api/rest-catalog-open-api.yaml | 20 +++++++++++--------- 2 files changed, 16 insertions(+), 14 deletions(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index 35448ba05800..a6a34f15a2b2 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -696,7 +696,7 @@ class TruncateToMonth(Action): class Sha256Global(Action): """ Applies SHA-256. Deterministic across all queries - and engines: the same input always produces the same output. + and readers: the same input always produces the same output. Input-to-bytes encoding by type: - string: UTF-8 encoded bytes @@ -725,9 +725,9 @@ class Sha256QueryLocal(Action): across queries while remaining consistent within a single query. The definition of a query is left to the implementation. - The engine must generate a cryptographically random salt of at least 16 bytes for each query. + The reader must generate a cryptographically random salt of at least 16 bytes for each query. - For each column value, the engine must encode the value to bytes using + For each column value, the reader must encode the value to bytes using sha-256-global's input rules, prepend the per-query salt, and compute SHA-256 over the result. @@ -1698,7 +1698,7 @@ class ReadRestrictions(BaseModel): """ Read restrictions for a table. A reader evaluates the row filter against original, untransformed column values, then applies required-column-projections to the surviving rows. Each action must produce a value of the same type as the input column. If a reader cannot apply any returned restriction (a filter expression or an action), it must fail the query and must not silently return raw, partial, or empty results. - If a projection targets a field, that action governs every value reachable through it; other projections in the same object that target a descendant of that field have no effect. + If a projection targets a nested-typed field (struct, list, or map), other projections in the same ReadRestrictions must not target any descendant field-id (struct subfields, list elements, or map keys/values) at any depth. This avoids ambiguity about which action governs a given leaf value. An empty ReadRestrictions object (no required-column-projections and no required-row-filter) imposes no restrictions and is equivalent to the field being absent from the response. """ @@ -1719,7 +1719,7 @@ class ReadRestrictions(BaseModel): ) = Field( None, alias='required-column-projections', - description='A list of columns that require specific actions to be applied when reading. A server must not return an action for a column whose type is not listed in that action\'s "Applicable to" set. If absent or empty, no required actions apply; columns not listed are not subject to any required action.\n1. For each column listed, the reader must apply the specified action before\n returning values for that column.\n\n2. The reader must replace all output references to the column with the result\n of the action, presenting the result under the original column name. For\n example, if the action for column \'cc\' is mask-alphanum, the reader must\n return the masked value as \'cc\' in the query output.\n\n3. A column must appear at most once in required-column-projections.\n', + description='A list of columns that require specific actions to be applied when reading. A server must not return an action for a column whose type is not listed in that action\'s "Applicable to" set. If absent or empty, no required actions apply; columns not listed are not subject to any required action.\n1. For each column listed, the reader must apply the specified action before\n returning values for that column.\n\n2. The reader must replace all output references to the column with the result\n of the action, presenting the result under the original field-id. For\n example, if the action for field-id `9` is mask-alphanum, the reader must\n return the masked value as field-id `9` in the query output.\n\n3. A column must appear at most once in required-column-projections.\n', ) required_row_filter: Expression | None = Field( None, diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index e5fc07300d68..a2bef48a8d00 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3860,9 +3860,11 @@ components: or an action), it must fail the query and must not silently return raw, partial, or empty results. - If a projection targets a field, that action governs every value reachable - through it; other projections in the same object that target a descendant - of that field have no effect. + If a projection targets a nested-typed field (struct, list, or map), + other projections in the same ReadRestrictions must not target any + descendant field-id (struct subfields, list elements, or map keys/values) + at any depth. This avoids ambiguity about which action governs a given + leaf value. An empty ReadRestrictions object (no required-column-projections and no required-row-filter) imposes no restrictions and is equivalent to the @@ -3895,9 +3897,9 @@ components: returning values for that column. 2. The reader must replace all output references to the column with the result - of the action, presenting the result under the original column name. For - example, if the action for column 'cc' is mask-alphanum, the reader must - return the masked value as 'cc' in the query output. + of the action, presenting the result under the original field-id. For + example, if the action for field-id `9` is mask-alphanum, the reader must + return the masked value as field-id `9` in the query output. 3. A column must appear at most once in required-column-projections. type: array @@ -4095,7 +4097,7 @@ components: Sha256Global: description: | Applies SHA-256. Deterministic across all queries - and engines: the same input always produces the same output. + and readers: the same input always produces the same output. Input-to-bytes encoding by type: - string: UTF-8 encoded bytes @@ -4125,9 +4127,9 @@ components: across queries while remaining consistent within a single query. The definition of a query is left to the implementation. - The engine must generate a cryptographically random salt of at least 16 bytes for each query. + The reader must generate a cryptographically random salt of at least 16 bytes for each query. - For each column value, the engine must encode the value to bytes using + For each column value, the reader must encode the value to bytes using sha-256-global's input rules, prepend the per-query salt, and compute SHA-256 over the result. From 78a3a69117146a2cf24672dd66d709bbbe73c0d0 Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Mon, 29 Jun 2026 10:46:35 -0700 Subject: [PATCH 35/51] Spec: Forbid projections on map key field-ids Add rule 4 to required-column-projections: a projection must not target a map's key field-id. Applying an action to keys can produce duplicate or null keys, which readers silently coalesce (Parquet/Avro/ORC/Flink all back map values with java.util.Map.put) or reject (map keys are required per spec.md:241), causing data loss. Addresses Sung's review comment on PR #13879. --- open-api/rest-catalog-open-api.py | 2 +- open-api/rest-catalog-open-api.yaml | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index a6a34f15a2b2..9e65c5d24ae8 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -1719,7 +1719,7 @@ class ReadRestrictions(BaseModel): ) = Field( None, alias='required-column-projections', - description='A list of columns that require specific actions to be applied when reading. A server must not return an action for a column whose type is not listed in that action\'s "Applicable to" set. If absent or empty, no required actions apply; columns not listed are not subject to any required action.\n1. For each column listed, the reader must apply the specified action before\n returning values for that column.\n\n2. The reader must replace all output references to the column with the result\n of the action, presenting the result under the original field-id. For\n example, if the action for field-id `9` is mask-alphanum, the reader must\n return the masked value as field-id `9` in the query output.\n\n3. A column must appear at most once in required-column-projections.\n', + description='A list of columns that require specific actions to be applied when reading. A server must not return an action for a column whose type is not listed in that action\'s "Applicable to" set. If absent or empty, no required actions apply; columns not listed are not subject to any required action.\n1. For each column listed, the reader must apply the specified action before\n returning values for that column.\n\n2. The reader must replace all output references to the column with the result\n of the action, presenting the result under the original field-id. For\n example, if the action for field-id `9` is mask-alphanum, the reader must\n return the masked value as field-id `9` in the query output.\n\n3. A column must appear at most once in required-column-projections.\n\n4. A projection must not target a map\'s key field-id. Applying an action\n to keys can produce duplicate or null keys, which readers silently\n coalesce or reject, causing data loss.\n', ) required_row_filter: Expression | None = Field( None, diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index a2bef48a8d00..37c3c9223586 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3902,6 +3902,10 @@ components: return the masked value as field-id `9` in the query output. 3. A column must appear at most once in required-column-projections. + + 4. A projection must not target a map's key field-id. Applying an action + to keys can produce duplicate or null keys, which readers silently + coalesce or reject, causing data loss. type: array items: $ref: '#/components/schemas/Action' From b65d8fc16ea80e24898d513bb276ba4dd673bf7f Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Mon, 29 Jun 2026 10:59:14 -0700 Subject: [PATCH 36/51] Spec: Regenerate rest-catalog-open-api.py with datamodel-code-generator 0.64.1 Pulled in via rebase onto main. The newer generator emits Action discriminator fields as `Literal[X] | None = None` (vs the older 0.60.0 form `Literal[X] = 'x'`) and adds a Summary __pydantic_extra__ rebuild helper. --- open-api/rest-catalog-open-api.py | 90 ++++++++++++++++--------------- 1 file changed, 46 insertions(+), 44 deletions(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index 9e65c5d24ae8..54e4bf3aabd0 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -18,7 +18,7 @@ from __future__ import annotations from datetime import date, timedelta -from typing import Literal +from typing import Dict, Literal from uuid import UUID from pydantic import Base64Str, BaseModel, ConfigDict, Field, RootModel @@ -279,6 +279,10 @@ class Summary(BaseModel): operation: Literal['append', 'replace', 'overwrite', 'delete'] +Summary.__annotations__['__pydantic_extra__'] = Dict[str, str] +Summary.model_rebuild(force=True) + + class Snapshot(BaseModel): snapshot_id: int = Field(..., alias='snapshot-id') parent_snapshot_id: int | None = Field(None, alias='parent-snapshot-id') @@ -371,17 +375,17 @@ class AssignUUIDUpdate(BaseUpdate): Assigning a UUID to a table/view should only be done when creating the table/view. It is not safe to re-assign the UUID if a table/view already has a UUID assigned """ - action: Literal['assign-uuid'] = 'assign-uuid' + action: Literal['assign-uuid'] uuid: str class UpgradeFormatVersionUpdate(BaseUpdate): - action: Literal['upgrade-format-version'] = 'upgrade-format-version' + action: Literal['upgrade-format-version'] format_version: int = Field(..., alias='format-version') class SetCurrentSchemaUpdate(BaseUpdate): - action: Literal['set-current-schema'] = 'set-current-schema' + action: Literal['set-current-schema'] schema_id: int = Field( ..., alias='schema-id', @@ -390,12 +394,12 @@ class SetCurrentSchemaUpdate(BaseUpdate): class AddPartitionSpecUpdate(BaseUpdate): - action: Literal['add-spec'] = 'add-spec' + action: Literal['add-spec'] spec: PartitionSpec class SetDefaultSpecUpdate(BaseUpdate): - action: Literal['set-default-spec'] = 'set-default-spec' + action: Literal['set-default-spec'] spec_id: int = Field( ..., alias='spec-id', @@ -404,12 +408,12 @@ class SetDefaultSpecUpdate(BaseUpdate): class AddSortOrderUpdate(BaseUpdate): - action: Literal['add-sort-order'] = 'add-sort-order' + action: Literal['add-sort-order'] sort_order: SortOrder = Field(..., alias='sort-order') class SetDefaultSortOrderUpdate(BaseUpdate): - action: Literal['set-default-sort-order'] = 'set-default-sort-order' + action: Literal['set-default-sort-order'] sort_order_id: int = Field( ..., alias='sort-order-id', @@ -418,47 +422,47 @@ class SetDefaultSortOrderUpdate(BaseUpdate): class AddSnapshotUpdate(BaseUpdate): - action: Literal['add-snapshot'] = 'add-snapshot' + action: Literal['add-snapshot'] snapshot: Snapshot class SetSnapshotRefUpdate(BaseUpdate, SnapshotReference): - action: Literal['set-snapshot-ref'] = 'set-snapshot-ref' + action: Literal['set-snapshot-ref'] ref_name: str = Field(..., alias='ref-name') class RemoveSnapshotsUpdate(BaseUpdate): - action: Literal['remove-snapshots'] = 'remove-snapshots' + action: Literal['remove-snapshots'] snapshot_ids: list[int] = Field(..., alias='snapshot-ids') class RemoveSnapshotRefUpdate(BaseUpdate): - action: Literal['remove-snapshot-ref'] = 'remove-snapshot-ref' + action: Literal['remove-snapshot-ref'] ref_name: str = Field(..., alias='ref-name') class SetLocationUpdate(BaseUpdate): - action: Literal['set-location'] = 'set-location' + action: Literal['set-location'] location: str class SetPropertiesUpdate(BaseUpdate): - action: Literal['set-properties'] = 'set-properties' + action: Literal['set-properties'] updates: dict[str, str] class RemovePropertiesUpdate(BaseUpdate): - action: Literal['remove-properties'] = 'remove-properties' + action: Literal['remove-properties'] removals: list[str] class AddViewVersionUpdate(BaseUpdate): - action: Literal['add-view-version'] = 'add-view-version' + action: Literal['add-view-version'] view_version: ViewVersion = Field(..., alias='view-version') class SetCurrentViewVersionUpdate(BaseUpdate): - action: Literal['set-current-view-version'] = 'set-current-view-version' + action: Literal['set-current-view-version'] view_version_id: int = Field( ..., alias='view-version-id', @@ -467,32 +471,32 @@ class SetCurrentViewVersionUpdate(BaseUpdate): class RemoveStatisticsUpdate(BaseUpdate): - action: Literal['remove-statistics'] = 'remove-statistics' + action: Literal['remove-statistics'] snapshot_id: int = Field(..., alias='snapshot-id') class RemovePartitionStatisticsUpdate(BaseUpdate): - action: Literal['remove-partition-statistics'] = 'remove-partition-statistics' + action: Literal['remove-partition-statistics'] snapshot_id: int = Field(..., alias='snapshot-id') class RemovePartitionSpecsUpdate(BaseUpdate): - action: Literal['remove-partition-specs'] = 'remove-partition-specs' + action: Literal['remove-partition-specs'] spec_ids: list[int] = Field(..., alias='spec-ids') class RemoveSchemasUpdate(BaseUpdate): - action: Literal['remove-schemas'] = 'remove-schemas' + action: Literal['remove-schemas'] schema_ids: list[int] = Field(..., alias='schema-ids') class AddEncryptionKeyUpdate(BaseUpdate): - action: Literal['add-encryption-key'] = 'add-encryption-key' + action: Literal['add-encryption-key'] encryption_key: EncryptedKey = Field(..., alias='encryption-key') class RemoveEncryptionKeyUpdate(BaseUpdate): - action: Literal['remove-encryption-key'] = 'remove-encryption-key' + action: Literal['remove-encryption-key'] key_id: str = Field(..., alias='key-id') @@ -525,7 +529,7 @@ class AssertRefSnapshotId(TableRequirement): """ - type: Literal['assert-ref-snapshot-id'] = 'assert-ref-snapshot-id' + type: Literal['assert-ref-snapshot-id'] ref: str snapshot_id: int = Field(..., alias='snapshot-id') @@ -535,7 +539,7 @@ class AssertLastAssignedFieldId(TableRequirement): The table's last assigned column id must match the requirement's `last-assigned-field-id` """ - type: Literal['assert-last-assigned-field-id'] = 'assert-last-assigned-field-id' + type: Literal['assert-last-assigned-field-id'] last_assigned_field_id: int = Field(..., alias='last-assigned-field-id') @@ -544,7 +548,7 @@ class AssertCurrentSchemaId(TableRequirement): The table's current schema id must match the requirement's `current-schema-id` """ - type: Literal['assert-current-schema-id'] = 'assert-current-schema-id' + type: Literal['assert-current-schema-id'] current_schema_id: int = Field(..., alias='current-schema-id') @@ -553,9 +557,7 @@ class AssertLastAssignedPartitionId(TableRequirement): The table's last assigned partition id must match the requirement's `last-assigned-partition-id` """ - type: Literal['assert-last-assigned-partition-id'] = ( - 'assert-last-assigned-partition-id' - ) + type: Literal['assert-last-assigned-partition-id'] last_assigned_partition_id: int = Field(..., alias='last-assigned-partition-id') @@ -564,7 +566,7 @@ class AssertDefaultSpecId(TableRequirement): The table's default spec id must match the requirement's `default-spec-id` """ - type: Literal['assert-default-spec-id'] = 'assert-default-spec-id' + type: Literal['assert-default-spec-id'] default_spec_id: int = Field(..., alias='default-spec-id') @@ -573,7 +575,7 @@ class AssertDefaultSortOrderId(TableRequirement): The table's default sort order id must match the requirement's `default-sort-order-id` """ - type: Literal['assert-default-sort-order-id'] = 'assert-default-sort-order-id' + type: Literal['assert-default-sort-order-id'] default_sort_order_id: int = Field(..., alias='default-sort-order-id') @@ -620,7 +622,7 @@ class MaskAlphanum(Action): """ - action: Literal['mask-alphanum'] = 'mask-alphanum' + action: Literal['mask-alphanum'] | None = None class MaskToFixedValue(Action): @@ -632,7 +634,7 @@ class MaskToFixedValue(Action): """ - action: Literal['mask-to-fixed-value'] = 'mask-to-fixed-value' + action: Literal['mask-to-fixed-value'] | None = None class ReplaceWithNull(Action): @@ -642,7 +644,7 @@ class ReplaceWithNull(Action): """ - action: Literal['replace-with-null'] = 'replace-with-null' + action: Literal['replace-with-null'] | None = None class ShowFirst4(Action): @@ -654,7 +656,7 @@ class ShowFirst4(Action): """ - action: Literal['show-first-4'] = 'show-first-4' + action: Literal['show-first-4'] | None = None class ShowLast4(Action): @@ -666,7 +668,7 @@ class ShowLast4(Action): """ - action: Literal['show-last-4'] = 'show-last-4' + action: Literal['show-last-4'] | None = None class TruncateToYear(Action): @@ -678,7 +680,7 @@ class TruncateToYear(Action): """ - action: Literal['truncate-to-year'] = 'truncate-to-year' + action: Literal['truncate-to-year'] | None = None class TruncateToMonth(Action): @@ -690,7 +692,7 @@ class TruncateToMonth(Action): """ - action: Literal['truncate-to-month'] = 'truncate-to-month' + action: Literal['truncate-to-month'] | None = None class Sha256Global(Action): @@ -716,7 +718,7 @@ class Sha256Global(Action): """ - action: Literal['sha-256-global'] = 'sha-256-global' + action: Literal['sha-256-global'] | None = None class Sha256QueryLocal(Action): @@ -739,7 +741,7 @@ class Sha256QueryLocal(Action): """ - action: Literal['sha-256-query-local'] = 'sha-256-query-local' + action: Literal['sha-256-query-local'] | None = None class LoadCredentialsResponse(BaseModel): @@ -1310,7 +1312,7 @@ class TransformTerm(BaseModel): class SetPartitionStatisticsUpdate(BaseUpdate): - action: Literal['set-partition-statistics'] = 'set-partition-statistics' + action: Literal['set-partition-statistics'] partition_statistics: PartitionStatisticsFile = Field( ..., alias='partition-statistics' ) @@ -1420,7 +1422,7 @@ class Term(RootModel[Reference | TransformTerm]): class SetStatisticsUpdate(BaseUpdate): - action: Literal['set-statistics'] = 'set-statistics' + action: Literal['set-statistics'] snapshot_id: int | None = Field( None, alias='snapshot-id', @@ -1684,7 +1686,7 @@ class ViewMetadata(BaseModel): class AddSchemaUpdate(BaseUpdate): - action: Literal['add-schema'] = 'add-schema' + action: Literal['add-schema'] schema_: Schema = Field(..., alias='schema') last_column_id: int | None = Field( None, @@ -1719,7 +1721,7 @@ class ReadRestrictions(BaseModel): ) = Field( None, alias='required-column-projections', - description='A list of columns that require specific actions to be applied when reading. A server must not return an action for a column whose type is not listed in that action\'s "Applicable to" set. If absent or empty, no required actions apply; columns not listed are not subject to any required action.\n1. For each column listed, the reader must apply the specified action before\n returning values for that column.\n\n2. The reader must replace all output references to the column with the result\n of the action, presenting the result under the original field-id. For\n example, if the action for field-id `9` is mask-alphanum, the reader must\n return the masked value as field-id `9` in the query output.\n\n3. A column must appear at most once in required-column-projections.\n\n4. A projection must not target a map\'s key field-id. Applying an action\n to keys can produce duplicate or null keys, which readers silently\n coalesce or reject, causing data loss.\n', + description='A list of columns that require specific actions to be applied when reading. A server must not return an action for a column whose type is not listed in that action\'s "Applicable to" set. If absent or empty, no required actions apply; columns not listed are not subject to any required action.\n1. For each column listed, the reader must apply the specified action before\n returning values for that column.\n\n2. The reader must replace all output references to the column with the result\n of the action, presenting the result under the original field-id. For\n example, if the action for field-id `9` is mask-alphanum, the reader must\n return the masked value as field-id `9` in the query output.\n\n3. A column must appear at most once in required-column-projections.\n4. A projection must not target a map\'s key field-id. Applying an action\n to keys can produce duplicate or null keys, which readers silently\n coalesce or reject, causing data loss.\n', ) required_row_filter: Expression | None = Field( None, From e653bf1b75cbd194b4056f7a883621564cfdba2b Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Mon, 29 Jun 2026 11:26:38 -0700 Subject: [PATCH 37/51] Spec: Use 'Iceberg predicates' instead of 'Iceberg expressions' for NULL note Per rdblue review: predicates (not expressions in general) are what cannot produce NULL. Updates the required-row-filter description in both rest-catalog-open-api.yaml and the regenerated .py. --- open-api/rest-catalog-open-api.py | 2 +- open-api/rest-catalog-open-api.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index 54e4bf3aabd0..9ed70f2608e5 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -1726,7 +1726,7 @@ class ReadRestrictions(BaseModel): required_row_filter: Expression | None = Field( None, alias='required-row-filter', - description='An expression that filters rows in the table that the authenticated principal does not have access to.\n1. The expression must evaluate to a boolean (TRUE or FALSE; Iceberg expressions\n never produce NULL). A reader must discard any row for which the filter\n evaluates to FALSE, and no information derived from discarded rows may be\n included in the query result.\n\n2. If this property is absent, null, or always true then no mandatory filtering is required.\n', + description='An expression that filters rows in the table that the authenticated principal does not have access to.\n1. The expression must evaluate to a boolean (TRUE or FALSE; Iceberg predicates\n never produce NULL). A reader must discard any row for which the filter\n evaluates to FALSE, and no information derived from discarded rows may be\n included in the query result.\n\n2. If this property is absent, null, or always true then no mandatory filtering is required.\n', ) diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index 37c3c9223586..5dfcc32e6ab7 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3913,7 +3913,7 @@ components: description: > An expression that filters rows in the table that the authenticated principal does not have access to. - 1. The expression must evaluate to a boolean (TRUE or FALSE; Iceberg expressions + 1. The expression must evaluate to a boolean (TRUE or FALSE; Iceberg predicates never produce NULL). A reader must discard any row for which the filter evaluates to FALSE, and no information derived from discarded rows may be included in the query result. From 6e87ef1c5e0a0223cda901a9e2db602acf344eee Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Mon, 29 Jun 2026 13:04:36 -0700 Subject: [PATCH 38/51] Spec: Exclude geometry/geography from MaskToFixedValue Per rdblue review: POINT EMPTY's WKB encoding is not defined in Iceberg's format spec (Appendix G defers to OGC SFA, which does not specify empty-geometry WKB), and is not formally standardized in GeoParquet, PostGIS, Apache Parquet's geospatial spec, or any other source that Iceberg cites. The "POINT(NaN, NaN)" convention is widely-used implementation lore but never canonicalized. This also aligns with format/spec.md L333 which states that columns of unknown, variant, geometry, and geography types must default to null and explicitly disallows non-null initial-default / write-default values for those types. Drops: - geometry: POINT EMPTY - geography: POINT EMPTY Tightens Applicable-to clause to exclude unknown, geometry, geography. variant is kept in the fixed-value table because Iceberg's Java API defines a canonical empty Variant byte sequence (EMPTY_V1_BUFFER), giving an implementation-anchorable encoding that geometry/geography lack. --- open-api/rest-catalog-open-api.py | 4 ++-- open-api/rest-catalog-open-api.yaml | 4 +--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index 9ed70f2608e5..380c34895ec5 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -628,9 +628,9 @@ class MaskAlphanum(Action): class MaskToFixedValue(Action): """ Replaces the column value with a type-specific fixed value. Readers must use exactly the values listed below to ensure consistency across implementations. - Fixed values by type: - boolean: false - int: 0 - long: 0 - float: 0.0 - double: 0.0 - decimal(p, s): 0 (the unscaled value is 0) - string: "XXXXXXXX" - date: 1970-01-01 - time: 00:00:00 - timestamp: 1970-01-01T00:00:00 - timestamptz: 1970-01-01T00:00:00+00:00 - timestamp_ns: 1970-01-01T00:00:00.000000000 - timestamptz_ns: 1970-01-01T00:00:00.000000000+00:00 - uuid: 00000000-0000-0000-0000-000000000000 - fixed(n): n zero bytes - binary: empty byte sequence - variant: {} - geometry: POINT EMPTY - geography: POINT EMPTY - list: empty list [] - map: empty map {} - struct: struct with each field set to its type-specific default (applied recursively) + Fixed values by type: - boolean: false - int: 0 - long: 0 - float: 0.0 - double: 0.0 - decimal(p, s): 0 (the unscaled value is 0) - string: "XXXXXXXX" - date: 1970-01-01 - time: 00:00:00 - timestamp: 1970-01-01T00:00:00 - timestamptz: 1970-01-01T00:00:00+00:00 - timestamp_ns: 1970-01-01T00:00:00.000000000 - timestamptz_ns: 1970-01-01T00:00:00.000000000+00:00 - uuid: 00000000-0000-0000-0000-000000000000 - fixed(n): n zero bytes - binary: empty byte sequence - variant: {} - list: empty list [] - map: empty map {} - struct: struct with each field set to its type-specific default (applied recursively) NULL input is also replaced with the type-specific fixed value; NULL is not preserved. - Applicable to: all data types + Applicable to: all data types except unknown, geometry, and geography. """ diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index 5dfcc32e6ab7..9483fb991e85 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3997,15 +3997,13 @@ components: - fixed(n): n zero bytes - binary: empty byte sequence - variant: {} - - geometry: POINT EMPTY - - geography: POINT EMPTY - list: empty list [] - map: empty map {} - struct: struct with each field set to its type-specific default (applied recursively) NULL input is also replaced with the type-specific fixed value; NULL is not preserved. - Applicable to: all data types + Applicable to: all data types except unknown, geometry, and geography. allOf: - $ref: '#/components/schemas/Action' properties: From a3127396306565f9f4ffed91a54cf220c4a1db02 Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Wed, 1 Jul 2026 10:57:38 -0700 Subject: [PATCH 39/51] Spec: Address review comments on read restrictions wording - Use 'reader' consistently in loadTable read-restrictions prose - Rename 'descendant field-id' to 'nested field-id' to match spec terminology - State both actors in the duplicate field-id rule (server must not return, reader must fail) - Forbid mask-to-fixed-value on containers nesting unknown/geometry/geography Regenerate rest-catalog-open-api.py. --- open-api/rest-catalog-open-api.py | 6 +++--- open-api/rest-catalog-open-api.yaml | 19 ++++++++++++------- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index 380c34895ec5..05679e2d2b1a 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -630,7 +630,7 @@ class MaskToFixedValue(Action): Replaces the column value with a type-specific fixed value. Readers must use exactly the values listed below to ensure consistency across implementations. Fixed values by type: - boolean: false - int: 0 - long: 0 - float: 0.0 - double: 0.0 - decimal(p, s): 0 (the unscaled value is 0) - string: "XXXXXXXX" - date: 1970-01-01 - time: 00:00:00 - timestamp: 1970-01-01T00:00:00 - timestamptz: 1970-01-01T00:00:00+00:00 - timestamp_ns: 1970-01-01T00:00:00.000000000 - timestamptz_ns: 1970-01-01T00:00:00.000000000+00:00 - uuid: 00000000-0000-0000-0000-000000000000 - fixed(n): n zero bytes - binary: empty byte sequence - variant: {} - list: empty list [] - map: empty map {} - struct: struct with each field set to its type-specific default (applied recursively) NULL input is also replaced with the type-specific fixed value; NULL is not preserved. - Applicable to: all data types except unknown, geometry, and geography. + Applicable to: all data types except unknown, geometry, and geography. Because those three types have no fixed value defined above, a catalog server must not return mask-to-fixed-value for a struct, list, or map that contains a field of type unknown, geometry, or geography at any nesting depth. """ @@ -1700,7 +1700,7 @@ class ReadRestrictions(BaseModel): """ Read restrictions for a table. A reader evaluates the row filter against original, untransformed column values, then applies required-column-projections to the surviving rows. Each action must produce a value of the same type as the input column. If a reader cannot apply any returned restriction (a filter expression or an action), it must fail the query and must not silently return raw, partial, or empty results. - If a projection targets a nested-typed field (struct, list, or map), other projections in the same ReadRestrictions must not target any descendant field-id (struct subfields, list elements, or map keys/values) at any depth. This avoids ambiguity about which action governs a given leaf value. + If a projection targets a nested-typed field (struct, list, or map), other projections in the same ReadRestrictions must not target any nested field-id (struct subfields, list elements, or map keys/values) at any depth. This avoids ambiguity about which action governs a given leaf value. An empty ReadRestrictions object (no required-column-projections and no required-row-filter) imposes no restrictions and is equivalent to the field being absent from the response. """ @@ -1721,7 +1721,7 @@ class ReadRestrictions(BaseModel): ) = Field( None, alias='required-column-projections', - description='A list of columns that require specific actions to be applied when reading. A server must not return an action for a column whose type is not listed in that action\'s "Applicable to" set. If absent or empty, no required actions apply; columns not listed are not subject to any required action.\n1. For each column listed, the reader must apply the specified action before\n returning values for that column.\n\n2. The reader must replace all output references to the column with the result\n of the action, presenting the result under the original field-id. For\n example, if the action for field-id `9` is mask-alphanum, the reader must\n return the masked value as field-id `9` in the query output.\n\n3. A column must appear at most once in required-column-projections.\n4. A projection must not target a map\'s key field-id. Applying an action\n to keys can produce duplicate or null keys, which readers silently\n coalesce or reject, causing data loss.\n', + description='A list of columns that require specific actions to be applied when reading. A server must not return an action for a column whose type is not listed in that action\'s "Applicable to" set. If absent or empty, no required actions apply; columns not listed are not subject to any required action.\n1. For each column listed, the reader must apply the specified action before\n returning values for that column.\n\n2. The reader must replace all output references to the column with the result\n of the action, presenting the result under the original field-id. For\n example, if the action for field-id `9` is mask-alphanum, the reader must\n return the masked value as field-id `9` in the query output.\n\n3. A server must not return more than one projection for the same field-id\n in required-column-projections. If a duplicate field-id appears, the reader\n must fail the query.\n\n4. A projection must not target a map\'s key field-id. Applying an action\n to keys can produce duplicate or null keys, which readers silently\n coalesce or reject, causing data loss.\n', ) required_row_filter: Expression | None = Field( None, diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index 9483fb991e85..0408cfb37ccb 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -1053,11 +1053,11 @@ paths: key. For example, "urn:ietf:params:oauth:token-type:jwt=". - The response may include a read-restrictions field. A client should support this field; - if it does, it must fail any read against the loaded table when it cannot apply a returned - restriction. This includes unrecognized action or expression types, and actions whose - definition cannot be loaded, parsed, or evaluated. These restrictions apply to every read - performed using this response (including subsequent planTableScan and fetchScanTasks + The response may include a read-restrictions field. A reader that supports read + restrictions must fail any read against the loaded table that cannot apply a returned + restriction in full. This includes unrecognized action or expression types, and actions + whose definition cannot be loaded, parsed, or evaluated. These restrictions apply to every + read performed using this response (including subsequent planTableScan and fetchScanTasks calls), only to the authenticated principal associated with the request, and must not be interpreted as global policy. parameters: @@ -3862,7 +3862,7 @@ components: If a projection targets a nested-typed field (struct, list, or map), other projections in the same ReadRestrictions must not target any - descendant field-id (struct subfields, list elements, or map keys/values) + nested field-id (struct subfields, list elements, or map keys/values) at any depth. This avoids ambiguity about which action governs a given leaf value. @@ -3901,7 +3901,9 @@ components: example, if the action for field-id `9` is mask-alphanum, the reader must return the masked value as field-id `9` in the query output. - 3. A column must appear at most once in required-column-projections. + 3. A server must not return more than one projection for the same field-id + in required-column-projections. If a duplicate field-id appears, the reader + must fail the query. 4. A projection must not target a map's key field-id. Applying an action to keys can produce duplicate or null keys, which readers silently @@ -4004,6 +4006,9 @@ components: NULL input is also replaced with the type-specific fixed value; NULL is not preserved. Applicable to: all data types except unknown, geometry, and geography. + Because those three types have no fixed value defined above, a catalog server + must not return mask-to-fixed-value for a struct, list, or map that contains a + field of type unknown, geometry, or geography at any nesting depth. allOf: - $ref: '#/components/schemas/Action' properties: From 5d7e73a1f573201773da715f127dfd2ca50fa5a5 Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Wed, 1 Jul 2026 14:16:30 -0700 Subject: [PATCH 40/51] Spec: Address Steven's comments on field-id presence in read schema --- open-api/rest-catalog-open-api.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index 0408cfb37ccb..dd160c1a1bd6 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3908,6 +3908,9 @@ components: 4. A projection must not target a map's key field-id. Applying an action to keys can produce duplicate or null keys, which readers silently coalesce or reject, causing data loss. + + 5. The reader must fail the query if a projection references a field-id + that is not present in the read schema. type: array items: $ref: '#/components/schemas/Action' From 4b4ffae4ac7ac55a213d64aa9dcbe9f8ca27062f Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Wed, 1 Jul 2026 14:30:45 -0700 Subject: [PATCH 41/51] Spec: Regenerate rest-catalog-open-api.py --- open-api/rest-catalog-open-api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index 05679e2d2b1a..258a3597fc9e 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -1721,7 +1721,7 @@ class ReadRestrictions(BaseModel): ) = Field( None, alias='required-column-projections', - description='A list of columns that require specific actions to be applied when reading. A server must not return an action for a column whose type is not listed in that action\'s "Applicable to" set. If absent or empty, no required actions apply; columns not listed are not subject to any required action.\n1. For each column listed, the reader must apply the specified action before\n returning values for that column.\n\n2. The reader must replace all output references to the column with the result\n of the action, presenting the result under the original field-id. For\n example, if the action for field-id `9` is mask-alphanum, the reader must\n return the masked value as field-id `9` in the query output.\n\n3. A server must not return more than one projection for the same field-id\n in required-column-projections. If a duplicate field-id appears, the reader\n must fail the query.\n\n4. A projection must not target a map\'s key field-id. Applying an action\n to keys can produce duplicate or null keys, which readers silently\n coalesce or reject, causing data loss.\n', + description='A list of columns that require specific actions to be applied when reading. A server must not return an action for a column whose type is not listed in that action\'s "Applicable to" set. If absent or empty, no required actions apply; columns not listed are not subject to any required action.\n1. For each column listed, the reader must apply the specified action before\n returning values for that column.\n\n2. The reader must replace all output references to the column with the result\n of the action, presenting the result under the original field-id. For\n example, if the action for field-id `9` is mask-alphanum, the reader must\n return the masked value as field-id `9` in the query output.\n\n3. A server must not return more than one projection for the same field-id\n in required-column-projections. If a duplicate field-id appears, the reader\n must fail the query.\n\n4. A projection must not target a map\'s key field-id. Applying an action\n to keys can produce duplicate or null keys, which readers silently\n coalesce or reject, causing data loss.\n\n5. The reader must fail the query if a projection references a field-id\n that is not present in the read schema.\n', ) required_row_filter: Expression | None = Field( None, From c9c978c1b1e76c7a088453431d232963111e9d06 Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Thu, 9 Jul 2026 10:18:41 -0700 Subject: [PATCH 42/51] Spec: Address review comments on ReadRestrictions rule 5 and wording - Reframe rule 5 around what the reader is actually reading per Ryan's sync framing, dropping the invented "read schema" vocabulary that Dan flagged as confusing. - Replace "authenticated principal" and "global policy" (undefined terms per Dan) with "authentication scope" in the loadTable caching note and drop the phrase from required-row-filter. - Scope the fail rule to conforming readers per huaxingao's nit. - Reword the "missing or empty" clause to drop the overloaded "field" per flyrain's nit. - Move the required/non-nullable note into ReplaceWithNull's "Applicable to" line per flyrain's nit. --- open-api/rest-catalog-open-api.yaml | 27 +++++++++++++-------------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index dd160c1a1bd6..c95d6cb570ed 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -1058,8 +1058,9 @@ paths: restriction in full. This includes unrecognized action or expression types, and actions whose definition cannot be loaded, parsed, or evaluated. These restrictions apply to every read performed using this response (including subsequent planTableScan and fetchScanTasks - calls), only to the authenticated principal associated with the request, and must not be - interpreted as global policy. + calls), but subsequent requests or requests made from a different authentication scope + may have different restrictions. The response should not be cached outside of the + authenticated scope. parameters: - $ref: '#/components/parameters/data-access' - name: If-None-Match @@ -3856,9 +3857,9 @@ components: A reader evaluates the row filter against original, untransformed column values, then applies required-column-projections to the surviving rows. Each action must produce a value of the same type as the input column. - If a reader cannot apply any returned restriction (a filter expression - or an action), it must fail the query and must not silently return raw, - partial, or empty results. + If a reader that supports read-restrictions cannot apply any returned + restriction (a filter expression or an action), it must fail the query + and must not silently return raw, partial, or empty results. If a projection targets a nested-typed field (struct, list, or map), other projections in the same ReadRestrictions must not target any @@ -3866,9 +3867,8 @@ components: at any depth. This avoids ambiguity about which action governs a given leaf value. - An empty ReadRestrictions object (no required-column-projections and no - required-row-filter) imposes no restrictions and is equivalent to the - field being absent from the response. + A missing or empty ReadRestrictions object (no required-column-projections and no + required-row-filter) imposes no restrictions. example: required-column-projections: - field-id: 4 @@ -3909,14 +3909,14 @@ components: to keys can produce duplicate or null keys, which readers silently coalesce or reject, causing data loss. - 5. The reader must fail the query if a projection references a field-id - that is not present in the read schema. + 5. A reader must enforce projections on the columns it is actually reading. + Projections referencing columns that are not being read do not apply. type: array items: $ref: '#/components/schemas/Action' required-row-filter: description: > - An expression that filters rows in the table that the authenticated principal does not have access to. + An expression that limits which rows the reader may return. 1. The expression must evaluate to a boolean (TRUE or FALSE; Iceberg predicates never produce NULL). A reader must discard any row for which the filter @@ -4021,10 +4021,9 @@ components: ReplaceWithNull: description: > - Replaces the entire column value with NULL. NULL input is preserved (NULL -> NULL). - A server must not return this action for a required (non-nullable) column. + Replaces the column value with NULL. NULL input is preserved (NULL -> NULL). - Applicable to: all optional types + Applicable to: all optional types. Applying to a required (non-nullable) column is invalid. allOf: - $ref: '#/components/schemas/Action' properties: From d6848eaed819b6398c040e04c42da9f9ce3928b4 Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Thu, 9 Jul 2026 10:18:46 -0700 Subject: [PATCH 43/51] Spec: Regenerate rest-catalog-open-api.py --- open-api/rest-catalog-open-api.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index 258a3597fc9e..8a9d21d1f0c6 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -639,8 +639,8 @@ class MaskToFixedValue(Action): class ReplaceWithNull(Action): """ - Replaces the entire column value with NULL. NULL input is preserved (NULL -> NULL). A server must not return this action for a required (non-nullable) column. - Applicable to: all optional types + Replaces the column value with NULL. NULL input is preserved (NULL -> NULL). + Applicable to: all optional types. Applying to a required (non-nullable) column is invalid. """ @@ -1699,9 +1699,9 @@ class AddSchemaUpdate(BaseUpdate): class ReadRestrictions(BaseModel): """ Read restrictions for a table. - A reader evaluates the row filter against original, untransformed column values, then applies required-column-projections to the surviving rows. Each action must produce a value of the same type as the input column. If a reader cannot apply any returned restriction (a filter expression or an action), it must fail the query and must not silently return raw, partial, or empty results. + A reader evaluates the row filter against original, untransformed column values, then applies required-column-projections to the surviving rows. Each action must produce a value of the same type as the input column. If a reader that supports read-restrictions cannot apply any returned restriction (a filter expression or an action), it must fail the query and must not silently return raw, partial, or empty results. If a projection targets a nested-typed field (struct, list, or map), other projections in the same ReadRestrictions must not target any nested field-id (struct subfields, list elements, or map keys/values) at any depth. This avoids ambiguity about which action governs a given leaf value. - An empty ReadRestrictions object (no required-column-projections and no required-row-filter) imposes no restrictions and is equivalent to the field being absent from the response. + A missing or empty ReadRestrictions object (no required-column-projections and no required-row-filter) imposes no restrictions. """ @@ -1721,12 +1721,12 @@ class ReadRestrictions(BaseModel): ) = Field( None, alias='required-column-projections', - description='A list of columns that require specific actions to be applied when reading. A server must not return an action for a column whose type is not listed in that action\'s "Applicable to" set. If absent or empty, no required actions apply; columns not listed are not subject to any required action.\n1. For each column listed, the reader must apply the specified action before\n returning values for that column.\n\n2. The reader must replace all output references to the column with the result\n of the action, presenting the result under the original field-id. For\n example, if the action for field-id `9` is mask-alphanum, the reader must\n return the masked value as field-id `9` in the query output.\n\n3. A server must not return more than one projection for the same field-id\n in required-column-projections. If a duplicate field-id appears, the reader\n must fail the query.\n\n4. A projection must not target a map\'s key field-id. Applying an action\n to keys can produce duplicate or null keys, which readers silently\n coalesce or reject, causing data loss.\n\n5. The reader must fail the query if a projection references a field-id\n that is not present in the read schema.\n', + description='A list of columns that require specific actions to be applied when reading. A server must not return an action for a column whose type is not listed in that action\'s "Applicable to" set. If absent or empty, no required actions apply; columns not listed are not subject to any required action.\n1. For each column listed, the reader must apply the specified action before\n returning values for that column.\n\n2. The reader must replace all output references to the column with the result\n of the action, presenting the result under the original field-id. For\n example, if the action for field-id `9` is mask-alphanum, the reader must\n return the masked value as field-id `9` in the query output.\n\n3. A server must not return more than one projection for the same field-id\n in required-column-projections. If a duplicate field-id appears, the reader\n must fail the query.\n\n4. A projection must not target a map\'s key field-id. Applying an action\n to keys can produce duplicate or null keys, which readers silently\n coalesce or reject, causing data loss.\n\n5. A reader must enforce projections on the columns it is actually reading.\n Projections referencing columns that are not being read do not apply.\n', ) required_row_filter: Expression | None = Field( None, alias='required-row-filter', - description='An expression that filters rows in the table that the authenticated principal does not have access to.\n1. The expression must evaluate to a boolean (TRUE or FALSE; Iceberg predicates\n never produce NULL). A reader must discard any row for which the filter\n evaluates to FALSE, and no information derived from discarded rows may be\n included in the query result.\n\n2. If this property is absent, null, or always true then no mandatory filtering is required.\n', + description='An expression that limits which rows the reader may return.\n1. The expression must evaluate to a boolean (TRUE or FALSE; Iceberg predicates\n never produce NULL). A reader must discard any row for which the filter\n evaluates to FALSE, and no information derived from discarded rows may be\n included in the query result.\n\n2. If this property is absent, null, or always true then no mandatory filtering is required.\n', ) From 5388e214ce4154c489e5fbf584ceba1700862ae0 Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Thu, 9 Jul 2026 11:21:22 -0700 Subject: [PATCH 44/51] Spec: Reframe MaskToFixedValue applicable-to as an allowlist MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per laurentgo's review nit on yaml:3835 — flip from an exclusion list ("all types except unknown, geometry, geography") to an inclusion by reference to the "Fixed values by type" list above. New Iceberg types are then implicitly blocked until the spec explicitly extends the fixed-values list to cover them. The container-nesting carve-out is dropped: it's redundant with the MUST NOT clause because a struct containing a field with no fixed value isn't a "type with a fixed value defined above" — the recursive definition simply doesn't apply. --- open-api/rest-catalog-open-api.yaml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index c95d6cb570ed..f2e0c4f2ab13 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -4008,10 +4008,8 @@ components: NULL input is also replaced with the type-specific fixed value; NULL is not preserved. - Applicable to: all data types except unknown, geometry, and geography. - Because those three types have no fixed value defined above, a catalog server - must not return mask-to-fixed-value for a struct, list, or map that contains a - field of type unknown, geometry, or geography at any nesting depth. + Applicable to: the types with a fixed value defined above. A catalog server + MUST NOT return mask-to-fixed-value for any other type. allOf: - $ref: '#/components/schemas/Action' properties: From f4872e2b1b415ed0506a592985eddc5da1686c41 Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Thu, 9 Jul 2026 11:21:22 -0700 Subject: [PATCH 45/51] Spec: Regenerate rest-catalog-open-api.py --- open-api/rest-catalog-open-api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index 8a9d21d1f0c6..6e7c22110c08 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -630,7 +630,7 @@ class MaskToFixedValue(Action): Replaces the column value with a type-specific fixed value. Readers must use exactly the values listed below to ensure consistency across implementations. Fixed values by type: - boolean: false - int: 0 - long: 0 - float: 0.0 - double: 0.0 - decimal(p, s): 0 (the unscaled value is 0) - string: "XXXXXXXX" - date: 1970-01-01 - time: 00:00:00 - timestamp: 1970-01-01T00:00:00 - timestamptz: 1970-01-01T00:00:00+00:00 - timestamp_ns: 1970-01-01T00:00:00.000000000 - timestamptz_ns: 1970-01-01T00:00:00.000000000+00:00 - uuid: 00000000-0000-0000-0000-000000000000 - fixed(n): n zero bytes - binary: empty byte sequence - variant: {} - list: empty list [] - map: empty map {} - struct: struct with each field set to its type-specific default (applied recursively) NULL input is also replaced with the type-specific fixed value; NULL is not preserved. - Applicable to: all data types except unknown, geometry, and geography. Because those three types have no fixed value defined above, a catalog server must not return mask-to-fixed-value for a struct, list, or map that contains a field of type unknown, geometry, or geography at any nesting depth. + Applicable to: the types with a fixed value defined above. A catalog server MUST NOT return mask-to-fixed-value for any other type. """ From d2465cc23d0c0da65393d13a153013f30bc5b844 Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Mon, 27 Jul 2026 11:53:42 -0700 Subject: [PATCH 46/51] Spec: Reword nested-overlap rule to fail-closed semantics Replace the inaccurate justification ("avoids ambiguity") with what we actually mean: the spec does not define how such actions combine, and a reader that receives such a response must fail the query. --- open-api/rest-catalog-open-api.py | 6 +----- open-api/rest-catalog-open-api.yaml | 4 ++-- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index 6e7c22110c08..ee28499e1ba8 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -279,10 +279,6 @@ class Summary(BaseModel): operation: Literal['append', 'replace', 'overwrite', 'delete'] -Summary.__annotations__['__pydantic_extra__'] = Dict[str, str] -Summary.model_rebuild(force=True) - - class Snapshot(BaseModel): snapshot_id: int = Field(..., alias='snapshot-id') parent_snapshot_id: int | None = Field(None, alias='parent-snapshot-id') @@ -1700,7 +1696,7 @@ class ReadRestrictions(BaseModel): """ Read restrictions for a table. A reader evaluates the row filter against original, untransformed column values, then applies required-column-projections to the surviving rows. Each action must produce a value of the same type as the input column. If a reader that supports read-restrictions cannot apply any returned restriction (a filter expression or an action), it must fail the query and must not silently return raw, partial, or empty results. - If a projection targets a nested-typed field (struct, list, or map), other projections in the same ReadRestrictions must not target any nested field-id (struct subfields, list elements, or map keys/values) at any depth. This avoids ambiguity about which action governs a given leaf value. + If a projection targets a nested-typed field (struct, list, or map), other projections in the same ReadRestrictions must not target any nested field-id (struct subfields, list elements, or map keys/values) at any depth. This specification does not define how such actions combine. A reader that receives such a response must fail the query. A missing or empty ReadRestrictions object (no required-column-projections and no required-row-filter) imposes no restrictions. """ diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index f2e0c4f2ab13..2f0060adf72e 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3864,8 +3864,8 @@ components: If a projection targets a nested-typed field (struct, list, or map), other projections in the same ReadRestrictions must not target any nested field-id (struct subfields, list elements, or map keys/values) - at any depth. This avoids ambiguity about which action governs a given - leaf value. + at any depth. This specification does not define how such actions + combine. A reader that receives such a response must fail the query. A missing or empty ReadRestrictions object (no required-column-projections and no required-row-filter) imposes no restrictions. From 4cd996170f6be67df7c5de836eb98f0668403a7a Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Mon, 27 Jul 2026 13:04:18 -0700 Subject: [PATCH 47/51] Spec: Use IdReference in required-row-filter and require field-id references Update the required-row-filter example to use the new expression format (IdReference + typed literal) and add rule requiring field-id references instead of column names for rename stability. --- open-api/rest-catalog-open-api.py | 326 +++++++++++++++++++++------- open-api/rest-catalog-open-api.yaml | 14 +- 2 files changed, 262 insertions(+), 78 deletions(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index ee28499e1ba8..d449cd1fdb5e 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -156,6 +156,11 @@ class ExpressionType(RootModel[str]): class TrueExpression(BaseModel): + """ + Deprecated. Use the bare boolean literal `true` as a predicate instead. + + """ + type: Literal['true'] = Field( ..., examples=[ @@ -185,6 +190,11 @@ class TrueExpression(BaseModel): class FalseExpression(BaseModel): + """ + Deprecated. Use the bare boolean literal `false` as a predicate instead. + + """ + type: Literal['false'] = Field( ..., examples=[ @@ -213,8 +223,49 @@ class FalseExpression(BaseModel): ) -class Reference(RootModel[str]): - root: str = Field(..., examples=[['column-name']]) +class IdReference(BaseModel): + """ + A bound reference to a field by field ID. + """ + + model_config = ConfigDict( + extra='forbid', + ) + type: Literal['reference'] + id: int + + +class NamedReference(BaseModel): + """ + An unbound reference to a field by name. + """ + + model_config = ConfigDict( + extra='forbid', + ) + type: Literal['reference'] + name: str + + +class Function(RootModel[str]): + root: str = Field(..., min_length=1) + + +class Function1(BaseModel): + model_config = ConfigDict( + extra='forbid', + ) + catalog: str | None = None + identifier: CatalogObjectIdentifier + + +class TermReference(RootModel[str]): + root: str = Field( + ..., + deprecated=True, + description='Deprecated string-form field reference used in older REST predicates. Use Reference (IdReference or NamedReference) instead.\n', + examples=[['column-name']], + ) class Transform(RootModel[str]): @@ -1301,10 +1352,64 @@ class RenameTableRequest(BaseModel): destination: TableIdentifier +class Literal1(BaseModel): + """ + A literal is either a bare value, an untyped literal object, or a typed literal object. + + """ + + model_config = ConfigDict( + extra='forbid', + ) + type: Literal['literal'] + value: PrimitiveTypeValue + data_type: PrimitiveType | None = Field(None, alias='data-type') + + +class LiteralModel(RootModel[PrimitiveTypeValue | Literal1]): + root: PrimitiveTypeValue | Literal1 = Field( + ..., + description='A literal is either a bare value, an untyped literal object, or a typed literal object.\n', + ) + + +class Literals1(BaseModel): + """ + Literals is either a bare array of literals or a typed object with an explicit data-type applied to all values. + + """ + + model_config = ConfigDict( + extra='forbid', + ) + type: Literal['literals'] + values: list[PrimitiveTypeValue] + data_type: PrimitiveType = Field(..., alias='data-type') + + +class Literals(RootModel[list[LiteralModel] | Literals1]): + root: list[LiteralModel] | Literals1 = Field( + ..., + description='Literals is either a bare array of literals or a typed object with an explicit data-type applied to all values.\n', + ) + + +class Reference(RootModel[IdReference | NamedReference]): + root: IdReference | NamedReference = Field( + ..., + description='A reference to a field. Either a bound reference (by field ID) or an unbound reference (by name). The context in which an expression is used determines which form is valid.\n', + ) + + class TransformTerm(BaseModel): + """ + Deprecated. Legacy transform-applied-to-a-term form. Use Apply with a Reference argument instead. + + """ + type: Literal['transform'] transform: Transform - term: Reference + term: TermReference class SetPartitionStatisticsUpdate(BaseUpdate): @@ -1413,8 +1518,12 @@ class FetchScanTasksRequest(BaseModel): plan_task: PlanTask = Field(..., alias='plan-task') -class Term(RootModel[Reference | TransformTerm]): - root: Reference | TransformTerm +class Term(RootModel[TermReference | TransformTerm]): + root: TermReference | TransformTerm = Field( + ..., + deprecated=True, + description='Deprecated. Legacy term form used by older REST predicates. Use Reference or Apply instead.\n', + ) class SetStatisticsUpdate(BaseUpdate): @@ -1452,8 +1561,39 @@ class FunctionDefinitionVersion(BaseModel): ) -class UnaryExpression(BaseModel): - type: Literal['is-null', 'not-null', 'is-nan', 'not-nan'] = Field( +class StructField(BaseModel): + id: int + name: str + type: Type + required: bool + doc: str | None = None + initial_default: PrimitiveTypeValue | None = Field(None, alias='initial-default') + write_default: PrimitiveTypeValue | None = Field(None, alias='write-default') + + +class StructType(BaseModel): + type: Literal['struct'] + fields: list[StructField] + + +class ListType(BaseModel): + type: Literal['list'] + element_id: int = Field(..., alias='element-id') + element: Type + element_required: bool = Field(..., alias='element-required') + + +class MapType(BaseModel): + type: Literal['map'] + key_id: int = Field(..., alias='key-id') + key: Type + value_id: int = Field(..., alias='value-id') + value: Type + value_required: bool = Field(..., alias='value-required') + + +class AndOrPredicate(BaseModel): + type: Literal['and', 'or'] = Field( ..., examples=[ [ @@ -1479,13 +1619,12 @@ class UnaryExpression(BaseModel): ] ], ) - term: Term + left: Predicate + right: Predicate -class LiteralExpression(BaseModel): - type: Literal[ - 'lt', 'lt-eq', 'gt', 'gt-eq', 'eq', 'not-eq', 'starts-with', 'not-starts-with' - ] = Field( +class NotPredicate(BaseModel): + type: Literal['not'] = Field( ..., examples=[ [ @@ -1511,12 +1650,19 @@ class LiteralExpression(BaseModel): ] ], ) - term: Term - value: PrimitiveTypeValue + child: Predicate -class SetExpression(BaseModel): - type: Literal['in', 'not-in'] = Field( +class UnaryPredicate(BaseModel): + """ + A predicate that tests a single value expression. Accepts either 'child' (preferred) or 'term' (deprecated) to identify the operand. + + """ + + model_config = ConfigDict( + extra='forbid', + ) + type: Literal['is-null', 'not-null', 'is-nan', 'not-nan'] = Field( ..., examples=[ [ @@ -1542,43 +1688,24 @@ class SetExpression(BaseModel): ] ], ) - term: Term - values: list[PrimitiveTypeValue] - - -class StructField(BaseModel): - id: int - name: str - type: Type - required: bool - doc: str | None = None - initial_default: PrimitiveTypeValue | None = Field(None, alias='initial-default') - write_default: PrimitiveTypeValue | None = Field(None, alias='write-default') - - -class StructType(BaseModel): - type: Literal['struct'] - fields: list[StructField] - - -class ListType(BaseModel): - type: Literal['list'] - element_id: int = Field(..., alias='element-id') - element: Type - element_required: bool = Field(..., alias='element-required') + child: ValueExpression | None = None + term: Term | None = Field( + None, deprecated=True, description="Deprecated. Use 'child' instead." + ) -class MapType(BaseModel): - type: Literal['map'] - key_id: int = Field(..., alias='key-id') - key: Type - value_id: int = Field(..., alias='value-id') - value: Type - value_required: bool = Field(..., alias='value-required') +class ComparisonPredicate(BaseModel): + """ + A predicate that compares two value expressions. Accepts either 'left'/'right' (preferred) or 'term'/'value' (deprecated). + """ -class AndOrExpression(BaseModel): - type: Literal['and', 'or'] = Field( + model_config = ConfigDict( + extra='forbid', + ) + type: Literal[ + 'lt', 'lt-eq', 'gt', 'gt-eq', 'eq', 'not-eq', 'starts-with', 'not-starts-with' + ] = Field( ..., examples=[ [ @@ -1604,12 +1731,26 @@ class AndOrExpression(BaseModel): ] ], ) - left: Expression - right: Expression + left: ValueExpression | None = None + right: ValueExpression | None = None + term: Term | None = Field( + None, deprecated=True, description="Deprecated. Use 'left' instead." + ) + value: LiteralModel | None = Field( + None, deprecated=True, description="Deprecated. Use 'right' instead." + ) -class NotExpression(BaseModel): - type: Literal['not'] = Field( +class SetPredicate(BaseModel): + """ + A predicate that tests whether a value is in a set of literals. Accepts either 'child' (preferred) or 'term' (deprecated) to identify the operand. + + """ + + model_config = ConfigDict( + extra='forbid', + ) + type: Literal['in', 'not-in'] = Field( ..., examples=[ [ @@ -1635,7 +1776,25 @@ class NotExpression(BaseModel): ] ], ) - child: Expression + child: ValueExpression | None = None + term: Term | None = Field( + None, deprecated=True, description="Deprecated. Use 'child' instead." + ) + values: Literals + + +class Apply(BaseModel): + """ + A function application on zero or more value expressions or predicates. + + """ + + model_config = ConfigDict( + extra='forbid', + ) + type: Literal['apply'] + function: Function | CatalogObjectIdentifier | Function1 + arguments: list[FunctionArgument] class TableMetadata(BaseModel): @@ -1719,10 +1878,10 @@ class ReadRestrictions(BaseModel): alias='required-column-projections', description='A list of columns that require specific actions to be applied when reading. A server must not return an action for a column whose type is not listed in that action\'s "Applicable to" set. If absent or empty, no required actions apply; columns not listed are not subject to any required action.\n1. For each column listed, the reader must apply the specified action before\n returning values for that column.\n\n2. The reader must replace all output references to the column with the result\n of the action, presenting the result under the original field-id. For\n example, if the action for field-id `9` is mask-alphanum, the reader must\n return the masked value as field-id `9` in the query output.\n\n3. A server must not return more than one projection for the same field-id\n in required-column-projections. If a duplicate field-id appears, the reader\n must fail the query.\n\n4. A projection must not target a map\'s key field-id. Applying an action\n to keys can produce duplicate or null keys, which readers silently\n coalesce or reject, causing data loss.\n\n5. A reader must enforce projections on the columns it is actually reading.\n Projections referencing columns that are not being read do not apply.\n', ) - required_row_filter: Expression | None = Field( + required_row_filter: Predicate | None = Field( None, alias='required-row-filter', - description='An expression that limits which rows the reader may return.\n1. The expression must evaluate to a boolean (TRUE or FALSE; Iceberg predicates\n never produce NULL). A reader must discard any row for which the filter\n evaluates to FALSE, and no information derived from discarded rows may be\n included in the query result.\n\n2. If this property is absent, null, or always true then no mandatory filtering is required.\n', + description='An expression that limits which rows the reader may return.\n1. The expression must evaluate to a boolean (TRUE or FALSE; Iceberg predicates\n never produce NULL). A reader must discard any row for which the filter\n evaluates to FALSE, and no information derived from discarded rows may be\n included in the query result.\n\n2. If this property is absent, null, or always true then no mandatory filtering is required.\n3. Column references within the expression must use field IDs (IdReference),\n not column names. This ensures the filter remains valid across column renames,\n consistent with required-column-projections which also reference columns by field-id.\n', ) @@ -1906,7 +2065,7 @@ class LoadViewResult(BaseModel): class ScanReport(BaseModel): table_name: str = Field(..., alias='table-name') snapshot_id: int = Field(..., alias='snapshot-id') - filter: Expression + filter: Predicate schema_id: int = Field(..., alias='schema-id') projected_field_ids: list[int] = Field(..., alias='projected-field-ids') projected_field_names: list[str] = Field(..., alias='projected-field-names') @@ -2063,8 +2222,8 @@ class PlanTableScanRequest(BaseModel): select: list[FieldName] | None = Field( None, description='List of selected schema fields' ) - filter: Expression | None = Field( - None, description='Expression used to filter the table data' + filter: Predicate | None = Field( + None, description='Predicate used to filter the table data' ) min_rows_requested: int | None = Field( None, @@ -2105,7 +2264,7 @@ class FileScanTask(BaseModel): alias='delete-file-references', description='A list of indices in the delete files array (0-based)', ) - residual_filter: Expression | None = Field( + residual_filter: Predicate | None = Field( None, alias='residual-filter', description='An optional filter to be applied to rows in this file scan task.\nIf the residual is not present, the client must produce the residual or use the original filter.', @@ -2121,25 +2280,34 @@ class Type(RootModel[VariantType | PrimitiveType | StructType | ListType | MapTy root: VariantType | PrimitiveType | StructType | ListType | MapType -class Expression( +class Predicate( RootModel[ - TrueExpression + bool + | TrueExpression | FalseExpression - | AndOrExpression - | NotExpression - | SetExpression - | LiteralExpression - | UnaryExpression + | AndOrPredicate + | NotPredicate + | UnaryPredicate + | ComparisonPredicate + | SetPredicate ] ): root: ( - TrueExpression + bool + | TrueExpression | FalseExpression - | AndOrExpression - | NotExpression - | SetExpression - | LiteralExpression - | UnaryExpression + | AndOrPredicate + | NotPredicate + | UnaryPredicate + | ComparisonPredicate + | SetPredicate + ) + + +class ValueExpression(RootModel[LiteralModel | Reference | Apply]): + root: LiteralModel | Reference | Apply = Field( + ..., + description='A value expression: a literal, a field reference, or a function application.\n', ) @@ -2255,6 +2423,10 @@ class FunctionDataType( ) +class FunctionArgument(RootModel[ValueExpression | Predicate]): + root: ValueExpression | Predicate + + class CompletedPlanningWithIDResult(CompletedPlanningResult): plan_id: str = Field( ..., alias='plan-id', description='ID used to track a planning request' @@ -2299,8 +2471,12 @@ class PlanTableScanResult( StructField.model_rebuild() ListType.model_rebuild() MapType.model_rebuild() -AndOrExpression.model_rebuild() -NotExpression.model_rebuild() +AndOrPredicate.model_rebuild() +NotPredicate.model_rebuild() +UnaryPredicate.model_rebuild() +ComparisonPredicate.model_rebuild() +SetPredicate.model_rebuild() +Apply.model_rebuild() TableMetadata.model_rebuild() ViewMetadata.model_rebuild() AddSchemaUpdate.model_rebuild() diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index 2f0060adf72e..0da7999cea91 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3883,8 +3883,12 @@ components: action: mask-alphanum required-row-filter: type: eq - term: region - value: US + left: + type: reference + id: 14 + right: + type: literal + value: "US" properties: required-column-projections: description: > @@ -3924,8 +3928,12 @@ components: included in the query result. 2. If this property is absent, null, or always true then no mandatory filtering is required. + + 3. Column references within the expression must use field IDs (IdReference), + not column names. This ensures the filter remains valid across column renames, + consistent with required-column-projections which also reference columns by field-id. allOf: - - $ref: '#/components/schemas/Expression' + - $ref: '#/components/schemas/Predicate' Action: discriminator: From 8f41ea94b45f77bd4855ed7bc5ab3d6187cdbced Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Tue, 25 Aug 2026 09:46:24 -0700 Subject: [PATCH 48/51] Spec: Name the server as actor in the nested-overlap rule Tie the prohibition on overlapping nested projections to the server, the way the sibling constraints on required-column-projections already do, and drop the redundant enumeration of nested field-ids (covered by "any field-id nested within it at any depth"). Regenerate rest-catalog-open-api.py. --- open-api/rest-catalog-open-api.py | 2 +- open-api/rest-catalog-open-api.yaml | 8 +++----- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index d449cd1fdb5e..4691ebc915aa 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -1855,7 +1855,7 @@ class ReadRestrictions(BaseModel): """ Read restrictions for a table. A reader evaluates the row filter against original, untransformed column values, then applies required-column-projections to the surviving rows. Each action must produce a value of the same type as the input column. If a reader that supports read-restrictions cannot apply any returned restriction (a filter expression or an action), it must fail the query and must not silently return raw, partial, or empty results. - If a projection targets a nested-typed field (struct, list, or map), other projections in the same ReadRestrictions must not target any nested field-id (struct subfields, list elements, or map keys/values) at any depth. This specification does not define how such actions combine. A reader that receives such a response must fail the query. + A server must not return projections on both a nested-typed field (struct, list, or map) and any field-id nested within it at any depth. A reader that receives such a response must fail the query. A missing or empty ReadRestrictions object (no required-column-projections and no required-row-filter) imposes no restrictions. """ diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index 0da7999cea91..641f7fd63ff3 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3861,11 +3861,9 @@ components: restriction (a filter expression or an action), it must fail the query and must not silently return raw, partial, or empty results. - If a projection targets a nested-typed field (struct, list, or map), - other projections in the same ReadRestrictions must not target any - nested field-id (struct subfields, list elements, or map keys/values) - at any depth. This specification does not define how such actions - combine. A reader that receives such a response must fail the query. + A server must not return projections on both a nested-typed field + (struct, list, or map) and any field-id nested within it at any + depth. A reader that receives such a response must fail the query. A missing or empty ReadRestrictions object (no required-column-projections and no required-row-filter) imposes no restrictions. From 09b55030ec4b1842090f255fe28b4ca3d0ee47f7 Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Tue, 25 Aug 2026 10:07:43 -0700 Subject: [PATCH 49/51] OpenAPI: Regenerate for datamodel-code-generator 0.73.0 Rebasing onto main picked up the bumps from 0.68.1 through 0.73.0, which emits the required const discriminator as `Literal[...]` rather than `Literal[...] | None = None`. --- open-api/rest-catalog-open-api.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index 4691ebc915aa..041623b86210 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -669,7 +669,7 @@ class MaskAlphanum(Action): """ - action: Literal['mask-alphanum'] | None = None + action: Literal['mask-alphanum'] class MaskToFixedValue(Action): @@ -681,7 +681,7 @@ class MaskToFixedValue(Action): """ - action: Literal['mask-to-fixed-value'] | None = None + action: Literal['mask-to-fixed-value'] class ReplaceWithNull(Action): @@ -691,7 +691,7 @@ class ReplaceWithNull(Action): """ - action: Literal['replace-with-null'] | None = None + action: Literal['replace-with-null'] class ShowFirst4(Action): @@ -703,7 +703,7 @@ class ShowFirst4(Action): """ - action: Literal['show-first-4'] | None = None + action: Literal['show-first-4'] class ShowLast4(Action): @@ -715,7 +715,7 @@ class ShowLast4(Action): """ - action: Literal['show-last-4'] | None = None + action: Literal['show-last-4'] class TruncateToYear(Action): @@ -727,7 +727,7 @@ class TruncateToYear(Action): """ - action: Literal['truncate-to-year'] | None = None + action: Literal['truncate-to-year'] class TruncateToMonth(Action): @@ -739,7 +739,7 @@ class TruncateToMonth(Action): """ - action: Literal['truncate-to-month'] | None = None + action: Literal['truncate-to-month'] class Sha256Global(Action): @@ -765,7 +765,7 @@ class Sha256Global(Action): """ - action: Literal['sha-256-global'] | None = None + action: Literal['sha-256-global'] class Sha256QueryLocal(Action): @@ -788,7 +788,7 @@ class Sha256QueryLocal(Action): """ - action: Literal['sha-256-query-local'] | None = None + action: Literal['sha-256-query-local'] class LoadCredentialsResponse(BaseModel): From e44a3cce51ce21c5957da8d6c0df8a7dbff00ce1 Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Mon, 31 Aug 2026 09:13:12 -0700 Subject: [PATCH 50/51] OpenAPI: Use a dummy email in redaction examples Replace the personal email in the MaskAlphanum and ShowFirst4 examples with iceberg16112018@apache.org. --- open-api/rest-catalog-open-api.py | 4 ++-- open-api/rest-catalog-open-api.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index 041623b86210..5f1bba67562e 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -663,7 +663,7 @@ class MaskAlphanum(Action): - All other Unicode characters (including letters, whitespace, and any punctuation not listed above) are replaced with 'x' - For example: "prashant010696@gmail.com" -> "xxxxxxxxnnnnnn@xxxxx.xxx" + For example: "iceberg16112018@apache.org" -> "xxxxxxxnnnnnnnn@xxxxxx.xxx" NULL input is preserved (NULL -> NULL). Applicable to: string @@ -697,7 +697,7 @@ class ReplaceWithNull(Action): class ShowFirst4(Action): """ Preserves the first 4 Unicode code points of the column value and redacts the remainder using mask-alphanum rules (see MaskAlphanum for the exact character rules). Values with 4 or fewer Unicode code points are returned unchanged. - For example: "prashant010696@gmail.com" -> "prasxxxxnnnnnn@xxxxx.xxx" + For example: "iceberg16112018@apache.org" -> "icebxxxnnnnnnnn@xxxxxx.xxx" NULL input is preserved (NULL -> NULL). Applicable to: string diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index 641f7fd63ff3..7b812043779c 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3972,7 +3972,7 @@ components: - All other Unicode characters (including letters, whitespace, and any punctuation not listed above) are replaced with 'x' - For example: "prashant010696@gmail.com" -> "xxxxxxxxnnnnnn@xxxxx.xxx" + For example: "iceberg16112018@apache.org" -> "xxxxxxxnnnnnnnn@xxxxxx.xxx" NULL input is preserved (NULL -> NULL). @@ -4041,7 +4041,7 @@ components: using mask-alphanum rules (see MaskAlphanum for the exact character rules). Values with 4 or fewer Unicode code points are returned unchanged. - For example: "prashant010696@gmail.com" -> "prasxxxxnnnnnn@xxxxx.xxx" + For example: "iceberg16112018@apache.org" -> "icebxxxnnnnnnnn@xxxxxx.xxx" NULL input is preserved (NULL -> NULL). From ace5184dc00ce16b132703357889b96adad1aa9b Mon Sep 17 00:00:00 2001 From: Prashant Singh Date: Tue, 1 Sep 2026 11:56:21 -0700 Subject: [PATCH 51/51] Spec: Clarify actor semantics and address cosmetic feedback Pin the responsible actor on two ReadRestrictions constraints that previously left it unstated, so a non-conforming catalog response is not something a reader may silently ignore: - required-column-projections rule 4 (map key field-id) - ReplaceWithNull on a required field Move the nested-overlap rule out of the ReadRestrictions description into required-column-projections as rule 6, since it constrains projections only. Cosmetic: name the variant fixed value as an empty variant object rather than JSON object syntax, and use lowercase "must not" in MaskToFixedValue to match the surrounding section. --- open-api/rest-catalog-open-api.py | 9 ++++----- open-api/rest-catalog-open-api.yaml | 24 +++++++++++++----------- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/open-api/rest-catalog-open-api.py b/open-api/rest-catalog-open-api.py index 5f1bba67562e..081604426ab0 100644 --- a/open-api/rest-catalog-open-api.py +++ b/open-api/rest-catalog-open-api.py @@ -675,9 +675,9 @@ class MaskAlphanum(Action): class MaskToFixedValue(Action): """ Replaces the column value with a type-specific fixed value. Readers must use exactly the values listed below to ensure consistency across implementations. - Fixed values by type: - boolean: false - int: 0 - long: 0 - float: 0.0 - double: 0.0 - decimal(p, s): 0 (the unscaled value is 0) - string: "XXXXXXXX" - date: 1970-01-01 - time: 00:00:00 - timestamp: 1970-01-01T00:00:00 - timestamptz: 1970-01-01T00:00:00+00:00 - timestamp_ns: 1970-01-01T00:00:00.000000000 - timestamptz_ns: 1970-01-01T00:00:00.000000000+00:00 - uuid: 00000000-0000-0000-0000-000000000000 - fixed(n): n zero bytes - binary: empty byte sequence - variant: {} - list: empty list [] - map: empty map {} - struct: struct with each field set to its type-specific default (applied recursively) + Fixed values by type: - boolean: false - int: 0 - long: 0 - float: 0.0 - double: 0.0 - decimal(p, s): 0 (the unscaled value is 0) - string: "XXXXXXXX" - date: 1970-01-01 - time: 00:00:00 - timestamp: 1970-01-01T00:00:00 - timestamptz: 1970-01-01T00:00:00+00:00 - timestamp_ns: 1970-01-01T00:00:00.000000000 - timestamptz_ns: 1970-01-01T00:00:00.000000000+00:00 - uuid: 00000000-0000-0000-0000-000000000000 - fixed(n): n zero bytes - binary: empty byte sequence - variant: an empty variant object - list: empty list [] - map: empty map {} - struct: struct with each field set to its type-specific default (applied recursively) NULL input is also replaced with the type-specific fixed value; NULL is not preserved. - Applicable to: the types with a fixed value defined above. A catalog server MUST NOT return mask-to-fixed-value for any other type. + Applicable to: the types with a fixed value defined above. A server must not return mask-to-fixed-value for any other type. """ @@ -687,7 +687,7 @@ class MaskToFixedValue(Action): class ReplaceWithNull(Action): """ Replaces the column value with NULL. NULL input is preserved (NULL -> NULL). - Applicable to: all optional types. Applying to a required (non-nullable) column is invalid. + Applicable to: all optional types. A server must not return replace-with-null for a required (non-nullable) field; a reader that receives one must fail the query. """ @@ -1855,7 +1855,6 @@ class ReadRestrictions(BaseModel): """ Read restrictions for a table. A reader evaluates the row filter against original, untransformed column values, then applies required-column-projections to the surviving rows. Each action must produce a value of the same type as the input column. If a reader that supports read-restrictions cannot apply any returned restriction (a filter expression or an action), it must fail the query and must not silently return raw, partial, or empty results. - A server must not return projections on both a nested-typed field (struct, list, or map) and any field-id nested within it at any depth. A reader that receives such a response must fail the query. A missing or empty ReadRestrictions object (no required-column-projections and no required-row-filter) imposes no restrictions. """ @@ -1876,7 +1875,7 @@ class ReadRestrictions(BaseModel): ) = Field( None, alias='required-column-projections', - description='A list of columns that require specific actions to be applied when reading. A server must not return an action for a column whose type is not listed in that action\'s "Applicable to" set. If absent or empty, no required actions apply; columns not listed are not subject to any required action.\n1. For each column listed, the reader must apply the specified action before\n returning values for that column.\n\n2. The reader must replace all output references to the column with the result\n of the action, presenting the result under the original field-id. For\n example, if the action for field-id `9` is mask-alphanum, the reader must\n return the masked value as field-id `9` in the query output.\n\n3. A server must not return more than one projection for the same field-id\n in required-column-projections. If a duplicate field-id appears, the reader\n must fail the query.\n\n4. A projection must not target a map\'s key field-id. Applying an action\n to keys can produce duplicate or null keys, which readers silently\n coalesce or reject, causing data loss.\n\n5. A reader must enforce projections on the columns it is actually reading.\n Projections referencing columns that are not being read do not apply.\n', + description='A list of columns that require specific actions to be applied when reading. A server must not return an action for a column whose type is not listed in that action\'s "Applicable to" set. If absent or empty, no required actions apply; columns not listed are not subject to any required action.\n1. For each column listed, the reader must apply the specified action before\n returning values for that column.\n\n2. The reader must replace all output references to the column with the result\n of the action, presenting the result under the original field-id. For\n example, if the action for field-id `9` is mask-alphanum, the reader must\n return the masked value as field-id `9` in the query output.\n\n3. A server must not return more than one projection for the same field-id\n in required-column-projections. If a duplicate field-id appears, the reader\n must fail the query.\n\n4. A server must not return a projection that targets a map\'s key field-id;\n a reader that receives one must fail the query. Applying an action to keys\n can produce duplicate or null keys, which readers silently coalesce or\n reject, causing data loss.\n\n5. A reader must enforce projections on the columns it is actually reading.\n Projections referencing columns that are not being read do not apply.\n\n6. A server must not return projections on both a nested-typed field\n (struct, list, or map) and any field-id nested within it at any depth.\n A reader that receives such a response must fail the query.\n', ) required_row_filter: Predicate | None = Field( None, diff --git a/open-api/rest-catalog-open-api.yaml b/open-api/rest-catalog-open-api.yaml index 7b812043779c..573b320a0f44 100644 --- a/open-api/rest-catalog-open-api.yaml +++ b/open-api/rest-catalog-open-api.yaml @@ -3861,10 +3861,6 @@ components: restriction (a filter expression or an action), it must fail the query and must not silently return raw, partial, or empty results. - A server must not return projections on both a nested-typed field - (struct, list, or map) and any field-id nested within it at any - depth. A reader that receives such a response must fail the query. - A missing or empty ReadRestrictions object (no required-column-projections and no required-row-filter) imposes no restrictions. example: @@ -3907,12 +3903,17 @@ components: in required-column-projections. If a duplicate field-id appears, the reader must fail the query. - 4. A projection must not target a map's key field-id. Applying an action - to keys can produce duplicate or null keys, which readers silently - coalesce or reject, causing data loss. + 4. A server must not return a projection that targets a map's key field-id; + a reader that receives one must fail the query. Applying an action to keys + can produce duplicate or null keys, which readers silently coalesce or + reject, causing data loss. 5. A reader must enforce projections on the columns it is actually reading. Projections referencing columns that are not being read do not apply. + + 6. A server must not return projections on both a nested-typed field + (struct, list, or map) and any field-id nested within it at any depth. + A reader that receives such a response must fail the query. type: array items: $ref: '#/components/schemas/Action' @@ -4007,15 +4008,15 @@ components: - uuid: 00000000-0000-0000-0000-000000000000 - fixed(n): n zero bytes - binary: empty byte sequence - - variant: {} + - variant: an empty variant object - list: empty list [] - map: empty map {} - struct: struct with each field set to its type-specific default (applied recursively) NULL input is also replaced with the type-specific fixed value; NULL is not preserved. - Applicable to: the types with a fixed value defined above. A catalog server - MUST NOT return mask-to-fixed-value for any other type. + Applicable to: the types with a fixed value defined above. A server must not + return mask-to-fixed-value for any other type. allOf: - $ref: '#/components/schemas/Action' properties: @@ -4027,7 +4028,8 @@ components: description: > Replaces the column value with NULL. NULL input is preserved (NULL -> NULL). - Applicable to: all optional types. Applying to a required (non-nullable) column is invalid. + Applicable to: all optional types. A server must not return replace-with-null for + a required (non-nullable) field; a reader that receives one must fail the query. allOf: - $ref: '#/components/schemas/Action' properties: