diff --git a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnectorMetadata.java b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnectorMetadata.java index 4c8f56c58b5162..1124f93c378984 100644 --- a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnectorMetadata.java +++ b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnectorMetadata.java @@ -626,7 +626,8 @@ public ConnectorTableSchema getTableSchema( ConnectorCapability.SUPPORTS_NESTED_COLUMN_PRUNE, ConnectorCapability.SUPPORTS_FIELD_ID_ACCESS_PATH, ConnectorCapability.SUPPORTS_STORAGE_PREDICATE_PRUNING, - ConnectorCapability.SUPPORTS_NESTED_COLUMN_SCHEMA_CHANGE)); + ConnectorCapability.SUPPORTS_NESTED_COLUMN_SCHEMA_CHANGE, + ConnectorCapability.SUPPORTS_WRITE_ONLY_DEFAULTS)); /** * Reflects the {@link #SIBLING_INHERITABLE_CAPABILITIES} subset of the owning sibling connector's @@ -636,7 +637,7 @@ public ConnectorTableSchema getTableSchema( * (hive) connector-wide set OR the table's own set, and NEVER consults the sibling connector directly, so * without this reflection an iceberg-on-HMS table would silently lose every such capability the iceberg * sibling declares connector-wide (auto-analyze / Top-N lazy / nested-column prune / storage predicate - * pruning / nested column DDL). + * pruning / nested column DDL / write-only defaults). * Returns the sibling schema unchanged when nothing is inherited and the sibling declared nothing itself * (e.g. a hudi sibling, which declares no capabilities at all). */ diff --git a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorMetadataSiblingDelegationTest.java b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorMetadataSiblingDelegationTest.java index d2f340f0f316fb..77a7b1529ebb59 100644 --- a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorMetadataSiblingDelegationTest.java +++ b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorMetadataSiblingDelegationTest.java @@ -525,7 +525,8 @@ public void foreignHandleSchemaReflectsSiblingScanCapabilitiesAsPerTableMarker() ConnectorCapability.SUPPORTS_TOPN_LAZY_MATERIALIZE, ConnectorCapability.SUPPORTS_NESTED_COLUMN_PRUNE, ConnectorCapability.SUPPORTS_FIELD_ID_ACCESS_PATH, - ConnectorCapability.SUPPORTS_STORAGE_PREDICATE_PRUNING); + ConnectorCapability.SUPPORTS_STORAGE_PREDICATE_PRUNING, + ConnectorCapability.SUPPORTS_WRITE_ONLY_DEFAULTS); HiveConnectorMetadata md = new HiveConnectorMetadata(null, HiveTestProperties.minimal(), new FakeConnectorContext(), SUPPLIER_MUST_NOT_BE_USED, SUPPLIER_MUST_NOT_BE_USED, handle -> new SiblingOwner(new CapabilityDeclaringSiblingConnector(siblingCaps), @@ -533,6 +534,9 @@ public void foreignHandleSchemaReflectsSiblingScanCapabilitiesAsPerTableMarker() ConnectorTableSchema schema = md.getTableSchema(session, foreignHandle); Set reflected = schema.getTableCapabilities(); + Assertions.assertTrue(reflected.contains(ConnectorCapability.SUPPORTS_WRITE_ONLY_DEFAULTS), + "an iceberg-on-HMS table must inherit write-only defaults, or DEFAULT(column) on it answers " + + "NULL from the read schema"); Assertions.assertTrue(reflected.contains(ConnectorCapability.SUPPORTS_COLUMN_AUTO_ANALYZE), "auto-analyze must survive the delegation as a per-table capability"); Assertions.assertTrue(reflected.contains(ConnectorCapability.SUPPORTS_TOPN_LAZY_MATERIALIZE), diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnector.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnector.java index 2d372f21eadb0d..ba5df0232cf70b 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnector.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnector.java @@ -911,6 +911,10 @@ public Set getCapabilities() { // (CREATE TABLE ... ORDER BY (...)). fe-core admits the ORDER BY clause for a plugin-driven CREATE TABLE // ONLY under this capability (replacing the legacy engine-name "iceberg" gate); the sort-column validation // (existence / type / duplicates) runs connector-side in IcebergConnectorMetadata.createTable. + // SUPPORTS_WRITE_ONLY_DEFAULTS: an iceberg write-default applies to new rows only, so it lives in the + // writer schema (IcebergWriteSchemaContext) and not in the read schema. fe-core resolves DEFAULT(column) + // against the writer schema pinned for the write target and rejects it in a plain SELECT, instead of + // answering NULL from the read schema. EnumSet capabilities = EnumSet.of(ConnectorCapability.SUPPORTS_MVCC_SNAPSHOT, ConnectorCapability.SUPPORTS_COLUMN_AUTO_ANALYZE, ConnectorCapability.SUPPORTS_TOPN_LAZY_MATERIALIZE, @@ -921,7 +925,8 @@ public Set getCapabilities() { ConnectorCapability.SUPPORTS_STORAGE_PREDICATE_PRUNING, ConnectorCapability.SUPPORTS_METADATA_PRELOAD, ConnectorCapability.SUPPORTS_SORT_ORDER, - ConnectorCapability.SUPPORTS_NESTED_COLUMN_SCHEMA_CHANGE); + ConnectorCapability.SUPPORTS_NESTED_COLUMN_SCHEMA_CHANGE, + ConnectorCapability.SUPPORTS_WRITE_ONLY_DEFAULTS); // SUPPORTS_USER_SESSION: only a REST catalog configured iceberg.rest.session=user projects the querying // user's delegated credential onto a per-request Iceberg REST SessionCatalog (#63068 re-migration). This // gates FE credential injection + shared-cache bypass; every other flavor/config authenticates with a diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorTest.java index ee671040606e22..09c99edc77a1ac 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorTest.java @@ -200,6 +200,18 @@ public void declaresColumnAutoAnalyzeAndTopNLazyMaterializeCapabilities() { + "materialization"); } + @Test + public void declaresWriteOnlyDefaultsCapability() { + // WHY: an iceberg write-default is carried by the writer schema only, so the read schema's columns have no + // default. fe-core resolves DEFAULT(column) against the pinned writer schema (and rejects it in a plain + // SELECT) ONLY under this capability. MUTATION: dropping the capability -> INSERT ... SELECT DEFAULT(c) + // writes NULL and UPDATE ... WHERE c = DEFAULT(c) silently matches nothing -> red. + IcebergConnector connector = new IcebergConnector(Collections.emptyMap(), new RecordingConnectorContext()); + Assertions.assertTrue(connector.getCapabilities() + .contains(ConnectorCapability.SUPPORTS_WRITE_ONLY_DEFAULTS), + "iceberg must declare SUPPORTS_WRITE_ONLY_DEFAULTS so DEFAULT(column) resolves to the write default"); + } + @Test public void declaresShowCreateDdlCapability() { // WHY: legacy IcebergExternalTable rendered LOCATION + PROPERTIES + PARTITION BY + ORDER BY in SHOW diff --git a/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/ConnectorCapability.java b/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/ConnectorCapability.java index 97e8d5461544d0..2c626b5fd1d971 100644 --- a/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/ConnectorCapability.java +++ b/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/ConnectorCapability.java @@ -304,5 +304,20 @@ public enum ConnectorCapability { * data tables while declining it on the subset of system tables whose readers cannot observe a * selected snapshot.

*/ - SUPPORTS_SCAN_PARAM_OPTIONS + SUPPORTS_SCAN_PARAM_OPTIONS, + /** + * Indicates the connector's column defaults apply to writes only: they are carried by the + * request-scoped writer schema ({@code ConnectorWritePlanProvider.getWriteColumns}), not by the read + * schema (e.g. Iceberg {@code write-default}). + * + *

fe-core then resolves {@code DEFAULT(column)} against the writer schema a statement pinned for its + * write target: in INSERT VALUES, by name against the target in INSERT ... SELECT, and in the + * predicates of UPDATE / MERGE on the target. Anywhere else, e.g. in a plain SELECT, {@code + * DEFAULT(column)} on such a table is rejected instead of answering NULL from the read schema. + * Connectors whose read schema carries the column defaults (hive) must NOT declare it.

+ * + *

Scope: catalog-wide OR per-table. An iceberg-on-HMS table inherits it through the per-table + * set the gateway reflects from its sibling, exactly like {@link #SUPPORTS_NESTED_COLUMN_PRUNE}.

+ */ + SUPPORTS_WRITE_ONLY_DEFAULTS } diff --git a/fe/fe-connector/fe-connector-spi/src/test/java/org/apache/doris/connector/spi/ConnectorPluginSurfaceTest.java b/fe/fe-connector/fe-connector-spi/src/test/java/org/apache/doris/connector/spi/ConnectorPluginSurfaceTest.java index cab3d00cb4b4be..a637b4ae597ffc 100644 --- a/fe/fe-connector/fe-connector-spi/src/test/java/org/apache/doris/connector/spi/ConnectorPluginSurfaceTest.java +++ b/fe/fe-connector/fe-connector-spi/src/test/java/org/apache/doris/connector/spi/ConnectorPluginSurfaceTest.java @@ -89,8 +89,9 @@ public void connectorApiMajorTracksTheRecordedSurfaceChange() throws IOException version.load(in); } // Major 20 adds deleteUsingMetadata to the connector write surface; major 21 adds - // typed filesystem credential descriptions to that surface. - Assertions.assertEquals("21.0", version.getProperty("api.version")); + // typed filesystem credential descriptions to that surface; major 22 adds the + // ConnectorCapability#SUPPORTS_WRITE_ONLY_DEFAULTS constant. + Assertions.assertEquals("22.0", version.getProperty("api.version")); } /** Root entry points plus provider/handle types returned to connector plugins. */ diff --git a/fe/fe-connector/fe-connector-spi/src/test/resources/connector-plugin-surface.txt b/fe/fe-connector/fe-connector-spi/src/test/resources/connector-plugin-surface.txt index 81918e6daa1431..4bc29adfec6fbf 100644 --- a/fe/fe-connector/fe-connector-spi/src/test/resources/connector-plugin-surface.txt +++ b/fe/fe-connector/fe-connector-spi/src/test/resources/connector-plugin-surface.txt @@ -35,6 +35,7 @@ org.apache.doris.connector.spi.ConnectorCapability#enum:SUPPORTS_SYS_TABLE_NESTE org.apache.doris.connector.spi.ConnectorCapability#enum:SUPPORTS_TOPN_LAZY_MATERIALIZE org.apache.doris.connector.spi.ConnectorCapability#enum:SUPPORTS_USER_SESSION org.apache.doris.connector.spi.ConnectorCapability#enum:SUPPORTS_VIEW +org.apache.doris.connector.spi.ConnectorCapability#enum:SUPPORTS_WRITE_ONLY_DEFAULTS org.apache.doris.connector.spi.ConnectorColumn#equals(java.lang.Object):boolean org.apache.doris.connector.spi.ConnectorColumn#getComment():java.lang.String org.apache.doris.connector.spi.ConnectorColumn#getDefaultValue():java.lang.String diff --git a/fe/fe-connector/pom.xml b/fe/fe-connector/pom.xml index 1d91dc0c412092..55ea25afcf097f 100644 --- a/fe/fe-connector/pom.xml +++ b/fe/fe-connector/pom.xml @@ -55,7 +55,7 @@ under the License. of the latter two means bumping this property as well (and fe-extension-spi means bumping all five families). --> - 21.0 + 22.0 diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/plugin/PluginDrivenExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/plugin/PluginDrivenExternalTable.java index 6cbea40af7505b..29cdb3044327b7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/plugin/PluginDrivenExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/plugin/PluginDrivenExternalTable.java @@ -446,6 +446,17 @@ public boolean supportsSampleAnalyze() { return hasCapability(ConnectorCapability.SUPPORTS_SAMPLE_ANALYZE); } + /** + * Returns whether THIS table's column defaults apply to writes only, i.e. only the request-scoped writer + * schema ({@link #resolveWriteColumns}) carries them. {@code InsertUtils} and {@code RewriteDefaultExpression} + * then resolve {@code DEFAULT(column)} against the writer schema a write statement pinned, and reject it + * anywhere else instead of answering NULL from the read schema. Resolved per-table via {@link #hasCapability} + * so an iceberg-on-HMS table inherits it, mirroring {@link #supportsNestedColumnSchemaChange}. + */ + public boolean hasWriteOnlyDefaults() { + return hasCapability(ConnectorCapability.SUPPORTS_WRITE_ONLY_DEFAULTS); + } + /** * Whether this table supports a table-scoped capability, resolved connector-wide OR per-table. A * uniform-format connector (iceberg — every table orc/parquet) declares the capability for all its tables diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/RewriteDefaultExpression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/RewriteDefaultExpression.java index 6dc8ab7c72ccce..9713bbce10de8e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/RewriteDefaultExpression.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/RewriteDefaultExpression.java @@ -18,7 +18,10 @@ package org.apache.doris.nereids.rules.expression.rules; import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.datasource.plugin.PluginDrivenExternalTable; import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.analyzer.UnboundFunction; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.parser.NereidsParser; @@ -46,6 +49,8 @@ * 1. DEFAULT(column) with default value -> the actual default value expression * 2. DEFAULT(column) with no default value but nullable -> NULL * 3. DEFAULT(column) with no default value and not nullable -> error + * A table whose defaults apply to writes only takes them from the writer schema that the statement + * pinned for its write target; DEFAULT(column) on any other such table is an error. */ public class RewriteDefaultExpression implements ExpressionPatternRuleFactory { @@ -82,6 +87,24 @@ private static Expression rewrite(ExpressionMatchingContext context) { + column.getName() + "'"); } + Optional originalTable = slotRef.getOriginalTable(); + if (originalTable.filter(PluginDrivenExternalTable.class::isInstance) + .filter(table -> ((PluginDrivenExternalTable) table).hasWriteOnlyDefaults()) + .isPresent()) { + column = pinnedWriteColumn(context.cascadesContext.getStatementContext(), originalTable.get(), column) + .orElseThrow(() -> new AnalysisException("DEFAULT(" + slotRef.toSql() + + ") must reference a column of the table that this statement writes")); + // The expressions around DEFAULT(column) were bound against the read column, so a default of another type + // cannot stand in for it. The types differ when the table changed outside Doris after its read schema was + // cached, e.g. by a type promotion. + DataType writeType = DataType.fromCatalogType(column.getType()); + if (!writeType.equals(targetType)) { + throw new AnalysisException("DEFAULT(" + slotRef.toSql() + ") reads the column as " + + targetType.toSql() + " but the table now writes it as " + writeType.toSql() + + "; the table changed outside Doris, refresh it and retry"); + } + } + String defaultValueSql = column.getDefaultValueSql(); if (defaultValueSql == null) { if (column.isAllowNull()) { @@ -101,4 +124,12 @@ private static Expression rewrite(ExpressionMatchingContext context) { return TypeCoercionUtils.castIfNotSameType(defaultValueExpr, targetType); } + + private static Optional pinnedWriteColumn(StatementContext statementContext, TableIf table, + Column readColumn) { + return statementContext.getConnectorWriteSchema(table.getId()) + .flatMap(columns -> columns.stream() + .filter(column -> column.getUniqueId() == readColumn.getUniqueId()) + .findFirst()); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ConnectorWriteSchemaUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ConnectorWriteSchemaUtils.java index c4a8d94f9b2e1c..a7f063eaba68ef 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ConnectorWriteSchemaUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ConnectorWriteSchemaUtils.java @@ -28,6 +28,8 @@ import org.apache.doris.nereids.trees.expressions.DefaultValueSlot; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.types.DataType; +import org.apache.doris.nereids.util.TypeCoercionUtils; import org.apache.doris.qe.ConnectContext; import com.google.common.collect.ImmutableList; @@ -101,7 +103,9 @@ public static Expression resolveDefaultReferences(Expression expression, List> writeOnlyDefaultColumns = writeOnlyDefaultColumns(table); if (!(query instanceof UnboundInlineTable)) { + if (writeOnlyDefaultColumns.isPresent()) { + return plan.withChildren(resolveSelectDefaultReferences( + query, writeOnlyDefaultColumns, table, unboundLogicalSink.getNameParts())); + } return plan; } @@ -444,7 +454,9 @@ private static Plan normalizePlanWithoutLock(LogicalPlan plan, TableIf table, null, rewriteContext, strictCast); } else { DataType targetType = DataType.fromCatalogType(sameNameColumn.getType()); - addColumnValue(analyzer, optimizedRowConstructor, values.get(i), + NamedExpression value = resolveDefaultReferences(values.get(i), + writeOnlyDefaultColumns, table, unboundLogicalSink.getNameParts()); + addColumnValue(analyzer, optimizedRowConstructor, value, targetType, rewriteContext, strictCast, sameNameColumn); } } @@ -465,7 +477,9 @@ private static Plan normalizePlanWithoutLock(LogicalPlan plan, TableIf table, null, rewriteContext, strictCast); } else { DataType targetType = DataType.fromCatalogType(columns.get(i).getType()); - addColumnValue(analyzer, optimizedRowConstructor, values.get(i), targetType, + NamedExpression value = resolveDefaultReferences(values.get(i), + writeOnlyDefaultColumns, table, unboundLogicalSink.getNameParts()); + addColumnValue(analyzer, optimizedRowConstructor, value, targetType, rewriteContext, strictCast, columns.get(i)); } } @@ -477,15 +491,67 @@ private static Plan normalizePlanWithoutLock(LogicalPlan plan, TableIf table, } static List connectorWriteSchema(TableIf table, boolean full) { + return pinnedConnectorWriteSchema(table).orElseGet(() -> table.getBaseSchema(full)); + } + + private static Optional> pinnedConnectorWriteSchema(TableIf table) { ConnectContext context = ConnectContext.get(); - if (context != null && context.getStatementContext() != null) { - Optional> pinned = - context.getStatementContext().getConnectorWriteSchema(table.getId()); - if (pinned.isPresent()) { - return pinned.get(); - } + if (context == null || context.getStatementContext() == null) { + return Optional.empty(); } - return table.getBaseSchema(full); + return context.getStatementContext().getConnectorWriteSchema(table.getId()); + } + + /** + * The pinned writer schema of a target whose column defaults apply to writes only. The read schema of such a + * table carries no defaults, so DEFAULT(column) in an INSERT into it must resolve against this schema. + */ + private static Optional> writeOnlyDefaultColumns(TableIf table) { + if (!(table instanceof PluginDrivenExternalTable) + || !((PluginDrivenExternalTable) table).hasWriteOnlyDefaults()) { + return Optional.empty(); + } + return pinnedConnectorWriteSchema(table); + } + + /** Resolve DEFAULT(column) in one inserted value by name against the target's pinned writer schema. */ + private static NamedExpression resolveDefaultReferences(NamedExpression value, + Optional> writeOnlyDefaultColumns, TableIf table, List targetNameParts) { + if (!writeOnlyDefaultColumns.isPresent()) { + return value; + } + return (NamedExpression) ConnectorWriteSchemaUtils.resolveDefaultReferences(value, + writeOnlyDefaultColumns.get(), (ExternalTable) table, ConnectContext.get(), targetNameParts, null); + } + + /** + * Resolve DEFAULT(column) in the select lists of an INSERT ... SELECT by name against the target's pinned + * writer schema, before the source relations are bound, so it names a target column whatever the source is. + * A derived table in FROM is a query of its own: DEFAULT(column) in it names a column of its own FROM clause, + * as without this resolution, so it is left alone. + */ + private static Plan resolveSelectDefaultReferences(Plan query, Optional> writeOnlyDefaultColumns, + TableIf table, List targetNameParts) { + if (query instanceof LogicalSubQueryAlias) { + return query; + } + Plan plan = query.withChildren(child -> resolveSelectDefaultReferences( + child, writeOnlyDefaultColumns, table, targetNameParts)); + if (plan instanceof LogicalProject) { + LogicalProject project = (LogicalProject) plan; + return project.withProjects(project.getProjects().stream() + .map(projection -> resolveDefaultReferences( + projection, writeOnlyDefaultColumns, table, targetNameParts)) + .collect(ImmutableList.toImmutableList())); + } + if (plan instanceof UnboundOneRowRelation) { + UnboundOneRowRelation relation = (UnboundOneRowRelation) plan; + return new UnboundOneRowRelation(relation.getRelationId(), relation.getProjects().stream() + .map(projection -> resolveDefaultReferences( + projection, writeOnlyDefaultColumns, table, targetNameParts)) + .collect(ImmutableList.toImmutableList())); + } + return plan; } static void pinConnectorWriteSchema(StatementContext statementContext, TableIf targetTableIf, diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/RewriteDefaultExpressionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/RewriteDefaultExpressionTest.java new file mode 100644 index 00000000000000..025bfa459763fa --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/RewriteDefaultExpressionTest.java @@ -0,0 +1,137 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.expression.rules; + +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.datasource.plugin.PluginDrivenExternalTable; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.rules.expression.ExpressionRewriteTestHelper; +import org.apache.doris.nereids.rules.expression.ExpressionRuleExecutor; +import org.apache.doris.nereids.trees.expressions.Default; +import org.apache.doris.nereids.trees.expressions.ExprId; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.functions.scalar.If; +import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; +import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; +import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; +import org.apache.doris.nereids.types.IntegerType; +import org.apache.doris.nereids.util.TypeCoercionUtils; + +import com.google.common.collect.ImmutableList; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +public class RewriteDefaultExpressionTest extends ExpressionRewriteTestHelper { + + private static final long TABLE_ID = 17L; + private static final int FIELD_ID = 3; + + @BeforeEach + public void setUp() { + executor = new ExpressionRuleExecutor(ImmutableList.of(bottomUp(RewriteDefaultExpression.INSTANCE))); + } + + @Test + public void writeOnlyDefaultResolvesAgainstPinnedWriterSchema() { + // UPDATE t ... WHERE score = DEFAULT(score): the scanned column carries no write-only default, the writer + // schema pinned for the statement's target does. + SlotReference score = scoreSlot(true); + pinWriterColumn("9"); + + Expression rewritten = executor.rewrite(new Default(score), context); + + Assertions.assertEquals( + TypeCoercionUtils.castIfNotSameType(PARSER.parseExpression("9"), IntegerType.INSTANCE), rewritten); + } + + @Test + public void writeOnlyDefaultOfAWriterColumnOfAnotherTypeIsRejected() { + // The read schema can still hold the INT column after it was promoted to BIGINT outside Doris, while the + // writer schema pinned for the statement has the BIGINT column and its new default, which an INT cannot hold. + SlotReference score = scoreSlot(true); + pinWriterColumn(PrimitiveType.BIGINT, "4900000001"); + + AnalysisException e = Assertions.assertThrows(AnalysisException.class, + () -> executor.rewrite(new Default(score), context)); + + Assertions.assertTrue(e.getMessage().contains("DEFAULT(score) reads the column as INT but the table now writes" + + " it as BIGINT; the table changed outside Doris, refresh it and retry"), e.getMessage()); + } + + @Test + public void writeOnlyDefaultOfAnotherTypeIsRejectedInsideAnExpressionBoundToTheReadType() { + // UPDATE t ... WHERE IF(TRUE, DEFAULT(score), 0) = 4900000001: IF was bound with the INT of the read column, + // so substituting the BIGINT default would later fold into CAST(4900000001 AS INT), NULL in non-strict mode. + SlotReference score = scoreSlot(true); + pinWriterColumn(PrimitiveType.BIGINT, "4900000001"); + If ifExpr = new If(BooleanLiteral.TRUE, new Default(score), new IntegerLiteral(0)); + Assertions.assertEquals(IntegerType.INSTANCE, ifExpr.getDataType()); + + AnalysisException e = Assertions.assertThrows(AnalysisException.class, + () -> executor.rewrite(ifExpr, context)); + + Assertions.assertTrue(e.getMessage().contains("refresh it and retry"), e.getMessage()); + } + + @Test + public void writeOnlyDefaultOutsideTheWriteTargetIsRejected() { + // SELECT DEFAULT(score) FROM t writes nothing, so nothing carries the write-only default. + SlotReference score = scoreSlot(true); + + AnalysisException e = Assertions.assertThrows(AnalysisException.class, + () -> executor.rewrite(new Default(score), context)); + + Assertions.assertTrue(e.getMessage().contains( + "DEFAULT(score) must reference a column of the table that this statement writes"), e.getMessage()); + } + + @Test + public void readSchemaDefaultIsKeptWithoutWriteOnlyDefaults() { + // A connector whose read schema carries its defaults (hive) resolves from the scanned column as before. + SlotReference score = scoreSlot(false); + pinWriterColumn("9"); + + Expression rewritten = executor.rewrite(new Default(score), context); + + Assertions.assertEquals(new NullLiteral(IntegerType.INSTANCE), rewritten); + } + + private SlotReference scoreSlot(boolean writeOnlyDefaults) { + PluginDrivenExternalTable table = Mockito.mock(PluginDrivenExternalTable.class); + Mockito.when(table.getId()).thenReturn(TABLE_ID); + Mockito.when(table.hasWriteOnlyDefaults()).thenReturn(writeOnlyDefaults); + Column readColumn = new Column("score", PrimitiveType.INT, true); + readColumn.setUniqueId(FIELD_ID); + return SlotReference.fromColumn(new ExprId(1), table, readColumn, ImmutableList.of()); + } + + private void pinWriterColumn(String defaultSql) { + pinWriterColumn(PrimitiveType.INT, defaultSql); + } + + private void pinWriterColumn(PrimitiveType type, String defaultSql) { + Column writeColumn = new Column("score", type, true); + writeColumn.setUniqueId(FIELD_ID); + writeColumn.setConnectorDefaultValueSql(defaultSql); + cascadesContext.getStatementContext().setConnectorWriteSchema(TABLE_ID, ImmutableList.of(writeColumn)); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ConnectorWriteSchemaUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ConnectorWriteSchemaUtilsTest.java index b61a51c1eadf08..32ce0f8827caca 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ConnectorWriteSchemaUtilsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ConnectorWriteSchemaUtilsTest.java @@ -36,8 +36,8 @@ import org.apache.doris.nereids.trees.expressions.literal.DecimalV3Literal; import org.apache.doris.nereids.trees.expressions.literal.DoubleLiteral; import org.apache.doris.nereids.trees.expressions.literal.FloatLiteral; +import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; import org.apache.doris.nereids.trees.expressions.literal.Literal; -import org.apache.doris.nereids.trees.expressions.literal.TinyIntLiteral; import org.apache.doris.nereids.types.DataType; import org.apache.doris.nereids.types.DecimalV3Type; import org.apache.doris.nereids.util.TypeCoercionUtils; @@ -96,12 +96,13 @@ public void connectorNonFiniteFloatingDefaultsParseAsTypedExpressions() { @Test public void defaultReferencesResolveAgainstPinnedWriteColumns() { - // DEFAULT(column) in DML resolves the statement-pinned write default; a name absent from the pinned writer - // schema is an error rather than a guess. + // DEFAULT(column) in DML resolves the statement-pinned write default, typed as its INT column rather than + // as the TINYINT the literal parses to; a name absent from the pinned writer schema is an error rather than + // a guess. ExternalTable target = targetTable(); List pinned = ImmutableList.of(pinnedColumn("score", 1, "9")); - Assertions.assertEquals(new TinyIntLiteral((byte) 9), ConnectorWriteSchemaUtils.resolveDefaultReferences( + Assertions.assertEquals(new IntegerLiteral(9), ConnectorWriteSchemaUtils.resolveDefaultReferences( new Default(new UnboundSlot("score")), pinned, target, ConnectContext.get(), TARGET_NAME_PARTS, null)); AnalysisException exception = Assertions.assertThrows(AnalysisException.class, @@ -121,7 +122,7 @@ public void defaultOfBoundSlotDoesNotFallBackToReusedColumnName() { Column replacement = new Column("reused_name", ScalarType.createType(PrimitiveType.INT)); replacement.setUniqueId(8); - Assertions.assertEquals(new TinyIntLiteral((byte) 9), ConnectorWriteSchemaUtils.resolveDefaultReferences( + Assertions.assertEquals(new IntegerLiteral(9), ConnectorWriteSchemaUtils.resolveDefaultReferences( new Default(boundSlot(target, pinnedField)), pinned, target, ConnectContext.get(), TARGET_NAME_PARTS, null)); AnalysisException exception = Assertions.assertThrows(AnalysisException.class, diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtilsTest.java index e624fecc4b0eb5..b0b9c326a46afb 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtilsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtilsTest.java @@ -28,17 +28,35 @@ import org.apache.doris.nereids.analyzer.UnboundAlias; import org.apache.doris.nereids.analyzer.UnboundConnectorTableSink; import org.apache.doris.nereids.analyzer.UnboundFunction; +import org.apache.doris.nereids.analyzer.UnboundInlineTable; +import org.apache.doris.nereids.analyzer.UnboundOneRowRelation; +import org.apache.doris.nereids.analyzer.UnboundSlot; import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.parser.NereidsParser; +import org.apache.doris.nereids.trees.expressions.Default; +import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; import org.apache.doris.nereids.trees.expressions.literal.TinyIntLiteral; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.RelationId; +import org.apache.doris.nereids.trees.plans.logical.LogicalInlineTable; +import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.trees.plans.logical.LogicalSubQueryAlias; +import org.apache.doris.nereids.trees.plans.logical.LogicalUnion; import org.apache.doris.nereids.trees.plans.logical.UnboundLogicalSink; +import org.apache.doris.nereids.types.IntegerType; +import org.apache.doris.nereids.util.MemoTestUtils; +import org.apache.doris.nereids.util.TypeCoercionUtils; +import org.apache.doris.qe.ConnectContext; import com.google.common.collect.ImmutableList; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.mockito.Mockito; +import java.util.List; import java.util.Optional; /** @@ -47,6 +65,7 @@ public class InsertUtilsTest { private static final int MAX_TOTAL_BYTES = 512; + private static final NereidsParser PARSER = new NereidsParser(); private String generateString(int length) { return generateString(length, "X"); @@ -434,6 +453,177 @@ public void explicitInvisibleColumnUsesInvisibleColumnErrorWithDataOnlyWriterSch "Cannot specify invisible column '_row_id' in INSERT statement", error.getMessage()); } + @Test + public void writeOnlyDefaultInValuesResolvesAgainstPinnedWriterSchema() { + // insert into t (score, id) values (DEFAULT(score), 4): the read schema of a table with write-only + // defaults has no default, so DEFAULT(score) must take the pinned writer column's default. + ConnectContext ctx = MemoTestUtils.createConnectContext(); + try { + PluginDrivenExternalTable table = pinWriterSchema(ctx, true); + Plan normalized = InsertUtils.normalizePlan(connectorSink(ImmutableList.of("score", "id"), + new UnboundInlineTable(ImmutableList.of(ImmutableList.of( + new UnboundAlias(new Default(new UnboundSlot("score"))), + new UnboundAlias(new IntegerLiteral(4)))))), + table, Optional.empty(), Optional.empty()); + + NamedExpression score = ((LogicalInlineTable) normalized.child(0)).getConstantExprsList().get(0).get(0); + Assertions.assertEquals(TypeCoercionUtils.castUnbound(PARSER.parseExpression("9"), IntegerType.INSTANCE), + score.child(0)); + + AnalysisException unknown = Assertions.assertThrows(AnalysisException.class, + () -> InsertUtils.normalizePlan(connectorSink(ImmutableList.of("id", "score"), + new UnboundInlineTable(ImmutableList.of(ImmutableList.of( + new UnboundAlias(new IntegerLiteral(6)), + new UnboundAlias(new Default(new UnboundSlot("no_such_column"))))))), + table, Optional.empty(), Optional.empty())); + Assertions.assertTrue(unknown.getMessage().contains("no_such_column"), unknown.getMessage()); + } finally { + ConnectContext.remove(); + } + } + + @Test + public void writeOnlyDefaultInSelectResolvesByTargetColumnName() { + // DEFAULT(name) in INSERT ... SELECT names the target column whatever the source is: here a staging + // relation whose own column is also called name, and a SELECT without FROM. + ConnectContext ctx = MemoTestUtils.createConnectContext(); + try { + PluginDrivenExternalTable table = pinWriterSchema(ctx, true); + Expression writeDefault = PARSER.parseExpression("'write-name'"); + + Plan fromStaging = InsertUtils.normalizePlan(connectorSink(ImmutableList.of("id", "name"), + PARSER.parseSingle("select id, DEFAULT(name) from " + + "(select 7 as id, 'source-name' as name) staging_source").child(0)), + table, Optional.empty(), Optional.empty()); + LogicalProject project = (LogicalProject) fromStaging.child(0); + Assertions.assertEquals(writeDefault, project.getProjects().get(1).child(0)); + + Plan noFrom = InsertUtils.normalizePlan(connectorSink(ImmutableList.of("id", "name"), + new UnboundOneRowRelation(new RelationId(1), ImmutableList.of( + new UnboundAlias(new IntegerLiteral(8)), + new UnboundAlias(new Default(new UnboundSlot("name")))))), + table, Optional.empty(), Optional.empty()); + UnboundOneRowRelation oneRow = (UnboundOneRowRelation) noFrom.child(0); + Assertions.assertEquals(writeDefault, oneRow.getProjects().get(1).child(0)); + } finally { + ConnectContext.remove(); + } + } + + @Test + public void writeOnlyDefaultInDerivedTableNamesItsOwnSource() { + // INSERT INTO t(score, name) SELECT v, DEFAULT(name) FROM (SELECT DEFAULT(score) AS v FROM src) q: the + // derived table is a query of its own, so its DEFAULT(score) names src.score, whose default can differ from + // the target's 9, and is left to binding, qualified or not. DEFAULT in the INSERT's own select lists, set + // operation branches included, still names the target column. + ConnectContext ctx = MemoTestUtils.createConnectContext(); + try { + PluginDrivenExternalTable table = pinWriterSchema(ctx, true); + Expression writeDefault = PARSER.parseExpression("'write-name'"); + for (String derived : ImmutableList.of("(select DEFAULT(score) as v from src) q", + "(select DEFAULT(s.score) as v from src s) q")) { + Plan normalized = InsertUtils.normalizePlan(connectorSink(ImmutableList.of("score", "name"), + PARSER.parseSingle("select v, DEFAULT(name) from " + derived).child(0)), + table, Optional.empty(), Optional.empty()); + LogicalProject outer = (LogicalProject) normalized.child(0); + Assertions.assertEquals(writeDefault, outer.getProjects().get(1).child(0), derived); + assertDerivedDefaultLeftToBinding(outer.child(0), derived); + } + + Plan union = InsertUtils.normalizePlan(connectorSink(ImmutableList.of("name"), + PARSER.parseSingle("select DEFAULT(name) union all " + + "select v from (select DEFAULT(name) as v from src) q").child(0)), + table, Optional.empty(), Optional.empty()); + LogicalUnion setOperation = (LogicalUnion) union.child(0); + UnboundOneRowRelation oneRow = (UnboundOneRowRelation) setOperation.child(0); + Assertions.assertEquals(writeDefault, oneRow.getProjects().get(0).child(0)); + assertDerivedDefaultLeftToBinding(setOperation.child(1).child(0), "union branch"); + } finally { + ConnectContext.remove(); + } + } + + private static void assertDerivedDefaultLeftToBinding(Plan derivedTable, String message) { + Assertions.assertInstanceOf(LogicalSubQueryAlias.class, derivedTable, message); + LogicalProject inner = (LogicalProject) derivedTable.child(0); + Assertions.assertInstanceOf(Default.class, inner.getProjects().get(0).child(0), message); + } + + @Test + public void defaultReferenceIsLeftToBindingWithoutWriteOnlyDefaults() { + // A connector whose read schema carries its defaults (hive) keeps the existing DEFAULT(column) handling. + ConnectContext ctx = MemoTestUtils.createConnectContext(); + try { + PluginDrivenExternalTable table = pinWriterSchema(ctx, false); + Plan normalized = InsertUtils.normalizePlan(connectorSink(ImmutableList.of("id", "name"), + new UnboundOneRowRelation(new RelationId(1), ImmutableList.of( + new UnboundAlias(new IntegerLiteral(8)), + new UnboundAlias(new Default(new UnboundSlot("name")))))), + table, Optional.empty(), Optional.empty()); + + UnboundOneRowRelation oneRow = (UnboundOneRowRelation) normalized.child(0); + Assertions.assertInstanceOf(Default.class, oneRow.getProjects().get(1).child(0)); + } finally { + ConnectContext.remove(); + } + } + + @Test + public void writeOnlyDefaultIsTypedAsItsColumnBeforeBinding() { + // bit_count(DEFAULT(bits)) on an INT column whose default is -1 must count the 32 bits of an INT, not the 8 + // of the TINYINT that -1 parses to; casting the finished value to the INT target cannot restore them. + ConnectContext ctx = MemoTestUtils.createConnectContext(); + try { + PluginDrivenExternalTable table = pinWriterSchema(ctx, true); + Expression intDefault = TypeCoercionUtils.castUnbound(PARSER.parseExpression("-1"), IntegerType.INSTANCE); + + Expression bitCount = new UnboundFunction("bit_count", + ImmutableList.of(new Default(new UnboundSlot("bits")))); + Plan values = InsertUtils.normalizePlan(connectorSink(ImmutableList.of("bits"), + new UnboundInlineTable(ImmutableList.of(ImmutableList.of(new UnboundAlias(bitCount))))), + table, Optional.empty(), Optional.empty()); + Assertions.assertEquals(intDefault, bitCountArgument( + ((LogicalInlineTable) values.child(0)).getConstantExprsList().get(0).get(0))); + + Plan select = InsertUtils.normalizePlan(connectorSink(ImmutableList.of("bits"), + PARSER.parseSingle("select bit_count(DEFAULT(bits))").child(0)), + table, Optional.empty(), Optional.empty()); + Assertions.assertEquals(intDefault, bitCountArgument( + ((UnboundOneRowRelation) select.child(0)).getProjects().get(0))); + Assertions.assertEquals(IntegerType.INSTANCE, intDefault.getDataType()); + } finally { + ConnectContext.remove(); + } + } + + private static Expression bitCountArgument(Expression value) { + UnboundFunction bitCount = value.collectFirst(UnboundFunction.class::isInstance).get(); + return bitCount.child(0); + } + + private static PluginDrivenExternalTable pinWriterSchema(ConnectContext ctx, boolean writeOnlyDefaults) { + PluginDrivenExternalTable table = Mockito.mock(PluginDrivenExternalTable.class); + Mockito.when(table.getId()).thenReturn(17L); + Mockito.when(table.getName()).thenReturn("t"); + Mockito.when(table.hasWriteOnlyDefaults()).thenReturn(writeOnlyDefaults); + Column id = new Column("id", PrimitiveType.INT, true); + id.setConnectorDefaultValueSql("NULL"); + Column name = new Column("name", PrimitiveType.STRING, true); + name.setConnectorDefaultValueSql("'write-name'"); + Column score = new Column("score", PrimitiveType.INT, true); + score.setConnectorDefaultValueSql("9"); + Column bits = new Column("bits", PrimitiveType.INT, true); + bits.setConnectorDefaultValueSql("-1"); + MemoTestUtils.createStatementContext(ctx, "") + .setConnectorWriteSchema(17L, ImmutableList.of(id, name, score, bits)); + return table; + } + + private static UnboundConnectorTableSink connectorSink(List columnNames, Plan query) { + return new UnboundConnectorTableSink<>(ImmutableList.of("ctl", "db", "t"), columnNames, + ImmutableList.of(), ImmutableList.of(), query); + } + @Test public void nativeExplicitInvisibleColumnRemainsSupported() { Column id = new Column("id", PrimitiveType.INT); diff --git a/regression-test/data/external_table_p0/iceberg/test_iceberg_initial_defaults.out b/regression-test/data/external_table_p0/iceberg/test_iceberg_initial_defaults.out index d2945f45cf8df0..a5ddf7fb1a88e6 100644 --- a/regression-test/data/external_table_p0/iceberg/test_iceberg_initial_defaults.out +++ b/regression-test/data/external_table_p0/iceberg/test_iceberg_initial_defaults.out @@ -387,6 +387,16 @@ missing_map_col map> Yes true \N 3 123e4567-e89b-12d3-a456-426614174000 0A0B0C0D 2A2B2C 2024-12-17 23:59:59.123456+00:00 4 123e4567-e89b-12d3-a456-426614174000 0A0B0C0D 2A2B2C 2024-12-17 23:59:59.123456+00:00 +-- !parquet_default_references -- +20 35 update-where-default +21 35 update-where-default +22 35 merge-on-default + +-- !orc_default_references -- +20 35 update-where-default +21 35 update-where-default +22 35 merge-on-default + -- !parquet_branch_overwrite_default -- 14 36 diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_initial_defaults.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_initial_defaults.groovy index 2ddeb9891a9a7c..e3afd11115f700 100644 --- a/regression-test/suites/external_table_p0/iceberg/test_iceberg_initial_defaults.groovy +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_initial_defaults.groovy @@ -599,6 +599,39 @@ suite("test_iceberg_initial_defaults", "p0,external") { sparkUpdateMergeVerification) } + // DEFAULT(column) elsewhere in a write must also resolve to the pinned write-default (35); the read + // schema carries none. Rows 20-22 come from VALUES and INSERT ... SELECT with and without FROM, then the + // UPDATE and MERGE predicates must match all of them. A plain SELECT writes nothing and is rejected. + tableNames.each { String format, String currentTable -> + sql """INSERT INTO ${currentTable} (id, default_int) VALUES (20, DEFAULT(default_int))""" + sql """ + INSERT INTO ${currentTable} (id, default_int) + SELECT 21, DEFAULT(default_int) FROM ${currentTable} WHERE id = 7 + """ + sql """INSERT INTO ${currentTable} (id, default_int) SELECT 22, DEFAULT(default_int)""" + sql """ + UPDATE ${currentTable} + SET default_string = 'update-where-default' + WHERE id IN (20, 21, 22) AND default_int = DEFAULT(default_int) + """ + sql """ + MERGE INTO ${currentTable} t + USING (SELECT 22 AS id) s + ON t.id = s.id AND t.default_int = DEFAULT(t.default_int) + WHEN MATCHED THEN UPDATE SET default_string = 'merge-on-default' + """ + "order_qt_${format}_default_references" """ + SELECT id, default_int, default_string + FROM ${currentTable} + WHERE id IN (20, 21, 22) + ORDER BY id + """ + } + test { + sql """SELECT DEFAULT(default_int) FROM ${parquetTable}""" + exception "must reference a column of the table that this statement writes" + } + // Pin a branch while default_int still has write-default 35, then evolve the table-current // schema to 36. Iceberg selects the branch as the parent snapshot but writes the new snapshot // with the table-current schema, so VALUES(DEFAULT) normalization and the writer must both use