From 43b09bfe6e5effce285d97a875eb70691c9c4495 Mon Sep 17 00:00:00 2001 From: morningman Date: Thu, 22 Oct 2026 21:43:36 +0800 Subject: [PATCH 1/5] [fix](fe) Resolve DEFAULT(column) on Iceberg tables from the write schema ### What problem does this PR solve? Issue Number: None Related PR: #66538, #57531, #65851 Problem Summary: An Iceberg write-default exists only in the writer schema that a write statement pins (IcebergWriteSchemaContext); the columns of the read schema carry no default. Master resolves DEFAULT(column) against that writer schema only in UPDATE / MERGE SET and in MERGE INSERT VALUES, and everywhere else falls back to the read column. On a v3 table whose column d has write-default 42, INSERT ... VALUES (DEFAULT(d)) and INSERT ... SELECT DEFAULT(d) without FROM fail with "Unknown column", INSERT ... SELECT DEFAULT(d) FROM t writes NULL, UPDATE ... WHERE d = DEFAULT(d) and MERGE ... ON ... d = DEFAULT(d) compare d with NULL and silently match no row, and SELECT DEFAULT(d) FROM t returns NULL. branch-4.1 and branch-4.2 (#66538) resolve all of these against the pinned write target, by name against the target in INSERT ... SELECT, and reject DEFAULT(column) on an Iceberg column in a statement that does not write the table. This change adds the connector capability SUPPORTS_WRITE_ONLY_DEFAULTS, declared by the Iceberg connector and inherited by iceberg-on-HMS tables. For a table that declares it, InsertUtils resolves DEFAULT(column) in INSERT VALUES and in the select lists of INSERT ... SELECT by name against the pinned writer schema, and RewriteDefaultExpression takes the default of a DEFAULT(column) bound to the table from the pinned writer column with the same field id, which covers the UPDATE and MERGE conditions, and rejects it when the statement pinned no writer schema for the table. Hive, Paimon and OLAP tables keep the current behavior. The new enum constant changes the connector plugin API surface, so the plugin API version goes from 21.0 to 22.0. ### Release note Fix: On Iceberg tables, DEFAULT(column) in INSERT VALUES, INSERT ... SELECT and the WHERE / ON conditions of UPDATE and MERGE now resolves to the column's write-default. Before, it failed, wrote NULL, or matched no row. SELECT DEFAULT(column) on an Iceberg table now fails instead of returning NULL. ### Check List (For Author) - Test: Regression test / Unit Test - test_iceberg_initial_defaults: writes rows 20-22 with DEFAULT(default_int) in VALUES and in INSERT ... SELECT with and without FROM, then updates them with UPDATE ... WHERE default_int = DEFAULT(default_int) and MERGE ... ON ... = DEFAULT(...), for Parquet and ORC; SELECT DEFAULT(default_int) must fail. Passes locally; with master's FE it fails at the first INSERT with "Unknown column 'default_int'" - InsertUtilsTest: VALUES and INSERT ... SELECT (from a staging relation and without FROM) resolve against the pinned writer schema; unknown column fails; a table without the capability is unchanged - RewriteDefaultExpressionTest: pinned writer column wins, no pinned schema fails, a table without the capability keeps the read default - IcebergConnectorTest, HiveConnectorMetadataSiblingDelegationTest, ConnectorPluginSurfaceTest - Behavior changed: Yes. See the release note. - Does this need documentation: No Co-Authored-By: Claude Opus 5.5 (1M context) --- .../connector/hive/HiveConnectorMetadata.java | 5 +- ...onnectorMetadataSiblingDelegationTest.java | 6 +- .../connector/iceberg/IcebergConnector.java | 7 +- .../iceberg/IcebergConnectorTest.java | 12 ++ .../connector/spi/ConnectorCapability.java | 17 ++- .../spi/ConnectorPluginSurfaceTest.java | 5 +- .../resources/connector-plugin-surface.txt | 1 + fe/fe-connector/pom.xml | 2 +- .../plugin/PluginDrivenExternalTable.java | 11 ++ .../rules/RewriteDefaultExpression.java | 22 ++++ .../plans/commands/insert/InsertUtils.java | 78 ++++++++++-- .../rules/RewriteDefaultExpressionTest.java | 101 ++++++++++++++++ .../commands/insert/InsertUtilsTest.java | 114 ++++++++++++++++++ .../iceberg/test_iceberg_initial_defaults.out | 10 ++ .../test_iceberg_initial_defaults.groovy | 33 +++++ 15 files changed, 407 insertions(+), 17 deletions(-) create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/RewriteDefaultExpressionTest.java diff --git a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnectorMetadata.java b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnectorMetadata.java index 4c8f56c58b5162..1124f93c378984 100644 --- a/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnectorMetadata.java +++ b/fe/fe-connector/fe-connector-hive/src/main/java/org/apache/doris/connector/hive/HiveConnectorMetadata.java @@ -626,7 +626,8 @@ public ConnectorTableSchema getTableSchema( ConnectorCapability.SUPPORTS_NESTED_COLUMN_PRUNE, ConnectorCapability.SUPPORTS_FIELD_ID_ACCESS_PATH, ConnectorCapability.SUPPORTS_STORAGE_PREDICATE_PRUNING, - ConnectorCapability.SUPPORTS_NESTED_COLUMN_SCHEMA_CHANGE)); + ConnectorCapability.SUPPORTS_NESTED_COLUMN_SCHEMA_CHANGE, + ConnectorCapability.SUPPORTS_WRITE_ONLY_DEFAULTS)); /** * Reflects the {@link #SIBLING_INHERITABLE_CAPABILITIES} subset of the owning sibling connector's @@ -636,7 +637,7 @@ public ConnectorTableSchema getTableSchema( * (hive) connector-wide set OR the table's own set, and NEVER consults the sibling connector directly, so * without this reflection an iceberg-on-HMS table would silently lose every such capability the iceberg * sibling declares connector-wide (auto-analyze / Top-N lazy / nested-column prune / storage predicate - * pruning / nested column DDL). + * pruning / nested column DDL / write-only defaults). * Returns the sibling schema unchanged when nothing is inherited and the sibling declared nothing itself * (e.g. a hudi sibling, which declares no capabilities at all). */ diff --git a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorMetadataSiblingDelegationTest.java b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorMetadataSiblingDelegationTest.java index d2f340f0f316fb..77a7b1529ebb59 100644 --- a/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorMetadataSiblingDelegationTest.java +++ b/fe/fe-connector/fe-connector-hive/src/test/java/org/apache/doris/connector/hive/HiveConnectorMetadataSiblingDelegationTest.java @@ -525,7 +525,8 @@ public void foreignHandleSchemaReflectsSiblingScanCapabilitiesAsPerTableMarker() ConnectorCapability.SUPPORTS_TOPN_LAZY_MATERIALIZE, ConnectorCapability.SUPPORTS_NESTED_COLUMN_PRUNE, ConnectorCapability.SUPPORTS_FIELD_ID_ACCESS_PATH, - ConnectorCapability.SUPPORTS_STORAGE_PREDICATE_PRUNING); + ConnectorCapability.SUPPORTS_STORAGE_PREDICATE_PRUNING, + ConnectorCapability.SUPPORTS_WRITE_ONLY_DEFAULTS); HiveConnectorMetadata md = new HiveConnectorMetadata(null, HiveTestProperties.minimal(), new FakeConnectorContext(), SUPPLIER_MUST_NOT_BE_USED, SUPPLIER_MUST_NOT_BE_USED, handle -> new SiblingOwner(new CapabilityDeclaringSiblingConnector(siblingCaps), @@ -533,6 +534,9 @@ public void foreignHandleSchemaReflectsSiblingScanCapabilitiesAsPerTableMarker() ConnectorTableSchema schema = md.getTableSchema(session, foreignHandle); Set reflected = schema.getTableCapabilities(); + Assertions.assertTrue(reflected.contains(ConnectorCapability.SUPPORTS_WRITE_ONLY_DEFAULTS), + "an iceberg-on-HMS table must inherit write-only defaults, or DEFAULT(column) on it answers " + + "NULL from the read schema"); Assertions.assertTrue(reflected.contains(ConnectorCapability.SUPPORTS_COLUMN_AUTO_ANALYZE), "auto-analyze must survive the delegation as a per-table capability"); Assertions.assertTrue(reflected.contains(ConnectorCapability.SUPPORTS_TOPN_LAZY_MATERIALIZE), diff --git a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnector.java b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnector.java index 2d372f21eadb0d..ba5df0232cf70b 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnector.java +++ b/fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergConnector.java @@ -911,6 +911,10 @@ public Set getCapabilities() { // (CREATE TABLE ... ORDER BY (...)). fe-core admits the ORDER BY clause for a plugin-driven CREATE TABLE // ONLY under this capability (replacing the legacy engine-name "iceberg" gate); the sort-column validation // (existence / type / duplicates) runs connector-side in IcebergConnectorMetadata.createTable. + // SUPPORTS_WRITE_ONLY_DEFAULTS: an iceberg write-default applies to new rows only, so it lives in the + // writer schema (IcebergWriteSchemaContext) and not in the read schema. fe-core resolves DEFAULT(column) + // against the writer schema pinned for the write target and rejects it in a plain SELECT, instead of + // answering NULL from the read schema. EnumSet capabilities = EnumSet.of(ConnectorCapability.SUPPORTS_MVCC_SNAPSHOT, ConnectorCapability.SUPPORTS_COLUMN_AUTO_ANALYZE, ConnectorCapability.SUPPORTS_TOPN_LAZY_MATERIALIZE, @@ -921,7 +925,8 @@ public Set getCapabilities() { ConnectorCapability.SUPPORTS_STORAGE_PREDICATE_PRUNING, ConnectorCapability.SUPPORTS_METADATA_PRELOAD, ConnectorCapability.SUPPORTS_SORT_ORDER, - ConnectorCapability.SUPPORTS_NESTED_COLUMN_SCHEMA_CHANGE); + ConnectorCapability.SUPPORTS_NESTED_COLUMN_SCHEMA_CHANGE, + ConnectorCapability.SUPPORTS_WRITE_ONLY_DEFAULTS); // SUPPORTS_USER_SESSION: only a REST catalog configured iceberg.rest.session=user projects the querying // user's delegated credential onto a per-request Iceberg REST SessionCatalog (#63068 re-migration). This // gates FE credential injection + shared-cache bypass; every other flavor/config authenticates with a diff --git a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorTest.java b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorTest.java index ee671040606e22..09c99edc77a1ac 100644 --- a/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorTest.java +++ b/fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergConnectorTest.java @@ -200,6 +200,18 @@ public void declaresColumnAutoAnalyzeAndTopNLazyMaterializeCapabilities() { + "materialization"); } + @Test + public void declaresWriteOnlyDefaultsCapability() { + // WHY: an iceberg write-default is carried by the writer schema only, so the read schema's columns have no + // default. fe-core resolves DEFAULT(column) against the pinned writer schema (and rejects it in a plain + // SELECT) ONLY under this capability. MUTATION: dropping the capability -> INSERT ... SELECT DEFAULT(c) + // writes NULL and UPDATE ... WHERE c = DEFAULT(c) silently matches nothing -> red. + IcebergConnector connector = new IcebergConnector(Collections.emptyMap(), new RecordingConnectorContext()); + Assertions.assertTrue(connector.getCapabilities() + .contains(ConnectorCapability.SUPPORTS_WRITE_ONLY_DEFAULTS), + "iceberg must declare SUPPORTS_WRITE_ONLY_DEFAULTS so DEFAULT(column) resolves to the write default"); + } + @Test public void declaresShowCreateDdlCapability() { // WHY: legacy IcebergExternalTable rendered LOCATION + PROPERTIES + PARTITION BY + ORDER BY in SHOW diff --git a/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/ConnectorCapability.java b/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/ConnectorCapability.java index 97e8d5461544d0..2c626b5fd1d971 100644 --- a/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/ConnectorCapability.java +++ b/fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/ConnectorCapability.java @@ -304,5 +304,20 @@ public enum ConnectorCapability { * data tables while declining it on the subset of system tables whose readers cannot observe a * selected snapshot.

*/ - SUPPORTS_SCAN_PARAM_OPTIONS + SUPPORTS_SCAN_PARAM_OPTIONS, + /** + * Indicates the connector's column defaults apply to writes only: they are carried by the + * request-scoped writer schema ({@code ConnectorWritePlanProvider.getWriteColumns}), not by the read + * schema (e.g. Iceberg {@code write-default}). + * + *

fe-core then resolves {@code DEFAULT(column)} against the writer schema a statement pinned for its + * write target: in INSERT VALUES, by name against the target in INSERT ... SELECT, and in the + * predicates of UPDATE / MERGE on the target. Anywhere else, e.g. in a plain SELECT, {@code + * DEFAULT(column)} on such a table is rejected instead of answering NULL from the read schema. + * Connectors whose read schema carries the column defaults (hive) must NOT declare it.

+ * + *

Scope: catalog-wide OR per-table. An iceberg-on-HMS table inherits it through the per-table + * set the gateway reflects from its sibling, exactly like {@link #SUPPORTS_NESTED_COLUMN_PRUNE}.

+ */ + SUPPORTS_WRITE_ONLY_DEFAULTS } diff --git a/fe/fe-connector/fe-connector-spi/src/test/java/org/apache/doris/connector/spi/ConnectorPluginSurfaceTest.java b/fe/fe-connector/fe-connector-spi/src/test/java/org/apache/doris/connector/spi/ConnectorPluginSurfaceTest.java index cab3d00cb4b4be..a637b4ae597ffc 100644 --- a/fe/fe-connector/fe-connector-spi/src/test/java/org/apache/doris/connector/spi/ConnectorPluginSurfaceTest.java +++ b/fe/fe-connector/fe-connector-spi/src/test/java/org/apache/doris/connector/spi/ConnectorPluginSurfaceTest.java @@ -89,8 +89,9 @@ public void connectorApiMajorTracksTheRecordedSurfaceChange() throws IOException version.load(in); } // Major 20 adds deleteUsingMetadata to the connector write surface; major 21 adds - // typed filesystem credential descriptions to that surface. - Assertions.assertEquals("21.0", version.getProperty("api.version")); + // typed filesystem credential descriptions to that surface; major 22 adds the + // ConnectorCapability#SUPPORTS_WRITE_ONLY_DEFAULTS constant. + Assertions.assertEquals("22.0", version.getProperty("api.version")); } /** Root entry points plus provider/handle types returned to connector plugins. */ diff --git a/fe/fe-connector/fe-connector-spi/src/test/resources/connector-plugin-surface.txt b/fe/fe-connector/fe-connector-spi/src/test/resources/connector-plugin-surface.txt index 81918e6daa1431..4bc29adfec6fbf 100644 --- a/fe/fe-connector/fe-connector-spi/src/test/resources/connector-plugin-surface.txt +++ b/fe/fe-connector/fe-connector-spi/src/test/resources/connector-plugin-surface.txt @@ -35,6 +35,7 @@ org.apache.doris.connector.spi.ConnectorCapability#enum:SUPPORTS_SYS_TABLE_NESTE org.apache.doris.connector.spi.ConnectorCapability#enum:SUPPORTS_TOPN_LAZY_MATERIALIZE org.apache.doris.connector.spi.ConnectorCapability#enum:SUPPORTS_USER_SESSION org.apache.doris.connector.spi.ConnectorCapability#enum:SUPPORTS_VIEW +org.apache.doris.connector.spi.ConnectorCapability#enum:SUPPORTS_WRITE_ONLY_DEFAULTS org.apache.doris.connector.spi.ConnectorColumn#equals(java.lang.Object):boolean org.apache.doris.connector.spi.ConnectorColumn#getComment():java.lang.String org.apache.doris.connector.spi.ConnectorColumn#getDefaultValue():java.lang.String diff --git a/fe/fe-connector/pom.xml b/fe/fe-connector/pom.xml index 1d91dc0c412092..55ea25afcf097f 100644 --- a/fe/fe-connector/pom.xml +++ b/fe/fe-connector/pom.xml @@ -55,7 +55,7 @@ under the License. of the latter two means bumping this property as well (and fe-extension-spi means bumping all five families). --> - 21.0 + 22.0 diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/plugin/PluginDrivenExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/plugin/PluginDrivenExternalTable.java index 6cbea40af7505b..29cdb3044327b7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/plugin/PluginDrivenExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/plugin/PluginDrivenExternalTable.java @@ -446,6 +446,17 @@ public boolean supportsSampleAnalyze() { return hasCapability(ConnectorCapability.SUPPORTS_SAMPLE_ANALYZE); } + /** + * Returns whether THIS table's column defaults apply to writes only, i.e. only the request-scoped writer + * schema ({@link #resolveWriteColumns}) carries them. {@code InsertUtils} and {@code RewriteDefaultExpression} + * then resolve {@code DEFAULT(column)} against the writer schema a write statement pinned, and reject it + * anywhere else instead of answering NULL from the read schema. Resolved per-table via {@link #hasCapability} + * so an iceberg-on-HMS table inherits it, mirroring {@link #supportsNestedColumnSchemaChange}. + */ + public boolean hasWriteOnlyDefaults() { + return hasCapability(ConnectorCapability.SUPPORTS_WRITE_ONLY_DEFAULTS); + } + /** * Whether this table supports a table-scoped capability, resolved connector-wide OR per-table. A * uniform-format connector (iceberg — every table orc/parquet) declares the capability for all its tables diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/RewriteDefaultExpression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/RewriteDefaultExpression.java index 6dc8ab7c72ccce..48e74e55a4a56e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/RewriteDefaultExpression.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/RewriteDefaultExpression.java @@ -18,7 +18,10 @@ package org.apache.doris.nereids.rules.expression.rules; import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.datasource.plugin.PluginDrivenExternalTable; import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.analyzer.UnboundFunction; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.parser.NereidsParser; @@ -46,6 +49,8 @@ * 1. DEFAULT(column) with default value -> the actual default value expression * 2. DEFAULT(column) with no default value but nullable -> NULL * 3. DEFAULT(column) with no default value and not nullable -> error + * A table whose defaults apply to writes only takes them from the writer schema that the statement + * pinned for its write target; DEFAULT(column) on any other such table is an error. */ public class RewriteDefaultExpression implements ExpressionPatternRuleFactory { @@ -82,6 +87,15 @@ private static Expression rewrite(ExpressionMatchingContext context) { + column.getName() + "'"); } + Optional originalTable = slotRef.getOriginalTable(); + if (originalTable.filter(PluginDrivenExternalTable.class::isInstance) + .filter(table -> ((PluginDrivenExternalTable) table).hasWriteOnlyDefaults()) + .isPresent()) { + column = pinnedWriteColumn(context.cascadesContext.getStatementContext(), originalTable.get(), column) + .orElseThrow(() -> new AnalysisException("DEFAULT(" + slotRef.toSql() + + ") must reference a column of the table that this statement writes")); + } + String defaultValueSql = column.getDefaultValueSql(); if (defaultValueSql == null) { if (column.isAllowNull()) { @@ -101,4 +115,12 @@ private static Expression rewrite(ExpressionMatchingContext context) { return TypeCoercionUtils.castIfNotSameType(defaultValueExpr, targetType); } + + private static Optional pinnedWriteColumn(StatementContext statementContext, TableIf table, + Column readColumn) { + return statementContext.getConnectorWriteSchema(table.getId()) + .flatMap(columns -> columns.stream() + .filter(column -> column.getUniqueId() == readColumn.getUniqueId()) + .findFirst()); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java index 36ee0bcddc8765..7366dd1b1970ed 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java @@ -29,6 +29,7 @@ import org.apache.doris.catalog.TableIf; import org.apache.doris.common.Config; import org.apache.doris.common.util.DebugPointUtil; +import org.apache.doris.datasource.ExternalTable; import org.apache.doris.datasource.connector.converter.ConnectorWriteValueConverter; import org.apache.doris.datasource.plugin.PluginDrivenExternalTable; import org.apache.doris.foundation.format.FormatOptions; @@ -41,6 +42,7 @@ import org.apache.doris.nereids.analyzer.UnboundDictionarySink; import org.apache.doris.nereids.analyzer.UnboundFunction; import org.apache.doris.nereids.analyzer.UnboundInlineTable; +import org.apache.doris.nereids.analyzer.UnboundOneRowRelation; import org.apache.doris.nereids.analyzer.UnboundSlot; import org.apache.doris.nereids.analyzer.UnboundStar; import org.apache.doris.nereids.analyzer.UnboundTableSink; @@ -61,9 +63,11 @@ import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.algebra.InlineTable; +import org.apache.doris.nereids.trees.plans.commands.ConnectorWriteSchemaUtils; import org.apache.doris.nereids.trees.plans.commands.info.DMLCommandType; import org.apache.doris.nereids.trees.plans.logical.LogicalInlineTable; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.trees.plans.logical.UnboundLogicalSink; import org.apache.doris.nereids.types.AggStateType; import org.apache.doris.nereids.types.DataType; @@ -368,7 +372,12 @@ private static Plan normalizePlanWithoutLock(LogicalPlan plan, TableIf table, } Plan query = unboundLogicalSink.child(); checkGeneratedColumnForInsertIntoSelect(table, unboundLogicalSink, insertCtx); + Optional> writeOnlyDefaultColumns = writeOnlyDefaultColumns(table); if (!(query instanceof UnboundInlineTable)) { + if (writeOnlyDefaultColumns.isPresent()) { + return plan.withChildren(resolveSelectDefaultReferences( + query, writeOnlyDefaultColumns, table, unboundLogicalSink.getNameParts())); + } return plan; } @@ -444,7 +453,9 @@ private static Plan normalizePlanWithoutLock(LogicalPlan plan, TableIf table, null, rewriteContext, strictCast); } else { DataType targetType = DataType.fromCatalogType(sameNameColumn.getType()); - addColumnValue(analyzer, optimizedRowConstructor, values.get(i), + NamedExpression value = resolveDefaultReferences(values.get(i), + writeOnlyDefaultColumns, table, unboundLogicalSink.getNameParts()); + addColumnValue(analyzer, optimizedRowConstructor, value, targetType, rewriteContext, strictCast, sameNameColumn); } } @@ -465,7 +476,9 @@ private static Plan normalizePlanWithoutLock(LogicalPlan plan, TableIf table, null, rewriteContext, strictCast); } else { DataType targetType = DataType.fromCatalogType(columns.get(i).getType()); - addColumnValue(analyzer, optimizedRowConstructor, values.get(i), targetType, + NamedExpression value = resolveDefaultReferences(values.get(i), + writeOnlyDefaultColumns, table, unboundLogicalSink.getNameParts()); + addColumnValue(analyzer, optimizedRowConstructor, value, targetType, rewriteContext, strictCast, columns.get(i)); } } @@ -477,15 +490,62 @@ private static Plan normalizePlanWithoutLock(LogicalPlan plan, TableIf table, } static List connectorWriteSchema(TableIf table, boolean full) { + return pinnedConnectorWriteSchema(table).orElseGet(() -> table.getBaseSchema(full)); + } + + private static Optional> pinnedConnectorWriteSchema(TableIf table) { ConnectContext context = ConnectContext.get(); - if (context != null && context.getStatementContext() != null) { - Optional> pinned = - context.getStatementContext().getConnectorWriteSchema(table.getId()); - if (pinned.isPresent()) { - return pinned.get(); - } + if (context == null || context.getStatementContext() == null) { + return Optional.empty(); + } + return context.getStatementContext().getConnectorWriteSchema(table.getId()); + } + + /** + * The pinned writer schema of a target whose column defaults apply to writes only. The read schema of such a + * table carries no defaults, so DEFAULT(column) in an INSERT into it must resolve against this schema. + */ + private static Optional> writeOnlyDefaultColumns(TableIf table) { + if (!(table instanceof PluginDrivenExternalTable) + || !((PluginDrivenExternalTable) table).hasWriteOnlyDefaults()) { + return Optional.empty(); } - return table.getBaseSchema(full); + return pinnedConnectorWriteSchema(table); + } + + /** Resolve DEFAULT(column) in one inserted value by name against the target's pinned writer schema. */ + private static NamedExpression resolveDefaultReferences(NamedExpression value, + Optional> writeOnlyDefaultColumns, TableIf table, List targetNameParts) { + if (!writeOnlyDefaultColumns.isPresent()) { + return value; + } + return (NamedExpression) ConnectorWriteSchemaUtils.resolveDefaultReferences(value, + writeOnlyDefaultColumns.get(), (ExternalTable) table, ConnectContext.get(), targetNameParts, null); + } + + /** + * Resolve DEFAULT(column) in the select lists of an INSERT ... SELECT by name against the target's pinned + * writer schema, before the source relations are bound, so it names a target column whatever the source is. + */ + private static Plan resolveSelectDefaultReferences(Plan query, Optional> writeOnlyDefaultColumns, + TableIf table, List targetNameParts) { + return query.rewriteUp(plan -> { + if (plan instanceof LogicalProject) { + LogicalProject project = (LogicalProject) plan; + return project.withProjects(project.getProjects().stream() + .map(projection -> resolveDefaultReferences( + projection, writeOnlyDefaultColumns, table, targetNameParts)) + .collect(ImmutableList.toImmutableList())); + } + if (plan instanceof UnboundOneRowRelation) { + UnboundOneRowRelation relation = (UnboundOneRowRelation) plan; + return new UnboundOneRowRelation(relation.getRelationId(), relation.getProjects().stream() + .map(projection -> resolveDefaultReferences( + projection, writeOnlyDefaultColumns, table, targetNameParts)) + .collect(ImmutableList.toImmutableList())); + } + return plan; + }); } static void pinConnectorWriteSchema(StatementContext statementContext, TableIf targetTableIf, diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/RewriteDefaultExpressionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/RewriteDefaultExpressionTest.java new file mode 100644 index 00000000000000..e1abcb11d3e286 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/RewriteDefaultExpressionTest.java @@ -0,0 +1,101 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.expression.rules; + +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.PrimitiveType; +import org.apache.doris.datasource.plugin.PluginDrivenExternalTable; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.rules.expression.ExpressionRewriteTestHelper; +import org.apache.doris.nereids.rules.expression.ExpressionRuleExecutor; +import org.apache.doris.nereids.trees.expressions.Default; +import org.apache.doris.nereids.trees.expressions.ExprId; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; +import org.apache.doris.nereids.types.IntegerType; +import org.apache.doris.nereids.util.TypeCoercionUtils; + +import com.google.common.collect.ImmutableList; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +public class RewriteDefaultExpressionTest extends ExpressionRewriteTestHelper { + + private static final long TABLE_ID = 17L; + private static final int FIELD_ID = 3; + + @BeforeEach + public void setUp() { + executor = new ExpressionRuleExecutor(ImmutableList.of(bottomUp(RewriteDefaultExpression.INSTANCE))); + } + + @Test + public void writeOnlyDefaultResolvesAgainstPinnedWriterSchema() { + // UPDATE t ... WHERE score = DEFAULT(score): the scanned column carries no write-only default, the writer + // schema pinned for the statement's target does. + SlotReference score = scoreSlot(true); + pinWriterColumn("9"); + + Expression rewritten = executor.rewrite(new Default(score), context); + + Assertions.assertEquals( + TypeCoercionUtils.castIfNotSameType(PARSER.parseExpression("9"), IntegerType.INSTANCE), rewritten); + } + + @Test + public void writeOnlyDefaultOutsideTheWriteTargetIsRejected() { + // SELECT DEFAULT(score) FROM t writes nothing, so nothing carries the write-only default. + SlotReference score = scoreSlot(true); + + AnalysisException e = Assertions.assertThrows(AnalysisException.class, + () -> executor.rewrite(new Default(score), context)); + + Assertions.assertTrue(e.getMessage().contains( + "DEFAULT(score) must reference a column of the table that this statement writes"), e.getMessage()); + } + + @Test + public void readSchemaDefaultIsKeptWithoutWriteOnlyDefaults() { + // A connector whose read schema carries its defaults (hive) resolves from the scanned column as before. + SlotReference score = scoreSlot(false); + pinWriterColumn("9"); + + Expression rewritten = executor.rewrite(new Default(score), context); + + Assertions.assertEquals(new NullLiteral(IntegerType.INSTANCE), rewritten); + } + + private SlotReference scoreSlot(boolean writeOnlyDefaults) { + PluginDrivenExternalTable table = Mockito.mock(PluginDrivenExternalTable.class); + Mockito.when(table.getId()).thenReturn(TABLE_ID); + Mockito.when(table.hasWriteOnlyDefaults()).thenReturn(writeOnlyDefaults); + Column readColumn = new Column("score", PrimitiveType.INT, true); + readColumn.setUniqueId(FIELD_ID); + return SlotReference.fromColumn(new ExprId(1), table, readColumn, ImmutableList.of()); + } + + private void pinWriterColumn(String defaultSql) { + Column writeColumn = new Column("score", PrimitiveType.INT, true); + writeColumn.setUniqueId(FIELD_ID); + writeColumn.setConnectorDefaultValueSql(defaultSql); + cascadesContext.getStatementContext().setConnectorWriteSchema(TABLE_ID, ImmutableList.of(writeColumn)); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtilsTest.java index e624fecc4b0eb5..1e2ac2c6dd48ac 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtilsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtilsTest.java @@ -28,17 +28,33 @@ import org.apache.doris.nereids.analyzer.UnboundAlias; import org.apache.doris.nereids.analyzer.UnboundConnectorTableSink; import org.apache.doris.nereids.analyzer.UnboundFunction; +import org.apache.doris.nereids.analyzer.UnboundInlineTable; +import org.apache.doris.nereids.analyzer.UnboundOneRowRelation; +import org.apache.doris.nereids.analyzer.UnboundSlot; import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.parser.NereidsParser; +import org.apache.doris.nereids.trees.expressions.Default; +import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; import org.apache.doris.nereids.trees.expressions.literal.TinyIntLiteral; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.RelationId; +import org.apache.doris.nereids.trees.plans.logical.LogicalInlineTable; +import org.apache.doris.nereids.trees.plans.logical.LogicalProject; import org.apache.doris.nereids.trees.plans.logical.UnboundLogicalSink; +import org.apache.doris.nereids.types.IntegerType; +import org.apache.doris.nereids.util.MemoTestUtils; +import org.apache.doris.nereids.util.TypeCoercionUtils; +import org.apache.doris.qe.ConnectContext; import com.google.common.collect.ImmutableList; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.mockito.Mockito; +import java.util.List; import java.util.Optional; /** @@ -47,6 +63,7 @@ public class InsertUtilsTest { private static final int MAX_TOTAL_BYTES = 512; + private static final NereidsParser PARSER = new NereidsParser(); private String generateString(int length) { return generateString(length, "X"); @@ -434,6 +451,103 @@ public void explicitInvisibleColumnUsesInvisibleColumnErrorWithDataOnlyWriterSch "Cannot specify invisible column '_row_id' in INSERT statement", error.getMessage()); } + @Test + public void writeOnlyDefaultInValuesResolvesAgainstPinnedWriterSchema() { + // insert into t (score, id) values (DEFAULT(score), 4): the read schema of a table with write-only + // defaults has no default, so DEFAULT(score) must take the pinned writer column's default. + ConnectContext ctx = MemoTestUtils.createConnectContext(); + try { + PluginDrivenExternalTable table = pinWriterSchema(ctx, true); + Plan normalized = InsertUtils.normalizePlan(connectorSink(ImmutableList.of("score", "id"), + new UnboundInlineTable(ImmutableList.of(ImmutableList.of( + new UnboundAlias(new Default(new UnboundSlot("score"))), + new UnboundAlias(new IntegerLiteral(4)))))), + table, Optional.empty(), Optional.empty()); + + NamedExpression score = ((LogicalInlineTable) normalized.child(0)).getConstantExprsList().get(0).get(0); + Assertions.assertEquals(TypeCoercionUtils.castUnbound(PARSER.parseExpression("9"), IntegerType.INSTANCE), + score.child(0)); + + AnalysisException unknown = Assertions.assertThrows(AnalysisException.class, + () -> InsertUtils.normalizePlan(connectorSink(ImmutableList.of("id", "score"), + new UnboundInlineTable(ImmutableList.of(ImmutableList.of( + new UnboundAlias(new IntegerLiteral(6)), + new UnboundAlias(new Default(new UnboundSlot("no_such_column"))))))), + table, Optional.empty(), Optional.empty())); + Assertions.assertTrue(unknown.getMessage().contains("no_such_column"), unknown.getMessage()); + } finally { + ConnectContext.remove(); + } + } + + @Test + public void writeOnlyDefaultInSelectResolvesByTargetColumnName() { + // DEFAULT(name) in INSERT ... SELECT names the target column whatever the source is: here a staging + // relation whose own column is also called name, and a SELECT without FROM. + ConnectContext ctx = MemoTestUtils.createConnectContext(); + try { + PluginDrivenExternalTable table = pinWriterSchema(ctx, true); + Expression writeDefault = PARSER.parseExpression("'write-name'"); + + Plan fromStaging = InsertUtils.normalizePlan(connectorSink(ImmutableList.of("id", "name"), + PARSER.parseSingle("select id, DEFAULT(name) from " + + "(select 7 as id, 'source-name' as name) staging_source").child(0)), + table, Optional.empty(), Optional.empty()); + LogicalProject project = (LogicalProject) fromStaging.child(0); + Assertions.assertEquals(writeDefault, project.getProjects().get(1).child(0)); + + Plan noFrom = InsertUtils.normalizePlan(connectorSink(ImmutableList.of("id", "name"), + new UnboundOneRowRelation(new RelationId(1), ImmutableList.of( + new UnboundAlias(new IntegerLiteral(8)), + new UnboundAlias(new Default(new UnboundSlot("name")))))), + table, Optional.empty(), Optional.empty()); + UnboundOneRowRelation oneRow = (UnboundOneRowRelation) noFrom.child(0); + Assertions.assertEquals(writeDefault, oneRow.getProjects().get(1).child(0)); + } finally { + ConnectContext.remove(); + } + } + + @Test + public void defaultReferenceIsLeftToBindingWithoutWriteOnlyDefaults() { + // A connector whose read schema carries its defaults (hive) keeps the existing DEFAULT(column) handling. + ConnectContext ctx = MemoTestUtils.createConnectContext(); + try { + PluginDrivenExternalTable table = pinWriterSchema(ctx, false); + Plan normalized = InsertUtils.normalizePlan(connectorSink(ImmutableList.of("id", "name"), + new UnboundOneRowRelation(new RelationId(1), ImmutableList.of( + new UnboundAlias(new IntegerLiteral(8)), + new UnboundAlias(new Default(new UnboundSlot("name")))))), + table, Optional.empty(), Optional.empty()); + + UnboundOneRowRelation oneRow = (UnboundOneRowRelation) normalized.child(0); + Assertions.assertInstanceOf(Default.class, oneRow.getProjects().get(1).child(0)); + } finally { + ConnectContext.remove(); + } + } + + private static PluginDrivenExternalTable pinWriterSchema(ConnectContext ctx, boolean writeOnlyDefaults) { + PluginDrivenExternalTable table = Mockito.mock(PluginDrivenExternalTable.class); + Mockito.when(table.getId()).thenReturn(17L); + Mockito.when(table.getName()).thenReturn("t"); + Mockito.when(table.hasWriteOnlyDefaults()).thenReturn(writeOnlyDefaults); + Column id = new Column("id", PrimitiveType.INT, true); + id.setConnectorDefaultValueSql("NULL"); + Column name = new Column("name", PrimitiveType.STRING, true); + name.setConnectorDefaultValueSql("'write-name'"); + Column score = new Column("score", PrimitiveType.INT, true); + score.setConnectorDefaultValueSql("9"); + MemoTestUtils.createStatementContext(ctx, "") + .setConnectorWriteSchema(17L, ImmutableList.of(id, name, score)); + return table; + } + + private static UnboundConnectorTableSink connectorSink(List columnNames, Plan query) { + return new UnboundConnectorTableSink<>(ImmutableList.of("ctl", "db", "t"), columnNames, + ImmutableList.of(), ImmutableList.of(), query); + } + @Test public void nativeExplicitInvisibleColumnRemainsSupported() { Column id = new Column("id", PrimitiveType.INT); diff --git a/regression-test/data/external_table_p0/iceberg/test_iceberg_initial_defaults.out b/regression-test/data/external_table_p0/iceberg/test_iceberg_initial_defaults.out index d2945f45cf8df0..a5ddf7fb1a88e6 100644 --- a/regression-test/data/external_table_p0/iceberg/test_iceberg_initial_defaults.out +++ b/regression-test/data/external_table_p0/iceberg/test_iceberg_initial_defaults.out @@ -387,6 +387,16 @@ missing_map_col map> Yes true \N 3 123e4567-e89b-12d3-a456-426614174000 0A0B0C0D 2A2B2C 2024-12-17 23:59:59.123456+00:00 4 123e4567-e89b-12d3-a456-426614174000 0A0B0C0D 2A2B2C 2024-12-17 23:59:59.123456+00:00 +-- !parquet_default_references -- +20 35 update-where-default +21 35 update-where-default +22 35 merge-on-default + +-- !orc_default_references -- +20 35 update-where-default +21 35 update-where-default +22 35 merge-on-default + -- !parquet_branch_overwrite_default -- 14 36 diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_initial_defaults.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_initial_defaults.groovy index 2ddeb9891a9a7c..e3afd11115f700 100644 --- a/regression-test/suites/external_table_p0/iceberg/test_iceberg_initial_defaults.groovy +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_initial_defaults.groovy @@ -599,6 +599,39 @@ suite("test_iceberg_initial_defaults", "p0,external") { sparkUpdateMergeVerification) } + // DEFAULT(column) elsewhere in a write must also resolve to the pinned write-default (35); the read + // schema carries none. Rows 20-22 come from VALUES and INSERT ... SELECT with and without FROM, then the + // UPDATE and MERGE predicates must match all of them. A plain SELECT writes nothing and is rejected. + tableNames.each { String format, String currentTable -> + sql """INSERT INTO ${currentTable} (id, default_int) VALUES (20, DEFAULT(default_int))""" + sql """ + INSERT INTO ${currentTable} (id, default_int) + SELECT 21, DEFAULT(default_int) FROM ${currentTable} WHERE id = 7 + """ + sql """INSERT INTO ${currentTable} (id, default_int) SELECT 22, DEFAULT(default_int)""" + sql """ + UPDATE ${currentTable} + SET default_string = 'update-where-default' + WHERE id IN (20, 21, 22) AND default_int = DEFAULT(default_int) + """ + sql """ + MERGE INTO ${currentTable} t + USING (SELECT 22 AS id) s + ON t.id = s.id AND t.default_int = DEFAULT(t.default_int) + WHEN MATCHED THEN UPDATE SET default_string = 'merge-on-default' + """ + "order_qt_${format}_default_references" """ + SELECT id, default_int, default_string + FROM ${currentTable} + WHERE id IN (20, 21, 22) + ORDER BY id + """ + } + test { + sql """SELECT DEFAULT(default_int) FROM ${parquetTable}""" + exception "must reference a column of the table that this statement writes" + } + // Pin a branch while default_int still has write-default 35, then evolve the table-current // schema to 36. Iceberg selects the branch as the parent snapshot but writes the new snapshot // with the table-current schema, so VALUES(DEFAULT) normalization and the writer must both use From 8b046c6e00c5e323e36f61ed0f938ba6bc8bd3fe Mon Sep 17 00:00:00 2001 From: morningman Date: Fri, 23 Oct 2026 16:21:16 +0800 Subject: [PATCH 2/5] [fix](fe) Type a resolved DEFAULT(column) as its writer column ### What problem does this PR solve? Issue Number: None Related PR: #66538 Problem Summary: Two review findings on the previous commit. ConnectorWriteSchemaUtils.resolveDefaultReferences parsed the default SQL of a writer column and substituted it without a cast, so the default kept the type the parser gives the literal: -1 is a TINYINT. For an INT column with write-default -1, INSERT INTO t(c) VALUES (bit_count(DEFAULT(c))) and INSERT INTO t(c) SELECT bit_count(DEFAULT(c)) evaluated bit_count on a TINYINT and wrote 8 instead of 32; casting the finished value to INT cannot restore that. The external UPDATE, MERGE and changelog paths use the same helper and typed DEFAULT(column) the same way. The resolved default is now cast to its column's type before the expression around it is bound, as RewriteDefaultExpression does for a bound DEFAULT(column). branch-4.2 builds write defaults as literals of the column type and is not affected. RewriteDefaultExpression took the result type from the read column before substituting the pinned writer column. After a type promotion committed outside Doris, the read schema can still hold an INT column while the writer column is a BIGINT whose default an INT cannot hold, such as 4900000001, and the default was cast to INT. The type now comes from the writer column the default is taken from. ### Release note None ### Check List (For Author) - Test: Unit Test - InsertUtilsTest#writeOnlyDefaultIsTypedAsItsColumnBeforeBinding: VALUES and SELECT pass DEFAULT(bits) to bit_count as an INT; with the previous commit it is a TINYINT - RewriteDefaultExpressionTest#writeOnlyDefaultTakesTheTypeOfThePinned WriterColumn: an INT read column with a BIGINT pinned writer column gives the BIGINT default; with the previous commit it is cast(4900000001 as INT) - InsertUtilsTest, RewriteDefaultExpressionTest, ConnectorWriteSchemaUtilsTest, ConnectorChangelogPlanBuilderTest, ChangelogRowLevelDmlTransformTest, ExternalRowLevelUpdatePlanBuilderTest, ExternalRowLevelMergePlanBuilderTest, PluginDrivenExternalTableTest: 93 tests pass - Behavior changed: No. DEFAULT(column) inside an expression of an external UPDATE / MERGE / changelog assignment now has the column's type, which is the type a bound DEFAULT(column) already had. - Does this need documentation: No Co-Authored-By: Claude Opus 5.5 (1M context) --- .../rules/RewriteDefaultExpression.java | 4 +- .../commands/ConnectorWriteSchemaUtils.java | 6 ++- .../rules/RewriteDefaultExpressionTest.java | 22 ++++++++++- .../commands/insert/InsertUtilsTest.java | 37 ++++++++++++++++++- 4 files changed, 65 insertions(+), 4 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/RewriteDefaultExpression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/RewriteDefaultExpression.java index 48e74e55a4a56e..eb8366089defb8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/RewriteDefaultExpression.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/RewriteDefaultExpression.java @@ -81,7 +81,6 @@ private static Expression rewrite(ExpressionMatchingContext context) { } Column column = columnOpt.get(); - DataType targetType = DataType.fromCatalogType(column.getType()); if (column.isGeneratedColumn()) { throw new AnalysisException("DEFAULT cannot be used on generated column '" + column.getName() + "'"); @@ -95,6 +94,9 @@ private static Expression rewrite(ExpressionMatchingContext context) { .orElseThrow(() -> new AnalysisException("DEFAULT(" + slotRef.toSql() + ") must reference a column of the table that this statement writes")); } + // The pinned writer column can be wider than the read column the slot was bound with, e.g. after a type + // promotion committed outside Doris, and the default is a value of the writer column. + DataType targetType = DataType.fromCatalogType(column.getType()); String defaultValueSql = column.getDefaultValueSql(); if (defaultValueSql == null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ConnectorWriteSchemaUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ConnectorWriteSchemaUtils.java index c4a8d94f9b2e1c..a7f063eaba68ef 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ConnectorWriteSchemaUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ConnectorWriteSchemaUtils.java @@ -28,6 +28,8 @@ import org.apache.doris.nereids.trees.expressions.DefaultValueSlot; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.types.DataType; +import org.apache.doris.nereids.util.TypeCoercionUtils; import org.apache.doris.qe.ConnectContext; import com.google.common.collect.ImmutableList; @@ -101,7 +103,9 @@ public static Expression resolveDefaultReferences(Expression expression, ListcollectFirst(UnboundFunction.class::isInstance).get(); + return bitCount.child(0); + } + private static PluginDrivenExternalTable pinWriterSchema(ConnectContext ctx, boolean writeOnlyDefaults) { PluginDrivenExternalTable table = Mockito.mock(PluginDrivenExternalTable.class); Mockito.when(table.getId()).thenReturn(17L); @@ -538,8 +571,10 @@ private static PluginDrivenExternalTable pinWriterSchema(ConnectContext ctx, boo name.setConnectorDefaultValueSql("'write-name'"); Column score = new Column("score", PrimitiveType.INT, true); score.setConnectorDefaultValueSql("9"); + Column bits = new Column("bits", PrimitiveType.INT, true); + bits.setConnectorDefaultValueSql("-1"); MemoTestUtils.createStatementContext(ctx, "") - .setConnectorWriteSchema(17L, ImmutableList.of(id, name, score)); + .setConnectorWriteSchema(17L, ImmutableList.of(id, name, score, bits)); return table; } From 95191a92c8384abefd4ecc54a4ecaa74eb06b3bc Mon Sep 17 00:00:00 2001 From: morningman Date: Fri, 23 Oct 2026 22:21:26 +0800 Subject: [PATCH 3/5] [test](fe) Expect a resolved DEFAULT(column) typed as its writer column ### What problem does this PR solve? Issue Number: None Related PR: #66538 Problem Summary: #69394 added two ConnectorWriteSchemaUtilsTest cases that expect resolveDefaultReferences to return the default with the type the parser gives its literal (9 is a TINYINT). The previous commit casts the resolved default to its writer column's type, so for the INT test column the result is now an INT literal 9, and both cases failed in FE UT once this branch met master. The two expectations now use the column's type. ### Release note None ### Check List (For Author) - Test: Unit Test - ConnectorWriteSchemaUtilsTest, RewriteDefaultExpressionTest, InsertUtilsTest - Behavior changed: No - Does this need documentation: No Co-Authored-By: Claude Opus 5.5 (1M context) --- .../plans/commands/ConnectorWriteSchemaUtilsTest.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ConnectorWriteSchemaUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ConnectorWriteSchemaUtilsTest.java index b61a51c1eadf08..32ce0f8827caca 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ConnectorWriteSchemaUtilsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ConnectorWriteSchemaUtilsTest.java @@ -36,8 +36,8 @@ import org.apache.doris.nereids.trees.expressions.literal.DecimalV3Literal; import org.apache.doris.nereids.trees.expressions.literal.DoubleLiteral; import org.apache.doris.nereids.trees.expressions.literal.FloatLiteral; +import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; import org.apache.doris.nereids.trees.expressions.literal.Literal; -import org.apache.doris.nereids.trees.expressions.literal.TinyIntLiteral; import org.apache.doris.nereids.types.DataType; import org.apache.doris.nereids.types.DecimalV3Type; import org.apache.doris.nereids.util.TypeCoercionUtils; @@ -96,12 +96,13 @@ public void connectorNonFiniteFloatingDefaultsParseAsTypedExpressions() { @Test public void defaultReferencesResolveAgainstPinnedWriteColumns() { - // DEFAULT(column) in DML resolves the statement-pinned write default; a name absent from the pinned writer - // schema is an error rather than a guess. + // DEFAULT(column) in DML resolves the statement-pinned write default, typed as its INT column rather than + // as the TINYINT the literal parses to; a name absent from the pinned writer schema is an error rather than + // a guess. ExternalTable target = targetTable(); List pinned = ImmutableList.of(pinnedColumn("score", 1, "9")); - Assertions.assertEquals(new TinyIntLiteral((byte) 9), ConnectorWriteSchemaUtils.resolveDefaultReferences( + Assertions.assertEquals(new IntegerLiteral(9), ConnectorWriteSchemaUtils.resolveDefaultReferences( new Default(new UnboundSlot("score")), pinned, target, ConnectContext.get(), TARGET_NAME_PARTS, null)); AnalysisException exception = Assertions.assertThrows(AnalysisException.class, @@ -121,7 +122,7 @@ public void defaultOfBoundSlotDoesNotFallBackToReusedColumnName() { Column replacement = new Column("reused_name", ScalarType.createType(PrimitiveType.INT)); replacement.setUniqueId(8); - Assertions.assertEquals(new TinyIntLiteral((byte) 9), ConnectorWriteSchemaUtils.resolveDefaultReferences( + Assertions.assertEquals(new IntegerLiteral(9), ConnectorWriteSchemaUtils.resolveDefaultReferences( new Default(boundSlot(target, pinnedField)), pinned, target, ConnectContext.get(), TARGET_NAME_PARTS, null)); AnalysisException exception = Assertions.assertThrows(AnalysisException.class, From fc049a2c65b38364415a1c2b5ee4d4cd552f404e Mon Sep 17 00:00:00 2001 From: morningman Date: Fri, 23 Oct 2026 22:43:09 +0800 Subject: [PATCH 4/5] [fix](fe) Reject a DEFAULT(column) whose writer type differs from its read type ### What problem does this PR solve? Issue Number: None Related PR: #66538 Problem Summary: Review follow-up. The previous commits typed the default that replaces a bound DEFAULT(column) as its pinned writer column. That type differs from the read column's only when the table changed outside Doris after Doris cached its read schema, e.g. an INT column promoted to BIGINT whose new write-default is 4900000001. The expressions around DEFAULT(column) were already bound against the read type and the rewriter keeps their signatures, so in UPDATE t SET ... WHERE IF(TRUE, DEFAULT(score), 0) = 4900000001 the IF keeps its INT result type and constant folding casts 4900000001 back to INT: NULL in non-strict mode, so no row matches, and an overflow error in strict mode. A substituted value cannot repair the types its consumers were bound with, so the rewrite now fails such a DEFAULT and asks to refresh the table. When the two types agree, which is every case without a type change made outside Doris, nothing changes. ### Release note None ### Check List (For Author) - Test: Unit Test - RewriteDefaultExpressionTest: a DEFAULT whose writer type differs is rejected, directly and under an IF bound to the read type - ConnectorWriteSchemaUtilsTest, InsertUtilsTest, ExternalRowLevelUpdatePlanBuilderTest, ExternalRowLevelMergePlanBuilderTest, ChangelogRowLevelDmlTransformTest, PluginDrivenExternalTableTest - Behavior changed: No - Does this need documentation: No Co-Authored-By: Claude Opus 5.5 (1M context) --- .../rules/RewriteDefaultExpression.java | 13 ++++++-- .../rules/RewriteDefaultExpressionTest.java | 30 ++++++++++++++----- 2 files changed, 33 insertions(+), 10 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/RewriteDefaultExpression.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/RewriteDefaultExpression.java index eb8366089defb8..9713bbce10de8e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/RewriteDefaultExpression.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/RewriteDefaultExpression.java @@ -81,6 +81,7 @@ private static Expression rewrite(ExpressionMatchingContext context) { } Column column = columnOpt.get(); + DataType targetType = DataType.fromCatalogType(column.getType()); if (column.isGeneratedColumn()) { throw new AnalysisException("DEFAULT cannot be used on generated column '" + column.getName() + "'"); @@ -93,10 +94,16 @@ private static Expression rewrite(ExpressionMatchingContext context) { column = pinnedWriteColumn(context.cascadesContext.getStatementContext(), originalTable.get(), column) .orElseThrow(() -> new AnalysisException("DEFAULT(" + slotRef.toSql() + ") must reference a column of the table that this statement writes")); + // The expressions around DEFAULT(column) were bound against the read column, so a default of another type + // cannot stand in for it. The types differ when the table changed outside Doris after its read schema was + // cached, e.g. by a type promotion. + DataType writeType = DataType.fromCatalogType(column.getType()); + if (!writeType.equals(targetType)) { + throw new AnalysisException("DEFAULT(" + slotRef.toSql() + ") reads the column as " + + targetType.toSql() + " but the table now writes it as " + writeType.toSql() + + "; the table changed outside Doris, refresh it and retry"); + } } - // The pinned writer column can be wider than the read column the slot was bound with, e.g. after a type - // promotion committed outside Doris, and the default is a value of the writer column. - DataType targetType = DataType.fromCatalogType(column.getType()); String defaultValueSql = column.getDefaultValueSql(); if (defaultValueSql == null) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/RewriteDefaultExpressionTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/RewriteDefaultExpressionTest.java index e9a3e97b89dbbb..025bfa459763fa 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/RewriteDefaultExpressionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/expression/rules/RewriteDefaultExpressionTest.java @@ -27,8 +27,10 @@ import org.apache.doris.nereids.trees.expressions.ExprId; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.SlotReference; +import org.apache.doris.nereids.trees.expressions.functions.scalar.If; +import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; +import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral; import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; -import org.apache.doris.nereids.types.BigIntType; import org.apache.doris.nereids.types.IntegerType; import org.apache.doris.nereids.util.TypeCoercionUtils; @@ -62,18 +64,32 @@ public void writeOnlyDefaultResolvesAgainstPinnedWriterSchema() { } @Test - public void writeOnlyDefaultTakesTheTypeOfThePinnedWriterColumn() { + public void writeOnlyDefaultOfAWriterColumnOfAnotherTypeIsRejected() { // The read schema can still hold the INT column after it was promoted to BIGINT outside Doris, while the // writer schema pinned for the statement has the BIGINT column and its new default, which an INT cannot hold. SlotReference score = scoreSlot(true); pinWriterColumn(PrimitiveType.BIGINT, "4900000001"); - Expression rewritten = executor.rewrite(new Default(score), context); + AnalysisException e = Assertions.assertThrows(AnalysisException.class, + () -> executor.rewrite(new Default(score), context)); - Assertions.assertEquals( - TypeCoercionUtils.castIfNotSameType(PARSER.parseExpression("4900000001"), BigIntType.INSTANCE), - rewritten); - Assertions.assertEquals(BigIntType.INSTANCE, rewritten.getDataType()); + Assertions.assertTrue(e.getMessage().contains("DEFAULT(score) reads the column as INT but the table now writes" + + " it as BIGINT; the table changed outside Doris, refresh it and retry"), e.getMessage()); + } + + @Test + public void writeOnlyDefaultOfAnotherTypeIsRejectedInsideAnExpressionBoundToTheReadType() { + // UPDATE t ... WHERE IF(TRUE, DEFAULT(score), 0) = 4900000001: IF was bound with the INT of the read column, + // so substituting the BIGINT default would later fold into CAST(4900000001 AS INT), NULL in non-strict mode. + SlotReference score = scoreSlot(true); + pinWriterColumn(PrimitiveType.BIGINT, "4900000001"); + If ifExpr = new If(BooleanLiteral.TRUE, new Default(score), new IntegerLiteral(0)); + Assertions.assertEquals(IntegerType.INSTANCE, ifExpr.getDataType()); + + AnalysisException e = Assertions.assertThrows(AnalysisException.class, + () -> executor.rewrite(ifExpr, context)); + + Assertions.assertTrue(e.getMessage().contains("refresh it and retry"), e.getMessage()); } @Test From 724071d42a8b1c8aa813d6c777055a7d017c6d25 Mon Sep 17 00:00:00 2001 From: morningman Date: Mon, 26 Oct 2026 11:38:03 +0800 Subject: [PATCH 5/5] [fix](fe) Keep DEFAULT(column) in a derived table of INSERT ... SELECT on its own FROM clause ### What problem does this PR solve? Issue Number: None Related PR: #69488 Problem Summary: For a target with write-only defaults (Iceberg), `InsertUtils.normalizePlan` resolves `DEFAULT(column)` in the select lists of an `INSERT ... SELECT` by name against the target's pinned writer schema, before the source relations are bound. It walked the whole query plan with `rewriteUp`, so it also rewrote the projections of a derived table in `FROM`, which is a query of its own. With a source `src(c INT DEFAULT 7)` and a target `t(c INT)` whose write-default is 9, INSERT INTO t(c) SELECT v FROM (SELECT DEFAULT(c) AS v FROM src) q wrote 9 instead of 7, and `DEFAULT(src.c)` inside the derived table failed with "column in assignment list is invalid", because its qualifier was checked against the target. branch-4.2's `resolveIcebergSelectDefaultReferences` walks the plan the same way. The walk now stops at `LogicalSubQueryAlias`. `DEFAULT(column)` inside a derived table is left to binding and names a column of that table's own `FROM` clause, as it does on master. The select lists of the INSERT's own query, set operation branches included, are resolved as before, still bottom-up. ### Release note None ### Check List (For Author) - Test: Unit Test - New `InsertUtilsTest#writeOnlyDefaultInDerivedTableNamesItsOwnSource` covers an unqualified and a qualified `DEFAULT(score)` inside a derived table (left to binding while the outer `DEFAULT(name)` resolves to the pinned default), and a UNION whose first branch resolves and whose second branch reads a derived table. It fails with the previous `rewriteUp` walk. - `InsertUtilsTest`, `ConnectorWriteSchemaUtilsTest`, `RewriteDefaultExpressionTest`, `IvmDeltaRewriterTest` and `IvmLinearDeltaHandlerTest` pass, as does the fe-core checkstyle. - Behavior changed: Yes. Within this PR, `DEFAULT(column)` in a derived table of an `INSERT ... SELECT` into an Iceberg table no longer resolves against the target; it keeps master's meaning. - Does this need documentation: No Co-Authored-By: Claude Opus 5.5 (1M context) --- .../plans/commands/insert/InsertUtils.java | 40 ++++++++++-------- .../commands/insert/InsertUtilsTest.java | 41 +++++++++++++++++++ 2 files changed, 64 insertions(+), 17 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java index 7366dd1b1970ed..6cbe68a6efc7ad 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtils.java @@ -68,6 +68,7 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalInlineTable; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.trees.plans.logical.LogicalSubQueryAlias; import org.apache.doris.nereids.trees.plans.logical.UnboundLogicalSink; import org.apache.doris.nereids.types.AggStateType; import org.apache.doris.nereids.types.DataType; @@ -526,26 +527,31 @@ private static NamedExpression resolveDefaultReferences(NamedExpression value, /** * Resolve DEFAULT(column) in the select lists of an INSERT ... SELECT by name against the target's pinned * writer schema, before the source relations are bound, so it names a target column whatever the source is. + * A derived table in FROM is a query of its own: DEFAULT(column) in it names a column of its own FROM clause, + * as without this resolution, so it is left alone. */ private static Plan resolveSelectDefaultReferences(Plan query, Optional> writeOnlyDefaultColumns, TableIf table, List targetNameParts) { - return query.rewriteUp(plan -> { - if (plan instanceof LogicalProject) { - LogicalProject project = (LogicalProject) plan; - return project.withProjects(project.getProjects().stream() - .map(projection -> resolveDefaultReferences( - projection, writeOnlyDefaultColumns, table, targetNameParts)) - .collect(ImmutableList.toImmutableList())); - } - if (plan instanceof UnboundOneRowRelation) { - UnboundOneRowRelation relation = (UnboundOneRowRelation) plan; - return new UnboundOneRowRelation(relation.getRelationId(), relation.getProjects().stream() - .map(projection -> resolveDefaultReferences( - projection, writeOnlyDefaultColumns, table, targetNameParts)) - .collect(ImmutableList.toImmutableList())); - } - return plan; - }); + if (query instanceof LogicalSubQueryAlias) { + return query; + } + Plan plan = query.withChildren(child -> resolveSelectDefaultReferences( + child, writeOnlyDefaultColumns, table, targetNameParts)); + if (plan instanceof LogicalProject) { + LogicalProject project = (LogicalProject) plan; + return project.withProjects(project.getProjects().stream() + .map(projection -> resolveDefaultReferences( + projection, writeOnlyDefaultColumns, table, targetNameParts)) + .collect(ImmutableList.toImmutableList())); + } + if (plan instanceof UnboundOneRowRelation) { + UnboundOneRowRelation relation = (UnboundOneRowRelation) plan; + return new UnboundOneRowRelation(relation.getRelationId(), relation.getProjects().stream() + .map(projection -> resolveDefaultReferences( + projection, writeOnlyDefaultColumns, table, targetNameParts)) + .collect(ImmutableList.toImmutableList())); + } + return plan; } static void pinConnectorWriteSchema(StatementContext statementContext, TableIf targetTableIf, diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtilsTest.java index 1ff47b9806ec6c..b0b9c326a46afb 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtilsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertUtilsTest.java @@ -43,6 +43,8 @@ import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.trees.plans.logical.LogicalInlineTable; import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.trees.plans.logical.LogicalSubQueryAlias; +import org.apache.doris.nereids.trees.plans.logical.LogicalUnion; import org.apache.doris.nereids.trees.plans.logical.UnboundLogicalSink; import org.apache.doris.nereids.types.IntegerType; import org.apache.doris.nereids.util.MemoTestUtils; @@ -508,6 +510,45 @@ public void writeOnlyDefaultInSelectResolvesByTargetColumnName() { } } + @Test + public void writeOnlyDefaultInDerivedTableNamesItsOwnSource() { + // INSERT INTO t(score, name) SELECT v, DEFAULT(name) FROM (SELECT DEFAULT(score) AS v FROM src) q: the + // derived table is a query of its own, so its DEFAULT(score) names src.score, whose default can differ from + // the target's 9, and is left to binding, qualified or not. DEFAULT in the INSERT's own select lists, set + // operation branches included, still names the target column. + ConnectContext ctx = MemoTestUtils.createConnectContext(); + try { + PluginDrivenExternalTable table = pinWriterSchema(ctx, true); + Expression writeDefault = PARSER.parseExpression("'write-name'"); + for (String derived : ImmutableList.of("(select DEFAULT(score) as v from src) q", + "(select DEFAULT(s.score) as v from src s) q")) { + Plan normalized = InsertUtils.normalizePlan(connectorSink(ImmutableList.of("score", "name"), + PARSER.parseSingle("select v, DEFAULT(name) from " + derived).child(0)), + table, Optional.empty(), Optional.empty()); + LogicalProject outer = (LogicalProject) normalized.child(0); + Assertions.assertEquals(writeDefault, outer.getProjects().get(1).child(0), derived); + assertDerivedDefaultLeftToBinding(outer.child(0), derived); + } + + Plan union = InsertUtils.normalizePlan(connectorSink(ImmutableList.of("name"), + PARSER.parseSingle("select DEFAULT(name) union all " + + "select v from (select DEFAULT(name) as v from src) q").child(0)), + table, Optional.empty(), Optional.empty()); + LogicalUnion setOperation = (LogicalUnion) union.child(0); + UnboundOneRowRelation oneRow = (UnboundOneRowRelation) setOperation.child(0); + Assertions.assertEquals(writeDefault, oneRow.getProjects().get(0).child(0)); + assertDerivedDefaultLeftToBinding(setOperation.child(1).child(0), "union branch"); + } finally { + ConnectContext.remove(); + } + } + + private static void assertDerivedDefaultLeftToBinding(Plan derivedTable, String message) { + Assertions.assertInstanceOf(LogicalSubQueryAlias.class, derivedTable, message); + LogicalProject inner = (LogicalProject) derivedTable.child(0); + Assertions.assertInstanceOf(Default.class, inner.getProjects().get(0).child(0), message); + } + @Test public void defaultReferenceIsLeftToBindingWithoutWriteOnlyDefaults() { // A connector whose read schema carries its defaults (hive) keeps the existing DEFAULT(column) handling.